Implement literal here-document support

25 files changed, 194 insertions, 123 deletions

diff --git a/build2/lexer b/build2/lexer
index 59150a9..e2cf07c 100644
--- a/build2/lexer
+++ b/build2/lexer
@@ -85,7 +85,9 @@ namespace build2
     // specifythe pair separator character (if the mode supports pairs).
     //
     virtual void
-    mode (lexer_mode, char pair_separator = '\0');
+    mode (lexer_mode,
+          char pair_separator = '\0',
+          const char* escapes = nullptr);
 
     // Expire the current mode early.
     //
@@ -119,6 +121,8 @@ namespace build2
       bool sep_space; // Are whitespaces separators (see skip_spaces())?
       bool quotes;    // Recognize quoted fragments.
 
+      const char* escapes; // Effective escape sequences to recognize.
+
       // Word separator characters. For two-character sequence put the first
       // one in sep_first and the second one in the corresponding position of
       // sep_second. If it's a single-character sequence, then put space in
@@ -170,16 +174,14 @@ namespace build2
         : char_scanner (is),
           fail ("error", &name_),
           name_ (n),
-          escapes_ (e),
           processor_ (p),
           sep_ (false)
     {
       if (sm)
-        mode (lexer_mode::normal, '@');
+        mode (lexer_mode::normal, '@', e);
     }
 
     const path name_;
-    const char* escapes_;
     void (*processor_) (token&, const lexer&);
 
     std::stack<state> state_;
diff --git a/build2/lexer.cxx b/build2/lexer.cxx
index b73c291..3c8eb5a 100644
--- a/build2/lexer.cxx
+++ b/build2/lexer.cxx
@@ -30,7 +30,7 @@ namespace build2
   }
 
   void lexer::
-  mode (lexer_mode m, char ps)
+  mode (lexer_mode m, char ps, const char* esc)
   {
     const char* s1 (nullptr);
     const char* s2 (nullptr);
@@ -76,7 +76,7 @@ namespace build2
     default: assert (false); // Unhandled custom mode.
     }
 
-    state_.push (state {m, ps, s, q, s1, s2});
+    state_.push (state {m, ps, s, q, esc, s1, s2});
   }
 
   token lexer::
@@ -329,8 +329,10 @@ namespace build2
         get ();
         xchar p (peek ());
 
-        if (escapes_ == nullptr ||
-            (!eos (p) && strchr (escapes_, p) != nullptr))
+        const char* esc (st.escapes);
+
+        if (esc == nullptr ||
+            (*esc != '\0' && !eos (p) && strchr (esc, p) != nullptr))
         {
           get ();
 
diff --git a/build2/test/script/lexer b/build2/test/script/lexer
index 5597e9a..b812f84 100644
--- a/build2/test/script/lexer
+++ b/build2/test/script/lexer
@@ -29,7 +29,8 @@ namespace build2
           second_token,    // Expires at the end of the token.
           variable_line,   // Expires at the end of the line.
           command_line,
-          here_line,
+          here_line_single,
+          here_line_double,
           description_line // Expires at the end of the line.
         };
 
@@ -48,10 +49,13 @@ namespace build2
                const path& name,
                lexer_mode m,
                const char* escapes = nullptr)
-            : base_lexer (is, name, escapes, nullptr, false) {mode (m);}
+            : base_lexer (is, name, nullptr, nullptr, false)
+        {
+          mode (m, '\0', escapes);
+        }
 
         virtual void
-        mode (base_mode, char = '\0') override;
+        mode (base_mode, char = '\0', const char* = nullptr) override;
 
         // Number of quoted (double or single) tokens since last reset.
         //
diff --git a/build2/test/script/lexer.cxx b/build2/test/script/lexer.cxx
index 19e7498..72fa85b 100644
--- a/build2/test/script/lexer.cxx
+++ b/build2/test/script/lexer.cxx
@@ -15,7 +15,7 @@ namespace build2
       using type = token_type;
 
       void lexer::
-      mode (base_mode m, char ps)
+      mode (base_mode m, char ps, const char* esc)
       {
         const char* s1 (nullptr);
         const char* s2 (nullptr);
@@ -76,7 +76,23 @@ namespace build2
             s = false;
             break;
           }
-        case lexer_mode::here_line:
+        case lexer_mode::here_line_single:
+          {
+            // This one is like a single-quoted string except it treats
+            // newlines as a separator. We also treat quotes as literals.
+            //
+            // Note that it might be tempting to enable line continuation
+            // escapes. However, we will then have to also enable escaping of
+            // the backslash, which makes it a lot less tempting.
+            //
+            s1 = "\n";
+            s2 = " ";
+            esc = ""; // Disable escape sequences.
+            s = false;
+            q = false;
+            break;
+          }
+        case lexer_mode::here_line_double:
           {
             // This one is like a double-quoted string except it treats
             // newlines as a separator. We also treat quotes as literals.
@@ -105,13 +121,13 @@ namespace build2
                     m == lexer_mode::eval ||
                     m == lexer_mode::attribute);
 
-            base_lexer::mode (m, ps);
+            base_lexer::mode (m, ps, esc);
             return;
           }
         }
 
         assert (ps == '\0');
-        state_.push (state {m, ps, s, q, s1, s2});
+        state_.push (state {m, ps, s, q, esc, s1, s2});
       }
 
       token lexer::
@@ -126,7 +142,8 @@ namespace build2
         case lexer_mode::second_token:
         case lexer_mode::variable_line:
         case lexer_mode::command_line:
-        case lexer_mode::here_line:
+        case lexer_mode::here_line_single:
+        case lexer_mode::here_line_double:
           r = next_line ();
           break;
         case lexer_mode::description_line:
@@ -184,7 +201,13 @@ namespace build2
               sep = true; // Treat newline as always separated.
               return make_token (type::newline);
             }
+          }
+        }
 
+        if (m != lexer_mode::here_line_single)
+        {
+          switch (c)
+          {
             // Variable expansion, function call, and evaluation context.
             //
           case '$': return make_token (type::dollar);
@@ -192,6 +215,7 @@ namespace build2
           }
         }
 
+
         if (m == lexer_mode::variable_line)
         {
           switch (c)
diff --git a/build2/test/script/parser b/build2/test/script/parser
index fdfbe11..da82df2 100644
--- a/build2/test/script/parser
+++ b/build2/test/script/parser
@@ -99,10 +99,10 @@ namespace build2
         {
           size_t expr;  // Index in command_expr.
           size_t pipe;  // Index in command_pipe.
-          size_t redir; // Redirect (0 - in, 1 - out, 2 - err).
-
+          int fd;       // Redirect fd (0 - in, 1 - out, 2 - err).
           string end;
-          bool no_newline;
+          bool literal;    // Literal (single-quote).
+          bool no_newline; // No final newline.
         };
         using here_docs = vector<here_doc>;
 
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index 7655ba9..9e2018f 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -1261,11 +1261,11 @@ namespace build2
         cleanup_type ct;  // Pending cleanup type.
         here_docs hd;     // Expected here-documents.
 
-        // Add the next word to either one of the pending positions or
-        // to program arguments by default.
+        // Add the next word to either one of the pending positions or to
+        // program arguments by default.
         //
-        auto add_word = [&expr, &c, &p, &nn, &app, &ct, &hd, this]
-          (string&& w, const location& l)
+        auto add_word =
+          [&c, &p, &nn, &app, &ct, this] (string&& w, const location& l)
         {
           auto add_merge = [&l, this] (redirect& r, const string& w, int fd)
           {
@@ -1290,13 +1290,6 @@ namespace build2
             r.str = move (w);
           };
 
-          auto add_here_end = [&expr, &hd, &nn] (size_t r, string&& w)
-          {
-            hd.push_back (
-              here_doc {
-                expr.size () - 1, expr.back ().pipe.size (), r, move (w), nn});
-          };
-
           auto parse_path = [&l, this] (string&& w, const char* what) -> path
           {
             try
@@ -1335,10 +1328,8 @@ namespace build2
           {
           case pending::none: c.arguments.push_back (move (w)); break;
           case pending::program:
-          {
             c.program = parse_path (move (w), "program path");
             break;
-          }
 
           case pending::out_merge: add_merge (c.out, w, 2); break;
           case pending::err_merge: add_merge (c.err, w, 1); break;
@@ -1347,21 +1338,19 @@ namespace build2
           case pending::out_string: add_here_str (c.out, move (w)); break;
           case pending::err_string: add_here_str (c.err, move (w)); break;
 
-          case pending::in_document:  add_here_end (0, move (w)); break;
-          case pending::out_document: add_here_end (1, move (w)); break;
-          case pending::err_document: add_here_end (2, move (w)); break;
+            // These are handled specially below.
+            //
+          case pending::in_document:
+          case pending::out_document:
+          case pending::err_document: assert (false); break;
 
           case pending::in_file:  add_file (c.in,  0, move (w)); break;
           case pending::out_file: add_file (c.out, 1, move (w)); break;
           case pending::err_file: add_file (c.err, 2, move (w)); break;
 
           case pending::clean:
-            {
-              c.cleanups.push_back (
-                {ct, parse_path (move (w), "cleanup path")});
-
-              break;
-            }
+            c.cleanups.push_back ({ct, parse_path (move (w), "cleanup path")});
+            break;
           }
 
           p = pending::none;
@@ -1692,7 +1681,9 @@ namespace build2
                     fail (t) << "partially-quoted here-document end marker";
                   }
 
-                  hd.push_back (here_doc {0, 0, 0, move (t.value), nn});
+                  hd.push_back (
+                    here_doc {
+                      0, 0, 0, move (t.value), qt == quote_type::single, nn});
                   break;
                 }
 
@@ -1774,6 +1765,40 @@ namespace build2
             }
           default:
             {
+              // Here-document end markers are literal (we verified that above
+              // during pre-parsing) and we need to know whether they were
+              // quoted. So handle this case specially.
+              //
+              {
+                int fd;
+                switch (p)
+                {
+                case pending::in_document:  fd =  0; break;
+                case pending::out_document: fd =  1; break;
+                case pending::err_document: fd =  2; break;
+                default:                    fd = -1; break;
+                }
+
+                if (fd != -1)
+                {
+                  hd.push_back (
+                    here_doc {
+                      expr.size () - 1,
+                      expr.back ().pipe.size (),
+                      fd,
+                      move (t.value),
+                      (t.qtype == quote_type::unquoted ||
+                       t.qtype == quote_type::single),
+                      nn});
+
+                  p = pending::none;
+                  nn = false;
+
+                  next (t, tt);
+                  break;
+                }
+              }
+
               // Parse the next chunk as simple names to get expansion, etc.
               // Note that we do it in the chunking mode to detect whether
               // anything in each chunk is quoted.
@@ -2060,10 +2085,12 @@ namespace build2
         //
         for (here_doc& h: p.second)
         {
-          // Switch to the here-line mode which is like double-quoted but
-          // recognized the newline as a separator.
+          // Switch to the here-line mode which is like single/double-quoted
+          // string but recognized the newline as a separator.
           //
-          mode (lexer_mode::here_line);
+          mode (h.literal
+                ? lexer_mode::here_line_single
+                : lexer_mode::here_line_double);
           next (t, tt);
 
           string v (parse_here_document (t, tt, h.end, h.no_newline));
@@ -2071,7 +2098,7 @@ namespace build2
           if (!pre_parse_)
           {
             command& c (p.first[h.expr].pipe[h.pipe]);
-            redirect& r (h.redir == 0 ? c.in : h.redir == 1 ? c.out : c.err);
+            redirect& r (h.fd == 0 ? c.in : h.fd == 1 ? c.out : c.err);
 
             r.doc.doc = move (v);
             r.doc.end = move (h.end);
diff --git a/doc/testscript.cli b/doc/testscript.cli
index 1ba9a4b..79c6836 100644
--- a/doc/testscript.cli
+++ b/doc/testscript.cli
@@ -1421,10 +1421,22 @@ error: no such table 'no_such_table'
 EOE
 \
 
-The lines in here-document are expanded as if they were double-quoted except
-that the double quote itself is not treated as special. This means we can use
-variables and evaluation contexts in here-documents but have to escape the
-\c{\\$(} character set.
+Here-strings can be single-quoted literals or double-quoted with expansion.
+This semantics is extended to here-documents as follows. If the end marker
+on the command line is single-quoted, then the here-document lines are
+parsed as if they were single-quoted except that the single quote itself
+is not treated as special. In this mode there are no expansions, escape
+sequences, not even line continuations \- each line is taken literally.
+
+If the end marker on the command line is double-quoted, then the here-document
+lines are parsed as if they were double-quoted except that the double quote
+itself is not treated as special. In this mode we can use variables
+expansions, function calls, and evaluation contexts. However, we have to
+escape the \c{$(\\} character set.
+
+If the end marker is not quoted then it is treated as if it were
+single-quoted. Note also that quoted end markers must be quoted \i{wholly},
+that is, from the beginning and until the end and without any interruptions.
 
 If the preceding command line starts with leading whitespaces, then the
 equivalent number is stripped (if present) from each here-document line
diff --git a/tests/expansion/type.test b/tests/expansion/type.test
index 1aae5b6..d524eea 100644
--- a/tests/expansion/type.test
+++ b/tests/expansion/type.test
@@ -10,30 +10,30 @@
 :
 $* <<EOI
 x = [bool] true
-y = \$x
-assert \(\$type\(\$y) == bool)
+y = $x
+assert ($type($y) == bool)
 EOI
 
 : eval
 :
 $* <<EOI
-y = \([bool] true)
-assert \(\$type\(\$y) == bool)
+y = ([bool] true)
+assert ($type($y) == bool)
 EOI
 
 : func
 :
 $* <<EOI
-y = \$identity\([bool] true)
-assert \(\$type\(\$y) == bool)
+y = $identity([bool] true)
+assert ($type($y) == bool)
 EOI
 
 : untypify
 :
 $* <<EOI
 x = [bool] true
-y = "\$x"
-assert \(\$type\(\$y) == "")
+y = "$x"
+assert ($type($y) == "")
 EOI
 
 : type-conflict
diff --git a/tests/function/builtin/testscript b/tests/function/builtin/testscript
index 1a4c5d1..6491a60 100644
--- a/tests/function/builtin/testscript
+++ b/tests/function/builtin/testscript
@@ -36,7 +36,7 @@ test.options += -q --buildfile - noop
 {
   $* <<EOI     >'true'                       : empty-untyped
   x =
-  print \$empty\(\$x)
+  print $empty($x)
   EOI
 
   $* <'print $empty([string])'     >'true'   : empty-typed
diff --git a/tests/function/path/testscript b/tests/function/path/testscript
index e1b08af..0f83ad6 100644
--- a/tests/function/path/testscript
+++ b/tests/function/path/testscript
@@ -34,6 +34,6 @@ if ($cxx.target.class != windows) # @@ TMP ternarry
 else
   p = c:/../foo
 end;
-$* <"\$path.normalize\('$p')" 2>>EOE != 0
+$* <"\$path.normalize\('$p')" 2>>"EOE" != 0
 error: invalid path: '$p'
 EOE
diff --git a/tests/test/script/integration/testscript b/tests/test/script/integration/testscript
index b927fe8..93ce08a 100644
--- a/tests/test/script/integration/testscript
+++ b/tests/test/script/integration/testscript
@@ -69,8 +69,7 @@ touch test/dummy &!test/dummy;
 $* <<EOI 2>>EOE
 ./: test{foo}
 EOI
-warning: working directory test/ exists and is not empty at the beginning \
-of the test
+warning: working directory test/ exists and is not empty at the beginning of the test
 EOE
 
 : wd-not-empty-after
diff --git a/tests/test/script/runner/cleanup.test b/tests/test/script/runner/cleanup.test
index 474ce48..e6cefd5 100644
--- a/tests/test/script/runner/cleanup.test
+++ b/tests/test/script/runner/cleanup.test
@@ -49,7 +49,7 @@ rm a
 : Test that file append redirect doesn't not register cleanup. If it did, that
 : cleanup would fail as the file would be already deleted by 'rm'.
 :
-$c <<EOI;
+$c <<"EOI";
 touch a &!a;
 $* -o foo >>>&a;
 rm a
@@ -222,7 +222,7 @@ EOE
 :
 : Test an explicit cleanup not being overwritten with the implicit one.
 :
-$c <<EOO;
+$c <<"EOO";
 $* &!a;
 $* -o foo >>>a
 EOO
diff --git a/tests/test/script/runner/status.test b/tests/test/script/runner/status.test
index f1a20ec..716f2d4 100644
--- a/tests/test/script/runner/status.test
+++ b/tests/test/script/runner/status.test
@@ -36,13 +36,13 @@ $b
 : eq-false
 :
 $c <"$* -s 1 == 0";
-$b 2>>EOE != 0
+$b 2>>"EOE" != 0
 testscript:1: error: ../../../driver$ext exit status 1 != 0
 EOE
 
 : ne-false
 :
 $c <"$* -s 1 != 1";
-$b 2>>EOE != 0
+$b 2>>"EOE" != 0
 testscript:1: error: ../../../driver$ext exit status 1 == 1
 EOE
diff --git a/unit-tests/function/call.test b/unit-tests/function/call.test
index 396090a..d16b91c 100644
--- a/unit-tests/function/call.test
+++ b/unit-tests/function/call.test
@@ -13,8 +13,8 @@ $* <'print $dummy.qual()' >'abc'
 : qual-fail
 :
 $* <'print $qual()' 2>>EOE != 0
-buildfile:1:8: error: unmatched call to qual\()
-  info: candidate: dummy.qual\()
+buildfile:1:8: error: unmatched call to qual()
+  info: candidate: dummy.qual()
 EOE
 
 : derived-base
@@ -32,42 +32,42 @@ $* <'print $variadic([bool] true)' >'1'
 :
 $* <'$fail()' 2>>EOE != 0
 error: failed
-buildfile:1:2: info: while calling fail\()
+buildfile:1:2: info: while calling fail()
 EOE
 
 : fail-invalid-arg
 :
 $* <'$fail_arg(abc)' 2>>EOE != 0
 error: invalid argument: invalid uint64 value: 'abc'
-buildfile:1:2: info: while calling fail_arg\(<untyped>)
+buildfile:1:2: info: while calling fail_arg(<untyped>)
 EOE
 
 : no-match-name
 :
 $* <'$bogus()' 2>>EOE != 0
-buildfile:1:2: error: unmatched call to bogus\()
+buildfile:1:2: error: unmatched call to bogus()
 EOE
 
 : no-match-count
 :
 $* <'$dummy0(abc)' 2>>EOE != 0
-buildfile:1:2: error: unmatched call to dummy0\(<untyped>)
-  info: candidate: dummy0\(), qualified name dummy.dummy0
+buildfile:1:2: error: unmatched call to dummy0(<untyped>)
+  info: candidate: dummy0(), qualified name dummy.dummy0
 EOE
 
 : no-match-type
 :
 $* <'$dummy1([uint64] 123)' 2>>EOE != 0
-buildfile:1:2: error: unmatched call to dummy1\(uint64)
-  info: candidate: dummy1\(string), qualified name dummy.dummy1
+buildfile:1:2: error: unmatched call to dummy1(uint64)
+  info: candidate: dummy1(string), qualified name dummy.dummy1
 EOE
 
 : ambig
 :
 $* <'$ambig(abc)' 2>- != 0 # @@ REGEX
-#buildfile:1:2: error: ambiguous call to ambig\(<untyped>)
-#  info: candidate: ambig\(<untyped> [, uint64]), qualified name dummy.ambig
-#  info: candidate: ambig\(<untyped> [, string]), qualified name dummy.ambig
+#buildfile:1:2: error: ambiguous call to ambig(<untyped>)
+#  info: candidate: ambig(<untyped> [, uint64]), qualified name dummy.ambig
+#  info: candidate: ambig(<untyped> [, string]), qualified name dummy.ambig
 #EOE
 
 : optional-absent
@@ -90,19 +90,19 @@ $* <'print $nullable(nonull)' >'false'
 :
 $* <'$dummy1([string null])' 2>>EOE != 0
 error: invalid argument: null value
-buildfile:1:2: info: while calling dummy1\(string)
+buildfile:1:2: info: while calling dummy1(string)
 EOE
 
 : print-call-1-untyped
 :
 $* <'$bogus(abc)' 2>>EOE != 0
-buildfile:1:2: error: unmatched call to bogus\(<untyped>)
+buildfile:1:2: error: unmatched call to bogus(<untyped>)
 EOE
 
 : print-call-1-typed
 :
 $* <'$bogus([uint64] 123)' 2>>EOE != 0
-buildfile:1:2: error: unmatched call to bogus\(uint64)
+buildfile:1:2: error: unmatched call to bogus(uint64)
 EOE
 
 #\
@@ -110,23 +110,23 @@ EOE
 : print-call-2
 :
 $* <'$bogus(abc, [uint64] 123)' 2>>EOE != 0
-buildfile:1:2: error: unmatched call to bogus\(<untyped>, uint64)
+buildfile:1:2: error: unmatched call to bogus(<untyped>, uint64)
 EOE
 #\
 
 : print-fovl
 :
 $* <'$ambig([bool] true)' 2>- != 0 # @@ REGEX
-#buildfile:1:2: error: unmatched call to ambig\(bool)
-#  info: candidate: ambig\(<untyped> [, uint64]), qualified name dummy.ambig
-#  info: candidate: ambig\(<untyped> [, string]), qualified name dummy.ambig
+#buildfile:1:2: error: unmatched call to ambig(bool)
+#  info: candidate: ambig(<untyped> [, uint64]), qualified name dummy.ambig
+#  info: candidate: ambig(<untyped> [, string]), qualified name dummy.ambig
 #EOE
 
 : print-fovl-variadic
 :
 $* <'$variadic(abc)' 2>>EOE != 0
-buildfile:1:2: error: unmatched call to variadic\(<untyped>)
-  info: candidate: variadic\(bool [, ...])
+buildfile:1:2: error: unmatched call to variadic(<untyped>)
+  info: candidate: variadic(bool [, ...])
 EOE
 
 : member-function
diff --git a/unit-tests/function/syntax.test b/unit-tests/function/syntax.test
index 9e653c8..d644fd1 100644
--- a/unit-tests/function/syntax.test
+++ b/unit-tests/function/syntax.test
@@ -19,9 +19,9 @@ $* <<EOI >>EOO
 foo = FOO
 bar = BAR
 
-print \$foo"\(\$bar)"
-print "\$foo"\(\$bar)
-print "\$foo""\(\$bar)"
+print $foo"($bar)"
+print "$foo"($bar)
+print "$foo""($bar)"
 EOI
 FOOBAR
 FOOBAR
diff --git a/unit-tests/lexer/comment.test b/unit-tests/lexer/comment.test
index 07d7ac5..65e768c 100644
--- a/unit-tests/lexer/comment.test
+++ b/unit-tests/lexer/comment.test
@@ -59,34 +59,34 @@ EOO
 #
 
 $* <<EOI >>:EOO # multi-only
-#\\
+#\
 comment
 comment
-#\\
+#\
 EOI
 EOO
 
 $* <<EOI >>:EOO # multi-empty
-#\\
-#\\
+#\
+#\
 EOI
 EOO
 
 $* <<EOI >>EOO # multi-start-same
-foo #\\
+foo #\
 comment
 comment
-#\\
+#\
 EOI
 'foo'
 <newline>
 EOO
 
 $* <<EOI >>EOO # multi-end-same
-#\\
+#\
 comment
 comment
-foo #\\
+foo #\
 bar
 EOI
 'bar'
@@ -94,10 +94,10 @@ EOI
 EOO
 
 $* <<EOI >>EOO # multi-end-not
-#\\
+#\
 comment
-#\\ not an end
-foo #\\
+#\ not an end
+foo #\
 bar
 EOI
 'bar'
@@ -105,7 +105,7 @@ EOI
 EOO
 
 $* <<EOI 2>>EOE != 0 # multi-unterm
-#\\
+#\
 comment
 EOI
 stdin:3:1: error: unterminated multi-line comment
diff --git a/unit-tests/lexer/quoting.test b/unit-tests/lexer/quoting.test
index 76fd904..aab02c3 100644
--- a/unit-tests/lexer/quoting.test
+++ b/unit-tests/lexer/quoting.test
@@ -48,7 +48,7 @@ EOO
 :
 $* <'"$foo"' >>EOO
 '' [D/P]
-\$
+$
 'foo' [D/P]
 <newline>
 EOO
@@ -58,7 +58,7 @@ EOO
 :
 $* <'"foo$"' >>EOO
 'foo' [D/P]
-\$
+$
 '' [D/P]
 <newline>
 EOO
diff --git a/unit-tests/test/script/lexer/driver.cxx b/unit-tests/test/script/lexer/driver.cxx
index abd32ba..3709191 100644
--- a/unit-tests/test/script/lexer/driver.cxx
+++ b/unit-tests/test/script/lexer/driver.cxx
@@ -34,7 +34,8 @@ namespace build2
           else if (s == "second-token")     m = lexer_mode::second_token;
           else if (s == "variable-line")    m = lexer_mode::variable_line;
           else if (s == "command-line")     m = lexer_mode::command_line;
-          else if (s == "here-line")        m = lexer_mode::here_line;
+          else if (s == "here-line-single") m = lexer_mode::here_line_single;
+          else if (s == "here-line-double") m = lexer_mode::here_line_double;
           else if (s == "description-line") m = lexer_mode::description_line;
           else if (s == "variable")         m = lexer_mode::variable;
           else                           assert (false);
diff --git a/unit-tests/test/script/parser/command-if.test b/unit-tests/test/script/parser/command-if.test
index 88cc7d6..4bbc016 100644
--- a/unit-tests/test/script/parser/command-if.test
+++ b/unit-tests/test/script/parser/command-if.test
@@ -249,7 +249,7 @@ if true
 else
   x = bar
 end;
-cmd \$x
+cmd $x
 EOI
 ? true
 cmd foo
diff --git a/unit-tests/test/script/parser/command-re-parse.test b/unit-tests/test/script/parser/command-re-parse.test
index aee4f78..335ff69 100644
--- a/unit-tests/test/script/parser/command-re-parse.test
+++ b/unit-tests/test/script/parser/command-re-parse.test
@@ -2,8 +2,8 @@
 # double-quote
 #
 $* <<EOI >>EOO
-x = cmd \\">-\\" "'<-'"
-\$x
+x = cmd \">-\" "'<-'"
+$x
 EOI
 cmd '>-' '<-'
 EOO
diff --git a/unit-tests/test/script/parser/description.test b/unit-tests/test/script/parser/description.test
index 1b3f358..b4ab435 100644
--- a/unit-tests/test/script/parser/description.test
+++ b/unit-tests/test/script/parser/description.test
@@ -180,7 +180,7 @@ EOE
 $* <<EOI >>EOO                                         # legal-var
 : foo bar
 x = y;
-cmd \$x
+cmd $x
 EOI
 : sm:foo bar
 cmd y
@@ -334,7 +334,7 @@ EOO
 :
 : No merge since test has description.
 :
-$* -s -i <<EOI >>EOO                    # 
+$* -s -i <<EOI >>EOO                    #
 {
   : foo-bar
   : foo bar
diff --git a/unit-tests/test/script/parser/expansion.test b/unit-tests/test/script/parser/expansion.test
index c23d598..1d10a63 100644
--- a/unit-tests/test/script/parser/expansion.test
+++ b/unit-tests/test/script/parser/expansion.test
@@ -6,7 +6,7 @@
 $* <<EOI >>EOO
 x = dir/ proj% proj%name proj%proj%dir/type{name   name   {name}}
 cmd dir/ proj% proj%name proj%proj%dir/type{name   name   {name}}
-cmd \$x
+cmd $x
 EOI
 cmd dir/ proj% proj%name proj%proj%dir/type{name name {name}}
 cmd dir/ proj% proj%name proj%proj%dir/type{name name {name}}
diff --git a/unit-tests/test/script/parser/here-document.test b/unit-tests/test/script/parser/here-document.test
index 6f26166..7cb9474 100644
--- a/unit-tests/test/script/parser/here-document.test
+++ b/unit-tests/test/script/parser/here-document.test
@@ -128,8 +128,8 @@
   :
   $* <<EOI >>EOO
   x = foo bar
-  cmd <<EOF
-    \$x
+  cmd <<"EOF"
+    $x
     EOF
   EOI
   cmd <<EOF
@@ -142,8 +142,8 @@
   :
   $* <<EOI >>EOO
   x = foo
-  cmd <<EOF
-     \$x bar \$x
+  cmd <<"EOF"
+     $x bar $x
     EOF
   EOI
   cmd <<EOF
@@ -185,7 +185,7 @@ EOO
 : Note: they are still recognized in eval contexts.
 :
 $* <<EOI >>EOO
-cmd <<EOF
+cmd <<"EOF"
 'single'
 "double"
 b'o't"h"
diff --git a/unit-tests/test/script/parser/include.test b/unit-tests/test/script/parser/include.test
index 65ce7ce..7a3b517 100644
--- a/unit-tests/test/script/parser/include.test
+++ b/unit-tests/test/script/parser/include.test
@@ -14,7 +14,7 @@ EOO
 touch foo.test;
 $* <<EOI
 x = foo.test
-.include\$x
+.include$x
 EOI
 
 : none
@@ -108,7 +108,7 @@ cat <<EOI >>>foo-$(build.version).test;
 cmd
 EOI
 $* <<EOI >>EOO
-.include foo-\$\(build.version\).test
+.include foo-$(build.version).test
 EOI
 cmd
 EOO
diff --git a/unit-tests/test/script/parser/scope.test b/unit-tests/test/script/parser/scope.test
index 3b10d01..6ddb265 100644
--- a/unit-tests/test/script/parser/scope.test
+++ b/unit-tests/test/script/parser/scope.test
@@ -74,7 +74,7 @@ EOO
 $* -s -i <<EOI >>EOO                    # test-scope-var
 {
   x = abc
-  cmd \$x
+  cmd $x
 }
 EOI
 {
@@ -88,7 +88,7 @@ $* -s -i <<EOI >>EOO                    # test-scope-setup
 {
   x = abc
   +setup
-  cmd \$x
+  cmd $x
 }
 EOI
 {
@@ -166,7 +166,7 @@ EOO
 $* -s <<EOI >>EOO                    # test-var
 cmd1;
 x = abc;
-cmd2 \$x
+cmd2 $x
 EOI
 {
   {
@@ -178,7 +178,7 @@ EOO
 
 $* -s <<EOI >>EOO                    # test-var-first
 x = abc;
-cmd \$x
+cmd $x
 EOI
 {
   {
@@ -189,7 +189,7 @@ EOO
 
 $* -s <<EOI >>EOO                    # var-setup-tdown
 x = abc
-cmd \$x
+cmd $x
 y = 123
 EOI
 {