Allow here-document end marker to be wholly quoted

7 files changed, 133 insertions, 16 deletions

diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index a116873..7655ba9 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -1652,8 +1652,45 @@ namespace build2
                   //
                   next (t, tt);
 
-                  if (tt != type::word || t.qtype != quote_type::unquoted)
-                    fail (l) << "expected here-document end marker";
+                  // We require the end marker to be an unquoted or completely
+                  // quoted word. The complete quoting becomes important for
+                  // cases like foo"$bar" (where we will see word 'foo').
+                  //
+                  // For good measure we could have also required it to be
+                  // separated from the following token, but out grammar
+                  // allows one to write >>EOO;. The problematic sequence
+                  // would be >>FOO$bar -- on reparse it will be expanded
+                  // as a single word.
+                  //
+                  if (tt != type::word)
+                    fail (t) << "expected here-document end marker";
+
+                  peek ();
+                  const token& p (peeked ());
+                  if (!p.separated)
+                  {
+                    switch (p.type)
+                    {
+                    case type::dollar:
+                    case type::lparen:
+                      fail (p) << "here-document end marker must be literal";
+                    }
+                  }
+
+                  quote_type qt (t.qtype);
+                  switch (qt)
+                  {
+                  case quote_type::unquoted:
+                    qt = quote_type::single; // Treat as single-quoted.
+                    break;
+                  case quote_type::single:
+                  case quote_type::double_:
+                    if (t.qcomp)
+                      break;
+                    // Fall through.
+                  case quote_type::mixed:
+                    fail (t) << "partially-quoted here-document end marker";
+                  }
 
                   hd.push_back (here_doc {0, 0, 0, move (t.value), nn});
                   break;
diff --git a/build2/test/script/script.cxx b/build2/test/script/script.cxx
index b128077..8fb8115 100644
--- a/build2/test/script/script.cxx
+++ b/build2/test/script/script.cxx
@@ -38,14 +38,16 @@ namespace build2
       }
 
       // Quote if empty or contains spaces or any of the special characters.
+      // Note that we use single quotes since double quotes still allow
+      // expansion.
       //
-      // @@ What if it contains quotes, escapes?
+      // @@ What if it contains single quotes?
       //
       static void
       to_stream_q (ostream& o, const string& s)
       {
-        if (s.empty () || s.find_first_of (" |&<>=") != string::npos)
-          o << '"' << s << '"';
+        if (s.empty () || s.find_first_of (" |&<>=\\\"") != string::npos)
+          o << '\'' << s << '\'';
         else
           o << s;
       };
@@ -100,7 +102,8 @@ namespace build2
               // Add another '>' or '<'. Note that here end marker never
               // needs to be quoted.
               //
-              o << d << (nl ? "" : ":") << r.doc.end;
+              o << d << (nl ? "" : ":");
+              to_stream_q (o, r.doc.end);
               break;
             }
           case redirect_type::file:
diff --git a/unit-tests/test/script/lexer/buildfile b/unit-tests/test/script/lexer/buildfile
index 16be005..ac833e4 100644
--- a/unit-tests/test/script/lexer/buildfile
+++ b/unit-tests/test/script/lexer/buildfile
@@ -10,6 +10,6 @@ test/script/{token lexer}
 
 exe{driver}: cxx{driver} ../../../../build2/cxx{$src} $libs \
 test{script-line command-line first-token second-token variable-line \
-     description-line variable comment}
+     description-line variable}
 
 include ../../../../build2/
diff --git a/unit-tests/test/script/parser/command-re-parse.test b/unit-tests/test/script/parser/command-re-parse.test
index 3b9ae3e..aee4f78 100644
--- a/unit-tests/test/script/parser/command-re-parse.test
+++ b/unit-tests/test/script/parser/command-re-parse.test
@@ -5,5 +5,5 @@ $* <<EOI >>EOO
 x = cmd \\">-\\" "'<-'"
 \$x
 EOI
-cmd ">-" "<-"
+cmd '>-' '<-'
 EOO
diff --git a/unit-tests/test/script/parser/here-document.test b/unit-tests/test/script/parser/here-document.test
index 4fa62d2..6f26166 100644
--- a/unit-tests/test/script/parser/here-document.test
+++ b/unit-tests/test/script/parser/here-document.test
@@ -1,3 +1,78 @@
+: end-marker
+:
+{
+  : missing-newline
+  :
+  $* <'cmd <<' 2>>EOE != 0
+  testscript:1:7: error: expected here-document end marker
+  EOE
+
+  : missing-exit
+  :
+  $* <'cmd << != 0' 2>>EOE != 0
+  testscript:1:8: error: expected here-document end marker
+  EOE
+
+  : unseparated-expansion
+  :
+  $* <'cmd <<FOO$foo' 2>>EOE != 0
+  testscript:1:10: error: here-document end marker must be literal
+  EOE
+
+  : quoted-single-partial
+  :
+  $* <"cmd <<F'O'O" 2>>EOE != 0
+  testscript:1:7: error: partially-quoted here-document end marker
+  EOE
+
+  : quoted-double-partial
+  :
+  $* <'cmd <<"FO"O' 2>>EOE != 0
+  testscript:1:7: error: partially-quoted here-document end marker
+  EOE
+
+  : quoted-mixed
+  :
+  $* <"cmd <<\"FO\"'O'" 2>>EOE != 0
+  testscript:1:7: error: partially-quoted here-document end marker
+  EOE
+
+  : unseparated
+  :
+  $* <<EOI >>EOO
+  cmd <<EOF!=0
+  foo
+  EOF
+  EOI
+  cmd <<EOF != 0
+  foo
+  EOF
+  EOO
+
+  : quoted-single
+  :
+  $* <<EOI >>EOO
+  cmd <<'EOF'
+  foo
+  EOF
+  EOI
+  cmd <<EOF
+  foo
+  EOF
+  EOO
+
+  : quoted-double
+  :
+  $* <<EOI >>EOO
+  cmd <<"EOF"
+  foo
+  EOF
+  EOI
+  cmd <<EOF
+  foo
+  EOF
+  EOO
+}
 
 : indent
 :
@@ -87,7 +162,9 @@
   EOE
 }
 
-$* <<EOI >>EOO  # blank-lines
+: blank
+:
+$* <<EOI >>EOO
 cmd <<EOF
 foo
 
@@ -103,10 +180,10 @@ bar
 EOF
 EOO
 
-# quote
-#
-# Note: they are still recognized in eval contexts.
-#
+: quote
+:
+: Note: they are still recognized in eval contexts.
+:
 $* <<EOI >>EOO
 cmd <<EOF
 'single'
diff --git a/unit-tests/test/script/parser/here-string.test b/unit-tests/test/script/parser/here-string.test
index 9f44bb2..9c4b68b 100644
--- a/unit-tests/test/script/parser/here-string.test
+++ b/unit-tests/test/script/parser/here-string.test
@@ -1,11 +1,11 @@
 $* <<EOI >>EOO # empty
 cmd <""
 EOI
-cmd <""
+cmd <''
 EOO
 
 $* <<EOI >>EOO # empty-nn
 cmd <:""
 EOI
-cmd <:""
+cmd <:''
 EOO
diff --git a/unit-tests/test/script/parser/redirect.test b/unit-tests/test/script/parser/redirect.test
index af4295a..2642834 100644
--- a/unit-tests/test/script/parser/redirect.test
+++ b/unit-tests/test/script/parser/redirect.test
@@ -11,7 +11,7 @@ EOO
 $* <<EOI >>EOO          # quote-file
 cmd 0<<<"a f" 1>>>"b f" 2>>>&"c f"
 EOI
-cmd <<<"a f" >>>"b f" 2>>>&"c f"
+cmd <<<'a f' >>>'b f' 2>>>&'c f'
 EOO
 
 $* <<EOI 2>>EOE !=0     # in-file-fail1