Add set builtin

8 files changed, 579 insertions, 18 deletions

diff --git a/build2/parser b/build2/parser
index e589948..df2797b 100644
--- a/build2/parser
+++ b/build2/parser
@@ -108,7 +108,7 @@ namespace build2
     apply_value_attributes (const variable*, // Optional.
                             value& lhs,
                             value&& rhs,
-                            token_type kind);
+                            token_type assign_kind);
 
     // Return the value pack (values can be NULL/typed). Note that for an
     // empty eval context ('()' potentially with whitespaces in between) the
diff --git a/build2/test/script/parser b/build2/test/script/parser
index cb51042..cb536e6 100644
--- a/build2/test/script/parser
+++ b/build2/test/script/parser
@@ -34,6 +34,19 @@ namespace build2
         void
         pre_parse (istream&, script&);
 
+        // Helpers.
+        //
+        // Parse attribute string and perform attribute-guided assignment.
+        // Issue diagnostics and throw failed in case of an error.
+        //
+        void
+        apply_value_attributes (const variable*, // Optional.
+                                value& lhs,
+                                value&& rhs,
+                                const string& attributes,
+                                token_type assign_kind,
+                                const path& name); // For diagnostics.
+
         // Recursive descent parser.
         //
         // Usually (but not always) parse functions receive the token/type
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index 59580a3..2ea42b5 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -3034,7 +3034,8 @@ namespace build2
                           ? scope_->assign (var)
                           : scope_->append (var));
 
-              apply_value_attributes (&var, lhs, move (rhs), kind);
+              build2::parser::apply_value_attributes (
+                &var, lhs, move (rhs), kind);
 
               // If we changes any of the test.* values, then reset the $*,
               // $N special aliases.
@@ -3203,13 +3204,23 @@ namespace build2
         // only look for buildfile variables.
         //
         // Otherwise, every variable that is ever set in a script has been
-        // pre-entered during pre-parse. Which means that if one is not found
-        // in the script pool then it can only possibly be set in the
-        // buildfile.
+        // pre-entered during pre-parse or introduced with the set builtin
+        // during test execution. Which means that if one is not found in the
+        // script pool then it can only possibly be set in the buildfile.
         //
-        const variable* pvar (scope_ != nullptr
-                              ? script_->var_pool.find (name)
-                              : nullptr);
+        // Note that we need to acquire the variable pool lock. The pool can
+        // be changed from multiple threads by the set builtin. The obtained
+        // variable pointer can safelly be used with no locking as the variable
+        // pool is an associative container (underneath) and we are only adding
+        // new variables into it.
+        //
+        const variable* pvar (nullptr);
+
+        if (scope_ != nullptr)
+        {
+          slock sl (script_->var_pool_mutex);
+          pvar = script_->var_pool.find (name);
+        }
 
         return pvar != nullptr
           ? scope_->find (*pvar)
@@ -3269,6 +3280,35 @@ namespace build2
         base_parser::lexer_ = l;
       }
 
+      void parser::
+      apply_value_attributes (const variable* var,
+                              value& lhs,
+                              value&& rhs,
+                              const string& attributes,
+                              token_type kind,
+                              const path& name)
+      {
+        path_ = &name;
+
+        istringstream is (attributes);
+        lexer l (is, name, lexer_mode::attribute);
+        set_lexer (&l);
+
+        token t;
+        type tt;
+        next (t, tt);
+
+        if (tt != type::lsbrace && tt != type::eos)
+          fail (t) << "expected '[' instead of " << t;
+
+        attributes_push (t, tt, true);
+
+        if (tt != type::eos)
+          fail (t) << "trailing junk after ']'";
+
+        build2::parser::apply_value_attributes (var, lhs, move (rhs), kind);
+      }
+
       // parser::parsed_doc
       //
       parser::parsed_doc::
diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx
index bc3c1ce..e8040e2 100644
--- a/build2/test/script/runner.cxx
+++ b/build2/test/script/runner.cxx
@@ -10,11 +10,13 @@
 #include <butl/fdstream> // fdopen_mode, fdnull(), fddup()
 
 #include <build2/regex>
+#include <build2/variable>
 #include <build2/filesystem>
 
 #include <build2/test/common>
 
 #include <build2/test/script/regex>
+#include <build2/test/script/parser>
 #include <build2/test/script/builtin>
 
 using namespace std;
@@ -792,6 +794,203 @@ namespace build2
                             : sp.wd_path.directory ());
       }
 
+      // The set pseudo-builtin: set variable from the stdin input.
+      //
+      // set [-e|--exact] [(-n|--newline)|(-w|--whitespace)] [<attr>] <var>
+      //
+      // -e|--exact
+      //    Unless the option is specified, a single final newline is ignored
+      //    in the input.
+      //
+      // -n|--newline
+      //    Split the input into a list of elements at newlines, including a
+      //    final blank element in case of -e. Multiple consecutive newlines
+      //    are not collapsed.
+      //
+      // -w|--whitespace
+      //    Split the input into a list of elements at whitespaces, including a
+      //    final blank element in case of -e. Multiple consecutive whitespaces
+      //    (including newlines) are collapsed.
+      //
+      // If the attr argument is specified, then it must contain a list of
+      // value attributes enclosed in [].
+      //
+      static void
+      set_builtin (scope& sp,
+                   const strings& args,
+                   auto_fd in,
+                   const location& ll)
+      {
+        try
+        {
+          // Do not throw when eofbit is set (end of stream reached), and
+          // when failbit is set (read operation failed to extract any
+          // character).
+          //
+          ifdstream cin  (move (in), ifdstream::badbit);
+
+          auto i (args.begin ());
+          auto e (args.end ());
+
+          // Process options.
+          //
+          bool exact (false);
+          bool newline (false);
+          bool whitespace (false);
+
+          for (; i != e; ++i)
+          {
+            const string& o (*i);
+
+            if (o == "-e" || o == "--exact")
+              exact = true;
+            else if (o == "-n" || o == "--newline")
+              newline = true;
+            else if (o == "-w" || o == "--whitespace")
+              whitespace = true;
+            else
+            {
+              if (*i == "--")
+                ++i;
+
+              break;
+            }
+          }
+
+          // Process arguments.
+          //
+          if (i == e)
+            fail (ll) << "missing variable name";
+
+          const string& a (*i++); // Either attributes or variable name.
+          const string* ats (i == e ? nullptr : &a);
+          const string& vname (i == e ? a : *i++);
+
+          if (i != e)
+	    fail (ll) << "unexpected argument";
+
+          if (ats != nullptr && ats->empty ())
+	    fail (ll) << "empty variable attributes";
+
+          if (vname.empty ())
+	    fail (ll) << "empty variable name";
+
+          // Read the input.
+          //
+          cin.peek (); // Sets eofbit for an empty stream.
+
+          names ns;
+          while (!cin.eof ())
+          {
+            // Read next element that depends on the whitespace mode being
+            // enabled or not. For the later case it also make sense to strip
+            // the trailing CRs that can appear while cross-testing Windows
+            // target or as a part of msvcrt junk production (see above).
+            //
+            string s;
+            if (whitespace)
+              cin >> s;
+            else
+            {
+              getline (cin, s);
+
+              while (!s.empty () && s.back () == '\r')
+                s.pop_back ();
+            }
+
+            // If failbit is set then we read nothing into the string as eof is
+            // reached. That in particular means that the stream has trailing
+            // whitespaces (possibly including newlines) if the whitespace mode
+            // is enabled, or the trailing newline otherwise. If so then
+            // we append the "blank" to the variable value in the exact mode
+            // prior to bailing out.
+            //
+            if (cin.fail ())
+            {
+              if (exact)
+              {
+                if (whitespace || newline)
+                  ns.emplace_back (move (s)); // Reuse empty string.
+                else if (ns.empty ())
+                  ns.emplace_back ("\n");
+                else
+                  ns[0].value += '\n';
+              }
+
+              break;
+            }
+
+            if (whitespace || newline || ns.empty ())
+              ns.emplace_back (move (s));
+            else
+            {
+              ns[0].value += '\n';
+              ns[0].value += s;
+            }
+          }
+
+          cin.close ();
+
+          // Set the variable value and attributes. Note that we need to aquire
+          // unique lock before potentially changing the script's variable
+          // pool. The obtained variable reference can safelly be used with no
+          // locking as the variable pool is an associative container
+          // (underneath) and we are only adding new variables into it.
+          //
+          ulock ul (sp.root->var_pool_mutex);
+          const variable& var (sp.root->var_pool.insert (move (vname)));
+          ul.unlock ();
+
+          value& lhs (sp.assign (var));
+
+          // If there are no attributes specified then the variable assignment
+          // is straightforward. Otherwise we will use the build2 parser helper
+          // function.
+          //
+          if (ats == nullptr)
+            lhs.assign (move (ns), &var);
+          else
+          {
+            // Come up with a "path" that contains both the expression line
+            // location as well as the attributes string. The resulting
+            // diagnostics will look like this:
+            //
+            // testscript:10:1: ([x]):1:1: error: unknown value attribute x
+            //
+            path name;
+            {
+              string n (ll.file->string ());
+              n += ':';
+
+              if (!ops.no_line ())
+              {
+                n += to_string (ll.line);
+                n += ':';
+
+                if (!ops.no_column ())
+                {
+                  n += to_string (ll.column);
+                  n += ':';
+                }
+              }
+
+              n += " (";
+              n += *ats;
+              n += ')';
+              name = path (move (n));
+            }
+
+            parser p;
+            p.apply_value_attributes(
+              &var, lhs, value (move (ns)), *ats, token_type::assign, name);
+          }
+        }
+        catch (const io_error& e)
+        {
+          fail (ll) << "set: " << e;
+        }
+      }
+
       static bool
       run_pipe (scope& sp,
                 command_pipe::const_iterator bc,
@@ -864,11 +1063,14 @@ namespace build2
           return normalize (move (p), sp, ll);
         };
 
+        const redirect& in  (c.in.effective ());
+        const redirect& out (c.out.effective ());
+        const redirect& err (c.err.effective ());
+
         // If stdin file descriptor is not open then this is the first pipeline
         // command. Open stdin descriptor according to the redirect specified.
         //
         path isp;
-        const redirect& in (c.in.effective ());
 
         if (ifd.get () != -1)
           assert (in.type == redirect_type::none); // No redirect expected.
@@ -973,6 +1175,40 @@ namespace build2
           }
         }
 
+        assert (ifd.get () != -1);
+
+        command_pipe::const_iterator nc (bc + 1);
+        bool last (nc == ec);
+
+        // Prior to follow up with opening file descriptors for command
+        // outputs redirects let's check if the command is the set builtin.
+        // Being a builtin syntactically it differs from the regular ones in a
+        // number of ways. It either succeeds or terminates abnormally, so
+        // redirecting stderr is meaningless. It also never produces any output
+        // and may appear only as a terminal command in a pipeline. That means
+        // we can short-circuit here calling the builtin and returning right
+        // after that. Checking that the user didn't specify any meaningless
+        // redirects or exit code check sounds as a right thing to do.
+        //
+        if (c.program.string () == "set")
+        {
+          if (!last)
+            fail (ll) << "set builtin must be the last command in a pipe";
+
+          if (out.type != redirect_type::none)
+            fail (ll) << "set builtin stdout must not be redirected";
+
+          if (err.type != redirect_type::none)
+            fail (ll) << "set builtin stderr must not be redirected";
+
+          if ((c.exit.comparison == exit_comparison::eq) !=
+              (c.exit.status == 0))
+            fail (ll) << "set builtin exit status must not be other than zero";
+
+          set_builtin (sp, c.arguments, move (ifd), ll);
+          return true;
+        }
+
         // Open a file for command output redirect if requested explicitly
         // (file overwrite/append redirects) or for the purpose of the output
         // validation (none, here_*, file comparison redirects), register the
@@ -1079,7 +1315,6 @@ namespace build2
         };
 
         path osp;
-        const redirect& out (c.out.effective ());
         auto_fd ofd;
 
         // If this is the last command in the pipeline than redirect the
@@ -1096,9 +1331,6 @@ namespace build2
         //    test failures investigation and for tests "tightening".
         //
         fdpipe p;
-        command_pipe::const_iterator nc (bc + 1);
-        bool last (nc == ec);
-
         if (last)
           ofd = open (out, 1, osp);
         else
@@ -1118,7 +1350,6 @@ namespace build2
         }
 
         path esp;
-        const redirect& err (c.err.effective ());
         auto_fd efd (open (err, 2, esp));
 
         // Merge standard streams.
@@ -1143,7 +1374,7 @@ namespace build2
 
         // All descriptors should be open to the date.
         //
-        assert (ifd.get () != -1 && ofd.get () != -1 && efd.get () != -1);
+        assert (ofd.get () != -1 && efd.get () != -1);
 
         optional<process_exit> exit;
         builtin* b (builtins.find (c.program.string ()));
diff --git a/build2/test/script/script b/build2/test/script/script
index 2438fa5..04c54db 100644
--- a/build2/test/script/script
+++ b/build2/test/script/script
@@ -495,6 +495,7 @@ namespace build2
 
       public:
         variable_pool var_pool;
+        mutable shared_mutex var_pool_mutex;
 
         const variable& test_var;      // test
         const variable& options_var;   // test.options
diff --git a/doc/testscript.cli b/doc/testscript.cli
index 918dd4a..5b4e8ee 100644
--- a/doc/testscript.cli
+++ b/doc/testscript.cli
@@ -2457,8 +2457,9 @@ list of elements at newlines, including a final blank element in case of
 
 If the \c{-w|--whitespace} option is specified, then the input is split into a
 list of elements at whitespaces, including a final blank element in case of
-\c{-e|--exact}. Multiple consecutive whitespaces (including newlines) are
-collapsed.
+\c{-e|--exact}. In this mode if \c{-e|--exact} is not specified, then all (and
+not just newline) trailing whitespaces are ignored. Multiple consecutive
+whitespaces (including newlines) are collapsed.
 
 If neither \c{-n|--newline} nor \c{-w|--whitespace} is specified, then the
 entire input is used as a single element, including a final newline in case
diff --git a/tests/test/script/runner/buildfile b/tests/test/script/runner/buildfile
index df37c6d..c3df228 100644
--- a/tests/test/script/runner/buildfile
+++ b/tests/test/script/runner/buildfile
@@ -2,7 +2,7 @@
 # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-./: test{cleanup expr if pipe redirect regex status} exe{driver} $b
+./: test{cleanup expr if pipe redirect regex set status} exe{driver} $b
 
 test{*}: target = exe{driver}
 
diff --git a/tests/test/script/runner/set.test b/tests/test/script/runner/set.test
new file mode 100644
index 0000000..7c24669
--- /dev/null
+++ b/tests/test/script/runner/set.test
@@ -0,0 +1,275 @@
+# file      : tests/test/script/runner/set.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+.include ../common.test
+
+: special
+:
+{
+  : pipelining
+  :
+  $c <'set foo | cat >bar' && $b 2>>EOE != 0
+  testscript:1:1: error: set builtin must be the last command in a pipe
+  EOE
+
+  : redirecting
+  :
+  {
+    : stdout
+    :
+    $c <'set foo >bar' && $b 2>>EOE != 0
+    testscript:1:1: error: set builtin stdout must not be redirected
+    EOE
+
+    : stderr
+    :
+    $c <'set foo 2>bar' && $b 2>>EOE != 0
+    testscript:1:1: error: set builtin stderr must not be redirected
+    EOE
+  }
+
+  : status
+  :
+  $c <'set foo == 1' && $b 2>>EOE != 0
+  testscript:1:1: error: set builtin exit status must not be other than zero
+  EOE
+}
+
+: arguments
+:
+{
+  : none
+  :
+  $c <'set -e' && $b 2>>EOE != 0
+  testscript:1:1: error: missing variable name
+  EOE
+
+  : unexpected
+  :
+  $c <'set foo bar baz' && $b 2>>EOE != 0
+  testscript:1:1: error: unexpected argument
+  EOE
+
+  : empty-attrs
+  :
+  $c <"set '' baz" && $b 2>>EOE != 0
+  testscript:1:1: error: empty variable attributes
+  EOE
+
+  : empty-var
+  :
+  $c <"set ''" && $b 2>>EOE != 0
+  testscript:1:1: error: empty variable name
+  EOE
+}
+
+: whitespace-separated-list
+:
+{
+  : non-exact
+  :
+  $c <<EOI && $b
+  set -w baz <' foo   bar ';
+  echo $baz >'foo bar'
+  EOI
+
+  : exact
+  :
+  {
+    : trailing-ws
+    :
+    $c <<EOI && $b
+    set -e -w baz <' foo   bar ';
+    echo $baz >'foo bar '
+    EOI
+
+    : no-trailing-ws
+    :
+    : Note that we need to strip the default trailing newline as well with the
+    : ':' modifier.
+    :
+    $c <<EOI && $b
+    set -e -w baz <:' foo bar';
+    echo $baz >'foo bar'
+    EOI
+  }
+}
+
+: newline-separated-list
+:
+{
+  : non-exact
+  :
+  $c <<EOI && $b
+  set -n baz <<EOF;
+
+  foo
+
+  bar
+
+  EOF
+  echo $baz >' foo  bar '
+  EOI
+
+  : exact
+  :
+  {
+    : trailing-newline
+    :
+    $c <<EOI && $b
+    set -e -n baz <<EOF;
+
+    foo
+
+    bar
+
+    EOF
+    echo $baz >' foo  bar  '
+    EOI
+
+    : no-trailing-newline
+    :
+    $c <<EOI && $b
+    set -e -n baz <<:EOF;
+
+    foo
+
+    bar
+    EOF
+    echo $baz >' foo  bar'
+    EOI
+  }
+}
+
+: string
+:
+{
+  : non-exact
+  :
+  $c <<EOI && $b
+  set baz <<EOF;
+
+  foo
+
+  bar
+
+  EOF
+  echo $baz >>EOO
+
+  foo
+
+  bar
+
+  EOO
+  EOI
+
+  : roundtrip
+  :
+  echo 'foo' | set bar;
+  echo "$bar" >'foo'
+
+  : exact
+  :
+  : Note that echo adds the trailing newline, so EOF and EOO here-documents
+  : differ by this newline.
+  :
+  {
+    : trailing-newline
+    :
+    $c <<EOI && $b
+    set -e baz <<EOF;
+
+    foo
+
+    bar
+    EOF
+    echo "$baz" >>EOO
+
+    foo
+
+    bar
+
+    EOO
+    EOI
+
+    : no-trailing-newline
+    :
+    $c <<EOI && $b
+    set -e baz <<:EOF;
+
+    foo
+
+    bar
+    EOF
+    echo "$baz" >>EOO
+
+    foo
+
+    bar
+    EOO
+    EOI
+  }
+}
+
+: attributes
+:
+{
+  : dir_path
+  :
+  $c <<EOI && $b
+  set [dir_path] bar <'foo';
+  echo $bar >/'foo/'
+  EOI
+
+  : null
+  :
+  $c <<EOI && $b
+  set [null] foo <-;
+  echo $foo >''
+  EOI
+
+  : none
+  :
+  $c <<EOI && $b 2>>EOE != 0
+  set -w baz <'foo bar';
+  echo "$baz"
+  EOI
+  testscript:2:8: error: concatenating variable expansion contains multiple values
+  EOE
+
+  # @@ Move the following tests to build2 parser unit tests when created.
+  #
+  : empty-brackets
+  :
+  $c <<EOI && $b 2>>EOE != 0
+  set -w '[]' baz <'foo bar';
+  echo "$baz"
+  EOI
+  testscript:2:8: error: concatenating variable expansion contains multiple values
+  EOE
+
+  : no-left-bracket
+  :
+  $c <<EOI && $b 2>>EOE != 0
+  set -w x baz
+  EOI
+  testscript:1:1: (x):1:1: error: expected '[' instead of 'x'
+  EOE
+
+  : unknown
+  :
+  $c <<EOI && $b 2>>EOE != 0
+  set -w [x] baz
+  EOI
+  testscript:1:1: ([x]):1:1: error: unknown value attribute x
+  EOE
+
+  : junk
+  :
+  $c <<EOI && $b 2>>EOE != 0
+  set -w '[string] x' baz
+  EOI
+  testscript:1:1: ([string] x):1:10: error: trailing junk after ']'
+  EOE
+}