Implement support for multiple scope/targets in variable assignment

Can now even do this:

foo/ file{*-bar} file{baz}: x = y

5 files changed, 207 insertions, 95 deletions

diff --git a/build/lexer.cxx b/build/lexer.cxx
index f6e34cb..864fc7b 100644
--- a/build/lexer.cxx
+++ b/build/lexer.cxx
@@ -75,7 +75,7 @@ namespace build
     // Handle pair separator.
     //
     if (m == lexer_mode::pairs && c == pair_separator_)
-      return token (type::pair_separator, sep, ln, cn);
+      return token (c, sep, ln, cn);
 
     // The following characters are not treated as special in the
     // value or pairs mode.
diff --git a/build/parser b/build/parser
index 4630110..2631ca2 100644
--- a/build/parser
+++ b/build/parser
@@ -7,11 +7,13 @@
 
 #include <string>
 #include <iosfwd>
-#include <utility> // move()
 
 #include <build/types>
-#include <build/token>
+#include <build/utility>
+
 #include <build/spec>
+#include <build/lexer>
+#include <build/token>
 #include <build/variable>    // list_value
 #include <build/diagnostics>
 
@@ -19,7 +21,6 @@ namespace build
 {
   class scope;
   class target;
-  class lexer;
 
   class parser
   {
@@ -180,6 +181,93 @@ namespace build
       return peek_;
     }
 
+    void
+    mode (lexer_mode m, char ps = '=')
+    {
+      if (replay_ != replay::play)
+        lexer_->mode (m, ps);
+    }
+
+    lexer_mode
+    mode () const
+    {
+      assert (replay_ != replay::play);
+      return lexer_->mode ();
+    }
+
+    void
+    expire_mode ()
+    {
+      if (replay_ != replay::play)
+        lexer_->expire_mode ();
+    }
+
+    // Token saving and replaying. Note that is can only be used in certain
+    // contexts. Specifically, the lexer mode should be the same and the code
+    // that parses a replay must not interact with the lexer directly (e.g.,
+    // the keyword() test). For now we don't enforce any of this.
+    //
+    // Note also that the peeked token is not part of the replay, until it
+    // is "got".
+    //
+    //
+    void
+    replay_save ()
+    {
+      assert (replay_ == replay::stop);
+      replay_ = replay::save;
+    }
+
+    void
+    replay_play ()
+    {
+      assert ((replay_ == replay::save && !replay_data_.empty ()) ||
+              (replay_ == replay::play && replay_i_ == replay_data_.size ()));
+
+      replay_i_ = 0;
+      replay_ = replay::play;
+    }
+
+    void
+    replay_stop ()
+    {
+      replay_data_.clear ();
+      replay_ = replay::stop;
+    }
+
+    const token&
+    replay_next ()
+    {
+      assert (replay_i_ != replay_data_.size ());
+      return replay_data_[replay_i_++];
+    }
+
+    struct replay_guard
+    {
+      replay_guard (parser& p, bool start = true)
+          : p_ (start ? &p : nullptr)
+      {
+        if (p_ != nullptr)
+          p_->replay_save ();
+      }
+
+      void
+      play ()
+      {
+        if (p_ != nullptr)
+          p_->replay_play ();
+      }
+
+      ~replay_guard ()
+      {
+        if (p_ != nullptr)
+          p_->replay_stop ();
+      }
+
+    private:
+      parser* p_;
+    };
+
     // Diagnostics.
     //
   protected:
@@ -196,8 +284,12 @@ namespace build
     target* default_target_;
     names_type export_value_;
 
-    token peek_ {token_type::eos, false, 0, 0};
-    bool peeked_ {false};
+    token peek_ = token (token_type::eos, false, 0, 0);
+    bool peeked_ = false;
+
+    enum class replay {stop, save, play} replay_ = replay::stop;
+    vector<token> replay_data_;
+    size_t replay_i_; // Position of the next token during replay.
   };
 }
 
diff --git a/build/parser.cxx b/build/parser.cxx
index 1893b70..ad17ae9 100644
--- a/build/parser.cxx
+++ b/build/parser.cxx
@@ -16,9 +16,6 @@
 #include <build/utility>
 #include <build/version>
 
-#include <build/token>
-#include <build/lexer>
-
 #include <build/scope>
 #include <build/target>
 #include <build/prerequisite>
@@ -309,93 +306,101 @@ namespace build
               tt == type::equal_plus ||
               tt == type::plus_equal)
           {
+            token at (t);
+            type att (tt);
+
             string v (variable_name (move (pns), ploc));
 
-            // Enter the target/scope and set it as current.
+            // If we have multiple targets/scopes, then we save the value
+            // tokens when parsing the first one and then replay them for
+            // the subsequent. We have to do it this way because the value
+            // may contain variable expansions that would be sensitive to
+            // the target/scope context in which they are evaluated.
             //
-            if (ns.size () != 1)
-              fail (nloc) << "multiple names in scope/target-specific "
-                          << "variable assignment";
-
-            name& n (ns[0]);
+            replay_guard rg (*this, ns.size () > 1);
 
-            if (n.qualified ())
-              fail (nloc) << "project name in scope/target " << n;
-
-            if (n.directory ())
+            for (name& n: ns)
             {
-              // The same code as in directory scope handling code above.
-              //
-              dir_path p (move (n.dir));
+              if (n.qualified ())
+                fail (nloc) << "project name in scope/target " << n;
 
-              if (p.relative ())
-                p = scope_->out_path () / p;
+              if (n.directory ())
+              {
+                // The same code as in directory scope handling code above.
+                //
+                dir_path p (move (n.dir));
 
-              p.normalize ();
+                if (p.relative ())
+                  p = scope_->out_path () / p;
 
-              scope* ors (root_);
-              scope* ocs (scope_);
-              switch_scope (p);
+                p.normalize ();
 
-              variable (t, tt, move (v), tt);
+                scope* ors (root_);
+                scope* ocs (scope_);
+                switch_scope (p);
 
-              scope_ = ocs;
-              root_ = ors;
-            }
-            else
-            {
-              // Figure out if this is a target or type/pattern specific
-              // variable.
-              //
-              size_t p (n.value.find ('*'));
+                variable (t, tt, v, att);
 
-              if (p == string::npos)
-              {
-                target* ot (target_);
-                target_ = &enter_target (move (n));
-                variable (t, tt, move (v), tt);
-                target_ = ot;
+                scope_ = ocs;
+                root_ = ors;
               }
               else
               {
-                // See tests/variable/type-pattern.
+                // Figure out if this is a target or type/pattern specific
+                // variable.
                 //
-                if (!n.dir.empty ())
-                  fail (nloc) << "directory in target type/pattern " << n;
+                size_t p (n.value.find ('*'));
 
-                if (n.value.find ('*', p + 1) != string::npos)
-                  fail (nloc) << "multiple wildcards in target type/pattern "
-                              << n;
+                if (p == string::npos)
+                {
+                  target* ot (target_);
+                  target_ = &enter_target (move (n));
+                  variable (t, tt, v, att);
+                  target_ = ot;
+                }
+                else
+                {
+                  // See tests/variable/type-pattern.
+                  //
+                  if (!n.dir.empty ())
+                    fail (nloc) << "directory in target type/pattern " << n;
 
-                // Resolve target type. If none is specified, use the root
-                // of the hierarchy.
-                //
-                const target_type* ti (
-                  n.untyped ()
-                  ? &target::static_type
-                  : scope_->find_target_type (n.type));
+                  if (n.value.find ('*', p + 1) != string::npos)
+                    fail (nloc) << "multiple wildcards in target type/pattern "
+                                << n;
 
-                if (ti == nullptr)
-                  fail (nloc) << "unknown target type " << n.type;
+                  // Resolve target type. If none is specified, use the root
+                  // of the hierarchy.
+                  //
+                  const target_type* ti (
+                    n.untyped ()
+                    ? &target::static_type
+                    : scope_->find_target_type (n.type));
 
-                if (tt == type::equal_plus)
-                  fail (t) << "prepend to target type/pattern-specific "
-                           << "variable " << v;
+                  if (ti == nullptr)
+                    fail (nloc) << "unknown target type " << n.type;
 
-                if (tt == type::plus_equal)
-                  fail (t) << "append to target type/pattern-specific "
-                           << "variable " << v;
+                  if (att == type::equal_plus)
+                    fail (at) << "prepend to target type/pattern-specific "
+                              << "variable " << v;
 
-                const auto& var (var_pool.find (move (v)));
+                  if (att == type::plus_equal)
+                    fail (at) << "append to target type/pattern-specific "
+                              << "variable " << v;
 
-                // Note: expand variables in the value in the context of
-                // the scope.
-                //
-                names_type vns (variable_value (t, tt, var));
-                value& val (
-                  scope_->target_vars[*ti][move (n.value)].assign (var).first);
-                val.assign (move (vns), var);
+                  const auto& var (var_pool.find (v));
+
+                  // Note: expand variables in the value in the context of
+                  // the scope.
+                  //
+                  names_type vns (variable_value (t, tt, var));
+                  value& val (scope_->target_vars[*ti][move (n.value)].assign (
+                                var).first);
+                  val.assign (move (vns), var);
+                }
               }
+
+              rg.play (); // Replay.
             }
           }
           // Dependency declaration.
@@ -500,7 +505,7 @@ namespace build
     // The rest should be a list of buildfiles. Parse them as names
     // to get variable expansion and directory prefixes.
     //
-    lexer_->mode (lexer_mode::value);
+    mode (lexer_mode::value);
     next (t, tt);
     const location l (get_location (t, &path_));
     names_type ns (tt != type::newline && tt != type::eos
@@ -581,7 +586,7 @@ namespace build
     // The rest should be a list of buildfiles. Parse them as names
     // to get variable expansion and directory prefixes.
     //
-    lexer_->mode (lexer_mode::value);
+    mode (lexer_mode::value);
     next (t, tt);
     const location l (get_location (t, &path_));
     names_type ns (tt != type::newline && tt != type::eos
@@ -746,7 +751,7 @@ namespace build
           ? &scope_->assign (*var)
           : &scope_->append (*var);
         next (t, tt); // Consume =/=+/+=.
-        lexer_->mode (lexer_mode::value);
+        mode (lexer_mode::value);
         next (t, tt);
       }
     }
@@ -797,7 +802,7 @@ namespace build
     // The rest is a value. Parse it as names to get variable expansion.
     // build::import() will check the names, if required.
     //
-    lexer_->mode (lexer_mode::value);
+    mode (lexer_mode::value);
     next (t, tt);
 
     if (tt != type::newline && tt != type::eos)
@@ -822,7 +827,7 @@ namespace build
     // The rest should be a list of module names. Parse them as names
     // to get variable expansion, etc.
     //
-    lexer_->mode (lexer_mode::pairs, '@');
+    mode (lexer_mode::pairs, '@');
     next (t, tt);
     const location l (get_location (t, &path_));
     names_type ns (tt != type::newline && tt != type::eos
@@ -1060,8 +1065,7 @@ namespace build
     // to the variable value lexing mode so that we don't treat special
     // characters (e.g., ':') as the end of the names.
     //
-    lexer_->mode (lexer_mode::value);
-
+    mode (lexer_mode::value);
     next (t, tt);
     names_type ns (tt != type::newline && tt != type::eos
                    ? names (t, tt)
@@ -1120,9 +1124,9 @@ namespace build
   variable_value (token& t, type& tt, const variable_type& var)
   {
     if (var.pairs != '\0')
-      lexer_->mode (lexer_mode::pairs, var.pairs);
+      mode (lexer_mode::pairs, var.pairs);
     else
-      lexer_->mode (lexer_mode::value);
+      mode (lexer_mode::value);
 
     next (t, tt);
     return (tt != type::newline && tt != type::eos
@@ -1133,7 +1137,7 @@ namespace build
   parser::names_type parser::
   eval (token& t, type& tt)
   {
-    lexer_->mode (lexer_mode::eval);
+    mode (lexer_mode::eval);
     next (t, tt);
 
     names_type ns (tt != type::rparen ? names (t, tt) : names_type ());
@@ -1490,7 +1494,7 @@ namespace build
           // it on and switch to the eval mode if what we get next
           // is a paren.
           //
-          lexer_->mode (lexer_mode::variable);
+          mode (lexer_mode::variable);
           next (t, tt);
           loc = get_location (t, &path_);
 
@@ -1499,7 +1503,7 @@ namespace build
             n = t.value;
           else if (tt == type::lparen)
           {
-            lexer_->expire_mode ();
+            expire_mode ();
             names_type ns (eval (t, tt));
 
             // Make sure the result of evaluation is a single, simple name.
@@ -1723,7 +1727,7 @@ namespace build
           count = 1;
         }
 
-        ns.back ().pair = lexer_->pair_separator ();
+        ns.back ().pair = t.pair;
         tt = peek ();
         continue;
       }
@@ -1802,6 +1806,7 @@ namespace build
   bool parser::
   keyword (token& t)
   {
+    assert (replay_ == replay::stop); // Can't be used in a replay.
     assert (t.type == type::name);
 
     // The goal here is to allow using keywords as variable names and
@@ -1873,7 +1878,7 @@ namespace build
     // Turn on pairs recognition with '@' as the pair separator (e.g.,
     // src_root/@out_root/exe{foo bar}).
     //
-    lexer_->mode (lexer_mode::pairs, '@');
+    mode (lexer_mode::pairs, '@');
 
     token t (type::eos, false, 0, 0);
     type tt;
@@ -1915,7 +1920,7 @@ namespace build
       if (tt != type::name    &&
           tt != type::lcbrace &&      // Untyped name group: '{foo ...'
           tt != type::dollar  &&      // Variable expansion: '$foo ...'
-          !(tt == type::lparen && lexer_->mode () == lexer_mode::quoted) &&
+          !(tt == type::lparen && mode () == lexer_mode::quoted) &&
           tt != type::pair_separator) // Empty pair LHS: '@foo ...'
         fail (t) << "operation or target expected instead of " << t;
 
@@ -2164,13 +2169,16 @@ namespace build
   type parser::
   next (token& t, type& tt)
   {
-    if (!peeked_)
-      t = lexer_->next ();
-    else
+    if (peeked_)
     {
       t = move (peek_);
       peeked_ = false;
     }
+    else
+      t = (replay_ == replay::play ? replay_next () : lexer_->next ());
+
+    if (replay_ == replay::save)
+      replay_data_.push_back (t);
 
     tt = t.type;
     return tt;
@@ -2181,7 +2189,7 @@ namespace build
   {
     if (!peeked_)
     {
-      peek_ = lexer_->next ();
+      peek_ = (replay_ == replay::play ? replay_next () : lexer_->next ());
       peeked_ = true;
     }
 
diff --git a/build/token b/build/token
index 7eed747..d98b2e1 100644
--- a/build/token
+++ b/build/token
@@ -37,6 +37,7 @@ namespace build
     bool separated; // Whitespace-separated from the previous token.
     bool quoted;    // Name (or some part of it) was quoted.
 
+    char pair;         // Only valid for pair_separator.
     std::string value; // Only valid for name.
 
     std::uint64_t line;
@@ -44,7 +45,15 @@ namespace build
 
   public:
     token (token_type t, bool s, std::uint64_t l, std::uint64_t c)
-        : type (t), separated (s), line (l), column (c) {}
+        : type (t), separated (s), quoted (false), line (l), column (c) {}
+
+    token (char p, bool s, std::uint64_t l, std::uint64_t c)
+        : type (token_type::pair_separator),
+          separated (s),
+          quoted (false),
+          pair (p),
+          line (l),
+          column (c) {}
 
     token (std::string n, bool s, bool q, std::uint64_t l, std::uint64_t c)
         : type (token_type::name),
diff --git a/build/types b/build/types
index 2dc9b04..3028fc2 100644
--- a/build/types
+++ b/build/types
@@ -9,6 +9,7 @@
 #include <string>
 #include <utility>     // pair
 #include <memory>      // unique_ptr, shared_ptr
+#include <cstddef>     // size_t
 #include <functional>  // reference_wrapper
 
 #include <butl/path>
@@ -21,13 +22,15 @@ namespace build
   // Commonly-used types.
   //
   using std::pair;
+  using std::size_t;
   using std::string;
   using std::unique_ptr;
   using std::shared_ptr;
   using std::reference_wrapper;
+  using std::vector;
 
-  using strings = std::vector<string>;
-  using cstrings = std::vector<const char*>;
+  using strings = vector<string>;
+  using cstrings = vector<const char*>;
 
   // <butl/path>
   //