Enable @-delimited pairs mode everywhere

12 files changed, 128 insertions, 82 deletions

diff --git a/build2/b.cxx b/build2/b.cxx
index 50398a2..2f94ad3 100644
--- a/build2/b.cxx
+++ b/build2/b.cxx
@@ -549,7 +549,7 @@ main (int argc, char* argv[])
           {
             for (const name& n: *l)
             {
-              if (n.pair != '\0')
+              if (n.pair)
                 continue; // Skip project names.
 
               if (out_base.sub (out_root / n.dir))
diff --git a/build2/context.cxx b/build2/context.cxx
index dc84e63..53ea467 100644
--- a/build2/context.cxx
+++ b/build2/context.cxx
@@ -71,9 +71,9 @@ namespace build2
       v.find<string> ("project");
       v.find<dir_path> ("amalgamation");
 
-      // Not typed since the value requires pre-processing.
+      // Not typed since the value requires pre-processing (see file.cxx).
       //
-      v.find ("subprojects", nullptr, '=');
+      v.find ("subprojects");
 
       v.find<string> ("extension");
     }
diff --git a/build2/file.cxx b/build2/file.cxx
index fb4f31f..09332f1 100644
--- a/build2/file.cxx
+++ b/build2/file.cxx
@@ -568,13 +568,11 @@ namespace build2
         else
         {
           // Pre-scan the value and convert it to the "canonical" form,
-          // that is, a list of simple=dir pairs.
+          // that is, a list of name@dir pairs.
           //
           for (auto i (v.data_.begin ()); i != v.data_.end (); ++i)
           {
-            bool p (i->pair != '\0');
-
-            if (p)
+            if (i->pair)
             {
               // Project name.
               //
@@ -593,7 +591,7 @@ namespace build2
 
             // Figure out the project name if the user didn't specify one.
             //
-            if (!p)
+            if (!i->pair)
             {
               // Pass fallback src_root since this is a subproject that
               // was specified by the user so it is most likely in our
@@ -608,7 +606,7 @@ namespace build2
 
               i = v.data_.emplace (i, move (n));
 
-              i->pair = '=';
+              i->pair = true;
               ++i;
             }
           }
@@ -681,7 +679,7 @@ namespace build2
     {
       for (const name& n: *l)
       {
-        if (n.pair != '\0')
+        if (n.pair)
           continue; // Skip project names.
 
         dir_path out_root (root.out_path () / n.dir);
diff --git a/build2/lexer b/build2/lexer
index 98feb75..001d52c 100644
--- a/build2/lexer
+++ b/build2/lexer
@@ -75,7 +75,7 @@ namespace build2
     next ();
 
     // Peek at the first character of the next token. Return the character
-    // or 0 if the next token will be eos. Also return an indicator of
+    // or '\0' if the next token will be eos. Also return an indicator of
     // whether the next token will be separated.
     //
     pair<char, bool>
diff --git a/build2/lexer.cxx b/build2/lexer.cxx
index 1b24a61..795387a 100644
--- a/build2/lexer.cxx
+++ b/build2/lexer.cxx
@@ -75,7 +75,7 @@ namespace build2
     // Handle pair separator.
     //
     if (m == lexer_mode::pairs && c == pair_separator_)
-      return token (c, sep, ln, cn);
+      return token (type::pair_separator, sep, ln, cn);
 
     // The following characters are not treated as special in the
     // value or pairs mode.
diff --git a/build2/name b/build2/name
index 74c111c..99c21f4 100644
--- a/build2/name
+++ b/build2/name
@@ -86,7 +86,7 @@ namespace build2
     dir_path dir;
     std::string type;
     std::string value;
-    char pair = '\0'; // Pair symbol, if any.
+    bool pair = false; // True if first half of a pair.
   };
 
   inline bool
diff --git a/build2/name.cxx b/build2/name.cxx
index 8ba9576..0d7211c 100644
--- a/build2/name.cxx
+++ b/build2/name.cxx
@@ -50,8 +50,8 @@ namespace build2
       ++i;
       os << n;
 
-      if (n.pair != '\0')
-        os << n.pair;
+      if (n.pair)
+        os << '@';
       else if (i != e)
         os << ' ';
     }
diff --git a/build2/parser b/build2/parser
index 5eb50e5..201e8ed 100644
--- a/build2/parser
+++ b/build2/parser
@@ -85,7 +85,7 @@ namespace build2
     variable_name (names_type&&, const location&);
 
     names_type
-    variable_value (token&, token_type&, const variable_type&);
+    variable_value (token&, token_type&);
 
     names_type
     eval (token&, token_type&);
@@ -208,7 +208,7 @@ namespace build2
         lexer_->expire_mode ();
     }
 
-    // Token saving and replaying. Note that is can only be used in certain
+    // Token saving and replaying. Note that it can only be used in certain
     // contexts. Specifically, the lexer mode should be the same and the code
     // that parses a replay must not interact with the lexer directly (e.g.,
     // the keyword() test). For now we don't enforce any of this.
diff --git a/build2/parser.cxx b/build2/parser.cxx
index d729a4c..c7f75b3 100644
--- a/build2/parser.cxx
+++ b/build2/parser.cxx
@@ -380,12 +380,12 @@ namespace build2
                     fail (at) << "append to target type/pattern-specific "
                               << "variable " << v;
 
-                  const auto& var (var_pool.find (v));
-
-                  // Note: expand variables in the value in the context of
+                  // Note: expanding variables in the value in the context of
                   // the scope.
                   //
-                  names_type vns (variable_value (t, tt, var));
+                  names_type vns (variable_value (t, tt));
+
+                  const auto& var (var_pool.find (v));
                   value& val (scope_->target_vars[*ti][move (n.value)].assign (
                                 var).first);
                   val.assign (move (vns), var);
@@ -495,7 +495,7 @@ namespace build2
     // The rest should be a list of buildfiles. Parse them as names
     // to get variable expansion and directory prefixes.
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::pairs, '@');
     next (t, tt);
     const location l (get_location (t, &path_));
     names_type ns (tt != type::newline && tt != type::eos
@@ -504,7 +504,7 @@ namespace build2
 
     for (name& n: ns)
     {
-      if (n.qualified () || n.empty () || n.value.empty ())
+      if (n.pair || n.qualified () || n.empty () || n.value.empty ())
         fail (l) << "expected buildfile instead of " << n;
 
       // Construct the buildfile path.
@@ -576,7 +576,7 @@ namespace build2
     // The rest should be a list of buildfiles. Parse them as names
     // to get variable expansion and directory prefixes.
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::pairs, '@');
     next (t, tt);
     const location l (get_location (t, &path_));
     names_type ns (tt != type::newline && tt != type::eos
@@ -585,7 +585,7 @@ namespace build2
 
     for (name& n: ns)
     {
-      if (n.qualified () || n.empty ())
+      if (n.pair || n.qualified () || n.empty ())
         fail (l) << "expected buildfile instead of " << n;
 
       // Construct the buildfile path. If it is a directory, then append
@@ -718,30 +718,92 @@ namespace build2
     if (root_->src_path_ == nullptr)
       fail (t) << "import during bootstrap";
 
-    next (t, tt);
-
     // General import format:
     //
     // import [<var>=](<project>|<project>/<target>])+
     //
+    type at; // Assignment type.
     value* val (nullptr);
     const build2::variable* var (nullptr);
 
-    type at; // Assignment type.
+    // We are now in the normal lexing mode and here is the problem: we need
+    // to switch to the value mode so that we don't treat certain characters
+    // as separators (e.g., + in 'libstdc++'). But at the same time we need
+    // to detect if we have the <var>= part. So what we are going to do is
+    // switch to the value mode, get the first token, and then re-parse it
+    // manually looking for =/=+/+=.
+    //
+    mode (lexer_mode::pairs, '@');
+    next (t, tt);
+
     if (tt == type::name)
     {
-      at = peek ();
+      // Split the token into the variable name and value at position (p) of
+      // '=', taking into account leading/trailing '+'. The variable name is
+      // returned while the token is set to value. If the resulting token
+      // value is empty, get the next token. Also set assignment type (at).
+      //
+      auto split = [&at, &t, &tt, this] (size_t p) -> string
+      {
+        string& v (t.value);
+        size_t e;
+
+        if (p != 0 && v[p - 1] == '+') // +=
+        {
+          e = p--;
+          at = type::append;
+        }
+        else if (p + 1 != v.size () && v[p + 1] == '+') // =+
+        {
+          e = p + 1;
+          at = type::prepend;
+        }
+        else // =
+        {
+          e = p;
+          at = type::assign;
+        }
 
-      if (at == type::assign || at == type::prepend || at == type::append)
+        string nv (v, e + 1); // value
+        v.resize (p);         // var name
+        v.swap (nv);
+
+        if (v.empty ())
+          next (t, tt);
+
+        return nv;
+      };
+
+      // Is this the 'foo=...' case?
+      //
+      size_t p (t.value.find ('='));
+
+      if (p != string::npos)
+        var = &var_pool.find (split (p));
+      //
+      // This could still be the 'foo =...' case.
+      //
+      else if (peek () == type::name)
       {
-        var = &var_pool.find (t.value);
+        const string& v (peeked ().value);
+        size_t n (v.size ());
+
+        // We should start with =/+=/=+.
+        //
+        if (n > 0 &&
+            (v[p = 0] == '=' ||
+             (n > 1 && v[0] == '+' && v[p = 1] == '=')))
+        {
+          var = &var_pool.find (t.value);
+          next (t, tt); // Get the peeked token.
+          split (p);    // Returned name should be empty.
+        }
+      }
+
+      if (var != nullptr)
         val = at == type::assign
           ? &scope_->assign (*var)
           : &scope_->append (*var);
-        next (t, tt); // Consume =/=+/+=.
-        mode (lexer_mode::value);
-        next (t, tt);
-      }
     }
 
     // The rest should be a list of projects and/or targets. Parse
@@ -754,6 +816,9 @@ namespace build2
 
     for (name& n: ns)
     {
+      if (n.pair)
+        fail (l) << "unexpected pair in import";
+
       // build2::import() will check the name, if required.
       //
       names_type r (build2::import (*scope_, move (n), l));
@@ -790,7 +855,7 @@ namespace build2
     // The rest is a value. Parse it as names to get variable expansion.
     // build2::import() will check the names, if required.
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::pairs, '@');
     next (t, tt);
 
     if (tt != type::newline && tt != type::eos)
@@ -1059,11 +1124,11 @@ namespace build2
   void parser::
   print (token& t, type& tt)
   {
-    // Parse the rest as names to get variable expansion, etc. Switch
-    // to the variable value lexing mode so that we don't treat special
-    // characters (e.g., ':') as the end of the names.
+    // Parse the rest as names to get variable expansion, etc. Switch to the
+    // variable lexing mode so that we don't treat special characters (e.g.,
+    // ':') as the end of the names.
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::pairs, '@');
     next (t, tt);
     names_type ns (tt != type::newline && tt != type::eos
                    ? names (t, tt)
@@ -1095,8 +1160,8 @@ namespace build2
   void parser::
   variable (token& t, type& tt, string name, type kind)
   {
+    names_type vns (variable_value (t, tt));
     const auto& var (var_pool.find (move (name)));
-    names_type vns (variable_value (t, tt, var));
 
     if (kind == type::assign)
     {
@@ -1119,13 +1184,9 @@ namespace build2
   }
 
   names parser::
-  variable_value (token& t, type& tt, const variable_type& var)
+  variable_value (token& t, type& tt)
   {
-    if (var.pairs != '\0')
-      mode (lexer_mode::pairs, var.pairs);
-    else
-      mode (lexer_mode::value);
-
+    mode (lexer_mode::pairs, '@');
     next (t, tt);
     return (tt != type::newline && tt != type::eos
             ? names (t, tt)
@@ -1204,7 +1265,7 @@ namespace build2
            false,
            (pair != 0
             ? pair
-            : (ns.empty () || ns.back ().pair == '\0' ? 0 : ns.size ())),
+            : (ns.empty () || ns.back ().pair ? ns.size () : 0)),
            pp, dp, tp);
     count = ns.size () - count;
 
@@ -1319,8 +1380,8 @@ namespace build2
     string concat;
 
     // Number of names in the last group. This is used to detect when
-    // we need to add an empty first pair element (e.g., {=y}) or when
-    // we have a for now unsupported multi-name LHS (e.g., {x y}=z).
+    // we need to add an empty first pair element (e.g., {@y}) or when
+    // we have a for now unsupported multi-name LHS (e.g., {x y}@z).
     //
     size_t count (0);
 
@@ -1343,7 +1404,7 @@ namespace build2
       {
         // If we are chunking, stop at the next separated token. Unless
         // current or next token is a pair separator, since we want the
-        // "x = y" pair to be parsed as a single chunk.
+        // "x @ y" pair to be parsed as a single chunk.
         //
         bool p (t.type == type::pair_separator); // Current token.
 
@@ -1711,7 +1772,7 @@ namespace build2
             {
               // Check that there are no nested pairs.
               //
-              if (n.pair != '\0')
+              if (n.pair)
                 fail (loc) << "nested pair in " << what;
 
               // And add another first half unless this is the first instance.
@@ -1755,7 +1816,7 @@ namespace build2
 
         if (count == 0)
         {
-          // Empty LHS, (e.g., {=y}), create an empty name.
+          // Empty LHS, (e.g., {@y}), create an empty name.
           //
           ns.emplace_back (pp,
                            (dp != nullptr ? *dp : dir_path ()),
@@ -1764,7 +1825,7 @@ namespace build2
           count = 1;
         }
 
-        ns.back ().pair = t.pair;
+        ns.back ().pair = true;
         tt = peek ();
         continue;
       }
@@ -1792,9 +1853,9 @@ namespace build2
         fail (t) << "expected name instead of " << t;
     }
 
-    // Handle the empty RHS in a pair, (e.g., {y=}).
+    // Handle the empty RHS in a pair, (e.g., {y@}).
     //
-    if (!ns.empty () && ns.back ().pair != '\0')
+    if (!ns.empty () && ns.back ().pair)
     {
       ns.emplace_back (pp,
                        (dp != nullptr ? *dp : dir_path ()),
@@ -1929,7 +1990,7 @@ namespace build2
   {
     // First it has to be a non-empty simple name.
     //
-    if (n.pair != '\0' || !n.simple () || n.empty ())
+    if (n.pair || !n.simple () || n.empty ())
       return false;
 
     // C identifier.
@@ -2073,7 +2134,7 @@ namespace build2
             // Do we have the src_base?
             //
             dir_path src_base;
-            if (i->pair != '\0')
+            if (i->pair)
             {
               if (i->typed ())
                 fail (l) << "expected target src_base instead of " << *i;
diff --git a/build2/token b/build2/token
index 31e7baa..6202f44 100644
--- a/build2/token
+++ b/build2/token
@@ -36,8 +36,7 @@ namespace build2
     bool separated; // Whitespace-separated from the previous token.
     bool quoted;    // Name (or some part of it) was quoted.
 
-    char pair;         // Only valid for pair_separator.
-    string value; // Only valid for name.
+    string value;   // Only valid for name.
 
     uint64_t line;
     uint64_t column;
@@ -46,14 +45,6 @@ namespace build2
     token (token_type t, bool s, uint64_t l, uint64_t c)
         : type (t), separated (s), quoted (false), line (l), column (c) {}
 
-    token (char p, bool s, uint64_t l, uint64_t c)
-        : type (token_type::pair_separator),
-          separated (s),
-          quoted (false),
-          pair (p),
-          line (l),
-          column (c) {}
-
     token (string n, bool s, bool q, uint64_t l, uint64_t c)
         : type (token_type::name),
           separated (s),
diff --git a/build2/variable b/build2/variable
index 15df6d6..a2490ac 100644
--- a/build2/variable
+++ b/build2/variable
@@ -50,7 +50,6 @@ namespace build2
     string name;
     const value_type* type;         // If NULL, then not (yet) typed.
     variable_visibility visibility;
-    char pairs;                     // Pair symbold or '\0' if not used.
   };
 
   inline bool
@@ -632,24 +631,23 @@ namespace build2
   {
     template <typename T>
     const variable&
-    find (string name, char p = '\0')
+    find (string name)
     {
-      return find (name, nullptr, &value_traits<T>::value_type, p);
+      return find (name, nullptr, &value_traits<T>::value_type);
     }
 
     const variable&
-    find (string name, const build2::value_type* t = nullptr, char p = '\0')
+    find (string name, const build2::value_type* t = nullptr)
     {
-      return find (name, nullptr, t, p);
+      return find (name, nullptr, t);
     }
 
     const variable&
     find (string name,
           variable_visibility v,
-          const build2::value_type* t = nullptr,
-          char p = '\0')
+          const build2::value_type* t = nullptr)
     {
-      return find (name, &v, t, p);
+      return find (name, &v, t);
     }
 
     using variable_pool_base::clear;
@@ -658,16 +656,14 @@ namespace build2
     const variable&
     find (string name,
           const variable_visibility* vv,
-          const build2::value_type* t,
-          char p)
+          const build2::value_type* t)
     {
       auto r (
         insert (
           variable {
             move (name),
             t,
-            vv != nullptr ? *vv : variable_visibility::normal,
-            p}));
+            vv != nullptr ? *vv : variable_visibility::normal}));
       const variable& v (*r.first);
 
       // Update type?
diff --git a/build2/variable.txx b/build2/variable.txx
index f2c55ff..54e48a3 100644
--- a/build2/variable.txx
+++ b/build2/variable.txx
@@ -64,7 +64,7 @@ namespace build2
     for (auto& p: m)
     {
       d->emplace_back (p.first); // Const, can't move.
-      d->back ().pair = '=';
+      d->back ().pair = true;
       d->emplace_back (move (p.second));
     }
 
@@ -106,7 +106,7 @@ namespace build2
     //
     for (auto i (v.begin ()); i != v.end (); ++i)
     {
-      if (i->pair == '\0')
+      if (!i->pair)
         fail << value_traits<std::map<K, V>>::value_type.name << " key-value "
              << "pair expected instead of '" << *i << "' "
              << "in variable '" << var.name << "'";