Various minor buildfile lexer/parser changes

4 files changed, 62 insertions, 52 deletions

diff --git a/build2/lexer b/build2/lexer
index f7f7b82..abaf273 100644
--- a/build2/lexer
+++ b/build2/lexer
@@ -40,11 +40,13 @@ namespace build2
 
   // Extendable/inheritable enum-like class.
   //
-  struct lexer_mode
+  struct lexer_mode: lexer_mode_base
   {
+    using base_type = lexer_mode_base;
+
     enum
     {
-      normal,
+      normal = base_type::value_next,
       variable,
       value,
       eval,
@@ -54,11 +56,9 @@ namespace build2
       value_next
     };
 
-    using value_type = uint16_t;
-
-    lexer_mode (value_type v = normal): v_ (v) {}
-    operator value_type () const {return v_;}
-    value_type v_;
+    lexer_mode () = default;
+    lexer_mode (value_type v): base_type (v) {}
+    lexer_mode (base_type v): base_type (v) {}
   };
 
   class lexer: protected butl::char_scanner
diff --git a/build2/parser b/build2/parser
index f038b97..c51f18f 100644
--- a/build2/parser
+++ b/build2/parser
@@ -184,24 +184,14 @@ namespace build2
     // Append names and return true if the parsed value is NOT NULL.
     //
     bool
-    parse_names (token& t, token_type& tt,
-                 names& ns,
-                 bool chunk = false,
-                 const char* what = "name")
-    {
-      return parse_names (
-        t, tt, ns, chunk, what, 0, nullptr, nullptr, nullptr);
-    }
-
-    bool
     parse_names (token&, token_type&,
                  names&,
-                 bool chunk,
-                 const char* what,
-                 size_t pair,
-                 const string* prj,
-                 const dir_path* dir,
-                 const string* type);
+                 bool chunk = false,
+                 const char* what = "name",
+                 size_t pair = 0,
+                 const string* prj = nullptr,
+                 const dir_path* dir = nullptr,
+                 const string* type = nullptr);
 
     size_t
     parse_names_trailer (token&, token_type&,
@@ -388,15 +378,6 @@ namespace build2
       parser* p_;
     };
 
-    struct replay_token
-    {
-      build2::token token;
-      lexer_mode mode;
-      char pair_separator;
-    };
-
-    using replay_tokens = vector<replay_token>;
-
     // Stop saving and get the data.
     //
     replay_tokens
diff --git a/build2/parser.cxx b/build2/parser.cxx
index a81aec7..03ba213 100644
--- a/build2/parser.cxx
+++ b/build2/parser.cxx
@@ -2017,13 +2017,13 @@ namespace build2
     // a@{b c d{e f} {}}.
     //
 
-    // Buffer that is used to collect the complete name in case of
-    // an unseparated variable expansion or eval context, e.g.,
-    // 'foo$bar($baz)fox'. The idea is to concatenate all the
-    // individual parts in this buffer and then re-inject it into
-    // the loop as a single token.
+    // Buffer that is used to collect the complete name in case of an
+    // unseparated variable expansion or eval context, e.g., foo$bar($baz)fox.
+    // The idea is to concatenate all the individual parts in this buffer and
+    // then re-inject it into the loop as a single token.
     //
-    string concat;
+    bool concat (false);
+    string concat_str;
 
     // Number of names in the last group. This is used to detect when
     // we need to add an empty first pair element (e.g., @y) or when
@@ -2037,14 +2037,14 @@ namespace build2
       // continue accumulating or inject. We inject if the next token is
       // not a word, var expansion, or eval context or if it is separated.
       //
-      if (!concat.empty () &&
+      if (concat &&
           ((tt != type::word   &&
             tt != type::dollar &&
             tt != type::lparen) || peeked ().separated))
       {
         tt = type::word;
-        t = token (move (concat), true, false, t.line, t.column);
-        concat.clear ();
+        t = token (move (concat_str), true, false, t.line, t.column);
+        concat = false;
       }
       else if (!first)
       {
@@ -2060,7 +2060,7 @@ namespace build2
       //
       if (tt == type::word)
       {
-        string name (t.value); //@@ move?
+        string name (move (t.value));
         tt = peek ();
 
         // Should we accumulate? If the buffer is not empty, then
@@ -2069,11 +2069,16 @@ namespace build2
         // the next token is a var expansion or eval context and it
         // is not separated, then we need to start accumulating.
         //
-        if (!concat.empty () ||                                // Continue.
+        if (concat ||                                       // Continue.
             ((tt == type::dollar ||
-              tt == type::lparen) && !peeked ().separated))    // Start.
+              tt == type::lparen) && !peeked ().separated)) // Start.
         {
-          concat += name;
+          if (concat_str.empty ())
+            concat_str = move (name);
+          else
+            concat_str += name;
+
+          concat = true;
           continue;
         }
 
@@ -2395,13 +2400,13 @@ namespace build2
         // the next token is a word or var expansion and it is not
         // separated, then we need to start accumulating.
         //
-        if (!concat.empty () ||                       // Continue.
+        if (concat ||                                 // Continue.
             ((tt == type::word   ||                   // Start.
               tt == type::dollar ||
               tt == type::lparen) && !peeked ().separated))
         {
-          // This should be a simple value or a simple directory. The
-          // token still points to the name (or closing paren).
+          // This should be a simple value or a simple directory. The token
+          // still points to the name (or closing paren).
           //
           if (lv.size () > 1)
             fail (loc) << "concatenating " << what << " contains multiple "
@@ -2424,10 +2429,14 @@ namespace build2
             // actually a directory path (think s/foo/bar/) so we have to
             // reverse it exactly.
             //
-            concat += n.original ? n.dir.representation () : n.dir.string ();
+            concat_str += n.original
+              ? n.dir.representation ()
+              : n.dir.string ();
           }
           else
-            concat += n.value;
+            concat_str += n.value;
+
+          concat = true;
         }
         else
         {
diff --git a/build2/token b/build2/token
index 517bf47..19e82dc 100644
--- a/build2/token
+++ b/build2/token
@@ -92,13 +92,33 @@ namespace build2
           printer (&token_printer) {}
   };
 
-  using tokens = vector<token>;
-
   // Output the token value in a format suitable for diagnostics.
   //
   inline ostream&
   operator<< (ostream& o, const token& t) {t.printer (o, t, true); return o;}
 
+  // Extendable/inheritable enum-like class.
+  //
+  struct lexer_mode_base
+  {
+    enum { value_next };
+
+    using value_type = uint16_t;
+
+    lexer_mode_base (value_type v = value_next): v_ (v) {}
+    operator value_type () const {return v_;}
+    value_type v_;
+  };
+
+  struct replay_token
+  {
+    build2::token token;
+    lexer_mode_base mode;
+    char pair_separator;
+  };
+
+  using replay_tokens = vector<replay_token>;
+
   // Diagnostics plumbing. We assume that any diag stream for which we can use
   // token as location has its aux data pointing to pointer to path.
   //