Get rid of lexer modes overriding pair separators

5 files changed, 30 insertions, 28 deletions

diff --git a/build2/lexer b/build2/lexer
index f0a0fa6..a539c2d 100644
--- a/build2/lexer
+++ b/build2/lexer
@@ -81,12 +81,11 @@ namespace build2
     const path&
     name () const {return fail.name_;}
 
-    // Note: sets mode for the next token. The second argument can be used
-    // to specify an alternative separator character (if the mode supports
-    // pair separators).
+    // Note: sets mode for the next token. The second argument can be used to
+    // specifythe pair separator character (if the mode supports pairs).
     //
     virtual void
-    mode (lexer_mode, char pair_separator = '@');
+    mode (lexer_mode, char pair_separator = '\0');
 
     // Expire the current mode early.
     //
@@ -186,7 +185,7 @@ namespace build2
           sep_ (false)
     {
       if (sm)
-        mode (lexer_mode::normal);
+        mode (lexer_mode::normal, '@');
     }
 
     const char* escapes_;
diff --git a/build2/lexer.cxx b/build2/lexer.cxx
index 0ce9770..2c728d6 100644
--- a/build2/lexer.cxx
+++ b/build2/lexer.cxx
@@ -34,7 +34,6 @@ namespace build2
   {
     const char* s1 (nullptr);
     const char* s2 (nullptr);
-    char p ('\0');
     bool s (true);
     bool q (true);
 
@@ -44,28 +43,24 @@ namespace build2
       {
         s1 = ":=+ $(){}[]#\t\n";
         s2 = "  =           ";
-        p = ps;
         break;
       }
     case lexer_mode::value:
       {
         s1 = " $(){}[]#\t\n";
         s2 = "           ";
-        p = ps;
         break;
       }
     case lexer_mode::attribute:
       {
         s1 = " $(]#\t\n";
         s2 = "       ";
-        p = ps;
         break;
       }
     case lexer_mode::eval:
       {
         s1 = ":<>=! $(){}[]#\t\n";
         s2 = "   ==           ";
-        p = ps;
         break;
       }
     case lexer_mode::single_quoted:
@@ -75,12 +70,13 @@ namespace build2
     case lexer_mode::variable:
       {
         // These are handled in an ad hoc way in word().
+        assert (ps == '\0');
         break;
       }
     default: assert (false); // Unhandled custom mode.
     }
 
-    state_.push (state {m, p, s, q, s1, s2});
+    state_.push (state {m, ps, s, q, s1, s2});
   }
 
   token lexer::
diff --git a/build2/parser b/build2/parser
index 0b3a807..3e6fc00 100644
--- a/build2/parser
+++ b/build2/parser
@@ -93,11 +93,10 @@ namespace build2
     string
     parse_variable_name (names&&, const location&);
 
-    // Note: calls attributes_push() that the caller must pop. Also expects
-    // the mode to auto-expire.
+    // Note: calls attributes_push() that the caller must pop.
     //
     value
-    parse_variable_value (token&, token_type&, lexer_mode = lexer_mode::value);
+    parse_variable_value (token&, token_type&);
 
     void
     apply_variable_attributes (const variable&);
@@ -288,7 +287,7 @@ namespace build2
     peek ();
 
     token_type
-    peek (lexer_mode m, char ps = '@')
+    peek (lexer_mode m, char ps = '\0')
     {
       // The idea is that if we already have something peeked, then it should
       // be in the same mode. We also don't re-set the mode since it may have
@@ -296,7 +295,7 @@ namespace build2
       //
       if (peeked_)
       {
-        assert (peek_.mode == m);
+        assert (peek_.mode == m && peek_.pair_separator == ps);
         return peek_.token.type;
       }
 
@@ -312,7 +311,7 @@ namespace build2
     }
 
     void
-    mode (lexer_mode m, char ps = '@')
+    mode (lexer_mode m, char ps = '\0')
     {
       if (replay_ != replay::play)
         lexer_->mode (m, ps);
diff --git a/build2/parser.cxx b/build2/parser.cxx
index eb9f463..bfbaa04 100644
--- a/build2/parser.cxx
+++ b/build2/parser.cxx
@@ -765,7 +765,7 @@ namespace build2
     // The rest should be a list of buildfiles. Parse them as names in the
     // value mode to get variable expansion and directory prefixes.
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::value, '@');
     next (t, tt);
     const location l (get_location (t));
     names ns (tt != type::newline && tt != type::eos
@@ -841,7 +841,7 @@ namespace build2
     // The rest should be a list of buildfiles. Parse them as names in the
     // value mode to get variable expansion and directory prefixes.
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::value, '@');
     next (t, tt);
     const location l (get_location (t));
     names ns (tt != type::newline && tt != type::eos
@@ -992,7 +992,7 @@ namespace build2
     // switch to the value mode, get the first token, and then re-parse it
     // manually looking for =/=+/+=.
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::value, '@');
     next (t, tt);
 
     // Get variable attributes, if any (note that here we will go into a
@@ -1172,7 +1172,7 @@ namespace build2
     // The rest should be a list of module names. Parse them as names in the
     // value mode to get variable expansion, etc.
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::value, '@');
     next (t, tt);
     const location l (get_location (t));
     names ns (tt != type::newline && tt != type::eos
@@ -1468,9 +1468,9 @@ namespace build2
   }
 
   value parser::
-  parse_variable_value (token& t, type& tt, lexer_mode m)
+  parse_variable_value (token& t, type& tt)
   {
-    mode (m);
+    mode (lexer_mode::value, '@');
     next (t, tt);
 
     // Parse value attributes if any. Note that it's ok not to have anything
@@ -1661,7 +1661,7 @@ namespace build2
   value parser::
   parse_eval (token& t, type& tt)
   {
-    mode (lexer_mode::eval);
+    mode (lexer_mode::eval, '@');
     next (t, tt);
     return parse_eval_trailer (t, tt);
   }
@@ -2792,7 +2792,7 @@ namespace build2
     // Turn on the value mode/pairs recognition with '@' as the pair separator
     // (e.g., src_root/@out_root/exe{foo bar}).
     //
-    mode (lexer_mode::value);
+    mode (lexer_mode::value, '@');
 
     token t;
     type tt;
diff --git a/build2/test/script/lexer.cxx b/build2/test/script/lexer.cxx
index 7a4e12c..5e6c66a 100644
--- a/build2/test/script/lexer.cxx
+++ b/build2/test/script/lexer.cxx
@@ -96,14 +96,22 @@ namespace build2
           }
         default:
           {
-            // Disable pair separator except for attributes.
+            // Make sure pair separators are only enabled where we expect
+            // them.
             //
-            base_lexer::mode (m, m != lexer_mode::attribute ? '\0' : ps);
+            // @@ Should we disable pair separators in the eval mode?
+            //
+            assert (ps == '\0' ||
+                    m == lexer_mode::eval ||
+                    m == lexer_mode::attribute);
+
+            base_lexer::mode (m, ps);
             return;
           }
         }
 
-        state_.push (state {m, '\0', s, q, s1, s2});
+        assert (ps == '\0');
+        state_.push (state {m, ps, s, q, s1, s2});
       }
 
       token lexer::