Tighten up attribute recognition during parsing

Now it should be possible to use `[]` for wildcard patterns, for example:

  foo = foo.[hit]xx

Note that a leading bracket expression will still be recognized as attributes
and escaping or quoting it will inhibit pattern matching. To resolve this case
we need to specify an empty attribute list:

  foo = [] [abc]-foo.cxx

2 files changed, 31 insertions, 24 deletions

diff --git a/libbuild2/test/script/lexer.cxx b/libbuild2/test/script/lexer.cxx
index 75c04c8..a65eb25 100644
--- a/libbuild2/test/script/lexer.cxx
+++ b/libbuild2/test/script/lexer.cxx
@@ -19,11 +19,14 @@ namespace build2
       void lexer::
       mode (base_mode m, char ps, optional<const char*> esc)
       {
+        bool a (false); // attributes
+
         const char* s1 (nullptr);
         const char* s2 (nullptr);
-        bool s (true);
-        bool n (true);
-        bool q (true);
+
+        bool s (true); // space
+        bool n (true); // newline
+        bool q (true); // quotes
 
         if (!esc)
         {
@@ -71,8 +74,8 @@ namespace build2
             // Note that we don't recognize ':' since having a trailing
             // variable assignment is illegal.
             //
-            s1 = "; $([]#\t\n";
-            s2 = "         ";
+            s1 = "; $(#\t\n";
+            s2 = "       ";
             break;
           }
 
@@ -128,7 +131,7 @@ namespace build2
             //
             assert (ps == '\0' ||
                     m == lexer_mode::eval ||
-                    m == lexer_mode::attribute);
+                    m == lexer_mode::attributes);
 
             base_lexer::mode (m, ps, esc);
             return;
@@ -136,7 +139,7 @@ namespace build2
         }
 
         assert (ps == '\0');
-        state_.push (state {m, ps, s, n, q, *esc, s1, s2});
+        state_.push (state {m, a, ps, s, n, q, *esc, s1, s2});
       }
 
       token lexer::
@@ -177,9 +180,6 @@ namespace build2
         xchar c (get ());
         uint64_t ln (c.line), cn (c.column);
 
-        if (eos (c))
-          return token (type::eos, sep, ln, cn, token_printer);
-
         state st (state_.top ()); // Make copy (see first/second_token).
         lexer_mode m (st.mode);
 
@@ -217,6 +217,22 @@ namespace build2
           return make_token (t, move (v));
         };
 
+        // Handle attributes (do it first to make sure the flag is cleared
+        // regardless of what we return).
+        //
+        if (st.attributes)
+        {
+          assert (m == lexer_mode::variable_line);
+
+          state_.top ().attributes = false;
+
+          if (c == '[')
+            return make_token (type::lsbrace);
+        }
+
+        if (eos (c))
+          return make_token (type::eos);
+
         // Expire certain modes at the end of the token. Do it early in case
         // we push any new mode (e.g., double quote).
         //
@@ -253,18 +269,6 @@ namespace build2
           }
         }
 
-
-        if (m == lexer_mode::variable_line)
-        {
-          switch (c)
-          {
-            // Attributes.
-            //
-          case '[': return make_token (type::lsbrace);
-          case ']': return make_token (type::rsbrace);
-          }
-        }
-
         // Line separators.
         //
         if (m == lexer_mode::command_line ||
diff --git a/libbuild2/test/script/parser.cxx b/libbuild2/test/script/parser.cxx
index f3f6ffa..aa78437 100644
--- a/libbuild2/test/script/parser.cxx
+++ b/libbuild2/test/script/parser.cxx
@@ -1288,10 +1288,11 @@ namespace build2
         // enter: assignment
         // leave: newline or semi
 
-        // We cannot reuse the value mode since it will recognize { which we
+        // We cannot reuse the value mode since it will recognize `{` which we
         // want to treat as a literal.
         //
         mode (lexer_mode::variable_line);
+        enable_attributes (); // @@ VAL
         next (t, tt);
 
         // Parse value attributes if any. Note that it's ok not to have
@@ -3446,11 +3447,13 @@ namespace build2
         path_ = &name;
 
         istringstream is (attributes);
-        lexer l (is, name, lexer_mode::attribute);
+        lexer l (is, name, lexer_mode::attributes);
         set_lexer (&l);
 
         token t;
         type tt;
+
+        enable_attributes (); // Enable `[` recognition.
         next (t, tt);
 
         if (tt != type::lsbrace && tt != type::eos)