Clean up testscript grammar and parser

5 files changed, 655 insertions, 604 deletions

diff --git a/build2/test/rule.cxx b/build2/test/rule.cxx
index 089f560..6eb3551 100644
--- a/build2/test/rule.cxx
+++ b/build2/test/rule.cxx
@@ -428,7 +428,7 @@ namespace build2
         p.pre_parse (s);
 
         script::concurrent_runner r;
-        p.parse (s, r);
+        p.execute (s, r);
       };
 
       for (target* pt: t.prerequisite_targets)
diff --git a/build2/test/script/parser b/build2/test/script/parser
index 23a52d4..a81ddf0 100644
--- a/build2/test/script/parser
+++ b/build2/test/script/parser
@@ -25,50 +25,33 @@ namespace build2
 
       class parser: protected build2::parser
       {
-      public:
-        // Issue diagnostics and throw failed in case of an error.
+        // Pre-parse. Issue diagnostics and throw failed in case of an error.
         //
+      public:
         void
         pre_parse (script&);
 
         void
         pre_parse (istream&, script&);
 
-        void
-        parse (script& s, runner& r)
-        {
-          if (!s.empty ())
-            parse (s, s, r);
-        }
-
         // Recursive descent parser.
         //
-        // Each parse function receives the token/type from which it should
-        // start consuming. On return the token/type should contain the first
-        // token that has not been consumed. For *_line() functions this is
-        // the newline.
+        // Usually (but not always) parse functions receive the token/type
+        // from which it should start consuming and in return the token/type
+        // should contain the first token that has not been consumed.
+        //
+        // Functions that are called parse_*() rather than pre_parse_*() are
+        // used for both stages.
         //
       protected:
-        void
-        parse (scope&, script&, runner&);
-
         bool
-        demote_group_scope (unique_ptr<scope>&);
+        pre_parse_demote_group_scope (unique_ptr<scope>&);
 
         token
         pre_parse_scope_body ();
 
-        void
-        parse_scope_body ();
-
         unique_ptr<group>
-        pre_parse_scope (token&, token_type&, const string&);
-
-        description
-        pre_parse_leading_description (token&, token_type&);
-
-        description
-        parse_trailing_description (token&, token_type&);
+        pre_parse_scope_block (token&, token_type&, const string&);
 
         bool
         pre_parse_line (token&, token_type&,
@@ -76,12 +59,6 @@ namespace build2
                         lines* = nullptr,
                         bool one = false);
 
-        void
-        parse_lines (lines::iterator, lines::iterator, size_t&, bool);
-
-        command_expr
-        parse_command_expr (token&, token_type&);
-
         bool
         pre_parse_if_else (token&, token_type&,
                            optional<description>&,
@@ -98,14 +75,23 @@ namespace build2
                                    lines&);
 
         void
-        parse_directive_line (token&, token_type&);
+        pre_parse_directive (token&, token_type&);
 
         void
-        perform_include (names, location);
+        pre_parse_include_line (names, location);
+
+        description
+        pre_parse_leading_description (token&, token_type&);
+
+        description
+        parse_trailing_description (token&, token_type&);
 
         value
         parse_variable_line (token&, token_type&);
 
+        command_expr
+        parse_command_line (token&, token_type&);
+
         // Ordered sequence of here-document redirects that we can expect to
         // see after the command line.
         //
@@ -121,7 +107,7 @@ namespace build2
         using here_docs = vector<here_doc>;
 
         pair<command_expr, here_docs>
-        parse_command_line (token&, token_type&);
+        parse_command_expr (token&, token_type&);
 
         command_exit
         parse_command_exit (token&, token_type&);
@@ -129,10 +115,25 @@ namespace build2
         void
         parse_here_documents (token&, token_type&,
                               pair<command_expr, here_docs>&);
-
         string
         parse_here_document (token&, token_type&, const string&, bool);
 
+        // Execute. Issue diagnostics and throw failed in case of an error.
+        //
+      public:
+        void
+        execute (script& s, runner& r) {if (!s.empty ()) execute (s, s, r);}
+
+        void
+        execute (scope&, script&, runner&);
+
+      protected:
+        void
+        exec_scope_body ();
+
+        void
+        exec_lines (lines::iterator, lines::iterator, size_t&, bool);
+
         // Customization hooks.
         //
       protected:
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index 6534667..10103f3 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -30,6 +30,10 @@ namespace build2
         return !s.empty ();
       }
 
+      //
+      // Pre-parse.
+      //
+
       void parser::
       pre_parse (script& s)
       {
@@ -84,28 +88,8 @@ namespace build2
         group_->end_loc_ = get_location (t);
       }
 
-      void parser::
-      parse (scope& sc, script& s, runner& r)
-      {
-        path_ = nullptr; // Set by replays.
-
-        pre_parse_ = false;
-
-        lexer_ = nullptr;
-        base_parser::lexer_ = nullptr;
-
-        script_ = &s;
-        runner_ = &r;
-        group_ = nullptr;
-        id_map_ = nullptr;
-        include_set_ = nullptr;
-        scope_ = &sc;
-
-        parse_scope_body ();
-      }
-
       bool parser::
-      demote_group_scope (unique_ptr<scope>& s)
+      pre_parse_demote_group_scope (unique_ptr<scope>& s)
       {
         // See if this turned out to be an explicit test scope. An explicit
         // test scope contains a single test, only variable assignments in
@@ -136,7 +120,8 @@ namespace build2
             !t->desc                                                 &&
             !t->if_cond_)
         {
-          if (g.if_chain != nullptr && !demote_group_scope (g.if_chain))
+          if (g.if_chain != nullptr &&
+              !pre_parse_demote_group_scope (g.if_chain))
             return false;
 
           // It would have been nice to reuse the test object and only throw
@@ -182,6 +167,9 @@ namespace build2
       token parser::
       pre_parse_scope_body ()
       {
+        // enter: next token is first token of scope body
+        // leave: rcbrace or eos (returned)
+
         token t;
         type tt;
 
@@ -228,10 +216,10 @@ namespace build2
                 ? d->id
                 : insert_id (id_prefix_ + to_string (sl.line), sl));
 
-              unique_ptr<scope> g (pre_parse_scope (t, tt, id));
+              unique_ptr<scope> g (pre_parse_scope_block (t, tt, id));
               g->desc = move (d);
 
-              demote_group_scope (g);
+              pre_parse_demote_group_scope (g);
               group_->scopes.push_back (move (g));
               continue;
             }
@@ -245,109 +233,11 @@ namespace build2
         }
       }
 
-      void parser::
-      parse_scope_body ()
-      {
-        size_t li (0);
-
-        runner_->enter (*scope_, scope_->start_loc_);
-
-        if (test* t = dynamic_cast<test*> (scope_))
-        {
-          parse_lines (t->tests_.begin (), t->tests_.end (), li, true);
-        }
-        else if (group* g = dynamic_cast<group*> (scope_))
-        {
-          parse_lines (g->setup_.begin (), g->setup_.end (), li, false);
-
-          for (const unique_ptr<scope>& chain: g->scopes)
-          {
-            // Pick a scope from the if-else chain.
-            //
-            // @@ Should we free the memory by dropping all but the choosen
-            //    scope?
-            //
-            scope* s (chain.get ());
-            for (; s != nullptr; s = s->if_chain.get ())
-            {
-              if (!s->if_cond_)
-              {
-                assert (s->if_chain == nullptr);
-                break;
-              }
-
-              line l (move (*s->if_cond_));
-              line_type lt (l.type);
-
-              replay_data (move (l.tokens));
-
-              token t;
-              type tt;
-
-              next (t, tt);
-              const location ll (get_location (t));
-              next (t, tt); // Skip to start of command.
-
-              bool take;
-              if (lt != line_type::cmd_else)
-              {
-                // Note: the line index count continues from setup.
-                //
-                command_expr ce (parse_command_expr (t, tt));
-                take = runner_->run_if (*scope_, ce, ++li, ll);
-
-                if (lt == line_type::cmd_ifn || lt == line_type::cmd_elifn)
-                  take = !take;
-              }
-              else
-              {
-                assert (tt == type::newline);
-                take = true;
-              }
-
-              replay_stop ();
-
-              if (take)
-              {
-                // Count the remaining conditions for the line index.
-                //
-                for (scope* r (s->if_chain.get ());
-                     r != nullptr && r->if_cond_->type != line_type::cmd_else;
-                     r = r->if_chain.get ())
-                  ++li;
-
-                break;
-              }
-            }
-
-            if (s != nullptr && !s->empty ())
-            {
-              // Hand it off to a sub-parser potentially in another thread.
-              // But we could also have handled it serially in this parser:
-              //
-              // scope* os (scope_);
-              // scope_ = s;
-              // parse_scope_body ();
-              // scope_ = os;
-              //
-              parser p;
-              p.parse (*s, *script_, *runner_);
-            }
-          }
-
-          parse_lines (g->tdown_.begin (), g->tdown_.end (), li, false);
-        }
-        else
-          assert (false);
-
-        runner_->leave (*scope_, scope_->end_loc_);
-      }
-
       unique_ptr<group> parser::
-      pre_parse_scope (token& t, type& tt, const string& id)
+      pre_parse_scope_block (token& t, type& tt, const string& id)
       {
         // enter: lcbrace
-        // leave: newline
+        // leave: newline after rcbrace
 
         const location sl (get_location (t));
 
@@ -391,183 +281,9 @@ namespace build2
         return g;
       }
 
-      description parser::
-      pre_parse_leading_description (token& t, type& tt)
-      {
-        // Note: token is only peeked at. On return tt is also only peeked at
-        // and in the first_token mode.
-        //
-        assert (tt == type::colon);
-
-        description r;
-        location loc (get_location (peeked ()));
-
-        string sp;     // Strip prefix.
-        size_t sn (0); // Strip prefix length.
-
-        for (size_t ln (1); tt == type::colon; ++ln)
-        {
-          next (t, tt); // Get ':'.
-
-          mode (lexer_mode::description_line);
-          next (t, tt);
-
-          // If it is empty, then we get newline right away.
-          //
-          const string& l (tt == type::word ? t.value : string ());
-
-          if (tt == type::word)
-            next (t, tt); // Get newline.
-
-          assert (tt == type::newline);
-
-          // If this is the first line, then get the "strip prefix", i.e.,
-          // the beginning of the line that contains only whitespaces. If
-          // the subsequent lines start with the same prefix, then we strip
-          // it.
-          //
-          if (ln == 1)
-          {
-            sn = l.find_first_not_of (" \t");
-            sp.assign (l, 0, sn == string::npos ? (sn = 0) : sn);
-          }
-
-          // Apply strip prefix.
-          //
-          size_t i (l.compare (0, sn, sp) == 0 ? sn : 0);
-
-          // Strip trailing whitespaces, as a courtesy to the user.
-          //
-          size_t j (l.find_last_not_of (" \t"));
-          j = j != string::npos ? j + 1 : i;
-
-          size_t n (j - i); // [i, j) is our data.
-
-          if (ln == 1)
-          {
-            // First line. Ignore if it's blank.
-            //
-            if (n == 0)
-              --ln; // Stay as if on the first line.
-            else
-            {
-              // Otherwise, see if it is the id. Failed that we assume it is
-              // the summary until we see the next line.
-              //
-              (l.find_first_of (" \t", i) >= j ? r.id : r.summary).
-                assign (l, i, n);
-            }
-          }
-          else if (ln == 2)
-          {
-            // If this is a blank then whatever we have in id/summary is good.
-            // Otherwise, if we have id, then assume this is summary until we
-            // see the next line. And if not, then move what we (wrongly)
-            // assumed to be the summary to details.
-            //
-            if (n != 0)
-            {
-              if (!r.id.empty ())
-                r.summary.assign (l, i, n);
-              else
-              {
-                r.details = move (r.summary);
-                r.details += '\n';
-                r.details.append (l, i, n);
-
-                r.summary.clear ();
-              }
-            }
-          }
-          // Don't treat line 3 as special if we have given up on id/summary.
-          //
-          else if (ln == 3 && r.details.empty ())
-          {
-            // If this is a blank and we have id and/or summary, then we are
-            // good. Otherwise, if we have both, then move what we (wrongly)
-            // assumed to be id and summary to details.
-            //
-            if (n != 0)
-            {
-              if (!r.id.empty () && !r.summary.empty ())
-              {
-                r.details = move (r.id);
-                r.details += '\n';
-                r.details += r.summary;
-                r.details += '\n';
-
-                r.id.clear ();
-                r.summary.clear ();
-              }
-
-              r.details.append (l, i, n);
-            }
-          }
-          else
-          {
-            if (!r.details.empty ())
-              r.details += '\n';
-
-            r.details.append (l, i, n);
-          }
-
-          tt = peek (lexer_mode::first_token);
-        }
-
-        // Zap trailing newlines in the details.
-        //
-        size_t p (r.details.find_last_not_of ('\n'));
-        if (p != string::npos && ++p != r.details.size ())
-          r.details.resize (p);
-
-        // Insert id into the id map if we have one.
-        //
-        if (!r.id.empty ())
-          insert_id (r.id, loc);
-
-        if (r.empty ())
-          fail (loc) << "empty description";
-
-        return r;
-      }
-
-      description parser::
-      parse_trailing_description (token& t, type& tt)
-      {
-        // Parse one-line trailing description.
-        //
-        description r;
-
-        // @@ Would be nice to omit trailing description from replay.
-        //
-        const location loc (get_location (t));
-
-        mode (lexer_mode::description_line);
-        next (t, tt);
-
-        // If it is empty, then we will get newline right away.
-        //
-        if (tt == type::word)
-        {
-          string l (move (t.value));
-          trim (l); // Strip leading/trailing whitespaces.
-
-          // Decide whether this is id or summary.
-          //
-          (l.find_first_of (" \t") == string::npos ? r.id : r.summary) =
-            move (l);
-
-          next (t, tt); // Get newline.
-        }
-
-        assert (tt == type::newline); // Lexer mode invariant.
-
-        if (r.empty ())
-          fail (loc) << "empty description";
-
-        return r;
-      }
-
+      // Parse a logical line (as well as scope-if since the only way to
+      // recognize it is to parse the if line).
+      //
       // If one is true then only parse one line returning an indication of
       // whether the line ended with a semicolon.
       //
@@ -577,6 +293,9 @@ namespace build2
                       lines* ls,
                       bool one)
       {
+        // enter: next token is peeked at (type in tt)
+        // leave: newline
+
         // Note: token is only peeked at.
         //
         const location ll (get_location (peeked ()));
@@ -648,7 +367,7 @@ namespace build2
                 if (ls != nullptr)
                   fail (ll) << "directive after ';'";
 
-                parse_directive_line (t, tt);
+                pre_parse_directive (t, tt);
                 assert (tt == type::newline);
                 return false;
               }
@@ -725,7 +444,7 @@ namespace build2
             pair<command_expr, here_docs> p;
 
             if (lt != line_type::cmd_else && lt != line_type::cmd_end)
-              p = parse_command_line (t, tt);
+              p = parse_command_expr (t, tt);
 
             // Colon and semicolon are only valid in test command lines and
             // after 'end' in if-else. Note that we still recognize them
@@ -929,235 +648,6 @@ namespace build2
         return semi;
       }
 
-      void parser::
-      parse_lines (lines::iterator i, lines::iterator e, size_t& li, bool test)
-      {
-        token t;
-        type tt;
-
-        for (; i != e; ++i)
-        {
-          line& l (*i);
-          line_type lt (l.type);
-
-          assert (path_ == nullptr);
-          replay_data (move (l.tokens)); // Set the tokens and start playing.
-
-          // We don't really need to change the mode since we already know
-          // the line type.
-          //
-          next (t, tt);
-          const location ll (get_location (t));
-
-          switch (lt)
-          {
-          case line_type::var:
-            {
-              // Parse.
-              //
-              string name (move (t.value));
-
-              next (t, tt);
-              type kind (tt); // Assignment kind.
-
-              value rhs (parse_variable_line (t, tt));
-
-              if (tt == type::semi)
-                next (t, tt);
-
-              assert (tt == type::newline);
-
-              // Assign.
-              //
-              const variable& var (script_->var_pool.insert (move (name)));
-
-              value& lhs (kind == type::assign
-                          ? scope_->assign (var)
-                          : scope_->append (var));
-
-              // @@ Need to adjust to make strings the default type.
-              //
-              apply_value_attributes (&var, lhs, move (rhs), kind);
-
-              // Handle the $*, $NN special aliases.
-              //
-              // The plan is as follows: here we detect modification of the
-              // source variables (test*), and (re)set $* to NULL on this
-              // scope (this is important to both invalidate any old values
-              // but also to "stake" the lookup position). This signals to
-              // the variable lookup function below that the $* and $NN
-              // values need to be recalculated from their sources. Note
-              // that we don't need to invalidate $NN since their lookup
-              // always checks $* first.
-              //
-              if (var.name == script_->test_var.name ||
-                  var.name == script_->opts_var.name ||
-                  var.name == script_->args_var.name)
-              {
-                scope_->assign (script_->cmd_var) = nullptr;
-              }
-
-              replay_stop ();
-              break;
-            }
-          case line_type::cmd:
-            {
-              // We use the 0 index to signal that this is the only command.
-              // Note that we only do this for test commands.
-              //
-              if (test && li == 0)
-              {
-                lines::iterator j (i);
-                for (++j; j != e && j->type == line_type::var; ++j) ;
-
-                if (j != e) // We have another command.
-                  ++li;
-              }
-              else
-                ++li;
-
-              command_expr ce (parse_command_expr (t, tt));
-              runner_->run (*scope_, ce, li, ll);
-
-              replay_stop ();
-              break;
-            }
-          case line_type::cmd_if:
-          case line_type::cmd_ifn:
-          case line_type::cmd_elif:
-          case line_type::cmd_elifn:
-          case line_type::cmd_else:
-            {
-              next (t, tt); // Skip to start of command.
-
-              bool take;
-              if (lt != line_type::cmd_else)
-              {
-                // Assume if-else always involves multiple commands.
-                //
-                command_expr ce (parse_command_expr (t, tt));
-                take = runner_->run_if (*scope_, ce, ++li, ll);
-
-                if (lt == line_type::cmd_ifn || lt == line_type::cmd_elifn)
-                  take = !take;
-              }
-              else
-              {
-                assert (tt == type::newline);
-                take = true;
-              }
-
-              replay_stop ();
-
-              // If end is true, then find the 'end' line. Otherwise, find the
-              // next if-else line. If skip is true then increment the command
-              // line index.
-              //
-              auto next = [e, &li]
-                (lines::iterator j, bool end, bool skip) -> lines::iterator
-              {
-                // We need to be aware of nested if-else chains.
-                //
-                size_t n (0);
-
-                for (++j; j != e; ++j)
-                {
-                  line_type lt (j->type);
-
-                  if (lt == line_type::cmd_if ||
-                      lt == line_type::cmd_ifn)
-                    ++n;
-
-                  // If we are nested then we just wait until we get back to
-                  // the surface.
-                  //
-                  if (n == 0)
-                  {
-                    switch (lt)
-                    {
-                    case line_type::cmd_elif:
-                    case line_type::cmd_elifn:
-                    case line_type::cmd_else: if (end) break; // Fall through.
-                    case line_type::cmd_end:  return j;
-                    default: break;
-                    }
-                  }
-
-                  if (lt == line_type::cmd_end)
-                    --n;
-
-                  if (skip)
-                  {
-                    // Note that we don't count else and end as commands.
-                    //
-                    switch (lt)
-                    {
-                    case line_type::cmd:
-                    case line_type::cmd_if:
-                    case line_type::cmd_ifn:
-                    case line_type::cmd_elif:
-                    case line_type::cmd_elifn: ++li; break;
-                    default:                         break;
-                    }
-                  }
-                }
-
-                assert (false); // Missing end.
-                return e;
-              };
-
-              // If we are taking this branch then we need to parse all the
-              // lines until the next if-else line and then skip all the lines
-              // until the end (unless next is already end).
-              //
-              // Otherwise, we need to skip all the lines until the next
-              // if-else line and then continue parsing.
-              //
-              if (take)
-              {
-                lines::iterator j (next (i, false, false)); // Next if-else.
-                parse_lines (i + 1, j, li, test);
-                i = j->type == line_type::cmd_end ? j : next (j, true, true);
-              }
-              else
-              {
-                i = next (i, false, true);
-                if (i->type != line_type::cmd_end)
-                  --i; // Continue with this line (e.g., elif or else).
-              }
-
-              break;
-            }
-          case line_type::cmd_end:
-            {
-              assert (false);
-            }
-          }
-        }
-      }
-
-      command_expr parser::
-      parse_command_expr (token& t, type& tt)
-      {
-        // enter: first token of the command line
-        // leave: <newline>
-
-        pair<command_expr, here_docs> p (parse_command_line (t, tt));
-
-        switch (tt)
-        {
-        case type::colon: parse_trailing_description (t, tt); break;
-        case type::semi: next (t, tt); break; // Get newline.
-        }
-
-        assert (tt == type::newline);
-
-        parse_here_documents (t, tt, p);
-        assert (tt == type::newline);
-
-        return move (p.first);
-      };
-
       bool parser::
       pre_parse_if_else (token& t, type& tt,
                          optional<description>& d,
@@ -1207,7 +697,7 @@ namespace build2
           next (t, tt); // Get '{'.
 
           {
-            unique_ptr<group> g (pre_parse_scope (t, tt, id));
+            unique_ptr<group> g (pre_parse_scope_block (t, tt, id));
 
             // If-condition.
             //
@@ -1287,7 +777,7 @@ namespace build2
           }
         }
 
-        demote_group_scope (root);
+        pre_parse_demote_group_scope (root);
         group_->scopes.push_back (move (root));
         return false; // We never end with a semi.
       }
@@ -1297,7 +787,7 @@ namespace build2
                                  optional<description>& d,
                                  lines& ls)
       {
-        // enter: peeked first token of next line
+        // enter: peeked first token of next line (type in tt)
         // leave: newline
 
         // Parse lines until we see closing 'end'. Nested if-else blocks are
@@ -1380,8 +870,11 @@ namespace build2
       }
 
       void parser::
-      parse_directive_line (token& t, type& tt)
+      pre_parse_directive (token& t, type& tt)
       {
+        // enter: directive name
+        // leave: newline
+
         string d (t.value);
         location l (get_location (t));
         next (t, tt);
@@ -1402,13 +895,13 @@ namespace build2
           fail (t) << t << " after directive";
 
         if (d == ".include")
-          perform_include (move (args), move (l));
+          pre_parse_include_line (move (args), move (l));
         else
           assert (false); // Unhandled directive.
       }
 
       void parser::
-      perform_include (names args, location dl)
+      pre_parse_include_line (names args, location dl)
       {
         auto i (args.begin ());
 
@@ -1504,9 +997,192 @@ namespace build2
         }
       }
 
+      description parser::
+      pre_parse_leading_description (token& t, type& tt)
+      {
+        // enter: peeked at colon (type in tt)
+        // leave: peeked at in the first_token mode (type in tt)
+
+        assert (tt == type::colon);
+
+        description r;
+        location loc (get_location (peeked ()));
+
+        string sp;     // Strip prefix.
+        size_t sn (0); // Strip prefix length.
+
+        for (size_t ln (1); tt == type::colon; ++ln)
+        {
+          next (t, tt); // Get ':'.
+
+          mode (lexer_mode::description_line);
+          next (t, tt);
+
+          // If it is empty, then we get newline right away.
+          //
+          const string& l (tt == type::word ? t.value : string ());
+
+          if (tt == type::word)
+            next (t, tt); // Get newline.
+
+          assert (tt == type::newline);
+
+          // If this is the first line, then get the "strip prefix", i.e.,
+          // the beginning of the line that contains only whitespaces. If
+          // the subsequent lines start with the same prefix, then we strip
+          // it.
+          //
+          if (ln == 1)
+          {
+            sn = l.find_first_not_of (" \t");
+            sp.assign (l, 0, sn == string::npos ? (sn = 0) : sn);
+          }
+
+          // Apply strip prefix.
+          //
+          size_t i (l.compare (0, sn, sp) == 0 ? sn : 0);
+
+          // Strip trailing whitespaces, as a courtesy to the user.
+          //
+          size_t j (l.find_last_not_of (" \t"));
+          j = j != string::npos ? j + 1 : i;
+
+          size_t n (j - i); // [i, j) is our data.
+
+          if (ln == 1)
+          {
+            // First line. Ignore if it's blank.
+            //
+            if (n == 0)
+              --ln; // Stay as if on the first line.
+            else
+            {
+              // Otherwise, see if it is the id. Failed that we assume it is
+              // the summary until we see the next line.
+              //
+              (l.find_first_of (" \t", i) >= j ? r.id : r.summary).
+                assign (l, i, n);
+            }
+          }
+          else if (ln == 2)
+          {
+            // If this is a blank then whatever we have in id/summary is good.
+            // Otherwise, if we have id, then assume this is summary until we
+            // see the next line. And if not, then move what we (wrongly)
+            // assumed to be the summary to details.
+            //
+            if (n != 0)
+            {
+              if (!r.id.empty ())
+                r.summary.assign (l, i, n);
+              else
+              {
+                r.details = move (r.summary);
+                r.details += '\n';
+                r.details.append (l, i, n);
+
+                r.summary.clear ();
+              }
+            }
+          }
+          // Don't treat line 3 as special if we have given up on id/summary.
+          //
+          else if (ln == 3 && r.details.empty ())
+          {
+            // If this is a blank and we have id and/or summary, then we are
+            // good. Otherwise, if we have both, then move what we (wrongly)
+            // assumed to be id and summary to details.
+            //
+            if (n != 0)
+            {
+              if (!r.id.empty () && !r.summary.empty ())
+              {
+                r.details = move (r.id);
+                r.details += '\n';
+                r.details += r.summary;
+                r.details += '\n';
+
+                r.id.clear ();
+                r.summary.clear ();
+              }
+
+              r.details.append (l, i, n);
+            }
+          }
+          else
+          {
+            if (!r.details.empty ())
+              r.details += '\n';
+
+            r.details.append (l, i, n);
+          }
+
+          tt = peek (lexer_mode::first_token);
+        }
+
+        // Zap trailing newlines in the details.
+        //
+        size_t p (r.details.find_last_not_of ('\n'));
+        if (p != string::npos && ++p != r.details.size ())
+          r.details.resize (p);
+
+        // Insert id into the id map if we have one.
+        //
+        if (!r.id.empty ())
+          insert_id (r.id, loc);
+
+        if (r.empty ())
+          fail (loc) << "empty description";
+
+        return r;
+      }
+
+      description parser::
+      parse_trailing_description (token& t, type& tt)
+      {
+        // enter: colon
+        // leave: newline
+
+        // Parse one-line trailing description.
+        //
+        description r;
+
+        // @@ Would be nice to omit trailing description from replay.
+        //
+        const location loc (get_location (t));
+
+        mode (lexer_mode::description_line);
+        next (t, tt);
+
+        // If it is empty, then we will get newline right away.
+        //
+        if (tt == type::word)
+        {
+          string l (move (t.value));
+          trim (l); // Strip leading/trailing whitespaces.
+
+          // Decide whether this is id or summary.
+          //
+          (l.find_first_of (" \t") == string::npos ? r.id : r.summary) =
+            move (l);
+
+          next (t, tt); // Get newline.
+        }
+
+        assert (tt == type::newline); // Lexer mode invariant.
+
+        if (r.empty ())
+          fail (loc) << "empty description";
+
+        return r;
+      }
+
       value parser::
       parse_variable_line (token& t, type& tt)
       {
+        // enter: assignment
+        // leave: newline or semi
+
         // We cannot reuse the value mode since it will recognize { which we
         // want to treat as a literal.
         //
@@ -1523,9 +1199,36 @@ namespace build2
           : value (names ());
       }
 
-      pair<command_expr, parser::here_docs> parser::
+      command_expr parser::
       parse_command_line (token& t, type& tt)
       {
+        // enter: first token of the command line
+        // leave: <newline>
+
+        // Note: this one is only used during execution.
+
+        pair<command_expr, here_docs> p (parse_command_expr (t, tt));
+
+        switch (tt)
+        {
+        case type::colon: parse_trailing_description (t, tt); break;
+        case type::semi: next (t, tt); break; // Get newline.
+        }
+
+        assert (tt == type::newline);
+
+        parse_here_documents (t, tt, p);
+        assert (tt == type::newline);
+
+        return move (p.first);
+      };
+
+      pair<command_expr, parser::here_docs> parser::
+      parse_command_expr (token& t, type& tt)
+      {
+        // enter: first token of the command line
+        // leave: <newline>
+
         command_expr expr {{expr_operator::log_and, {}}};
         command c; // Command being assembled.
 
@@ -2277,6 +1980,9 @@ namespace build2
       command_exit parser::
       parse_command_exit (token& t, type& tt)
       {
+        // enter: equal/not_equal
+        // leave: token after exit status (one parse_names() chunk)
+
         exit_comparison comp (tt == type::equal
                               ? exit_comparison::eq
                               : exit_comparison::ne);
@@ -2315,6 +2021,9 @@ namespace build2
       parse_here_documents (token& t, type& tt,
                             pair<command_expr, here_docs>& p)
       {
+        // enter: newline
+        // leave: newline
+
         // Parse here-document fragments in the order they were mentioned on
         // the command line.
         //
@@ -2344,6 +2053,9 @@ namespace build2
       string parser::
       parse_here_document (token& t, type& tt, const string& em, bool nn)
       {
+        // enter: first token on first line
+        // leave: newline (after end marker)
+
         string r;
 
         // Here-documents can be indented. The leading whitespaces of the end
@@ -2496,6 +2208,339 @@ namespace build2
         return r;
       }
 
+      //
+      // Execute.
+      //
+
+      void parser::
+      execute (scope& sc, script& s, runner& r)
+      {
+        path_ = nullptr; // Set by replays.
+
+        pre_parse_ = false;
+
+        lexer_ = nullptr;
+        base_parser::lexer_ = nullptr;
+
+        script_ = &s;
+        runner_ = &r;
+        group_ = nullptr;
+        id_map_ = nullptr;
+        include_set_ = nullptr;
+        scope_ = &sc;
+
+        exec_scope_body ();
+      }
+
+      void parser::
+      exec_scope_body ()
+      {
+        size_t li (0);
+
+        runner_->enter (*scope_, scope_->start_loc_);
+
+        if (test* t = dynamic_cast<test*> (scope_))
+        {
+          exec_lines (t->tests_.begin (), t->tests_.end (), li, true);
+        }
+        else if (group* g = dynamic_cast<group*> (scope_))
+        {
+          exec_lines (g->setup_.begin (), g->setup_.end (), li, false);
+
+          for (const unique_ptr<scope>& chain: g->scopes)
+          {
+            // Pick a scope from the if-else chain.
+            //
+            // @@ Should we free the memory by dropping all but the choosen
+            //    scope?
+            //
+            scope* s (chain.get ());
+            for (; s != nullptr; s = s->if_chain.get ())
+            {
+              if (!s->if_cond_)
+              {
+                assert (s->if_chain == nullptr);
+                break;
+              }
+
+              line l (move (*s->if_cond_));
+              line_type lt (l.type);
+
+              replay_data (move (l.tokens));
+
+              token t;
+              type tt;
+
+              next (t, tt);
+              const location ll (get_location (t));
+              next (t, tt); // Skip to start of command.
+
+              bool take;
+              if (lt != line_type::cmd_else)
+              {
+                // Note: the line index count continues from setup.
+                //
+                command_expr ce (parse_command_line (t, tt));
+                take = runner_->run_if (*scope_, ce, ++li, ll);
+
+                if (lt == line_type::cmd_ifn || lt == line_type::cmd_elifn)
+                  take = !take;
+              }
+              else
+              {
+                assert (tt == type::newline);
+                take = true;
+              }
+
+              replay_stop ();
+
+              if (take)
+              {
+                // Count the remaining conditions for the line index.
+                //
+                for (scope* r (s->if_chain.get ());
+                     r != nullptr && r->if_cond_->type != line_type::cmd_else;
+                     r = r->if_chain.get ())
+                  ++li;
+
+                break;
+              }
+            }
+
+            if (s != nullptr && !s->empty ())
+            {
+              // Hand it off to a sub-parser potentially in another thread.
+              // But we could also have handled it serially in this parser:
+              //
+              // scope* os (scope_);
+              // scope_ = s;
+              // exec_scope_body ();
+              // scope_ = os;
+              //
+              parser p;
+              p.execute (*s, *script_, *runner_);
+            }
+          }
+
+          exec_lines (g->tdown_.begin (), g->tdown_.end (), li, false);
+        }
+        else
+          assert (false);
+
+        runner_->leave (*scope_, scope_->end_loc_);
+      }
+
+      void parser::
+      exec_lines (lines::iterator i, lines::iterator e, size_t& li, bool test)
+      {
+        token t;
+        type tt;
+
+        for (; i != e; ++i)
+        {
+          line& l (*i);
+          line_type lt (l.type);
+
+          assert (path_ == nullptr);
+          replay_data (move (l.tokens)); // Set the tokens and start playing.
+
+          // We don't really need to change the mode since we already know
+          // the line type.
+          //
+          next (t, tt);
+          const location ll (get_location (t));
+
+          switch (lt)
+          {
+          case line_type::var:
+            {
+              // Parse.
+              //
+              string name (move (t.value));
+
+              next (t, tt);
+              type kind (tt); // Assignment kind.
+
+              value rhs (parse_variable_line (t, tt));
+
+              if (tt == type::semi)
+                next (t, tt);
+
+              assert (tt == type::newline);
+
+              // Assign.
+              //
+              const variable& var (script_->var_pool.insert (move (name)));
+
+              value& lhs (kind == type::assign
+                          ? scope_->assign (var)
+                          : scope_->append (var));
+
+              // @@ Need to adjust to make strings the default type.
+              //
+              apply_value_attributes (&var, lhs, move (rhs), kind);
+
+              // Handle the $*, $NN special aliases.
+              //
+              // The plan is as follows: here we detect modification of the
+              // source variables (test*), and (re)set $* to NULL on this
+              // scope (this is important to both invalidate any old values
+              // but also to "stake" the lookup position). This signals to
+              // the variable lookup function below that the $* and $NN
+              // values need to be recalculated from their sources. Note
+              // that we don't need to invalidate $NN since their lookup
+              // always checks $* first.
+              //
+              if (var.name == script_->test_var.name ||
+                  var.name == script_->opts_var.name ||
+                  var.name == script_->args_var.name)
+              {
+                scope_->assign (script_->cmd_var) = nullptr;
+              }
+
+              replay_stop ();
+              break;
+            }
+          case line_type::cmd:
+            {
+              // We use the 0 index to signal that this is the only command.
+              // Note that we only do this for test commands.
+              //
+              if (test && li == 0)
+              {
+                lines::iterator j (i);
+                for (++j; j != e && j->type == line_type::var; ++j) ;
+
+                if (j != e) // We have another command.
+                  ++li;
+              }
+              else
+                ++li;
+
+              command_expr ce (parse_command_line (t, tt));
+              runner_->run (*scope_, ce, li, ll);
+
+              replay_stop ();
+              break;
+            }
+          case line_type::cmd_if:
+          case line_type::cmd_ifn:
+          case line_type::cmd_elif:
+          case line_type::cmd_elifn:
+          case line_type::cmd_else:
+            {
+              next (t, tt); // Skip to start of command.
+
+              bool take;
+              if (lt != line_type::cmd_else)
+              {
+                // Assume if-else always involves multiple commands.
+                //
+                command_expr ce (parse_command_line (t, tt));
+                take = runner_->run_if (*scope_, ce, ++li, ll);
+
+                if (lt == line_type::cmd_ifn || lt == line_type::cmd_elifn)
+                  take = !take;
+              }
+              else
+              {
+                assert (tt == type::newline);
+                take = true;
+              }
+
+              replay_stop ();
+
+              // If end is true, then find the 'end' line. Otherwise, find the
+              // next if-else line. If skip is true then increment the command
+              // line index.
+              //
+              auto next = [e, &li]
+                (lines::iterator j, bool end, bool skip) -> lines::iterator
+              {
+                // We need to be aware of nested if-else chains.
+                //
+                size_t n (0);
+
+                for (++j; j != e; ++j)
+                {
+                  line_type lt (j->type);
+
+                  if (lt == line_type::cmd_if ||
+                      lt == line_type::cmd_ifn)
+                    ++n;
+
+                  // If we are nested then we just wait until we get back to
+                  // the surface.
+                  //
+                  if (n == 0)
+                  {
+                    switch (lt)
+                    {
+                    case line_type::cmd_elif:
+                    case line_type::cmd_elifn:
+                    case line_type::cmd_else: if (end) break; // Fall through.
+                    case line_type::cmd_end:  return j;
+                    default: break;
+                    }
+                  }
+
+                  if (lt == line_type::cmd_end)
+                    --n;
+
+                  if (skip)
+                  {
+                    // Note that we don't count else and end as commands.
+                    //
+                    switch (lt)
+                    {
+                    case line_type::cmd:
+                    case line_type::cmd_if:
+                    case line_type::cmd_ifn:
+                    case line_type::cmd_elif:
+                    case line_type::cmd_elifn: ++li; break;
+                    default:                         break;
+                    }
+                  }
+                }
+
+                assert (false); // Missing end.
+                return e;
+              };
+
+              // If we are taking this branch then we need to parse all the
+              // lines until the next if-else line and then skip all the lines
+              // until the end (unless next is already end).
+              //
+              // Otherwise, we need to skip all the lines until the next
+              // if-else line and then continue parsing.
+              //
+              if (take)
+              {
+                lines::iterator j (next (i, false, false)); // Next if-else.
+                exec_lines (i + 1, j, li, test);
+                i = j->type == line_type::cmd_end ? j : next (j, true, true);
+              }
+              else
+              {
+                i = next (i, false, true);
+                if (i->type != line_type::cmd_end)
+                  --i; // Continue with this line (e.g., elif or else).
+              }
+
+              break;
+            }
+          case line_type::cmd_end:
+            {
+              assert (false);
+            }
+          }
+        }
+      }
+
+      //
+      // The rest.
+      //
+
       lookup parser::
       lookup_variable (name&& qual, string&& name, const location& loc)
       {
@@ -2518,7 +2563,7 @@ namespace build2
 
         // Handle the $*, $NN special aliases.
         //
-        // See the parse_lines() for the overall plan.
+        // See the exec_lines() for the overall plan.
         //
         // @@ MT: we are potentially changing outer scopes. Could force
         //    lookup before executing tests in each group scope. Poblem is
diff --git a/doc/testscript.cli b/doc/testscript.cli
index 123846f..1ba9a4b 100644
--- a/doc/testscript.cli
+++ b/doc/testscript.cli
@@ -708,14 +708,13 @@ script:
 
 scope-body:
   *setup
-  *(scope|test|include)
-  *teardown
+  *(directive|scope|test)
+  *tdown
 
-setup:
-  variable-line|setup-line
+directive:
+  include
 
-teardown:
-  variable-line|teardown-line
+include: '.include' (' '+'--once')*(' '+<path>)*
 
 scope:
   ?description
@@ -726,54 +725,60 @@ scope-block:
   scope-body
   '}'
 
-scope-if: ('if'|'if!') command-expr
+scope-if: ('if'|'if!') command-line
   scope-block
   *scope-elif
   ?scope-else
 
-scope-elif: ('elif'|'elif!') command-expr
+scope-elif: ('elif'|'elif!') command-line
   scope-block
 
 scope-else: 'else'
   scope-block
 
-test:
-  ?description
-  *((variable-line|test-line) ';')
-  test-line (':' <text>)?
-
-include: '.include' (' '+'--once')*(' '+<path>)*
-
 description:
   +(':' <text>)
 
+setup:
+  variable-line|setup-line
+
+tdown:
+  variable-line|tdown-line
+
+setup-line: '+' command-like
+tdown-line: '-' command-like
+
+test:
+  ?description
+  +(variable-line|command-like)
+
 variable-line: <variable> ('='|'+='|'=+') value-attributes? <value>
 value-attributes: '[' <key-value-pairs> ']'
 
-setup-line:    '+' command-line
-teardown-line: '-' command-line
-test-line:         command-line
+command-like:
+  command-line|command-if
 
-command-line:
-  command-expr|command-if
+# Note: ':' and ';' only valid directly in test.
+#
+command-line: command-expr (';'|(':' <text>))?
+  *here-document
 
-command-if: ('if'|'if!') command-expr
+command-if: ('if'|'if!') command-line
   command-if-body
   *command-elif
   ?command-else
-  'end'
+  'end' (';'|(':' <text>))?
 
-command-elif: ('elif'|'elif!') command-expr
+command-elif: ('elif'|'elif!') command-line
   command-if-body
 
 command-else: 'else'
   command-if-body
 
 command-if-body:
-  *(variable-line|command-line)
+  *(variable-line|command-like)
 
 command-expr: command-pipe (('||'|'&&') command-pipe)*
-  *here-document
 
 command-pipe: command ('|' command)*
 
diff --git a/unit-tests/test/script/parser/driver.cxx b/unit-tests/test/script/parser/driver.cxx
index de34da7..0fcba38 100644
--- a/unit-tests/test/script/parser/driver.cxx
+++ b/unit-tests/test/script/parser/driver.cxx
@@ -198,7 +198,7 @@ namespace build2
           p.pre_parse (cin, s);
 
           print_runner r (scope, id, line);
-          p.parse (s, r);
+          p.execute (s, r);
         }
         catch (const failed&)
         {