Add ability for meta-operation to preprocess buildspec

8 files changed, 118 insertions, 47 deletions

diff --git a/build2/b.cxx b/build2/b.cxx
index d247958..6e5aef9 100644
--- a/build2/b.cxx
+++ b/build2/b.cxx
@@ -349,22 +349,20 @@ main (int argc, char* argv[])
 
     // If not NULL, then lifted points to the operation that has been "lifted"
     // to the meta-operaion (see the logic below for details). Skip is the
-    // position of the next operation. Dirty indicated whether we managed to
-    // execute anything before lifting an operation.
+    // position of the next operation.
     //
     opspec* lifted (nullptr);
     size_t skip (0);
-    bool dirty (false); // We already (re)set for the first run.
+
+    // The dirty flag indicated whether we managed to execute anything before
+    // lifting an operation.
+    //
+    bool dirty (false); // Already (re)set for the first run.
 
     for (auto mit (bspec.begin ()); mit != bspec.end (); )
     {
       vector_view<opspec> opspecs;
 
-      const string& mname (lifted == nullptr ? mit->name : lifted->name);
-      const values& mparams (lifted == nullptr ? mit->params : lifted->params);
-
-      current_mname = &mname;
-
       if (lifted == nullptr)
       {
         metaopspec& ms (*mit);
@@ -402,9 +400,36 @@ main (int argc, char* argv[])
         dirty = false;
       }
 
+      const path p ("<buildspec>");
+      const location l (&p, 0, 0); //@@ TODO
+
       meta_operation_id mid (0); // Not yet translated.
       const meta_operation_info* mif (nullptr);
 
+      // See if this meta-operation wants to pre-process the opspecs. Note
+      // that this functionality can only be used for build-in meta-operations
+      // that were explicitly specified on the command line (so cannot be used
+      // for perform) and that will be lifted early (see below).
+      //
+      values& mparams (lifted == nullptr ? mit->params : lifted->params);
+      {
+        const string& mname (lifted == nullptr ? mit->name : lifted->name);
+        current_mname = &mname;
+
+        if (!mname.empty ())
+        {
+          if (meta_operation_id m = meta_operation_table.find (mname))
+          {
+            // Can modify params, opspec, change meta-operation name.
+            //
+            if (auto f = meta_operation_table[m].process)
+              current_mname = &f (mparams, opspecs, lifted != nullptr, l);
+          }
+        }
+      }
+
+      const string& mname (*current_mname);
+
       for (auto oit (opspecs.begin ()); oit != opspecs.end (); ++oit)
       {
         opspec& os (*oit);
@@ -422,9 +447,6 @@ main (int argc, char* argv[])
         if (os.empty ()) // Default target: dir{}.
           os.push_back (targetspec (name ("dir", string ())));
 
-        const path p ("<buildspec>");
-        const location l (&p, 0, 0); //@@ TODO
-
         operation_id oid (0); // Not yet translated.
         const operation_info* oif (nullptr);
 
@@ -434,14 +456,42 @@ main (int argc, char* argv[])
         operation_id post_oid (0);
         const operation_info* post_oif (nullptr);
 
+        // Return true if this operation is lifted.
+        //
+        auto lift = [&oname, &mname, &os, &mit, &lifted, &skip, &l, &trace] ()
+        {
+          meta_operation_id m (meta_operation_table.find (oname));
+
+          if (m != 0)
+          {
+            if (!mname.empty ())
+              fail (l) << "nested meta-operation " << mname << '(' << oname << ')';
+
+            l5 ([&]{trace << "lifting operation " << oname
+                          << ", id " << uint16_t (m);});
+
+            lifted = &os;
+            skip = lifted - mit->data () + 1;
+          }
+
+          return m != 0;
+        };
+
         // We do meta-operation and operation batches sequentially (no
         // parallelism). But multiple targets in an operation batch can be
         // done in parallel.
 
-        // First bootstrap projects for all the target so that all the
-        // variable overrides are set (if we also load/search/match in the
-        // same loop then we may end up loading a project (via import) before
-        // this happends.
+        // First see if we can lift this operation early by checking if it
+        // is one of the built-in meta-operations. This is important to make
+        // sure we pre-process the opspec before loading anything.
+        //
+        if (!oname.empty () && lift ())
+          break;
+
+        // Next bootstrap projects for all the target so that all the variable
+        // overrides are set (if we also load/search/match in the same loop
+        // then we may end up loading a project (via import) before this
+        // happends.
         //
         for (targetspec& ts: os)
         {
@@ -720,28 +770,12 @@ main (int argc, char* argv[])
           // known.
           //
           {
+            if (!oname.empty () && lift ())
+              break; // Out of targetspec loop.
+
             meta_operation_id m (0);
             operation_id o (0);
 
-            if (!oname.empty ())
-            {
-              m = meta_operation_table.find (oname);
-
-              if (m != 0)
-              {
-                if (!mname.empty ())
-                  fail (l) << "nested meta-operation " << mname
-                           << '(' << oname << ')';
-
-                l5 ([&]{trace << "lifting operation " << oname
-                              << ", id " << uint16_t (m);});
-
-                lifted = &os;
-                skip = lifted - mit->data () + 1;
-                break; // Out of targetspec loop.
-              }
-            }
-
             if (!mname.empty ())
             {
               m = meta_operation_table.find (mname);
@@ -779,14 +813,14 @@ main (int argc, char* argv[])
               }
             }
 
-            // The default meta-operation is perform. The default
-            // operation is assigned by the meta-operation below.
+            // The default meta-operation is perform. The default operation is
+            // assigned by the meta-operation below.
             //
             if (m == 0)
               m = perform_id;
 
-            // If this is the first target in the meta-operation batch,
-            // then set the batch meta-operation id.
+            // If this is the first target in the meta-operation batch, then
+            // set the batch meta-operation id.
             //
             if (mid == 0)
             {
@@ -795,7 +829,7 @@ main (int argc, char* argv[])
 
               if (mif == nullptr)
                 fail (l) << "target " << tn << " does not support meta-"
-                         << "operation " << meta_operation_table[m];
+                         << "operation " << meta_operation_table[m].name;
 
               l5 ([&]{trace << "start meta-operation batch " << mif->name
                             << ", id " << static_cast<uint16_t> (mid);});
@@ -819,7 +853,7 @@ main (int argc, char* argv[])
 
               if (mi == nullptr)
                 fail (l) << "target " << tn << " does not support meta-"
-                         << "operation " << meta_operation_table[mid];
+                         << "operation " << meta_operation_table[mid].name;
 
               if (mi != mif)
                 fail (l) << "different implementations of meta-operation "
diff --git a/build2/operation b/build2/operation
index ca51ee5..1f3e217 100644
--- a/build2/operation
+++ b/build2/operation
@@ -17,6 +17,7 @@ namespace build2
   class location;
   class scope;
   class target_key;
+  struct opspec;
 
   // While we are using uint8_t for the meta/operation ids, we assume
   // that each is limited to 4 bits (max 128 entries) so that we can
@@ -343,7 +344,33 @@ namespace build2
   // its semantics. It would be strange to have an operation called
   // test that does two very different things in different projects.
   //
-  extern butl::string_table<meta_operation_id> meta_operation_table;
+  // A built-in/pre-defined meta-operation can also provide a pre-processor
+  // callback that will be called for operation-specs before any project
+  // discovery/bootstrap is performed.
+  //
+  struct meta_operation_data
+  {
+    // The processor may modify the parameters, opspec, and change the
+    // meta-operation by returning a different name.
+    //
+    // If lifted is true then the operation name in opspec is bogus (has
+    // been lifted) and the default/empty name should be assumed instead.
+    //
+    using process_func = const string& (values&,
+                                        vector_view<opspec>&,
+                                        bool lifted,
+                                        const location&);
+
+    meta_operation_data () = default;
+    meta_operation_data (const char* n, process_func p = nullptr)
+        : name (n), process (p) {}
+
+    string name;
+    process_func* process;
+  };
+
+  extern butl::string_table<meta_operation_id,
+                            meta_operation_data> meta_operation_table;
   extern butl::string_table<operation_id> operation_table;
 
   // These are "sparse" in the sense that we may have "holes" that
@@ -393,4 +420,14 @@ namespace build2
   using operations = sparse_vector<const operation_info>;
 }
 
+namespace butl
+{
+  template <>
+  struct string_table_traits<build2::meta_operation_data>
+  {
+    static const std::string&
+    key (const build2::meta_operation_data& d) {return d.name;}
+  };
+}
+
 #endif // BUILD2_OPERATION
diff --git a/build2/operation.cxx b/build2/operation.cxx
index 9656e2a..2099ec6 100644
--- a/build2/operation.cxx
+++ b/build2/operation.cxx
@@ -371,6 +371,6 @@ namespace build2
 
   // Tables.
   //
-  string_table<meta_operation_id> meta_operation_table;
+  string_table<meta_operation_id, meta_operation_data> meta_operation_table;
   string_table<operation_id> operation_table;
 }
diff --git a/unit-tests/function/buildfile b/unit-tests/function/buildfile
index 0f40dd4..8c14899 100644
--- a/unit-tests/function/buildfile
+++ b/unit-tests/function/buildfile
@@ -9,7 +9,7 @@ src = token lexer diagnostics utility variable name b-options types-parsers \
 context scope parser target operation rule prerequisite file module function \
 functions-builtin functions-path functions-process-path functions-string \
 functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation}
+config/{utility init operation} spec
 
 exe{driver}: cxx{driver} ../../build2/cxx{$src} ../../build2/liba{b} $libs test{call syntax}
 
diff --git a/unit-tests/lexer/buildfile b/unit-tests/lexer/buildfile
index 47c9e45..efcd45c 100644
--- a/unit-tests/lexer/buildfile
+++ b/unit-tests/lexer/buildfile
@@ -9,7 +9,7 @@ src = token lexer diagnostics utility variable name b-options types-parsers \
 context scope parser target operation rule prerequisite file module function \
 functions-builtin functions-path functions-process-path functions-string \
 functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation}
+config/{utility init operation} spec
 
 exe{driver}: cxx{driver} ../../build2/cxx{$src} ../../build2/liba{b} $libs \
 test{*}
diff --git a/unit-tests/scheduler/buildfile b/unit-tests/scheduler/buildfile
index 5854c31..2742c73 100644
--- a/unit-tests/scheduler/buildfile
+++ b/unit-tests/scheduler/buildfile
@@ -9,7 +9,7 @@ src = token lexer diagnostics utility variable name b-options types-parsers \
 context scope parser target operation rule prerequisite file module function \
 functions-builtin functions-path functions-process-path functions-string \
 functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation}
+config/{utility init operation} spec
 
 exe{driver}: cxx{driver} ../../build2/cxx{$src} ../../build2/liba{b} $libs
 
diff --git a/unit-tests/test/script/lexer/buildfile b/unit-tests/test/script/lexer/buildfile
index eb42935..f46f835 100644
--- a/unit-tests/test/script/lexer/buildfile
+++ b/unit-tests/test/script/lexer/buildfile
@@ -9,7 +9,7 @@ src = token lexer diagnostics utility variable name b-options types-parsers \
 context scope parser target operation rule prerequisite file module function \
 functions-builtin functions-path functions-process-path functions-string \
 functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation} test/script/{token lexer}
+config/{utility init operation} test/script/{token lexer} spec
 
 exe{driver}: cxx{driver} ../../../../build2/cxx{$src} ../../../../build2/liba{b} $libs \
 test{command-line first-token second-token command-expansion variable-line \
diff --git a/unit-tests/test/script/parser/buildfile b/unit-tests/test/script/parser/buildfile
index 85668c9..1c06ad2 100644
--- a/unit-tests/test/script/parser/buildfile
+++ b/unit-tests/test/script/parser/buildfile
@@ -10,7 +10,7 @@ scope prerequisite file module operation rule b-options algorithm search \
 filesystem function functions-builtin functions-path functions-process-path \
 functions-string functions-target-triplet config/{utility init operation} \
 dump types-parsers test/{target script/{token lexer parser regex script}} \
-scheduler
+scheduler spec
 
 exe{driver}: cxx{driver} ../../../../build2/cxx{$src} ../../../../build2/liba{b} $libs \
 test{cleanup command-if command-re-parse description directive exit \