Implement pre/post operation support

Also, extend execution mode/postponed logic to propagate the postponed
target state. At the top, we now re-try postponed targets. This results
in the expected behavior when, for example, cleaning two targets with
one depending on the other.

7 files changed, 283 insertions, 88 deletions

diff --git a/build/algorithm.cxx b/build/algorithm.cxx
index 83a6510..290bfa5 100644
--- a/build/algorithm.cxx
+++ b/build/algorithm.cxx
@@ -339,35 +339,39 @@ namespace build
   target_state
   execute_prerequisites (action a, target& t)
   {
-    target_state ts (target_state::unchanged);
+    target_state r (target_state::unchanged);
 
     for (target* pt: t.prerequisite_targets)
     {
       if (pt == nullptr) // Skipped.
         continue;
 
-      if (execute (a, *pt) == target_state::changed)
-        ts = target_state::changed;
+      target_state ts (execute (a, *pt));
+      if (ts == target_state::changed ||
+          (ts == target_state::postponed && r == target_state::unchanged))
+        r = ts;
     }
 
-    return ts;
+    return r;
   }
 
   target_state
   reverse_execute_prerequisites (action a, target& t)
   {
-    target_state ts (target_state::unchanged);
+    target_state r (target_state::unchanged);
 
     for (target* pt: reverse_iterate (t.prerequisite_targets))
     {
       if (pt == nullptr) // Skipped.
         continue;
 
-      if (execute (a, *pt) == target_state::changed)
-        ts = target_state::changed;
+      target_state ts (execute (a, *pt));
+      if (ts == target_state::changed ||
+          (ts == target_state::postponed && r == target_state::unchanged))
+        r = ts;
     }
 
-    return ts;
+    return r;
   }
 
   bool
diff --git a/build/b.cxx b/build/b.cxx
index 7280ecb..2500196 100644
--- a/build/b.cxx
+++ b/build/b.cxx
@@ -216,7 +216,11 @@ main (int argc, char* argv[])
         operation_id oid (0); // Not yet translated.
         const operation_info* oif (nullptr);
 
-        action act (0, 0); // Not yet initialized.
+        operation_id pre_oid (0);
+        const operation_info* pre_oif (nullptr);
+
+        operation_id post_oid (0);
+        const operation_info* post_oif (nullptr);
 
         // We do meta-operation and operation batches sequentially (no
         // parallelism). But multiple targets in an operation batch
@@ -590,6 +594,8 @@ main (int argc, char* argv[])
 
               if (mif->meta_operation_pre != nullptr)
                 mif->meta_operation_pre ();
+
+              current_mif = mif;
             }
             //
             // Otherwise, check that all the targets in a meta-operation
@@ -630,11 +636,19 @@ main (int argc, char* argv[])
                                   << ", id " << static_cast<uint16_t> (oid);});
               }
 
-              act = action (mid, oid);
+              // Handle pre/post operations.
+              //
+              if (oif->pre != nullptr && (pre_oid = oif->pre (mid)) != 0)
+              {
+                assert (pre_oid != default_id);
+                pre_oif = &rs.operations[pre_oid].get ();
+              }
 
-              current_mif = mif;
-              current_oif = oif;
-              current_mode = oif->mode;
+              if (oif->post != nullptr && (post_oid = oif->post (mid)) != 0)
+              {
+                assert (post_oid != default_id);
+                post_oif = &rs.operations[post_oid].get ();
+              }
             }
             //
             // Similar to meta-operations, check that all the targets in
@@ -646,6 +660,22 @@ main (int argc, char* argv[])
                   oif != &rs.operations[oid].get ()) // Not the same impl.
                 fail (l) << "different operation implementations "
                          << "in an operation batch";
+
+              if (pre_oid != 0)
+              {
+                if (pre_oid > rs.operations.size () ||
+                    pre_oif != &rs.operations[pre_oid].get ())
+                  fail (l) << "different pre-operation implementations "
+                           << "in an operation batch";
+              }
+
+              if (post_oid != 0)
+              {
+                if (post_oid > rs.operations.size () ||
+                    post_oif != &rs.operations[post_oid].get ())
+                  fail (l) << "different post-operation implementations "
+                           << "in an operation batch";
+              }
             }
           }
 
@@ -672,7 +702,7 @@ main (int argc, char* argv[])
           //
           mif->load (bf, rs, out_base, src_base, l);
 
-          // Next resolve and match the target. We don't want to start
+          // Next search and match the targets. We don't want to start
           // building before we know how to for all the targets in this
           // operation batch.
           //
@@ -695,13 +725,63 @@ main (int argc, char* argv[])
 
             d.normalize ();
 
-            mif->match (act, rs, target_key {ti, &d, &tn.value, &e}, l, tgs);
+            mif->search (rs, target_key {ti, &d, &tn.value, &e}, l, tgs);
           }
         }
 
-        // Now execute the action on the list of targets.
-        //
-        mif->execute (act, tgs);
+        if (pre_oid != 0)
+        {
+          level4 ([&]{trace << "start pre-operation batch " << pre_oif->name
+                            << ", id " << static_cast<uint16_t> (pre_oid);});
+
+          if (mif->operation_pre != nullptr)
+            mif->operation_pre (pre_oid); // Cannot be translated.
+
+          current_oif = pre_oif;
+          current_mode = pre_oif->mode;
+
+          action a (mid, pre_oid, oid);
+
+          mif->match (a, tgs);
+          mif->execute (a, tgs);
+
+          if (mif->operation_post != nullptr)
+            mif->operation_post (pre_oid);
+
+          level4 ([&]{trace << "end pre-operation batch " << pre_oif->name
+                            << ", id " << static_cast<uint16_t> (pre_oid);});
+        }
+
+        current_oif = oif;
+        current_mode = oif->mode;
+
+        action a (mid, oid, 0);
+
+        mif->match (a, tgs);
+        mif->execute (a, tgs);
+
+        if (post_oid != 0)
+        {
+          level4 ([&]{trace << "start post-operation batch " << post_oif->name
+                            << ", id " << static_cast<uint16_t> (post_oid);});
+
+          if (mif->operation_pre != nullptr)
+            mif->operation_pre (post_oid); // Cannot be translated.
+
+          current_oif = post_oif;
+          current_mode = post_oif->mode;
+
+          action a (mid, post_oid, oid);
+
+          mif->match (a, tgs);
+          mif->execute (a, tgs);
+
+          if (mif->operation_post != nullptr)
+            mif->operation_post (post_oid);
+
+          level4 ([&]{trace << "end post-operation batch " << post_oif->name
+                            << ", id " << static_cast<uint16_t> (post_oid);});
+        }
 
         if (mif->operation_post != nullptr)
           mif->operation_post (oid);
diff --git a/build/bin/rule.cxx b/build/bin/rule.cxx
index 0fc0e40..0eb8363 100644
--- a/build/bin/rule.cxx
+++ b/build/bin/rule.cxx
@@ -114,15 +114,25 @@ namespace build
       if (current_mode == execution_mode::last)
         swap (m1, m2);
 
-      target_state ts (target_state::unchanged);
+      target_state r (target_state::unchanged), ts;
 
-      if (m1 != nullptr && execute (a, *m1) == target_state::changed)
-        ts = target_state::changed;
+      if (m1 != nullptr)
+      {
+        ts = execute (a, *m1);
+        if (ts == target_state::changed ||
+            (ts == target_state::postponed && r == target_state::unchanged))
+          r = ts;
+      }
 
-      if (m2 != nullptr && execute (a, *m2) == target_state::changed)
-        ts = target_state::changed;
+      if (m2 != nullptr)
+      {
+        ts = execute (a, *m2);
+        if (ts == target_state::changed ||
+            (ts == target_state::postponed && r == target_state::unchanged))
+          r = ts;
+      }
 
-      return ts;
+      return r;
     }
   }
 }
diff --git a/build/config/operation.cxx b/build/config/operation.cxx
index 48d4d19..a79ddbf 100644
--- a/build/config/operation.cxx
+++ b/build/config/operation.cxx
@@ -215,6 +215,7 @@ namespace build
       nullptr, // meta-operation pre
       &configure_operation_pre,
       &load,   // normal load
+      &search, // normal search
       &match,  // normal match
       &configure_execute,
       nullptr, // operation post
@@ -244,17 +245,19 @@ namespace build
     }
 
     static void
-    disfigure_match (action,
-                     scope& root,
-                     const target_key&,
-                     const location&,
-                     action_targets& ts)
+    disfigure_search (scope& root,
+                      const target_key&,
+                      const location&,
+                      action_targets& ts)
     {
-      tracer trace ("disfigure_match");
+      tracer trace ("disfigure_search");
       level5 ([&]{trace << "collecting " << root.path ();});
       ts.push_back (&root);
     }
 
+    static void
+    disfigure_match (action, action_targets&) {}
+
     static bool
     disfigure_project (action a, scope& root)
     {
@@ -403,6 +406,7 @@ namespace build
       nullptr, // meta-operation pre
       &disfigure_operation_pre,
       &disfigure_load,
+      &disfigure_search,
       &disfigure_match,
       &disfigure_execute,
       nullptr, // operation post
diff --git a/build/operation b/build/operation
index be483a9..75d6526 100644
--- a/build/operation
+++ b/build/operation
@@ -33,24 +33,61 @@ namespace build
   using operation_id = std::uint8_t;
   using action_id = std::uint8_t;
 
+  // Meta-operations and operations are not the end of the story. We
+  // also have operation nesting (currently only one level deep) which
+  // is used to implement pre/post operations (currently, but may be
+  // useful for other things). Here is the idea: the test operation
+  // needs to make sure that the targets that it needs to test are
+  // up-to-date. So it runs update as its pre-operation. It is almost
+  // like an ordinary update except that it has test as its outer
+  // operation (the meta-operations are always the same). This way a
+  // rule can recognize that this is "update for test" and do something
+  // differently. For example, if an executable is not a test, then
+  // there is no use updating it. At the same time, most rules will
+  // ignore the fact that this is a nested update and for them it is
+  // "update as usual".
+  //
   struct action
   {
-    action (meta_operation_id m, operation_id o): id ((m << 4) | o) {}
+    action (): inner_id (0), outer_id (0) {} // Invalid action.
+
+    bool
+    valid () const {return inner_id != 0;}
+
+    // If this is not a nested operation, then outer should be 0.
+    //
+    action (meta_operation_id m, operation_id inner, operation_id outer)
+        : inner_id ((m << 4) | inner),
+          outer_id (outer == 0 ? 0 : (m << 4) | outer) {}
 
     meta_operation_id
-    meta_operation () const {return id >> 4;}
+    meta_operation () const {return inner_id >> 4;}
 
     operation_id
-    operation () const {return id & 0xF;}
+    operation () const {return inner_id & 0xF;}
+
+    operation_id
+    outer_operation () const {return outer_id & 0xF;}
 
     // Implicit conversion operator to action_id for the switch()
-    // statement, etc.
+    // statement, etc. Most places will only care about the inner
+    // operation.
     //
-    operator action_id () const {return id;}
+    operator action_id () const {return inner_id;}
 
-    action_id id;
+    action_id inner_id;
+    action_id outer_id;
   };
 
+  inline bool
+  operator== (action x, action y)
+  {
+    return x.inner_id == y.inner_id && x.outer_id == y.outer_id;
+  }
+
+  inline bool
+  operator!= (action x, action y) {return !(x == y);}
+
   std::ostream&
   operator<< (std::ostream&, action);
 
@@ -130,10 +167,15 @@ namespace build
     const std::string name_doing;        // E.g., [while] 'configuring'.
     const std::string name_already_done; // E.g., [already] 'configured'.
 
+    // If operation_pre() is not NULL, then it may translate default_id
+    // (and only default_id) to some other operation. If not translated,
+    // then default_id is used. If, however, operation_pre() is NULL,
+    // then default_id is translated to update_id.
+    //
     void (*meta_operation_pre) (); // Start of meta-operation batch.
     operation_id (*operation_pre) (operation_id); // Start of operation batch.
 
-    // Meta-operation-specific logic to load the buildfile, resolve and match
+    // Meta-operation-specific logic to load the buildfile, search and match
     // the targets, and execute the action on the targets.
     //
     void (*load) (const path& buildfile,
@@ -142,11 +184,12 @@ namespace build
                   const dir_path& src_base,
                   const location&);
 
-    void (*match) (action,
-                   scope& root,
-                   const target_key&,
-                   const location&,
-                   action_targets&);
+    void (*search) (scope& root,
+                    const target_key&,
+                    const location&,
+                    action_targets&);
+
+    void (*match) (action, action_targets&);
 
     void (*execute) (action, const action_targets&);
 
@@ -172,11 +215,14 @@ namespace build
         const dir_path& src_base,
         const location&);
 
-  // Resolve and match the target. This is the default implementation
+  // Search and match the target. This is the default implementation
   // that does just that and adds a pointer to the target to the list.
   //
   void
-  match (action, scope&, const target_key&, const location&, action_targets&);
+  search (scope&, const target_key&, const location&, action_targets&);
+
+  void
+  match (action, action_targets&);
 
   // Execute the action on the list of targets. This is the default
   // implementation that does just that while issuing appropriate
@@ -204,6 +250,13 @@ namespace build
     const std::string name_already_done; // E.g., [already] 'up to date'.
 
     const execution_mode mode;
+
+    // If the returned operation_id's are not 0, then they are injected
+    // as pre/post operations for this operation. Can be NULL if unused.
+    // The returned operation_id shall not be default_id.
+    //
+    operation_id (*pre) (meta_operation_id);
+    operation_id (*post) (meta_operation_id);
   };
 
   // Build-in operations.
diff --git a/build/operation.cxx b/build/operation.cxx
index 4c32d42..6b3b08d 100644
--- a/build/operation.cxx
+++ b/build/operation.cxx
@@ -8,6 +8,8 @@
 #include <cassert>
 #include <functional> // reference_wrapper
 
+#include <butl/utility> // reverse_iterate
+
 #include <build/scope>
 #include <build/target>
 #include <build/file>
@@ -16,6 +18,7 @@
 #include <build/dump>
 
 using namespace std;
+using namespace butl;
 
 namespace build
 {
@@ -24,10 +27,24 @@ namespace build
   ostream&
   operator<< (ostream& os, action a)
   {
-    return os << '('
-              << static_cast<uint16_t> (a.meta_operation ()) << ','
-              << static_cast<uint16_t> (a.operation ())
-              << ')';
+    uint16_t
+      m (a.meta_operation ()),
+      i (a.operation ()),
+      o (a.outer_operation ());
+
+    os << '(' << m << ',';
+
+    if (o != 0)
+      os << o << '(';
+
+    os << i;
+
+    if (o != 0)
+      os << ')';
+
+    os << ')';
+
+    return os;
   }
 
   // perform
@@ -59,30 +76,37 @@ namespace build
   }
 
   void
-  match (action a,
-         scope&,
-         const target_key& tk,
-         const location& l,
-         action_targets& ts)
+  search (scope&,
+          const target_key& tk,
+          const location& l,
+          action_targets& ts)
   {
-    tracer trace ("match");
+    tracer trace ("search");
 
     auto i (targets.find (tk, trace));
     if (i == targets.end ())
       fail (l) << "unknown target " << tk;
 
-    target& t (**i);
+    ts.push_back (i->get ());
+  }
+
+  void
+  match (action a, action_targets& ts)
+  {
+    tracer trace ("match");
 
     if (verb >= 5)
       dump (a);
 
-    level4 ([&]{trace << "matching " << t;});
-    match (a, t);
+    for (void* vt: ts)
+    {
+      target& t (*static_cast<target*> (vt));
+      level4 ([&]{trace << "matching " << t;});
+      match (a, t);
+    }
 
     if (verb >= 5)
       dump (a);
-
-    ts.push_back (&t);
   }
 
   void
@@ -94,38 +118,50 @@ namespace build
     //
     vector<reference_wrapper<target>> psp;
 
-    for (void* v: ts)
-    {
-      target& t (*static_cast<target*> (v));
+    // Execute targets in reverse if the execution mode is 'last'.
+    //
+    auto body (
+      [a, &psp, &trace] (void* v)
+      {
+        target& t (*static_cast<target*> (v));
 
-      level4 ([&]{trace << diag_doing (a, t);});
+        level4 ([&]{trace << diag_doing (a, t);});
 
-      switch (execute (a, t))
-      {
-      case target_state::postponed:
+        switch (execute (a, t))
         {
-          info << diag_doing (a, t) << " is postponed";
-          psp.push_back (t);
+        case target_state::postponed:
+          {
+            info << diag_doing (a, t) << " is postponed";
+            psp.push_back (t);
+            break;
+          }
+        case target_state::unchanged:
+          {
+            info << diag_already_done (a, t);
+            break;
+          }
+        case target_state::changed:
           break;
+        case target_state::failed:
+          //@@ This could probably happen in a parallel build.
+        default:
+          assert (false);
         }
-      case target_state::unchanged:
-        {
-          info << diag_already_done (a, t);
-          break;
-        }
-      case target_state::changed:
-        break;
-      case target_state::failed:
-        //@@ This could probably happen in a parallel build.
-      default:
-        assert (false);
-      }
-    }
+      });
+
+    if (current_mode == execution_mode::first)
+      for (void* v: ts) body (v);
+    else
+      for (void* v: reverse_iterate (ts)) body (v);
 
-    // Re-examine postponed targets.
+    // Re-examine postponed targets. Note that we will do it in the
+    // order added, so no need to check the execution mode.
     //
     for (target& t: psp)
     {
+      if (t.state () == target_state::postponed)
+        execute_direct (a, t); // Try again, now ignoring the execution mode.
+
       switch (t.state ())
       {
       case target_state::postponed:
@@ -157,6 +193,7 @@ namespace build
     nullptr, // meta-operation pre
     nullptr, // operation pre
     &load,
+    &search,
     &match,
     &execute,
     nullptr, // operation post
@@ -170,7 +207,9 @@ namespace build
     "",
     "",
     "",
-    execution_mode::first
+    execution_mode::first,
+    nullptr,
+    nullptr
   };
 
   operation_info update {
@@ -178,7 +217,9 @@ namespace build
     "update",
     "updating",
     "up to date",
-    execution_mode::first
+    execution_mode::first,
+    nullptr,
+    nullptr
   };
 
   operation_info clean {
@@ -186,5 +227,8 @@ namespace build
     "clean",
     "cleaning",
     "clean",
-    execution_mode::last};
+    execution_mode::last,
+    nullptr,
+    nullptr
+  };
 }
diff --git a/build/target b/build/target
index ee61c7b..d164f72 100644
--- a/build/target
+++ b/build/target
@@ -306,10 +306,10 @@ namespace build
     typedef build::recipe recipe_type;
 
     const recipe_type&
-    recipe (action_id a) const {return action_ == a ? recipe_ : empty_recipe;}
+    recipe (action a) const {return action_ == a ? recipe_ : empty_recipe;}
 
     void
-    recipe (action_id a, recipe_type r)
+    recipe (action a, recipe_type r)
     {
       assert (action_ != a || !recipe_);
       action_ = a;
@@ -348,7 +348,7 @@ namespace build
     static const target_type static_type;
 
   private:
-    action_id action_ {0}; // Action id of this recipe.
+    action action_; // Action this recipe is for.
     recipe_type recipe_;
   };