Add support for update=unmatch|match to ad hoc recipes

10 files changed, 217 insertions, 48 deletions

diff --git a/build2/cli/rule.cxx b/build2/cli/rule.cxx
index 99b6bee..5c03d9c 100644
--- a/build2/cli/rule.cxx
+++ b/build2/cli/rule.cxx
@@ -122,7 +122,9 @@ namespace build2
             if (g == nullptr)
               g = &t.ctx.targets.insert<cli_cxx> (t.dir, t.out, t.name, trace);
 
-            g->prerequisites (prerequisites {p->as_prerequisite ()});
+            prerequisites ps;
+            ps.push_back (p->as_prerequisite ());
+            g->prerequisites (move (ps));
           }
         }
 
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx
index e9bd2c6..c4f8833 100644
--- a/libbuild2/adhoc-rule-buildscript.cxx
+++ b/libbuild2/adhoc-rule-buildscript.cxx
@@ -301,10 +301,13 @@ namespace build2
       return execute_inner;
     }
 
+    context& ctx (xt.ctx);
+    const scope& bs (xt.base_scope ());
+
     // Inject pattern's ad hoc group members, if any.
     //
     if (pattern != nullptr)
-      pattern->apply_adhoc_members (a, xt, me);
+      pattern->apply_adhoc_members (a, xt, bs, me);
 
     // Derive file names for the target and its ad hoc group members, if any.
     //
@@ -326,18 +329,145 @@ namespace build2
 
     // Match prerequisites.
     //
-    match_prerequisite_members (a, xt);
-
-    // Inject pattern's prerequisites, if any.
+    // This is essentially match_prerequisite_members() but with support
+    // for update=unmatch|match.
     //
-    if (pattern != nullptr)
-      pattern->apply_prerequisites (a, xt, me);
+    auto& pts (xt.prerequisite_targets[a]);
+    {
+      // Re-create the clean semantics as in match_prerequisite_members().
+      //
+      bool clean (a.operation () == clean_id && !xt.is_a<alias> ());
+
+      // Add target's prerequisites.
+      //
+      for (prerequisite_member p: group_prerequisite_members (a, xt))
+      {
+        // Note that we have to recognize update=unmatch|match for *(update),
+        // not just perform(update). But only actually do anything about it
+        // for perform(update).
+        //
+        lookup l; // The `update` variable value, if any.
+        include_type pi (
+          include (a, xt, p, a.operation () == update_id ? &l : nullptr));
+
+        // Use bit 2 of prerequisite_target::include to signal update during
+        // match and bit 3 -- unmatch.
+        //
+        uintptr_t mask (0);
+        if (l)
+        {
+          const string& v (cast<string> (l));
+
+          if (v == "match")
+          {
+            if (a == perform_update_id)
+              mask = 2;
+          }
+          else if (v == "unmatch")
+          {
+            if (a == perform_update_id)
+              mask = 4;
+          }
+          else if (v != "false" && v != "true")
+          {
+            fail << "unrecognized update variable value '" << v
+                 << "' specified for prerequisite " << p.prerequisite;
+          }
+        }
+
+        // Skip excluded.
+        //
+        if (!pi)
+          continue;
+
+        const target& pt (p.search (xt));
+
+        if (clean && !pt.in (*bs.root_scope ()))
+          continue;
+
+        prerequisite_target pto (&pt, pi);
+
+        if (mask != 0)
+          pto.include |= mask;
+
+        pts.push_back (move (pto));
+      }
+
+      // Inject pattern's prerequisites, if any.
+      //
+      if (pattern != nullptr)
+        pattern->apply_prerequisites (a, xt, bs, me);
+
+      // Start asynchronous matching of prerequisites. Wait with unlocked
+      // phase to allow phase switching.
+      //
+      wait_guard wg (ctx, ctx.count_busy (), xt[a].task_count, true);
+
+      for (const prerequisite_target& pt: pts)
+      {
+        if (pt.target == dir)
+          continue;
+
+        match_async (a, *pt.target, ctx.count_busy (), xt[a].task_count);
+      }
+
+      wg.wait ();
+
+      // Finish matching all the targets that we have started.
+      //
+      for (prerequisite_target& pt: pts)
+      {
+        if (pt.target == dir)
+          continue;
+
+        // Handle update=unmatch.
+        //
+        unmatch um ((pt.include & 4) != 0 ? unmatch::safe : unmatch::none);
+
+        pair<bool, target_state> mr (build2::match (a, *pt.target, um));
+
+        if (um != unmatch::none)
+        {
+          l6 ([&]{trace << "unmatch " << *pt.target << ": " << mr.first;});
+
+          // If we managed to unmatch, blank it out so that it's not executed,
+          // etc. Otherwise, convert it to ad hoc (we also automatically avoid
+          // hashing it and updating it during match in exec_depdb_dyndep()).
+          //
+          // The hashing part is tricky: by not hashing it we won't detect the
+          // case where it was removed as a prerequisite altogether. The
+          // thinking is that it was added with update=unmatch to extract some
+          // information (e.g., poptions from a library) and those will be
+          // change-tracked.
+          //
+          if (mr.first)
+            pt.target = nullptr;
+          else
+            pt.include |= 1;
+        }
+      }
+    }
 
     // See if we are providing the standard clean as a fallback.
     //
     if (me.fallback)
       return &perform_clean_file;
 
+    // If we have any update during match prerequisites, now is the time to
+    // update them.
+    //
+    // Note also that we ignore the result and whether it renders us out of
+    // date, leaving it to the common execute logic in perform_update_*().
+    //
+    if (a == perform_update_id)
+    {
+      for (const prerequisite_target& pt: pts)
+      {
+        if ((pt.include & 2) != 0)
+          update_during_match (trace, a, *pt.target);
+      }
+    }
+
     // See if this is not update or not on a file-based target.
     //
     if (a != perform_update_id || !xt.is_a<file> ())
@@ -380,11 +510,8 @@ namespace build2
     // where you can often find more detailed rationale for some of the steps
     // performed (like the fsdir update below).
     //
-    context& ctx (xt.ctx);
-
     file& t (xt.as<file> ());
     const path& tp (t.path ());
-    const scope& bs (t.base_scope ());
 
     if (dir != nullptr)
       fsdir_rule::perform_update_direct (a, t);
@@ -394,7 +521,6 @@ namespace build2
     // them to the auxiliary data member in prerequisite_target (see
     // execute_update_prerequisites() for details).
     //
-    auto& pts (t.prerequisite_targets[a]);
     for (prerequisite_target& p: pts)
     {
       // Note that fsdir{} injected above is adhoc.
@@ -429,6 +555,9 @@ namespace build2
              p.adhoc ()          ? reinterpret_cast<target*> (p.data) :
              nullptr))
         {
+          if ((p.include & 4) != 0) // Skip update=unmatch.
+            continue;
+
           hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage);
         }
       }
@@ -1039,6 +1168,9 @@ namespace build2
              p.adhoc ()          ? reinterpret_cast<target*> (p.data)
              : nullptr))
         {
+          if ((p.include & 4) != 0) // Skip update=unmatch.
+            continue;
+
           hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage);
         }
       }
@@ -1216,7 +1348,8 @@ namespace build2
 
   // Update prerequisite targets.
   //
-  // Each prerequisite target should be in one of the following states:
+  // Each (non-NULL) prerequisite target should be in one of the following
+  // states:
   //
   // target  adhoc  data
   // --------------------
diff --git a/libbuild2/adhoc-rule-regex-pattern.cxx b/libbuild2/adhoc-rule-regex-pattern.cxx
index b0de827..637a16f 100644
--- a/libbuild2/adhoc-rule-regex-pattern.cxx
+++ b/libbuild2/adhoc-rule-regex-pattern.cxx
@@ -158,11 +158,18 @@ namespace build2
     auto find_prereq = [a, &t] (const target_type& tt) -> optional<target_key>
     {
       // We use the standard logic that one would use in the rule::match()
-      // implementation.
+      // implementation. Except we support the unmatch and match values in
+      // the update variable.
       //
       for (prerequisite_member p: group_prerequisite_members (a, t))
       {
-        if (include (a, t, p) == include_type::normal && p.is_a (tt))
+        // Note that here we don't validate the update operation override
+        // value (since we may not match). Instead the rule does this in
+        // apply().
+        //
+        lookup l;
+        if (include (a, t, p, a.operation () == update_id ? &l : nullptr) ==
+              include_type::normal && p.is_a (tt))
           return p.key ().tk;
       }
       return nullopt;
@@ -293,7 +300,7 @@ namespace build2
   }
 
   void adhoc_rule_regex_pattern::
-  apply_adhoc_members (action, target& t, match_extra&) const
+  apply_adhoc_members (action, target& t, const scope&, match_extra&) const
   {
     const auto& mr (t.data<regex_match_results> ());
 
@@ -334,27 +341,17 @@ namespace build2
   }
 
   void adhoc_rule_regex_pattern::
-  apply_prerequisites (action a, target& t, match_extra&) const
+  apply_prerequisites (action a, target& t,
+                       const scope& bs,
+                       match_extra&) const
   {
     const auto& mr (t.data<regex_match_results> ());
 
-    // Resolve and cache target scope lazily.
-    //
-    auto base_scope = [&t, bs = (const scope*) nullptr] () mutable
-      -> const scope&
-    {
-      if (bs == nullptr)
-        bs = &t.base_scope ();
-
-      return *bs;
-    };
-
     // Re-create the same clean semantics as in match_prerequisite_members().
     //
     bool clean (a.operation () == clean_id && !t.is_a<alias> ());
 
     auto& pts (t.prerequisite_targets[a]);
-    size_t start (pts.size ());
 
     for (const element& e: prereqs_)
     {
@@ -382,7 +379,7 @@ namespace build2
         n = name (e.name.dir,
                   e.name.type,
                   substitute (t, mr, e.name.value, "prerequisite"));
-        s = &base_scope ();
+        s = &bs;
       }
       else
       {
@@ -392,18 +389,15 @@ namespace build2
 
       const target& pt (search (t, move (n), *s, &e.type));
 
-      if (clean && !pt.in (*base_scope ().root_scope ()))
+      if (clean && !pt.in (*bs.root_scope ()))
         continue;
 
       // @@ TODO: it could be handy to mark a prerequisite (e.g., a tool)
       //    ad hoc so that it doesn't interfere with the $< list. Also
-      //    clean=false.
+      //    clean=false. Also update=match|unmatch.
       //
       pts.push_back (prerequisite_target (&pt, false /* adhoc */));
     }
-
-    if (start != pts.size ())
-      match_members (a, t, pts, start);
   }
 
   void adhoc_rule_regex_pattern::
diff --git a/libbuild2/adhoc-rule-regex-pattern.hxx b/libbuild2/adhoc-rule-regex-pattern.hxx
index 4327e72..e6fb938 100644
--- a/libbuild2/adhoc-rule-regex-pattern.hxx
+++ b/libbuild2/adhoc-rule-regex-pattern.hxx
@@ -35,10 +35,14 @@ namespace build2
     match (action, target&, const string&, match_extra&) const override;
 
     virtual void
-    apply_adhoc_members (action, target&, match_extra&) const override;
+    apply_adhoc_members (action, target&,
+                         const scope&,
+                         match_extra&) const override;
 
     virtual void
-    apply_prerequisites (action, target&, match_extra&) const override;
+    apply_prerequisites (action, target&,
+                         const scope&,
+                         match_extra&) const override;
 
     virtual void
     dump (ostream&) const override;
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index e9c8d9c..d9bfef6 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -1625,6 +1625,19 @@ namespace build2
                p.adhoc ()          ? reinterpret_cast<target*> (p.data)
                : nullptr))
           {
+            // Automatically skip update=unmatch that we could not unmatch.
+            //
+            // Note that we don't skip update=match here (unless filtered out)
+            // in order to incorporate the result into our out-of-date'ness.
+            // So there is a nuanced interaction between update=match and
+            // --update-*.
+            //
+            if ((p.include & 4) != 0)
+            {
+              l6 ([&]{trace << "skipping unmatched " << *pt;});
+              continue;
+            }
+
             // Apply the --update-* filter.
             //
             if (!p.adhoc () && !filters.empty ())
diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx
index aaaa43c..e7e90ad 100644
--- a/libbuild2/cc/compile-rule.cxx
+++ b/libbuild2/cc/compile-rule.cxx
@@ -1010,7 +1010,7 @@ namespace build2
         // @@ If for some reason unmatch fails, this messes up the for_install
         //    logic because we will update this library during match. Perhaps
         //    we should postpone updating them until execute if we failed to
-        //    unmatch.
+        //    unmatch. See how we do this in ad hoc rule.
         //
         pair<bool, target_state> mr (
           build2::match (
@@ -5889,7 +5889,10 @@ namespace build2
       ps.push_back (prerequisite (lt));
       for (prerequisite_member p: group_prerequisite_members (a, lt))
       {
-        if (include (a, lt, p) != include_type::normal) // Excluded/ad hoc.
+        // Ignore update=match.
+        //
+        lookup l;
+        if (include (a, lt, p, &l) != include_type::normal) // Excluded/ad hoc.
           continue;
 
         if (p.is_a<libx> () ||
@@ -6123,7 +6126,10 @@ namespace build2
         //
         for (prerequisite_member p: group_prerequisite_members (a, t))
         {
-          if (include (a, t, p) != include_type::normal) // Excluded/ad hoc.
+          // Ignore update=match.
+          //
+          lookup l;
+          if (include (a, t, p, &l) != include_type::normal) // Excluded/ad hoc.
             continue;
 
           if (p.is_a<libx> () ||
diff --git a/libbuild2/cc/link-rule.cxx b/libbuild2/cc/link-rule.cxx
index c993df6..d90cd75 100644
--- a/libbuild2/cc/link-rule.cxx
+++ b/libbuild2/cc/link-rule.cxx
@@ -286,7 +286,8 @@ namespace build2
         // value (since we may not match). Instead we do this in apply().
         //
         lookup l;
-        if (include (a, t, p, &l) != include_type::normal)
+        if (include (a, t, p, a.operation () == update_id ? &l : nullptr) !=
+              include_type::normal)
           continue;
 
         if (p.is_a (x_src)                        ||
@@ -1199,7 +1200,7 @@ namespace build2
       //
       if (update_match)
       {
-        for (prerequisite_target& pto: pts)
+        for (const prerequisite_target& pto: pts)
         {
           if ((pto.include & 2) != 0)
             update_during_match (trace, a, *pto.target);
@@ -1559,7 +1560,8 @@ namespace build2
           if (!pt->has_prerequisites () &&
               (!group || !rt.has_prerequisites ()))
           {
-            prerequisites ps {p.as_prerequisite ()}; // Source.
+            prerequisites ps;
+            ps.push_back (p.as_prerequisite ()); // Source.
 
             // Add our lib*{} (see the export.* machinery for details) and
             // bmi*{} (both original and chained; see module search logic)
@@ -1578,7 +1580,7 @@ namespace build2
             // might depend on the imported one(s) which we will never "see"
             // unless we start with this library.
             //
-            // Note: have similar logic in make_module_sidebuild().
+            // Note: have similar logic in make_{module,header}_sidebuild().
             //
             size_t j (start);
             for (prerequisite_member p: group_prerequisite_members (a, t))
diff --git a/libbuild2/rule.hxx b/libbuild2/rule.hxx
index 364e3ff..73492fe 100644
--- a/libbuild2/rule.hxx
+++ b/libbuild2/rule.hxx
@@ -275,10 +275,19 @@ namespace build2
     match (action, target&, const string&, match_extra&) const = 0;
 
     virtual void
-    apply_adhoc_members (action, target&, match_extra&) const = 0;
-
+    apply_adhoc_members (action, target&,
+                         const scope& base,
+                         match_extra&) const = 0;
+
+    // The implementation should append pattern prerequisites to
+    // t.prerequisite_targets[a] but not match. It should set bit 2 in
+    // prerequisite_target::include to indicate update=match and bit 3
+    // to indicate update=unmatch.
+    //
     virtual void
-    apply_prerequisites (action, target&, match_extra&) const = 0;
+    apply_prerequisites (action, target&,
+                         const scope& base,
+                         match_extra&) const = 0;
 
     // Dump support.
     //
diff --git a/libbuild2/target.hxx b/libbuild2/target.hxx
index efc3291..6b79e1b 100644
--- a/libbuild2/target.hxx
+++ b/libbuild2/target.hxx
@@ -1089,7 +1089,8 @@ namespace build2
       return member != nullptr ? member : prerequisite.target.load (mo);
     }
 
-    // Return as a new prerequisite instance.
+    // Return as a new prerequisite instance. Note that it includes a copy
+    // of prerequisite-specific variables.
     //
     prerequisite_type
     as_prerequisite () const;
diff --git a/libbuild2/target.ixx b/libbuild2/target.ixx
index a5ad32b..8f0768e 100644
--- a/libbuild2/target.ixx
+++ b/libbuild2/target.ixx
@@ -443,7 +443,12 @@ namespace build2
     //
     assert (!member->adhoc_group_member ());
 
-    return prerequisite_type (*member);
+    // Feels like copying the prerequisite's variables to member is more
+    // correct than not (consider for_install, for example).
+    //
+    prerequisite_type p (*member);
+    p.vars = prerequisite.vars;
+    return p;
   }
 
   inline prerequisite_key prerequisite_member::