Rename rule-adhoc-* to adhoc-rule-*

1 files changed, 603 insertions, 0 deletions

diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx
new file mode 100644
index 0000000..c68f4b2
--- /dev/null
+++ b/libbuild2/adhoc-rule-buildscript.cxx
@@ -0,0 +1,603 @@
+// file      : libbuild2/adhoc-rule-buildscript.cxx -*- C++ -*-
+// license   : MIT; see accompanying LICENSE file
+
+#include <libbuild2/adhoc-rule-buildscript.hxx>
+
+#include <sstream>
+
+#include <libbuild2/depdb.hxx>
+#include <libbuild2/scope.hxx>
+#include <libbuild2/target.hxx>
+#include <libbuild2/context.hxx>
+#include <libbuild2/algorithm.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+#include <libbuild2/parser.hxx> // attributes
+
+#include <libbuild2/build/script/parser.hxx>
+#include <libbuild2/build/script/runner.hxx>
+
+using namespace std;
+
+namespace build2
+{
+  bool adhoc_buildscript_rule::
+  recipe_text (context& ctx, const target& tg, string&& t, attributes& as)
+  {
+    // Handle and erase recipe-specific attributes.
+    //
+    optional<string> diag;
+    for (auto i (as.begin ()); i != as.end (); )
+    {
+      attribute& a (*i);
+      const string& n (a.name);
+
+      if (n == "diag")
+      try
+      {
+        diag = convert<string> (move (a.value));
+      }
+      catch (const invalid_argument& e)
+      {
+        fail (as.loc) << "invalid " << n << " attribute value: " << e;
+      }
+      else
+      {
+        ++i;
+        continue;
+      }
+
+      i = as.erase (i);
+    }
+
+    checksum = sha256 (t).string ();
+
+    istringstream is (move (t));
+    build::script::parser p (ctx);
+
+    script = p.pre_parse (tg,
+                          is, loc.file, loc.line + 1,
+                          move (diag), as.loc);
+
+    return false;
+  }
+
+  void adhoc_buildscript_rule::
+  dump_attributes (ostream& os) const
+  {
+    // For now we dump it as an attribute whether it was specified or derived
+    // from the script. Maybe that's ok (we use this in tests)?
+    //
+    if (script.diag_name)
+    {
+      os << " [";
+      os << "diag=";
+      to_stream (os, name (*script.diag_name), true /* quote */, '@');
+      os << ']';
+    }
+  }
+
+  void adhoc_buildscript_rule::
+  dump_text (ostream& os, string& ind) const
+  {
+    os << ind << string (braces, '{') << endl;
+    ind += "  ";
+
+    if (script.depdb_clear)
+      os << ind << "depdb clear" << endl;
+
+    script::dump (os, ind, script.depdb_lines);
+
+    if (script.diag_line)
+    {
+      os << ind; script::dump (os, *script.diag_line, true /* newline */);
+    }
+
+    script::dump (os, ind, script.lines);
+    ind.resize (ind.size () - 2);
+    os << ind << string (braces, '}');
+  }
+
+  bool adhoc_buildscript_rule::
+  match (action a, target& t, const string&, optional<action> fb) const
+  {
+    if (!fb)
+      ;
+    // If this is clean for a file target and we are supplying the update,
+    // then we will also supply the standard clean.
+    //
+    else if (a   == perform_clean_id  &&
+             *fb == perform_update_id &&
+             t.is_a<file> ())
+      ;
+    else
+      return false;
+
+    // It's unfortunate we have to resort to this but we need to remember this
+    // in apply().
+    //
+    t.data (fb.has_value ());
+
+    return true;
+  }
+
+  recipe adhoc_buildscript_rule::
+  apply (action a, target& t) const
+  {
+    // If this is an outer operation (e.g., update-for-test), then delegate to
+    // the inner.
+    //
+    if (a.outer ())
+    {
+      match_inner (a, t);
+      return execute_inner;
+    }
+
+    // Derive file names for the target and its ad hoc group members, if any.
+    //
+    if (a == perform_update_id || a == perform_clean_id)
+    {
+      for (target* m (&t); m != nullptr; m = m->adhoc_member)
+      {
+        if (auto* p = m->is_a<path_target> ())
+          p->derive_path ();
+      }
+    }
+
+    // Inject dependency on the output directory.
+    //
+    // We do it always instead of only if one of the targets is path-based in
+    // case the recipe creates temporary files or some such.
+    //
+    inject_fsdir (a, t);
+
+    // Match prerequisites.
+    //
+    match_prerequisite_members (a, t);
+
+    // See if we are providing the standard clean as a fallback.
+    //
+    if (t.data<bool> ())
+      return &perform_clean_depdb;
+
+    if (a == perform_update_id && t.is_a<file> ())
+    {
+      return [this] (action a, const target& t)
+      {
+        return perform_update_file (a, t);
+      };
+    }
+    else
+    {
+      return [this] (action a, const target& t)
+      {
+        return default_action (a, t);
+      };
+    }
+  }
+
+  target_state adhoc_buildscript_rule::
+  perform_update_file (action a, const target& xt) const
+  {
+    tracer trace ("adhoc_buildscript_rule::perform_update_file");
+
+    context& ctx (xt.ctx);
+
+    const file& t (xt.as<file> ());
+    const path& tp (t.path ());
+
+    // How should we hash target and prerequisite sets ($> and $<)? We could
+    // hash them as target names (i.e., the same as the $>/< content) or as
+    // paths (only for path-based targets). While names feel more general,
+    // they are also more expensive to compute. And for path-based targets,
+    // path is generally a good proxy for the target name. Since the bulk of
+    // the ad hoc recipes will presumably be operating exclusively on
+    // path-based targets, let's do it both ways.
+    //
+    auto hash_target = [ns = names ()] (sha256& cs, const target& t) mutable
+    {
+      if (const path_target* pt = t.is_a<path_target> ())
+        cs.append (pt->path ().string ());
+      else
+      {
+        ns.clear ();
+        t.as_name (ns);
+        for (const name& n: ns)
+          to_checksum (cs, n);
+      }
+    };
+
+    // Update prerequisites and determine if any of them render this target
+    // out-of-date.
+    //
+    timestamp mt (t.load_mtime ());
+    optional<target_state> ps;
+
+    sha256 pcs, ecs;
+    {
+      // This is essentially ps=execute_prerequisites(a, t, mt) which we
+      // cannot use because we need to see ad hoc prerequisites.
+      //
+      size_t busy (ctx.count_busy ());
+      size_t exec (ctx.count_executed ());
+
+      target_state rs (target_state::unchanged);
+
+      wait_guard wg (ctx, busy, t[a].task_count);
+
+      for (const target*& pt: t.prerequisite_targets[a])
+      {
+        if (pt == nullptr) // Skipped.
+          continue;
+
+        target_state s (execute_async (a, *pt, busy, t[a].task_count));
+
+        if (s == target_state::postponed)
+        {
+          rs |= s;
+          pt = nullptr;
+        }
+      }
+
+      wg.wait ();
+
+      bool e (mt == timestamp_nonexistent);
+      for (prerequisite_target& p: t.prerequisite_targets[a])
+      {
+        if (p == nullptr)
+          continue;
+
+        const target& pt (*p.target);
+
+        const auto& tc (pt[a].task_count);
+        if (tc.load (memory_order_acquire) >= busy)
+          ctx.sched.wait (exec, tc, scheduler::work_none);
+
+        target_state s (pt.executed_state (a));
+        rs |= s;
+
+        // Compare our timestamp to this prerequisite's.
+        //
+        if (!e)
+        {
+          // If this is an mtime-based target, then compare timestamps.
+          //
+          if (const mtime_target* mpt = pt.is_a<mtime_target> ())
+          {
+            if (mpt->newer (mt, s))
+              e = true;
+          }
+          else
+          {
+            // Otherwise we assume the prerequisite is newer if it was
+            // changed.
+            //
+            if (s == target_state::changed)
+              e = true;
+          }
+        }
+
+        if (p.adhoc)
+          p.target = nullptr; // Blank out.
+
+        // As part of this loop calculate checksums that need to include ad
+        // hoc prerequisites (unless the script tracks changes itself).
+        //
+        if (script.depdb_clear)
+          continue;
+
+        hash_target (pcs, pt);
+
+        // The script can reference a program in one of four ways:
+        //
+        // 1. As an (imported) target (e.g., $cli)
+        //
+        // 2. As a process_path_ex (e.g., $cxx.path).
+        //
+        // 3. As a builtin (e.g., sed)
+        //
+        // 4. As a program path/name.
+        //
+        // When it comes to change tracking, there is nothing we can do for
+        // (4) and there is nothing to do for (3) (assuming builtin semantics
+        // is stable/backwards-compatible). The (2) case is handled
+        // automatically by hashing all the variable values referenced by the
+        // script (see below), which in case of process_path_ex includes the
+        // checksum, if available.
+        //
+        // This leaves the (1) case, which itself splits into two sub-cases:
+        // the target comes with the dependency information (e.g., imported
+        // from a project via an export stub) or it does not (e.g., imported
+        // as installed). We don't need to do anything extra for the first
+        // sub-case since the target's state/mtime can be relied upon like any
+        // other prerequisite. Which cannot be said about the second sub-case,
+        // where we reply on checksum that may be included as part of the
+        // target metadata.
+        //
+        // So what we are going to do is hash checksum metadata of every
+        // executable prerequisite target that has it (we do it here in order
+        // to include ad hoc prerequisites, which feels like the right thing
+        // to do; the user may mark tools as ad hoc in order to omit them from
+        // $<).
+        //
+        if (auto* e = pt.is_a<exe> ())
+        {
+          if (auto* c = e->lookup_metadata<string> ("checksum"))
+          {
+            ecs.append (*c);
+          }
+        }
+      }
+
+      if (!e)
+        ps = rs;
+    }
+
+    bool update (!ps);
+
+    // We use depdb to track changes to the script itself, input/output file
+    // names, tools, etc.
+    //
+    depdb dd (tp + ".d");
+
+    // First should come the rule name/version.
+    //
+    if (dd.expect ("<ad hoc buildscript recipe> 1") != nullptr)
+      l4 ([&]{trace << "rule mismatch forcing update of " << t;});
+
+    // Then the script checksum.
+    //
+    // Ideally, to detect changes to the script semantics, we would hash the
+    // text with all the variables expanded but without executing any
+    // commands. In practice, this is easier said than done (think the set
+    // builtin that receives output of a command that modifies the
+    // filesystem).
+    //
+    // So as the next best thing we are going to hash the unexpanded text as
+    // well as values of all the variables expanded in it (which we get as a
+    // side effect of pre-parsing the script). This approach has a number of
+    // drawbacks:
+    //
+    // - We can't handle computed variable names (e.g., $($x ? X : Y)).
+    //
+    // - We may "overhash" by including variables that are actually
+    //   script-local.
+    //
+    // - There are functions like $install.resolve() with result based on
+    //   external (to the script) information.
+    //
+    if (dd.expect (checksum) != nullptr)
+      l4 ([&]{trace << "recipe text change forcing update of " << t;});
+
+    // Track the variables, targets, and prerequisites changes, unless the
+    // script doesn't track the dependency changes itself.
+    //
+
+    // For each variable hash its name, undefined/null/non-null indicator,
+    // and the value if non-null.
+    //
+    // Note that this excludes the special $< and $> variables which we
+    // handle below.
+    //
+    if (!script.depdb_clear)
+    {
+      sha256 cs;
+      names storage;
+
+      for (const string& n: script.vars)
+      {
+        cs.append (n);
+
+        lookup l;
+
+        if (const variable* var = ctx.var_pool.find (n))
+          l = t[var];
+
+        cs.append (!l.defined () ? '\x1' : l->null ? '\x2' : '\x3');
+
+        if (l)
+        {
+          storage.clear ();
+          names_view ns (reverse (*l, storage));
+
+          for (const name& n: ns)
+            to_checksum (cs, n);
+        }
+      }
+
+      if (dd.expect (cs.string ()) != nullptr)
+        l4 ([&]{trace << "recipe variable change forcing update of " << t;});
+    }
+
+    // Target and prerequisite sets ($> and $<).
+    //
+    if (!script.depdb_clear)
+    {
+      sha256 tcs;
+      for (const target* m (&t); m != nullptr; m = m->adhoc_member)
+        hash_target (tcs, *m);
+
+      if (dd.expect (tcs.string ()) != nullptr)
+        l4 ([&]{trace << "target set change forcing update of " << t;});
+
+      if (dd.expect (pcs.string ()) != nullptr)
+        l4 ([&]{trace << "prerequisite set change forcing update of " << t;});
+    }
+
+    // Finally the programs checksum.
+    //
+    if (!script.depdb_clear)
+    {
+      if (dd.expect (ecs.string ()) != nullptr)
+        l4 ([&]{trace << "program checksum change forcing update of " << t;});
+    }
+
+    const scope* bs (nullptr);
+    const scope* rs (nullptr);
+
+    // Execute the custom dependency change tracking commands, if present.
+    //
+    if (!script.depdb_lines.empty ())
+    {
+      bs = &t.base_scope ();
+      rs = bs->root_scope ();
+
+      // While it would have been nice to reuse the environment for both
+      // dependency tracking and execution, there are complications (creating
+      // temporary directory, etc).
+      //
+      build::script::environment e (a, t, false /* temp_dir */);
+      build::script::parser p (ctx);
+
+      for (const script::line& l: script.depdb_lines)
+      {
+        names ns (p.execute_special (*rs, *bs, e, l));
+
+        // These should have been enforced during pre-parsing.
+        //
+        assert (!ns.empty ());         //         <cmd> ... <newline>
+        assert (l.tokens.size () > 2); // 'depdb' <cmd> ... <newline>
+
+        const string& cmd (ns[0].value);
+
+        location loc (l.tokens[0].location ());
+
+        if (cmd == "hash")
+        {
+          sha256 cs;
+          for (auto i (ns.begin () + 1); i != ns.end (); ++i) // Skip <cmd>.
+            to_checksum (cs, *i);
+
+          if (dd.expect (cs.string ()) != nullptr)
+            l4 ([&] {
+                diag_record dr (trace);
+                dr << "'depdb hash' argument change forcing update of " << t <<
+                  info (loc); script::dump (dr.os, l);
+              });
+        }
+        else if (cmd == "string")
+        {
+          string s;
+          try
+          {
+            s = convert<string> (names (make_move_iterator (ns.begin () + 1),
+                                        make_move_iterator (ns.end ())));
+          }
+          catch (const invalid_argument& e)
+          {
+            fail (l.tokens[2].location ())
+              << "invalid 'depdb string' argument: " << e;
+          }
+
+          if (dd.expect (s) != nullptr)
+            l4 ([&] {
+                diag_record dr (trace);
+                dr << "'depdb string' argument change forcing update of "
+                   << t <<
+                  info (loc); script::dump (dr.os, l);
+              });
+        }
+        else
+          assert (false);
+      }
+    }
+
+    // Update if depdb mismatch.
+    //
+    if (dd.writing () || dd.mtime > mt)
+      update = true;
+
+    dd.close ();
+
+    // If nothing changed, then we are done.
+    //
+    if (!update)
+      return *ps;
+
+    if (!ctx.dry_run || verb != 0)
+    {
+      if (bs == nullptr)
+      {
+        bs = &t.base_scope ();
+        rs = bs->root_scope ();
+      }
+
+      build::script::environment e (a, t, script.temp_dir);
+      build::script::parser p (ctx);
+
+      if (verb == 1)
+      {
+        if (script.diag_line)
+        {
+          text << p.execute_special (*rs, *bs, e, *script.diag_line);
+        }
+        else
+        {
+          // @@ TODO (and below):
+          //
+          // - we are printing target, not source (like in most other places)
+          //
+          // - printing of ad hoc target group (the {hxx cxx}{foo} idea)
+          //
+          // - if we are printing prerequisites, should we print all of them
+          //   (including tools)?
+          //
+          text << *script.diag_name << ' ' << t;
+        }
+      }
+
+      if (!ctx.dry_run || verb >= 2)
+      {
+        build::script::default_runner r;
+        p.execute (*rs, *bs, e, script, r);
+
+        if (!ctx.dry_run)
+          dd.check_mtime (tp);
+      }
+    }
+
+    t.mtime (system_clock::now ());
+    return target_state::changed;
+  }
+
+  target_state adhoc_buildscript_rule::
+  default_action (action a, const target& t) const
+  {
+    tracer trace ("adhoc_buildscript_rule::default_action");
+
+    context& ctx (t.ctx);
+
+    execute_prerequisites (a, t);
+
+    if (!ctx.dry_run || verb != 0)
+    {
+      const scope& bs (t.base_scope ());
+      const scope& rs (*bs.root_scope ());
+
+      build::script::environment e (a, t, script.temp_dir);
+      build::script::parser p (ctx);
+
+      if (verb == 1)
+      {
+        if (script.diag_line)
+        {
+          text << p.execute_special (rs, bs, e, *script.diag_line);
+        }
+        else
+        {
+          // @@ TODO: as above
+          //
+          text << *script.diag_name << ' ' << t;
+        }
+      }
+
+      if (!ctx.dry_run || verb >= 2)
+      {
+        build::script::default_runner r;
+        p.execute (rs, bs, e, script, r);
+      }
+    }
+
+    return target_state::changed;
+  }
+}