aboutsummaryrefslogtreecommitdiff
path: root/libbuild2/adhoc-rule-buildscript.cxx
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2020-07-12 10:24:08 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2020-07-12 10:24:08 +0200
commit70b4532ae118accdbe11f1983a81a26927fc9065 (patch)
tree01b272cbd8783926fe3283ef60d2d7014536c9c3 /libbuild2/adhoc-rule-buildscript.cxx
parentec203677f1de13c200e54813db73a8ed5be8d4c9 (diff)
Rename rule-adhoc-* to adhoc-rule-*
Diffstat (limited to 'libbuild2/adhoc-rule-buildscript.cxx')
-rw-r--r--libbuild2/adhoc-rule-buildscript.cxx603
1 files changed, 603 insertions, 0 deletions
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx
new file mode 100644
index 0000000..c68f4b2
--- /dev/null
+++ b/libbuild2/adhoc-rule-buildscript.cxx
@@ -0,0 +1,603 @@
+// file : libbuild2/adhoc-rule-buildscript.cxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#include <libbuild2/adhoc-rule-buildscript.hxx>
+
+#include <sstream>
+
+#include <libbuild2/depdb.hxx>
+#include <libbuild2/scope.hxx>
+#include <libbuild2/target.hxx>
+#include <libbuild2/context.hxx>
+#include <libbuild2/algorithm.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+#include <libbuild2/parser.hxx> // attributes
+
+#include <libbuild2/build/script/parser.hxx>
+#include <libbuild2/build/script/runner.hxx>
+
+using namespace std;
+
+namespace build2
+{
+ bool adhoc_buildscript_rule::
+ recipe_text (context& ctx, const target& tg, string&& t, attributes& as)
+ {
+ // Handle and erase recipe-specific attributes.
+ //
+ optional<string> diag;
+ for (auto i (as.begin ()); i != as.end (); )
+ {
+ attribute& a (*i);
+ const string& n (a.name);
+
+ if (n == "diag")
+ try
+ {
+ diag = convert<string> (move (a.value));
+ }
+ catch (const invalid_argument& e)
+ {
+ fail (as.loc) << "invalid " << n << " attribute value: " << e;
+ }
+ else
+ {
+ ++i;
+ continue;
+ }
+
+ i = as.erase (i);
+ }
+
+ checksum = sha256 (t).string ();
+
+ istringstream is (move (t));
+ build::script::parser p (ctx);
+
+ script = p.pre_parse (tg,
+ is, loc.file, loc.line + 1,
+ move (diag), as.loc);
+
+ return false;
+ }
+
+ void adhoc_buildscript_rule::
+ dump_attributes (ostream& os) const
+ {
+ // For now we dump it as an attribute whether it was specified or derived
+ // from the script. Maybe that's ok (we use this in tests)?
+ //
+ if (script.diag_name)
+ {
+ os << " [";
+ os << "diag=";
+ to_stream (os, name (*script.diag_name), true /* quote */, '@');
+ os << ']';
+ }
+ }
+
+ void adhoc_buildscript_rule::
+ dump_text (ostream& os, string& ind) const
+ {
+ os << ind << string (braces, '{') << endl;
+ ind += " ";
+
+ if (script.depdb_clear)
+ os << ind << "depdb clear" << endl;
+
+ script::dump (os, ind, script.depdb_lines);
+
+ if (script.diag_line)
+ {
+ os << ind; script::dump (os, *script.diag_line, true /* newline */);
+ }
+
+ script::dump (os, ind, script.lines);
+ ind.resize (ind.size () - 2);
+ os << ind << string (braces, '}');
+ }
+
+ bool adhoc_buildscript_rule::
+ match (action a, target& t, const string&, optional<action> fb) const
+ {
+ if (!fb)
+ ;
+ // If this is clean for a file target and we are supplying the update,
+ // then we will also supply the standard clean.
+ //
+ else if (a == perform_clean_id &&
+ *fb == perform_update_id &&
+ t.is_a<file> ())
+ ;
+ else
+ return false;
+
+ // It's unfortunate we have to resort to this but we need to remember this
+ // in apply().
+ //
+ t.data (fb.has_value ());
+
+ return true;
+ }
+
+ recipe adhoc_buildscript_rule::
+ apply (action a, target& t) const
+ {
+ // If this is an outer operation (e.g., update-for-test), then delegate to
+ // the inner.
+ //
+ if (a.outer ())
+ {
+ match_inner (a, t);
+ return execute_inner;
+ }
+
+ // Derive file names for the target and its ad hoc group members, if any.
+ //
+ if (a == perform_update_id || a == perform_clean_id)
+ {
+ for (target* m (&t); m != nullptr; m = m->adhoc_member)
+ {
+ if (auto* p = m->is_a<path_target> ())
+ p->derive_path ();
+ }
+ }
+
+ // Inject dependency on the output directory.
+ //
+ // We do it always instead of only if one of the targets is path-based in
+ // case the recipe creates temporary files or some such.
+ //
+ inject_fsdir (a, t);
+
+ // Match prerequisites.
+ //
+ match_prerequisite_members (a, t);
+
+ // See if we are providing the standard clean as a fallback.
+ //
+ if (t.data<bool> ())
+ return &perform_clean_depdb;
+
+ if (a == perform_update_id && t.is_a<file> ())
+ {
+ return [this] (action a, const target& t)
+ {
+ return perform_update_file (a, t);
+ };
+ }
+ else
+ {
+ return [this] (action a, const target& t)
+ {
+ return default_action (a, t);
+ };
+ }
+ }
+
+ target_state adhoc_buildscript_rule::
+ perform_update_file (action a, const target& xt) const
+ {
+ tracer trace ("adhoc_buildscript_rule::perform_update_file");
+
+ context& ctx (xt.ctx);
+
+ const file& t (xt.as<file> ());
+ const path& tp (t.path ());
+
+ // How should we hash target and prerequisite sets ($> and $<)? We could
+ // hash them as target names (i.e., the same as the $>/< content) or as
+ // paths (only for path-based targets). While names feel more general,
+ // they are also more expensive to compute. And for path-based targets,
+ // path is generally a good proxy for the target name. Since the bulk of
+ // the ad hoc recipes will presumably be operating exclusively on
+ // path-based targets, let's do it both ways.
+ //
+ auto hash_target = [ns = names ()] (sha256& cs, const target& t) mutable
+ {
+ if (const path_target* pt = t.is_a<path_target> ())
+ cs.append (pt->path ().string ());
+ else
+ {
+ ns.clear ();
+ t.as_name (ns);
+ for (const name& n: ns)
+ to_checksum (cs, n);
+ }
+ };
+
+ // Update prerequisites and determine if any of them render this target
+ // out-of-date.
+ //
+ timestamp mt (t.load_mtime ());
+ optional<target_state> ps;
+
+ sha256 pcs, ecs;
+ {
+ // This is essentially ps=execute_prerequisites(a, t, mt) which we
+ // cannot use because we need to see ad hoc prerequisites.
+ //
+ size_t busy (ctx.count_busy ());
+ size_t exec (ctx.count_executed ());
+
+ target_state rs (target_state::unchanged);
+
+ wait_guard wg (ctx, busy, t[a].task_count);
+
+ for (const target*& pt: t.prerequisite_targets[a])
+ {
+ if (pt == nullptr) // Skipped.
+ continue;
+
+ target_state s (execute_async (a, *pt, busy, t[a].task_count));
+
+ if (s == target_state::postponed)
+ {
+ rs |= s;
+ pt = nullptr;
+ }
+ }
+
+ wg.wait ();
+
+ bool e (mt == timestamp_nonexistent);
+ for (prerequisite_target& p: t.prerequisite_targets[a])
+ {
+ if (p == nullptr)
+ continue;
+
+ const target& pt (*p.target);
+
+ const auto& tc (pt[a].task_count);
+ if (tc.load (memory_order_acquire) >= busy)
+ ctx.sched.wait (exec, tc, scheduler::work_none);
+
+ target_state s (pt.executed_state (a));
+ rs |= s;
+
+ // Compare our timestamp to this prerequisite's.
+ //
+ if (!e)
+ {
+ // If this is an mtime-based target, then compare timestamps.
+ //
+ if (const mtime_target* mpt = pt.is_a<mtime_target> ())
+ {
+ if (mpt->newer (mt, s))
+ e = true;
+ }
+ else
+ {
+ // Otherwise we assume the prerequisite is newer if it was
+ // changed.
+ //
+ if (s == target_state::changed)
+ e = true;
+ }
+ }
+
+ if (p.adhoc)
+ p.target = nullptr; // Blank out.
+
+ // As part of this loop calculate checksums that need to include ad
+ // hoc prerequisites (unless the script tracks changes itself).
+ //
+ if (script.depdb_clear)
+ continue;
+
+ hash_target (pcs, pt);
+
+ // The script can reference a program in one of four ways:
+ //
+ // 1. As an (imported) target (e.g., $cli)
+ //
+ // 2. As a process_path_ex (e.g., $cxx.path).
+ //
+ // 3. As a builtin (e.g., sed)
+ //
+ // 4. As a program path/name.
+ //
+ // When it comes to change tracking, there is nothing we can do for
+ // (4) and there is nothing to do for (3) (assuming builtin semantics
+ // is stable/backwards-compatible). The (2) case is handled
+ // automatically by hashing all the variable values referenced by the
+ // script (see below), which in case of process_path_ex includes the
+ // checksum, if available.
+ //
+ // This leaves the (1) case, which itself splits into two sub-cases:
+ // the target comes with the dependency information (e.g., imported
+ // from a project via an export stub) or it does not (e.g., imported
+ // as installed). We don't need to do anything extra for the first
+ // sub-case since the target's state/mtime can be relied upon like any
+ // other prerequisite. Which cannot be said about the second sub-case,
+ // where we reply on checksum that may be included as part of the
+ // target metadata.
+ //
+ // So what we are going to do is hash checksum metadata of every
+ // executable prerequisite target that has it (we do it here in order
+ // to include ad hoc prerequisites, which feels like the right thing
+ // to do; the user may mark tools as ad hoc in order to omit them from
+ // $<).
+ //
+ if (auto* e = pt.is_a<exe> ())
+ {
+ if (auto* c = e->lookup_metadata<string> ("checksum"))
+ {
+ ecs.append (*c);
+ }
+ }
+ }
+
+ if (!e)
+ ps = rs;
+ }
+
+ bool update (!ps);
+
+ // We use depdb to track changes to the script itself, input/output file
+ // names, tools, etc.
+ //
+ depdb dd (tp + ".d");
+
+ // First should come the rule name/version.
+ //
+ if (dd.expect ("<ad hoc buildscript recipe> 1") != nullptr)
+ l4 ([&]{trace << "rule mismatch forcing update of " << t;});
+
+ // Then the script checksum.
+ //
+ // Ideally, to detect changes to the script semantics, we would hash the
+ // text with all the variables expanded but without executing any
+ // commands. In practice, this is easier said than done (think the set
+ // builtin that receives output of a command that modifies the
+ // filesystem).
+ //
+ // So as the next best thing we are going to hash the unexpanded text as
+ // well as values of all the variables expanded in it (which we get as a
+ // side effect of pre-parsing the script). This approach has a number of
+ // drawbacks:
+ //
+ // - We can't handle computed variable names (e.g., $($x ? X : Y)).
+ //
+ // - We may "overhash" by including variables that are actually
+ // script-local.
+ //
+ // - There are functions like $install.resolve() with result based on
+ // external (to the script) information.
+ //
+ if (dd.expect (checksum) != nullptr)
+ l4 ([&]{trace << "recipe text change forcing update of " << t;});
+
+ // Track the variables, targets, and prerequisites changes, unless the
+ // script doesn't track the dependency changes itself.
+ //
+
+ // For each variable hash its name, undefined/null/non-null indicator,
+ // and the value if non-null.
+ //
+ // Note that this excludes the special $< and $> variables which we
+ // handle below.
+ //
+ if (!script.depdb_clear)
+ {
+ sha256 cs;
+ names storage;
+
+ for (const string& n: script.vars)
+ {
+ cs.append (n);
+
+ lookup l;
+
+ if (const variable* var = ctx.var_pool.find (n))
+ l = t[var];
+
+ cs.append (!l.defined () ? '\x1' : l->null ? '\x2' : '\x3');
+
+ if (l)
+ {
+ storage.clear ();
+ names_view ns (reverse (*l, storage));
+
+ for (const name& n: ns)
+ to_checksum (cs, n);
+ }
+ }
+
+ if (dd.expect (cs.string ()) != nullptr)
+ l4 ([&]{trace << "recipe variable change forcing update of " << t;});
+ }
+
+ // Target and prerequisite sets ($> and $<).
+ //
+ if (!script.depdb_clear)
+ {
+ sha256 tcs;
+ for (const target* m (&t); m != nullptr; m = m->adhoc_member)
+ hash_target (tcs, *m);
+
+ if (dd.expect (tcs.string ()) != nullptr)
+ l4 ([&]{trace << "target set change forcing update of " << t;});
+
+ if (dd.expect (pcs.string ()) != nullptr)
+ l4 ([&]{trace << "prerequisite set change forcing update of " << t;});
+ }
+
+ // Finally the programs checksum.
+ //
+ if (!script.depdb_clear)
+ {
+ if (dd.expect (ecs.string ()) != nullptr)
+ l4 ([&]{trace << "program checksum change forcing update of " << t;});
+ }
+
+ const scope* bs (nullptr);
+ const scope* rs (nullptr);
+
+ // Execute the custom dependency change tracking commands, if present.
+ //
+ if (!script.depdb_lines.empty ())
+ {
+ bs = &t.base_scope ();
+ rs = bs->root_scope ();
+
+ // While it would have been nice to reuse the environment for both
+ // dependency tracking and execution, there are complications (creating
+ // temporary directory, etc).
+ //
+ build::script::environment e (a, t, false /* temp_dir */);
+ build::script::parser p (ctx);
+
+ for (const script::line& l: script.depdb_lines)
+ {
+ names ns (p.execute_special (*rs, *bs, e, l));
+
+ // These should have been enforced during pre-parsing.
+ //
+ assert (!ns.empty ()); // <cmd> ... <newline>
+ assert (l.tokens.size () > 2); // 'depdb' <cmd> ... <newline>
+
+ const string& cmd (ns[0].value);
+
+ location loc (l.tokens[0].location ());
+
+ if (cmd == "hash")
+ {
+ sha256 cs;
+ for (auto i (ns.begin () + 1); i != ns.end (); ++i) // Skip <cmd>.
+ to_checksum (cs, *i);
+
+ if (dd.expect (cs.string ()) != nullptr)
+ l4 ([&] {
+ diag_record dr (trace);
+ dr << "'depdb hash' argument change forcing update of " << t <<
+ info (loc); script::dump (dr.os, l);
+ });
+ }
+ else if (cmd == "string")
+ {
+ string s;
+ try
+ {
+ s = convert<string> (names (make_move_iterator (ns.begin () + 1),
+ make_move_iterator (ns.end ())));
+ }
+ catch (const invalid_argument& e)
+ {
+ fail (l.tokens[2].location ())
+ << "invalid 'depdb string' argument: " << e;
+ }
+
+ if (dd.expect (s) != nullptr)
+ l4 ([&] {
+ diag_record dr (trace);
+ dr << "'depdb string' argument change forcing update of "
+ << t <<
+ info (loc); script::dump (dr.os, l);
+ });
+ }
+ else
+ assert (false);
+ }
+ }
+
+ // Update if depdb mismatch.
+ //
+ if (dd.writing () || dd.mtime > mt)
+ update = true;
+
+ dd.close ();
+
+ // If nothing changed, then we are done.
+ //
+ if (!update)
+ return *ps;
+
+ if (!ctx.dry_run || verb != 0)
+ {
+ if (bs == nullptr)
+ {
+ bs = &t.base_scope ();
+ rs = bs->root_scope ();
+ }
+
+ build::script::environment e (a, t, script.temp_dir);
+ build::script::parser p (ctx);
+
+ if (verb == 1)
+ {
+ if (script.diag_line)
+ {
+ text << p.execute_special (*rs, *bs, e, *script.diag_line);
+ }
+ else
+ {
+ // @@ TODO (and below):
+ //
+ // - we are printing target, not source (like in most other places)
+ //
+ // - printing of ad hoc target group (the {hxx cxx}{foo} idea)
+ //
+ // - if we are printing prerequisites, should we print all of them
+ // (including tools)?
+ //
+ text << *script.diag_name << ' ' << t;
+ }
+ }
+
+ if (!ctx.dry_run || verb >= 2)
+ {
+ build::script::default_runner r;
+ p.execute (*rs, *bs, e, script, r);
+
+ if (!ctx.dry_run)
+ dd.check_mtime (tp);
+ }
+ }
+
+ t.mtime (system_clock::now ());
+ return target_state::changed;
+ }
+
+ target_state adhoc_buildscript_rule::
+ default_action (action a, const target& t) const
+ {
+ tracer trace ("adhoc_buildscript_rule::default_action");
+
+ context& ctx (t.ctx);
+
+ execute_prerequisites (a, t);
+
+ if (!ctx.dry_run || verb != 0)
+ {
+ const scope& bs (t.base_scope ());
+ const scope& rs (*bs.root_scope ());
+
+ build::script::environment e (a, t, script.temp_dir);
+ build::script::parser p (ctx);
+
+ if (verb == 1)
+ {
+ if (script.diag_line)
+ {
+ text << p.execute_special (rs, bs, e, *script.diag_line);
+ }
+ else
+ {
+ // @@ TODO: as above
+ //
+ text << *script.diag_name << ' ' << t;
+ }
+ }
+
+ if (!ctx.dry_run || verb >= 2)
+ {
+ build::script::default_runner r;
+ p.execute (rs, bs, e, script, r);
+ }
+ }
+
+ return target_state::changed;
+ }
+}