From b5083221dad8084deb4a7949cb9fc487aa09e080 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 16 Nov 2021 10:46:19 +0200 Subject: WIP: apply/perform_update --- libbuild2/adhoc-rule-buildscript.cxx | 734 ++++++++++++++++++++++++----------- libbuild2/adhoc-rule-buildscript.hxx | 13 +- libbuild2/build/script/parser.cxx | 191 ++++++--- libbuild2/build/script/parser.hxx | 78 +++- libbuild2/build/script/script.hxx | 11 +- libbuild2/cc/compile-rule.cxx | 65 +--- libbuild2/cc/compile-rule.hxx | 4 +- libbuild2/dynamic.cxx | 76 ++++ libbuild2/dynamic.hxx | 33 ++ 9 files changed, 845 insertions(+), 360 deletions(-) create mode 100644 libbuild2/dynamic.cxx create mode 100644 libbuild2/dynamic.hxx diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx index 61b4cb2..aa320e2 100644 --- a/libbuild2/adhoc-rule-buildscript.cxx +++ b/libbuild2/adhoc-rule-buildscript.cxx @@ -9,6 +9,7 @@ #include #include #include +#include #include #include // path_perms(), auto_rmfile #include @@ -22,6 +23,110 @@ using namespace std; namespace build2 { + static inline void + hash_script_vars (sha256& cs, + const build::script::script& s, + const target& t, + names& storage) + { + context& ctx (t.ctx); + + for (const string& n: s.vars) + { + cs.append (n); + + lookup l; + + if (const variable* var = ctx.var_pool.find (n)) + l = t[var]; + + cs.append (!l.defined () ? '\x1' : l->null ? '\x2' : '\x3'); + + if (l) + { + storage.clear (); + names_view ns (reverse (*l, storage)); + + for (const name& n: ns) + to_checksum (cs, n); + } + } + } + + // How should we hash target and prerequisite sets ($> and $<)? We could + // hash them as target names (i.e., the same as the $>/< content) or as + // paths (only for path-based targets). While names feel more general, they + // are also more expensive to compute. And for path-based targets, path is + // generally a good proxy for the target name. Since the bulk of the ad hoc + // recipes will presumably be operating exclusively on path-based targets, + // let's do it both ways. + // + static inline void + hash_target (sha256& cs, const target& t, names& storage) + { + if (const path_target* pt = t.is_a ()) + cs.append (pt->path ().string ()); + else + { + storage.clear (); + t.as_name (storage); + for (const name& n: storage) + to_checksum (cs, n); + } + }; + + // The script can reference a program in one of four ways: + // + // 1. As an (imported) target (e.g., $cli) + // + // 2. As a process_path_ex (e.g., $cxx.path). + // + // 3. As a builtin (e.g., sed) + // + // 4. As a program path/name. + // + // When it comes to change tracking, there is nothing we can do for (4) (the + // user can track its environment manually with depdb-env) and there is + // nothing to do for (3) (assuming builtin semantics is stable/backwards- + // compatible). The (2) case is handled automatically by hashing all the + // variable values referenced by the script (see below), which in case of + // process_path_ex includes the checksums (both executable and environment), + // if available. + // + // This leaves the (1) case, which itself splits into two sub-cases: the + // target comes with the dependency information (e.g., imported from a + // project via an export stub) or it does not (e.g., imported as installed). + // We don't need to do anything extra for the first sub-case since the + // target's state/mtime can be relied upon like any other prerequisite. + // Which cannot be said about the second sub-case, where we reply on + // checksum that may be included as part of the target metadata. + // + // So what we are going to do is hash checksum metadata of every executable + // prerequisite target that has it (we do it here in order to include ad hoc + // prerequisites, which feels like the right thing to do; the user may mark + // tools as ad hoc in order to omit them from $<). + // + static inline void + hash_prerequisite_target (sha256& cs, sha256& exe_cs, sha256& env_cs, + const target& pt, + names& storage) + { + hash_target (cs, pt, storage); + + if (const exe* et = pt.is_a ()) + { + if (const string* c = et->lookup_metadata ("checksum")) + { + exe_cs.append (*c); + } + + if (const strings* e = et->lookup_metadata ("environment")) + { + hash_environment (env_cs, *e); + } + } + } + bool adhoc_buildscript_rule:: recipe_text (const scope& s, const target_type& tt, @@ -113,6 +218,19 @@ namespace build2 perform_update_id) != actions.end (); } + struct adhoc_buildscript_rule::match_data + { + match_data (action a, const target& t, bool temp_dir) + : env (a, t, temp_dir) {} + + build::script::environment env; + build::script::default_runner run; + + const scope* bs; + timestamp mt; + path dd; + }; + bool adhoc_buildscript_rule:: match (action a, target& t, const string& h, match_extra& me) const { @@ -140,15 +258,17 @@ namespace build2 recipe adhoc_buildscript_rule:: apply (action a, - target& t, + target& xt, match_extra& me, const optional& d) const { + tracer trace ("adhoc_buildscript_rule::apply"); + // We don't support deadlines for any of these cases (see below). // if (d && (a.outer () || me.fallback || - (a == perform_update_id && t.is_a ()))) + (a == perform_update_id && xt.is_a ()))) return empty_recipe; // If this is an outer operation (e.g., update-for-test), then delegate to @@ -156,20 +276,20 @@ namespace build2 // if (a.outer ()) { - match_inner (a, t); + match_inner (a, xt); return execute_inner; } // Inject pattern's ad hoc group members, if any. // if (pattern != nullptr) - pattern->apply_adhoc_members (a, t, me); + pattern->apply_adhoc_members (a, xt, me); // Derive file names for the target and its ad hoc group members, if any. // if (a == perform_update_id || a == perform_clean_id) { - for (target* m (&t); m != nullptr; m = m->adhoc_member) + for (target* m (&xt); m != nullptr; m = m->adhoc_member) { if (auto* p = m->is_a ()) p->derive_path (); @@ -181,68 +301,297 @@ namespace build2 // We do it always instead of only if one of the targets is path-based in // case the recipe creates temporary files or some such. // - inject_fsdir (a, t); + const fsdir* dir (inject_fsdir (a, xt)); // Match prerequisites. // - match_prerequisite_members (a, t); + match_prerequisite_members (a, xt); // Inject pattern's prerequisites, if any. // if (pattern != nullptr) - pattern->apply_prerequisites (a, t, me); + pattern->apply_prerequisites (a, xt, me); // See if we are providing the standard clean as a fallback. // if (me.fallback) return &perform_clean_depdb; - if (a == perform_update_id && t.is_a ()) + // See if this is not update or not on a file-based target. + // + if (a != perform_update_id || !xt.is_a ()) + { + return [d, this] (action a, const target& t) + { + return default_action (a, t, d); + }; + } + + // See if this is the simple case with only static dependencies. + // + if (!script.depdb_pre_dynamic) { return [this] (action a, const target& t) { - return perform_update_file (a, t); + return perform_update_file_static (a, t); }; } + + // This is a perform update on a file target with extraction of dynamic + // dependency information in the depdb preamble (depdb-pre-dynamic). + // + // This means we may need to add additional prerequisites (or even target + // group members). We also have to save any such additional prerequisites + // in depdb so that we can check if any of them have changed on subsequent + // updates. So all this means that have to take care of depdb here in + // apply() instead of perform_*() like we normally do. We also do things + // in slightly different order due to the restrictions impose by the match + // phase. + // + // Note that the C/C++ header dependency extraction is the canonical + // example and all this logic is based on the prior work in the cc module + // where you can often find more detailed rationale for some of the steps + // performed (like the fsdir update below). + // + context& ctx (xt.ctx); + + const file& t (xt.as ()); + const path& tp (t.path ()); + + if (dir != nullptr) + fsdir_rule::perform_update_direct (a, t); + + // Because the depdb preamble can access $<, we have to blank out all the + // ad hoc prerequisites. Since we will still need them later, we "move" + // them to the auxiliary data member in prerequisite_target (which also + // means we cannot use the standard execute_prerequisites()). + // + auto& pts (t.prerequisite_targets[a]); + for (prerequisite_target& p: pts) + { + if (p.target != nullptr && p.adhoc) + { + // Blank out injected fsdir{} for good. + // + if (p.target != dir) + p.data = reinterpret_cast (p.target); + } + } + + // NOTE: see the "static dependencies" version (with comments) below. + // + depdb dd (tp + ".d"); + + if (dd.expect (" 1") != nullptr) + l4 ([&]{trace << "rule mismatch forcing update of " << t;}); + + if (dd.expect (checksum) != nullptr) + l4 ([&]{trace << "recipe text change forcing update of " << t;}); + + if (!script.depdb_clear) + { + names storage; + + sha256 prq_cs, exe_cs, env_cs; + + for (const prerequisite_target& p: pts) + { + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : + nullptr)) + { + hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage); + } + } + + { + sha256 cs; + hash_script_vars (cs, script, t, storage); + + if (dd.expect (cs.string ()) != nullptr) + l4 ([&]{trace << "recipe variable change forcing update of " << t;}); + } + + { + sha256 tcs; + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + hash_target (tcs, *m, storage); + + if (dd.expect (tcs.string ()) != nullptr) + l4 ([&]{trace << "target set change forcing update of " << t;}); + + if (dd.expect (prq_cs.string ()) != nullptr) + l4 ([&]{trace << "prerequisite set change forcing update of " << t;}); + } + + { + if (dd.expect (exe_cs.string ()) != nullptr) + l4 ([&]{trace << "program checksum change forcing update of " << t;}); + + if (dd.expect (env_cs.string ()) != nullptr) + l4 ([&]{trace << "environment change forcing update of " << t;}); + } + } + + const scope& bs (t.base_scope ()); + + unique_ptr md ( + new match_data (a, t, script.depdb_preamble_temp_dir)); + + build::script::environment& env (md->env); + build::script::default_runner& run (md->run); + + run.enter (env, script.start_loc); + + // Run the first half of the preamble (before depdb-pre-dynamic). + // + { + build::script::parser p (ctx); + p.execute_depdb_preamble (bs, env, script, run, dd); + } + + // Determine if we need to do an update based on the above checks. + // + bool update; + timestamp mt; + + if (dd.writing ()) + update = true; else { - return [d, this] (action a, const target& t) + if ((mt = t.mtime ()) == timestamp_unknown) + t.mtime (mt = mtime (tp)); // Cache. + + update = dd.mtime > mt; + } + + if (update) + mt = timestamp_nonexistent; + + // Update our prerequisite targets. While strictly speaking we only need + // to update those that are referenced by depdb-pre-dynamic, communicating + // this is both tedious and error-prone. So we update them all. + // + for (const prerequisite_target& p: pts) + { + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : nullptr)) { - return default_action (a, t, d); - }; + update = dyndep_rule::update ( + trace, a, *pt, update ? timestamp_unknown : mt) || update; + } } + + // Run the second half of the preamble (depdb-pre-dynamic commands) to + // extract dynamic dependencies. + // + // Note that this should be the last update to depdb (the invalidation + // order semantics). + { + build::script::parser p (ctx); + p.execute_depdb_preamble_dynamic (bs, env, script, run, dd, update, mt); + } + + if (update && dd.reading () && !ctx.dry_run) + dd.touch = true; + + dd.close (); + md->dd = move (dd.path); + + // Pass on base scope and update/mtime. + // + md->bs = &bs; + md->mt = update ? timestamp_nonexistent : mt; + + // @@ TMP: re-enable once recipe becomes move_only_function. + // +#if 0 + return [this, md = move (md)] (action a, const target& t) mutable + { + auto r (perform_update_file_dynamic (a, t, *md)); + md.reset (); // @@ TMP: is this really necessary (+mutable)? + return r; + }; +#else + t.data (move (md)); + return recipe ([this] (action a, const target& t) mutable + { + auto md (move (t.data> ())); + return perform_update_file_dynamic (a, t, *md); + }); +#endif } target_state adhoc_buildscript_rule:: - perform_update_file (action a, const target& xt) const + perform_update_file_dynamic (action a, const target& xt, + match_data& md) const { - tracer trace ("adhoc_buildscript_rule::perform_update_file"); + tracer trace ("adhoc_buildscript_rule::perform_update_file_dynamic"); context& ctx (xt.ctx); const file& t (xt.as ()); const path& tp (t.path ()); - // How should we hash target and prerequisite sets ($> and $<)? We could - // hash them as target names (i.e., the same as the $>/< content) or as - // paths (only for path-based targets). While names feel more general, - // they are also more expensive to compute. And for path-based targets, - // path is generally a good proxy for the target name. Since the bulk of - // the ad hoc recipes will presumably be operating exclusively on - // path-based targets, let's do it both ways. + // While we've updated all our prerequisites in apply(), we still need to + // execute them here to keep the dependency counts straight. // - auto hash_target = [ns = names ()] (sha256& cs, const target& t) mutable + for (const prerequisite_target& p: t.prerequisite_targets[a]) { - if (const path_target* pt = t.is_a ()) - cs.append (pt->path ().string ()); - else + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : nullptr)) { - ns.clear (); - t.as_name (ns); - for (const name& n: ns) - to_checksum (cs, n); + target_state ts (execute (a, *pt)); + assert (ts == target_state::unchanged || ts == target_state::changed); } - }; + } + + build::script::environment& env (md.env); + build::script::default_runner& run (md.run); + + if (md.mt != timestamp_nonexistent) + { + run.leave (env, script.end_loc); + return target_state::unchanged; + } + + // Sequence start time for mtime checks below. + // + timestamp start (!ctx.dry_run && depdb::mtime_check () + ? system_clock::now () + : timestamp_unknown); + + if (!ctx.dry_run || verb != 0) + { + if (execute_update_file (*md.bs, a, t, env, run)) + ; + else + run.leave (env, script.end_loc); + } + else + run.leave (env, script.end_loc); + + timestamp now (system_clock::now ()); + + if (!ctx.dry_run) + depdb::check_mtime (start, md.dd, tp, now); + + t.mtime (now); + return target_state::changed; + } + + target_state adhoc_buildscript_rule:: + perform_update_file_static (action a, const target& xt) const + { + tracer trace ("adhoc_buildscript_rule::perform_update_file_static"); + + context& ctx (xt.ctx); + + const file& t (xt.as ()); + const path& tp (t.path ()); // Update prerequisites and determine if any of them render this target // out-of-date. @@ -250,6 +599,8 @@ namespace build2 timestamp mt (t.load_mtime ()); optional ps; + names storage; + sha256 prq_cs, exe_cs, env_cs; { // This is essentially ps=execute_prerequisites(a, t, mt) which we @@ -262,7 +613,9 @@ namespace build2 wait_guard wg (ctx, busy, t[a].task_count); - for (const target*& pt: t.prerequisite_targets[a]) + auto& pts (t.prerequisite_targets[a]); + + for (const target*& pt: pts) { if (pt == nullptr) // Skipped. continue; @@ -279,7 +632,7 @@ namespace build2 wg.wait (); bool e (mt == timestamp_nonexistent); - for (prerequisite_target& p: t.prerequisite_targets[a]) + for (prerequisite_target& p: pts) { if (p == nullptr) continue; @@ -318,56 +671,10 @@ namespace build2 // As part of this loop calculate checksums that need to include ad // hoc prerequisites (unless the script tracks changes itself). // - if (script.depdb_clear) - continue; - - hash_target (prq_cs, pt); - - // The script can reference a program in one of four ways: - // - // 1. As an (imported) target (e.g., $cli) + // @@ TODO: skip fsdir{}? // - // 2. As a process_path_ex (e.g., $cxx.path). - // - // 3. As a builtin (e.g., sed) - // - // 4. As a program path/name. - // - // When it comes to change tracking, there is nothing we can do for - // (4) (the user can track its environment manually with depdb-env) - // and there is nothing to do for (3) (assuming builtin semantics is - // stable/backwards-compatible). The (2) case is handled automatically - // by hashing all the variable values referenced by the script (see - // below), which in case of process_path_ex includes the checksums - // (both executable and environment), if available. - // - // This leaves the (1) case, which itself splits into two sub-cases: - // the target comes with the dependency information (e.g., imported - // from a project via an export stub) or it does not (e.g., imported - // as installed). We don't need to do anything extra for the first - // sub-case since the target's state/mtime can be relied upon like any - // other prerequisite. Which cannot be said about the second sub-case, - // where we reply on checksum that may be included as part of the - // target metadata. - // - // So what we are going to do is hash checksum metadata of every - // executable prerequisite target that has it (we do it here in order - // to include ad hoc prerequisites, which feels like the right thing - // to do; the user may mark tools as ad hoc in order to omit them from - // $<). - // - if (auto* et = pt.is_a ()) - { - if (auto* c = et->lookup_metadata ("checksum")) - { - exe_cs.append (*c); - } - - if (auto* e = et->lookup_metadata ("environment")) - { - hash_environment (env_cs, *e); - } - } + if (!script.depdb_clear) + hash_prerequisite_target (prq_cs, exe_cs, env_cs, pt, storage); } if (!e) @@ -379,6 +686,8 @@ namespace build2 // We use depdb to track changes to the script itself, input/output file // names, tools, etc. // + // NOTE: see the "dynamic dependencies" version above. + // depdb dd (tp + ".d"); // First should come the rule name/version. @@ -411,76 +720,53 @@ namespace build2 l4 ([&]{trace << "recipe text change forcing update of " << t;}); // Track the variables, targets, and prerequisites changes, unless the - // script doesn't track the dependency changes itself. - // - - // For each variable hash its name, undefined/null/non-null indicator, - // and the value if non-null. - // - // Note that this excludes the special $< and $> variables which we - // handle below. - // - // @@ TODO: maybe detect and decompose process_path_ex in order to - // properly attribute checksum and environment changes? + // script tracks the dependency changes itself. // if (!script.depdb_clear) { - sha256 cs; - names storage; - - for (const string& n: script.vars) + // For each variable hash its name, undefined/null/non-null indicator, + // and the value if non-null. + // + // Note that this excludes the special $< and $> variables which we + // handle below. + // + // @@ TODO: maybe detect and decompose process_path_ex in order to + // properly attribute checksum and environment changes? + // { - cs.append (n); + sha256 cs; + hash_script_vars (cs, script, t, storage); - lookup l; - - if (const variable* var = ctx.var_pool.find (n)) - l = t[var]; - - cs.append (!l.defined () ? '\x1' : l->null ? '\x2' : '\x3'); - - if (l) - { - storage.clear (); - names_view ns (reverse (*l, storage)); - - for (const name& n: ns) - to_checksum (cs, n); - } + if (dd.expect (cs.string ()) != nullptr) + l4 ([&]{trace << "recipe variable change forcing update of " << t;}); } - if (dd.expect (cs.string ()) != nullptr) - l4 ([&]{trace << "recipe variable change forcing update of " << t;}); - } - - // Target and prerequisite sets ($> and $<). - // - if (!script.depdb_clear) - { - sha256 tcs; - for (const target* m (&t); m != nullptr; m = m->adhoc_member) - hash_target (tcs, *m); + // Target and prerequisite sets ($> and $<). + // + { + sha256 tcs; + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + hash_target (tcs, *m, storage); - if (dd.expect (tcs.string ()) != nullptr) - l4 ([&]{trace << "target set change forcing update of " << t;}); + if (dd.expect (tcs.string ()) != nullptr) + l4 ([&]{trace << "target set change forcing update of " << t;}); - if (dd.expect (prq_cs.string ()) != nullptr) - l4 ([&]{trace << "prerequisite set change forcing update of " << t;}); - } + if (dd.expect (prq_cs.string ()) != nullptr) + l4 ([&]{trace << "prerequisite set change forcing update of " << t;}); + } - // Finally the programs and environment checksums. - // - if (!script.depdb_clear) - { - if (dd.expect (exe_cs.string ()) != nullptr) - l4 ([&]{trace << "program checksum change forcing update of " << t;}); + // Finally the programs and environment checksums. + // + { + if (dd.expect (exe_cs.string ()) != nullptr) + l4 ([&]{trace << "program checksum change forcing update of " << t;}); - if (dd.expect (env_cs.string ()) != nullptr) - l4 ([&]{trace << "environment change forcing update of " << t;}); + if (dd.expect (env_cs.string ()) != nullptr) + l4 ([&]{trace << "environment change forcing update of " << t;}); + } } const scope* bs (nullptr); - const scope* rs (nullptr); // Execute the custom dependency change tracking commands, if present. // @@ -507,20 +793,19 @@ namespace build2 } build::script::environment env (a, t, false /* temp_dir */); - build::script::default_runner r; + build::script::default_runner run; if (depdb_preamble) { bs = &t.base_scope (); - rs = bs->root_scope (); if (script.depdb_preamble_temp_dir) env.set_temp_dir_variable (); build::script::parser p (ctx); - r.enter (env, script.start_loc); - p.execute_depdb_preamble (*rs, *bs, env, script, r, dd); + run.enter (env, script.start_loc); + p.execute_depdb_preamble (*bs, env, script, run, dd); } // Update if depdb mismatch. @@ -539,104 +824,118 @@ namespace build2 // below). // if (depdb_preamble) - r.leave (env, script.end_loc); + run.leave (env, script.end_loc); return *ps; } if (!ctx.dry_run || verb != 0) { - // Prepare to executing the script diag line and/or body. - // - // Note that it doesn't make much sense to use the temporary directory - // variable ($~) in the 'diag' builtin call, so we postpone setting it - // until the script body execution, that can potentially be omitted. + // Prepare to execute the script diag line and/or body. // if (bs == nullptr) - { bs = &t.base_scope (); - rs = bs->root_scope (); + + if (execute_update_file (*bs, a, t, env, run)) + { + if (!ctx.dry_run) + dd.check_mtime (tp); } + else if (depdb_preamble) + run.leave (env, script.end_loc); + } + else if (depdb_preamble) + run.leave (env, script.end_loc); - build::script::parser p (ctx); + t.mtime (system_clock::now ()); + return target_state::changed; + } - if (verb == 1) + bool adhoc_buildscript_rule:: + execute_update_file (const scope& bs, + action, const file& t, + build::script::environment& env, + build::script::default_runner& run) const + { + context& ctx (t.ctx); + + // Note that it doesn't make much sense to use the temporary directory + // variable ($~) in the 'diag' builtin call, so we postpone setting it + // until the script body execution, that can potentially be omitted. + // + build::script::parser p (ctx); + + if (verb == 1) + { + if (script.diag_line) { - if (script.diag_line) - { - text << p.execute_special (*rs, *bs, env, *script.diag_line); - } - else - { - // @@ TODO (and below): - // - // - we are printing target, not source (like in most other places) - // - // - printing of ad hoc target group (the {hxx cxx}{foo} idea) - // - // - if we are printing prerequisites, should we print all of them - // (including tools)? - // - text << *script.diag_name << ' ' << t; - } + text << p.execute_special (bs, env, *script.diag_line); } - - if (!ctx.dry_run || verb >= 2) + else { - // On failure remove the target files that may potentially exist but - // be invalid. + // @@ TODO (and in default_action() below): // - small_vector rms; + // - we are printing target, not source (like in most other places) + // + // - printing of ad hoc target group (the {hxx cxx}{foo} idea) + // + // - if we are printing prerequisites, should we print all of them + // (including tools)? + // + text << *script.diag_name << ' ' << t; + } + } - if (!ctx.dry_run) + if (!ctx.dry_run || verb >= 2) + { + // On failure remove the target files that may potentially exist but + // be invalid. + // + small_vector rms; + + if (!ctx.dry_run) + { + for (const target* m (&t); m != nullptr; m = m->adhoc_member) { - for (const target* m (&t); m != nullptr; m = m->adhoc_member) - { - if (auto* f = m->is_a ()) - rms.emplace_back (f->path ()); - } + if (auto* f = m->is_a ()) + rms.emplace_back (f->path ()); } + } - if (script.body_temp_dir && !script.depdb_preamble_temp_dir) - env.set_temp_dir_variable (); + if (script.body_temp_dir && !script.depdb_preamble_temp_dir) + env.set_temp_dir_variable (); - p.execute_body (*rs, *bs, env, script, r, !depdb_preamble); + p.execute_body (bs, env, script, run, script.depdb_preamble.empty ()); - if (!ctx.dry_run) - { - // If this is an executable, let's be helpful to the user and set - // the executable bit on POSIX. - // + if (!ctx.dry_run) + { + // If this is an executable, let's be helpful to the user and set + // the executable bit on POSIX. + // #ifndef _WIN32 - auto chmod = [] (const path& p) - { - path_perms (p, - (path_perms (p) | - permissions::xu | - permissions::xg | - permissions::xo)); - }; - - for (const target* m (&t); m != nullptr; m = m->adhoc_member) - { - if (auto* p = m->is_a ()) - chmod (p->path ()); - } -#endif - dd.check_mtime (tp); - - for (auto& rm: rms) - rm.cancel (); + auto chmod = [] (const path& p) + { + path_perms (p, + (path_perms (p) | + permissions::xu | + permissions::xg | + permissions::xo)); + }; + + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (auto* p = m->is_a ()) + chmod (p->path ()); } +#endif + for (auto& rm: rms) + rm.cancel (); } - else if (depdb_preamble) - r.leave (env, script.end_loc); - } - else if (depdb_preamble) - r.leave (env, script.end_loc); - t.mtime (system_clock::now ()); - return target_state::changed; + return true; + } + else + return false; } target_state adhoc_buildscript_rule:: @@ -653,7 +952,6 @@ namespace build2 if (!ctx.dry_run || verb != 0) { const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); build::script::environment e (a, t, script.body_temp_dir, deadline); build::script::parser p (ctx); @@ -662,11 +960,11 @@ namespace build2 { if (script.diag_line) { - text << p.execute_special (rs, bs, e, *script.diag_line); + text << p.execute_special (bs, e, *script.diag_line); } else { - // @@ TODO: as above + // @@ TODO: as above (execute_update_file()). // text << *script.diag_name << ' ' << t; } @@ -675,7 +973,7 @@ namespace build2 if (!ctx.dry_run || verb >= 2) { build::script::default_runner r; - p.execute_body (rs, bs, e, script, r); + p.execute_body (bs, e, script, r); } } diff --git a/libbuild2/adhoc-rule-buildscript.hxx b/libbuild2/adhoc-rule-buildscript.hxx index 7f9c10a..7ec647b 100644 --- a/libbuild2/adhoc-rule-buildscript.hxx +++ b/libbuild2/adhoc-rule-buildscript.hxx @@ -36,7 +36,18 @@ namespace build2 const optional&) const override; target_state - perform_update_file (action, const target&) const; + perform_update_file_static (action, const target&) const; + + struct match_data; + + target_state + perform_update_file_dynamic (action, const target&, match_data&) const; + + bool + execute_update_file (const scope&, + action a, const file&, + build::script::environment&, + build::script::default_runner&) const; target_state default_action (action, const target&, const optional&) const; diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index 0d0ad0a..c6d7d49 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -5,6 +5,7 @@ #include +#include #include #include @@ -128,6 +129,8 @@ namespace build2 // Save the custom dependency change tracking lines, if present. // s.depdb_clear = depdb_clear_.has_value (); + if (depdb_pre_dynamic_) + s.depdb_pre_dynamic = depdb_pre_dynamic_->second; s.depdb_preamble = move (depdb_preamble_); return s; @@ -494,7 +497,7 @@ namespace build2 v != "hash" && v != "string" && v != "env" && - v != "dep")) + v != "pre-dynamic")) { fail (get_location (t)) << "expected 'depdb' builtin command instead of " << t; @@ -534,32 +537,34 @@ namespace build2 // the referenced variable list, since it won't be used. // depdb_clear_ = l; - save_line_ = nullptr; + save_line_ = nullptr; script_->vars.clear (); } else { - // Verify depdb-dep is last. + // Verify depdb-pre-dynamic is last. // - if (v == "dep") + if (v == "pre-dynamic") { - // Note that for now we do not allow multiple depdb-dep calls. - // But we may wan to relax this later (though alternating - // targets with prerequisites may be tricky -- maybe still - // only allow additional targets in the first call). + // Note that for now we do not allow multiple pre-dynamic + // calls. But we may wan to relax this later (though + // alternating targets with prerequisites may be tricky -- + // maybe still only allow additional targets in the first + // call). // - if (!depdb_dep_) - depdb_dep_ = l; + if (!depdb_pre_dynamic_) + depdb_pre_dynamic_ = make_pair (l, depdb_preamble_.size ()); else - fail (l) << "multiple 'depdb dep' calls" << - info (*depdb_dep_) << "previous call is here"; + fail (l) << "multiple 'depdb pre-dynamic' calls" << + info (depdb_pre_dynamic_->first) << "previous call is here"; } else { - if (depdb_dep_) - fail (l) << "'depdb " << v << "' after 'depdb dep'" << - info (*depdb_dep_) << "'depdb dep' call is here"; + if (depdb_pre_dynamic_) + fail (l) << "'depdb " << v << "' after 'depdb pre-dynamic'" << + info (depdb_pre_dynamic_->first) + << "'depdb pre-dynamic' call is here"; } // Move the script body to the end of the depdb preamble. @@ -880,11 +885,11 @@ namespace build2 } void parser:: - execute_body (const scope& rs, const scope& bs, + execute_body (const scope& bs, environment& e, const script& s, runner& r, bool enter, bool leave) { - pre_exec (rs, bs, e, &s, &r); + pre_exec (*bs.root_scope (), bs, e, &s, &r); if (enter) runner_->enter (e, s.start_loc); @@ -914,35 +919,37 @@ namespace build2 } void parser:: - execute_depdb_preamble (const scope& rs, const scope& bs, - environment& e, const script& s, runner& r, - depdb& dd) + exec_depdb_preamble (const scope& bs, + environment& e, const script& s, runner& r, + lines_iterator begin, lines_iterator end, + depdb& dd, bool* update, optional mt) { - tracer trace ("execute_depdb_preamble"); + tracer trace ("exec_depdb_preamble"); // The only valid lines in the depdb preamble are the depdb builtin // itself as well as the variable assignments, including via the set // builtin. - pre_exec (rs, bs, e, &s, &r); + pre_exec (*bs.root_scope (), bs, e, &s, &r); // Let's "wrap up" the objects we operate upon into the single object // to rely on "small function object" optimization. // struct { + tracer& trace; environment& env; const script& scr; depdb& dd; - tracer& trace; - } ctx {e, s, dd, trace}; - - auto exec_cmd = [&ctx, this] - (token& t, - build2::script::token_type& tt, - size_t li, - bool /* single */, - const location& ll) + bool* update; + optional mt; + } data {trace, e, s, dd, update, mt}; + + auto exec_cmd = [this, &data] (token& t, + build2::script::token_type& tt, + size_t li, + bool /* single */, + const location& ll) { // Note that we never reset the line index to zero (as we do in // execute_body()) assuming that there are some script body @@ -958,9 +965,14 @@ namespace build2 string cmd (move (t.value)); - if (cmd == "dep") + if (cmd == "pre-dynamic") { - exec_depdb_dep (t, tt, li, ll); + exec_depdb_pre_dynamic (t, tt, + li, ll, + data.env.target, + data.dd, + *data.update, + *data.mt); } else { @@ -972,11 +984,11 @@ namespace build2 for (const name& n: ns) to_checksum (cs, n); - if (ctx.dd.expect (cs.string ()) != nullptr) + if (data.dd.expect (cs.string ()) != nullptr) l4 ([&] { - ctx.trace (ll) + data.trace (ll) << "'depdb hash' argument change forcing update of " - << ctx.env.target;}); + << data.env.target;}); } else if (cmd == "string") { @@ -990,11 +1002,11 @@ namespace build2 fail (ll) << "invalid 'depdb string' argument: " << e; } - if (ctx.dd.expect (s) != nullptr) + if (data.dd.expect (s) != nullptr) l4 ([&] { - ctx.trace (ll) + data.trace (ll) << "'depdb string' argument change forcing update of " - << ctx.env.target;}); + << data.env.target;}); } else if (cmd == "env") { @@ -1015,11 +1027,11 @@ namespace build2 fail (ll) << pf << e; } - if (ctx.dd.expect (cs.string ()) != nullptr) + if (data.dd.expect (cs.string ()) != nullptr) l4 ([&] { - ctx.trace (ll) + data.trace (ll) << "'depdb env' environment change forcing update of " - << ctx.env.target;}); + << data.env.target;}); } else assert (false); @@ -1040,7 +1052,7 @@ namespace build2 p.recall.string () == "set"; }) == ce.end ()) { - const replay_tokens& rt (ctx.scr.depdb_preamble.back ().tokens); + const replay_tokens& rt (data.scr.depdb_preamble.back ().tokens); assert (!rt.empty ()); fail (ll) << "disallowed command in depdb preamble" << @@ -1053,7 +1065,7 @@ namespace build2 } }; - exec_lines (s.depdb_preamble, exec_cmd); + exec_lines (begin, end, exec_cmd); } void parser:: @@ -1085,7 +1097,7 @@ namespace build2 } void parser:: - exec_lines (const lines& lns, + exec_lines (lines_iterator begin, lines_iterator end, const function& exec_cmd) { // Note that we rely on "small function object" optimization for the @@ -1124,7 +1136,7 @@ namespace build2 return runner_->run_if (*environment_, ce, li, ll); }; - build2::script::parser::exec_lines (lns.begin (), lns.end (), + build2::script::parser::exec_lines (begin, end, exec_set, exec_cmd, exec_if, environment_->exec_line, &environment_->var_pool); @@ -1145,12 +1157,15 @@ namespace build2 } names parser:: - execute_special (const scope& rs, const scope& bs, + execute_special (const scope& bs, environment& e, const line& ln, bool omit_builtin) { - pre_exec (rs, bs, e, nullptr /* script */, nullptr /* runner */); + pre_exec (*bs.root_scope (), bs, + e, + nullptr /* script */, + nullptr /* runner */); // Copy the tokens and start playing. // @@ -1167,9 +1182,12 @@ namespace build2 } void parser:: - exec_depdb_dep (token& t, build2::script::token_type& tt, - size_t li, - const location& ll) + exec_depdb_pre_dynamic (token& t, build2::script::token_type& tt, + size_t li, const location& ll, + const target& tgt, + depdb& dd, + bool& update, + timestamp /*mt*/) { // Similar approach to parse_env_builtin(). // @@ -1197,14 +1215,14 @@ namespace build2 location l (get_location (t)); if (!start_names (tt)) - fail (l) << "depdb dep: expected option or '--' separator " + fail (l) << "depdb pre-dynamic: expected option or '--' separator " << "instead of " << t; parse_names (t, tt, ns, pattern_mode::ignore, true /* chunk */, - "depdb dep builtin argument", + "depdb pre-dynamic builtin argument", nullptr); for (name& n: ns) @@ -1229,7 +1247,8 @@ namespace build2 next (t, tt); // Skip '--'. if (tt == type::newline || tt == type::eos) - fail (t) << "depdb dep: expected program name instead of " << t; + fail (t) << "depdb pre-dynamic: expected program name instead of " + << t; } // Parse the options. @@ -1241,12 +1260,65 @@ namespace build2 ops = depdb_dep_options (scan); if (scan.more ()) - fail (ll) << "depdb dep: unexpected argument '" << scan.next () - << "'"; + fail (ll) << "depdb pre-dynamic: unexpected argument '" + << scan.next () << "'"; } catch (const cli::exception& e) { - fail (ll) << "depdb dep: " << e; + fail (ll) << "depdb pre-dynamic: " << e; + } + + // This code is based on the prior work in the cc module (specifically + // extract_headers()) where you can often find more detailed rationale + // for some of the steps performed. + + // If things go wrong (and they often do in this area), give the user + // a bit extra context. + // + auto df = make_diag_frame ( + [this, &tgt](const diag_record& dr) + { + if (verb != 0) + dr << info << "while extracting dynamic dependencies for " << tgt; + }); + + // If nothing so far has invalidated the dependency database, then try + // the cached data before running the program. + // + bool cache (!update); + size_t skip_count (0); + + for (bool restart (true); restart; cache = false) + { + restart = false; + + if (cache) + { + // If any, this is always the first run. + // + assert (skip_count == 0); + + // We should always end with a blank line. + // + for (;;) + { + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + restart = true; + break; + } + + if (l->empty ()) // Done, nothing changed. + return; + } + } + else + { + } } optional file; @@ -1277,7 +1349,8 @@ namespace build2 // @@ TODO: improve diagnostics. // if (!file && ce.size () != 1) - fail (ll) << "depdb dep: command cannot contain logical operators"; + fail (ll) << "depdb pre-dynamic: command cannot contain " + << "logical operators"; string s; build2::script::run (*environment_, @@ -1295,7 +1368,7 @@ namespace build2 // Assume file is one of the prerequisites. // if (!file) - fail (ll) << "depdb dep: program or --file expected"; + fail (ll) << "depdb pre-dynamic: program or --file expected"; } } diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx index d2c99d9..f957f82 100644 --- a/libbuild2/build/script/parser.hxx +++ b/libbuild2/build/script/parser.hxx @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -83,27 +82,54 @@ namespace build2 // execution. // void - execute_body (const scope& root, const scope& base, + execute_body (const scope& base, environment&, const script&, runner&, bool enter = true, bool leave = true); + // Execute the first or the second (pre-dynamic) half of the depdb + // preamble. + // // Note that it's the caller's responsibility to make sure that the // runner's enter() function is called before the first preamble/body // command execution and leave() -- after the last command. // void - execute_depdb_preamble (const scope& root, const scope& base, - environment&, const script&, runner&, - depdb&); + execute_depdb_preamble (const scope& base, + environment& e, const script& s, runner& r, + depdb& dd) + { + auto b (s.depdb_preamble.begin ()); + exec_depdb_preamble ( + base, + e, s, r, + b, + (s.depdb_pre_dynamic + ? b + *s.depdb_pre_dynamic + : s.depdb_preamble.end ()), + dd); + } + void + execute_depdb_preamble_dynamic ( + const scope& base, + environment& e, const script& s, runner& r, + depdb& dd, bool& update, timestamp mt) + { + exec_depdb_preamble ( + base, + e, s, r, + s.depdb_preamble.begin () + *s.depdb_pre_dynamic, + s.depdb_preamble.end (), + dd, &update, mt); + } // Parse a special builtin line into names, performing the variable // and pattern expansions. If omit_builtin is true, then omit the // builtin name from the result. // names - execute_special (const scope& root, const scope& base, + execute_special (const scope& base, environment&, const line&, bool omit_builtin = true); @@ -115,16 +141,36 @@ namespace build2 pre_exec (const scope& root, const scope& base, environment&, const script*, runner*); + using lines_iterator = lines::const_iterator; + void - exec_lines (const lines&, const function&); + exec_lines (lines_iterator, lines_iterator, + const function&); + + void + exec_lines (const lines& l, const function& c) + { + exec_lines (l.begin (), l.end (), c); + } names exec_special (token&, build2::script::token_type&, bool skip_first); void - exec_depdb_dep (token&, build2::script::token_type&, - size_t line_index, - const location&); + exec_depdb_preamble (const scope& base, + environment&, const script&, runner&, + lines_iterator begin, lines_iterator end, + depdb&, + bool* update = nullptr, + optional mt = nullopt); + + void + exec_depdb_pre_dynamic (token&, build2::script::token_type&, + size_t line_index, const location&, + const target&, + depdb&, + bool& update, + timestamp); // Helpers. // @@ -223,12 +269,14 @@ namespace build2 // depdb env - Track the environment variables change as a // hash. // - // depdb dep ... - Extract additional dependency information. - // Can only be the last depdb builtin call. + // depdb pre-dynamic ... - Extract dynamic dependency information. + // Can only be the last depdb builtin call + // in the preamble. // - optional depdb_clear_; // depdb-clear location if any. - optional depdb_dep_; // depdb-dep location if any. - lines depdb_preamble_; // Note: excludes depdb-clear. + optional depdb_clear_; // depdb-clear location. + optional> + depdb_pre_dynamic_; // depdb-pre-dynamic location. + lines depdb_preamble_; // Note: excluding depdb-clear. // If present, the first impure function called in the body of the // script that performs update of a file-based target. diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx index e11cb45..bdfbe51 100644 --- a/libbuild2/build/script/script.hxx +++ b/libbuild2/build/script/script.hxx @@ -29,6 +29,10 @@ namespace build2 using build2::script::deadline; using build2::script::timeout; + // Forward declarations. + // + class default_runner; + // Notes: // // - Once parsed, the script can be executed in multiple threads with @@ -70,9 +74,10 @@ namespace build2 // The script's custom dependency change tracking lines (see the // script parser for details). // - bool depdb_clear; - lines_type depdb_preamble; - bool depdb_preamble_temp_dir = false; // True if references $~. + bool depdb_clear; + optional depdb_pre_dynamic; // Position of first pre-dynamic. + lines_type depdb_preamble; + bool depdb_preamble_temp_dir = false; // True if refs $~. location start_loc; location end_loc; diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx index 87fce90..5a3ed95 100644 --- a/libbuild2/cc/compile-rule.cxx +++ b/libbuild2/cc/compile-rule.cxx @@ -775,69 +775,6 @@ namespace build2 } } - // Update the target during the match phase. Return true if it has changed - // or if the passed timestamp is not timestamp_unknown and is older than - // the target. - // - // This function is used to make sure header dependencies are up to date. - // - // There would normally be a lot of headers for every source file (think - // all the system headers) and just calling execute_direct() on all of - // them can get expensive. At the same time, most of these headers are - // existing files that we will never be updating (again, system headers, - // for example) and the rule that will match them is the fallback - // file_rule. That rule has an optimization: it returns noop_recipe (which - // causes the target state to be automatically set to unchanged) if the - // file is known to be up to date. So we do the update "smartly". - // - static bool - update (tracer& trace, action a, const target& t, timestamp ts) - { - const path_target* pt (t.is_a ()); - - if (pt == nullptr) - ts = timestamp_unknown; - - target_state os (t.matched_state (a)); - - if (os == target_state::unchanged) - { - if (ts == timestamp_unknown) - return false; - else - { - // We expect the timestamp to be known (i.e., existing file). - // - timestamp mt (pt->mtime ()); - assert (mt != timestamp_unknown); - return mt > ts; - } - } - else - { - // We only want to return true if our call to execute() actually - // caused an update. In particular, the target could already have been - // in target_state::changed because of a dependency extraction run for - // some other source file. - // - // @@ MT perf: so we are going to switch the phase and execute for - // any generated header. - // - phase_switch ps (t.ctx, run_phase::execute); - target_state ns (execute_direct (a, t)); - - if (ns != os && ns != target_state::unchanged) - { - l6 ([&]{trace << "updated " << t - << "; old state " << os - << "; new state " << ns;}); - return true; - } - else - return ts != timestamp_unknown ? pt->newer (ts, ns) : false; - } - } - recipe compile_rule:: apply (action a, target& xt) const { @@ -1097,6 +1034,8 @@ namespace build2 md.symexport = l ? cast (l) : symexport; } + // NOTE: see similar code in adhoc_buildscript_rule::apply(). + // Make sure the output directory exists. // // Is this the right thing to do? It does smell a bit, but then we do diff --git a/libbuild2/cc/compile-rule.hxx b/libbuild2/cc/compile-rule.hxx index f573968..a211590 100644 --- a/libbuild2/cc/compile-rule.hxx +++ b/libbuild2/cc/compile-rule.hxx @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -37,7 +38,8 @@ namespace build2 }; class LIBBUILD2_CC_SYMEXPORT compile_rule: public simple_rule, - virtual common + virtual common, + dyndep_rule { public: compile_rule (data&&); diff --git a/libbuild2/dynamic.cxx b/libbuild2/dynamic.cxx new file mode 100644 index 0000000..6dd439b --- /dev/null +++ b/libbuild2/dynamic.cxx @@ -0,0 +1,76 @@ +// file : libbuild2/dynamic.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include +#include +#include +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + bool + update_during_match (tracer& trace, action a, const target& t, timestamp ts) + { + // In particular, this function is used to make sure header dependencies + // are up to date. + // + // There would normally be a lot of headers for every source file (think + // all the system headers) and just calling execute_direct() on all of + // them can get expensive. At the same time, most of these headers are + // existing files that we will never be updating (again, system headers, + // for example) and the rule that will match them is the fallback + // file_rule. That rule has an optimization: it returns noop_recipe (which + // causes the target state to be automatically set to unchanged) if the + // file is known to be up to date. So we do the update "smartly". + // + const path_target* pt (t.is_a ()); + + if (pt == nullptr) + ts = timestamp_unknown; + + target_state os (t.matched_state (a)); + + if (os == target_state::unchanged) + { + if (ts == timestamp_unknown) + return false; + else + { + // We expect the timestamp to be known (i.e., existing file). + // + timestamp mt (pt->mtime ()); + assert (mt != timestamp_unknown); + return mt > ts; + } + } + else + { + // We only want to return true if our call to execute() actually caused + // an update. In particular, the target could already have been in + // target_state::changed because of the dynamic dependency extraction + // run for some other target. + // + // @@ MT perf: so we are going to switch the phase and execute for + // any generated header. + // + phase_switch ps (t.ctx, run_phase::execute); + target_state ns (execute_direct (a, t)); + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "updated " << t + << "; old state " << os + << "; new state " << ns;}); + return true; + } + else + return ts != timestamp_unknown ? pt->newer (ts, ns) : false; + } + } +} diff --git a/libbuild2/dynamic.hxx b/libbuild2/dynamic.hxx new file mode 100644 index 0000000..395a839 --- /dev/null +++ b/libbuild2/dynamic.hxx @@ -0,0 +1,33 @@ +// file : libbuild2/dynamic.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_DYNAMIC_HXX +#define LIBBUILD2_DYNAMIC_HXX + +#include +#include +#include + +#include +#include + +#include + +// Additional functionality that is normally only useful for implementing +// rules with dynamic dependencies. +// +namespace build2 +{ + class LIBBUILD2_SYMEXPORT dyndep_rule + { + public: + // Update the target during the match phase. Return true if it has changed + // or if the passed timestamp is not timestamp_unknown and is older than + // the target. + // + static bool + update (tracer&, action, const target&, timestamp); + }; +} + +#endif // LIBBUILD2_DYNAMIC_HXX -- cgit v1.1