From b236b111e52d08245d9bc1caadd6b78f7723f42c Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 4 Jan 2022 14:56:56 +0200 Subject: Add depdb-dyndep --update-{include,exclude} options These options specify prerequisite targets/patterns to include/exclude (from the static prerequisite set) for update during match as part of dynamic dependency extraction (those excluded will be updated during execute). For example: depdb dyndep ... --update-exclude libue{hello-meta} ... depdb dyndep ... --update-exclude libue{*} ... depdb dyndep ... --update-include $moc --update-include hxx{*} ... The order in which these options are specified is significant with the first target/pattern that matches determining the result. If only the --update-include options are specified, then only the explicitly included prerequisites will be updated. Otherwise, all prerequisites that are not explicitly excluded will be updated. If none of these options is specified, then all the static prerequisites are updated during match. Note also that these options do not apply to ad hoc prerequisites which are always updated during match. --- libbuild2/adhoc-rule-buildscript.cxx | 290 +++++++++++++++++++---------------- 1 file changed, 155 insertions(+), 135 deletions(-) (limited to 'libbuild2/adhoc-rule-buildscript.cxx') diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx index e0e69dc..f4f3af9 100644 --- a/libbuild2/adhoc-rule-buildscript.cxx +++ b/libbuild2/adhoc-rule-buildscript.cxx @@ -391,8 +391,8 @@ namespace build2 // Because the depdb preamble can access $<, we have to blank out all the // ad hoc prerequisites. Since we will still need them later, we "move" - // them to the auxiliary data member in prerequisite_target (which also - // means we cannot use the standard execute_prerequisites()). + // them to the auxiliary data member in prerequisite_target (see + // execute_update_prerequisites() for details). // auto& pts (t.prerequisite_targets[a]); for (prerequisite_target& p: pts) @@ -426,7 +426,7 @@ namespace build2 { if (const target* pt = (p.target != nullptr ? p.target : - p.data != 0 ? reinterpret_cast (p.data) : + p.adhoc ? reinterpret_cast (p.data) : nullptr)) { hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage); @@ -504,46 +504,32 @@ namespace build2 if (update) mt = timestamp_nonexistent; - // Update our prerequisite targets. While strictly speaking we only need - // to update those that are referenced by depdb-dyndep, communicating - // this is both tedious and error-prone. So we update them all. - // - for (const prerequisite_target& p: pts) - { - if (const target* pt = - (p.target != nullptr ? p.target : - p.data != 0 ? reinterpret_cast (p.data) : nullptr)) - { - update = dyndep_rule::update ( - trace, a, *pt, update ? timestamp_unknown : mt) || update; - } - } - if (script.depdb_dyndep_byproduct) { // If we have the dynamic dependency information as byproduct of the // recipe body, then do the first part: verify the entries in depdb // unless we are already updating. Essentially, this is the `if(cache)` // equivalent of the restart loop in exec_depdb_dyndep(). - // - // Do we really need to update our prerequisite targets in this case (as - // we do above)? While it may seem like we should be able to avoid it by - // triggering update on encountering any non-existent files in depbd, we - // may actually incorrectly "validate" some number of depdb entires - // while having an out-of-date main source file. We could probably avoid - // the update if we are already updating. using dyndep = dyndep_rule; - // Extract the depdb-dyndep command's information (we may also execute - // some variable assignments). + // Update our prerequisite targets and extract the depdb-dyndep + // command's information (we may also execute some variable + // assignments). + // + // Do we really need to update our prerequisite targets in this case? + // While it may seem like we should be able to avoid it by triggering + // update on encountering any non-existent files in depbd, we may + // actually incorrectly "validate" some number of depdb entires while + // having an out-of-date main source file. We could probably avoid the + // update if we are already updating. // { build::script::parser p (ctx); mdb->byp = p.execute_depdb_preamble_dyndep_byproduct ( a, bs, t, env, script, run, - dd); + dd, update, mt); } mdb->pts_n = pts.size (); @@ -582,10 +568,16 @@ namespace build2 fp, true /* cache */, true /* normalized */, map_ext, *byp.default_type).first) { + // Note: mark the injected prerequisite target as updated (see + // execute_update_prerequisites() for details). + // if (optional u = dyndep::inject_existing_file ( trace, what, a, t, - *ft, mt, false /* fail */)) + *ft, mt, + false /* fail */, + false /* adhoc */, + 1 /* data */)) { skip_count++; return *u; @@ -671,8 +663,8 @@ namespace build2 } else { - // Run the second half of the preamble (depdb-dyndep commands) to - // extract dynamic dependencies. + // Run the second half of the preamble (depdb-dyndep commands) to update + // our prerequisite targets and extract dynamic dependencies. // // Note that this should be the last update to depdb (the invalidation // order semantics). @@ -684,8 +676,8 @@ namespace build2 env, script, run, dd, update, - deferred_failure, - mt); + mt, + deferred_failure); } if (update && dd.reading () && !ctx.dry_run) @@ -733,21 +725,13 @@ namespace build2 const file& t (xt.as ()); - // While we've updated all our prerequisites in apply(), we still need to - // execute them here to keep the dependency counts straight. + // Note that even if we've updated all our prerequisites in apply(), we + // still need to execute them here to keep the dependency counts straight. // - const auto& pts (t.prerequisite_targets[a]); + optional ps (execute_update_prerequisites (a, t, md.mt)); - for (const prerequisite_target& p: pts) - { - if (const target* pt = - (p.target != nullptr ? p.target : - p.data != 0 ? reinterpret_cast (p.data) : nullptr)) - { - target_state ts (execute_wait (a, *pt)); - assert (ts == target_state::unchanged || ts == target_state::changed); - } - } + if (!ps) + md.mt = timestamp_nonexistent; // Update. build::script::environment& env (md.env); build::script::default_runner& run (md.run); @@ -755,7 +739,7 @@ namespace build2 if (md.mt != timestamp_nonexistent) { run.leave (env, script.end_loc); - return target_state::unchanged; + return *ps; } const scope& bs (*md.bs); @@ -805,6 +789,7 @@ namespace build2 // Note that fp is expected to be absolute. // size_t skip (md.skip_count); + const auto& pts (t.prerequisite_targets[a]); auto add = [&trace, what, a, &bs, &t, &pts, pts_n = md.pts_n, @@ -818,7 +803,8 @@ namespace build2 fp, false /* cache */, true /* normalized */, map_ext, *byp.default_type).first) { - // Skip if this is one of the static prerequisites. + // Skip if this is one of the static prerequisites provided it was + // updated. // for (size_t i (0); i != pts_n; ++i) { @@ -826,10 +812,10 @@ namespace build2 if (const target* pt = (p.target != nullptr ? p.target : - p.data != 0 ? reinterpret_cast (p.data) : + p.adhoc ? reinterpret_cast (p.data) : nullptr)) { - if (ft == pt) + if (ft == pt && (p.adhoc || p.data == 1)) return; } } @@ -855,6 +841,9 @@ namespace build2 // Verify it has noop recipe. // + // @@ Currently we will issue an imprecise diagnostics if this is + // a static prerequisite that was not updated (see above). + // dyndep::verify_existing_file (trace, what, a, t, *ft); } @@ -974,19 +963,13 @@ namespace build2 const file& t (xt.as ()); - // While we've updated all our prerequisites in apply(), we still need to - // execute them here to keep the dependency counts straight. + // Note that even if we've updated all our prerequisites in apply(), we + // still need to execute them here to keep the dependency counts straight. // - for (const prerequisite_target& p: t.prerequisite_targets[a]) - { - if (const target* pt = - (p.target != nullptr ? p.target : - p.data != 0 ? reinterpret_cast (p.data) : nullptr)) - { - target_state ts (execute_wait (a, *pt)); - assert (ts == target_state::unchanged || ts == target_state::changed); - } - } + optional ps (execute_update_prerequisites (a, t, md.mt)); + + if (!ps) + md.mt = timestamp_nonexistent; // Update. build::script::environment& env (md.env); build::script::default_runner& run (md.run); @@ -996,7 +979,7 @@ namespace build2 if (md.mt != timestamp_nonexistent && !md.deferred_failure) { run.leave (env, script.end_loc); - return target_state::unchanged; + return *ps; } // Sequence start time for mtime checks below. @@ -1035,86 +1018,30 @@ namespace build2 // out-of-date. // timestamp mt (t.load_mtime ()); - optional ps; - names storage; + // This is essentially ps=execute_prerequisites(a, t, mt) which we + // cannot use because we need to see ad hoc prerequisites. + // + optional ps (execute_update_prerequisites (a, t, mt)); + // Calculate prerequisite checksums (that need to include ad hoc + // prerequisites) unless the script tracks changes itself. + // + names storage; sha256 prq_cs, exe_cs, env_cs; - { - // This is essentially ps=execute_prerequisites(a, t, mt) which we - // cannot use because we need to see ad hoc prerequisites. - // - size_t busy (ctx.count_busy ()); - size_t exec (ctx.count_executed ()); - - target_state rs (target_state::unchanged); - - wait_guard wg (ctx, busy, t[a].task_count); - - auto& pts (t.prerequisite_targets[a]); - - for (const target*& pt: pts) - { - if (pt == nullptr) // Skipped. - continue; - - target_state s (execute_async (a, *pt, busy, t[a].task_count)); - - if (s == target_state::postponed) - { - rs |= s; - pt = nullptr; - } - } - wg.wait (); - - bool e (mt == timestamp_nonexistent); - for (prerequisite_target& p: pts) + if (!script.depdb_clear) + { + for (const prerequisite_target& p: t.prerequisite_targets[a]) { - if (p == nullptr) - continue; - - const target& pt (*p.target); - - ctx.sched.wait (exec, pt[a].task_count, scheduler::work_none); - - target_state s (pt.executed_state (a)); - rs |= s; - - // Compare our timestamp to this prerequisite's. - // - if (!e) + if (const target* pt = + (p.target != nullptr ? p.target : + p.adhoc ? reinterpret_cast (p.data) + : nullptr)) { - // If this is an mtime-based target, then compare timestamps. - // - if (const mtime_target* mpt = pt.is_a ()) - { - if (mpt->newer (mt, s)) - e = true; - } - else - { - // Otherwise we assume the prerequisite is newer if it was - // changed. - // - if (s == target_state::changed) - e = true; - } + hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage); } - - if (p.adhoc) - p.target = nullptr; // Blank out. - - // As part of this loop calculate checksums that need to include ad - // hoc prerequisites (unless the script tracks changes itself). - // - if (!script.depdb_clear) - hash_prerequisite_target (prq_cs, exe_cs, env_cs, pt, storage); } - - if (!e) - ps = rs; } bool update (!ps); @@ -1287,6 +1214,99 @@ namespace build2 return target_state::changed; } + // Update prerequisite targets. + // + // Each prerequisite target should be in one of the following states: + // + // target adhoc data + // -------------------- + // !NULL false 0 - normal prerequisite to be updated + // !NULL false 1 - normal prerequisite already updated + // !NULL true 0 - ad hoc prerequisite to be updated and blanked + // NULL true !NULL - ad hoc prerequisite already updated and blanked + // + // Note that we still execute already updated prerequisites to keep the + // dependency counts straight. But we don't consider them for the "renders + // us out-of-date" check assuming this has already been done. + // + optional adhoc_buildscript_rule:: + execute_update_prerequisites (action a, const target& t, timestamp mt) const + { + context& ctx (t.ctx); + + // This is essentially a customized execute_prerequisites(a, t, mt). + // + size_t busy (ctx.count_busy ()); + size_t exec (ctx.count_executed ()); + + target_state rs (target_state::unchanged); + + wait_guard wg (ctx, busy, t[a].task_count); + + auto& pts (t.prerequisite_targets[a]); + + for (const prerequisite_target& p: pts) + { + if (const target* pt = + (p.target != nullptr ? p.target : + p.adhoc ? reinterpret_cast (p.data) : nullptr)) + { + target_state s (execute_async (a, *pt, busy, t[a].task_count)); + assert (s != target_state::postponed); + } + } + + wg.wait (); + + bool e (mt == timestamp_nonexistent); + for (prerequisite_target& p: pts) + { + if (const target* pt = + (p.target != nullptr ? p.target : + p.adhoc ? reinterpret_cast (p.data) : nullptr)) + { + ctx.sched.wait (exec, (*pt)[a].task_count, scheduler::work_none); + + if (p.data == 0) + { + target_state s (pt->executed_state (a)); + rs |= s; + + // Compare our timestamp to this prerequisite's. + // + if (!e) + { + // If this is an mtime-based target, then compare timestamps. + // + if (const mtime_target* mpt = pt->is_a ()) + { + if (mpt->newer (mt, s)) + e = true; + } + else + { + // Otherwise we assume the prerequisite is newer if it was + // changed. + // + if (s == target_state::changed) + e = true; + } + } + + // Blank out adhoc. + // + if (p.adhoc) + { + p.data = reinterpret_cast (p.target); + p.target = nullptr; + } + } + } + } + + return e ? nullopt : optional (rs); + } + // Return true if execute_body() was called and thus the caller should call // run.leave(). // -- cgit v1.1