diff options
Diffstat (limited to 'libbuild2/operation.cxx')
| -rw-r--r-- | libbuild2/operation.cxx | 579 |
1 files changed, 500 insertions, 79 deletions
diff --git a/libbuild2/operation.cxx b/libbuild2/operation.cxx index 9fbf870..6f88e38 100644 --- a/libbuild2/operation.cxx +++ b/libbuild2/operation.cxx @@ -18,6 +18,10 @@ #include <libbuild2/algorithm.hxx> #include <libbuild2/diagnostics.hxx> +#if 0 +#include <libbuild2/adhoc-rule-buildscript.hxx> // @@ For a hack below. +#endif + using namespace std; using namespace butl; @@ -60,7 +64,7 @@ namespace build2 true, // bootstrap_outer nullptr, // meta-operation pre nullptr, // operation pre - &load, + &perform_load, nullptr, // search nullptr, // match nullptr, // execute @@ -72,16 +76,17 @@ namespace build2 // perform // void - load (const values&, - scope& root, - const path& bf, - const dir_path& out_base, - const dir_path& src_base, - const location&) + perform_load (const values&, + scope& root, + const path& bf, + const dir_path& out_base, + const dir_path& src_base, + const location&) { // Load project's root.build. // - load_root (root); + if (!root.root_extra->loaded) + load_root (root); // Create the base scope. Note that its existence doesn't mean it was // already setup as a base scope; it can be the same as root. @@ -96,15 +101,15 @@ namespace build2 } void - search (const values&, - const scope&, - const scope& bs, - const path& bf, - const target_key& tk, - const location& l, - action_targets& ts) + perform_search (const values&, + const scope&, + const scope& bs, + const path& bf, + const target_key& tk, + const location& l, + action_targets& ts) { - tracer trace ("search"); + tracer trace ("perform_search"); context& ctx (bs.ctx); phase_lock pl (ctx, run_phase::match); @@ -159,8 +164,9 @@ namespace build2 // map.reserve (ctx.targets.size () / 2); - bool e (false); + size_t count_matched (ctx.count_matched ()); + bool e (false); for (size_t pass (1); pass != 3; ++pass) { for (const auto& pt: ctx.targets) @@ -175,8 +181,7 @@ namespace build2 // const target::opstate& s (t->state[a]); - if (s.task_count.load (memory_order_relaxed) - ctx.count_base () < - target::offset_matched) + if (s.task_count.load (memory_order_relaxed) < count_matched) continue; // Skip if for some reason the path is not assigned. @@ -226,9 +231,19 @@ namespace build2 } else if (t->decl != target_decl::real) { - dr << info << "target " << *t << " is not explicitly declared " - << "in any buildfile" << - info << "perhaps it is a dynamic dependency?"; + if (t->decl == target_decl::implied) + { + dr << info << "target " << *t << " is implied by a buildfile"; + } + else + { + dr << info << "target " << *t << " is not declared in a buildfile"; + + if (t->decl == target_decl::prereq_file) + dr << " but has corresponding existing file"; + + dr << info << "perhaps it is a dynamic dependency?"; + } } } } @@ -238,9 +253,10 @@ namespace build2 } void - match (const values&, action a, action_targets& ts, uint16_t diag, bool prog) + perform_match (const values&, action a, action_targets& ts, + uint16_t diag, bool prog) { - tracer trace ("match"); + tracer trace ("perform_match"); if (ts.empty ()) return; @@ -266,10 +282,12 @@ namespace build2 // the up-to-date check on some projects (e.g., Boost). So we jump // through a few hoops to make sure we don't overindulge. // - md.incr = stderr_term ? 5 : 100; // Scale depending on output type. + md.incr = stderr_term // Scale depending on output type. + ? (ctx.sched->serial () ? 1 : 5) + : 100; md.what = " targets to " + diag_do (ctx, a); - mg = ctx.sched.monitor ( + mg = ctx.sched->monitor ( ctx.target_count, md.incr, [&md] (size_t c) -> size_t @@ -301,6 +319,7 @@ namespace build2 // many we have started. Wait with unlocked phase to allow phase // switching. // + bool fail (false); size_t i (0), n (ts.size ()); { atomic_count task_count (0); @@ -311,21 +330,85 @@ namespace build2 const target& t (ts[i].as<target> ()); l5 ([&]{trace << diag_doing (a, t);}); - target_state s (match_async (a, t, 0, task_count, false)); + target_state s (match_async (a, t, + 0, task_count, + match_extra::all_options, + false /* fail */)); // Bail out if the target has failed and we weren't instructed to // keep going. // - if (s == target_state::failed && !ctx.keep_going) + if (s == target_state::failed) { - ++i; - break; + fail = true; + + if (!ctx.keep_going) + { + ++i; + break; + } } } wg.wait (); } + // If we have any targets with post hoc prerequisites, match those. + // + // See match_posthoc() for the overall approach description. + // + bool posthoc_fail (false); + if (!ctx.current_posthoc_targets.empty () && (!fail || ctx.keep_going)) + { + using posthoc_target = context::posthoc_target; + using posthoc_prerequisite_target = posthoc_target::prerequisite_target; + + // Note that on each iteration we may end up with new entries at the + // back. Since we start and end each iteration in serial execution, we + // don't need to mess with the mutex. + // + for (const posthoc_target& p: ctx.current_posthoc_targets) + { + action a (p.action); // May not be the same as argument action. + const target& t (p.target); + + auto df = make_diag_frame ( + [a, &t](const diag_record& dr) + { + if (verb != 0) + dr << info << "while matching to " << diag_do (t.ctx, a) + << " post hoc prerequisites of " << t; + }); + + // Cannot use normal match because incrementing dependency counts in + // the face of cycles does not work well (we will deadlock for the + // reverse execution mode). + // + // @@ PERF: match in parallel (need match_direct_async(), etc). + // + for (const posthoc_prerequisite_target& pt: p.prerequisite_targets) + { + if (pt.target != nullptr) + { + target_state s (match_direct_sync (a, *pt.target, + pt.match_options, + false /* fail */)); + + if (s == target_state::failed) + { + posthoc_fail = true; + + if (!ctx.keep_going) + break; + } + } + } + + if (posthoc_fail && !ctx.keep_going) + break; + } + } + // Clear the progress if present. // if (mg) @@ -336,15 +419,25 @@ namespace build2 // We are now running serially. Re-examine targets that we have matched. // - bool fail (false); for (size_t j (0); j != n; ++j) { action_target& at (ts[j]); const target& t (at.as<target> ()); - target_state s (j < i - ? match_complete (a, t, false) - : target_state::postponed); + // We cannot attribute post hoc failures to specific targets so it + // seems the best we can do is just fail them all. + // + target_state s; + if (j < i) + { + s = match_complete (a, t, match_extra::all_options, false /* fail */); + + if (posthoc_fail) + s = /*t.state[a].state =*/ target_state::failed; + } + else + s = target_state::postponed; + switch (s) { case target_state::postponed: @@ -395,16 +488,103 @@ namespace build2 } void - execute (const values&, action a, action_targets& ts, - uint16_t diag, bool prog) + perform_execute (const values&, action a, action_targets& ts, + uint16_t diag, bool prog) { - tracer trace ("execute"); + tracer trace ("perform_execute"); if (ts.empty ()) return; context& ctx (ts[0].as<target> ().ctx); + bool posthoc_fail (false); + auto execute_posthoc = [&ctx, &posthoc_fail] () + { + using posthoc_target = context::posthoc_target; + using posthoc_prerequisite_target = posthoc_target::prerequisite_target; + + for (const posthoc_target& p: ctx.current_posthoc_targets) + { + action a (p.action); // May not be the same as argument action. + const target& t (p.target); + + auto df = make_diag_frame ( + [a, &t](const diag_record& dr) + { + if (verb != 0) + dr << info << "while " << diag_doing (t.ctx, a) + << " post hoc prerequisites of " << t; + }); + +#if 0 + for (const posthoc_prerequisite_target& pt: p.prerequisite_targets) + { + if (pt.target != nullptr) + { + target_state s ( + execute_direct_sync (a, *pt.target, false /* fail */)); + + if (s == target_state::failed) + { + posthoc_fail = true; + + if (!ctx.keep_going) + break; + } + } + } +#else + // Note: similar logic/reasoning to below except we use direct + // execution. + // + atomic_count tc (0); + wait_guard wg (ctx, tc); + + for (const posthoc_prerequisite_target& pt: p.prerequisite_targets) + { + if (pt.target != nullptr) + { + target_state s ( + execute_direct_async (a, *pt.target, 0, tc, false /*fail*/)); + + if (s == target_state::failed) + { + posthoc_fail = true; + + if (!ctx.keep_going) + break; + } + } + } + + wg.wait (); + + // Process the result. + // + for (const posthoc_prerequisite_target& pt: p.prerequisite_targets) + { + if (pt.target != nullptr) + { + // Similar to below, no need to wait. + // + target_state s (pt.target->executed_state (a, false /* fail */)); + + if (s == target_state::failed) + { + // Note: no need to keep going. + // + posthoc_fail = true; + break; + } + } + } +#endif + if (posthoc_fail && !ctx.keep_going) + break; + } + }; + // Reverse the order of targets if the execution mode is 'last'. // if (ctx.current_mode == execution_mode::last) @@ -412,6 +592,7 @@ namespace build2 phase_lock pl (ctx, run_phase::execute); // Never switched. + bool fail (false); { // Tune the scheduler. // @@ -420,7 +601,7 @@ namespace build2 switch (ctx.current_inner_oif->concurrency) { - case 0: sched_tune = tune_guard (ctx.sched, 1); break; // Run serially. + case 0: sched_tune = tune_guard (*ctx.sched, 1); break; // Run serially. case 1: break; // Run as is. default: assert (false); // Not supported. } @@ -443,7 +624,7 @@ namespace build2 { what = "% of targets " + diag_did (ctx, a); - mg = ctx.sched.monitor ( + mg = ctx.sched->monitor ( ctx.target_count, init - incr, [init, incr, &what, &ctx] (size_t c) -> size_t @@ -468,9 +649,18 @@ namespace build2 } } + // In the 'last' execution mode run post hoc first. + // + if (ctx.current_mode == execution_mode::last) + { + if (!ctx.current_posthoc_targets.empty ()) + execute_posthoc (); + } + // Similar logic to execute_members(): first start asynchronous // execution of all the top-level targets. // + if (!posthoc_fail || ctx.keep_going) { atomic_count task_count (0); wait_guard wg (ctx, task_count); @@ -486,13 +676,24 @@ namespace build2 // Bail out if the target has failed and we weren't instructed to // keep going. // - if (s == target_state::failed && !ctx.keep_going) - break; + if (s == target_state::failed) + { + fail = true; + + if (!ctx.keep_going) + break; + } } wg.wait (); } + if (ctx.current_mode == execution_mode::first) + { + if (!ctx.current_posthoc_targets.empty () && (!fail || ctx.keep_going)) + execute_posthoc (); + } + // We are now running serially. // @@ -528,15 +729,24 @@ namespace build2 // Re-examine all the targets and print diagnostics. // - bool fail (false); for (action_target& at: ts) { const target& t (at.as<target> ()); - // Note that here we call executed_state() directly instead of - // execute_complete() since we know there is no need to wait. + // Similar to match we cannot attribute post hoc failures to specific + // targets so it seems the best we can do is just fail them all. // - switch ((at.state = t.executed_state (a, false))) + if (!posthoc_fail) + { + // Note that here we call executed_state() directly instead of + // execute_complete() since we know there is no need to wait. + // + at.state = t.executed_state (a, false /* fail */); + } + else + at.state = /*t.state[a].state =*/ target_state::failed; + + switch (at.state) { case target_state::unknown: { @@ -581,26 +791,191 @@ namespace build2 if (fail) throw failed (); - // We should have executed every target that we matched, provided we +#ifndef NDEBUG + size_t base (ctx.count_base ()); + + // For now we disable these checks if we've performed any group member + // resolutions that required a match (with apply()) but not execute. + // + if (ctx.target_count.load (memory_order_relaxed) != 0 && + ctx.resolve_count.load (memory_order_relaxed) != 0) + { + // These counts are only tracked for the inner operation. + // + action ia (a.outer () ? a.inner_action () : a); + + // While it may seem that just decrementing the counters for every + // target with the resolve_counted flag set should be enough, this will + // miss any prerequisites that this target has matched but did not + // execute, which may affect both task_count and dependency_count. Note + // that this applies recursively and we effectively need to pretend to + // execute this target and all its prerequisites, recursively without + // actually executing any of their recepies. + // + // That last bit means we must be able to interpret the populated + // prerequisite_targets generically, which is a requirement we place on + // rules that resolve groups in apply (see target::group_members() for + // details). It so happens that our own adhoc_buildscript_rule doesn't + // follow this rule (see execute_update_prerequisites()) so we detect + // and handle this with a hack. + // + // @@ Hm, but there is no guarantee that this holds recursively since + // prerequisites may not be see-through groups. For this to work we + // would have to impose this restriction globally. Which we could + // probably do, just need to audit things carefully (especially + // cc::link_rule). But we already sort of rely on that for dump! Maybe + // should just require it everywhere and fix adhoc_buildscript_rule. + // + // @@ There are special recipes that don't populate prerequisite_targets + // like group_recipe! Are we banning any user-defined such recipes? + // Need to actually look if we have anything else like this. There + // is also inner_recipe, though doesn't apply here (only for outer). + // + // @@ TMP: do and enable after the 0.16.0 release. + // + // Note: recursive lambda. + // +#if 0 + auto pretend_execute = [base, ia] (target& t, + const auto& pretend_execute) -> void + { + context& ctx (t.ctx); + + // Note: tries to emulate the execute_impl() functions semantics. + // + auto execute_impl = [base, ia, &ctx, &pretend_execute] (target& t) + { + target::opstate& s (t.state[ia]); + + size_t gd (ctx.dependency_count.fetch_sub (1, memory_order_relaxed)); + size_t td (s.dependents.fetch_sub (1, memory_order_release)); + assert (td != 0 && gd != 0); + + // Execute unless already executed. + // + if (s.task_count.load (memory_order_relaxed) != + base + target::offset_executed) + pretend_execute (t, pretend_execute); + }; + + target::opstate& s (t.state[ia]); + + if (s.state != target_state::unchanged) // Noop recipe. + { + if (s.recipe_group_action) + { + execute_impl (const_cast<target&> (*t.group)); + } + else + { + // @@ Special hack for adhoc_buildscript_rule (remember to drop + // include above if getting rid of). + // + bool adhoc ( + ia == perform_update_id && + s.rule != nullptr && + dynamic_cast<const adhoc_buildscript_rule*> ( + &s.rule->second.get ()) != nullptr); + + for (const prerequisite_target& p: t.prerequisite_targets[ia]) + { + const target* pt; + + if (adhoc) + pt = (p.target != nullptr ? p.target : + p.adhoc () ? reinterpret_cast<target*> (p.data) : + nullptr); + else + pt = p.target; + + if (pt != nullptr) + execute_impl (const_cast<target&> (*pt)); + } + + ctx.target_count.fetch_sub (1, memory_order_relaxed); + if (s.resolve_counted) + { + s.resolve_counted = false; + ctx.resolve_count.fetch_sub (1, memory_order_relaxed); + } + } + + s.state = target_state::changed; + } + + s.task_count.store (base + target::offset_executed, + memory_order_relaxed); + }; +#endif + + for (const auto& pt: ctx.targets) + { + target& t (*pt); + target::opstate& s (t.state[ia]); + + // We are only interested in the targets that have been matched for + // this operation and are in the applied state. + // + if (s.task_count.load (memory_order_relaxed) != + base + target::offset_applied) + continue; + + if (s.resolve_counted) + { +#if 0 + pretend_execute (t, pretend_execute); + + if (ctx.resolve_count.load (memory_order_relaxed) == 0) + break; +#else + return; // Skip all the below checks. +#endif + } + } + } + + // We should have executed every target that we have matched, provided we // haven't failed (in which case we could have bailed out early). // assert (ctx.target_count.load (memory_order_relaxed) == 0); + assert (ctx.resolve_count.load (memory_order_relaxed) == 0); // Sanity check. -#ifndef NDEBUG if (ctx.dependency_count.load (memory_order_relaxed) != 0) { + auto dependents = [base] (action a, const target& t) + { + const target::opstate& s (t.state[a]); + + // Only consider targets that have been matched for this operation + // (since matching is what causes the dependents count reset). + // + size_t c (s.task_count.load (memory_order_relaxed)); + + return (c >= base + target::offset_applied + ? s.dependents.load (memory_order_relaxed) + : 0); + }; + diag_record dr; dr << info << "detected unexecuted matched targets:"; for (const auto& pt: ctx.targets) { const target& t (*pt); - if (size_t n = t[a].dependents.load (memory_order_relaxed)) + + if (size_t n = dependents (a, t)) dr << text << t << ' ' << n; + + if (a.outer ()) + { + if (size_t n = dependents (a.inner_action (), t)) + dr << text << t << ' ' << n; + } } } -#endif + assert (ctx.dependency_count.load (memory_order_relaxed) == 0); +#endif } const meta_operation_info mo_perform { @@ -613,10 +988,10 @@ namespace build2 true, // bootstrap_outer nullptr, // meta-operation pre nullptr, // operation pre - &load, - &search, - &match, - &execute, + &perform_load, + &perform_search, + &perform_match, + &perform_execute, nullptr, // operation post nullptr, // meta-operation post nullptr // include @@ -627,31 +1002,66 @@ namespace build2 // Note: similar approach to forward() in configure. // - static bool - info_json (const values& params, - const char* mo = nullptr, - const location& l = location ()) + struct info_params { + bool json = false; + bool subprojects = true; + }; + + // Note: should not fail if mo is NULL (see info_subprojects() below). + // + static info_params + info_parse_params (const values& params, + const char* mo = nullptr, + const location& l = location ()) + { + info_params r; + if (params.size () == 1) { - const names& ns (cast<names> (params[0])); + for (const name& n: cast<names> (params[0])) + { + if (n.simple ()) + { + if (n.value == "json") + { + r.json = true; + continue; + } + + if (n.value == "no_subprojects") + { + r.subprojects = false; + continue; + } + + // Fall through. + } - if (ns.size () == 1 && ns[0].simple () && ns[0].value == "json") - return true; - else if (!ns.empty ()) - fail (l) << "unexpected parameter '" << ns << "' for " - << "meta-operation " << mo; + if (mo != nullptr) + fail (l) << "unexpected parameter '" << n << "' for " + << "meta-operation " << mo; + } } else if (!params.empty ()) + { + if (mo != nullptr) fail (l) << "unexpected parameters for meta-operation " << mo; + } - return false; + return r; + } + + bool + info_subprojects (const values& params) + { + return info_parse_params (params).subprojects; } static void info_pre (context&, const values& params, const location& l) { - info_json (params, "info", l); // Validate. + info_parse_params (params, "info", l); // Validate. } static operation_id @@ -704,7 +1114,7 @@ namespace build2 } static void - info_execute_lines (action_targets& ts) + info_execute_lines (action_targets& ts, bool subp) { for (size_t i (0); i != ts.size (); ++i) { @@ -737,7 +1147,7 @@ namespace build2 // auto print_mods = [&rs] () { - for (const module_state& ms: rs.root_extra->modules) + for (const module_state& ms: rs.root_extra->loaded_modules) cout << ' ' << ms.name; }; @@ -778,8 +1188,13 @@ namespace build2 << "url:" ; print_empty (cast_empty<string> (rs[ctx.var_project_url])); cout << endl << "src_root:" ; print_dir (cast<dir_path> (rs[ctx.var_src_root])); cout << endl << "out_root:" ; print_dir (cast<dir_path> (rs[ctx.var_out_root])); cout << endl - << "amalgamation:" ; print_pdir (*rs.root_extra->amalgamation); cout << endl - << "subprojects:" ; print_null (*rs.root_extra->subprojects); cout << endl + << "amalgamation:" ; print_pdir (*rs.root_extra->amalgamation); cout << endl; + if (subp) + { + cout + << "subprojects:" ; print_null (*rs.root_extra->subprojects); cout << endl; + } + cout << "operations:" ; print_ops (rs.root_extra->operations, ctx.operation_table); cout << endl << "meta-operations:"; print_ops (rs.root_extra->meta_operations, ctx.meta_operation_table); cout << endl << "modules:" ; print_mods (); cout << endl; @@ -788,7 +1203,7 @@ namespace build2 #ifndef BUILD2_BOOTSTRAP static void - info_execute_json (action_targets& ts) + info_execute_json (action_targets& ts, bool subp) { json::stream_serializer s (cout); s.begin_array (); @@ -859,6 +1274,7 @@ namespace build2 // Print subprojects. // + if (subp) { const subprojects* sps (*rs.root_extra->subprojects); @@ -902,12 +1318,12 @@ namespace build2 // Print modules. // - if (!rs.root_extra->modules.empty ()) + if (!rs.root_extra->loaded_modules.empty ()) { s.member_name ("modules", false /* check */); s.begin_array (); - for (const module_state& ms: rs.root_extra->modules) + for (const module_state& ms: rs.root_extra->loaded_modules) s.value (ms.name, false /* check */); s.end_array (); @@ -921,7 +1337,7 @@ namespace build2 } #else static void - info_execute_json (action_targets&) + info_execute_json (action_targets&, bool) { } #endif //BUILD2_BOOTSTRAP @@ -933,14 +1349,16 @@ namespace build2 uint16_t, bool) { + info_params ip (info_parse_params (params)); + // Note that both outputs will not be "ideal" if the user does something // like `b info(foo/) info(bar/)` instead of `b info(foo/ bar/)`. Oh, // well. // - if (info_json (params)) - info_execute_json (ts); + if (ip.json) + info_execute_json (ts, ip.subprojects); else - info_execute_lines (ts); + info_execute_lines (ts, ip.subprojects); } const meta_operation_info mo_info { @@ -968,7 +1386,6 @@ namespace build2 default_id, 0, "<default>", - nullptr, "", "", "", @@ -978,6 +1395,8 @@ namespace build2 nullptr, nullptr, nullptr, + nullptr, + nullptr, nullptr }; @@ -996,7 +1415,6 @@ namespace build2 0, "update", "update", - "update", "updating", "updated", "is up to date", @@ -1005,6 +1423,8 @@ namespace build2 nullptr, nullptr, nullptr, + nullptr, + nullptr, nullptr }; @@ -1013,7 +1433,6 @@ namespace build2 0, "clean", "clean", - "clean", "cleaning", "cleaned", "is clean", @@ -1022,6 +1441,8 @@ namespace build2 nullptr, nullptr, nullptr, + nullptr, + nullptr, nullptr }; } |