From 6cc8301e4fb819393c1245cea0fbfb89e69b90b4 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Fri, 23 Aug 2019 08:02:15 +0200 Subject: Meta/operations and counts --- build2/b.cxx | 27 ++++--- build2/cc/compile-rule.cxx | 6 +- build2/cc/link-rule.cxx | 9 ++- libbuild2/algorithm.cxx | 81 +++++++++++--------- libbuild2/algorithm.ixx | 15 ++-- libbuild2/config/init.cxx | 4 +- libbuild2/config/operation.cxx | 8 +- libbuild2/config/utility.cxx | 12 +-- libbuild2/config/utility.txx | 2 +- libbuild2/context.cxx | 26 ++----- libbuild2/context.hxx | 170 +++++++++++++++++++++++------------------ libbuild2/diagnostics.cxx | 38 ++++----- libbuild2/diagnostics.hxx | 7 +- libbuild2/dist/operation.cxx | 14 ++-- libbuild2/dump.cxx | 2 +- libbuild2/operation.cxx | 26 +++---- libbuild2/prerequisite.cxx | 7 +- libbuild2/rule.cxx | 4 +- libbuild2/target.cxx | 7 +- libbuild2/target.hxx | 12 +-- libbuild2/target.ixx | 9 ++- libbuild2/test/rule.cxx | 4 +- 22 files changed, 258 insertions(+), 232 deletions(-) diff --git a/build2/b.cxx b/build2/b.cxx index 6a49cb7..cb9c2c0 100644 --- a/build2/b.cxx +++ b/build2/b.cxx @@ -107,12 +107,15 @@ namespace build2 if (ops.structured_result ()) { + const target& t (at.as_target ()); + context& ctx (t.ctx); + cout << at.state - << ' ' << current_mif->name - << ' ' << current_inner_oif->name; + << ' ' << ctx.current_mif->name + << ' ' << ctx.current_inner_oif->name; - if (current_outer_oif != nullptr) - cout << '(' << current_outer_oif->name << ')'; + if (ctx.current_outer_oif != nullptr) + cout << '(' << ctx.current_outer_oif->name << ')'; // There are two ways one may wish to identify the target of the // operation: as something specific but inherently non-portable (say, @@ -131,7 +134,7 @@ namespace build2 stream_verbosity sv (stream_verb (cout)); stream_verb (cout, stream_verbosity (1, 0)); - cout << ' ' << at.as_target () << endl; + cout << ' ' << t << endl; stream_verb (cout, sv); } @@ -773,7 +776,7 @@ main (int argc, char* argv[]) values& mparams (lifted == nullptr ? mit->params : lifted->params); string mname (lifted == nullptr ? mit->name : lifted->name); - current_mname = mname; // Set early. + ctx->current_mname = mname; // Set early. if (!mname.empty ()) { @@ -782,7 +785,7 @@ main (int argc, char* argv[]) // Can modify params, opspec, change meta-operation name. // if (auto f = meta_operation_table[m].process) - mname = current_mname = f ( + mname = ctx->current_mname = f ( *ctx, mparams, opspecs, lifted != nullptr, l); } } @@ -802,7 +805,7 @@ main (int argc, char* argv[]) const values& oparams (lifted == nullptr ? os.params : values ()); const string& oname (lifted == nullptr ? os.name : empty_string); - current_oname = oname; // Set early. + ctx->current_oname = oname; // Set early. if (lifted != nullptr) lifted = nullptr; // Clear for the next iteration. @@ -1242,7 +1245,7 @@ main (int argc, char* argv[]) fail (l) << "unexpected parameters for meta-operation " << mif->name; - ctx->current_mif (*mif); + ctx->current_meta_operation (*mif); dirty = true; } @@ -1562,7 +1565,7 @@ main (int argc, char* argv[]) if (mif->operation_pre != nullptr) mif->operation_pre (mparams, pre_oid); // Cannot be translated. - ctx->current_oif (*pre_oif, oif); + ctx->current_operation (*pre_oif, oif); action a (mid, pre_oid, oid); @@ -1589,7 +1592,7 @@ main (int argc, char* argv[]) tgs.reset (); } - ctx->current_oif (*oif, outer_oif); + ctx->current_operation (*oif, outer_oif); action a (mid, oid, oif->outer_id); @@ -1617,7 +1620,7 @@ main (int argc, char* argv[]) if (mif->operation_pre != nullptr) mif->operation_pre (mparams, post_oid); // Cannot be translated. - ctx->current_oif (*post_oif, oif); + ctx->current_operation (*post_oif, oif); action a (mid, post_oid, oid); diff --git a/build2/cc/compile-rule.cxx b/build2/cc/compile-rule.cxx index f600f76..6902bb6 100644 --- a/build2/cc/compile-rule.cxx +++ b/build2/cc/compile-rule.cxx @@ -698,7 +698,7 @@ namespace build2 // Start asynchronous matching of prerequisites. Wait with unlocked // phase to allow phase switching. // - wait_guard wg (t.ctx, target::count_busy (), t[a].task_count, true); + wait_guard wg (t.ctx, t.ctx.count_busy (), t[a].task_count, true); size_t start (pts.size ()); // Index of the first to be added. for (prerequisite_member p: group_prerequisite_members (a, t)) @@ -760,7 +760,7 @@ namespace build2 continue; } - match_async (a, *pt, target::count_busy (), t[a].task_count); + match_async (a, *pt, t.ctx.count_busy (), t[a].task_count); pts.push_back (prerequisite_target (pt, pi)); } @@ -5585,7 +5585,7 @@ namespace build2 { touch (tp, false, 2); t.mtime (system_clock::now ()); - skip_count.fetch_add (1, memory_order_relaxed); + t.ctx.skip_count.fetch_add (1, memory_order_relaxed); } // Note: else mtime should be cached. diff --git a/build2/cc/link-rule.cxx b/build2/cc/link-rule.cxx index 57772a9..3d99535 100644 --- a/build2/cc/link-rule.cxx +++ b/build2/cc/link-rule.cxx @@ -484,6 +484,7 @@ namespace build2 tracer trace (x, "link_rule::apply"); file& t (xt.as ()); + context& ctx (t.ctx); // Note that for_install is signalled by install_rule and therefore // can only be relied upon during execute. @@ -698,7 +699,7 @@ namespace build2 // Note that ad hoc inputs have to be explicitly marked with the // include=adhoc prerequisite-specific variable. // - if (current_outer_oif != nullptr) + if (ctx.current_outer_oif != nullptr) continue; } @@ -1164,7 +1165,7 @@ namespace build2 bool u; if ((u = pt->is_a ()) || pt->is_a ()) { - const variable& var (t.ctx.var_pool["bin.whole"]); // @@ Cache. + const variable& var (ctx.var_pool["bin.whole"]); // @@ Cache. // See the bin module for the lookup semantics discussion. Note // that the variable is not overridable so we omit find_override() @@ -1198,7 +1199,7 @@ namespace build2 // Wait with unlocked phase to allow phase switching. // - wait_guard wg (t.ctx, target::count_busy (), t[a].task_count, true); + wait_guard wg (ctx, ctx.count_busy (), t[a].task_count, true); i = start; for (prerequisite_member p: group_prerequisite_members (a, t)) @@ -1230,7 +1231,7 @@ namespace build2 } } - match_async (a, *pt, target::count_busy (), t[a].task_count); + match_async (a, *pt, ctx.count_busy (), t[a].task_count); mark (pt, m); } diff --git a/libbuild2/algorithm.cxx b/libbuild2/algorithm.cxx index 0af18c8..12cd61f 100644 --- a/libbuild2/algorithm.cxx +++ b/libbuild2/algorithm.cxx @@ -174,7 +174,7 @@ namespace build2 // Most likely the target's state is (count_touched - 1), that is, 0 or // previously executed, so let's start with that. // - size_t b (target::count_base ()); + size_t b (ct.ctx.count_base ()); size_t e (b + target::offset_touched - 1); size_t appl (b + target::offset_applied); @@ -255,7 +255,7 @@ namespace build2 // Set the task count and wake up any threads that might be waiting for // this target. // - task_count.store (offset + target::count_base (), memory_order_release); + task_count.store (offset + t.ctx.count_base (), memory_order_release); sched.resume (task_count); } @@ -800,7 +800,7 @@ namespace build2 // Start asynchronous matching of prerequisites. Wait with unlocked phase // to allow phase switching. // - wait_guard wg (t.ctx, target::count_busy (), t[a].task_count, true); + wait_guard wg (t.ctx, t.ctx.count_busy (), t[a].task_count, true); size_t i (pts.size ()); // Index of the first to be added. for (auto&& p: forward (r)) @@ -819,7 +819,7 @@ namespace build2 if (pt.target == nullptr || (s != nullptr && !pt.target->in (*s))) continue; - match_async (a, *pt.target, target::count_busy (), t[a].task_count); + match_async (a, *pt.target, t.ctx.count_busy (), t[a].task_count); pts.push_back (move (pt)); } @@ -857,7 +857,7 @@ namespace build2 // Pretty much identical to match_prerequisite_range() except we don't // search. // - wait_guard wg (t.ctx, target::count_busy (), t[a].task_count, true); + wait_guard wg (t.ctx, t.ctx.count_busy (), t[a].task_count, true); for (size_t i (0); i != n; ++i) { @@ -866,7 +866,7 @@ namespace build2 if (m == nullptr || marked (m)) continue; - match_async (a, *m, target::count_busy (), t[a].task_count); + match_async (a, *m, t.ctx.count_busy (), t[a].task_count); } wg.wait (); @@ -1524,7 +1524,7 @@ namespace build2 { target::opstate& s (t[a]); - assert (s.task_count.load (memory_order_consume) == target::count_busy () + assert (s.task_count.load (memory_order_consume) == t.ctx.count_busy () && s.state == target_state::unknown); target_state ts; @@ -1569,7 +1569,7 @@ namespace build2 { recipe_function** f (s.recipe.target ()); if (f == nullptr || *f != &group_action) - target_count.fetch_sub (1, memory_order_relaxed); + t.ctx.target_count.fetch_sub (1, memory_order_relaxed); } // Decrement the task count (to count_executed) and wake up any threads @@ -1578,7 +1578,7 @@ namespace build2 size_t tc (s.task_count.fetch_sub ( target::offset_busy - target::offset_executed, memory_order_release)); - assert (tc == target::count_busy ()); + assert (tc == t.ctx.count_busy ()); sched.resume (s.task_count); return ts; @@ -1593,9 +1593,11 @@ namespace build2 target& t (const_cast (ct)); // MT-aware. target::opstate& s (t[a]); + context& ctx (t.ctx); + // Update dependency counts and make sure they are not skew. // - size_t gd (dependency_count.fetch_sub (1, memory_order_relaxed)); + size_t gd (ctx.dependency_count.fetch_sub (1, memory_order_relaxed)); size_t td (s.dependents.fetch_sub (1, memory_order_release)); assert (td != 0 && gd != 0); td--; @@ -1621,15 +1623,15 @@ namespace build2 // thread. For other threads the state will still be unknown (until they // try to execute it). // - if (current_mode == execution_mode::last && td != 0) + if (ctx.current_mode == execution_mode::last && td != 0) return target_state::postponed; // Try to atomically change applied to busy. // - size_t tc (target::count_applied ()); + size_t tc (ctx.count_applied ()); - size_t exec (target::count_executed ()); - size_t busy (target::count_busy ()); + size_t exec (ctx.count_executed ()); + size_t busy (ctx.count_busy ()); if (s.task_count.compare_exchange_strong ( tc, @@ -1690,10 +1692,10 @@ namespace build2 // Similar logic to match() above except we execute synchronously. // - size_t tc (target::count_applied ()); + size_t tc (t.ctx.count_applied ()); - size_t exec (target::count_executed ()); - size_t busy (target::count_busy ()); + size_t exec (t.ctx.count_executed ()); + size_t busy (t.ctx.count_busy ()); if (s.task_count.compare_exchange_strong ( tc, @@ -1748,9 +1750,12 @@ namespace build2 { target_state r (target_state::unchanged); + size_t busy (ctx.count_busy ()); + size_t exec (ctx.count_executed ()); + // Start asynchronous execution of prerequisites. // - wait_guard wg (ctx, target::count_busy (), tc); + wait_guard wg (ctx, busy, tc); n += p; for (size_t i (p); i != n; ++i) @@ -1760,7 +1765,7 @@ namespace build2 if (mt == nullptr) // Skipped. continue; - target_state s (execute_async (a, *mt, target::count_busy (), tc)); + target_state s (execute_async (a, *mt, busy, tc)); if (s == target_state::postponed) { @@ -1785,8 +1790,8 @@ namespace build2 // If the target is still busy, wait for its completion. // const auto& tc (mt[a].task_count); - if (tc.load (memory_order_acquire) >= target::count_busy ()) - sched.wait (target::count_executed (), tc, scheduler::work_none); + if (tc.load (memory_order_acquire) >= busy) + sched.wait (exec, tc, scheduler::work_none); r |= mt.executed_state (a); @@ -1805,7 +1810,10 @@ namespace build2 // target_state r (target_state::unchanged); - wait_guard wg (ctx, target::count_busy (), tc); + size_t busy (ctx.count_busy ()); + size_t exec (ctx.count_executed ()); + + wait_guard wg (ctx, busy, tc); n = p - n; for (size_t i (p); i != n; ) @@ -1815,7 +1823,7 @@ namespace build2 if (mt == nullptr) continue; - target_state s (execute_async (a, *mt, target::count_busy (), tc)); + target_state s (execute_async (a, *mt, busy, tc)); if (s == target_state::postponed) { @@ -1834,8 +1842,8 @@ namespace build2 const target& mt (*ts[i]); const auto& tc (mt[a].task_count); - if (tc.load (memory_order_acquire) >= target::count_busy ()) - sched.wait (target::count_executed (), tc, scheduler::work_none); + if (tc.load (memory_order_acquire) >= busy) + sched.wait (exec, tc, scheduler::work_none); r |= mt.executed_state (a); @@ -1869,7 +1877,12 @@ namespace build2 const timestamp& mt, const execute_filter& ef, size_t n) { - assert (current_mode == execution_mode::first); + context& ctx (t.ctx); + + assert (ctx.current_mode == execution_mode::first); + + size_t busy (ctx.count_busy ()); + size_t exec (ctx.count_executed ()); auto& pts (t.prerequisite_targets[a]); @@ -1880,7 +1893,7 @@ namespace build2 // target_state rs (target_state::unchanged); - wait_guard wg (t.ctx, target::count_busy (), t[a].task_count); + wait_guard wg (ctx, busy, t[a].task_count); for (size_t i (0); i != n; ++i) { @@ -1889,9 +1902,7 @@ namespace build2 if (pt == nullptr) // Skipped. continue; - target_state s ( - execute_async ( - a, *pt, target::count_busy (), t[a].task_count)); + target_state s (execute_async (a, *pt, busy, t[a].task_count)); if (s == target_state::postponed) { @@ -1915,8 +1926,8 @@ namespace build2 const target& pt (*p.target); const auto& tc (pt[a].task_count); - if (tc.load (memory_order_acquire) >= target::count_busy ()) - sched.wait (target::count_executed (), tc, scheduler::work_none); + if (tc.load (memory_order_acquire) >= busy) + sched.wait (exec, tc, scheduler::work_none); target_state s (pt.executed_state (a)); rs |= s; @@ -1980,7 +1991,7 @@ namespace build2 target_state gs (execute (a, g)); if (gs == target_state::busy) - sched.wait (target::count_executed (), + sched.wait (t.ctx.count_executed (), g[a].task_count, scheduler::work_none); @@ -2187,7 +2198,7 @@ namespace build2 // if (tr != target_state::changed && er == target_state::changed) { - if (verb > (current_diag_noise ? 0 : 1) && verb < 3) + if (verb > (ft.ctx.current_diag_noise ? 0 : 1) && verb < 3) { if (ed) text << "rm -r " << path_cast (ep); @@ -2264,7 +2275,7 @@ namespace build2 if (tr != target_state::changed && er == target_state::changed) { - if (verb > (current_diag_noise ? 0 : 1) && verb < 3) + if (verb > (g.ctx.current_diag_noise ? 0 : 1) && verb < 3) text << "rm " << ep; } diff --git a/libbuild2/algorithm.ixx b/libbuild2/algorithm.ixx index b409b7c..f865992 100644 --- a/libbuild2/algorithm.ixx +++ b/libbuild2/algorithm.ixx @@ -275,7 +275,7 @@ namespace build2 inline void match_inc_dependens (action a, const target& t) { - dependency_count.fetch_add (1, memory_order_relaxed); + t.ctx.dependency_count.fetch_add (1, memory_order_relaxed); t[a].dependents.fetch_add (1, memory_order_release); } @@ -368,7 +368,8 @@ namespace build2 inline void set_recipe (target_lock& l, recipe&& r) { - target::opstate& s ((*l.target)[l.action]); + target& t (*l.target); + target::opstate& s (t[l.action]); s.recipe = move (r); @@ -399,7 +400,7 @@ namespace build2 if (l.action.inner ()) { if (f == nullptr || *f != &group_action) - target_count.fetch_add (1, memory_order_relaxed); + t.ctx.target_count.fetch_add (1, memory_order_relaxed); } } } @@ -544,7 +545,7 @@ namespace build2 execute_wait (action a, const target& t) { if (execute (a, t) == target_state::busy) - sched.wait (target::count_executed (), + sched.wait (t.ctx.count_executed (), t[a].task_count, scheduler::work_none); @@ -601,7 +602,7 @@ namespace build2 inline target_state execute_prerequisites (action a, const target& t, size_t c) { - return current_mode == execution_mode::first + return t.ctx.current_mode == execution_mode::first ? straight_execute_prerequisites (a, t, c) : reverse_execute_prerequisites (a, t, c); } @@ -636,7 +637,7 @@ namespace build2 inline target_state execute_prerequisites_inner (action a, const target& t, size_t c) { - return current_mode == execution_mode::first + return t.ctx.current_mode == execution_mode::first ? straight_execute_prerequisites_inner (a, t, c) : reverse_execute_prerequisites_inner (a, t, c); } @@ -694,7 +695,7 @@ namespace build2 inline target_state execute_members (action a, const target& t, const target* ts[], size_t n) { - return current_mode == execution_mode::first + return t.ctx.current_mode == execution_mode::first ? straight_execute_members (a, t, ts, n, 0) : reverse_execute_members (a, t, ts, n, n); } diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx index 9bdfef9..6998017 100644 --- a/libbuild2/config/init.cxx +++ b/libbuild2/config/init.cxx @@ -29,8 +29,8 @@ namespace build2 l5 ([&]{trace << "for " << rs;}); - const string& mname (current_mname); - const string& oname (current_oname); + const string& mname (rs.ctx.current_mname); + const string& oname (rs.ctx.current_oname); // Only create the module if we are configuring or creating. This is a // bit tricky since the build2 core may not yet know if this is the diff --git a/libbuild2/config/operation.cxx b/libbuild2/config/operation.cxx index 6abfd33..92c80b8 100644 --- a/libbuild2/config/operation.cxx +++ b/libbuild2/config/operation.cxx @@ -539,6 +539,8 @@ namespace build2 if (rs == nullptr) fail << "out of project target " << t; + context& ctx (t.ctx); + const operations& ops (rs->root_extra->operations); for (operation_id id (default_id + 1); // Skip default_id. @@ -552,9 +554,9 @@ namespace build2 if (oif->id != id) continue; - rs->ctx.current_oif (*oif); + ctx.current_operation (*oif); - phase_lock pl (t.ctx, run_phase::match); + phase_lock pl (ctx, run_phase::match); match (action (configure_id, id), t); } } @@ -916,7 +918,7 @@ namespace build2 fail (l) << "invalid module name: " << e.what (); } - current_oname = empty_string; // Make sure valid. + ctx.current_oname = empty_string; // Make sure valid. // Now handle each target in each operation spec. // diff --git a/libbuild2/config/utility.cxx b/libbuild2/config/utility.cxx index a89fac6..355e896 100644 --- a/libbuild2/config/utility.cxx +++ b/libbuild2/config/utility.cxx @@ -46,7 +46,7 @@ namespace build2 } } - if (l.defined () && current_mif->id == configure_id) + if (l.defined () && r.ctx.current_mif->id == configure_id) save_variable (r, var); return pair (l, n); @@ -55,7 +55,7 @@ namespace build2 lookup optional (scope& r, const variable& var) { - if (current_mif->id == configure_id) + if (r.ctx.current_mif->id == configure_id) save_variable (r, var); auto l (r[var]); @@ -103,7 +103,7 @@ namespace build2 const variable& var ( rs.ctx.var_pool.rw (rs).insert ("config." + n + ".configured")); - if (current_mif->id == configure_id) + if (rs.ctx.current_mif->id == configure_id) save_variable (rs, var); auto l (rs[var]); // Include inherited values. @@ -118,7 +118,7 @@ namespace build2 const variable& var ( rs.ctx.var_pool.rw (rs).insert ("config." + n + ".configured")); - if (current_mif->id == configure_id) + if (rs.ctx.current_mif->id == configure_id) save_variable (rs, var); value& x (rs.assign (var)); @@ -135,7 +135,7 @@ namespace build2 void save_variable (scope& r, const variable& var, uint64_t flags) { - if (current_mif->id != configure_id) + if (r.ctx.current_mif->id != configure_id) return; // The project might not be using the config module. But then how @@ -148,7 +148,7 @@ namespace build2 void save_module (scope& r, const char* name, int prio) { - if (current_mif->id != configure_id) + if (r.ctx.current_mif->id != configure_id) return; if (module* m = r.lookup_module (module::name)) diff --git a/libbuild2/config/utility.txx b/libbuild2/config/utility.txx index 841c408..9c1455f 100644 --- a/libbuild2/config/utility.txx +++ b/libbuild2/config/utility.txx @@ -19,7 +19,7 @@ namespace build2 { // Note: see also omitted() if changing anything here. - if (current_mif->id == configure_id) + if (root.ctx.current_mif->id == configure_id) save_variable (root, var, save_flags); pair org (root.find_original (var)); diff --git a/libbuild2/context.cxx b/libbuild2/context.cxx index 82a2cbb..c5b31ad 100644 --- a/libbuild2/context.cxx +++ b/libbuild2/context.cxx @@ -514,7 +514,7 @@ namespace build2 } void context:: - current_mif (const meta_operation_info& mif) + current_meta_operation (const meta_operation_info& mif) { if (current_mname != mif.name) { @@ -522,14 +522,14 @@ namespace build2 global_scope.rw ().assign (var_build_meta_operation) = mif.name; } - build2::current_mif = &mif; + current_mif = &mif; current_on = 0; // Reset. } void context:: - current_oif (const operation_info& inner_oif, - const operation_info* outer_oif, - bool diag_noise) + current_operation (const operation_info& inner_oif, + const operation_info* outer_oif, + bool diag_noise) { current_oname = (outer_oif == nullptr ? inner_oif : *outer_oif).name; current_inner_oif = &inner_oif; @@ -736,6 +736,8 @@ namespace build2 { phase_lock* pl (phase_lock_instance); + // This is tricky: we might be switching to another context. + // if (pl != nullptr && &pl->ctx == &ctx) assert (pl->phase == phase); else @@ -850,20 +852,6 @@ namespace build2 //text << this_thread::get_id () << " phase restore " << n << " " << o; } - string current_mname; - string current_oname; - - const meta_operation_info* current_mif; - const operation_info* current_inner_oif; - const operation_info* current_outer_oif; - size_t current_on; - execution_mode current_mode; - bool current_diag_noise; - - atomic_count dependency_count; - atomic_count target_count; - atomic_count skip_count; - bool keep_going = false; bool dry_run = false; diff --git a/libbuild2/context.hxx b/libbuild2/context.hxx index 394f600..4680fd1 100644 --- a/libbuild2/context.hxx +++ b/libbuild2/context.hxx @@ -8,9 +8,9 @@ #include #include -// NOTE: this file is included by pretty much every other "data model" header -// (scope, target, variable, etc) so including any of them here is -// probably a non-starter. +// NOTE: this file is included by pretty much every other build state header +// (scope, target, variable, etc) so including any of them here is most +// likely a non-starter. // #include #include @@ -41,31 +41,6 @@ namespace build2 // LIBBUILD2_SYMEXPORT extern scheduler sched; - // A "tri-mutex" that keeps all the threads in one of the three phases. When - // a thread wants to switch a phase, it has to wait for all the other - // threads to do the same (or release their phase locks). The load phase is - // exclusive. - // - // The interleaving match and execute is interesting: during match we read - // the "external state" (e.g., filesystem entries, modifications times, etc) - // and capture it in the "internal state" (our dependency graph). During - // execute we are modifying the external state with controlled modifications - // of the internal state to reflect the changes (e.g., update mtimes). If - // you think about it, it's pretty clear that we cannot safely perform both - // of these actions simultaneously. A good example would be running a code - // generator and header dependency extraction simultaneously: the extraction - // process may pick up headers as they are being generated. As a result, we - // either have everyone treat the external state as read-only or write-only. - // - // There is also one more complication: if we are returning from a load - // phase that has failed, then the build state could be seriously messed up - // (things like scopes not being setup completely, etc). And once we release - // the lock, other threads that are waiting will start relying on this - // messed up state. So a load phase can mark the phase_mutex as failed in - // which case all currently blocked and future lock()/relock() calls return - // false. Note that in this case we still switch to the desired phase. See - // the phase_{lock,switch,unlock} implementations for details. - // class LIBBUILD2_SYMEXPORT run_phase_mutex { public: @@ -167,10 +142,95 @@ namespace build2 // "within the islands". // run_phase phase = run_phase::load; - run_phase_mutex phase_mutex; size_t load_generation = 0; - // Scopes, targets, and variables. + // A "tri-mutex" that keeps all the threads in one of the three phases. + // When a thread wants to switch a phase, it has to wait for all the other + // threads to do the same (or release their phase locks). The load phase + // is exclusive. + // + // The interleaving match and execute is interesting: during match we read + // the "external state" (e.g., filesystem entries, modifications times, + // etc) and capture it in the "internal state" (our dependency graph). + // During execute we are modifying the external state with controlled + // modifications of the internal state to reflect the changes (e.g., + // update mtimes). If you think about it, it's pretty clear that we cannot + // safely perform both of these actions simultaneously. A good example + // would be running a code generator and header dependency extraction + // simultaneously: the extraction process may pick up headers as they are + // being generated. As a result, we either have everyone treat the + // external state as read-only or write-only. + // + // There is also one more complication: if we are returning from a load + // phase that has failed, then the build state could be seriously messed + // up (things like scopes not being setup completely, etc). And once we + // release the lock, other threads that are waiting will start relying on + // this messed up state. So a load phase can mark the phase_mutex as + // failed in which case all currently blocked and future lock()/relock() + // calls return false. Note that in this case we still switch to the + // desired phase. See the phase_{lock,switch,unlock} implementations for + // details. + // + run_phase_mutex phase_mutex; + + // Current action (meta/operation). + // + // The names unlike info are available during boot but may not yet be + // lifted. The name is always for an outer operation (or meta operation + // that hasn't been recognized as such yet). + // + string current_mname; + string current_oname; + + const meta_operation_info* current_mif; + const operation_info* current_inner_oif; + const operation_info* current_outer_oif; + + // Current operation number (1-based) in the meta-operation batch. + // + size_t current_on; + + // Note: we canote use the corresponding target::offeset_* values. + // + size_t count_base () const {return 5 * (current_on - 1);} + + size_t count_touched () const {return 1 + count_base ();} + size_t count_tried () const {return 2 + count_base ();} + size_t count_matched () const {return 3 + count_base ();} + size_t count_applied () const {return 4 + count_base ();} + size_t count_executed () const {return 5 + count_base ();} + size_t count_busy () const {return 6 + count_base ();} + + // Execution mode. + // + execution_mode current_mode; + + // Some diagnostics (for example output directory creation/removal by the + // fsdir rule) is just noise at verbosity level 1 unless it is the only + // thing that is printed. So we can only suppress it in certain situations + // (e.g., dist) where we know we have already printed something. + // + bool current_diag_noise; + + // Total number of dependency relationships and targets with non-noop + // recipe in the current action. + // + // Together with target::dependents the dependency count is incremented + // during the rule search & match phase and is decremented during + // execution with the expectation of it reaching 0. Used as a sanity + // check. + // + // The target count is incremented after a non-noop recipe is matched and + // decremented after such recipe has been executed. If such a recipe has + // skipped executing the operation, then it should increment the skip + // count. These two counters are used for progress monitoring and + // diagnostics. + // + atomic_count dependency_count; + atomic_count target_count; + atomic_count skip_count; + + // Build state (scopes, targets, variables, etc). // const scope_map& scopes; const scope& global_scope; @@ -184,13 +244,15 @@ namespace build2 explicit context (scheduler&, const strings& cmd_vars = {}); + // Set current meta-operation and operation. + // void - current_mif (const meta_operation_info&); + current_meta_operation (const meta_operation_info&); void - current_oif (const operation_info& inner, - const operation_info* outer = nullptr, - bool diag_noise = true); + current_operation (const operation_info& inner, + const operation_info* outer = nullptr, + bool diag_noise = true); context (context&&) = delete; context& operator= (context&&) = delete; @@ -325,47 +387,7 @@ namespace build2 bool phase; }; - // Current action (meta/operation). - // - // The names unlike info are available during boot but may not yet be - // lifted. The name is always for an outer operation (or meta operation - // that hasn't been recognized as such yet). - // - LIBBUILD2_SYMEXPORT extern string current_mname; - LIBBUILD2_SYMEXPORT extern string current_oname; - - LIBBUILD2_SYMEXPORT extern const meta_operation_info* current_mif; - LIBBUILD2_SYMEXPORT extern const operation_info* current_inner_oif; - LIBBUILD2_SYMEXPORT extern const operation_info* current_outer_oif; - - // Current operation number (1-based) in the meta-operation batch. - // - LIBBUILD2_SYMEXPORT extern size_t current_on; - LIBBUILD2_SYMEXPORT extern execution_mode current_mode; - - // Some diagnostics (for example output directory creation/removal by the - // fsdir rule) is just noise at verbosity level 1 unless it is the only - // thing that is printed. So we can only suppress it in certain situations - // (e.g., dist) where we know we have already printed something. - // - LIBBUILD2_SYMEXPORT extern bool current_diag_noise; - - // Total number of dependency relationships and targets with non-noop - // recipe in the current action. - // - // Together with target::dependents the dependency count is incremented - // during the rule search & match phase and is decremented during execution - // with the expectation of it reaching 0. Used as a sanity check. - // - // The target count is incremented after a non-noop recipe is matched and - // decremented after such recipe has been executed. If such a recipe has - // skipped executing the operation, then it should increment the skip count. - // These two counters are used for progress monitoring and diagnostics. - // - LIBBUILD2_SYMEXPORT extern atomic_count dependency_count; - LIBBUILD2_SYMEXPORT extern atomic_count target_count; - LIBBUILD2_SYMEXPORT extern atomic_count skip_count; // Keep going flag. // diff --git a/libbuild2/diagnostics.cxx b/libbuild2/diagnostics.cxx index 3375e00..71f3d48 100644 --- a/libbuild2/diagnostics.cxx +++ b/libbuild2/diagnostics.cxx @@ -141,14 +141,14 @@ namespace build2 const fail_mark fail ("error"); const fail_end endf; - // diag_do(), etc. + // diag_do(), etc. // string - diag_do (const action&) + diag_do (context& ctx, const action&) { - const meta_operation_info& m (*current_mif); - const operation_info& io (*current_inner_oif); - const operation_info* oo (current_outer_oif); + const meta_operation_info& m (*ctx.current_mif); + const operation_info& io (*ctx.current_inner_oif); + const operation_info* oo (ctx.current_outer_oif); string r; @@ -181,15 +181,15 @@ namespace build2 void diag_do (ostream& os, const action& a, const target& t) { - os << diag_do (a) << ' ' << t; + os << diag_do (t.ctx, a) << ' ' << t; } string - diag_doing (const action&) + diag_doing (context& ctx, const action&) { - const meta_operation_info& m (*current_mif); - const operation_info& io (*current_inner_oif); - const operation_info* oo (current_outer_oif); + const meta_operation_info& m (*ctx.current_mif); + const operation_info& io (*ctx.current_inner_oif); + const operation_info* oo (ctx.current_outer_oif); string r; @@ -218,15 +218,15 @@ namespace build2 void diag_doing (ostream& os, const action& a, const target& t) { - os << diag_doing (a) << ' ' << t; + os << diag_doing (t.ctx, a) << ' ' << t; } string - diag_did (const action&) + diag_did (context& ctx, const action&) { - const meta_operation_info& m (*current_mif); - const operation_info& io (*current_inner_oif); - const operation_info* oo (current_outer_oif); + const meta_operation_info& m (*ctx.current_mif); + const operation_info& io (*ctx.current_inner_oif); + const operation_info* oo (ctx.current_outer_oif); string r; @@ -259,15 +259,15 @@ namespace build2 void diag_did (ostream& os, const action& a, const target& t) { - os << diag_did (a) << ' ' << t; + os << diag_did (t.ctx, a) << ' ' << t; } void diag_done (ostream& os, const action&, const target& t) { - const meta_operation_info& m (*current_mif); - const operation_info& io (*current_inner_oif); - const operation_info* oo (current_outer_oif); + const meta_operation_info& m (*t.ctx.current_mif); + const operation_info& io (*t.ctx.current_inner_oif); + const operation_info* oo (t.ctx.current_outer_oif); // perform(update(x)) -> "x is up to date" // configure(update(x)) -> "updating x is configured" diff --git a/libbuild2/diagnostics.hxx b/libbuild2/diagnostics.hxx index dbc8351..5d69132 100644 --- a/libbuild2/diagnostics.hxx +++ b/libbuild2/diagnostics.hxx @@ -439,6 +439,7 @@ namespace build2 // class scope; class target; + class context; struct action; struct diag_phrase @@ -456,7 +457,7 @@ namespace build2 } LIBBUILD2_SYMEXPORT string - diag_do (const action&); + diag_do (context&, const action&); LIBBUILD2_SYMEXPORT void diag_do (ostream&, const action&, const target&); @@ -468,7 +469,7 @@ namespace build2 } LIBBUILD2_SYMEXPORT string - diag_doing (const action&); + diag_doing (context&, const action&); LIBBUILD2_SYMEXPORT void diag_doing (ostream&, const action&, const target&); @@ -480,7 +481,7 @@ namespace build2 } LIBBUILD2_SYMEXPORT string - diag_did (const action&); + diag_did (context&, const action&); LIBBUILD2_SYMEXPORT void diag_did (ostream&, const action&, const target&); diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx index cdeb9ee..6db3626 100644 --- a/libbuild2/dist/operation.cxx +++ b/libbuild2/dist/operation.cxx @@ -169,7 +169,7 @@ namespace build2 if (operation_id pid = oif->pre (params, dist_id, loc)) { const operation_info* poif (ops[pid]); - ctx.current_oif (*poif, oif, false /* diag_noise */); + ctx.current_operation (*poif, oif, false /* diag_noise */); action a (dist_id, poif->id, oif->id); match (params, a, ts, 1 /* diag (failures only) */, @@ -177,7 +177,7 @@ namespace build2 } } - ctx.current_oif (*oif, nullptr, false /* diag_noise */); + ctx.current_operation (*oif, nullptr, false /* diag_noise */); action a (dist_id, oif->id); match (params, a, ts, 1 /* diag (failures only) */, @@ -188,7 +188,7 @@ namespace build2 if (operation_id pid = oif->post (params, dist_id)) { const operation_info* poif (ops[pid]); - ctx.current_oif (*poif, oif, false /* diag_noise */); + ctx.current_operation (*poif, oif, false /* diag_noise */); action a (dist_id, poif->id, oif->id); match (params, a, ts, 1 /* diag (failures only) */, @@ -306,14 +306,14 @@ namespace build2 // // Note also that we don't do any structured result printing. // - size_t on (current_on); - ctx.current_mif (mo_perform); - current_on = on + 1; + size_t on (ctx.current_on); + ctx.current_meta_operation (mo_perform); + ctx.current_on = on + 1; if (mo_perform.operation_pre != nullptr) mo_perform.operation_pre (params, update_id); - ctx.current_oif (op_update, nullptr, false /* diag_noise */); + ctx.current_operation (op_update, nullptr, false /* diag_noise */); action a (perform_id, update_id); diff --git a/libbuild2/dump.cxx b/libbuild2/dump.cxx index 27cd9e2..738ef36 100644 --- a/libbuild2/dump.cxx +++ b/libbuild2/dump.cxx @@ -276,7 +276,7 @@ namespace build2 if (size_t c = t[inner].task_count.load (memory_order_relaxed)) { - if (c == target::count_applied () || c == target::count_executed ()) + if (c == t.ctx.count_applied () || c == t.ctx.count_executed ()) { bool f (false); for (const target* pt: t.prerequisite_targets[inner]) diff --git a/libbuild2/operation.cxx b/libbuild2/operation.cxx index 289b893..19f6e4e 100644 --- a/libbuild2/operation.cxx +++ b/libbuild2/operation.cxx @@ -149,10 +149,10 @@ namespace build2 { size_t incr (stderr_term ? 1 : 10); // Scale depending on output type. - what = " targets to " + diag_do (a); + what = " targets to " + diag_do (ctx, a); mg = sched.monitor ( - target_count, + ctx.target_count, incr, [incr, &what] (size_t c) -> size_t { @@ -267,12 +267,12 @@ namespace build2 // Reverse the order of targets if the execution mode is 'last'. // - if (current_mode == execution_mode::last) + if (ctx.current_mode == execution_mode::last) reverse (ts.begin (), ts.end ()); // Tune the scheduler. // - switch (current_inner_oif->concurrency) + switch (ctx.current_inner_oif->concurrency) { case 0: sched.tune (1); break; // Run serially. case 1: break; // Run as is. @@ -292,20 +292,20 @@ namespace build2 if (prog && show_progress (1 /* max_verb */)) { - size_t init (target_count.load (memory_order_relaxed)); + size_t init (ctx.target_count.load (memory_order_relaxed)); size_t incr (init > 100 ? init / 100 : 1); // 1%. if (init != incr) { - what = "% of targets " + diag_did (a); + what = "% of targets " + diag_did (ctx, a); mg = sched.monitor ( - target_count, + ctx.target_count, init - incr, - [init, incr, &what] (size_t c) -> size_t + [init, incr, &what, &ctx] (size_t c) -> size_t { size_t p ((init - c) * 100 / init); - size_t s (skip_count.load (memory_order_relaxed)); + size_t s (ctx.skip_count.load (memory_order_relaxed)); diag_progress_lock pl; diag_progress = ' '; @@ -372,9 +372,9 @@ namespace build2 // if (verb != 0) { - if (size_t s = skip_count.load (memory_order_relaxed)) + if (size_t s = ctx.skip_count.load (memory_order_relaxed)) { - text << "skipped " << diag_doing (a) << ' ' << s << " targets"; + text << "skipped " << diag_doing (ctx, a) << ' ' << s << " targets"; } } @@ -433,8 +433,8 @@ namespace build2 // We should have executed every target that we matched, provided we // haven't failed (in which case we could have bailed out early). // - assert (target_count.load (memory_order_relaxed) == 0); - assert (dependency_count.load (memory_order_relaxed) == 0); + assert (ctx.target_count.load (memory_order_relaxed) == 0); + assert (ctx.dependency_count.load (memory_order_relaxed) == 0); } const meta_operation_info mo_perform { diff --git a/libbuild2/prerequisite.cxx b/libbuild2/prerequisite.cxx index b73aa4e..0883b75 100644 --- a/libbuild2/prerequisite.cxx +++ b/libbuild2/prerequisite.cxx @@ -113,8 +113,9 @@ namespace build2 // Call the meta-operation override, if any (currently used by dist). // - return current_mif->include == nullptr - ? r - : current_mif->include (a, t, prerequisite_member {p, m}, r); + if (auto f = t.ctx.current_mif->include) + r = f (a, t, prerequisite_member {p, m}, r); + + return r; } } diff --git a/libbuild2/rule.cxx b/libbuild2/rule.cxx index 0ade8a3..d69e817 100644 --- a/libbuild2/rule.cxx +++ b/libbuild2/rule.cxx @@ -192,7 +192,7 @@ namespace build2 { if (verb >= 2) text << "mkdir " << d; - else if (verb && current_diag_noise) + else if (verb && t.ctx.current_diag_noise) text << "mkdir " << t; }; @@ -279,7 +279,7 @@ namespace build2 // (or is current working directory). In this case rmdir() will issue a // warning when appropriate. // - target_state ts (rmdir (t.dir, t, current_diag_noise ? 1 : 2) + target_state ts (rmdir (t.dir, t, t.ctx.current_diag_noise ? 1 : 2) ? target_state::changed : target_state::unchanged); diff --git a/libbuild2/target.cxx b/libbuild2/target.cxx index 977c75c..5e6db5a 100644 --- a/libbuild2/target.cxx +++ b/libbuild2/target.cxx @@ -544,10 +544,11 @@ namespace build2 // Similar logic to matched_state_impl(). // const opstate& s (state[action () /* inner */]); - size_t o (s.task_count.load (memory_order_relaxed) - // Synchronized. - target::count_base ()); - if (o != target::offset_applied && o != target::offset_executed) + // Note: already synchronized. + size_t o (s.task_count.load (memory_order_relaxed) - ctx.count_base ()); + + if (o != offset_applied && o != offset_executed) break; } // Fall through. diff --git a/libbuild2/target.hxx b/libbuild2/target.hxx index 5be1aab..62b6753 100644 --- a/libbuild2/target.hxx +++ b/libbuild2/target.hxx @@ -455,6 +455,9 @@ namespace build2 // the target is synchronized, then we can access and modify (second case) // its state etc. // + // NOTE: see also the corresponding count_*() fuctions in context (must be + // kept in sync). + // static const size_t offset_touched = 1; // Target has been locked. static const size_t offset_tried = 2; // Rule match has been tried. static const size_t offset_matched = 3; // Rule has been matched. @@ -462,15 +465,6 @@ namespace build2 static const size_t offset_executed = 5; // Recipe has been executed. static const size_t offset_busy = 6; // Match/execute in progress. - static size_t count_base () {return 5 * (current_on - 1);} - - static size_t count_touched () {return offset_touched + count_base ();} - static size_t count_tried () {return offset_tried + count_base ();} - static size_t count_matched () {return offset_matched + count_base ();} - static size_t count_applied () {return offset_applied + count_base ();} - static size_t count_executed () {return offset_executed + count_base ();} - static size_t count_busy () {return offset_busy + count_base ();} - // Inner/outer operation state. See for details. // class LIBBUILD2_SYMEXPORT opstate diff --git a/libbuild2/target.ixx b/libbuild2/target.ixx index e7fef17..980df7c 100644 --- a/libbuild2/target.ixx +++ b/libbuild2/target.ixx @@ -93,16 +93,17 @@ namespace build2 // Note that the "tried" state is "final". // const opstate& s (state[a]); - size_t o (s.task_count.load (memory_order_relaxed) - // Synchronized. - target::count_base ()); - if (o == target::offset_tried) + // Note: already synchronized. + size_t o (s.task_count.load (memory_order_relaxed) - ctx.count_base ()); + + if (o == offset_tried) return make_pair (false, target_state::unknown); else { // Normally applied but can also be already executed. // - assert (o == target::offset_applied || o == target::offset_executed); + assert (o == offset_applied || o == offset_executed); return make_pair (true, (group_state (a) ? group->state[a] : s).state); } } diff --git a/libbuild2/test/rule.cxx b/libbuild2/test/rule.cxx index 61bfba4..c4767cf 100644 --- a/libbuild2/test/rule.cxx +++ b/libbuild2/test/rule.cxx @@ -490,7 +490,7 @@ namespace build2 wait_guard wg; if (!dry_run) - wg = wait_guard (t.ctx, target::count_busy (), t[a].task_count); + wg = wait_guard (t.ctx, t.ctx.count_busy (), t[a].task_count); // Result vector. // @@ -538,7 +538,7 @@ namespace build2 { scope_state& r (res.back ()); - if (!sched.async (target::count_busy (), + if (!sched.async (t.ctx.count_busy (), t[a].task_count, [this] (const diag_frame* ds, scope_state& r, -- cgit v1.1