diff options
Diffstat (limited to 'libbuild2/algorithm.cxx')
-rw-r--r-- | libbuild2/algorithm.cxx | 2862 |
1 files changed, 2220 insertions, 642 deletions
diff --git a/libbuild2/algorithm.cxx b/libbuild2/algorithm.cxx index 54ddf78..16f1503 100644 --- a/libbuild2/algorithm.cxx +++ b/libbuild2/algorithm.cxx @@ -54,29 +54,45 @@ namespace build2 const target& search (const target& t, const prerequisite_key& pk) { - assert (t.ctx.phase == run_phase::match); + context& ctx (t.ctx); + + assert (ctx.phase == run_phase::match); // If this is a project-qualified prerequisite, then this is import's - // business. + // business (phase 2). // if (pk.proj) - return import (t.ctx, pk); + return import2 (ctx, pk); - if (const target* pt = pk.tk.type->search (t, pk)) + if (const target* pt = pk.tk.type->search (ctx, &t, pk)) return *pt; - return create_new_target (t.ctx, pk); + if (pk.tk.out->empty ()) + return create_new_target (ctx, pk); + + // If this is triggered, then you are probably not passing scope to + // search() (which leads to search_existing_file() being skipped). + // + fail << "no existing source file for prerequisite " << pk << endf; } pair<target&, ulock> search_locked (const target& t, const prerequisite_key& pk) { - assert (t.ctx.phase == run_phase::match && !pk.proj); + context& ctx (t.ctx); + + assert (ctx.phase == run_phase::match && !pk.proj); - if (const target* pt = pk.tk.type->search (t, pk)) + if (const target* pt = pk.tk.type->search (ctx, &t, pk)) return {const_cast<target&> (*pt), ulock ()}; - return create_new_target_locked (t.ctx, pk); + if (pk.tk.out->empty ()) + return create_new_target_locked (ctx, pk); + + // If this is triggered, then you are probably not passing scope to + // search() (which leads to search_existing_file() being skipped). + // + fail << "no existing source file for prerequisite " << pk << endf; } const target* @@ -84,11 +100,33 @@ namespace build2 { return pk.proj ? import_existing (ctx, pk) - : search_existing_target (ctx, pk); + : pk.tk.type->search (ctx, nullptr /* existing */, pk); } const target& - search (const target& t, name n, const scope& s, const target_type* tt) + search_new (context& ctx, const prerequisite_key& pk) + { + assert (ctx.phase == run_phase::load || ctx.phase == run_phase::match); + + if (const target* pt = search_existing_target (ctx, pk, true /*out_only*/)) + return *pt; + + return create_new_target (ctx, pk); + } + + pair<target&, ulock> + search_new_locked (context& ctx, const prerequisite_key& pk) + { + assert (ctx.phase == run_phase::load || ctx.phase == run_phase::match); + + if (const target* pt = search_existing_target (ctx, pk, true /*out_only*/)) + return {const_cast<target&> (*pt), ulock ()}; + + return create_new_target_locked (ctx, pk); + } + + const target& + search (const target& t, name&& n, const scope& s, const target_type* tt) { assert (t.ctx.phase == run_phase::match); @@ -132,19 +170,22 @@ namespace build2 return nullptr; if (!n.dir.empty ()) - n.dir.normalize (false, true); // Current dir collapses to an empty one. + { + // More often than not a non-empty directory will already be normalized. + // + // Note that we collapse current dir to an empty one. + // + if (!n.dir.normalized () || n.dir.string () == ".") + n.dir.normalize (false, true); + } bool q (cn.qualified ()); - - // @@ OUT: for now we assume the prerequisite's out is undetermined. - // Would need to pass a pair of names. - // prerequisite_key pk { n.proj, {tt, &n.dir, q ? &empty_dir_path : &out, &n.value, ext}, &s}; return q ? import_existing (s.ctx, pk) - : search_existing_target (s.ctx, pk); + : tt->search (s.ctx, nullptr /* existing */, pk); } const target* @@ -191,8 +232,14 @@ namespace build2 // If the work_queue is absent, then we don't wait. // + // While already applied or executed targets are normally not locked, if + // options contain any bits that are not already in cur_options, then the + // target is locked even in these states. + // target_lock - lock_impl (action a, const target& ct, optional<scheduler::work_queue> wq) + lock_impl (action a, const target& ct, + optional<scheduler::work_queue> wq, + uint64_t options) { context& ctx (ct.ctx); @@ -207,7 +254,8 @@ namespace build2 size_t appl (b + target::offset_applied); size_t busy (b + target::offset_busy); - atomic_count& task_count (ct[a].task_count); + const target::opstate& cs (ct[a]); + atomic_count& task_count (cs.task_count); while (!task_count.compare_exchange_strong ( e, @@ -227,7 +275,7 @@ namespace build2 fail << "dependency cycle detected involving target " << ct; if (!wq) - return target_lock {a, nullptr, e - b}; + return target_lock {a, nullptr, e - b, false}; // We also unlock the phase for the duration of the wait. Why? // Consider this scenario: we are trying to match a dir{} target whose @@ -236,14 +284,20 @@ namespace build2 // to switch the phase to load. Which would result in a deadlock // unless we release the phase. // - phase_unlock u (ct.ctx, true /* unlock */, true /* delay */); - e = ctx.sched.wait (busy - 1, task_count, u, *wq); + phase_unlock u (ct.ctx, true /* delay */); + e = ctx.sched->wait (busy - 1, task_count, u, *wq); } - // We don't lock already applied or executed targets. + // We don't lock already applied or executed targets unless there + // are new options. + // + // Note: we don't have the lock yet so we must use atomic cur_options. + // We also have to re-check this once we've grabbed the lock. // - if (e >= appl) - return target_lock {a, nullptr, e - b}; + if (e >= appl && + (cs.match_extra.cur_options_.load (memory_order_relaxed) & options) + == options) + return target_lock {a, nullptr, e - b, false}; } // We now have the lock. Analyze the old value and decide what to do. @@ -252,24 +306,41 @@ namespace build2 target::opstate& s (t[a]); size_t offset; - if (e <= b) + bool first; + if ((first = (e <= b))) { // First lock for this operation. // + // Note that we use 0 match_extra::cur_options_ as an indication of not + // being applied yet. In particular, in the match phase, this is used to + // distinguish between the "busy because not applied yet" and "busy + // because relocked to reapply match options" cases. See + // target::matched() for details. + // s.rule = nullptr; s.dependents.store (0, memory_order_release); + s.match_extra.cur_options_.store (0, memory_order_relaxed); offset = target::offset_touched; } else { + // Re-check the options if already applied or worse. + // + if (e >= appl && (s.match_extra.cur_options & options) == options) + { + // Essentially unlock_impl(). + // + task_count.store (e, memory_order_release); + ctx.sched->resume (task_count); + + return target_lock {a, nullptr, e - b, false}; + } + offset = e - b; - assert (offset == target::offset_touched || - offset == target::offset_tried || - offset == target::offset_matched); } - return target_lock {a, &t, offset}; + return target_lock {a, &t, offset, first}; } void @@ -285,7 +356,7 @@ namespace build2 // this target. // task_count.store (offset + ctx.count_base (), memory_order_release); - ctx.sched.resume (task_count); + ctx.sched->resume (task_count); } target& @@ -293,7 +364,8 @@ namespace build2 const target_type& tt, dir_path dir, dir_path out, - string n) + string n, + optional<string> ext) { tracer trace ("add_adhoc_member"); @@ -308,101 +380,342 @@ namespace build2 move (dir), move (out), move (n), - nullopt /* ext */, + move (ext), target_decl::implied, - trace)); - - assert (r.second); + trace, + true /* skip_find */)); target& m (r.first); - *mp = &m; + + if (!r.second) + fail << "target " << m << " already exists and cannot be made " + << "ad hoc member of group " << t; + m.group = &t; + *mp = &m; return m; }; - // Return the matching rule or NULL if no match and try_match is true. - // - const rule_match* - match_rule (action a, target& t, const rule* skip, bool try_match) + pair<target&, bool> + add_adhoc_member_identity (target& t, + const target_type& tt, + dir_path dir, + dir_path out, + string n, + optional<string> ext, + const location& loc) { - const scope& bs (t.base_scope ()); + // NOTE: see similar code in parser::enter_adhoc_members(). - // Match rules in project environment. - // - auto_project_env penv; - if (const scope* rs = bs.root_scope ()) - penv = auto_project_env (*rs); + tracer trace ("add_adhoc_member_identity"); - match_extra& me (t[a].match_extra); + pair<target&, ulock> r ( + t.ctx.targets.insert_locked (tt, + move (dir), + move (out), + move (n), + move (ext), + target_decl::implied, + trace, + true /* skip_find */)); + target& m (r.first); - // First check for an ad hoc recipe. + // Add as an ad hoc member at the end of the chain skipping duplicates. // - // Note that a fallback recipe is preferred over a non-fallback rule. + const_ptr<target>* mp (&t.adhoc_member); + for (; *mp != nullptr; mp = &(*mp)->adhoc_member) + { + if (*mp == &m) + return {m, false}; + } + + if (!r.second) + fail (loc) << "target " << m << " already exists and cannot be made " + << "ad hoc member of group " << t; + + m.group = &t; + *mp = &m; + + return {m, true}; + } + + static bool + trace_target (const target& t, const vector<name>& ns) + { + for (const name& n: ns) + { + if (n.untyped () || n.qualified () || n.pattern) + fail << "unsupported trace target name '" << n << "'" << + info << "unqualified, typed, non-pattern name expected"; + + if (!n.dir.empty ()) + { + if (n.dir.relative () || !n.dir.normalized ()) + fail << "absolute and normalized trace target directory expected"; + + if (t.dir != n.dir) + continue; + } + + if (n.type == t.type ().name && n.value == t.name) + return true; + } + + return false; + } + + void + set_rule_trace (target_lock& l, const rule_match* rm) + { + action a (l.action); + target& t (*l.target); + + // Note: see similar code in execute_impl() for execute. // - if (!t.adhoc_recipes.empty ()) + if (trace_target (t, *t.ctx.trace_match)) { - auto df = make_diag_frame ( - [a, &t](const diag_record& dr) - { - if (verb != 0) - dr << info << "while matching ad hoc recipe to " << diag_do (a, t); - }); + diag_record dr (info); + + dr << "matching to " << diag_do (a, t); - auto match = [a, &t, &me] (const adhoc_rule& r, bool fallback) -> bool + if (rm != nullptr) { - me.init (fallback); + const rule& r (rm->second); - if (auto* f = (a.outer () - ? t.ctx.current_outer_oif - : t.ctx.current_inner_oif)->adhoc_match) - return f (r, a, t, string () /* hint */, me); + if (const adhoc_rule* ar = dynamic_cast<const adhoc_rule*> (&r)) + { + dr << info (ar->loc); + + if (ar->pattern != nullptr) + dr << "using ad hoc pattern rule "; + else + dr << "using ad hoc recipe "; + } else - return r.match (a, t, string () /* hint */, me); - }; + dr << info << "using rule "; - // The action could be Y-for-X while the ad hoc recipes are always for - // X. So strip the Y-for part for comparison (but not for the match() - // calls; see below for the hairy inner/outer semantics details). - // - action ca (a.inner () - ? a - : action (a.meta_operation (), a.outer_operation ())); + dr << rm->first; + } + else + dr << info << "using directly-assigned recipe"; + } + + t[a].rule = rm; + } + // Note: not static since also called by rule::sub_match(). + // + const rule_match* + match_adhoc_recipe (action a, target& t, match_extra& me) + { + auto df = make_diag_frame ( + [a, &t](const diag_record& dr) + { + if (verb != 0) + dr << info << "while matching ad hoc recipe to " << diag_do (a, t); + }); + + auto match = [a, &t, &me] (const adhoc_rule& r, bool fallback) -> bool + { + me.reinit (fallback); + + if (auto* f = (a.outer () + ? t.ctx.current_outer_oif + : t.ctx.current_inner_oif)->adhoc_match) + return f (r, a, t, string () /* hint */, me); + else + return r.match (a, t, string () /* hint */, me); + }; + + // The action could be Y-for-X while the ad hoc recipes are always for + // X. So strip the Y-for part for comparison (but not for the match() + // calls; see below for the hairy inner/outer semantics details). + // + action ca (a.inner () + ? a + : action (a.meta_operation (), a.outer_operation ())); + + // If returned rule_match is NULL, then the second half indicates whether + // the rule was found (but did not match). + // + auto find_match = [&t, &match] (action ca) -> pair<const rule_match*, bool> + { + // Note that there can be at most one recipe for any action. + // auto b (t.adhoc_recipes.begin ()), e (t.adhoc_recipes.end ()); auto i (find_if ( b, e, - [&match, ca] (const shared_ptr<adhoc_rule>& r) + [ca] (const shared_ptr<adhoc_rule>& r) { auto& as (r->actions); - return (find (as.begin (), as.end (), ca) != as.end () && - match (*r, false)); + return find (as.begin (), as.end (), ca) != as.end (); })); - if (i == e) + bool f (i != e); + if (f) + { + if (!match (**i, false /* fallback */)) + i = e; + } + else { // See if we have a fallback implementation. // // See the adhoc_rule::reverse_fallback() documentation for details on // what's going on here. // + // Note that it feels natural not to look for a fallback if a custom + // recipe was provided but did not match. + // + const target_type& tt (t.type ()); i = find_if ( b, e, - [&match, ca, &t] (const shared_ptr<adhoc_rule>& r) + [ca, &tt] (const shared_ptr<adhoc_rule>& r) { - auto& as (r->actions); - - // Note that the rule could be there but not match (see above), - // thus this extra check. + // Only the rule that provides the "forward" action can provide + // "reverse", so there can be at most one such rule. // - return (find (as.begin (), as.end (), ca) == as.end () && - r->reverse_fallback (ca, t.type ()) && - match (*r, true)); + return r->reverse_fallback (ca, tt); }); + + f = (i != e); + if (f) + { + if (!match (**i, true /* fallback */)) + i = e; + } + } + + return pair<const rule_match*, bool> ( + i != e ? &(*i)->rule_match : nullptr, + f); + }; + + pair<const rule_match*, bool> r (find_match (ca)); + + // Provide the "add dist_* and configure_* actions for every perform_* + // action unless there is a custom one" semantics (see the equivalent ad + // hoc rule registration code in the parser for background). + // + // Note that handling this in the parser by adding the extra actions is + // difficult because we store recipe actions in the recipe itself ( + // adhoc_rule::actions) and a recipe could be shared among multiple + // targets, some of which may provide a "custom one" as another recipe. On + // the other hand, handling it here is relatively straightforward. + // + if (r.first == nullptr && !r.second) + { + meta_operation_id mo (ca.meta_operation ()); + if (mo == configure_id || mo == dist_id) + { + action pa (perform_id, ca.operation ()); + r = find_match (pa); + } + } + + return r.first; + } + + // Return the matching rule or NULL if no match and try_match is true. + // + const rule_match* + match_rule_impl (action a, target& t, + uint64_t options, + const rule* skip, + bool try_match, + match_extra* pme) + { + using fallback_rule = adhoc_rule_pattern::fallback_rule; + + auto adhoc_rule_match = [] (const rule_match& r) + { + return dynamic_cast<const adhoc_rule*> (&r.second.get ()); + }; + + auto fallback_rule_match = [] (const rule_match& r) + { + return dynamic_cast<const fallback_rule*> (&r.second.get ()); + }; + + // Note: we copy the options value to me.new_options after successfully + // matching the rule to make sure rule::match() implementations don't rely + // on it. + // + match_extra& me (pme == nullptr ? t[a].match_extra : *pme); + + if (const target* g = t.group) + { + // If this is a group with dynamic members, then match it with the + // group's rule automatically. See dyndep_rule::inject_group_member() + // for background. + // + if ((g->type ().flags & target_type::flag::dyn_members) == + target_type::flag::dyn_members) + { + if (g->matched (a, memory_order_acquire)) + { + const rule_match* r (g->state[a].rule); + assert (r != nullptr); // Shouldn't happen with dyn_members. + + me.new_options = options; + return r; + } + + // Assume static member and fall through. } - if (i != e) - return &(*i)->rule_match; + // If this is a member of group-based target, then first try to find a + // matching ad hoc recipe/rule by matching (to an ad hoc recipe/rule) + // the group but applying to the member. See adhoc_rule::match() for + // background, including for why const_cast should be safe. + // + // To put it another way, if a group is matched by an ad hoc + // recipe/rule, then we want all the member to be matched to the same + // recipe/rule. + // + // Note that such a group is dyn_members so we would have tried the + // "already matched" case above. + // + if (g->is_a<group> ()) + { + // We cannot init match_extra from the target if it's unlocked so use + // a temporary (it shouldn't be modified if unlocked). + // + match_extra gme (false /* locked */); + if (const rule_match* r = match_rule_impl (a, const_cast<target&> (*g), + 0 /* options */, + skip, + true /* try_match */, + &gme)) + { + me.new_options = options; + return r; + } + + // Fall through to normal match of the member. + } + } + + const scope& bs (t.base_scope ()); + + // Match rules in project environment. + // + auto_project_env penv; + if (const scope* rs = bs.root_scope ()) + penv = auto_project_env (*rs); + + // First check for an ad hoc recipe. + // + // Note that a fallback recipe is preferred over a non-fallback rule. + // + if (!t.adhoc_recipes.empty ()) + { + if (const rule_match* r = match_adhoc_recipe (a, t, me)) + { + me.new_options = options; + return r; + } } // If this is an outer operation (Y-for-X), then we look for rules @@ -418,183 +731,214 @@ namespace build2 meta_operation_id mo (a.meta_operation ()); operation_id o (a.inner () ? a.operation () : a.outer_operation ()); - for (auto tt (&t.type ()); tt != nullptr; tt = tt->base) + // Our hint semantics applies regardless of the meta-operation. This works + // reasonably well except for the default/fallback rules provided by some + // meta-operations (e.g., dist, config), which naturally do not match the + // hint. + // + // The way we solve this problem is by trying a hint-less match as a + // fallback for non-perform meta-operations. @@ Ideally we would want to + // only consider such default/fallback rules, which we may do in the + // future (we could just decorate their names with some special marker, + // e.g., `dist.file.*` but that would be visible in diagnostics). + // + // It seems the only potential problem with this approach is the inability + // by the user to specify the hint for this specific meta-operation (e.g., + // to resolve an ambiguity between two rules or override a matched rule), + // which seems quite remote at the moment. Maybe/later we can invent a + // syntax for that. + // + const string* hint; + for (bool retry (false);; retry = true) { - // Search scopes outwards, stopping at the project root. - // - for (const scope* s (&bs); - s != nullptr; - s = s->root () ? &s->global_scope () : s->parent_scope ()) - { - const operation_rule_map* om (s->rules[mo]); - - if (om == nullptr) - continue; // No entry for this meta-operation id. + hint = retry + ? &empty_string + : &t.find_hint (o); // MT-safe (target locked). - // First try the map for the actual operation. If that doesn't yeld - // anything, try the wildcard map. + for (auto tt (&t.type ()); tt != nullptr; tt = tt->base) + { + // Search scopes outwards, stopping at the project root. For retry + // only look in the root and global scopes. // - for (operation_id oi (o), oip (o); oip != 0; oip = oi, oi = 0) + for (const scope* s (retry ? bs.root_scope () : &bs); + s != nullptr; + s = s->root () ? &s->global_scope () : s->parent_scope ()) { - const target_type_rule_map* ttm ((*om)[oi]); - - if (ttm == nullptr) - continue; // No entry for this operation id. - - if (ttm->empty ()) - continue; // Empty map for this operation id. + const operation_rule_map* om (s->rules[mo]); - auto i (ttm->find (tt)); + if (om == nullptr) + continue; // No entry for this meta-operation id. - if (i == ttm->end () || i->second.empty ()) - continue; // No rules registered for this target type. - - const auto& rules (i->second); // Hint map. - - // @@ TODO hint - // - // Different rules can be used for different operations (update vs - // test is a good example). So, at some point, we will probably have - // to support a list of hints or even an operation-hint map (e.g., - // 'hint=cxx test=foo' if cxx supports the test operation but we - // want the foo rule instead). This is also the place where the - // '{build clean}=cxx' construct (which we currently do not support) - // can come handy. - // - // Also, ignore the hint (that is most likely ment for a different - // operation) if this is a unique match. + // First try the map for the actual operation. If that doesn't yeld + // anything, try the wildcard map. // - string hint; - auto rs (rules.size () == 1 - ? make_pair (rules.begin (), rules.end ()) - : rules.find_sub (hint)); - - for (auto i (rs.first); i != rs.second; ++i) + for (operation_id oi (o), oip (o); oip != 0; oip = oi, oi = 0) { - const rule_match* r (&*i); + const target_type_rule_map* ttm ((*om)[oi]); - // In a somewhat hackish way we reuse operation wildcards to plumb - // the ad hoc rule's reverse operation fallback logic. - // - // The difficulty is two-fold: - // - // 1. It's difficult to add the fallback flag to the rule map - // because of rule_match which is used throughout. - // - // 2. Even if we could do that, we pass the reverse action to - // reverse_fallback() rather than it returning (a list) of - // reverse actions, which would be necessary to register them. - // - using fallback_rule = adhoc_rule_pattern::fallback_rule; + if (ttm == nullptr) + continue; // No entry for this operation id. - auto find_fallback = [mo, o, tt] (const fallback_rule& fr) - -> const rule_match* - { - for (const shared_ptr<adhoc_rule>& ar: fr.rules) - if (ar->reverse_fallback (action (mo, o), *tt)) - return &ar->rule_match; + if (ttm->empty ()) + continue; // Empty map for this operation id. - return nullptr; - }; + auto i (ttm->find (tt)); - if (oi == 0) - { - if (auto* fr = - dynamic_cast<const fallback_rule*> (&r->second.get ())) - { - if ((r = find_fallback (*fr)) == nullptr) - continue; - } - } + if (i == ttm->end () || i->second.empty ()) + continue; // No rules registered for this target type. - const string& n (r->first); - const rule& ru (r->second); + const auto& rules (i->second); // Name map. - if (&ru == skip) - continue; + // Filter against the hint, if any. + // + auto rs (hint->empty () + ? make_pair (rules.begin (), rules.end ()) + : rules.find_sub (*hint)); - me.init (oi == 0 /* fallback */); + for (auto i (rs.first); i != rs.second; ++i) { - auto df = make_diag_frame ( - [a, &t, &n](const diag_record& dr) - { - if (verb != 0) - dr << info << "while matching rule " << n << " to " - << diag_do (a, t); - }); + const rule_match* r (&*i); - if (!ru.match (a, t, hint, me)) - continue; - } - - // Do the ambiguity test. - // - bool ambig (false); + // In a somewhat hackish way we reuse operation wildcards to + // plumb the ad hoc rule's reverse operation fallback logic. + // + // The difficulty is two-fold: + // + // 1. It's difficult to add the fallback flag to the rule map + // because of rule_match which is used throughout. + // + // 2. Even if we could do that, we pass the reverse action to + // reverse_fallback() rather than it returning (a list) of + // reverse actions, which would be necessary to register them. + // + auto find_fallback = [mo, o, tt] (const fallback_rule& fr) + -> const rule_match* + { + for (const shared_ptr<adhoc_rule>& ar: fr.rules) + if (ar->reverse_fallback (action (mo, o), *tt)) + return &ar->rule_match; - diag_record dr; - for (++i; i != rs.second; ++i) - { - const rule_match* r1 (&*i); + return nullptr; + }; if (oi == 0) { - if (auto* fr = - dynamic_cast<const fallback_rule*> (&r1->second.get ())) + if (const fallback_rule* fr = fallback_rule_match (*r)) { - if ((r1 = find_fallback (*fr)) == nullptr) + if ((r = find_fallback (*fr)) == nullptr) continue; } } - const string& n1 (r1->first); - const rule& ru1 (r1->second); + // Skip non-ad hoc rules if the target is not locked (see above; + // note that in this case match_extra is a temporary which we + // can reinit). + // + if (!me.locked && !adhoc_rule_match (*r)) + continue; + + const string& n (r->first); + const rule& ru (r->second); + + if (&ru == skip) + continue; + me.reinit (oi == 0 /* fallback */); { auto df = make_diag_frame ( - [a, &t, &n1](const diag_record& dr) + [a, &t, &n](const diag_record& dr) { if (verb != 0) - dr << info << "while matching rule " << n1 << " to " + dr << info << "while matching rule " << n << " to " << diag_do (a, t); }); - // @@ TODO: this makes target state in match() undetermined - // so need to fortify rules that modify anything in match - // to clear things. - // - // @@ Can't we temporarily swap things out in target? - // - match_extra me1; - me1.init (oi == 0); - if (!ru1.match (a, t, hint, me1)) + if (!ru.match (a, t, *hint, me)) continue; } - if (!ambig) + // Do the ambiguity test. + // + bool ambig (false); + + diag_record dr; + for (++i; i != rs.second; ++i) { - dr << fail << "multiple rules matching " << diag_doing (a, t) - << info << "rule " << n << " matches"; - ambig = true; + const rule_match* r1 (&*i); + + if (oi == 0) + { + if (const fallback_rule* fr = fallback_rule_match (*r1)) + { + if ((r1 = find_fallback (*fr)) == nullptr) + continue; + } + } + + if (!me.locked && !adhoc_rule_match (*r1)) + continue; + + const string& n1 (r1->first); + const rule& ru1 (r1->second); + + { + auto df = make_diag_frame ( + [a, &t, &n1](const diag_record& dr) + { + if (verb != 0) + dr << info << "while matching rule " << n1 << " to " + << diag_do (a, t); + }); + + // @@ TODO: this makes target state in match() undetermined + // so need to fortify rules that modify anything in match + // to clear things. + // + // @@ Can't we temporarily swap things out in target? + // + match_extra me1 (me.locked, oi == 0 /* fallback */); + if (!ru1.match (a, t, *hint, me1)) + continue; + } + + if (!ambig) + { + dr << fail << "multiple rules matching " << diag_doing (a, t) + << info << "rule " << n << " matches"; + ambig = true; + } + + dr << info << "rule " << n1 << " also matches"; } - dr << info << "rule " << n1 << " also matches"; + if (!ambig) + { + me.new_options = options; + return r; + } + else + dr << info << "use rule hint to disambiguate this match"; } - - if (!ambig) - return r; - else - dr << info << "use rule hint to disambiguate this match"; } } } + + if (mo == perform_id || hint->empty () || retry) + break; } me.free (); if (!try_match) { - diag_record dr; - dr << fail << "no rule to " << diag_do (a, t); + diag_record dr (fail); + + if (hint->empty ()) + dr << "no rule to "; + else + dr << "no rule with hint " << *hint << " to "; + + dr << diag_do (a, t); // Try to give some hints of the common causes. // @@ -684,66 +1028,280 @@ namespace build2 recipe re (ar != nullptr ? f (*ar, a, t, me) : ru.apply (a, t, me)); - me.free (); + me.free (); // Note: cur_options are still in use. + assert (me.cur_options != 0); // Match options cannot be 0 after apply(). + me.cur_options_.store (me.cur_options, memory_order_relaxed); return re; } + static void + apply_posthoc_impl ( + action a, target& t, + const pair<const string, reference_wrapper<const rule>>& m, + context::posthoc_target& pt) + { + const scope& bs (t.base_scope ()); + + // Apply rules in project environment. + // + auto_project_env penv; + if (const scope* rs = bs.root_scope ()) + penv = auto_project_env (*rs); + + const rule& ru (m.second); + match_extra& me (t[a].match_extra); + me.posthoc_prerequisite_targets = &pt.prerequisite_targets; + + auto df = make_diag_frame ( + [a, &t, &m](const diag_record& dr) + { + if (verb != 0) + dr << info << "while applying rule " << m.first << " to " + << diag_do (a, t) << " for post hoc prerequisites"; + }); + + // Note: for now no adhoc_apply_posthoc(). + // + ru.apply_posthoc (a, t, me); + } + + static void + reapply_impl (action a, + target& t, + const pair<const string, reference_wrapper<const rule>>& m) + { + const scope& bs (t.base_scope ()); + + // Reapply rules in project environment. + // + auto_project_env penv; + if (const scope* rs = bs.root_scope ()) + penv = auto_project_env (*rs); + + const rule& ru (m.second); + match_extra& me (t[a].match_extra); + // Note: me.posthoc_prerequisite_targets carried over. + + auto df = make_diag_frame ( + [a, &t, &m](const diag_record& dr) + { + if (verb != 0) + dr << info << "while reapplying rule " << m.first << " to " + << diag_do (a, t); + }); + + // Note: for now no adhoc_reapply(). + // + ru.reapply (a, t, me); + assert (me.cur_options != 0); // Match options cannot be 0 after reapply(). + me.cur_options_.store (me.cur_options, memory_order_relaxed); + } + + // If anything goes wrong, set target state to failed and return nullopt. + // Otherwise return the pointer to the new posthoc_target entry if any post + // hoc prerequisites were present or NULL otherwise. Note that the returned + // entry is stable (because we use a list) and should only be accessed + // during the match phase if the holding the target lock. + // + // Note: must be called while holding target_lock. + // + static optional<context::posthoc_target*> + match_posthoc (action a, target& t) + { + // The plan is to, while holding the lock, search and collect all the post + // hoc prerequisited and add an entry to context::current_posthoc_targets. + // The actual matching happens as post-pass in the meta-operation's match + // function. + // + // While it may seem like we could do matching here by unlocking (or + // unstacking) the lock for this target, that will only work for simple + // cases. In particular, consider: + // + // lib{foo}: ... + // lib{plug}: ... lib{foo} + // libs{foo}: libs{plug}: include = posthoc + // + // The chain we will end up with: + // + // lib{foo}->libs{foo}=>libs{plug}->lib{foo} + // + // This will trip up the cycle detection for group lib{foo}, not for + // libs{foo}. + // + // In the end, matching (and execution) "inline" (i.e., as we match/ + // execute the corresponding target) appears to be unworkable in the + // face of cycles. + // + // Note also that this delayed match also helps with allowing the rule to + // adjust match options of post hoc prerequisites without needing the + // rematch support (see match_extra::posthoc_prerequisites). + // + // @@ Anything we need to do for group members (see through)? Feels quite + // far-fetched. + // + using posthoc_target = context::posthoc_target; + using posthoc_prerequisite_target = posthoc_target::prerequisite_target; + + vector<posthoc_prerequisite_target> pts; + try + { + for (const prerequisite& p: group_prerequisites (t)) + { + // Note that we have to ignore any operation-specific values for + // non-posthoc prerequisites. See include_impl() for details. + // + lookup l; + if (include (a, t, p, &l) == include_type::posthoc) + { + if (l) + { + const string& v (cast<string> (l)); + + // The only valid values are true and false and the latter would + // have been translated to include_type::exclude. + // + if (v != "true") + { + fail << "unrecognized " << *l.var << " variable value " + << "'" << v << "' specified for prerequisite " << p; + } + } + + pts.push_back ( + posthoc_prerequisite_target { + &search (t, p), // May fail. + match_extra::all_options}); + } + } + } + catch (const failed&) + { + t.state[a].state = target_state::failed; + return nullopt; + } + + if (!pts.empty ()) + { + context& ctx (t.ctx); + + mlock l (ctx.current_posthoc_targets_mutex); + ctx.current_posthoc_targets.push_back (posthoc_target {a, t, move (pts)}); + return &ctx.current_posthoc_targets.back (); // Stable. + } + + return nullptr; + } + // If step is true then perform only one step of the match/apply sequence. // // If try_match is true, then indicate whether there is a rule match with // the first half of the result. // static pair<bool, target_state> - match_impl (target_lock& l, - bool step = false, - bool try_match = false) + match_impl_impl (target_lock& l, + uint64_t options, + bool step = false, + bool try_match = false) { + // With regards to options, the semantics that we need to achieve for each + // target::offeset_*: + // + // tried -- nothing to do (no match) + // touched -- set to new_options + // matched -- add to new_options + // applied -- reapply if any new options + // executed -- check and fail if any new options + // busy -- postpone until *_complete() call + // + // Note that if options is 0 (see resolve_{members,group}_impl()), then + // all this can be skipped. + assert (l.target != nullptr); action a (l.action); target& t (*l.target); target::opstate& s (t[a]); - // Intercept and handle matching an ad hoc group member. - // - if (t.adhoc_group_member ()) + try { - assert (!step); + // Intercept and handle matching an ad hoc group member. + // + if (t.adhoc_group_member ()) + { + // It feels natural to "convert" this call to the one for the group, + // including the try_match part. Semantically, we want to achieve the + // following: + // + // [try_]match (a, g); + // match_recipe (l, group_recipe); + // + // Currently, ad hoc group members cannot have options. An alternative + // semantics could be to call the goup's rule to translate member + // options to group options and then (re)match the group with that. + // The implementation of this semantics could look like this: + // + // 1. Lock the group. + // 2. If not already offset_matched, do one step to get the rule. + // 3. Call the rule to translate options. + // 4. Continue matching the group passing the translated options. + // 5. Keep track of member options in member's cur_options to handle + // member rematches (if already offset_{applied,executed}). + // + // Note: see also similar semantics but for explicit groups in + // adhoc-rule-*.cxx. - const target& g (*t.group); + assert (!step && options == match_extra::all_options); - // It feels natural to "convert" this call to the one for the group, - // including the try_match part. Semantically, we want to achieve the - // following: - // - // [try_]match (a, g); - // match_recipe (l, group_recipe); - // - auto df = make_diag_frame ( - [a, &t](const diag_record& dr) - { - if (verb != 0) - dr << info << "while matching group rule to " << diag_do (a, t); - }); + const target& g (*t.group); - pair<bool, target_state> r (match (a, g, 0, nullptr, try_match)); + // What should we do with options? After some rumination it fells most + // natural to treat options for the group and for its ad hoc member as + // the same entity ... or not. + // + auto df = make_diag_frame ( + [a, &t](const diag_record& dr) + { + if (verb != 0) + dr << info << "while matching group rule to " << diag_do (a, t); + }); - if (r.first) - { - if (r.second != target_state::failed) + pair<bool, target_state> r ( + match_impl (a, g, 0 /* options */, 0, nullptr, try_match)); + + if (r.first) { - match_inc_dependens (a, g); - match_recipe (l, group_recipe); + if (r.second != target_state::failed) + { + // Note: in particular, passing all_options makes sure we will + // never re-lock this member if already applied/executed. + // + match_inc_dependents (a, g); + match_recipe (l, group_recipe, match_extra::all_options); + + // Note: no need to call match_posthoc() since an ad hoc member + // has no own prerequisites and the group's ones will be matched + // by the group. + } + else + { + // Similar to catch(failed) below. + // + s.state = target_state::failed; + l.offset = target::offset_applied; + + // Make sure we don't relock a failed target. + // + match_extra& me (s.match_extra); + me.cur_options = match_extra::all_options; + me.cur_options_.store (me.cur_options, memory_order_relaxed); + } } - } - else - l.offset = target::offset_tried; + else + l.offset = target::offset_tried; - return r; // Group state (must be consistent with matched_state()). - } + return r; // Group state (must be consistent with matched_state()). + } - try - { // Continue from where the target has been left off. // switch (l.offset) @@ -762,13 +1320,14 @@ namespace build2 // // Clear the rule-specific variables, resolved targets list, and the - // data pad before calling match(). The rule is free to modify these - // in its match() (provided that it matches) in order to, for - // example, convey some information to apply(). + // auxiliary data storage before calling match(). The rule is free + // to modify these in its match() (provided that it matches) in + // order to, for example, convey some information to apply(). // clear_target (a, t); - const rule_match* r (match_rule (a, t, nullptr, try_match)); + const rule_match* r ( + match_rule_impl (a, t, options, nullptr, try_match)); assert (l.offset != target::offset_tried); // Should have failed. @@ -778,7 +1337,7 @@ namespace build2 return make_pair (false, target_state::unknown); } - s.rule = r; + set_rule (l, r); l.offset = target::offset_matched; if (step) @@ -790,25 +1349,86 @@ namespace build2 // Fall through. case target::offset_matched: { + // Add any new options. + // + s.match_extra.new_options |= options; + // Apply. // set_recipe (l, apply_impl (a, t, *s.rule)); l.offset = target::offset_applied; + + if (t.has_group_prerequisites ()) // Ok since already matched. + { + if (optional<context::posthoc_target*> p = match_posthoc (a, t)) + { + if (*p != nullptr) + { + // It would have been more elegant to do this before calling + // apply_impl() and then expose the post hoc prerequisites to + // apply(). The problem is the group may not be resolved until + // the call to apply(). And so we resort to the separate + // apply_posthoc() function. + // + apply_posthoc_impl (a, t, *s.rule, **p); + } + } + else + s.state = target_state::failed; + } + + break; + } + case target::offset_applied: + { + // Reapply if any new options. + // + match_extra& me (s.match_extra); + me.new_options = options & ~me.cur_options; // Clear existing. + assert (me.new_options != 0); // Otherwise should not have locked. + + // Feels like this can only be a logic bug since to end up with a + // subset of options requires a rule (see match_extra for details). + // + assert (s.rule != nullptr); + + reapply_impl (a, t, *s.rule); break; } + case target::offset_executed: + { + // Diagnose new options after execute. + // + match_extra& me (s.match_extra); + assert ((me.cur_options & options) != options); // Otherwise no lock. + + fail << "change of match options after " << diag_do (a, t) + << " has been executed" << + info << "executed options 0x" << hex << me.cur_options << + info << "requested options 0x" << hex << options << endf; + } default: assert (false); } } catch (const failed&) { + s.state = target_state::failed; + l.offset = target::offset_applied; + + // Make sure we don't relock a failed target. + // + match_extra& me (s.match_extra); + me.cur_options = match_extra::all_options; + me.cur_options_.store (me.cur_options, memory_order_relaxed); + } + + if (s.state == target_state::failed) + { // As a sanity measure clear the target data since it can be incomplete // or invalid (mark()/unmark() should give you some ideas). // clear_target (a, t); - - s.state = target_state::failed; - l.offset = target::offset_applied; } return make_pair (true, s.state); @@ -818,11 +1438,10 @@ namespace build2 // the first half of the result. // pair<bool, target_state> - match (action a, - const target& ct, - size_t start_count, - atomic_count* task_count, - bool try_match) + match_impl (action a, const target& ct, + uint64_t options, + size_t start_count, atomic_count* task_count, + bool try_match) { // If we are blocking then work our own queue one task at a time. The // logic here is that we may have already queued other tasks before this @@ -843,17 +1462,16 @@ namespace build2 ct, task_count == nullptr ? optional<scheduler::work_queue> (scheduler::work_none) - : nullopt)); + : nullopt, + options)); if (l.target != nullptr) { - assert (l.offset < target::offset_applied); // Shouldn't lock otherwise. - if (try_match && l.offset == target::offset_tried) return make_pair (false, target_state::unknown); if (task_count == nullptr) - return match_impl (l, false /* step */, try_match); + return match_impl_impl (l, options, false /* step */, try_match); // Pass "disassembled" lock since the scheduler queue doesn't support // task destruction. @@ -863,12 +1481,18 @@ namespace build2 // Also pass our diagnostics and lock stacks (this is safe since we // expect the caller to wait for completion before unwinding its stack). // - if (ct.ctx.sched.async ( + // Note: pack captures and arguments a bit to reduce the storage space + // requrements. + // + bool first (ld.first); + + if (ct.ctx.sched->async ( start_count, *task_count, - [a, try_match] (const diag_frame* ds, - const target_lock* ls, - target& t, size_t offset) + [a, try_match, first] (const diag_frame* ds, + const target_lock* ls, + target& t, size_t offset, + uint64_t options) { // Switch to caller's diag and lock stacks. // @@ -879,17 +1503,18 @@ namespace build2 { phase_lock pl (t.ctx, run_phase::match); // Throws. { - target_lock l {a, &t, offset}; // Reassemble. - match_impl (l, false /* step */, try_match); - // Unlock within the match phase. + // Note: target_lock must be unlocked within the match phase. + // + target_lock l {a, &t, offset, first}; // Reassemble. + match_impl_impl (l, options, false /* step */, try_match); } } catch (const failed&) {} // Phase lock failure. }, diag_frame::stack (), target_lock::stack (), - ref (*ld.target), - ld.offset)) + ref (*ld.target), ld.offset, + options)) return make_pair (true, target_state::postponed); // Queued. // Matched synchronously, fall through. @@ -907,9 +1532,39 @@ namespace build2 return ct.try_matched_state (a, false); } + void + match_only_sync (action a, const target& t, uint64_t options) + { + assert (t.ctx.phase == run_phase::match); + + target_lock l (lock_impl (a, t, scheduler::work_none, options)); + + if (l.target != nullptr) + { + if (l.offset != target::offset_matched) + { + if (match_impl_impl (l, + options, + true /* step */).second == target_state::failed) + throw failed (); + } + else + { + // If the target is already matched, then we need to add any new + // options but not call apply() (thus cannot use match_impl_impl()). + // + (*l.target)[a].match_extra.new_options |= options; + } + } + } + + // Note: lock is a reference to avoid the stacking overhead. + // static group_view - resolve_members_impl (action a, const target& g, target_lock l) + resolve_members_impl (action a, const target& g, target_lock&& l) { + assert (a.inner ()); + // Note that we will be unlocked if the target is already applied. // group_view r; @@ -923,7 +1578,9 @@ namespace build2 { // Match (locked). // - if (match_impl (l, true).second == target_state::failed) + if (match_impl_impl (l, + 0 /* options */, + true /* step */).second == target_state::failed) throw failed (); if ((r = g.group_members (a)).members != nullptr) @@ -934,22 +1591,52 @@ namespace build2 // Fall through. case target::offset_matched: { - // @@ Doing match without execute messes up our target_count. Does - // not seem like it will be easy to fix (we don't know whether - // someone else will execute this target). - // - // @@ What if we always do match & execute together? After all, - // if a group can be resolved in apply(), then it can be - // resolved in match()! - // - // Apply (locked). // - if (match_impl (l, true).second == target_state::failed) + pair<bool, target_state> s ( + match_impl_impl (l, 0 /* options */, true /* step */)); + + if (s.second == target_state::failed) throw failed (); if ((r = g.group_members (a)).members != nullptr) + { + // Doing match without execute messes up our target_count. There + // doesn't seem to be a clean way to solve this. Well, just always + // executing if we've done the match would have been clean but quite + // heavy-handed (it would be especially surprising if otherwise + // there is nothing else to do, which can happen, for example, + // during update-for-test when there are no tests to run). + // + // So our solution is as follows: + // + // 1. Keep track both of the targets that ended up in this situation + // (the target::resolve_counted flag) as well as their total + // count (the context::resolve_count member). Only do this if + // set_recipe() (called by match_impl()) would have incremented + // target_count. + // + // 2. If we happen to execute such a target (common case), then + // clear the flag and decrement the count. + // + // 3. When it's time to assert that target_count==0 (i.e., all the + // matched targets have been executed), check if resolve_count is + // 0. If it's not, then find every target with the flag set, + // pretend-execute it, and decrement both counts. See + // perform_execute() for further details on this step. + // + if (s.second != target_state::unchanged) + { + target::opstate& s (l.target->state[a]); // Inner. + + if (!s.recipe_group_action) + { + s.resolve_counted = true; + g.ctx.resolve_count.fetch_add (1, memory_order_relaxed); + } + } break; + } // Unlock and to execute ... // @@ -960,15 +1647,19 @@ namespace build2 { // Execute (unlocked). // - // Note that we use execute_direct() rather than execute() here to - // sidestep the dependents count logic. In this context, this is by - // definition the first attempt to execute this rule (otherwise we - // would have already known the members list) and we really do need + // Note that we use execute_direct_sync() rather than execute_sync() + // here to sidestep the dependents count logic. In this context, this + // is by definition the first attempt to execute this rule (otherwise + // we would have already known the members list) and we really do need // to execute it now. // + // Note that while it might be tempting to decrement resolve_count + // here, there is no guarantee that we were the ones who have matched + // this target. + // { phase_switch ps (g.ctx, run_phase::execute); - execute_direct (a, g); + execute_direct_sync (a, g); } r = g.group_members (a); @@ -1014,10 +1705,23 @@ namespace build2 return r; } + // Note: lock is a reference to avoid the stacking overhead. + // void - resolve_group_impl (action, const target&, target_lock l) + resolve_group_impl (target_lock&& l) { - match_impl (l, true /* step */, true /* try_match */); + assert (l.action.inner ()); + + pair<bool, target_state> r ( + match_impl_impl (l, + 0 /* options */, + true /* step */, + true /* try_match */)); + + l.unlock (); + + if (r.first && r.second == target_state::failed) + throw failed (); } template <typename R, typename S> @@ -1025,16 +1729,33 @@ namespace build2 match_prerequisite_range (action a, target& t, R&& r, const S& ms, - const scope* s) + const scope* s, + bool search_only) { auto& pts (t.prerequisite_targets[a]); + size_t i (pts.size ()); // Index of the first to be added. + + // Avoid duplicating fsdir{} that may have already been injected by + // inject_fsdir() (in which case it is expected to be first). + // + const target* dir (nullptr); + if (i != 0) + { + const prerequisite_target& pt (pts.front ()); + + if (pt.target != nullptr && pt.adhoc () && pt.target->is_a<fsdir> ()) + dir = pt.target; + } + // Start asynchronous matching of prerequisites. Wait with unlocked phase // to allow phase switching. // - wait_guard wg (t.ctx, t.ctx.count_busy (), t[a].task_count, true); + wait_guard wg ( + search_only + ? wait_guard () + : wait_guard (t.ctx, t.ctx.count_busy (), t[a].task_count, true)); - size_t i (pts.size ()); // Index of the first to be added. for (auto&& p: forward<R> (r)) { // Ignore excluded. @@ -1048,13 +1769,20 @@ namespace build2 ? ms (a, t, p, pi) : prerequisite_target (&search (t, p), pi)); - if (pt.target == nullptr || (s != nullptr && !pt.target->in (*s))) + if (pt.target == nullptr || + pt.target == dir || + (s != nullptr && !pt.target->in (*s))) continue; - match_async (a, *pt.target, t.ctx.count_busy (), t[a].task_count); + if (!search_only) + match_async (a, *pt.target, t.ctx.count_busy (), t[a].task_count); + pts.push_back (move (pt)); } + if (search_only) + return; + wg.wait (); // Finish matching all the targets that we have started. @@ -1062,29 +1790,38 @@ namespace build2 for (size_t n (pts.size ()); i != n; ++i) { const target& pt (*pts[i]); - match (a, pt); + match_complete (a, pt); } } void match_prerequisites (action a, target& t, const match_search& ms, - const scope* s) + const scope* s, + bool search_only) { - match_prerequisite_range (a, t, group_prerequisites (t), ms, s); + match_prerequisite_range (a, t, + group_prerequisites (t), + ms, + s, + search_only); } void match_prerequisite_members (action a, target& t, const match_search_member& msm, - const scope* s) + const scope* s, + bool search_only) { - match_prerequisite_range (a, t, group_prerequisite_members (a, t), msm, s); + match_prerequisite_range (a, t, + group_prerequisite_members (a, t), + msm, + s, + search_only); } - template <typename T> void - match_members (action a, target& t, T const* ts, size_t n) + match_members (action a, const target& t, const target* const* ts, size_t n) { // Pretty much identical to match_prerequisite_range() except we don't // search. @@ -1112,22 +1849,52 @@ namespace build2 if (m == nullptr || marked (m)) continue; - match (a, *m); + match_complete (a, *m); } } - // Instantiate only for what we need. - // - template LIBBUILD2_SYMEXPORT void - match_members<const target*> (action, target&, - const target* const*, size_t); + void + match_members (action a, + const target& t, + prerequisite_targets& ts, + size_t s, + pair<uintptr_t, uintptr_t> imv) + { + size_t n (ts.size ()); + + wait_guard wg (t.ctx, t.ctx.count_busy (), t[a].task_count, true); + + for (size_t i (s); i != n; ++i) + { + const prerequisite_target& pt (ts[i]); + const target* m (pt.target); + + if (m == nullptr || + marked (m) || + (imv.first != 0 && (pt.include & imv.first) != imv.second)) + continue; + + match_async (a, *m, t.ctx.count_busy (), t[a].task_count); + } + + wg.wait (); + + for (size_t i (s); i != n; ++i) + { + const prerequisite_target& pt (ts[i]); + const target* m (pt.target); + + if (m == nullptr || + marked (m) || + (imv.first != 0 && (pt.include & imv.first) != imv.second)) + continue; - template LIBBUILD2_SYMEXPORT void - match_members<prerequisite_target> (action, target&, - prerequisite_target const*, size_t); + match_complete (a, *m); + } + } const fsdir* - inject_fsdir (action a, target& t, bool parent) + inject_fsdir_impl (target& t, bool prereq, bool parent) { tracer trace ("inject_fsdir"); @@ -1148,6 +1915,7 @@ namespace build2 // subprojects (e.g., tests/). // const fsdir* r (nullptr); + if (rs != nullptr && !d.sub (rs->src_path ())) { l6 ([&]{trace << d << " for " << t;}); @@ -1156,7 +1924,7 @@ namespace build2 // r = &search<fsdir> (t, d, dir_path (), string (), nullptr, nullptr); } - else + else if (prereq) { // See if one was mentioned explicitly. // @@ -1175,13 +1943,45 @@ namespace build2 } } + return r; + } + + const fsdir* + inject_fsdir (action a, target& t, bool match, bool prereq, bool parent) + { + auto& pts (t.prerequisite_targets[a]); + + assert (!prereq || pts.empty ()); // This prerequisite target must be first. + + const fsdir* r (inject_fsdir_impl (t, prereq, parent)); + if (r != nullptr) { + if (match) + match_sync (a, *r); + // Make it ad hoc so that it doesn't end up in prerequisite_targets // after execution. // - match (a, *r); - t.prerequisite_targets[a].emplace_back (r, include_type::adhoc); + pts.emplace_back (r, include_type::adhoc); + } + + return r; + } + + const fsdir* + inject_fsdir_direct (action a, target& t, bool prereq, bool parent) + { + auto& pts (t.prerequisite_targets[a]); + + assert (!prereq || pts.empty ()); // This prerequisite target must be first. + + const fsdir* r (inject_fsdir_impl (t, prereq, parent)); + + if (r != nullptr) + { + match_direct_sync (a, *r); + pts.emplace_back (r, include_type::adhoc); } return r; @@ -1294,11 +2094,26 @@ namespace build2 return ts; } - void - update_backlink (const file& f, const path& l, bool changed, backlink_mode m) + static inline const char* + update_backlink_name (backlink_mode m, bool to_dir) { using mode = backlink_mode; + const char* r (nullptr); + switch (m) + { + case mode::link: + case mode::symbolic: r = verb >= 3 ? "ln -sf" : verb >= 2 ? "ln -s" : "ln"; break; + case mode::hard: r = verb >= 3 ? "ln -f" : "ln"; break; + case mode::copy: + case mode::overwrite: r = to_dir ? "cp -r" : "cp"; break; + } + return r; + } + + void + update_backlink (const file& f, const path& l, bool changed, backlink_mode m) + { const path& p (f.path ()); dir_path d (l.directory ()); @@ -1310,28 +2125,20 @@ namespace build2 // actually updated to signal to the user that the updated out target is // now available in src. // - if (verb <= 2) + if (verb == 1 || verb == 2) { if (changed || !butl::entry_exists (l, false /* follow_symlinks */, true /* ignore_errors */)) { - const char* c (nullptr); - switch (m) - { - case mode::link: - case mode::symbolic: c = verb >= 2 ? "ln -s" : "ln"; break; - case mode::hard: c = "ln"; break; - case mode::copy: - case mode::overwrite: c = l.to_directory () ? "cp -r" : "cp"; break; - } + const char* c (update_backlink_name (m, l.to_directory ())); - // Note: 'ln foo/ bar/' means a different thing. + // Note: 'ln foo/ bar/' means a different thing (and below). // - if (verb >= 2) + if (verb == 2) text << c << ' ' << p.string () << ' ' << l.string (); else - text << c << ' ' << f << " -> " << d; + print_diag (c, f, d); } } @@ -1351,30 +2158,25 @@ namespace build2 { // As above but with a slightly different diagnostics. - using mode = backlink_mode; - dir_path d (l.directory ()); - if (verb <= 2) + if (verb == 1 || verb == 2) { if (changed || !butl::entry_exists (l, false /* follow_symlinks */, true /* ignore_errors */)) { - const char* c (nullptr); - switch (m) - { - case mode::link: - case mode::symbolic: c = verb >= 2 ? "ln -s" : "ln"; break; - case mode::hard: c = "ln"; break; - case mode::copy: - case mode::overwrite: c = l.to_directory () ? "cp -r" : "cp"; break; - } + const char* c (update_backlink_name (m, l.to_directory ())); + // Note: 'ln foo/ bar/' means a different thing (and above) so strip + // trailing directory separator (but keep as path for relative). + // if (verb >= 2) text << c << ' ' << p.string () << ' ' << l.string (); else - text << c << ' ' << p.string () << " -> " << d; + print_diag (c, + p.to_directory () ? path (p.string ()) : p, + d); } } @@ -1426,6 +2228,8 @@ namespace build2 const path& p, const path& l, backlink_mode om, uint16_t verbosity) { + assert (verbosity >= 2); + using mode = backlink_mode; bool d (l.to_directory ()); @@ -1435,17 +2239,8 @@ namespace build2 { if (verb >= verbosity) { - const char* c (nullptr); - switch (m) - { - case mode::link: - case mode::symbolic: c = "ln -sf"; break; - case mode::hard: c = "ln -f"; break; - case mode::copy: - case mode::overwrite: c = d ? "cp -r" : "cp"; break; - } - - text << c << ' ' << p.string () << ' ' << l.string (); + text << update_backlink_name (m, d) << ' ' << p.string () << ' ' + << l.string (); } }; @@ -1507,8 +2302,7 @@ namespace build2 try_mkdir (to); - for (const auto& de: - dir_iterator (fr, false /* ignore_dangling */)) + for (const auto& de: dir_iterator (fr, dir_iterator::no_follow)) { path f (fr / de.path ()); path t (to / de.path ()); @@ -1561,6 +2355,11 @@ namespace build2 // // Note that here the dry-run mode is handled by the filesystem functions. + // Note that if we ever need to support level 1 for some reason, maybe + // consider showing the target, for example, `unlink exe{hello} <- dir/`? + // + assert (v >= 2); + using mode = backlink_mode; if (l.to_directory ()) @@ -1595,9 +2394,15 @@ namespace build2 struct backlink: auto_rm<path> { using path_type = build2::path; + using target_type = build2::target; reference_wrapper<const path_type> target; - backlink_mode mode; + backlink_mode mode; + + // Ad hoc group-specific information for diagnostics (see below). + // + const target_type* member = nullptr; + bool print = true; backlink (const path_type& t, path_type&& l, backlink_mode m, bool active) : auto_rm<path_type> (move (l), active), target (t), mode (m) @@ -1619,33 +2424,65 @@ namespace build2 }; // Normally (i.e., on sane platforms that don't have things like PDBs, etc) - // there will be just one backlink so optimize for that. + // there will be just one or two backlinks so optimize for that. // - using backlinks = small_vector<backlink, 1>; + using backlinks = small_vector<backlink, 2>; - static optional<backlink_mode> - backlink_test (const target& t, const lookup& l) + static optional<pair<backlink_mode, bool>> + backlink_test (const target& t, const lookup& l, optional<backlink_mode> gm) { using mode = backlink_mode; - optional<mode> r; - const string& v (cast<string> (l)); + const names& ns (cast<names> (l)); - if (v == "true") r = mode::link; - else if (v == "symbolic") r = mode::symbolic; - else if (v == "hard") r = mode::hard; - else if (v == "copy") r = mode::copy; - else if (v == "overwrite") r = mode::overwrite; - else if (v != "false") - fail << "invalid backlink variable value '" << v << "' " + if (ns.size () != 1 && ns.size () != 2) + { + fail << "invalid backlink variable value '" << ns << "' " << "specified for target " << t; + } - return r; + optional<mode> m; + for (;;) // Breakout loop. + { + const name& n (ns.front ()); + + if (n.simple ()) + { + const string& v (n.value); + + if (v == "true") {m = mode::link; break;} + else if (v == "symbolic") {m = mode::symbolic; break;} + else if (v == "hard") {m = mode::hard; break;} + else if (v == "copy") {m = mode::copy; break;} + else if (v == "overwrite") {m = mode::overwrite; break;} + else if (v == "false") { break;} + else if (v == "group") {if ((m = gm)) break;} + } + + fail << "invalid backlink variable value mode component '" << n << "' " + << "specified for target " << t << endf; + } + + bool np (false); // "not print" + if (ns.size () == 2) + { + const name& n (ns.back ()); + + if (n.simple () && (n.value == "true" || (np = (n.value == "false")))) + ; + else + fail << "invalid backlink variable value print component '" << n + << "' specified for target " << t; + } + + return m ? optional<pair<mode, bool>> (make_pair (*m, !np)) : nullopt; } static optional<backlink_mode> backlink_test (action a, target& t) { + using mode = backlink_mode; + context& ctx (t.ctx); // Note: the order of these checks is from the least to most expensive. @@ -1655,9 +2492,20 @@ namespace build2 if (a.outer () || (a != perform_update_id && a != perform_clean_id)) return nullopt; - // Only file-based targets in the out tree can be backlinked. + // Only targets in the out tree can be backlinked. // - if (!t.out.empty () || !t.is_a<file> ()) + if (!t.out.empty ()) + return nullopt; + + // Only file-based targets or groups containing file-based targets can be + // backlinked. Note that we don't do the "file-based" check of the latter + // case here since they can still be execluded. So instead we are prepared + // to handle the empty backlinks list. + // + // @@ Potentially members could only be resolved in execute. I guess we + // don't support backlink for such groups at the moment. + // + if (!t.is_a<file> () && t.group_members (a).members == nullptr) return nullopt; // Neither an out-of-project nor in-src configuration can be forwarded. @@ -1681,7 +2529,13 @@ namespace build2 if (!l.defined ()) l = ctx.global_scope.lookup (*ctx.var_backlink, t.key ()); - return l ? backlink_test (t, l) : nullopt; + optional<pair<mode, bool>> r (l ? backlink_test (t, l, nullopt) : nullopt); + + if (r && !r->second) + fail << "backlink variable value print component cannot be false " + << "for primary target " << t; + + return r ? optional<mode> (r->first) : nullopt; } static backlinks @@ -1689,58 +2543,104 @@ namespace build2 { using mode = backlink_mode; + context& ctx (t.ctx); const scope& s (t.base_scope ()); backlinks bls; - auto add = [&bls, &s] (const path& p, mode m) + auto add = [&bls, &s] (const path& p, + mode m, + const target* mt = nullptr, + bool print = true) { bls.emplace_back (p, s.src_path () / p.leaf (s.out_path ()), m, !s.ctx.dry_run /* active */); + + if (mt != nullptr) + { + backlink& bl (bls.back ()); + bl.member = mt; + bl.print = print; + } }; - // First the target itself. + // Check for a custom backlink mode for this member. If none, then + // inherit the one from the group (so if the user asked to copy + // .exe, we will also copy .pdb). // - add (t.as<file> ().path (), m); + // Note that we want to avoid group or tt/patter-spec lookup. And + // since this is an ad hoc member (which means it was either declared + // in the buildfile or added by the rule), we assume that the value, + // if any, will be set as a target or rule-specific variable. + // + auto member_mode = [a, m, &ctx] (const target& mt) + -> optional<pair<mode, bool>> + { + lookup l (mt.state[a].vars[ctx.var_backlink]); + + if (!l) + l = mt.vars[ctx.var_backlink]; + + return l ? backlink_test (mt, l, m) : make_pair (m, true); + }; - // Then ad hoc group file/fsdir members, if any. + // @@ Currently we don't handle the following cases: // - for (const target* mt (t.adhoc_member); - mt != nullptr; - mt = mt->adhoc_member) + // 1. File-based explicit groups. + // + // 2. Ad hoc subgroups in explicit groups. + // + // Note: see also the corresponding code in backlink_update_post(). + // + if (file* f = t.is_a<file> ()) { - const path* p (nullptr); + // First the target itself. + // + add (f->path (), m, f, true); // Note: always printed. - if (const file* f = mt->is_a<file> ()) + // Then ad hoc group file/fsdir members, if any. + // + for (const target* mt (t.adhoc_member); + mt != nullptr; + mt = mt->adhoc_member) { - p = &f->path (); + const path* p (nullptr); - if (p->empty ()) // The "trust me, it's somewhere" case. - p = nullptr; - } - else if (const fsdir* d = mt->is_a<fsdir> ()) - p = &d->dir; + if (const file* f = mt->is_a<file> ()) + { + p = &f->path (); - if (p != nullptr) - { - // Check for a custom backlink mode for this member. If none, then - // inherit the one from the group (so if the user asked to copy .exe, - // we will also copy .pdb). - // - // Note that we want to avoid group or tt/patter-spec lookup. And - // since this is an ad hoc member (which means it was either declared - // in the buildfile or added by the rule), we assume that the value, - // if any, will be set as a rule-specific variable (since setting it - // as a target-specific wouldn't be MT-safe). @@ Don't think this - // applies to declared ad hoc members. - // - lookup l (mt->state[a].vars[t.ctx.var_backlink]); + if (p->empty ()) // The "trust me, it's somewhere" case. + p = nullptr; + } + else if (const fsdir* d = mt->is_a<fsdir> ()) + p = &d->dir; - optional<mode> bm (l ? backlink_test (*mt, l) : m); + if (p != nullptr) + { + if (optional<pair<mode, bool>> m = member_mode (*mt)) + add (*p, m->first, mt, m->second); + } + } + } + else + { + // Explicit group. + // + group_view gv (t.group_members (a)); + assert (gv.members != nullptr); - if (bm) - add (*p, *bm); + for (size_t i (0); i != gv.count; ++i) + { + if (const target* mt = gv.members[i]) + { + if (const file* f = mt->is_a<file> ()) + { + if (optional<pair<mode, bool>> m = member_mode (*mt)) + add (f->path (), m->first); + } + } } } @@ -1754,29 +2654,89 @@ namespace build2 } static void - backlink_update_post (target& t, target_state ts, backlinks& bls) + backlink_update_post (target& t, target_state ts, + backlink_mode m, backlinks& bls) { if (ts == target_state::failed) return; // Let auto rm clean things up. - // Make backlinks. - // - for (auto b (bls.begin ()), i (b); i != bls.end (); ++i) + context& ctx (t.ctx); + + file* ft (t.is_a<file> ()); + + if (ft != nullptr && bls.size () == 1) { - const backlink& bl (*i); + // Single file-based target. + // + const backlink& bl (bls.front ()); - if (i == b) - update_backlink (t.as<file> (), - bl.path, - ts == target_state::changed, - bl.mode); - else - update_backlink (t.ctx, bl.target, bl.path, bl.mode); + update_backlink (*ft, + bl.path, + ts == target_state::changed, + bl.mode); + } + else + { + // Explicit or ad hoc group. + // + // What we have below is a custom variant of update_backlink(file). + // + dir_path d (bls.front ().path.directory ()); + + // First print the verbosity level 1 diagnostics. Level 2 and higher are + // handled by the update_backlink() calls below. + // + if (verb == 1) + { + bool changed (ts == target_state::changed); + + if (!changed) + { + for (const backlink& bl: bls) + { + changed = !butl::entry_exists (bl.path, + false /* follow_symlinks */, + true /* ignore_errors */); + if (changed) + break; + } + } + + if (changed) + { + const char* c (update_backlink_name (m, false /* to_dir */)); + + // For explicit groups we only print the group target. For ad hoc + // groups we print all the members except those explicitly excluded. + // + if (ft == nullptr) + print_diag (c, t, d); + else + { + vector<target_key> tks; + tks.reserve (bls.size ()); + + for (const backlink& bl: bls) + if (bl.print) + tks.push_back (bl.member->key ()); + + print_diag (c, move (tks), d); + } + } + } + + if (!exists (d)) + mkdir_p (d, 2 /* verbosity */); + + // Make backlinks. + // + for (const backlink& bl: bls) + update_backlink (ctx, bl.target, bl.path, bl.mode, 2 /* verbosity */); } // Cancel removal. // - if (!t.ctx.dry_run) + if (!ctx.dry_run) { for (backlink& bl: bls) bl.cancel (); @@ -1817,15 +2777,57 @@ namespace build2 // which is ok since such targets are probably not interesting for // backlinking. // + // Note also that for group members (both ad hoc and non) backlinking + // is handled when updating/cleaning the group. + // backlinks bls; - optional<backlink_mode> blm (backlink_test (a, t)); + optional<backlink_mode> blm; - if (blm) + if (t.group == nullptr) // Matched so must be already resolved. { - if (a == perform_update_id) - bls = backlink_update_pre (a, t, *blm); + blm = backlink_test (a, t); + + if (blm) + { + if (a == perform_update_id) + { + bls = backlink_update_pre (a, t, *blm); + if (bls.empty ()) + blm = nullopt; + } + else + backlink_clean_pre (a, t, *blm); + } + } + + // Note: see similar code in set_rule_trace() for match. + // + if (ctx.trace_execute != nullptr && trace_target (t, *ctx.trace_execute)) + { + diag_record dr (info); + + dr << diag_doing (a, t); + + if (s.rule != nullptr) + { + const rule& r (s.rule->second); + + if (const adhoc_rule* ar = dynamic_cast<const adhoc_rule*> (&r)) + { + dr << info (ar->loc); + + if (ar->pattern != nullptr) + dr << "using ad hoc pattern rule "; + else + dr << "using ad hoc recipe "; + } + else + dr << info << "using rule "; + + dr << s.rule->first; + } else - backlink_clean_pre (a, t, *blm); + dr << info << "using directly-assigned recipe"; } ts = execute_recipe (a, t, s.recipe); @@ -1833,7 +2835,7 @@ namespace build2 if (blm) { if (a == perform_update_id) - backlink_update_post (t, ts, bls); + backlink_update_post (t, ts, *blm, bls); } } catch (const failed&) @@ -1844,13 +2846,29 @@ namespace build2 ts = s.state = target_state::failed; } + // Clear the recipe to release any associated memory. Note that + // s.recipe_group_action may be used further (see, for example, + // group_state()) and should retain its value. + // + if (!s.recipe_keep) + s.recipe = nullptr; + // Decrement the target count (see set_recipe() for details). // - if (a.inner ()) + // Note that here we cannot rely on s.state being group because of the + // postponment logic (see excute_recipe() for details). + // + if (a.inner () && !s.recipe_group_action) { - recipe_function** f (s.recipe.target<recipe_function*> ()); - if (f == nullptr || *f != &group_action) - ctx.target_count.fetch_sub (1, memory_order_relaxed); + // See resolve_members_impl() for background. + // + if (s.resolve_counted) + { + s.resolve_counted = false; + ctx.resolve_count.fetch_sub (1, memory_order_relaxed); + } + + ctx.target_count.fetch_sub (1, memory_order_relaxed); } // Decrement the task count (to count_executed) and wake up any threads @@ -1860,17 +2878,19 @@ namespace build2 target::offset_busy - target::offset_executed, memory_order_release)); assert (tc == ctx.count_busy ()); - ctx.sched.resume (s.task_count); + ctx.sched->resume (s.task_count); return ts; } target_state - execute (action a, - const target& ct, - size_t start_count, - atomic_count* task_count) + execute_impl (action a, + const target& ct, + size_t start_count, + atomic_count* task_count) { + // NOTE: see also pretend_execute lambda in perform_execute(). + target& t (const_cast<target&> (ct)); // MT-aware. target::opstate& s (t[a]); @@ -1881,7 +2901,6 @@ namespace build2 size_t gd (ctx.dependency_count.fetch_sub (1, memory_order_relaxed)); size_t td (s.dependents.fetch_sub (1, memory_order_release)); assert (td != 0 && gd != 0); - td--; // Handle the "last" execution mode. // @@ -1904,7 +2923,7 @@ namespace build2 // thread. For other threads the state will still be unknown (until they // try to execute it). // - if (ctx.current_mode == execution_mode::last && td != 0) + if (ctx.current_mode == execution_mode::last && --td != 0) return target_state::postponed; // Try to atomically change applied to busy. @@ -1914,6 +2933,7 @@ namespace build2 size_t exec (ctx.count_executed ()); size_t busy (ctx.count_busy ()); + optional<target_state> r; if (s.task_count.compare_exchange_strong ( tc, busy, @@ -1926,32 +2946,35 @@ namespace build2 { // There could still be scope operations. // - if (t.is_a<dir> ()) - execute_recipe (a, t, nullptr /* recipe */); + r = t.is_a<dir> () + ? execute_recipe (a, t, nullptr /* recipe */) + : s.state; s.task_count.store (exec, memory_order_release); - ctx.sched.resume (s.task_count); + ctx.sched->resume (s.task_count); } else { if (task_count == nullptr) - return execute_impl (a, t); - - // Pass our diagnostics stack (this is safe since we expect the - // caller to wait for completion before unwinding its diag stack). - // - if (ctx.sched.async (start_count, - *task_count, - [a] (const diag_frame* ds, target& t) - { - diag_frame::stack_guard dsg (ds); - execute_impl (a, t); - }, - diag_frame::stack (), - ref (t))) - return target_state::unknown; // Queued. - - // Executed synchronously, fall through. + r = execute_impl (a, t); + else + { + // Pass our diagnostics stack (this is safe since we expect the + // caller to wait for completion before unwinding its diag stack). + // + if (ctx.sched->async (start_count, + *task_count, + [a] (const diag_frame* ds, target& t) + { + diag_frame::stack_guard dsg (ds); + execute_impl (a, t); + }, + diag_frame::stack (), + ref (t))) + return target_state::unknown; // Queued. + + // Executed synchronously, fall through. + } } } else @@ -1962,24 +2985,28 @@ namespace build2 else assert (tc == exec); } - return t.executed_state (a, false); + return r ? *r : t.executed_state (a, false /* fail */); } target_state - execute_direct (action a, const target& ct) + execute_direct_impl (action a, + const target& ct, + size_t start_count, + atomic_count* task_count) { context& ctx (ct.ctx); target& t (const_cast<target&> (ct)); // MT-aware. target::opstate& s (t[a]); - // Similar logic to match() above except we execute synchronously. + // Similar logic to execute_impl() above. // size_t tc (ctx.count_applied ()); size_t exec (ctx.count_executed ()); size_t busy (ctx.count_busy ()); + optional<target_state> r; if (s.task_count.compare_exchange_strong ( tc, busy, @@ -1987,33 +3014,436 @@ namespace build2 memory_order_acquire)) // Synchronize on failure. { if (s.state == target_state::unknown) - execute_impl (a, t); + { + if (task_count == nullptr) + r = execute_impl (a, t); + else + { + if (ctx.sched->async (start_count, + *task_count, + [a] (const diag_frame* ds, target& t) + { + diag_frame::stack_guard dsg (ds); + execute_impl (a, t); + }, + diag_frame::stack (), + ref (t))) + return target_state::unknown; // Queued. + + // Executed synchronously, fall through. + } + } else { assert (s.state == target_state::unchanged || s.state == target_state::failed); - if (s.state == target_state::unchanged) - { - if (t.is_a<dir> ()) - execute_recipe (a, t, nullptr /* recipe */); - } + r = s.state == target_state::unchanged && t.is_a<dir> () + ? execute_recipe (a, t, nullptr /* recipe */) + : s.state; s.task_count.store (exec, memory_order_release); - ctx.sched.resume (s.task_count); + ctx.sched->resume (s.task_count); } } else { - // If the target is busy, wait for it. + // Either busy or already executed. + // + if (tc >= busy) return target_state::busy; + else assert (tc == exec); + } + + return r ? *r : t.executed_state (a, false /* fail */); + } + + bool + update_during_match (tracer& trace, action a, const target& t, timestamp ts) + { + // NOTE: see also clean_during_match() if changing anything here. + + assert (a == perform_update_id); + + // Note: this function is used to make sure header dependencies are up to + // date (and which is where it originated). + // + // There would normally be a lot of headers for every source file (think + // all the system headers) and just calling execute_direct_sync() on all + // of them can get expensive. At the same time, most of these headers are + // existing files that we will never be updating (again, system headers, + // for example) and the rule that will match them is the fallback + // file_rule. That rule has an optimization: it returns noop_recipe (which + // causes the target state to be automatically set to unchanged) if the + // file is known to be up to date. So we do the update "smartly". + // + // Also, now that we do header pre-generation by default, there is a good + // chance the header has already been updated. So we also detect that and + // avoid switching the phase. + // + const path_target* pt (t.is_a<path_target> ()); + + if (pt == nullptr) + ts = timestamp_unknown; + + target_state os (t.matched_state (a)); + + if (os == target_state::unchanged) + { + if (ts == timestamp_unknown) + return false; + else + { + // We expect the timestamp to be known (i.e., existing file). // - if (tc >= busy) - ctx.sched.wait (exec, s.task_count, scheduler::work_none); - else - assert (tc == exec); + timestamp mt (pt->mtime ()); + assert (mt != timestamp_unknown); + return mt > ts; + } + } + else + { + // We only want to return true if our call to execute() actually caused + // an update. In particular, the target could already have been in + // target_state::changed because of the dynamic dependency extraction + // run for some other target. + // + target_state ns; + if (os != target_state::changed) + { + phase_switch ps (t.ctx, run_phase::execute); + ns = execute_direct_sync (a, t); + } + else + ns = os; + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "updated " << t + << "; old state " << os + << "; new state " << ns;}); + return true; + } + else + return ts != timestamp_unknown ? pt->newer (ts, ns) : false; } + } + + bool + update_during_match_prerequisites (tracer& trace, + action a, target& t, + uintptr_t mask) + { + // NOTE: see also clean_during_match_prerequisites() if changing anything + // here. + + assert (a == perform_update_id); + + prerequisite_targets& pts (t.prerequisite_targets[a]); + + // On the first pass detect and handle unchanged tragets. Note that we + // have to do it in a separate pass since we cannot call matched_state() + // once we've switched the phase. + // + size_t n (0); + + for (prerequisite_target& p: pts) + { + if (mask == 0 || (p.include & mask) != 0) + { + if (p.target != nullptr) + { + const target& pt (*p.target); + + target_state os (pt.matched_state (a)); + + if (os != target_state::unchanged) + { + ++n; + p.data = static_cast<uintptr_t> (os); + continue; + } + } + + p.data = 0; + } + } + + // If all unchanged, we are done. + // + if (n == 0) + return false; + + // Provide additional information on what's going on. + // + auto df = make_diag_frame ( + [&t](const diag_record& dr) + { + if (verb != 0) + dr << info << "while updating during match prerequisites of " + << "target " << t; + }); + + context& ctx (t.ctx); - return t.executed_state (a); + phase_switch ps (ctx, run_phase::execute); + + bool r (false); + + // @@ Maybe we should optimize for n == 1? Maybe we should just call + // smarter update_during_match() in this case? + // +#if 0 + for (prerequisite_target& p: pts) + { + if ((mask == 0 || (p.include & mask) != 0) && p.data != 0) + { + const target& pt (*p.target); + + target_state os (static_cast<target_state> (p.data)); + target_state ns (execute_direct_sync (a, pt)); + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "updated " << pt + << "; old state " << os + << "; new state " << ns;}); + r = true; + } + + p.data = 0; + } + } +#else + + // Start asynchronous execution of prerequisites. Similar logic to + // straight_execute_members(). + // + // Note that the target's task count is expected to be busy (since this + // function is called during match). And there don't seem to be any + // problems in using it for execute. + // + atomic_count& tc (t[a].task_count); + + size_t busy (ctx.count_busy ()); + + wait_guard wg (ctx, busy, tc); + + for (prerequisite_target& p: pts) + { + if ((mask == 0 || (p.include & mask) != 0) && p.data != 0) + { + execute_direct_async (a, *p.target, busy, tc); + } + } + + wg.wait (); + + // Finish execution and process the result. + // + for (prerequisite_target& p: pts) + { + if ((mask == 0 || (p.include & mask) != 0) && p.data != 0) + { + const target& pt (*p.target); + target_state ns (execute_complete (a, pt)); + target_state os (static_cast<target_state> (p.data)); + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "updated " << pt + << "; old state " << os + << "; new state " << ns;}); + r = true; + } + + p.data = 0; + } + } +#endif + + return r; + } + + bool + clean_during_match (tracer& trace, action a, const target& t) + { + // Let's keep this as close to update_during_match() semantically as + // possible until we see a clear reason to deviate. + + // We have a problem with fsdir{}: if the directory is not empty because + // there are other targets that depend on it and we execute it here and + // now, it will not remove the directory (because it's not yet empty) but + // will cause the target to be in the executed state, which means that + // when other targets try to execute it, it will be a noop and the + // directory will be left behind. + + assert (a == perform_clean_id && !t.is_a<fsdir> ()); + + target_state os (t.matched_state (a)); + + if (os == target_state::unchanged) + return false; + else + { + target_state ns; + if (os != target_state::changed) + { + phase_switch ps (t.ctx, run_phase::execute); + ns = execute_direct_sync (a, t); + } + else + ns = os; + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "cleaned " << t + << "; old state " << os + << "; new state " << ns;}); + return true; + } + else + return false; + } + } + + bool + clean_during_match_prerequisites (tracer& trace, + action a, target& t, + uintptr_t mask) + { + // Let's keep this as close to update_during_match_prerequisites() + // semantically as possible until we see a clear reason to deviate. + // + // Currently the only substantial change is the reverse iteration order. + + assert (a == perform_clean_id); + + prerequisite_targets& pts (t.prerequisite_targets[a]); + + // On the first pass detect and handle unchanged tragets. Note that we + // have to do it in a separate pass since we cannot call matched_state() + // once we've switched the phase. + // + size_t n (0); + + for (prerequisite_target& p: pts) + { + if (mask == 0 || (p.include & mask) != 0) + { + if (p.target != nullptr) + { + const target& pt (*p.target); + + assert (!pt.is_a<fsdir> ()); // See above. + + target_state os (pt.matched_state (a)); + + if (os != target_state::unchanged) + { + ++n; + p.data = static_cast<uintptr_t> (os); + continue; + } + } + + p.data = 0; + } + } + + // If all unchanged, we are done. + // + if (n == 0) + return false; + + // Provide additional information on what's going on. + // + auto df = make_diag_frame ( + [&t](const diag_record& dr) + { + if (verb != 0) + dr << info << "while cleaning during match prerequisites of " + << "target " << t; + }); + + context& ctx (t.ctx); + + phase_switch ps (ctx, run_phase::execute); + + bool r (false); + + // @@ Maybe we should optimize for n == 1? Maybe we should just call + // smarter clean_during_match() in this case? + // +#if 0 + for (prerequisite_target& p: reverse_iterate (pts)) + { + if ((mask == 0 || (p.include & mask) != 0) && p.data != 0) + { + const target& pt (*p.target); + + target_state os (static_cast<target_state> (p.data)); + target_state ns (execute_direct_sync (a, pt)); + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "cleaned " << pt + << "; old state " << os + << "; new state " << ns;}); + r = true; + } + + p.data = 0; + } + } +#else + + // Start asynchronous execution of prerequisites. Similar logic to + // straight_execute_members(). + // + // Note that the target's task count is expected to be busy (since this + // function is called during match). And there don't seem to be any + // problems in using it for execute. + // + atomic_count& tc (t[a].task_count); + + size_t busy (ctx.count_busy ()); + + wait_guard wg (ctx, busy, tc); + + for (prerequisite_target& p: reverse_iterate (pts)) + { + if ((mask == 0 || (p.include & mask) != 0) && p.data != 0) + { + execute_direct_async (a, *p.target, busy, tc); + } + } + + wg.wait (); + + // Finish execution and process the result. + // + for (prerequisite_target& p: reverse_iterate (pts)) + { + if ((mask == 0 || (p.include & mask) != 0) && p.data != 0) + { + const target& pt (*p.target); + target_state ns (execute_complete (a, pt)); + target_state os (static_cast<target_state> (p.data)); + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "cleaned " << pt + << "; old state " << os + << "; new state " << ns;}); + r = true; + } + + p.data = 0; + } + } +#endif + + return r; } static inline void @@ -2024,7 +3454,7 @@ namespace build2 static inline void blank_adhoc_member (prerequisite_target& pt) { - if (pt.adhoc) + if (pt.adhoc ()) pt.target = nullptr; } @@ -2036,7 +3466,6 @@ namespace build2 target_state r (target_state::unchanged); size_t busy (ctx.count_busy ()); - size_t exec (ctx.count_executed ()); // Start asynchronous execution of prerequisites. // @@ -2071,12 +3500,7 @@ namespace build2 continue; const target& mt (*ts[i]); - - // If the target is still busy, wait for its completion. - // - ctx.sched.wait (exec, mt[a].task_count, scheduler::work_none); - - r |= mt.executed_state (a); + r |= execute_complete (a, mt); blank_adhoc_member (ts[i]); } @@ -2094,7 +3518,6 @@ namespace build2 target_state r (target_state::unchanged); size_t busy (ctx.count_busy ()); - size_t exec (ctx.count_executed ()); wait_guard wg (ctx, busy, tc); @@ -2123,10 +3546,7 @@ namespace build2 continue; const target& mt (*ts[i]); - - ctx.sched.wait (exec, mt[a].task_count, scheduler::work_none); - - r |= mt.executed_state (a); + r |= execute_complete (a, mt); blank_adhoc_member (ts[i]); } @@ -2158,12 +3578,11 @@ namespace build2 const timestamp& mt, const execute_filter& ef, size_t n) { - context& ctx (t.ctx); + assert (a == perform_update_id); - assert (ctx.current_mode == execution_mode::first); + context& ctx (t.ctx); size_t busy (ctx.count_busy ()); - size_t exec (ctx.count_executed ()); auto& pts (t.prerequisite_targets[a]); @@ -2195,7 +3614,7 @@ namespace build2 wg.wait (); bool e (mt == timestamp_nonexistent); - const target* rt (tt != nullptr ? nullptr : &t); + const target* rt (nullptr); for (size_t i (0); i != n; ++i) { @@ -2205,15 +3624,102 @@ namespace build2 continue; const target& pt (*p.target); + target_state s (execute_complete (a, pt)); + rs |= s; - ctx.sched.wait (exec, pt[a].task_count, scheduler::work_none); + // Should we compare the timestamp to this target's? + // + if (!e && (p.adhoc () || !ef || ef (pt, i))) + { + // If this is an mtime-based target, then compare timestamps. + // + if (const mtime_target* mpt = pt.is_a<mtime_target> ()) + { + if (mpt->newer (mt, s)) + e = true; + } + else + { + // Otherwise we assume the prerequisite is newer if it was changed. + // + if (s == target_state::changed) + e = true; + } + } + + if (p.adhoc ()) + p.target = nullptr; // Blank out. + else if (tt != nullptr) + { + if (rt == nullptr && pt.is_a (*tt)) + rt = &pt; + } + } + + assert (tt == nullptr || rt != nullptr); + + return pair<optional<target_state>, const target*> ( + e ? optional<target_state> () : rs, rt); + } + + pair<optional<target_state>, const target*> + reverse_execute_prerequisites (const target_type* tt, + action a, const target& t, + const timestamp& mt, const execute_filter& ef, + size_t n) + { + assert (a == perform_update_id); + + context& ctx (t.ctx); + + size_t busy (ctx.count_busy ()); - target_state s (pt.executed_state (a)); + auto& pts (t.prerequisite_targets[a]); + + if (n == 0) + n = pts.size (); + + // Pretty much as reverse_execute_members() but hairier. + // + target_state rs (target_state::unchanged); + + wait_guard wg (ctx, busy, t[a].task_count); + + for (size_t i (n); i != 0; ) + { + const target*& pt (pts[--i]); + + if (pt == nullptr) // Skipped. + continue; + + target_state s (execute_async (a, *pt, busy, t[a].task_count)); + + if (s == target_state::postponed) + { + rs |= s; + pt = nullptr; + } + } + + wg.wait (); + + bool e (mt == timestamp_nonexistent); + const target* rt (nullptr); + + for (size_t i (n); i != 0; ) + { + prerequisite_target& p (pts[--i]); + + if (p == nullptr) + continue; + + const target& pt (*p.target); + target_state s (execute_complete (a, pt)); rs |= s; // Should we compare the timestamp to this target's? // - if (!e && (p.adhoc || !ef || ef (pt, i))) + if (!e && (p.adhoc () || !ef || ef (pt, i))) { // If this is an mtime-based target, then compare timestamps. // @@ -2231,26 +3737,27 @@ namespace build2 } } - if (p.adhoc) + if (p.adhoc ()) p.target = nullptr; // Blank out. - else + else if (tt != nullptr) { - if (rt == nullptr && pt.is_a (*tt)) + // Note that here we need last. + // + if (pt.is_a (*tt)) rt = &pt; } } - assert (rt != nullptr); + assert (tt == nullptr || rt != nullptr); return pair<optional<target_state>, const target*> ( - e ? optional<target_state> () : rs, - tt != nullptr ? rt : nullptr); + e ? optional<target_state> () : rs, rt); } target_state noop_action (action a, const target& t) { - text << "noop action triggered for " << diag_doing (a, t); + error << "noop action triggered for " << diag_doing (a, t); assert (false); // We shouldn't be called (see set_recipe()). return target_state::unchanged; } @@ -2264,10 +3771,13 @@ namespace build2 // const target& g (*t.group); - target_state gs (execute (a, g)); + // This is execute_sync(a, t, false) but that saves a call to + // executed_state() (which we don't need). + // + target_state gs (execute_impl (a, g, 0, nullptr)); if (gs == target_state::busy) - ctx.sched.wait (ctx.count_executed (), + ctx.sched->wait (ctx.count_executed (), g[a].task_count, scheduler::work_none); @@ -2295,102 +3805,105 @@ namespace build2 return execute_prerequisites (a, t); } - target_state - perform_clean_extra (action a, const file& ft, - const clean_extras& extras, - const clean_adhoc_extras& adhoc_extras) + static target_state + clean_extra (context& ctx, + const path& fp, + const clean_extras& es, + path& ep, bool& ed) { - // Clean the extras first and don't print the commands at verbosity level - // below 3. Note the first extra file/directory that actually got removed - // for diagnostics below. - // - // Note that dry-run is taken care of by the filesystem functions. - // - target_state er (target_state::unchanged); - bool ed (false); - path ep; + assert (!fp.empty ()); // Must be assigned. - context& ctx (ft.ctx); + target_state er (target_state::unchanged); - auto clean_extra = [&er, &ed, &ep, &ctx] (const file& f, - const path* fp, - const clean_extras& es) + for (const char* e: es) { - for (const char* e: es) - { - size_t n; - if (e == nullptr || (n = strlen (e)) == 0) - continue; + size_t n; + if (e == nullptr || (n = strlen (e)) == 0) + continue; - path p; - bool d; + path p; + bool d; - if (path::traits_type::absolute (e)) - { - p = path (e); - d = p.to_directory (); - } - else - { - if ((d = (e[n - 1] == '/'))) - --n; + if (path::traits_type::absolute (e)) + { + p = path (e); + d = p.to_directory (); + } + else + { + if ((d = (e[n - 1] == '/'))) + --n; - if (fp == nullptr) - { - fp = &f.path (); - assert (!fp->empty ()); // Must be assigned. - } + p = fp; + for (; *e == '-'; ++e) + p = p.base (); - p = *fp; - for (; *e == '-'; ++e) - p = p.base (); + p.append (e, n); + } - p.append (e, n); - } + target_state r (target_state::unchanged); - target_state r (target_state::unchanged); + if (d) + { + dir_path dp (path_cast<dir_path> (p)); - if (d) + switch (rmdir_r (ctx, dp, true, 3)) { - dir_path dp (path_cast<dir_path> (p)); - - switch (rmdir_r (ctx, dp, true, 3)) + case rmdir_status::success: { - case rmdir_status::success: - { - r = target_state::changed; - break; - } - case rmdir_status::not_empty: - { - if (verb >= 3) - text << dp << " is current working directory, not removing"; - break; - } - case rmdir_status::not_exist: + r = target_state::changed; break; } + case rmdir_status::not_empty: + { + if (verb >= 3) + info << dp << " is current working directory, not removing"; + break; + } + case rmdir_status::not_exist: + break; } - else - { - if (rmfile (ctx, p, 3)) - r = target_state::changed; - } - - if (r == target_state::changed && ep.empty ()) - { - ed = d; - ep = move (p); - } + } + else + { + if (rmfile (ctx, p, 3)) + r = target_state::changed; + } - er |= r; + if (r == target_state::changed && ep.empty ()) + { + ed = d; + ep = move (p); } - }; + + er |= r; + } + + return er; + } + + target_state + perform_clean_extra (action a, const file& ft, + const clean_extras& extras, + const clean_adhoc_extras& adhoc_extras, + bool show_adhoc) + { + context& ctx (ft.ctx); + + // Clean the extras first and don't print the commands at verbosity level + // below 3. Note the first extra file/directory that actually got removed + // for diagnostics below. + // + // Note that dry-run is taken care of by the filesystem functions. + // + target_state er (target_state::unchanged); + bool ed (false); + path ep; const path& fp (ft.path ()); if (!fp.empty () && !extras.empty ()) - clean_extra (ft, nullptr, extras); + er |= clean_extra (ctx, fp, extras, ep, ed); target_state tr (target_state::unchanged); @@ -2405,6 +3918,12 @@ namespace build2 // Now clean the ad hoc group file members, if any. // + // While at it, also collect the group target keys if we are showing + // the members. But only those that exist (since we don't want to + // print any diagnostics if none of them exist). + // + vector<target_key> tks; + for (const target* m (ft.adhoc_member); m != nullptr; m = m->adhoc_member) @@ -2425,7 +3944,7 @@ namespace build2 })); if (i != adhoc_extras.end ()) - clean_extra (*mf, mp, i->extras); + er |= clean_extra (ctx, *mp, i->extras, ep, ed); } if (!clean) @@ -2445,29 +3964,45 @@ namespace build2 ? target_state::changed : target_state::unchanged); - if (r == target_state::changed && ep.empty ()) - ep = *mp; - - er |= r; + if (r == target_state::changed) + { + if (show_adhoc && verb == 1) + tks.push_back (mf->key ()); + else if (ep.empty ()) + { + ep = *mp; + er |= r; + } + } } } // Now clean the primary target and its prerequisited in the reverse order // of update: first remove the file, then clean the prerequisites. // - if (clean && !fp.empty () && rmfile (fp, ft)) - tr = target_state::changed; + if (clean && !fp.empty ()) + { + if (show_adhoc && verb == 1 && !tks.empty ()) + { + if (rmfile (fp, ft, 2 /* verbosity */)) + tks.insert (tks.begin (), ft.key ()); + + print_diag ("rm", move (tks)); + tr = target_state::changed; + } + else + { + if (rmfile (fp, ft)) + tr = target_state::changed; + } + } // Update timestamp in case there are operations after us that could use // the information. // ft.mtime (timestamp_nonexistent); - // Clean prerequisites. - // - tr |= reverse_execute_prerequisites (a, ft); - - // Factor the result of removing the extra files into the target state. + // We factor the result of removing the extra files into the target state. // While strictly speaking removing them doesn't change the target state, // if we don't do this, then we may end up removing the file but still // saying that everything is clean (e.g., if someone removes the target @@ -2480,41 +4015,45 @@ namespace build2 { if (verb > (ctx.current_diag_noise ? 0 : 1) && verb < 3) { - if (ed) - text << "rm -r " << path_cast<dir_path> (ep); - else - text << "rm " << ep; + if (verb >= 2) + { + if (ed) + text << "rm -r " << path_cast<dir_path> (ep); + else + text << "rm " << ep; + } + else if (verb) + { + if (ed) + print_diag ("rm -r", path_cast<dir_path> (ep)); + else + print_diag ("rm", ep); + } } } + // Clean prerequisites. + // + tr |= reverse_execute_prerequisites (a, ft); + tr |= er; return tr; } target_state - perform_clean (action a, const target& t) + perform_clean_group_extra (action a, const mtime_target& g, + const clean_extras& extras) { - const file& f (t.as<file> ()); - assert (!f.path ().empty ()); - return perform_clean_extra (a, f, {}); - } + context& ctx (g.ctx); - target_state - perform_clean_depdb (action a, const target& t) - { - const file& f (t.as<file> ()); - assert (!f.path ().empty ()); - return perform_clean_extra (a, f, {".d"}); - } + target_state er (target_state::unchanged); + bool ed (false); + path ep; - target_state - perform_clean_group (action a, const target& xg) - { - const mtime_target& g (xg.as<mtime_target> ()); + if (!extras.empty ()) + er |= clean_extra (ctx, g.dir / path (g.name), extras, ep, ed); - // Similar logic to perform_clean_extra() above. - // - target_state r (target_state::unchanged); + target_state tr (target_state::unchanged); if (cast_true<bool> (g[g.ctx.var_clean])) { @@ -2522,54 +4061,93 @@ namespace build2 { if (const target* m = gv.members[gv.count - 1]) { - if (rmfile (m->as<file> ().path (), *m)) - r |= target_state::changed; + // Note that at the verbosity level 1 we don't show the removal of + // each group member. This is consistent with what is normally shown + // during update. + // + if (rmfile (m->as<file> ().path (), *m, 2 /* verbosity */)) + tr |= target_state::changed; } } + + if (tr == target_state::changed && verb == 1) + print_diag ("rm", g); } g.mtime (timestamp_nonexistent); - r |= reverse_execute_prerequisites (a, g); - return r; + if (tr != target_state::changed && er == target_state::changed) + { + if (verb > (ctx.current_diag_noise ? 0 : 1) && verb < 3) + { + if (verb >= 2) + { + if (ed) + text << "rm -r " << path_cast<dir_path> (ep); + else + text << "rm " << ep; + } + else if (verb) + { + if (ed) + print_diag ("rm -r", path_cast<dir_path> (ep)); + else + print_diag ("rm", ep); + } + } + } + + tr |= reverse_execute_prerequisites (a, g); + + tr |= er; + return tr; + } + + target_state + perform_clean (action a, const target& t) + { + const file& f (t.as<file> ()); + assert (!f.path ().empty ()); + return perform_clean_extra (a, f, {}); } target_state - perform_clean_group_depdb (action a, const target& g) + perform_clean_depdb (action a, const target& t) { - context& ctx (g.ctx); + const file& f (t.as<file> ()); + assert (!f.path ().empty ()); + return perform_clean_extra (a, f, {".d"}); + } - // The same twisted target state merging logic as in perform_clean_extra(). - // - target_state er (target_state::unchanged); - path ep; + target_state + perform_clean_group (action a, const target& t) + { + return perform_clean_group_extra (a, t.as<mtime_target> (), {}); + } - group_view gv (g.group_members (a)); - if (gv.count != 0) + target_state + perform_clean_group_depdb (action a, const target& t) + { + path d; + clean_extras extras; { - for (size_t i (0); i != gv.count; ++i) + group_view gv (t.group_members (a)); + if (gv.count != 0) { - if (const target* m = gv.members[i]) + for (size_t i (0); i != gv.count; ++i) { - ep = m->as<file> ().path () + ".d"; - break; + if (const target* m = gv.members[i]) + { + d = m->as<file> ().path () + ".d"; + break; + } } - } - assert (!ep.empty ()); - if (rmfile (ctx, ep, 3)) - er = target_state::changed; - } - - target_state tr (perform_clean_group (a, g)); - - if (tr != target_state::changed && er == target_state::changed) - { - if (verb > (ctx.current_diag_noise ? 0 : 1) && verb < 3) - text << "rm " << ep; + assert (!d.empty ()); + extras.push_back (d.string ().c_str ()); + } } - tr |= er; - return tr; + return perform_clean_group_extra (a, t.as<mtime_target> (), extras); } } |