diff options
Diffstat (limited to 'libbuild2/adhoc-rule-buildscript.cxx')
| -rw-r--r-- | libbuild2/adhoc-rule-buildscript.cxx | 1529 |
1 files changed, 1331 insertions, 198 deletions
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx index 03e810d..3e868a6 100644 --- a/libbuild2/adhoc-rule-buildscript.cxx +++ b/libbuild2/adhoc-rule-buildscript.cxx @@ -5,6 +5,8 @@ #include <sstream> +#include <libbutl/filesystem.hxx> // try_rm_file(), path_entry() + #include <libbuild2/depdb.hxx> #include <libbuild2/scope.hxx> #include <libbuild2/target.hxx> @@ -27,10 +29,11 @@ namespace build2 static inline void hash_script_vars (sha256& cs, const build::script::script& s, + const scope& bs, const target& t, names& storage) { - context& ctx (t.ctx); + auto& vp (bs.var_pool ()); for (const string& n: s.vars) { @@ -38,7 +41,7 @@ namespace build2 lookup l; - if (const variable* var = ctx.var_pool.find (n)) + if (const variable* var = vp.find (n)) l = t[var]; cs.append (!l.defined () ? '\x1' : l->null ? '\x2' : '\x3'); @@ -46,7 +49,7 @@ namespace build2 if (l) { storage.clear (); - names_view ns (reverse (*l, storage)); + names_view ns (reverse (*l, storage, true /* reduce */)); for (const name& n: ns) to_checksum (cs, n); @@ -198,11 +201,7 @@ namespace build2 os << ind << "depdb clear" << endl; script::dump (os, ind, script.depdb_preamble); - - if (script.diag_line) - { - os << ind; script::dump (os, *script.diag_line, true /* newline */); - } + script::dump (os, ind, script.diag_preamble); script::dump (os, ind, script.body); ind.resize (ind.size () - 2); @@ -212,22 +211,28 @@ namespace build2 bool adhoc_buildscript_rule:: reverse_fallback (action a, const target_type& tt) const { - // We can provide clean for a file target if we are providing update. + // We can provide clean for a file or group target if we are providing + // update. // - return a == perform_clean_id && tt.is_a<file> () && - find (actions.begin (), actions.end (), - perform_update_id) != actions.end (); + return (a == perform_clean_id && + (tt.is_a<file> () || tt.is_a<group> ()) && + find (actions.begin (), actions.end (), + perform_update_id) != actions.end ()); } + using dynamic_target = build::script::parser::dynamic_target; + using dynamic_targets = build::script::parser::dynamic_targets; + struct adhoc_buildscript_rule::match_data { - match_data (action a, const target& t, bool temp_dir) - : env (a, t, temp_dir) {} + match_data (action a, const target& t, const scope& bs, bool temp_dir) + : env (a, t, bs, temp_dir) {} build::script::environment env; build::script::default_runner run; path dd; + dynamic_targets dyn_targets; const scope* bs; timestamp mt; @@ -236,8 +241,10 @@ namespace build2 struct adhoc_buildscript_rule::match_data_byproduct { - match_data_byproduct (action a, const target& t, bool temp_dir) - : env (a, t, temp_dir) {} + match_data_byproduct (action a, const target& t, + const scope& bs, + bool temp_dir) + : env (a, t, bs, temp_dir) {} build::script::environment env; build::script::default_runner run; @@ -253,22 +260,27 @@ namespace build2 }; bool adhoc_buildscript_rule:: - match (action a, target& t, const string& h, match_extra& me) const + match (action a, target& xt, const string& h, match_extra& me) const { + const target& t (xt); // See adhoc_rule::match(). + // We pre-parsed the script with the assumption it will be used on a - // non/file-based target. Note that this should not be possible with - // patterns. + // non/file-based (or file group-based) target. Note that this should not + // be possible with patterns. // if (pattern == nullptr) { - if ((t.is_a<file> () != nullptr) != ttype->is_a<file> ()) - { + // Let's not allow mixing file/group. + // + if ((t.is_a<file> () != nullptr) == ttype->is_a<file> () || + (t.is_a<group> () != nullptr) == ttype->is_a<group> ()) + ; + else fail (loc) << "incompatible target types used with shared recipe" << - info << "all targets must be file-based or non-file-based"; - } + info << "all targets must be file- or file group-based or non"; } - return adhoc_rule::match (a, t, h, me); + return adhoc_rule::match (a, xt, h, me); } recipe adhoc_buildscript_rule:: @@ -279,17 +291,28 @@ namespace build2 recipe adhoc_buildscript_rule:: apply (action a, - target& xt, + target& t, match_extra& me, - const optional<timestamp>& d) const + const optional<timestamp>& deadline) const { tracer trace ("adhoc_buildscript_rule::apply"); + // Handle matching group members (see adhoc_rule::match() for background). + // + if (const group* g = t.group != nullptr ? t.group->is_a<group> () : nullptr) + { + // Note: this looks very similar to how we handle ad hoc group members. + // + match_sync (a, *g, 0 /* options */); + return group_recipe; // Execute the group's recipe. + } + // We don't support deadlines for any of these cases (see below). // - if (d && (a.outer () || - me.fallback || - (a == perform_update_id && xt.is_a<file> ()))) + if (deadline && (a.outer () || + me.fallback || + (a == perform_update_id && + (t.is_a<file> () || t.is_a<group> ())))) return empty_recipe; // If this is an outer operation (e.g., update-for-test), then delegate to @@ -297,26 +320,98 @@ namespace build2 // if (a.outer ()) { - match_inner (a, xt); - return execute_inner; + match_inner (a, t); + return inner_recipe; } - context& ctx (xt.ctx); - const scope& bs (xt.base_scope ()); + context& ctx (t.ctx); + const scope& bs (t.base_scope ()); - // Inject pattern's ad hoc group members, if any. + group* g (t.is_a<group> ()); // Explicit group. + + // Inject pattern's ad hoc group members, if any (explicit group members + // are injected after reset below). // - if (pattern != nullptr) - pattern->apply_adhoc_members (a, xt, bs, me); + if (g == nullptr && pattern != nullptr) + pattern->apply_group_members (a, t, bs, me); - // Derive file names for the target and its ad hoc group members, if any. + // Derive file names for the target and its static/ad hoc group members, + // if any. // if (a == perform_update_id || a == perform_clean_id) { - for (target* m (&xt); m != nullptr; m = m->adhoc_member) + if (g != nullptr) + { + g->reset_members (a); // See group::group_members() for background. + + // Note that we rely on the fact that if the group has static members, + // then they always come first in members and the first static member + // is a file. + // + for (const target& m: g->static_members) + g->members.push_back (&m); + + g->members_static = g->members.size (); + + if (pattern != nullptr) + { + pattern->apply_group_members (a, *g, bs, me); + g->members_static = g->members.size (); + } + + if (g->members_static == 0) + { + if (!script.depdb_dyndep_dyn_target) + fail << "group " << *g << " has no static or dynamic members"; + } + else + { + if (!g->members.front ()->is_a<file> ()) + { + // We use the first static member to derive depdb path, get mtime, + // etc. So it must be file-based. + // + fail << "first static member " << g->members.front () + << " of group " << *g << " is not a file"; + } + + // Derive paths for all the static members. + // + for (const target* m: g->members) + if (auto* p = m->is_a<path_target> ()) + p->derive_path (); + } + } + else + { + for (target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (auto* p = m->is_a<path_target> ()) + p->derive_path (); + } + } + } + else if (g != nullptr) + { + // This could be, for example, configure/dist update which could need a + // "representative sample" of members (in order to be able to match the + // rules). So add static members unless we already have something + // cached. + // + if (g->group_members (a).members == nullptr) // Note: not g->member. { - if (auto* p = m->is_a<path_target> ()) - p->derive_path (); + g->reset_members (a); + + for (const target& m: g->static_members) + g->members.push_back (&m); + + g->members_static = g->members.size (); + + if (pattern != nullptr) + { + pattern->apply_group_members (a, *g, bs, me); + g->members_static = g->members.size (); + } } } @@ -325,22 +420,26 @@ namespace build2 // We do it always instead of only if one of the targets is path-based in // case the recipe creates temporary files or some such. // - const fsdir* dir (inject_fsdir (a, xt)); + // Note that we disable the prerequisite search for fsdir{} because of the + // prerequisites injected by the pattern. So we have to handle this ad hoc + // below. + // + const fsdir* dir (inject_fsdir (a, t, true /*match*/, false /*prereq*/)); // Match prerequisites. // // This is essentially match_prerequisite_members() but with support // for update=unmatch|match. // - auto& pts (xt.prerequisite_targets[a]); + auto& pts (t.prerequisite_targets[a]); { // Re-create the clean semantics as in match_prerequisite_members(). // - bool clean (a.operation () == clean_id && !xt.is_a<alias> ()); + bool clean (a.operation () == clean_id && !t.is_a<alias> ()); // Add target's prerequisites. // - for (prerequisite_member p: group_prerequisite_members (a, xt)) + for (prerequisite_member p: group_prerequisite_members (a, t)) { // Note that we have to recognize update=unmatch|match for *(update), // not just perform(update). But only actually do anything about it @@ -348,10 +447,10 @@ namespace build2 // lookup l; // The `update` variable value, if any. include_type pi ( - include (a, xt, p, a.operation () == update_id ? &l : nullptr)); + include (a, t, p, a.operation () == update_id ? &l : nullptr)); - // Use bit 2 of prerequisite_target::include to signal update during - // match and bit 3 -- unmatch. + // Use prerequisite_target::include to signal update during match or + // unmatch. // uintptr_t mask (0); if (l) @@ -361,12 +460,12 @@ namespace build2 if (v == "match") { if (a == perform_update_id) - mask = 2; + mask = prerequisite_target::include_udm; } else if (v == "unmatch") { if (a == perform_update_id) - mask = 4; + mask = include_unmatch; } else if (v != "false" && v != "true" && v != "execute") { @@ -380,7 +479,10 @@ namespace build2 if (!pi) continue; - const target& pt (p.search (xt)); + const target& pt (p.search (t)); + + if (&pt == dir) // Don't add injected fsdir{} twice. + continue; if (clean && !pt.in (*bs.root_scope ())) continue; @@ -396,19 +498,19 @@ namespace build2 // Inject pattern's prerequisites, if any. // if (pattern != nullptr) - pattern->apply_prerequisites (a, xt, bs, me); + pattern->apply_prerequisites (a, t, bs, me); // Start asynchronous matching of prerequisites. Wait with unlocked // phase to allow phase switching. // - wait_guard wg (ctx, ctx.count_busy (), xt[a].task_count, true); + wait_guard wg (ctx, ctx.count_busy (), t[a].task_count, true); for (const prerequisite_target& pt: pts) { - if (pt.target == dir) + if (pt.target == dir) // Don't match injected fsdir{} twice. continue; - match_async (a, *pt.target, ctx.count_busy (), xt[a].task_count); + match_async (a, *pt.target, ctx.count_busy (), t[a].task_count); } wg.wait (); @@ -417,12 +519,14 @@ namespace build2 // for (prerequisite_target& pt: pts) { - if (pt.target == dir) + if (pt.target == dir) // See above. continue; // Handle update=unmatch. // - unmatch um ((pt.include & 4) != 0 ? unmatch::safe : unmatch::none); + unmatch um ((pt.include & include_unmatch) != 0 + ? unmatch::safe + : unmatch::none); pair<bool, target_state> mr (match_complete (a, *pt.target, um)); @@ -431,7 +535,7 @@ namespace build2 l6 ([&]{trace << "unmatch " << *pt.target << ": " << mr.first;}); // If we managed to unmatch, blank it out so that it's not executed, - // etc. Otherwise, convert it to ad hoc (we also automatically avoid + // etc. Otherwise, leave it as is (but we still automatically avoid // hashing it, updating it during match in exec_depdb_dyndep(), and // making us out of date in execute_update_prerequisites()). // @@ -441,18 +545,213 @@ namespace build2 // information (e.g., poptions from a library) and those will be // change-tracked. // + // Note: set the include_target flag for the updated_during_match() + // check. + // if (mr.first) + { + pt.data = reinterpret_cast<uintptr_t> (pt.target); pt.target = nullptr; - else - pt.include |= 1; + pt.include |= prerequisite_target::include_target; + + // Note that this prerequisite could also be ad hoc and we must + // clear that flag if we managed to unmatch (failed that we will + // treat it as ordinary ad hoc since it has the target pointer in + // data). + // + // But that makes it impossible to distinguish ad hoc unmatch from + // ordinary unmatch prerequisites later when setting $<. Another + // flag to the rescue. + // + if ((pt.include & prerequisite_target::include_adhoc) != 0) + { + pt.include &= ~prerequisite_target::include_adhoc; + pt.include |= include_unmatch_adhoc; + } + } } } } + // Read the list of dynamic targets and, optionally, fsdir{} prerequisites + // from depdb, if exists (used in a few depdb-dyndep --dyn-target handling + // places below). + // + auto read_dyn_targets = [] (path ddp, bool fsdir) + -> pair<dynamic_targets, dir_paths> + { + depdb dd (move (ddp), true /* read_only */); + + pair<dynamic_targets, dir_paths> r; + while (dd.reading ()) // Breakout loop. + { + string* l; + auto read = [&dd, &l] () -> bool + { + return (l = dd.read ()) != nullptr; + }; + + if (!read ()) // Rule id. + break; + + // We can omit this for as long as we don't break our blank line + // anchors semantics. + // +#if 0 + if (*l != rule_id_) + fail << "unable to clean dynamic target group " << t + << " with old depdb"; +#endif + + // Note that we cannot read out expected lines since there can be + // custom depdb builtins. So we use the blank lines as anchors to + // skip to the parts we need. + // + // Skip until the first blank that separated custom depdb entries from + // the prerequisites list. + { + bool g; + while ((g = read ()) && !l->empty ()) ; + if (!g) + break; + } + + // Next read the prerequisites, detecting fsdir{} entries if asked. + // + { + bool g; + while ((g = read ()) && !l->empty ()) + { + if (fsdir) + { + path p (*l); + if (p.to_directory ()) + r.second.push_back (path_cast<dir_path> (move (p))); + } + } + + if (!g) + break; + } + + // Read the dynamic target files. We should always end with a blank + // line. + // + for (;;) + { + if (!read () || l->empty ()) + break; + + // Split into type and path. + // + size_t p (l->find (' ')); + if (p == string::npos || // Invalid format. + p == 0 || // Empty type. + p + 1 == l->size ()) // Empty path. + break; + + r.first.push_back ( + dynamic_target {string (*l, 0, p), path (*l, p + 1, string::npos)}); + } + + break; + } + + return r; + }; + + // Target path to derive the depdb path, query mtime (if file), etc. + // + // To derive the depdb path for a group with at least one static member we + // use the path of the first member. For a group without any static + // members we use the group name with the target type name as the + // second-level extension. + // + auto target_path = [&t, g, p = path ()] () mutable -> const path& + { + return + g == nullptr ? t.as<file> ().path () : + g->members_static != 0 ? g->members.front ()->as<file> ().path () : + (p = g->dir / (g->name + '.' + g->type ().name)); + }; + // See if we are providing the standard clean as a fallback. // if (me.fallback) - return &perform_clean_file; + { + // For depdb-dyndep --dyn-target use depdb to clean dynamic targets. + // + if (script.depdb_dyndep_dyn_target) + { + // Note that only removing the relevant filesystem entries is not + // enough: we actually have to populate the group with members since + // this information could be used to clean derived targets (for + // example, object files). So we just do that and let the standard + // clean logic take care of them the same as static members. + // + // NOTE that this logic should be consistent with what we have in + // exec_depdb_dyndep(). + // + using dyndep = dyndep_rule; + + function<dyndep::group_filter_func> filter; + if (g != nullptr) + { + filter = [] (mtime_target& g, const build2::file& m) + { + auto& ms (g.as<group> ().members); + return find (ms.begin (), ms.end (), &m) == ms.end (); + }; + } + + pair<dynamic_targets, dir_paths> p ( + read_dyn_targets (target_path () + ".d", true)); + + for (dynamic_target& dt: p.first) + { + path& f (dt.path); + + // Resolve target type. Clean it as file if unable to. + // + const target_type* tt (bs.find_target_type (dt.type)); + if (tt == nullptr) + tt = &file::static_type; + + if (g != nullptr) + { + pair<const build2::file&, bool> r ( + dyndep::inject_group_member (a, bs, *g, move (f), *tt, filter)); + + if (r.second) + g->members.push_back (&r.first); + } + else + { + // Note that here we don't bother cleaning any old dynamic targets + // -- the more we can clean, the merrier. + // + dyndep::inject_adhoc_group_member (a, bs, t, move (f), *tt); + } + } + + // Enter fsdir{} prerequisites. + // + // See the add lambda in exec_depdb_dyndep() for background. + // + for (dir_path& d: p.second) + { + dir_path o; string n; // For GCC 13 -Wdangling-reference. + const fsdir& dt (search<fsdir> (t, + move (d), + move (o), + move (n), nullptr, nullptr)); + match_sync (a, dt); + pts.push_back (prerequisite_target (&dt, true /* adhoc */)); + } + } + + return g == nullptr ? perform_clean_file : perform_clean_group; + } // If we have any update during match prerequisites, now is the time to // update them. @@ -464,17 +763,17 @@ namespace build2 // prerequisite_target::data. // if (a == perform_update_id) - update_during_match_prerequisites (trace, a, xt, 2 /* mask */); + update_during_match_prerequisites (trace, a, t); - // See if this is not update or not on a file-based target. + // See if this is not update or not on a file/group-based target. // - if (a != perform_update_id || !xt.is_a<file> ()) + if (a != perform_update_id || !(g != nullptr || t.is_a<file> ())) { // Make sure we get small object optimization. // - if (d) + if (deadline) { - return [dv = *d, this] (action a, const target& t) + return [dv = *deadline, this] (action a, const target& t) { return default_action (a, t, dv); }; @@ -488,20 +787,22 @@ namespace build2 } } + // This is a perform update on a file or group target. + // // See if this is the simple case with only static dependencies. // if (!script.depdb_dyndep) { return [this] (action a, const target& t) { - return perform_update_file (a, t); + return perform_update_file_or_group (a, t); }; } - // This is a perform update on a file target with extraction of dynamic - // dependency information either in the depdb preamble (depdb-dyndep - // without --byproduct) or as a byproduct of the recipe body execution - // (depdb-dyndep with --byproduct). + // This is a perform update on a file or group target with extraction of + // dynamic dependency information either in the depdb preamble + // (depdb-dyndep without --byproduct) or as a byproduct of the recipe body + // execution (depdb-dyndep with --byproduct). // // For the former case, we may need to add additional prerequisites (or // even target group members). We also have to save any such additional @@ -519,18 +820,38 @@ namespace build2 // example and all this logic is based on the prior work in the cc module // where you can often find more detailed rationale for some of the steps // performed (like the fsdir update below). + + // Re-acquire fsdir{} specified by the user, similar to inject_fsdir() + // (which we have disabled; see above). // - file& t (xt.as<file> ()); - const path& tp (t.path ()); + if (dir == nullptr) + { + for (const target* pt: pts) + { + if (pt != nullptr) + { + if (const fsdir* dt = pt->is_a<fsdir> ()) + { + if (dt->dir == t.dir) + { + dir = dt; + break; + } + } + } + } + } if (dir != nullptr) - fsdir_rule::perform_update_direct (a, t); + fsdir_rule::perform_update_direct (a, *dir); // Because the depdb preamble can access $<, we have to blank out all the // ad hoc prerequisites. Since we will still need them later, we "move" // them to the auxiliary data member in prerequisite_target (see // execute_update_prerequisites() for details). // + // Note: set the include_target flag for the updated_during_match() check. + // for (prerequisite_target& p: pts) { // Note that fsdir{} injected above is adhoc. @@ -539,13 +860,57 @@ namespace build2 { p.data = reinterpret_cast<uintptr_t> (p.target); p.target = nullptr; + p.include |= prerequisite_target::include_target; } } + const path& tp (target_path ()); + + // Note that while it's tempting to turn match_data* into recipes, some of + // their members are not movable. And in the end we will have the same + // result: one dynamic memory allocation. + // + unique_ptr<match_data> md; + unique_ptr<match_data_byproduct> mdb; + + dynamic_targets old_dyn_targets; + + if (script.depdb_dyndep_byproduct) + { + mdb.reset (new match_data_byproduct ( + a, t, bs, script.depdb_preamble_temp_dir)); + } + else + { + md.reset (new match_data (a, t, bs, script.depdb_preamble_temp_dir)); + + // If the set of dynamic targets can change based on changes to the + // inputs (say, each entity, such as a type, in the input file gets its + // own output file), then we can end up with a large number of old + // output files laying around because they are not part of the new + // dynamic target set. So we try to clean them up based on the old depdb + // information, similar to how we do it for perform_clean above (except + // here we will just keep the list of old files). + // + // Note: do before opening depdb, which can start over-writing it. + // + // We also have to do this speculatively, without knowing whether we + // will need to update. Oh, well, being dynamic ain't free. + // + if (script.depdb_dyndep_dyn_target) + old_dyn_targets = read_dyn_targets (tp + ".d", false).first; + } + depdb dd (tp + ".d"); // NOTE: see the "static dependencies" version (with comments) below. // + // NOTE: We use blank lines as anchors to skip directly to certain entries + // (e.g., dynamic targets). So make sure none of the other entries + // can be blank (for example, see `depdb string` builtin). + // + // NOTE: KEEP IN SYNC WITH read_dyn_targets ABOVE! + // if (dd.expect ("<ad hoc buildscript recipe> 1") != nullptr) l4 ([&]{trace << "rule mismatch forcing update of " << t;}); @@ -565,7 +930,7 @@ namespace build2 p.adhoc () ? reinterpret_cast<target*> (p.data) : nullptr)) { - if ((p.include & 4) != 0) // Skip update=unmatch. + if ((p.include & include_unmatch) != 0) // Skip update=unmatch. continue; hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage); @@ -574,16 +939,38 @@ namespace build2 { sha256 cs; - hash_script_vars (cs, script, t, storage); + hash_script_vars (cs, script, bs, t, storage); if (dd.expect (cs.string ()) != nullptr) l4 ([&]{trace << "recipe variable change forcing update of " << t;}); } + // Static targets and prerequisites (there can also be dynamic targets; + // see dyndep --dyn-target). + // { sha256 tcs; - for (const target* m (&t); m != nullptr; m = m->adhoc_member) - hash_target (tcs, *m, storage); + if (g == nullptr) + { + // There is a nuance: in an operation batch (e.g., `b update + // update`) we will already have the dynamic targets as members on + // the subsequent operations and we need to make sure we don't treat + // them as static. Using target_decl to distinguish the two seems + // like a natural way. + // + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (m->decl == target_decl::real) + hash_target (tcs, *m, storage); + } + } + else + { + // Feels like there is not much sense in hashing the group itself. + // + for (const target* m: g->members) + hash_target (tcs, *m, storage); + } if (dd.expect (tcs.string ()) != nullptr) l4 ([&]{trace << "target set change forcing update of " << t;}); @@ -601,22 +988,8 @@ namespace build2 } } - // Note that while it's tempting to turn match_data* into recipes, some of - // their members are not movable. And in the end we will have the same - // result: one dynamic memory allocation. + // Get ready to run the depdb preamble. // - unique_ptr<match_data> md; - unique_ptr<match_data_byproduct> mdb; - - if (script.depdb_dyndep_byproduct) - { - mdb.reset (new match_data_byproduct ( - a, t, script.depdb_preamble_temp_dir)); - } - else - md.reset (new match_data (a, t, script.depdb_preamble_temp_dir)); - - build::script::environment& env (mdb != nullptr ? mdb->env : md->env); build::script::default_runner& run (mdb != nullptr ? mdb->run : md->run); @@ -627,21 +1000,51 @@ namespace build2 { build::script::parser p (ctx); p.execute_depdb_preamble (a, bs, t, env, script, run, dd); + + // Write a blank line after the custom depdb entries and before + // prerequisites, which we use as an anchor (see read_dyn_targets + // above). We only do it for the new --dyn-target mode in order not to + // invalidate the existing depdb instances. + // + if (script.depdb_dyndep_dyn_target) + dd.expect (""); } // Determine if we need to do an update based on the above checks. // - bool update; + bool update (false); timestamp mt; if (dd.writing ()) update = true; else { - if ((mt = t.mtime ()) == timestamp_unknown) - t.mtime (mt = mtime (tp)); // Cache. + if (g == nullptr) + { + const file& ft (t.as<file> ()); - update = dd.mtime > mt; + if ((mt = ft.mtime ()) == timestamp_unknown) + ft.mtime (mt = mtime (tp)); // Cache. + } + else + { + // Use static member, old dynamic, or force update. + // + const path* p ( + g->members_static != 0 + ? &tp /* first static member path */ + : (!old_dyn_targets.empty () + ? &old_dyn_targets.front ().path + : nullptr)); + + if (p != nullptr) + mt = g->load_mtime (*p); + else + update = true; + } + + if (!update) + update = dd.mtime > mt; } if (update) @@ -665,7 +1068,8 @@ namespace build2 // update on encountering any non-existent files in depbd, we may // actually incorrectly "validate" some number of depdb entires while // having an out-of-date main source file. We could probably avoid the - // update if we are already updating. + // update if we are already updating (or not: there is pre-generation + // to consider; see inject_existing_file() for details). // { build::script::parser p (ctx); @@ -701,7 +1105,7 @@ namespace build2 size_t& skip_count (mdb->skip_count); auto add = [&trace, what, - a, &bs, &t, + a, &bs, &t, pts_n = mdb->pts_n, &byp, &map_ext, &skip_count, mt] (path fp) -> optional<bool> { @@ -716,7 +1120,7 @@ namespace build2 // if (optional<bool> u = dyndep::inject_existing_file ( trace, what, - a, t, + a, t, pts_n, *ft, mt, false /* fail */, false /* adhoc */, @@ -776,7 +1180,7 @@ namespace build2 // Note that in case of dry run we will have an incomplete (but valid) // database which will be updated on the next non-dry run. // - if (!update || ctx.dry_run) + if (!update || ctx.dry_run_option) dd.close (false /* mtime_check */); else mdb->dd = dd.close_to_reopen (); @@ -788,59 +1192,107 @@ namespace build2 return [this, md = move (mdb)] (action a, const target& t) { - return perform_update_file_dyndep_byproduct (a, t, *md); + return perform_update_file_or_group_dyndep_byproduct (a, t, *md); }; } else { // Run the second half of the preamble (depdb-dyndep commands) to update - // our prerequisite targets and extract dynamic dependencies. + // our prerequisite targets and extract dynamic dependencies (targets + // and prerequisites). // // Note that this should be the last update to depdb (the invalidation // order semantics). // - bool deferred_failure (false); + md->deferred_failure = false; { build::script::parser p (ctx); p.execute_depdb_preamble_dyndep (a, bs, t, env, script, run, dd, + md->dyn_targets, update, mt, - deferred_failure); + md->deferred_failure); } - if (update && dd.reading () && !ctx.dry_run) + if (update && dd.reading () && !ctx.dry_run_option) dd.touch = timestamp_unknown; dd.close (false /* mtime_check */); - md->dd = move (dd.path); - // Pass on base scope and update/mtime. + // Remove previous dynamic targets since their set may change with + // changes to the inputs. + // + // The dry-run mode complicates things: if we don't remove the old + // files, then that information will be gone (since we update depdb even + // in the dry-run mode). But if we remove everything in the dry-run + // mode, then we may also remove some of the current files, which would + // be incorrect. So let's always remove but only files that are not in + // the current set. + // + // Note that we used to do this in perform_update_file_or_group_dyndep() + // but that had a tricky issue: if we end up performing match but not + // execute (e.g., via the resolve_members() logic), then we will not + // cleanup old targets but loose this information (since the depdb has + // be updated). So now we do it here, which is a bit strange, but it + // sort of fits into that dry-run logic above. Note also that we do this + // unconditionally, update or not, since if everything is up to date, + // then old and new sets should be the same. + // + for (const dynamic_target& dt: old_dyn_targets) + { + const path& f (dt.path); + + if (find_if (md->dyn_targets.begin (), md->dyn_targets.end (), + [&f] (const dynamic_target& dt) + { + return dt.path == f; + }) == md->dyn_targets.end ()) + { + // This is an optimization so best effort. + // + if (optional<rmfile_status> s = butl::try_rmfile_ignore_error (f)) + { + if (s == rmfile_status::success && verb >= 2) + text << "rm " << f; + } + } + } + + // Pass on the base scope, depdb path, and update/mtime. // md->bs = &bs; + md->dd = move (dd.path); md->mt = update ? timestamp_nonexistent : mt; - md->deferred_failure = deferred_failure; return [this, md = move (md)] (action a, const target& t) { - return perform_update_file_dyndep (a, t, *md); + return perform_update_file_or_group_dyndep (a, t, *md); }; } } target_state adhoc_buildscript_rule:: - perform_update_file_dyndep_byproduct (action a, - const target& xt, - match_data_byproduct& md) const + perform_update_file_or_group_dyndep_byproduct ( + action a, const target& t, match_data_byproduct& md) const { // Note: using shared function name among the three variants. // - tracer trace ("adhoc_buildscript_rule::perform_update_file"); + tracer trace ( + "adhoc_buildscript_rule::perform_update_file_or_group_dyndep_byproduct"); - context& ctx (xt.ctx); + context& ctx (t.ctx); - const file& t (xt.as<file> ()); + // For a group we use the first (for now static) member as a source of + // mtime. + // + // @@ TODO: expl: byproduct: Note that until we support dynamic targets in + // the byproduct mode, we verify there is at least one static member in + // apply() above. Once we do support this, we will need to verify after + // the dependency extraction below. + // + const group* g (t.is_a<group> ()); // Note that even if we've updated all our prerequisites in apply(), we // still need to execute them here to keep the dependency counts straight. @@ -869,7 +1321,14 @@ namespace build2 if (!ctx.dry_run || verb != 0) { - execute_update_file (bs, a, t, env, run); + if (g == nullptr) + execute_update_file (bs, a, t.as<file> (), env, run); + else + { + // Note: no dynamic members yet. + // + execute_update_group (bs, a, *g, env, run); + } } // Extract the dynamic dependency information as byproduct of the recipe @@ -909,15 +1368,25 @@ namespace build2 const auto& pts (t.prerequisite_targets[a]); auto add = [&trace, what, - a, &bs, &t, &pts, pts_n = md.pts_n, + a, &bs, &t, g, &pts, pts_n = md.pts_n, &byp, &map_ext, &dd, &skip] (path fp) { normalize_external (fp, what); + // Note that unless we take into account dynamic targets, the skip + // logic below falls apart since we neither see targets entered via + // prerequsites (skip static prerequisites) nor by the cache=true code + // above (skip depdb entries). + // + // If this turns out to be racy (which is the reason we would skip + // dynamic targets; see the fine_file() implementation for details), + // then the only answer for now is to not use the byproduct mode. + // if (const build2::file* ft = dyndep::find_file ( trace, what, a, bs, t, fp, false /* cache */, true /* normalized */, + true /* dynamic */, map_ext, *byp.default_type).first) { // Skip if this is one of the static prerequisites provided it was @@ -937,15 +1406,25 @@ namespace build2 } } - // Skip if this is one of the targets. + // Skip if this is one of the targets (see the non-byproduct version + // for background). // if (byp.drop_cycles) { - for (const target* m (&t); m != nullptr; m = m->adhoc_member) + if (g != nullptr) { - if (ft == m) + auto& ms (g->members); + if (find (ms.begin (), ms.end (), ft) != ms.end ()) return; } + else + { + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (ft == m) + return; + } + } } // Skip until where we left off. @@ -961,7 +1440,7 @@ namespace build2 // @@ Currently we will issue an imprecise diagnostics if this is // a static prerequisite that was not updated (see above). // - dyndep::verify_existing_file (trace, what, a, t, *ft); + dyndep::verify_existing_file (trace, what, a, t, pts_n, *ft); } dd.write (fp); @@ -1022,7 +1501,7 @@ namespace build2 if (r.second.empty ()) continue; - // @@ TODO: what should we do about targets? + // Note: no support for dynamic targets in byproduct mode. // if (r.first == make_type::target) continue; @@ -1032,10 +1511,11 @@ namespace build2 if (f.relative ()) { if (!byp.cwd) - fail (il) << "relative path '" << f << "' in make dependency" - << " declaration" << + fail (il) << "relative " << what + << " prerequisite path '" << f + << "' in make dependency declaration" << info << "consider using --cwd to specify relative path " - << "base"; + << "base"; f = *byp.cwd / f; } @@ -1050,6 +1530,52 @@ namespace build2 break; } + case dyndep_format::lines: + { + for (string l;; ++il.line) // Reuse the buffer. + { + if (eof (getline (is, l))) + break; + + if (l.empty ()) + fail (il) << "blank line in prerequisites list"; + + if (l.front () == ' ') + fail (il) << "non-existent prerequisite in --byproduct mode"; + + path f; + try + { + f = path (l); + + // fsdir{} prerequisites only make sense with dynamic targets. + // + if (f.to_directory ()) + throw invalid_path (""); + + if (f.relative ()) + { + if (!byp.cwd) + fail (il) << "relative " << what + << " prerequisite path '" << f + << "' in lines dependency declaration" << + info << "consider using --cwd to specify " + << "relative path base"; + + f = *byp.cwd / f; + } + } + catch (const invalid_path&) + { + fail (il) << "invalid " << what << " prerequisite path '" + << l << "'"; + } + + add (move (f)); + } + + break; + } } // Add the terminating blank line. @@ -1057,6 +1583,8 @@ namespace build2 dd.expect (""); dd.close (); + //@@ TODO: expl: byproduct: verify have at least one member. + md.dd.path = move (dd.path); // For mtime check below. } @@ -1065,20 +1593,36 @@ namespace build2 timestamp now (system_clock::now ()); if (!ctx.dry_run) - depdb::check_mtime (start, md.dd.path, t.path (), now); + { + // Only now we know for sure there must be a member in the group. + // + const file& ft ((g == nullptr ? t : *g->members.front ()).as<file> ()); + + depdb::check_mtime (start, md.dd.path, ft.path (), now); + } + + (g == nullptr + ? static_cast<const mtime_target&> (t.as<file> ()) + : static_cast<const mtime_target&> (*g)).mtime (now); - t.mtime (now); return target_state::changed; } target_state adhoc_buildscript_rule:: - perform_update_file_dyndep (action a, const target& xt, match_data& md) const + perform_update_file_or_group_dyndep ( + action a, const target& t, match_data& md) const { - tracer trace ("adhoc_buildscript_rule::perform_update_file"); + tracer trace ( + "adhoc_buildscript_rule::perform_update_file_or_group_dyndep"); - context& ctx (xt.ctx); + context& ctx (t.ctx); - const file& t (xt.as<file> ()); + // For a group we use the first (static or dynamic) member as a source of + // mtime. Note that in this case there must be at least one since we fail + // if we were unable to extract any dynamic members and there are no + // static (see exec_depdb_dyndep()). + // + const group* g (t.is_a<group> ()); // Note that even if we've updated all our prerequisites in apply(), we // still need to execute them here to keep the dependency counts straight. @@ -1107,7 +1651,11 @@ namespace build2 if (!ctx.dry_run || verb != 0) { - execute_update_file (*md.bs, a, t, env, run, md.deferred_failure); + if (g == nullptr) + execute_update_file ( + *md.bs, a, t.as<file> (), env, run, md.deferred_failure); + else + execute_update_group (*md.bs, a, *g, env, run, md.deferred_failure); } run.leave (env, script.end_loc); @@ -1115,26 +1663,67 @@ namespace build2 timestamp now (system_clock::now ()); if (!ctx.dry_run) - depdb::check_mtime (start, md.dd, t.path (), now); + { + // Note: in case of deferred failure we may not have any members. + // + const file& ft ((g == nullptr ? t : *g->members.front ()).as<file> ()); + depdb::check_mtime (start, md.dd, ft.path (), now); + } + + (g == nullptr + ? static_cast<const mtime_target&> (t) + : static_cast<const mtime_target&> (*g)).mtime (now); - t.mtime (now); return target_state::changed; } target_state adhoc_buildscript_rule:: - perform_update_file (action a, const target& xt) const + perform_update_file_or_group (action a, const target& t) const { - tracer trace ("adhoc_buildscript_rule::perform_update_file"); + tracer trace ("adhoc_buildscript_rule::perform_update_file_or_group"); - context& ctx (xt.ctx); + context& ctx (t.ctx); + const scope& bs (t.base_scope ()); + + // For a group we use the first (static) member to derive depdb path, as a + // source of mtime, etc. Note that in this case there must be a static + // member since in this version of perform_update we don't extract dynamic + // dependencies (see apply() details). + // + const group* g (t.is_a<group> ()); - const file& t (xt.as<file> ()); - const path& tp (t.path ()); + const file& ft ((g == nullptr ? t : *g->members.front ()).as<file> ()); + const path& tp (ft.path ()); + + // Support creating file symlinks using ad hoc recipes. + // + auto path_symlink = [&tp] () + { + pair<bool, butl::entry_stat> r ( + butl::path_entry (tp, + false /* follow_symlinks */, + true /* ignore_errors */)); + + return r.first && r.second.type == butl::entry_type::symlink; + }; // Update prerequisites and determine if any of them render this target // out-of-date. // - timestamp mt (t.load_mtime ()); + // If the file entry exists, check if its a symlink. + // + bool symlink (false); + timestamp mt; + + if (g == nullptr) + { + mt = ft.load_mtime (); + + if (mt != timestamp_nonexistent) + symlink = path_symlink (); + } + else + mt = g->load_mtime (tp); // This is essentially ps=execute_prerequisites(a, t, mt) which we // cannot use because we need to see ad hoc prerequisites. @@ -1156,7 +1745,7 @@ namespace build2 p.adhoc () ? reinterpret_cast<target*> (p.data) : nullptr)) { - if ((p.include & 4) != 0) // Skip update=unmatch. + if ((p.include & include_unmatch) != 0) // Skip update=unmatch. continue; hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage); @@ -1218,7 +1807,7 @@ namespace build2 // { sha256 cs; - hash_script_vars (cs, script, t, storage); + hash_script_vars (cs, script, bs, t, storage); if (dd.expect (cs.string ()) != nullptr) l4 ([&]{trace << "recipe variable change forcing update of " << t;}); @@ -1228,8 +1817,18 @@ namespace build2 // { sha256 tcs; - for (const target* m (&t); m != nullptr; m = m->adhoc_member) - hash_target (tcs, *m, storage); + if (g == nullptr) + { + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + hash_target (tcs, *m, storage); + } + else + { + // Feels like there is not much sense in hashing the group itself. + // + for (const target* m: g->members) + hash_target (tcs, *m, storage); + } if (dd.expect (tcs.string ()) != nullptr) l4 ([&]{trace << "target set change forcing update of " << t;}); @@ -1249,8 +1848,6 @@ namespace build2 } } - const scope* bs (nullptr); - // Execute the custom dependency change tracking commands, if present. // // Note that we share the environment between the execute_depdb_preamble() @@ -1265,7 +1862,10 @@ namespace build2 if (!depdb_preamble) { - if (dd.writing () || dd.mtime > mt) + // If this is a symlink, depdb mtime could be greater than the symlink + // target. + // + if (dd.writing () || (dd.mtime > mt && !symlink)) update = true; if (!update) @@ -1275,25 +1875,23 @@ namespace build2 } } - build::script::environment env (a, t, false /* temp_dir */); + build::script::environment env (a, t, bs, false /* temp_dir */); build::script::default_runner run; if (depdb_preamble) { - bs = &t.base_scope (); - if (script.depdb_preamble_temp_dir) env.set_temp_dir_variable (); build::script::parser p (ctx); run.enter (env, script.start_loc); - p.execute_depdb_preamble (a, *bs, t, env, script, run, dd); + p.execute_depdb_preamble (a, bs, t, env, script, run, dd); } // Update if depdb mismatch. // - if (dd.writing () || dd.mtime > mt) + if (dd.writing () || (dd.mtime > mt && !symlink)) update = true; dd.close (); @@ -1315,22 +1913,44 @@ namespace build2 bool r (false); if (!ctx.dry_run || verb != 0) { - // Prepare to execute the script diag line and/or body. + // Prepare to execute the script diag preamble and/or body. // - if (bs == nullptr) - bs = &t.base_scope (); + r = g == nullptr + ? execute_update_file (bs, a, ft, env, run) + : execute_update_group (bs, a, *g, env, run); - if ((r = execute_update_file (*bs, a, t, env, run))) + if (r) { if (!ctx.dry_run) - dd.check_mtime (tp); + { + if (g == nullptr) + symlink = path_symlink (); + + // Again, if this is a symlink, depdb mtime will be greater than + // the symlink target. + // + if (!symlink) + dd.check_mtime (tp); + } } } if (r || depdb_preamble) run.leave (env, script.end_loc); - t.mtime (system_clock::now ()); + // Symlinks don't play well with dry-run: we can't extract accurate target + // timestamp without creating the symlink. Overriding the dry-run doesn't + // seem to be an option since we don't know whether it will be a symlink + // until it's created. At least we are being pessimistic rather than + // optimistic here. + // + (g == nullptr + ? static_cast<const mtime_target&> (ft) + : static_cast<const mtime_target&> (*g)).mtime ( + symlink + ? build2::mtime (tp) + : system_clock::now ()); + return target_state::changed; } @@ -1345,6 +1965,7 @@ namespace build2 // !NULL false 1 - normal prerequisite already updated // !NULL true 0 - ad hoc prerequisite to be updated and blanked // NULL true !NULL - ad hoc prerequisite already updated and blanked + // NULL false !NULL - unmatched prerequisite (ignored by this function) // // Note that we still execute already updated prerequisites to keep the // dependency counts straight. But we don't consider them for the "renders @@ -1352,6 +1973,8 @@ namespace build2 // // See also environment::set_special_variables(). // + // See also perform_execute() which has to deal with these shenanigans. + // optional<target_state> adhoc_buildscript_rule:: execute_update_prerequisites (action a, const target& t, timestamp mt) const { @@ -1396,7 +2019,7 @@ namespace build2 // Compare our timestamp to this prerequisite's skipping // update=unmatch. // - if (!e && (p.include & 4) == 0) + if (!e && (p.include & include_unmatch) == 0) { // If this is an mtime-based target, then compare timestamps. // @@ -1417,10 +2040,14 @@ namespace build2 // Blank out adhoc. // + // Note: set the include_target flag for the updated_during_match() + // check. + // if (p.adhoc ()) { p.data = reinterpret_cast<uintptr_t> (p.target); p.target = nullptr; + p.include |= prerequisite_target::include_target; } } } @@ -1429,16 +2056,18 @@ namespace build2 return e ? nullopt : optional<target_state> (rs); } - // Return true if execute_body() was called and thus the caller should call - // run.leave(). + // Return true if execute_diag_preamble() and/or execute_body() were called + // and thus the caller should call run.leave(). // bool adhoc_buildscript_rule:: execute_update_file (const scope& bs, - action, const file& t, + action a, const file& t, build::script::environment& env, build::script::default_runner& run, bool deferred_failure) const { + // NOTE: similar to execute_update_group() below. + // context& ctx (t.ctx); const scope& rs (*bs.root_scope ()); @@ -1449,28 +2078,73 @@ namespace build2 // build::script::parser p (ctx); - if (verb == 1) + bool exec_body (!ctx.dry_run || verb >= 2); + bool exec_diag (!script.diag_preamble.empty () && (exec_body || verb == 1)); + bool exec_depdb (!script.depdb_preamble.empty ()); + + if (script.diag_name) { - if (script.diag_line) - { - text << p.execute_special (rs, bs, env, *script.diag_line); - } - else + if (verb == 1) { - // @@ TODO (and in default_action() below): + // By default we print the first non-ad hoc prerequisite target as the + // "main" prerequisite, unless there isn't any or it's not file-based, + // in which case we fallback to the second form without the + // prerequisite. Potential future improvements: // - // - we are printing target, not source (like in most other places) + // - Somehow detect that the first prerequisite target is a tool being + // executed and fallback to the second form. It's tempting to just + // exclude all exe{} targets, but this could be a rule for something + // like strip. // - // - printing of ad hoc target group (the {hxx cxx}{foo} idea) - // - // - if we are printing prerequisites, should we print all of them - // (including tools)? - // - text << *script.diag_name << ' ' << t; + const file* pt (nullptr); + for (const prerequisite_target& p: t.prerequisite_targets[a]) + { + // See execute_update_prerequisites(). + // + if (p.target != nullptr && !p.adhoc ()) + { + pt = p.target->is_a<file> (); + break; + } + } + + if (t.adhoc_member == nullptr) + { + if (pt != nullptr) + print_diag (script.diag_name->c_str (), *pt, t); + else + print_diag (script.diag_name->c_str (), t); + } + else + { + vector<target_key> ts; + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + ts.push_back (m->key ()); + + if (pt != nullptr) + print_diag (script.diag_name->c_str (), pt->key (), move (ts)); + else + print_diag (script.diag_name->c_str (), move (ts)); + } } } + else if (exec_diag) + { + if (script.diag_preamble_temp_dir && !script.depdb_preamble_temp_dir) + env.set_temp_dir_variable (); + + pair<names, location> diag ( + p.execute_diag_preamble (rs, bs, + env, script, run, + verb == 1 /* diag */, + !exec_depdb /* enter */, + false /* leave */)); - if (!ctx.dry_run || verb >= 2) + if (verb == 1) + print_custom_diag (bs, move (diag.first), diag.second); + } + + if (exec_body) { // On failure remove the target files that may potentially exist but // be invalid. @@ -1486,13 +2160,15 @@ namespace build2 } } - if (script.body_temp_dir && !script.depdb_preamble_temp_dir) + if (script.body_temp_dir && + !script.depdb_preamble_temp_dir && + !script.diag_preamble_temp_dir) env.set_temp_dir_variable (); p.execute_body (rs, bs, env, script, run, - script.depdb_preamble.empty () /* enter */, - false /* leave */); + !exec_depdb && !exec_diag /* enter */, + false /* leave */); if (!ctx.dry_run) { @@ -1521,11 +2197,131 @@ namespace build2 for (auto& rm: rms) rm.cancel (); } + } + + return exec_diag || exec_body; + } - return true; + bool adhoc_buildscript_rule:: + execute_update_group (const scope& bs, + action a, const group& g, + build::script::environment& env, + build::script::default_runner& run, + bool deferred_failure) const + { + // Note: similar to execute_update_file() above (see there for comments). + // + // NOTE: when called from perform_update_file_or_group_dyndep_byproduct(), + // the group does not contain dynamic members yet and thus could + // have no members at all. + // + context& ctx (g.ctx); + + const scope& rs (*bs.root_scope ()); + + build::script::parser p (ctx); + + bool exec_body (!ctx.dry_run || verb >= 2); + bool exec_diag (!script.diag_preamble.empty () && (exec_body || verb == 1)); + bool exec_depdb (!script.depdb_preamble.empty ()); + + if (script.diag_name) + { + if (verb == 1) + { + const file* pt (nullptr); + for (const prerequisite_target& p: g.prerequisite_targets[a]) + { + if (p.target != nullptr && !p.adhoc ()) + { + pt = p.target->is_a<file> (); + break; + } + } + + if (pt != nullptr) + print_diag (script.diag_name->c_str (), *pt, g); + else + print_diag (script.diag_name->c_str (), g); + } } - else - return false; + else if (exec_diag) + { + if (script.diag_preamble_temp_dir && !script.depdb_preamble_temp_dir) + env.set_temp_dir_variable (); + + pair<names, location> diag ( + p.execute_diag_preamble (rs, bs, + env, script, run, + verb == 1 /* diag */, + !exec_depdb /* enter */, + false /* leave */)); + if (verb == 1) + print_custom_diag (bs, move (diag.first), diag.second); + } + + if (exec_body) + { + // On failure remove the target files that may potentially exist but + // be invalid. + // + // Note: we may leave dynamic members if we don't know about them yet. + // Feels natural enough. + // + small_vector<auto_rmfile, 8> rms; + + if (!ctx.dry_run) + { + for (const target* m: g.members) + { + if (auto* f = m->is_a<file> ()) + rms.emplace_back (f->path ()); + } + } + + if (script.body_temp_dir && + !script.depdb_preamble_temp_dir && + !script.diag_preamble_temp_dir) + env.set_temp_dir_variable (); + + p.execute_body (rs, bs, + env, script, run, + !exec_depdb && !exec_diag /* enter */, + false /* leave */); + + if (!ctx.dry_run) + { + if (deferred_failure) + fail << "expected error exit status from recipe body"; + + // @@ TODO: expl: byproduct + // + // Note: will not work for dynamic members if we don't know about them + // yet. Could probably fix by doing this later, after the dynamic + // dependency extraction. + // +#ifndef _WIN32 + auto chmod = [] (const path& p) + { + path_perms (p, + (path_perms (p) | + permissions::xu | + permissions::xg | + permissions::xo)); + }; + + for (const target* m: g.members) + { + if (auto* p = m->is_a<exe> ()) + chmod (p->path ()); + } +#endif + for (auto& rm: rms) + rm.cancel (); + } + } + + return exec_diag || exec_body; } target_state adhoc_buildscript_rule:: @@ -1540,7 +2336,41 @@ namespace build2 // temporary directory ($~) is that it's next to other output which makes // it easier to examine during recipe troubleshooting. // - return perform_clean_extra (a, t.as<file> (), {".d", ".t"}); + // Finally, we print the entire ad hoc group at verbosity level 1, similar + // to the default update diagnostics. + // + // @@ TODO: .t may also be a temporary directory (and below). + // + return perform_clean_extra (a, + t.as<file> (), + {".d", ".t"}, + {}, + true /* show_adhoc_members */); + } + + target_state adhoc_buildscript_rule:: + perform_clean_group (action a, const target& xt) + { + const group& g (xt.as<group> ()); + + path d, t; + if (g.members_static != 0) + { + const path& p (g.members.front ()->as<file> ().path ()); + d = p + ".d"; + t = p + ".t"; + } + else + { + // See target_path lambda in apply(). + // + t = g.dir / (g.name + '.' + g.type ().name); + d = t + ".d"; + t += ".t"; + } + + return perform_clean_group_extra (a, g, {d.string ().c_str (), + t.string ().c_str ()}); } target_state adhoc_buildscript_rule:: @@ -1552,37 +2382,340 @@ namespace build2 context& ctx (t.ctx); - execute_prerequisites (a, t); + target_state ts (target_state::unchanged); - if (!ctx.dry_run || verb != 0) + if (ctx.current_mode == execution_mode::first) + ts |= straight_execute_prerequisites (a, t); + + bool exec (!ctx.dry_run || verb != 0); + + // Special handling for fsdir{} (which is the recommended if somewhat + // hackish way to represent directory symlinks). See fsdir_rule for + // background. + // + // @@ Note that because there is no depdb, we cannot detect the target + // directory change (or any other changes in the script). + // + if (exec && + (a == perform_update_id || a == perform_clean_id) && + t.is_a<fsdir> ()) + { + // For update we only want to skip if it's a directory. For clean we + // want to (try) to clean up any filesystem entry, including a dangling + // symlink. + // + exec = a == perform_update_id + ? !exists (t.dir, true /* ignore_errors */) + : build2::entry_exists (t.dir, false /* follow_symlinks */); + } + + if (exec) { const scope& bs (t.base_scope ()); const scope& rs (*bs.root_scope ()); - build::script::environment e (a, t, script.body_temp_dir, deadline); + build::script::environment e (a, t, bs, false /* temp_dir */, deadline); build::script::parser p (ctx); + build::script::default_runner r; - if (verb == 1) + bool exec_body (!ctx.dry_run || verb >= 2); + bool exec_diag (!script.diag_preamble.empty () && + (exec_body || verb == 1)); + + if (script.diag_name) { - if (script.diag_line) + if (verb == 1) { - text << p.execute_special (rs, bs, e, *script.diag_line); + // For operations other than update (as well as for non-file + // targets), we default to the second form (without the + // prerequisite). Think test. + // + if (t.adhoc_member == nullptr) + print_diag (script.diag_name->c_str (), t); + else + { + vector<target_key> ts; + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + ts.push_back (m->key ()); + + print_diag (script.diag_name->c_str (), move (ts)); + } } - else + } + else if (exec_diag) + { + if (script.diag_preamble_temp_dir) + e.set_temp_dir_variable (); + + pair<names, location> diag ( + p.execute_diag_preamble (rs, bs, + e, script, r, + verb == 1 /* diag */, + true /* enter */, + !exec_body /* leave */)); + + if (verb == 1) + print_custom_diag (bs, move (diag.first), diag.second); + } + + if (exec_body) + { + if (script.body_temp_dir && !script.diag_preamble_temp_dir) + e.set_temp_dir_variable (); + + p.execute_body (rs, bs, e, script, r, !exec_diag /* enter */); + } + + ts |= target_state::changed; + } + + if (ctx.current_mode == execution_mode::last) + ts |= reverse_execute_prerequisites (a, t); + + return ts; + } + + void adhoc_buildscript_rule:: + print_custom_diag (const scope& bs, names&& ns, const location& l) const + { + // The straightforward thing to do would be to just print the diagnostics + // as specified by the user. But that will make some of the tidying up + // done by print_diag() unavailable to custom diagnostics. Things like + // omitting the out-qualification as well as compact printing of the + // groups. Also, in the future we may want to support colorization of the + // diagnostics, which will be difficult to achive with such a "just print" + // approach. + // + // So instead we are going to parse the custom diagnostics, translate + // names back to targets (where appropriate), and call one of the + // print_diag() functions. Specifically, we expect the custom diagnostics + // to be in one of the following two forms (which correspond to the two + // forms of pring_diag()): + // + // diag <prog> <l-target> <comb> <r-target>... + // diag <prog> <r-target>... + // + // And the way we are going to disambiguate this is by analyzing name + // types. Specifically, we expect <comb> to be a simple name that also + // does not contain any directory separators (so we can distinguish it + // from both target names as well as paths, which can be specified on + // either side). We will also recognize `-` as the special stdout path + // name (so <comb> cannot be `-`). Finally, <l-target> (but not + // <r-target>) can be a string (e.g., an argument) but that should not + // pose an ambiguity. + // + // With this approach, the way to re-create the default diagnostics would + // be: + // + // diag <prog> ($<[0]) -> $> + // diag <prog> $> + // + auto i (ns.begin ()), e (ns.end ()); + + // <prog> + // + if (i == e) + fail (l) << "missing program name in diag builtin"; + + if (!i->simple () || i->empty ()) + fail (l) << "expected simple name as program name in diag builtin"; + + const char* prog (i->value.c_str ()); + ++i; + + // <l-target> + // + const target* l_t (nullptr); + path l_p; + string l_s; + + auto parse_target = [&bs, &l, &i, &e] () -> const target& + { + name& n (*i++); + name o; + + if (n.pair) + { + if (i == e) + fail (l) << "invalid target name pair in diag builtin"; + + o = move (*i++); + } + + // Similar to to_target() in $target.*(). + // + if (const target* r = search_existing (n, bs, o.dir)) + return *r; + + fail (l) << "target " + << (n.pair ? names {move (n), move (o)} : names {move (n)}) + << " not found in diag builtin" << endf; + }; + + auto parse_first = [&l, &i, &e, + &parse_target] (const target*& t, path& p, string& s, + const char* after) + { + if (i == e) + fail (l) << "missing target after " << after << " in diag builtin"; + + try + { + if (i->typed ()) { - // @@ TODO: as above (execute_update_file()). - // - text << *script.diag_name << ' ' << t; + t = &parse_target (); + return; // i is already incremented. + } + else if (!i->dir.empty ()) + { + p = move (i->dir); + p /= i->value; } + else if (path_traits::find_separator (i->value) != string::npos) + { + p = path (move (i->value)); + } + else if (!i->value.empty ()) + { + s = move (i->value); + } + else + fail (l) << "expected target, path, or argument after " + << after << " in diag builtin"; } + catch (const invalid_path& e) + { + fail (l) << "invalid path '" << e.path << "' after " + << after << " in diag builtin"; + } + + ++i; + }; + + parse_first (l_t, l_p, l_s, "program name"); - if (!ctx.dry_run || verb >= 2) + // Now detect which form it is. + // + if (i != e && + i->simple () && + !i->empty () && + path_traits::find_separator (i->value) == string::npos) + { + // The first form. + + // <comb> + // + const char* comb (i->value.c_str ()); + ++i; + + // <r-target> + // + const target* r_t (nullptr); + path r_p; + string r_s; + + parse_first (r_t, r_p, r_s, "combiner"); + + path_name r_pn; + + if (r_t != nullptr) + ; + else if (!r_p.empty ()) + r_pn = path_name (&r_p); + else { - build::script::default_runner r; - p.execute_body (rs, bs, e, script, r); + if (r_s != "-") + fail (l) << "expected target or path instead of '" << r_s + << "' after combiner in diag builtin"; + + r_pn = path_name (move (r_s)); } + + if (i == e) + { + if (r_t != nullptr) + { + if (l_t != nullptr) print_diag (prog, *l_t, *r_t, comb); + else if (!l_p.empty ()) print_diag (prog, l_p, *r_t, comb); + else print_diag (prog, l_s, *r_t, comb); + } + else + { + if (l_t != nullptr) print_diag (prog, *l_t, r_pn, comb); + else if (!l_p.empty ()) print_diag (prog, l_p, r_pn, comb); + else print_diag (prog, l_s, r_pn, comb); + } + + return; + } + + // We can only have multiple targets, not paths. + // + if (r_t == nullptr) + fail (l) << "unexpected name after path in diag builtin"; + + // <r-target>... + // + vector<target_key> r_ts {r_t->key ()}; + + do r_ts.push_back (parse_target ().key ()); while (i != e); + + if (l_t != nullptr) print_diag (prog, l_t->key (), move (r_ts), comb); + else if (!l_p.empty ()) print_diag (prog, l_p, move (r_ts), comb); + else print_diag (prog, l_s, move (r_ts), comb); } + else + { + // The second form. - return target_state::changed; + // First "absorb" the l_* values as the first <r-target>. + // + const target* r_t (nullptr); + path_name r_pn; + + if (l_t != nullptr) + r_t = l_t; + else if (!l_p.empty ()) + r_pn = path_name (&l_p); + else + { + if (l_s != "-") + { + diag_record dr (fail (l)); + + dr << "expected target or path instead of '" << l_s + << "' after program name in diag builtin"; + + if (i != e) + dr << info << "alternatively, missing combiner after '" + << l_s << "'"; + } + + r_pn = path_name (move (l_s)); + } + + if (i == e) + { + if (r_t != nullptr) + print_diag (prog, *r_t); + else + print_diag (prog, r_pn); + + return; + } + + // We can only have multiple targets, not paths. + // + if (r_t == nullptr) + fail (l) << "unexpected name after path in diag builtin"; + + // <r-target>... + // + vector<target_key> r_ts {r_t->key ()}; + + do r_ts.push_back (parse_target ().key ()); while (i != e); + + print_diag (prog, move (r_ts)); + } } } |