From 76f1988539c477ad3b906f254654929aec04283c Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 30 Nov 2021 10:15:33 +0200 Subject: Add support for dynamic dependencies as byproduct of script body Specifically, the `depdb dyndep` builtin now has the --byproduct option (which must come first). In this mode only the --file input is supported. For example: obje{hello.o}: cxx{hello} {{ o = $path($>) t = $(o).t depdb dyndep --byproduct --what=header --default-type=h --file $t diag c++ ($<[0]) $cxx.path $cxx.poptions $cc.poptions $cc.coptions $cxx.coptions $cxx.mode -o $o -MD -MF $t -c $path($<[0]) }} Naturally, this mode does not support dynamic auto-generated prerequisites. If present, such prerequisites must be specified statically in the buildfile. Note also that the --default-prereq-type option has been rename to --default-type. --- libbuild2/adhoc-rule-buildscript.cxx | 573 +++++++++++++++++++++++++++++++---- 1 file changed, 509 insertions(+), 64 deletions(-) (limited to 'libbuild2/adhoc-rule-buildscript.cxx') diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx index b981b8d..db3be30 100644 --- a/libbuild2/adhoc-rule-buildscript.cxx +++ b/libbuild2/adhoc-rule-buildscript.cxx @@ -13,6 +13,7 @@ #include #include // path_perms(), auto_rmfile #include +#include #include // attributes @@ -227,11 +228,30 @@ namespace build2 build::script::default_runner run; path dd; + const scope* bs; timestamp mt; bool deferred_failure; }; + struct adhoc_buildscript_rule::match_data_byproduct + { + match_data_byproduct (action a, const target& t, bool temp_dir) + : env (a, t, temp_dir) {} + + build::script::environment env; + build::script::default_runner run; + + build::script::parser::dyndep_byproduct byp; + + depdb::reopen_state dd; + size_t skip_count = 0; + size_t pts_n; // Number of static prerequisites in prerequisite_targets. + + const scope* bs; + timestamp mt; + }; + bool adhoc_buildscript_rule:: match (action a, target& t, const string& h, match_extra& me) const { @@ -339,15 +359,21 @@ namespace build2 } // This is a perform update on a file target with extraction of dynamic - // dependency information in the depdb preamble (depdb-dyndep). + // dependency information either in the depdb preamble (depdb-dyndep + // without --byproduct) or as a byproduct of the recipe body execution + // (depdb-dyndep with --byproduct). // - // This means we may need to add additional prerequisites (or even target - // group members). We also have to save any such additional prerequisites - // in depdb so that we can check if any of them have changed on subsequent - // updates. So all this means that have to take care of depdb here in - // apply() instead of perform_*() like we normally do. We also do things - // in slightly different order due to the restrictions impose by the match - // phase. + // For the former case, we may need to add additional prerequisites (or + // even target group members). We also have to save any such additional + // prerequisites in depdb so that we can check if any of them have changed + // on subsequent updates. So all this means that we have to take care of + // depdb here in apply() instead of perform_*() like we normally do. We + // also do things in slightly different order due to the restrictions + // impose by the match phase. + // + // The latter case (depdb-dyndep --byproduct) is sort of a combination + // of the normal dyndep and the static case: we check the depdb during + // match but save after executing the recipe. // // Note that the C/C++ header dependency extraction is the canonical // example and all this logic is based on the prior work in the cc module @@ -358,6 +384,7 @@ namespace build2 file& t (xt.as ()); const path& tp (t.path ()); + const scope& bs (t.base_scope ()); if (dir != nullptr) fsdir_rule::perform_update_direct (a, t); @@ -379,10 +406,10 @@ namespace build2 } } - // NOTE: see the "static dependencies" version (with comments) below. - // depdb dd (tp + ".d"); + // NOTE: see the "static dependencies" version (with comments) below. + // if (dd.expect (" 1") != nullptr) l4 ([&]{trace << "rule mismatch forcing update of " << t;}); @@ -435,13 +462,20 @@ namespace build2 } } - const scope& bs (t.base_scope ()); + unique_ptr md; + unique_ptr mdb; - unique_ptr md ( - new match_data (a, t, script.depdb_preamble_temp_dir)); + if (script.depdb_dyndep_byproduct) + { + mdb.reset (new match_data_byproduct ( + a, t, script.depdb_preamble_temp_dir)); + } + else + md.reset (new match_data (a, t, script.depdb_preamble_temp_dir)); - build::script::environment& env (md->env); - build::script::default_runner& run (md->run); + + build::script::environment& env (mdb != nullptr ? mdb->env : md->env); + build::script::default_runner& run (mdb != nullptr ? mdb->run : md->run); run.enter (env, script.start_loc); @@ -485,63 +519,471 @@ namespace build2 } } - // Run the second half of the preamble (depdb-dyndep commands) to extract - // dynamic dependencies. + if (script.depdb_dyndep_byproduct) + { + // If we have the dynamic dependency information as byproduct of the + // recipe body, then do the first part: verify the entries in depdb + // unless we are already updating. Essentially, this is the `if(cache)` + // equivalent of the restart loop in exec_depdb_dyndep(). + // + // Do we really need to update our prerequisite targets in this case (as + // we do above)? While it may seem like we should be able to avoid it by + // triggering update on encountering any non-existent files in depbd, we + // may actually incorrectly "validate" some number of depdb entires + // while having an out-of-date main source file. We could probably avoid + // the update if we are already updating. + + using dyndep = dyndep_rule; + + // Extract the depdb-dyndep command's information (we may also execute + // some variable assignments). + // + { + build::script::parser p (ctx); + mdb->byp = p.execute_depdb_preamble_dyndep_byproduct ( + a, bs, t, + env, script, run, + dd); + } + + mdb->pts_n = pts.size (); + + if (!update) + { + const auto& byp (mdb->byp); + const char* what (byp.what.c_str ()); + const location& ll (byp.location); + + function map_ext ( + [] (const scope& bs, const string& n, const string& e) + { + // NOTE: another version in exec_depdb_dyndep(). + + return dyndep::map_extension (bs, n, e, nullptr); + }); + + // Similar to exec_depdb_dyndep()::add() but only for cache=true and + // without support for generated files. + // + // Note that we have to update each file for the same reason as the + // main source file -- if any of them changed, then we must assume the + // subsequent entries are invalid. + // + size_t& skip_count (mdb->skip_count); + + auto add = [this, &trace, what, + a, &bs, &t, &pts, pts_n = mdb->pts_n, + &byp, &map_ext, + &skip_count, mt] (path fp) -> optional + { + if (const build2::file* ft = dyndep::enter_file ( + trace, what, + a, bs, t, + fp, true /* cache */, true /* normalized */, + map_ext, *byp.default_type).first) + { + // Skip if this is one of the static prerequisites. + // + for (size_t i (0); i != pts_n; ++i) + { + const prerequisite_target& p (pts[i]); + + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : + nullptr)) + { + if (pt == ft) + { + // Note that we have to increment the skip count since we + // skip before performing this test. + // + skip_count++; + return false; + } + } + } + + if (optional u = dyndep::inject_existing_file ( + trace, what, + a, t, + *ft, mt, false /* fail */)) + { + skip_count++; + return *u; + } + } + + return nullopt; + }; + + auto df = make_diag_frame ( + [this, &ll, &t] (const diag_record& dr) + { + if (verb != 0) + dr << info (ll) << "while extracting dynamic dependencies for " + << t; + }); + + while (!update) + { + // We should always end with a blank line. + // + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + update = true; + break; + } + + if (l->empty ()) // Done, nothing changed. + break; + + if (optional r = add (path (move (*l)))) + { + if (*r) + update = true; + } + else + { + // Invalidate this line and trigger update. + // + dd.write (); + update = true; + } + + if (update) + l6 ([&]{trace << "restarting (cache)";}); + } + } + + // Note that in case of dry run we will have an incomplete (but valid) + // database which will be updated on the next non-dry run. + // + if (!update || ctx.dry_run) + dd.close (false /* mtime_check */); + else + mdb->dd = dd.close_to_reopen (); + + // Pass on base scope and update/mtime. + // + mdb->bs = &bs; + mdb->mt = update ? timestamp_nonexistent : mt; + + // @@ TMP: re-enable once recipe becomes move_only_function. + // +#if 0 + return [this, md = move (mdb)] (action a, const target& t) mutable + { + auto r (perform_update_file_dyndep_byproduct (a, t, *md)); + md.reset (); // @@ TMP: is this really necessary (+mutable)? + return r; + }; +#else + t.data (move (mdb)); + return recipe ([this] (action a, const target& t) mutable + { + auto md (move (t.data> ())); + return perform_update_file_dyndep_byproduct (a, t, *md); + }); +#endif + } + else + { + // Run the second half of the preamble (depdb-dyndep commands) to + // extract dynamic dependencies. + // + // Note that this should be the last update to depdb (the invalidation + // order semantics). + // + bool deferred_failure (false); + { + build::script::parser p (ctx); + p.execute_depdb_preamble_dyndep (a, bs, t, + env, script, run, + dd, + update, + deferred_failure, + mt); + } + + if (update && dd.reading () && !ctx.dry_run) + dd.touch = timestamp_unknown; + + dd.close (false /* mtime_check */); + md->dd = move (dd.path); + + // Pass on base scope and update/mtime. + // + md->bs = &bs; + md->mt = update ? timestamp_nonexistent : mt; + md->deferred_failure = deferred_failure; + + // @@ TMP: re-enable once recipe becomes move_only_function. + // +#if 0 + return [this, md = move (md)] (action a, const target& t) mutable + { + auto r (perform_update_file_dyndep (a, t, *md)); + md.reset (); // @@ TMP: is this really necessary (+mutable)? + return r; + }; +#else + t.data (move (md)); + return recipe ([this] (action a, const target& t) mutable + { + auto md (move (t.data> ())); + return perform_update_file_dyndep (a, t, *md); + }); +#endif + } + } + + target_state adhoc_buildscript_rule:: + perform_update_file_dyndep_byproduct (action a, + const target& xt, + match_data_byproduct& md) const + { + // Note: using shared function name among the three variants. // - // Note that this should be the last update to depdb (the invalidation - // order semantics). + tracer trace ("adhoc_buildscript_rule::perform_update_file"); + + context& ctx (xt.ctx); + + const file& t (xt.as ()); + + // While we've updated all our prerequisites in apply(), we still need to + // execute them here to keep the dependency counts straight. // - bool deferred_failure (false); + const auto& pts (t.prerequisite_targets[a]); + + for (const prerequisite_target& p: pts) { - build::script::parser p (ctx); - p.execute_depdb_preamble_dyndep (a, bs, t, - env, script, run, - dd, - update, - deferred_failure, - mt); + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : nullptr)) + { + target_state ts (execute_wait (a, *pt)); + assert (ts == target_state::unchanged || ts == target_state::changed); + } } - if (update && dd.reading () && !ctx.dry_run) - dd.touch = timestamp_unknown; + build::script::environment& env (md.env); + build::script::default_runner& run (md.run); + + if (md.mt != timestamp_nonexistent) + { + run.leave (env, script.end_loc); + return target_state::unchanged; + } - dd.close (false /* mtime_check */); - md->dd = move (dd.path); + const scope& bs (*md.bs); - // Pass on base scope and update/mtime. + // Sequence start time for mtime checks below. // - md->bs = &bs; - md->mt = update ? timestamp_nonexistent : mt; - md->deferred_failure = deferred_failure; + timestamp start (!ctx.dry_run && depdb::mtime_check () + ? system_clock::now () + : timestamp_unknown); - // @@ TMP: re-enable once recipe becomes move_only_function. - // -#if 0 - return [this, md = move (md)] (action a, const target& t) mutable + if (!ctx.dry_run || verb != 0) { - auto r (perform_update_file_dyndep (a, t, *md)); - md.reset (); // @@ TMP: is this really necessary (+mutable)? - return r; - }; -#else - t.data (move (md)); - return recipe ([this] (action a, const target& t) mutable + execute_update_file (bs, a, t, env, run); + } + + // Extract the dynamic dependency information as byproduct of the recipe + // body. Essentially, this is the `if(!cache)` equivalent of the restart + // loop in exec_depdb_dyndep(). + // + if (!ctx.dry_run) { - auto md (move (t.data> ())); - return perform_update_file_dyndep (a, t, *md); - }); -#endif + using dyndep = dyndep_rule; + using dyndep_format = build::script::parser::dyndep_format; + + depdb dd (move (md.dd)); + + const auto& byp (md.byp); + const location& ll (byp.location); + const char* what (byp.what.c_str ()); + const path& file (byp.file); + + env.clean ({build2::script::cleanup_type::always, file}, + true /* implicit */); + + function map_ext ( + [] (const scope& bs, const string& n, const string& e) + { + // NOTE: another version in exec_depdb_dyndep() and above. + + return dyndep::map_extension (bs, n, e, nullptr); + }); + + // Analogous to exec_depdb_dyndep()::add() but only for cache=false. + // The semantics is quite different, however: instead of updating the + // dynamic prerequisites we verify they are not generated. + // + // Note that fp is expected to be absolute. + // + auto add = [this, &trace, what, + a, &bs, &t, &pts, pts_n = md.pts_n, + &byp, &map_ext, &dd] (path fp) + { + normalize_external (fp, what); + + if (const build2::file* ft = dyndep::find_file ( + trace, what, + a, bs, t, + fp, false /* cache */, true /* normalized */, + map_ext, *byp.default_type).first) + { + // Skip if this is one of the static prerequisites. + // + for (size_t i (0); i != pts_n; ++i) + { + const prerequisite_target& p (pts[i]); + + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : + nullptr)) + { + if (pt == ft) + return; + } + } + + // Verify it has noop recipe. + // + dyndep::verify_existing_file (trace, what, a, t, *ft); + } + + dd.write (fp); + }; + + auto df = make_diag_frame ( + [this, &ll, &t] (const diag_record& dr) + { + if (verb != 0) + dr << info (ll) << "while extracting dynamic dependencies for " + << t; + }); + + ifdstream is (ifdstream::badbit); + try + { + is.open (file); + } + catch (const io_error& e) + { + fail (ll) << "unable to open file " << file << ": " << e; + } + + location il (file, 1); + size_t skip (md.skip_count); + + // The way we parse things is format-specific. + // + // Note: similar code in exec_depdb_dyndep(). Except here we just add + // the paths to depdb without entering them as targets. + // + switch (md.byp.format) + { + case dyndep_format::make: + { + using make_state = make_parser; + using make_type = make_parser::type; + + make_parser make; + + for (string l;; ++il.line) // Reuse the buffer. + { + if (eof (getline (is, l))) + { + if (make.state != make_state::end) + fail (il) << "incomplete make dependency declaration"; + + break; + } + + size_t pos (0); + do + { + // Note that we don't really need a diag frame that prints the + // line being parsed since we are always parsing the file. + // + pair r ( + make.next (l, pos, il, false /* strict */)); + + if (r.second.empty ()) + continue; + + // @@ TODO: what should we do about targets? + // + if (r.first == make_type::target) + continue; + + // Skip until where we left off. + // + if (skip != 0) + { + skip--; + continue; + } + + path f (move (r.second)); + + if (f.relative ()) + { + if (!byp.cwd) + fail (il) << "relative path " << f << " in make dependency " + << "declaration" << + info << "consider using --cwd to specify relative path " + << "base"; + + f = *byp.cwd / f; + } + + add (move (f)); + } + while (pos != l.size ()); + + if (make.state == make_state::end) + break; + } + + break; + } + } + + // Add the terminating blank line. + // + dd.expect (""); + dd.close (); + + md.dd.path = move (dd.path); // For mtime check below. + } + + run.leave (env, script.end_loc); + + timestamp now (system_clock::now ()); + + if (!ctx.dry_run) + depdb::check_mtime (start, md.dd.path, t.path (), now); + + t.mtime (now); + return target_state::changed; } target_state adhoc_buildscript_rule:: perform_update_file_dyndep (action a, const target& xt, match_data& md) const { - tracer trace ("adhoc_buildscript_rule::perform_update_file_dyndep"); + tracer trace ("adhoc_buildscript_rule::perform_update_file"); context& ctx (xt.ctx); const file& t (xt.as ()); - const path& tp (t.path ()); // While we've updated all our prerequisites in apply(), we still need to // execute them here to keep the dependency counts straight. @@ -576,18 +1018,15 @@ namespace build2 if (!ctx.dry_run || verb != 0) { - if (execute_update_file (*md.bs, a, t, env, run, md.deferred_failure)) - ; - else - run.leave (env, script.end_loc); + execute_update_file (*md.bs, a, t, env, run, md.deferred_failure); } - else - run.leave (env, script.end_loc); + + run.leave (env, script.end_loc); timestamp now (system_clock::now ()); if (!ctx.dry_run) - depdb::check_mtime (start, md.dd, tp, now); + depdb::check_mtime (start, md.dd, t.path (), now); t.mtime (now); return target_state::changed; @@ -837,6 +1276,7 @@ namespace build2 return *ps; } + bool r (false); if (!ctx.dry_run || verb != 0) { // Prepare to execute the script diag line and/or body. @@ -844,21 +1284,23 @@ namespace build2 if (bs == nullptr) bs = &t.base_scope (); - if (execute_update_file (*bs, a, t, env, run)) + if ((r = execute_update_file (*bs, a, t, env, run))) { if (!ctx.dry_run) dd.check_mtime (tp); } - else if (depdb_preamble) - run.leave (env, script.end_loc); } - else if (depdb_preamble) + + if (r || depdb_preamble) run.leave (env, script.end_loc); t.mtime (system_clock::now ()); return target_state::changed; } + // Return true if execute_body() was called and thus the caller should call + // run.leave(). + // bool adhoc_buildscript_rule:: execute_update_file (const scope& bs, action, const file& t, @@ -916,7 +1358,10 @@ namespace build2 if (script.body_temp_dir && !script.depdb_preamble_temp_dir) env.set_temp_dir_variable (); - p.execute_body (rs, bs, env, script, run, script.depdb_preamble.empty ()); + p.execute_body (rs, bs, + env, script, run, + script.depdb_preamble.empty () /* enter */, + false /* leave */); if (!ctx.dry_run) { -- cgit v1.1