From 76f1988539c477ad3b906f254654929aec04283c Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Tue, 30 Nov 2021 10:15:33 +0200 Subject: Add support for dynamic dependencies as byproduct of script body Specifically, the `depdb dyndep` builtin now has the --byproduct option (which must come first). In this mode only the --file input is supported. For example: obje{hello.o}: cxx{hello} {{ o = $path($>) t = $(o).t depdb dyndep --byproduct --what=header --default-type=h --file $t diag c++ ($<[0]) $cxx.path $cxx.poptions $cc.poptions $cc.coptions $cxx.coptions $cxx.mode -o $o -MD -MF $t -c $path($<[0]) }} Naturally, this mode does not support dynamic auto-generated prerequisites. If present, such prerequisites must be specified statically in the buildfile. Note also that the --default-prereq-type option has been rename to --default-type. --- libbuild2/adhoc-rule-buildscript.cxx | 573 +++++++++++++++++++++++++---- libbuild2/adhoc-rule-buildscript.hxx | 5 + libbuild2/build/script/builtin-options.cxx | 83 +++-- libbuild2/build/script/builtin-options.hxx | 35 +- libbuild2/build/script/builtin-options.ixx | 102 +++-- libbuild2/build/script/builtin.cli | 30 +- libbuild2/build/script/parser.cxx | 214 +++++++---- libbuild2/build/script/parser.hxx | 60 ++- libbuild2/build/script/script.hxx | 3 +- libbuild2/cc/compile-rule.cxx | 38 +- libbuild2/cc/compile-rule.hxx | 2 +- libbuild2/dyndep.cxx | 242 ++++++++---- libbuild2/dyndep.hxx | 76 +++- 13 files changed, 1136 insertions(+), 327 deletions(-) (limited to 'libbuild2') diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx index b981b8d..db3be30 100644 --- a/libbuild2/adhoc-rule-buildscript.cxx +++ b/libbuild2/adhoc-rule-buildscript.cxx @@ -13,6 +13,7 @@ #include #include // path_perms(), auto_rmfile #include +#include #include // attributes @@ -227,11 +228,30 @@ namespace build2 build::script::default_runner run; path dd; + const scope* bs; timestamp mt; bool deferred_failure; }; + struct adhoc_buildscript_rule::match_data_byproduct + { + match_data_byproduct (action a, const target& t, bool temp_dir) + : env (a, t, temp_dir) {} + + build::script::environment env; + build::script::default_runner run; + + build::script::parser::dyndep_byproduct byp; + + depdb::reopen_state dd; + size_t skip_count = 0; + size_t pts_n; // Number of static prerequisites in prerequisite_targets. + + const scope* bs; + timestamp mt; + }; + bool adhoc_buildscript_rule:: match (action a, target& t, const string& h, match_extra& me) const { @@ -339,15 +359,21 @@ namespace build2 } // This is a perform update on a file target with extraction of dynamic - // dependency information in the depdb preamble (depdb-dyndep). + // dependency information either in the depdb preamble (depdb-dyndep + // without --byproduct) or as a byproduct of the recipe body execution + // (depdb-dyndep with --byproduct). // - // This means we may need to add additional prerequisites (or even target - // group members). We also have to save any such additional prerequisites - // in depdb so that we can check if any of them have changed on subsequent - // updates. So all this means that have to take care of depdb here in - // apply() instead of perform_*() like we normally do. We also do things - // in slightly different order due to the restrictions impose by the match - // phase. + // For the former case, we may need to add additional prerequisites (or + // even target group members). We also have to save any such additional + // prerequisites in depdb so that we can check if any of them have changed + // on subsequent updates. So all this means that we have to take care of + // depdb here in apply() instead of perform_*() like we normally do. We + // also do things in slightly different order due to the restrictions + // impose by the match phase. + // + // The latter case (depdb-dyndep --byproduct) is sort of a combination + // of the normal dyndep and the static case: we check the depdb during + // match but save after executing the recipe. // // Note that the C/C++ header dependency extraction is the canonical // example and all this logic is based on the prior work in the cc module @@ -358,6 +384,7 @@ namespace build2 file& t (xt.as ()); const path& tp (t.path ()); + const scope& bs (t.base_scope ()); if (dir != nullptr) fsdir_rule::perform_update_direct (a, t); @@ -379,10 +406,10 @@ namespace build2 } } - // NOTE: see the "static dependencies" version (with comments) below. - // depdb dd (tp + ".d"); + // NOTE: see the "static dependencies" version (with comments) below. + // if (dd.expect (" 1") != nullptr) l4 ([&]{trace << "rule mismatch forcing update of " << t;}); @@ -435,13 +462,20 @@ namespace build2 } } - const scope& bs (t.base_scope ()); + unique_ptr md; + unique_ptr mdb; - unique_ptr md ( - new match_data (a, t, script.depdb_preamble_temp_dir)); + if (script.depdb_dyndep_byproduct) + { + mdb.reset (new match_data_byproduct ( + a, t, script.depdb_preamble_temp_dir)); + } + else + md.reset (new match_data (a, t, script.depdb_preamble_temp_dir)); - build::script::environment& env (md->env); - build::script::default_runner& run (md->run); + + build::script::environment& env (mdb != nullptr ? mdb->env : md->env); + build::script::default_runner& run (mdb != nullptr ? mdb->run : md->run); run.enter (env, script.start_loc); @@ -485,63 +519,471 @@ namespace build2 } } - // Run the second half of the preamble (depdb-dyndep commands) to extract - // dynamic dependencies. + if (script.depdb_dyndep_byproduct) + { + // If we have the dynamic dependency information as byproduct of the + // recipe body, then do the first part: verify the entries in depdb + // unless we are already updating. Essentially, this is the `if(cache)` + // equivalent of the restart loop in exec_depdb_dyndep(). + // + // Do we really need to update our prerequisite targets in this case (as + // we do above)? While it may seem like we should be able to avoid it by + // triggering update on encountering any non-existent files in depbd, we + // may actually incorrectly "validate" some number of depdb entires + // while having an out-of-date main source file. We could probably avoid + // the update if we are already updating. + + using dyndep = dyndep_rule; + + // Extract the depdb-dyndep command's information (we may also execute + // some variable assignments). + // + { + build::script::parser p (ctx); + mdb->byp = p.execute_depdb_preamble_dyndep_byproduct ( + a, bs, t, + env, script, run, + dd); + } + + mdb->pts_n = pts.size (); + + if (!update) + { + const auto& byp (mdb->byp); + const char* what (byp.what.c_str ()); + const location& ll (byp.location); + + function map_ext ( + [] (const scope& bs, const string& n, const string& e) + { + // NOTE: another version in exec_depdb_dyndep(). + + return dyndep::map_extension (bs, n, e, nullptr); + }); + + // Similar to exec_depdb_dyndep()::add() but only for cache=true and + // without support for generated files. + // + // Note that we have to update each file for the same reason as the + // main source file -- if any of them changed, then we must assume the + // subsequent entries are invalid. + // + size_t& skip_count (mdb->skip_count); + + auto add = [this, &trace, what, + a, &bs, &t, &pts, pts_n = mdb->pts_n, + &byp, &map_ext, + &skip_count, mt] (path fp) -> optional + { + if (const build2::file* ft = dyndep::enter_file ( + trace, what, + a, bs, t, + fp, true /* cache */, true /* normalized */, + map_ext, *byp.default_type).first) + { + // Skip if this is one of the static prerequisites. + // + for (size_t i (0); i != pts_n; ++i) + { + const prerequisite_target& p (pts[i]); + + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : + nullptr)) + { + if (pt == ft) + { + // Note that we have to increment the skip count since we + // skip before performing this test. + // + skip_count++; + return false; + } + } + } + + if (optional u = dyndep::inject_existing_file ( + trace, what, + a, t, + *ft, mt, false /* fail */)) + { + skip_count++; + return *u; + } + } + + return nullopt; + }; + + auto df = make_diag_frame ( + [this, &ll, &t] (const diag_record& dr) + { + if (verb != 0) + dr << info (ll) << "while extracting dynamic dependencies for " + << t; + }); + + while (!update) + { + // We should always end with a blank line. + // + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + update = true; + break; + } + + if (l->empty ()) // Done, nothing changed. + break; + + if (optional r = add (path (move (*l)))) + { + if (*r) + update = true; + } + else + { + // Invalidate this line and trigger update. + // + dd.write (); + update = true; + } + + if (update) + l6 ([&]{trace << "restarting (cache)";}); + } + } + + // Note that in case of dry run we will have an incomplete (but valid) + // database which will be updated on the next non-dry run. + // + if (!update || ctx.dry_run) + dd.close (false /* mtime_check */); + else + mdb->dd = dd.close_to_reopen (); + + // Pass on base scope and update/mtime. + // + mdb->bs = &bs; + mdb->mt = update ? timestamp_nonexistent : mt; + + // @@ TMP: re-enable once recipe becomes move_only_function. + // +#if 0 + return [this, md = move (mdb)] (action a, const target& t) mutable + { + auto r (perform_update_file_dyndep_byproduct (a, t, *md)); + md.reset (); // @@ TMP: is this really necessary (+mutable)? + return r; + }; +#else + t.data (move (mdb)); + return recipe ([this] (action a, const target& t) mutable + { + auto md (move (t.data> ())); + return perform_update_file_dyndep_byproduct (a, t, *md); + }); +#endif + } + else + { + // Run the second half of the preamble (depdb-dyndep commands) to + // extract dynamic dependencies. + // + // Note that this should be the last update to depdb (the invalidation + // order semantics). + // + bool deferred_failure (false); + { + build::script::parser p (ctx); + p.execute_depdb_preamble_dyndep (a, bs, t, + env, script, run, + dd, + update, + deferred_failure, + mt); + } + + if (update && dd.reading () && !ctx.dry_run) + dd.touch = timestamp_unknown; + + dd.close (false /* mtime_check */); + md->dd = move (dd.path); + + // Pass on base scope and update/mtime. + // + md->bs = &bs; + md->mt = update ? timestamp_nonexistent : mt; + md->deferred_failure = deferred_failure; + + // @@ TMP: re-enable once recipe becomes move_only_function. + // +#if 0 + return [this, md = move (md)] (action a, const target& t) mutable + { + auto r (perform_update_file_dyndep (a, t, *md)); + md.reset (); // @@ TMP: is this really necessary (+mutable)? + return r; + }; +#else + t.data (move (md)); + return recipe ([this] (action a, const target& t) mutable + { + auto md (move (t.data> ())); + return perform_update_file_dyndep (a, t, *md); + }); +#endif + } + } + + target_state adhoc_buildscript_rule:: + perform_update_file_dyndep_byproduct (action a, + const target& xt, + match_data_byproduct& md) const + { + // Note: using shared function name among the three variants. // - // Note that this should be the last update to depdb (the invalidation - // order semantics). + tracer trace ("adhoc_buildscript_rule::perform_update_file"); + + context& ctx (xt.ctx); + + const file& t (xt.as ()); + + // While we've updated all our prerequisites in apply(), we still need to + // execute them here to keep the dependency counts straight. // - bool deferred_failure (false); + const auto& pts (t.prerequisite_targets[a]); + + for (const prerequisite_target& p: pts) { - build::script::parser p (ctx); - p.execute_depdb_preamble_dyndep (a, bs, t, - env, script, run, - dd, - update, - deferred_failure, - mt); + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : nullptr)) + { + target_state ts (execute_wait (a, *pt)); + assert (ts == target_state::unchanged || ts == target_state::changed); + } } - if (update && dd.reading () && !ctx.dry_run) - dd.touch = timestamp_unknown; + build::script::environment& env (md.env); + build::script::default_runner& run (md.run); + + if (md.mt != timestamp_nonexistent) + { + run.leave (env, script.end_loc); + return target_state::unchanged; + } - dd.close (false /* mtime_check */); - md->dd = move (dd.path); + const scope& bs (*md.bs); - // Pass on base scope and update/mtime. + // Sequence start time for mtime checks below. // - md->bs = &bs; - md->mt = update ? timestamp_nonexistent : mt; - md->deferred_failure = deferred_failure; + timestamp start (!ctx.dry_run && depdb::mtime_check () + ? system_clock::now () + : timestamp_unknown); - // @@ TMP: re-enable once recipe becomes move_only_function. - // -#if 0 - return [this, md = move (md)] (action a, const target& t) mutable + if (!ctx.dry_run || verb != 0) { - auto r (perform_update_file_dyndep (a, t, *md)); - md.reset (); // @@ TMP: is this really necessary (+mutable)? - return r; - }; -#else - t.data (move (md)); - return recipe ([this] (action a, const target& t) mutable + execute_update_file (bs, a, t, env, run); + } + + // Extract the dynamic dependency information as byproduct of the recipe + // body. Essentially, this is the `if(!cache)` equivalent of the restart + // loop in exec_depdb_dyndep(). + // + if (!ctx.dry_run) { - auto md (move (t.data> ())); - return perform_update_file_dyndep (a, t, *md); - }); -#endif + using dyndep = dyndep_rule; + using dyndep_format = build::script::parser::dyndep_format; + + depdb dd (move (md.dd)); + + const auto& byp (md.byp); + const location& ll (byp.location); + const char* what (byp.what.c_str ()); + const path& file (byp.file); + + env.clean ({build2::script::cleanup_type::always, file}, + true /* implicit */); + + function map_ext ( + [] (const scope& bs, const string& n, const string& e) + { + // NOTE: another version in exec_depdb_dyndep() and above. + + return dyndep::map_extension (bs, n, e, nullptr); + }); + + // Analogous to exec_depdb_dyndep()::add() but only for cache=false. + // The semantics is quite different, however: instead of updating the + // dynamic prerequisites we verify they are not generated. + // + // Note that fp is expected to be absolute. + // + auto add = [this, &trace, what, + a, &bs, &t, &pts, pts_n = md.pts_n, + &byp, &map_ext, &dd] (path fp) + { + normalize_external (fp, what); + + if (const build2::file* ft = dyndep::find_file ( + trace, what, + a, bs, t, + fp, false /* cache */, true /* normalized */, + map_ext, *byp.default_type).first) + { + // Skip if this is one of the static prerequisites. + // + for (size_t i (0); i != pts_n; ++i) + { + const prerequisite_target& p (pts[i]); + + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : + nullptr)) + { + if (pt == ft) + return; + } + } + + // Verify it has noop recipe. + // + dyndep::verify_existing_file (trace, what, a, t, *ft); + } + + dd.write (fp); + }; + + auto df = make_diag_frame ( + [this, &ll, &t] (const diag_record& dr) + { + if (verb != 0) + dr << info (ll) << "while extracting dynamic dependencies for " + << t; + }); + + ifdstream is (ifdstream::badbit); + try + { + is.open (file); + } + catch (const io_error& e) + { + fail (ll) << "unable to open file " << file << ": " << e; + } + + location il (file, 1); + size_t skip (md.skip_count); + + // The way we parse things is format-specific. + // + // Note: similar code in exec_depdb_dyndep(). Except here we just add + // the paths to depdb without entering them as targets. + // + switch (md.byp.format) + { + case dyndep_format::make: + { + using make_state = make_parser; + using make_type = make_parser::type; + + make_parser make; + + for (string l;; ++il.line) // Reuse the buffer. + { + if (eof (getline (is, l))) + { + if (make.state != make_state::end) + fail (il) << "incomplete make dependency declaration"; + + break; + } + + size_t pos (0); + do + { + // Note that we don't really need a diag frame that prints the + // line being parsed since we are always parsing the file. + // + pair r ( + make.next (l, pos, il, false /* strict */)); + + if (r.second.empty ()) + continue; + + // @@ TODO: what should we do about targets? + // + if (r.first == make_type::target) + continue; + + // Skip until where we left off. + // + if (skip != 0) + { + skip--; + continue; + } + + path f (move (r.second)); + + if (f.relative ()) + { + if (!byp.cwd) + fail (il) << "relative path " << f << " in make dependency " + << "declaration" << + info << "consider using --cwd to specify relative path " + << "base"; + + f = *byp.cwd / f; + } + + add (move (f)); + } + while (pos != l.size ()); + + if (make.state == make_state::end) + break; + } + + break; + } + } + + // Add the terminating blank line. + // + dd.expect (""); + dd.close (); + + md.dd.path = move (dd.path); // For mtime check below. + } + + run.leave (env, script.end_loc); + + timestamp now (system_clock::now ()); + + if (!ctx.dry_run) + depdb::check_mtime (start, md.dd.path, t.path (), now); + + t.mtime (now); + return target_state::changed; } target_state adhoc_buildscript_rule:: perform_update_file_dyndep (action a, const target& xt, match_data& md) const { - tracer trace ("adhoc_buildscript_rule::perform_update_file_dyndep"); + tracer trace ("adhoc_buildscript_rule::perform_update_file"); context& ctx (xt.ctx); const file& t (xt.as ()); - const path& tp (t.path ()); // While we've updated all our prerequisites in apply(), we still need to // execute them here to keep the dependency counts straight. @@ -576,18 +1018,15 @@ namespace build2 if (!ctx.dry_run || verb != 0) { - if (execute_update_file (*md.bs, a, t, env, run, md.deferred_failure)) - ; - else - run.leave (env, script.end_loc); + execute_update_file (*md.bs, a, t, env, run, md.deferred_failure); } - else - run.leave (env, script.end_loc); + + run.leave (env, script.end_loc); timestamp now (system_clock::now ()); if (!ctx.dry_run) - depdb::check_mtime (start, md.dd, tp, now); + depdb::check_mtime (start, md.dd, t.path (), now); t.mtime (now); return target_state::changed; @@ -837,6 +1276,7 @@ namespace build2 return *ps; } + bool r (false); if (!ctx.dry_run || verb != 0) { // Prepare to execute the script diag line and/or body. @@ -844,21 +1284,23 @@ namespace build2 if (bs == nullptr) bs = &t.base_scope (); - if (execute_update_file (*bs, a, t, env, run)) + if ((r = execute_update_file (*bs, a, t, env, run))) { if (!ctx.dry_run) dd.check_mtime (tp); } - else if (depdb_preamble) - run.leave (env, script.end_loc); } - else if (depdb_preamble) + + if (r || depdb_preamble) run.leave (env, script.end_loc); t.mtime (system_clock::now ()); return target_state::changed; } + // Return true if execute_body() was called and thus the caller should call + // run.leave(). + // bool adhoc_buildscript_rule:: execute_update_file (const scope& bs, action, const file& t, @@ -916,7 +1358,10 @@ namespace build2 if (script.body_temp_dir && !script.depdb_preamble_temp_dir) env.set_temp_dir_variable (); - p.execute_body (rs, bs, env, script, run, script.depdb_preamble.empty ()); + p.execute_body (rs, bs, + env, script, run, + script.depdb_preamble.empty () /* enter */, + false /* leave */); if (!ctx.dry_run) { diff --git a/libbuild2/adhoc-rule-buildscript.hxx b/libbuild2/adhoc-rule-buildscript.hxx index 51d37d4..c39a0c0 100644 --- a/libbuild2/adhoc-rule-buildscript.hxx +++ b/libbuild2/adhoc-rule-buildscript.hxx @@ -39,10 +39,15 @@ namespace build2 perform_update_file (action, const target&) const; struct match_data; + struct match_data_byproduct; target_state perform_update_file_dyndep (action, const target&, match_data&) const; + target_state + perform_update_file_dyndep_byproduct ( + action, const target&, match_data_byproduct&) const; + bool execute_update_file (const scope&, action a, const file&, diff --git a/libbuild2/build/script/builtin-options.cxx b/libbuild2/build/script/builtin-options.cxx index cf99b12..f66fe47 100644 --- a/libbuild2/build/script/builtin-options.cxx +++ b/libbuild2/build/script/builtin-options.cxx @@ -397,11 +397,11 @@ namespace build2 { namespace script { - // depdb_dep_options + // depdb_dyndep_options // - depdb_dep_options:: - depdb_dep_options () + depdb_dyndep_options:: + depdb_dyndep_options () : file_ (), file_specified_ (false), format_ (), @@ -410,12 +410,14 @@ namespace build2 what_specified_ (false), include_path_ (), include_path_specified_ (false), - default_prereq_type_ (), - default_prereq_type_specified_ (false) + default_type_ (), + default_type_specified_ (false), + cwd_ (), + cwd_specified_ (false) { } - bool depdb_dep_options:: + bool depdb_dyndep_options:: parse (int& argc, char** argv, bool erase, @@ -427,7 +429,7 @@ namespace build2 return r; } - bool depdb_dep_options:: + bool depdb_dyndep_options:: parse (int start, int& argc, char** argv, @@ -440,7 +442,7 @@ namespace build2 return r; } - bool depdb_dep_options:: + bool depdb_dyndep_options:: parse (int& argc, char** argv, int& end, @@ -454,7 +456,7 @@ namespace build2 return r; } - bool depdb_dep_options:: + bool depdb_dyndep_options:: parse (int start, int& argc, char** argv, @@ -469,7 +471,7 @@ namespace build2 return r; } - bool depdb_dep_options:: + bool depdb_dyndep_options:: parse (::build2::build::script::cli::scanner& s, ::build2::build::script::cli::unknown_mode opt, ::build2::build::script::cli::unknown_mode arg) @@ -479,44 +481,47 @@ namespace build2 } typedef - std::map - _cli_depdb_dep_options_map; + std::map + _cli_depdb_dyndep_options_map; - static _cli_depdb_dep_options_map _cli_depdb_dep_options_map_; + static _cli_depdb_dyndep_options_map _cli_depdb_dyndep_options_map_; - struct _cli_depdb_dep_options_map_init + struct _cli_depdb_dyndep_options_map_init { - _cli_depdb_dep_options_map_init () - { - _cli_depdb_dep_options_map_["--file"] = - &::build2::build::script::cli::thunk< depdb_dep_options, path, &depdb_dep_options::file_, - &depdb_dep_options::file_specified_ >; - _cli_depdb_dep_options_map_["--format"] = - &::build2::build::script::cli::thunk< depdb_dep_options, string, &depdb_dep_options::format_, - &depdb_dep_options::format_specified_ >; - _cli_depdb_dep_options_map_["--what"] = - &::build2::build::script::cli::thunk< depdb_dep_options, string, &depdb_dep_options::what_, - &depdb_dep_options::what_specified_ >; - _cli_depdb_dep_options_map_["--include-path"] = - &::build2::build::script::cli::thunk< depdb_dep_options, dir_paths, &depdb_dep_options::include_path_, - &depdb_dep_options::include_path_specified_ >; - _cli_depdb_dep_options_map_["-I"] = - &::build2::build::script::cli::thunk< depdb_dep_options, dir_paths, &depdb_dep_options::include_path_, - &depdb_dep_options::include_path_specified_ >; - _cli_depdb_dep_options_map_["--default-prereq-type"] = - &::build2::build::script::cli::thunk< depdb_dep_options, string, &depdb_dep_options::default_prereq_type_, - &depdb_dep_options::default_prereq_type_specified_ >; + _cli_depdb_dyndep_options_map_init () + { + _cli_depdb_dyndep_options_map_["--file"] = + &::build2::build::script::cli::thunk< depdb_dyndep_options, path, &depdb_dyndep_options::file_, + &depdb_dyndep_options::file_specified_ >; + _cli_depdb_dyndep_options_map_["--format"] = + &::build2::build::script::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::format_, + &depdb_dyndep_options::format_specified_ >; + _cli_depdb_dyndep_options_map_["--what"] = + &::build2::build::script::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::what_, + &depdb_dyndep_options::what_specified_ >; + _cli_depdb_dyndep_options_map_["--include-path"] = + &::build2::build::script::cli::thunk< depdb_dyndep_options, dir_paths, &depdb_dyndep_options::include_path_, + &depdb_dyndep_options::include_path_specified_ >; + _cli_depdb_dyndep_options_map_["-I"] = + &::build2::build::script::cli::thunk< depdb_dyndep_options, dir_paths, &depdb_dyndep_options::include_path_, + &depdb_dyndep_options::include_path_specified_ >; + _cli_depdb_dyndep_options_map_["--default-type"] = + &::build2::build::script::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::default_type_, + &depdb_dyndep_options::default_type_specified_ >; + _cli_depdb_dyndep_options_map_["--cwd"] = + &::build2::build::script::cli::thunk< depdb_dyndep_options, dir_path, &depdb_dyndep_options::cwd_, + &depdb_dyndep_options::cwd_specified_ >; } }; - static _cli_depdb_dep_options_map_init _cli_depdb_dep_options_map_init_; + static _cli_depdb_dyndep_options_map_init _cli_depdb_dyndep_options_map_init_; - bool depdb_dep_options:: + bool depdb_dyndep_options:: _parse (const char* o, ::build2::build::script::cli::scanner& s) { - _cli_depdb_dep_options_map::const_iterator i (_cli_depdb_dep_options_map_.find (o)); + _cli_depdb_dyndep_options_map::const_iterator i (_cli_depdb_dyndep_options_map_.find (o)); - if (i != _cli_depdb_dep_options_map_.end ()) + if (i != _cli_depdb_dyndep_options_map_.end ()) { (*(i->second)) (*this, s); return true; @@ -525,7 +530,7 @@ namespace build2 return false; } - bool depdb_dep_options:: + bool depdb_dyndep_options:: _parse (::build2::build::script::cli::scanner& s, ::build2::build::script::cli::unknown_mode opt_mode, ::build2::build::script::cli::unknown_mode arg_mode) diff --git a/libbuild2/build/script/builtin-options.hxx b/libbuild2/build/script/builtin-options.hxx index 85d67b9..15119f4 100644 --- a/libbuild2/build/script/builtin-options.hxx +++ b/libbuild2/build/script/builtin-options.hxx @@ -297,10 +297,10 @@ namespace build2 { namespace script { - class depdb_dep_options + class depdb_dyndep_options { public: - depdb_dep_options (); + depdb_dyndep_options (); // Return true if anything has been parsed. // @@ -404,19 +404,34 @@ namespace build2 include_path_specified (bool); const string& - default_prereq_type () const; + default_type () const; string& - default_prereq_type (); + default_type (); void - default_prereq_type (const string&); + default_type (const string&); bool - default_prereq_type_specified () const; + default_type_specified () const; void - default_prereq_type_specified (bool); + default_type_specified (bool); + + const dir_path& + cwd () const; + + dir_path& + cwd (); + + void + cwd (const dir_path&); + + bool + cwd_specified () const; + + void + cwd_specified (bool); // Implementation details. // @@ -439,8 +454,10 @@ namespace build2 bool what_specified_; dir_paths include_path_; bool include_path_specified_; - string default_prereq_type_; - bool default_prereq_type_specified_; + string default_type_; + bool default_type_specified_; + dir_path cwd_; + bool cwd_specified_; }; } } diff --git a/libbuild2/build/script/builtin-options.ixx b/libbuild2/build/script/builtin-options.ixx index 06575c8..c6266d0 100644 --- a/libbuild2/build/script/builtin-options.ixx +++ b/libbuild2/build/script/builtin-options.ixx @@ -176,157 +176,187 @@ namespace build2 { namespace script { - // depdb_dep_options + // depdb_dyndep_options // - inline const path& depdb_dep_options:: + inline const path& depdb_dyndep_options:: file () const { return this->file_; } - inline path& depdb_dep_options:: + inline path& depdb_dyndep_options:: file () { return this->file_; } - inline void depdb_dep_options:: + inline void depdb_dyndep_options:: file (const path& x) { this->file_ = x; } - inline bool depdb_dep_options:: + inline bool depdb_dyndep_options:: file_specified () const { return this->file_specified_; } - inline void depdb_dep_options:: + inline void depdb_dyndep_options:: file_specified (bool x) { this->file_specified_ = x; } - inline const string& depdb_dep_options:: + inline const string& depdb_dyndep_options:: format () const { return this->format_; } - inline string& depdb_dep_options:: + inline string& depdb_dyndep_options:: format () { return this->format_; } - inline void depdb_dep_options:: + inline void depdb_dyndep_options:: format (const string& x) { this->format_ = x; } - inline bool depdb_dep_options:: + inline bool depdb_dyndep_options:: format_specified () const { return this->format_specified_; } - inline void depdb_dep_options:: + inline void depdb_dyndep_options:: format_specified (bool x) { this->format_specified_ = x; } - inline const string& depdb_dep_options:: + inline const string& depdb_dyndep_options:: what () const { return this->what_; } - inline string& depdb_dep_options:: + inline string& depdb_dyndep_options:: what () { return this->what_; } - inline void depdb_dep_options:: + inline void depdb_dyndep_options:: what (const string& x) { this->what_ = x; } - inline bool depdb_dep_options:: + inline bool depdb_dyndep_options:: what_specified () const { return this->what_specified_; } - inline void depdb_dep_options:: + inline void depdb_dyndep_options:: what_specified (bool x) { this->what_specified_ = x; } - inline const dir_paths& depdb_dep_options:: + inline const dir_paths& depdb_dyndep_options:: include_path () const { return this->include_path_; } - inline dir_paths& depdb_dep_options:: + inline dir_paths& depdb_dyndep_options:: include_path () { return this->include_path_; } - inline void depdb_dep_options:: + inline void depdb_dyndep_options:: include_path (const dir_paths& x) { this->include_path_ = x; } - inline bool depdb_dep_options:: + inline bool depdb_dyndep_options:: include_path_specified () const { return this->include_path_specified_; } - inline void depdb_dep_options:: + inline void depdb_dyndep_options:: include_path_specified (bool x) { this->include_path_specified_ = x; } - inline const string& depdb_dep_options:: - default_prereq_type () const + inline const string& depdb_dyndep_options:: + default_type () const { - return this->default_prereq_type_; + return this->default_type_; } - inline string& depdb_dep_options:: - default_prereq_type () + inline string& depdb_dyndep_options:: + default_type () { - return this->default_prereq_type_; + return this->default_type_; } - inline void depdb_dep_options:: - default_prereq_type (const string& x) + inline void depdb_dyndep_options:: + default_type (const string& x) { - this->default_prereq_type_ = x; + this->default_type_ = x; } - inline bool depdb_dep_options:: - default_prereq_type_specified () const + inline bool depdb_dyndep_options:: + default_type_specified () const { - return this->default_prereq_type_specified_; + return this->default_type_specified_; } - inline void depdb_dep_options:: - default_prereq_type_specified (bool x) + inline void depdb_dyndep_options:: + default_type_specified (bool x) { - this->default_prereq_type_specified_ = x; + this->default_type_specified_ = x; + } + + inline const dir_path& depdb_dyndep_options:: + cwd () const + { + return this->cwd_; + } + + inline dir_path& depdb_dyndep_options:: + cwd () + { + return this->cwd_; + } + + inline void depdb_dyndep_options:: + cwd (const dir_path& x) + { + this->cwd_ = x; + } + + inline bool depdb_dyndep_options:: + cwd_specified () const + { + return this->cwd_specified_; + } + + inline void depdb_dyndep_options:: + cwd_specified (bool x) + { + this->cwd_specified_ = x; } } } diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli index 3ed3659..fafb330 100644 --- a/libbuild2/build/script/builtin.cli +++ b/libbuild2/build/script/builtin.cli @@ -15,17 +15,33 @@ namespace build2 { // Pseudo-builtin options. // - class depdb_dep_options + class depdb_dyndep_options { // Note that --byproduct, if any, must be the first option and is // handled ad hoc, kind of as a sub-command. // - path --file; // Read from file rather than stdin. - string --format; // Dependency format: make (default). - string --what; // Dependency kind, e.g., "header". - dir_paths --include-path|-I; // Search paths for generated files. - string --default-prereq-type; // Default prerequisite type to use - // if none could be derived from ext. + // Note that in the future we may extend --cwd support to the non- + // byproduct mode where it will also have the `env --cwd` semantics + // (thus the matching name). Note that it will also be incompatible + // with support for generated files (and thus -I) at least in the make + // format where we use relative paths for non-existent files. + // + // Note on naming: whenever we (may) have two options, one for target + // and the other for prerequisite, we omit "prerequisite" as that's + // what we extract by default and most commonly. For example: + // + // --what --what-target + // --default-type --default-target-type + // + path --file; // Read from file rather than stdin. + string --format; // Dependency format: make (default). + string --what; // Dependency kind, e.g., "header". + dir_paths --include-path|-I; // Search paths for generated files. + string --default-type; // Default prerequisite type to use + // if none could be derived from ext. + dir_path --cwd; // Builtin's working directory used + // to complete relative paths (only + // in --byproduct mode). }; } } diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index 67dbf69..15545cf 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -135,7 +135,10 @@ namespace build2 // s.depdb_clear = depdb_clear_.has_value (); if (depdb_dyndep_) + { s.depdb_dyndep = depdb_dyndep_->second; + s.depdb_dyndep_byproduct = depdb_dyndep_byproduct_; + } s.depdb_preamble = move (depdb_preamble_); return s; @@ -548,7 +551,7 @@ namespace build2 } else { - // Verify depdb-dyndep is last. + // Verify depdb-dyndep is last and detect the byproduct flavor. // if (v == "dyndep") { @@ -563,10 +566,8 @@ namespace build2 fail (l) << "multiple 'depdb dyndep' calls" << info (depdb_dyndep_->first) << "previous call is here"; -#if 0 if (peek () == type::word && peeked ().value == "--byproduct") - ; -#endif + depdb_dyndep_byproduct_ = true; } else { @@ -933,7 +934,8 @@ namespace build2 depdb& dd, bool* update, bool* deferred_failure, - optional mt) + optional mt, + dyndep_byproduct* byp) { tracer trace ("exec_depdb_preamble"); @@ -961,8 +963,9 @@ namespace build2 bool* update; bool* deferred_failure; optional mt; + dyndep_byproduct* byp; - } data {trace, a, bs, t, e, s, dd, update, deferred_failure, mt}; + } data {trace, a, bs, t, e, s, dd, update, deferred_failure, mt, byp}; auto exec_cmd = [this, &data] (token& t, build2::script::token_type& tt, @@ -986,7 +989,8 @@ namespace build2 if (cmd == "dyndep") { - // Note: cast is safe since this is always executed in apply(). + // Note: the cast is safe since the part where the target is + // modified is always executed in apply(). // exec_depdb_dyndep (t, tt, li, ll, @@ -994,7 +998,8 @@ namespace build2 data.dd, *data.update, *data.deferred_failure, - *data.mt); + *data.mt, + data.byp); } else { @@ -1207,7 +1212,8 @@ namespace build2 depdb& dd, bool& update, bool& deferred_failure, - timestamp mt) + timestamp mt, + dyndep_byproduct* byprod_result) { tracer trace ("exec_depdb_dyndep"); @@ -1215,13 +1221,22 @@ namespace build2 // Similar approach to parse_env_builtin(). // - depdb_dep_options ops; + depdb_dyndep_options ops; bool prog (false); + bool byprod (false); { auto& t (lt); auto& tt (ltt); - next (t, tt); // Skip 'dep' command. + next (t, tt); // Skip the 'dyndep' command. + + if (tt == type::word && t.value == "--byproduct") + { + byprod = true; + next (t, tt); + } + + assert (byprod == (byprod_result != nullptr)); // Note that an option name and value can belong to different name // chunks. That's why we parse the arguments in the chunking mode @@ -1273,6 +1288,10 @@ namespace build2 if (prog) { + if (byprod) + fail (t) << "depdb dyndep: --byproduct cannot be used with " + << "program"; + next (t, tt); // Skip '--'. if (tt == type::newline || tt == type::eos) @@ -1311,12 +1330,10 @@ namespace build2 continue; } -#if 0 // Handle --byproduct in the wrong place. // if (strcmp (a, "--byproduct") == 0) fail (ll) << "depdb dyndep: --byproduct must be first option"; -#endif // Handle unknown option. // @@ -1334,6 +1351,64 @@ namespace build2 } } + // --what + // + const char* what (ops.what_specified () + ? ops.what ().c_str () + : "file"); + + // --format + // + dyndep_format format (dyndep_format::make); + + if (ops.format_specified ()) + { + const string& f (ops.format ()); + + if (f != "make") + fail (ll) << "depdb dyndep: invalid --format option value '" + << f << "'"; + } + + // --cwd + // + optional cwd; + + if (ops.cwd_specified ()) + { + if (!byprod) + fail (ll) << "depdb dyndep: --cwd only valid in --byproduct mode"; + + cwd = move (ops.cwd ()); + + if (cwd->relative ()) + fail (ll) << "depdb dyndep: relative path specified with --cwd"; + } + + // --file + // + // Note that if --file is specified without a program, then we assume + // it is one of the static prerequisites. + // + optional file; + + if (ops.file_specified ()) + { + file = move (ops.file ()); + + if (file->relative ()) + { + if (!cwd) + fail (ll) << "depdb dyndep: relative path specified with --file"; + + *file = *cwd / *file; + } + } + else if (!prog) + fail (ll) << "depdb dyndep: program or --file expected"; + + // --default-type + // // Get the default prerequisite type falling back to file{} if not // specified. // @@ -1344,17 +1419,34 @@ namespace build2 // system headers to h{} targets analogous to the c module's rule. // const target_type* def_pt; - if (ops.default_prereq_type_specified ()) + if (ops.default_type_specified ()) { - const string& t (ops.default_prereq_type ()); + const string& t (ops.default_type ()); def_pt = bs.find_target_type (t); if (def_pt == nullptr) - fail (ll) << "unknown target type '" << t << "'"; + fail (ll) << "unknown target type '" << t << "' specific with " + << "--default-type"; } else def_pt = &file::static_type; + if (byprod) + { + if (!ops.include_path ().empty ()) + fail (ll) << "depdb dyndep: -I specified with --byproduct"; + + *byprod_result = dyndep_byproduct { + ll, + format, + move (cwd), + move (*file), + ops.what_specified () ? move (ops.what ()) : string (what), + def_pt}; + + return; + } + // This code is based on the prior work in the cc module (specifically // extract_headers()) where you can often find more detailed rationale // for some of the steps performed. @@ -1369,6 +1461,8 @@ namespace build2 function map_ext ( [] (const scope& bs, const string& n, const string& e) { + // NOTE: another version in adhoc_buildscript_rule::apply(). + // @@ TODO: allow specifying base target types. // // Feels like the only reason one would want to specify base types @@ -1403,7 +1497,7 @@ namespace build2 { tracer& trace; const location& ll; - const depdb_dep_options& ops; + const depdb_dyndep_options& ops; optional map; } pfx_data {trace, ll, ops, nullopt}; @@ -1440,11 +1534,6 @@ namespace build2 }; } - optional file; - enum class format {make} fmt (format::make); - command_expr cmd; - srcout_map so_map; - // Parse the remainder of the command line as a program (which can be // a pipe). If file is absent, then we save the command's stdout to a // pipe. Otherwise, assume the command writes to file and add it to @@ -1454,31 +1543,13 @@ namespace build2 // could do other broken tools). However, the user can always merge // stderr to stdout (2>&1). // + command_expr cmd; + srcout_map so_map; + auto init_run = [this, &ctx, <, <t, &ll, - &ops, prog, &file, &cmd, &so_map] () + prog, &file, &ops, &cmd, &so_map] () { - // --format - // - if (ops.format_specified ()) - { - const string& f (ops.format ()); - - if (f != "make") - fail (ll) << "depdb dyndep: invalid --format option value '" - << f << "'"; - } - - // --file - // - if (ops.file_specified ()) - { - file = move (ops.file ()); - - if (file->relative ()) - fail (ll) << "depdb dyndep: relative path specified with --file"; - } - // Populate the srcout map with the -I$out_base -I$src_base pairs. // { @@ -1515,25 +1586,20 @@ namespace build2 {build2::script::cleanup_type::always, *file}, true /* implicit */); } - else - { - // Assume file is one of the prerequisites. - // - if (!file) - fail (ll) << "depdb dyndep: program or --file expected"; - } }; // Enter as a target, update, and add to the list of prerequisite // targets a file. // - const char* what (ops.what_specified () - ? ops.what ().c_str () - : "file"); - + // Note that these targets don't end up in $< (which is the right + // thing) because that variable has already been initialized (in the + // environment ctor). + // size_t skip_count (0); + auto& pts (t.prerequisite_targets[a]); + auto add = [this, &trace, what, - a, &bs, &t, + a, &bs, &t, &pts, pts_n = pts.size (), &map_ext, def_pt, &pfx_map, &so_map, &dd, &skip_count] (path fp, bool cache, @@ -1567,16 +1633,38 @@ namespace build2 if (const build2::file* ft = dyndep::enter_file ( trace, what, a, bs, t, - move (fp), cache, false /* normalize */, + fp, cache, cache /* normalized */, map_ext, *def_pt, pfx_map, so_map).first) { + // Skip if this is one of the static prerequisites. + // + for (size_t i (0); i != pts_n; ++i) + { + const prerequisite_target& p (pts[i]); + + if (const target* pt = + (p.target != nullptr ? p.target : + p.data != 0 ? reinterpret_cast (p.data) : + nullptr)) + { + if (pt == ft) + { + // Note that we have to increment the skip count since we + // skip before performing this test. + // + skip_count++; + return false; + } + } + } + if (optional u = dyndep::inject_file ( trace, what, a, t, *ft, mt, false /* fail */)) { if (!cache) - dd.expect (ft->path ()); + dd.expect (ft->path ()); // @@ Use fp (or verify match)? skip_count++; return *u; @@ -1721,14 +1809,16 @@ namespace build2 : path_name ("")); location il (in, 1); + size_t skip (skip_count); // The way we parse things is format-specific. // - size_t skip (skip_count); - - switch (fmt) + // Note: similar code in + // adhoc_buildscript_rule::perform_update_file_dyndep_byproduct(). + // + switch (format) { - case format::make: + case dyndep_format::make: { using make_state = make_parser; using make_type = make_parser::type; diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx index da15509..bbdb052 100644 --- a/libbuild2/build/script/parser.hxx +++ b/libbuild2/build/script/parser.hxx @@ -127,6 +127,46 @@ namespace build2 dd, &update, &deferred_failure, mt); } + // This version doesn't actually execute the depdb-dyndep builtin (but + // may execute some variable assignments) instead returning all the + // information (extracted from options) necessary to implement the + // depdb-dyndep --byproduct logic (which fits better into the rule + // implementation). + // + enum class dyndep_format {make}; + + struct dyndep_byproduct + { + location_value location; + dyndep_format format; + optional cwd; + path file; + string what; + const target_type* default_type; + }; + + dyndep_byproduct + execute_depdb_preamble_dyndep_byproduct ( + action a, const scope& base, const file& t, + environment& e, const script& s, runner& r, + depdb& dd) + { + // This is getting really ugly (we also don't really need to pass + // depdb here). One day we will find a better way... + // + bool update, deferred_failure; // Dymmy. + timestamp mt; // Dummy. + + dyndep_byproduct v; + exec_depdb_preamble ( + a, base, t, + e, s, r, + s.depdb_preamble.begin () + *s.depdb_dyndep, + s.depdb_preamble.end (), + dd, &update, &deferred_failure, mt, &v); + return v; + } + // Parse a special builtin line into names, performing the variable // and pattern expansions. If omit_builtin is true, then omit the // builtin name from the result. @@ -166,7 +206,8 @@ namespace build2 depdb&, bool* update = nullptr, bool* deferred_failure = nullptr, - optional mt = nullopt); + optional mt = nullopt, + dyndep_byproduct* = nullptr); void exec_depdb_dyndep (token&, build2::script::token_type&, @@ -175,7 +216,8 @@ namespace build2 depdb&, bool& update, bool& deferred_failure, - timestamp); + timestamp, + dyndep_byproduct*); // Helpers. // @@ -274,13 +316,21 @@ namespace build2 // depdb env - Track the environment variables change as a // hash. // - // depdb dyndep ... - Extract dynamic dependency information. - // Can only be the last depdb builtin call - // in the preamble. + // depdb dyndep ... - Extract dynamic dependency information. Can + // only be the last depdb builtin call in the + // preamble. Note that such dependencies don't + // end up in $<. We also don't cause clean of + // such dependencies (since there may be no .d + // file) -- they should also be listed as + // static prerequisites of some other target + // (e.g., lib{} for headers) or a custom clean + // recipe should be provided. + // // optional depdb_clear_; // depdb-clear location. optional> depdb_dyndep_; // depdb-dyndep location/position. + bool depdb_dyndep_byproduct_ = false; // --byproduct lines depdb_preamble_; // Note: excluding depdb-clear. // If present, the first impure function called in the body of the diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx index 9d7567c..4e88785 100644 --- a/libbuild2/build/script/script.hxx +++ b/libbuild2/build/script/script.hxx @@ -75,7 +75,8 @@ namespace build2 // script parser for details). // bool depdb_clear; - optional depdb_dyndep; // Position of first depdb-dyndep. + optional depdb_dyndep; // Pos of first dyndep. + bool depdb_dyndep_byproduct = false; // dyndep --byproduct lines_type depdb_preamble; bool depdb_preamble_temp_dir = false; // True if refs $~. diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx index db9e6cb..44d1fd5 100644 --- a/libbuild2/cc/compile-rule.cxx +++ b/libbuild2/cc/compile-rule.cxx @@ -2046,9 +2046,10 @@ namespace build2 try { pair er ( - enter_header (a, bs, t, li, - move (f), false /* cache */, false /* norm */, - pfx_map, so_map)); + enter_header ( + a, bs, t, li, + f, false /* cache */, false /* normalized */, + pfx_map, so_map)); ht = er.first; remapped = er.second; @@ -2574,9 +2575,10 @@ namespace build2 if (exists) { pair r ( - enter_header (a, bs, t, li, - move (f), false /* cache */, false /* norm */, - pfx_map, so_map)); + enter_header ( + a, bs, t, li, + f, false /* cache */, false /* normalized */, + pfx_map, so_map)); if (!r.second) // Shouldn't be remapped. ht = r.first; @@ -2601,9 +2603,10 @@ namespace build2 try { pair er ( - enter_header (a, bs, t, li, - move (f), false /* cache */, false /* norm */, - pfx_map, so_map)); + enter_header ( + a, bs, t, li, + f, false /* cache */, false /* normalized */, + pfx_map, so_map)); ht = er.first; remapped = er.second; @@ -2796,7 +2799,7 @@ namespace build2 // pair compile_rule:: enter_header (action a, const scope& bs, file& t, linfo li, - path&& f, bool cache, bool norm, + path& fp, bool cache, bool norm, optional& pfx_map, const srcout_map& so_map) const { @@ -2814,7 +2817,7 @@ namespace build2 return enter_file ( trace, "header", a, bs, t, - move (f), cache, norm, + fp, cache, norm, [this] (const scope& bs, const string& n, const string& e) { return map_extension (bs, n, e, x_inc); @@ -3566,9 +3569,10 @@ namespace build2 dr << endf; }; - if (const file* ht = enter_header (a, bs, t, li, - move (hp), cache, false /* norm */, - pfx_map, so_map).first) + if (const file* ht = enter_header ( + a, bs, t, li, + hp, cache, cache /* normalized */, + pfx_map, so_map).first) { // If we are reading the cache, then it is possible the file has // since been removed (think of a header in /usr/local/include that @@ -3576,12 +3580,14 @@ namespace build2 // /usr/include). This will lead to the match failure which we // translate to a restart. // + // @@ Won't this fail in enter_header() rather? + // if (optional u = inject_header (a, t, *ht, mt, false /*fail*/)) { // Verify/add it to the dependency database. // if (!cache) - dd.expect (ht->path ()); + dd.expect (ht->path ()); // @@ Use hp (or verify match)? skip_count++; return *u; @@ -3612,7 +3618,7 @@ namespace build2 const file* ht ( enter_header (a, bs, t, li, - move (hp), true /* cache */, true /* norm */, + hp, true /* cache */, false /* normalized */, pfx_map, so_map).first); if (ht == nullptr) diff --git a/libbuild2/cc/compile-rule.hxx b/libbuild2/cc/compile-rule.hxx index 568c04b..dbb2dd5 100644 --- a/libbuild2/cc/compile-rule.hxx +++ b/libbuild2/cc/compile-rule.hxx @@ -120,7 +120,7 @@ namespace build2 pair enter_header (action, const scope&, file&, linfo, - path&&, bool, bool, + path&, bool, bool, optional&, const srcout_map&) const; optional diff --git a/libbuild2/dyndep.cxx b/libbuild2/dyndep.cxx index 51fa7bc..3740e21 100644 --- a/libbuild2/dyndep.cxx +++ b/libbuild2/dyndep.cxx @@ -80,7 +80,8 @@ namespace build2 action a, target& t, const file& pt, timestamp mt, - bool f) + bool f, + bool ah) { // Even if failing we still use try_match() in order to issue consistent // (with other places) diagnostics (rather than the generic "not rule to @@ -103,11 +104,72 @@ namespace build2 // Add to our prerequisite target list. // + t.prerequisite_targets[a].push_back (prerequisite_target (&pt, ah)); + + return r; + } + + optional dyndep_rule:: + inject_existing_file (tracer& trace, const char* what, + action a, target& t, + const file& pt, + timestamp mt, + bool f) + { + if (!try_match (a, pt).first) + { + if (!f) + return nullopt; + + diag_record dr; + dr << fail << what << ' ' << pt << " not found and no rule to " + << "generate it"; + + if (verb < 4) + dr << info << "re-run with --verbose=4 for more information"; + } + + recipe_function* const* rf (pt[a].recipe.target ()); + if (rf == nullptr || *rf != &noop_action) + { + fail << what << ' ' << pt << " has non-noop recipe" << + info << "consider listing it as static prerequisite of " << t; + } + + bool r (update (trace, a, pt, mt)); + + // Add to our prerequisite target list. + // t.prerequisite_targets[a].push_back (&pt); return r; } + void dyndep_rule:: + verify_existing_file (tracer&, const char* what, + action a, const target& t, + const file& pt) + { + diag_record dr; + + if (pt.matched (a)) + { + recipe_function* const* rf (pt[a].recipe.target ()); + if (rf == nullptr || *rf != &noop_action) + { + dr << fail << what << ' ' << pt << " has non-noop recipe"; + } + } + else if (pt.decl == target_decl::real) + { + dr << fail << what << ' ' << pt << " is explicitly declared as " + << "target and may have non-noop recipe"; + } + + if (!dr.empty ()) + dr << info << "consider listing it as static prerequisite of " << t; + } + // Reverse-lookup target type(s) from file name/extension. // // If the list of base target types is specified, then only these types and @@ -367,14 +429,16 @@ namespace build2 return false; } - pair dyndep_rule:: - enter_file (tracer& trace, const char* what, - action a, const scope& bs, target& t, - path&& f, bool cache, bool norm, - const function& map_extension, - const target_type& fallback, - const function& get_pfx_map, - const srcout_map& so_map) + static pair + enter_file_impl ( + tracer& trace, const char* what, + action a, const scope& bs, const target& t, + path& fp, bool cache, bool norm, + bool insert, + const function& map_extension, + const target_type& fallback, + const function& get_pfx_map, + const dyndep_rule::srcout_map& so_map) { // Find or maybe insert the target. The directory is only moved from if // insert is true. Note that it must be normalized. @@ -545,85 +609,87 @@ namespace build2 // If still relative then it does not exist. // - if (f.relative ()) + if (fp.relative ()) { // This is probably as often an error as an auto-generated file, so // trace at level 4. // - l4 ([&]{trace << "non-existent " << what << " '" << f << "'";}); + l4 ([&]{trace << "non-existent " << what << " '" << fp << "'";}); - f.normalize (); - - // The relative path might still contain '..' (e.g., ../foo.hxx; - // presumably ""-include'ed). We don't attempt to support auto- - // generated files with such inclusion styles. - // - if (get_pfx_map != nullptr && f.normalized ()) + if (get_pfx_map != nullptr) { - const prefix_map& pfx_map (get_pfx_map (a, bs, t)); + fp.normalize (); - // First try the whole file. Then just the directory. + // The relative path might still contain '..' (e.g., ../foo.hxx; + // presumably ""-include'ed). We don't attempt to support auto- + // generated files with such inclusion styles. // - // @@ Has to be a separate map since the prefix can be the same as - // the file name. - // - // auto i (pfx_map->find (f)); - - // Find the most qualified prefix of which we are a sub-path. - // - if (!pfx_map.empty ()) + if (fp.normalized ()) { - dir_path d (f.directory ()); - auto p (pfx_map.sup_range (d)); + const dyndep_rule::prefix_map& pfx_map (get_pfx_map (a, bs, t)); - if (p.first != p.second) - { - // Note that we can only have multiple entries for the - // prefixless mapping. - // - dir_path pd; // Reuse. - for (auto i (p.first); i != p.second; ++i) - { - // Note: value in pfx_map is not necessarily canonical. - // - pd = i->second.directory; - pd.canonicalize (); + // First try the whole file. Then just the directory. + // + // @@ Has to be a separate map since the prefix can be the same as + // the file name. + // + // auto i (pfx_map->find (f)); - l4 ([&]{trace << "try prefix '" << d << "' mapped to " << pd;}); + // Find the most qualified prefix of which we are a sub-path. + // + if (!pfx_map.empty ()) + { + dir_path d (fp.directory ()); + auto p (pfx_map.sup_range (d)); - // If this is a prefixless mapping, then only use it if we can - // resolve it to an existing target (i.e., it is explicitly - // spelled out in a buildfile). @@ Hm, I wonder why, it's not - // like we can generate any file without an explicit target. - // Maybe for diagnostics (i.e., we will actually try to build - // something there instead of just saying no mapping). + if (p.first != p.second) + { + // Note that we can only have multiple entries for the + // prefixless mapping. // - pt = find (pd / d, f.leaf (), !i->first.empty ()); - if (pt != nullptr) + dir_path pd; // Reuse. + for (auto i (p.first); i != p.second; ++i) { - f = pd / f; - l4 ([&]{trace << "mapped as auto-generated " << f;}); - break; + // Note: value in pfx_map is not necessarily canonical. + // + pd = i->second.directory; + pd.canonicalize (); + + l4 ([&]{trace << "try prefix '" << d << "' mapped to " << pd;}); + + // If this is a prefixless mapping, then only use it if we can + // resolve it to an existing target (i.e., it is explicitly + // spelled out in a buildfile). @@ Hm, I wonder why, it's not + // like we can generate any file without an explicit target. + // Maybe for diagnostics (i.e., we will actually try to build + // something there instead of just saying no mapping). + // + pt = find (pd / d, fp.leaf (), insert && !i->first.empty ()); + if (pt != nullptr) + { + fp = pd / fp; + l4 ([&]{trace << "mapped as auto-generated " << fp;}); + break; + } + else + l4 ([&]{trace << "no explicit target in " << pd;}); } - else - l4 ([&]{trace << "no explicit target in " << pd;}); } + else + l4 ([&]{trace << "no prefix map entry for '" << d << "'";}); } else - l4 ([&]{trace << "no prefix map entry for '" << d << "'";}); + l4 ([&]{trace << "prefix map is empty";}); } - else - l4 ([&]{trace << "prefix map is empty";}); } } else { - // Normalize the path unless it comes from the depdb, in which case - // we've already done that (normally). This is also where we handle - // src-out remap (again, not needed if cached). + // Normalize the path unless it is already normalized. This is also + // where we handle src-out remap which is not needed if cached. // - if (!cache || norm) - normalize_external (f, what); + if (!norm) + normalize_external (fp, what); if (!cache) { @@ -631,7 +697,7 @@ namespace build2 { // Find the most qualified prefix of which we are a sub-path. // - auto i (so_map.find_sup (f)); + auto i (so_map.find_sup (fp)); if (i != so_map.end ()) { // Ok, there is an out tree for this file. Remap to a path from @@ -639,16 +705,16 @@ namespace build2 // value in so_map is not necessarily canonical. // dir_path d (i->second); - d /= f.leaf (i->first).directory (); + d /= fp.leaf (i->first).directory (); d.canonicalize (); - pt = find (move (d), f.leaf (), false); // d is not moved from. + pt = find (move (d), fp.leaf (), false); // d is not moved from. if (pt != nullptr) { - path p (d / f.leaf ()); - l4 ([&]{trace << "remapping " << f << " to " << p;}); - f = move (p); + path p (d / fp.leaf ()); + l4 ([&]{trace << "remapping " << fp << " to " << p;}); + fp = move (p); remapped = true; } } @@ -657,11 +723,43 @@ namespace build2 if (pt == nullptr) { - l6 ([&]{trace << "entering " << f;}); - pt = find (f.directory (), f.leaf (), true); + l6 ([&]{trace << (insert ? "entering " : "finding ") << fp;}); + pt = find (fp.directory (), fp.leaf (), insert); } } return make_pair (pt, remapped); } + + pair dyndep_rule:: + enter_file (tracer& trace, const char* what, + action a, const scope& bs, target& t, + path& fp, bool cache, bool norm, + const function& map_ext, + const target_type& fallback, + const function& pfx_map, + const srcout_map& so_map) + { + return enter_file_impl (trace, what, + a, bs, t, + fp, cache, norm, + true /* insert */, + map_ext, fallback, pfx_map, so_map); + } + + pair dyndep_rule:: + find_file (tracer& trace, const char* what, + action a, const scope& bs, const target& t, + path& fp, bool cache, bool norm, + const function& map_ext, + const target_type& fallback, + const function& pfx_map, + const srcout_map& so_map) + { + return enter_file_impl (trace, what, + a, bs, t, + fp, cache, norm, + false /* insert */, + map_ext, fallback, pfx_map, so_map); + } } diff --git a/libbuild2/dyndep.hxx b/libbuild2/dyndep.hxx index 3ba0c09..cfd3c7e 100644 --- a/libbuild2/dyndep.hxx +++ b/libbuild2/dyndep.hxx @@ -37,12 +37,45 @@ namespace build2 // then issue diagnostics and fail if the fail argument is true and return // nullopt otherwise. // + // If adhoc is true, then add it as ad hoc to prerequisite targets. At + // first it may seem like such dynamic prerequisites should always be ad + // hoc. But on the other hand, taking headers as an example, if the same + // header is listed as a static prerequisite, it will most definitely not + // going to be ad hoc. So we leave it to the caller to make this decision. + // static optional inject_file (tracer&, const char* what, action, target&, const file& prerequiste, timestamp, - bool fail); + bool fail, + bool adhoc = false); + + // As above but verify the file is matched with noop_recipe and issue + // diagnostics and fail otherwise (regardless of the fail flag). + // + // This version (together with verify_existing_file() below) is primarily + // useful for handling dynamic dependencies that are produced as a + // byproduct of recipe execution (and thus must have all the generated + // prerequisites specified statically). + // + static optional + inject_existing_file (tracer&, const char* what, + action, target&, + const file& prerequiste, + timestamp, + bool fail); + + // Verify the file is matched with noop_recipe and issue diagnostics and + // fail otherwise. If the file is not matched, then fail if the target is + // not implied (that is, declared in a buildfile). + // + // Note: can only be called in the execute phase. + // + static void + verify_existing_file (tracer&, const char* what, + action, const target&, + const file& prerequiste); // Reverse-lookup target type(s) from file name/extension. // @@ -124,29 +157,31 @@ namespace build2 dir_path diff_; }; - // Enter a prerequisite file as a target. If the path is relative, then - // assume this a non-existent generated file. + // Find or insert a prerequisite file path as a target. If the path is + // relative, then assume this is a non-existent generated file. // // Depending on the cache flag, the path is assumed to either have come - // from the depdb cache or from the compiler run. In the former case - // assume the path is already normalized unless the normalize flag is - // true. + // from the depdb cache or from the compiler run. If normalized is true, + // then assume the absolute path is already normalized. // // Return the file target and an indication of whether it was remapped or - // NULL if the file does not exist and cannot be generated. In the latter - // case the passed file path is guaranteed to still be valid but might - // have been adjusted (e.g., normalized, etc). + // NULL if the file does not exist and cannot be generated. The passed by + // reference file path is guaranteed to still be valid but might have been + // adjusted (e.g., completed, normalized, remapped, etc). If the result is + // not NULL, then it is the absolute and normalized path to the actual + // file. If the result is NULL, then it can be used in diagnostics to + // identify the origial file path. // // The map_extension function is used to reverse-map a file extension to // the target type. The fallback target type is used if it's NULL or // didn't return anything but only in situations where we are sure the - // file is (or should be there; see the implementation for details). + // file is or should be there (see the implementation for details). // // The prefix map function is only called if this is a non-existent // generated file (so it can be initialized lazily). If it's NULL, then // generated files will not be supported. The srcout map is only consulted - // if cache is false (so its initialization can be delayed until the call - // with cache=false). + // if cache is false to re-map generated files (so its initialization can + // be delayed until the call with cache=false). // using map_extension_func = small_vector ( const scope& base, const string& name, const string& ext); @@ -157,11 +192,22 @@ namespace build2 static pair enter_file (tracer&, const char* what, action, const scope& base, target&, - path&& prerequisite, bool cache, bool norm, + path& prerequisite, bool cache, bool normalized, const function&, const target_type& fallback, - const function&, - const srcout_map&); + const function& = nullptr, + const srcout_map& = {}); + + // As above but do not insert the target if it doesn't already exist. + // + static pair + find_file (tracer&, const char* what, + action, const scope& base, const target&, + path& prerequisite, bool cache, bool normalized, + const function&, + const target_type& fallback, + const function& = nullptr, + const srcout_map& = {}); }; } -- cgit v1.1