aboutsummaryrefslogtreecommitdiff
path: root/libbuild2
diff options
context:
space:
mode:
Diffstat (limited to 'libbuild2')
-rw-r--r--libbuild2/adhoc-rule-buildscript.cxx227
-rw-r--r--libbuild2/build/script/builtin-options.cxx17
-rw-r--r--libbuild2/build/script/builtin-options.hxx51
-rw-r--r--libbuild2/build/script/builtin-options.ixx90
-rw-r--r--libbuild2/build/script/builtin.cli41
-rw-r--r--libbuild2/build/script/parser.cxx421
-rw-r--r--libbuild2/build/script/parser.hxx12
-rw-r--r--libbuild2/build/script/script.hxx3
-rw-r--r--libbuild2/dyndep.cxx101
-rw-r--r--libbuild2/dyndep.hxx21
-rw-r--r--libbuild2/script/script.cxx4
-rw-r--r--libbuild2/target.hxx4
12 files changed, 864 insertions, 128 deletions
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx
index ce4ab8b..f3e3560 100644
--- a/libbuild2/adhoc-rule-buildscript.cxx
+++ b/libbuild2/adhoc-rule-buildscript.cxx
@@ -5,6 +5,8 @@
#include <sstream>
+#include <libbutl/filesystem.hxx> // try_rm_file()
+
#include <libbuild2/depdb.hxx>
#include <libbuild2/scope.hxx>
#include <libbuild2/target.hxx>
@@ -225,6 +227,8 @@ namespace build2
build::script::default_runner run;
path dd;
+ paths dyn_targets;
+ paths old_dyn_targets;
const scope* bs;
timestamp mt;
@@ -457,10 +461,120 @@ namespace build2
}
}
+ // Read the list of dynamic targets from depdb, if exists (used in a few
+ // depdb-dyndep --dyn-target handling places below).
+ //
+ auto read_dyn_targets = [] (path ddp) -> paths
+ {
+ depdb dd (move (ddp), true /* read_only */);
+
+ paths r;
+ while (dd.reading ()) // Breakout loop.
+ {
+ string* l;
+ auto read = [&dd, &l] () -> bool
+ {
+ return (l = dd.read ()) != nullptr;
+ };
+
+ if (!read ()) // Rule id.
+ break;
+
+ // We can omit this for as long as we don't break our blank line
+ // anchors semantics (e.g., by adding another blank somewhere, say
+ // after the custom depdb builtins).
+ //
+#if 0
+ if (*l != rule_id_)
+ fail << "unable to clean dynamic target group " << t
+ << " with old depdb";
+#endif
+
+ // Note that we cannot read out expected lines since there can be
+ // custom depdb builtins. We also need to skip the prerequisites
+ // list. So we read until the blank line that always terminates the
+ // prerequisites list.
+ //
+ {
+ bool r;
+ while ((r = read ()) && !l->empty ()) ;
+ if (!r)
+ break;
+ }
+
+ // Read the dynamic target files. We should always end with a blank
+ // line.
+ //
+ for (;;)
+ {
+ if (!read () || l->empty ())
+ break;
+
+ r.push_back (path (*l));
+ }
+
+ break;
+ }
+
+ return r;
+ };
+
// See if we are providing the standard clean as a fallback.
//
if (me.fallback)
- return &perform_clean_file;
+ {
+ // For depdb-dyndep --dyn-target use depdb to clean dynamic targets.
+ //
+ if (script.depdb_dyndep && script.depdb_dyndep_dyn_target)
+ {
+ file& t (xt.as<file> ());
+
+ // Note that only removing the relevant filesystem entries is not
+ // enough: we actually have to populate the group with members since
+ // this information could be used to clean derived targets (for
+ // example, object files). So we just do that and let the standard
+ // clean logic take care of them the same as static members.
+ //
+ using dyndep = dyndep_rule;
+
+ function<dyndep::map_extension_func> map_ext (
+ [] (const scope& bs, const string& n, const string& e)
+ {
+ return dyndep::map_extension (bs, n, e, nullptr);
+ });
+
+ for (path& f: read_dyn_targets (t.path () + ".d"))
+ {
+ // Note that this logic should be consistent with what we have in
+ // exec_depdb_dyndep().
+ //
+ // Note that here we don't bother cleaning any old dynamic targets
+ // -- the more we can clean, the merrier.
+ //
+ // @@ We don't have --target-what, --target-default-type here.
+ // Could we do the same thing as byproduct to get them? That
+ // would require us running the first half of the depdb preamble
+ // but ignoring all the depdb builtins (we still want all the
+ // variable assignments -- maybe we could automatically skip them
+ // if we see depdb is not open). Wonder if there would be any
+ // other complications...
+ //
+ // BTW, this sort of works for --target-default-type since we
+ // just clean them as file{} targets (but diagnostics is off). It
+ // does break, however, if there is s batch, since then we end up
+ // detecting different targets sharing a path. This will also not
+ // work at all if/when we support specifying custom extension to
+ // type mapping in order to resolve ambiguities.
+ //
+ dyndep::inject_adhoc_group_member ("file",
+ a, bs, t,
+ move (f),
+ map_ext, file::static_type);
+ }
+ }
+
+ return perform_clean_file;
+ }
// If we have any update during match prerequisites, now is the time to
// update them.
@@ -575,10 +689,49 @@ namespace build2
}
}
+ // Note that while it's tempting to turn match_data* into recipes, some of
+ // their members are not movable. And in the end we will have the same
+ // result: one dynamic memory allocation.
+ //
+ unique_ptr<match_data> md;
+ unique_ptr<match_data_byproduct> mdb;
+
+ if (script.depdb_dyndep_byproduct)
+ {
+ mdb.reset (new match_data_byproduct (
+ a, t, bs, script.depdb_preamble_temp_dir));
+ }
+ else
+ {
+ md.reset (new match_data (a, t, bs, script.depdb_preamble_temp_dir));
+
+ // If the set of dynamic targets can change based on changes to the
+ // inputs (say, each entity, such as a type, in the input file gets its
+ // own output file), then we can end up with a large number of old
+ // output files laying around because they are not part of the new
+ // dynamic target set. So we try to clean them up based on the old depdb
+ // information, similar to how we do it for perform_clean above (except
+ // here we will just keep the list of old files).
+ //
+ // Note: do before opening depdb, which can start over-writing it.
+ //
+ // We also have to do this speculatively, without knowing whether we
+ // will need to update. Oh, well, being dynamic ain't free.
+ //
+ if (script.depdb_dyndep_dyn_target)
+ md->old_dyn_targets = read_dyn_targets (tp + ".d");
+ }
+
depdb dd (tp + ".d");
// NOTE: see the "static dependencies" version (with comments) below.
//
+ // NOTE: We use blank lines as anchors to skip directly to certain entries
+ // (e.g., dynamic targets). So make sure none of the other entries
+ // can be blank (for example, see `depdb string` builtin).
+ //
+ // NOTE: KEEP IN SYNC WITH read_dyn_targets ABOVE!
+ //
if (dd.expect ("<ad hoc buildscript recipe> 1") != nullptr)
l4 ([&]{trace << "rule mismatch forcing update of " << t;});
@@ -613,10 +766,21 @@ namespace build2
l4 ([&]{trace << "recipe variable change forcing update of " << t;});
}
+ // Static targets and prerequisites (there can also be dynamic targets;
+ // see dyndep --dyn-target).
+ //
+ // There is a nuance: in an operation batch (e.g., `b update update`) we
+ // will already have the dynamic targets as members on the subsequent
+ // operations and we need to make sure we don't treat them as static.
+ // Using target_decl to distinguish the two seems like a natural way.
+ //
{
sha256 tcs;
for (const target* m (&t); m != nullptr; m = m->adhoc_member)
- hash_target (tcs, *m, storage);
+ {
+ if (m->decl == target_decl::real)
+ hash_target (tcs, *m, storage);
+ }
if (dd.expect (tcs.string ()) != nullptr)
l4 ([&]{trace << "target set change forcing update of " << t;});
@@ -634,22 +798,8 @@ namespace build2
}
}
- // Note that while it's tempting to turn match_data* into recipes, some of
- // their members are not movable. And in the end we will have the same
- // result: one dynamic memory allocation.
+ // Get ready to run the depdb preamble.
//
- unique_ptr<match_data> md;
- unique_ptr<match_data_byproduct> mdb;
-
- if (script.depdb_dyndep_byproduct)
- {
- mdb.reset (new match_data_byproduct (
- a, t, bs, script.depdb_preamble_temp_dir));
- }
- else
- md.reset (new match_data (a, t, bs, script.depdb_preamble_temp_dir));
-
-
build::script::environment& env (mdb != nullptr ? mdb->env : md->env);
build::script::default_runner& run (mdb != nullptr ? mdb->run : md->run);
@@ -828,20 +978,22 @@ namespace build2
else
{
// Run the second half of the preamble (depdb-dyndep commands) to update
- // our prerequisite targets and extract dynamic dependencies.
+ // our prerequisite targets and extract dynamic dependencies (targets and
+ // prerequisites).
//
// Note that this should be the last update to depdb (the invalidation
// order semantics).
//
- bool deferred_failure (false);
+ md->deferred_failure = false;
{
build::script::parser p (ctx);
p.execute_depdb_preamble_dyndep (a, bs, t,
env, script, run,
dd,
+ md->dyn_targets,
update,
mt,
- deferred_failure);
+ md->deferred_failure);
}
if (update && dd.reading () && !ctx.dry_run)
@@ -854,7 +1006,6 @@ namespace build2
//
md->bs = &bs;
md->mt = update ? timestamp_nonexistent : mt;
- md->deferred_failure = deferred_failure;
return [this, md = move (md)] (action a, const target& t)
{
@@ -1066,7 +1217,7 @@ namespace build2
if (r.second.empty ())
continue;
- // @@ TODO: what should we do about targets?
+ // Note: no support for dynamic targets in byproduct mode.
//
if (r.first == make_type::target)
continue;
@@ -1076,10 +1227,10 @@ namespace build2
if (f.relative ())
{
if (!byp.cwd)
- fail (il) << "relative path '" << f << "' in make dependency"
- << " declaration" <<
+ fail (il) << "relative prerequisite path '" << f
+ << "' in make dependency declaration" <<
info << "consider using --cwd to specify relative path "
- << "base";
+ << "base";
f = *byp.cwd / f;
}
@@ -1143,6 +1294,30 @@ namespace build2
return *ps;
}
+ // Remove previous dynamic targets since their set may change with changes
+ // to the inputs (see apply() for details).
+ //
+ // The dry-run mode complicates things: if we don't remove the old files,
+ // then that information will be gone (since we update depdb even in the
+ // dry-run mode). But if we remove everything in the dry-run mode, then we
+ // may also remove some of the current files, which would be incorrect. So
+ // let's always remove but only files that are not in the current set.
+ //
+ for (const path& f: md.old_dyn_targets)
+ {
+ if (find (md.dyn_targets.begin (), md.dyn_targets.end (), f) ==
+ md.dyn_targets.end ())
+ {
+ // This is an optimization so best effort.
+ //
+ if (optional<rmfile_status> s = butl::try_rmfile_ignore_error (f))
+ {
+ if (s == rmfile_status::success && verb >= 2)
+ text << "rm " << f;
+ }
+ }
+ }
+
// Sequence start time for mtime checks below.
//
timestamp start (!ctx.dry_run && depdb::mtime_check ()
@@ -1627,6 +1802,8 @@ namespace build2
// Finally, we print the entire ad hoc group at verbosity level 1, similar
// to the default update diagnostics.
//
+ // @@ TODO: .t may also be a temporary directory.
+ //
return perform_clean_extra (a,
t.as<file> (),
{".d", ".t"},
diff --git a/libbuild2/build/script/builtin-options.cxx b/libbuild2/build/script/builtin-options.cxx
index 9e4213d..3b64de1 100644
--- a/libbuild2/build/script/builtin-options.cxx
+++ b/libbuild2/build/script/builtin-options.cxx
@@ -289,7 +289,13 @@ namespace build2
adhoc_ (),
cwd_ (),
cwd_specified_ (false),
- drop_cycles_ ()
+ drop_cycles_ (),
+ target_what_ (),
+ target_what_specified_ (false),
+ target_default_type_ (),
+ target_default_type_specified_ (false),
+ target_cwd_ (),
+ target_cwd_specified_ (false)
{
}
@@ -391,6 +397,15 @@ namespace build2
&depdb_dyndep_options::cwd_specified_ >;
_cli_depdb_dyndep_options_map_["--drop-cycles"] =
&::build2::build::cli::thunk< depdb_dyndep_options, &depdb_dyndep_options::drop_cycles_ >;
+ _cli_depdb_dyndep_options_map_["--target-what"] =
+ &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_what_,
+ &depdb_dyndep_options::target_what_specified_ >;
+ _cli_depdb_dyndep_options_map_["--target-default-type"] =
+ &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_default_type_,
+ &depdb_dyndep_options::target_default_type_specified_ >;
+ _cli_depdb_dyndep_options_map_["--target-cwd"] =
+ &::build2::build::cli::thunk< depdb_dyndep_options, dir_path, &depdb_dyndep_options::target_cwd_,
+ &depdb_dyndep_options::target_cwd_specified_ >;
}
};
diff --git a/libbuild2/build/script/builtin-options.hxx b/libbuild2/build/script/builtin-options.hxx
index 590d3b2..2a00072 100644
--- a/libbuild2/build/script/builtin-options.hxx
+++ b/libbuild2/build/script/builtin-options.hxx
@@ -174,6 +174,51 @@ namespace build2
void
drop_cycles (const bool&);
+ const string&
+ target_what () const;
+
+ string&
+ target_what ();
+
+ void
+ target_what (const string&);
+
+ bool
+ target_what_specified () const;
+
+ void
+ target_what_specified (bool);
+
+ const string&
+ target_default_type () const;
+
+ string&
+ target_default_type ();
+
+ void
+ target_default_type (const string&);
+
+ bool
+ target_default_type_specified () const;
+
+ void
+ target_default_type_specified (bool);
+
+ const dir_path&
+ target_cwd () const;
+
+ dir_path&
+ target_cwd ();
+
+ void
+ target_cwd (const dir_path&);
+
+ bool
+ target_cwd_specified () const;
+
+ void
+ target_cwd_specified (bool);
+
// Implementation details.
//
protected:
@@ -201,6 +246,12 @@ namespace build2
dir_path cwd_;
bool cwd_specified_;
bool drop_cycles_;
+ string target_what_;
+ bool target_what_specified_;
+ string target_default_type_;
+ bool target_default_type_specified_;
+ dir_path target_cwd_;
+ bool target_cwd_specified_;
};
}
}
diff --git a/libbuild2/build/script/builtin-options.ixx b/libbuild2/build/script/builtin-options.ixx
index ea06a0f..3e3787f 100644
--- a/libbuild2/build/script/builtin-options.ixx
+++ b/libbuild2/build/script/builtin-options.ixx
@@ -233,6 +233,96 @@ namespace build2
{
this->drop_cycles_ = x;
}
+
+ inline const string& depdb_dyndep_options::
+ target_what () const
+ {
+ return this->target_what_;
+ }
+
+ inline string& depdb_dyndep_options::
+ target_what ()
+ {
+ return this->target_what_;
+ }
+
+ inline void depdb_dyndep_options::
+ target_what (const string& x)
+ {
+ this->target_what_ = x;
+ }
+
+ inline bool depdb_dyndep_options::
+ target_what_specified () const
+ {
+ return this->target_what_specified_;
+ }
+
+ inline void depdb_dyndep_options::
+ target_what_specified (bool x)
+ {
+ this->target_what_specified_ = x;
+ }
+
+ inline const string& depdb_dyndep_options::
+ target_default_type () const
+ {
+ return this->target_default_type_;
+ }
+
+ inline string& depdb_dyndep_options::
+ target_default_type ()
+ {
+ return this->target_default_type_;
+ }
+
+ inline void depdb_dyndep_options::
+ target_default_type (const string& x)
+ {
+ this->target_default_type_ = x;
+ }
+
+ inline bool depdb_dyndep_options::
+ target_default_type_specified () const
+ {
+ return this->target_default_type_specified_;
+ }
+
+ inline void depdb_dyndep_options::
+ target_default_type_specified (bool x)
+ {
+ this->target_default_type_specified_ = x;
+ }
+
+ inline const dir_path& depdb_dyndep_options::
+ target_cwd () const
+ {
+ return this->target_cwd_;
+ }
+
+ inline dir_path& depdb_dyndep_options::
+ target_cwd ()
+ {
+ return this->target_cwd_;
+ }
+
+ inline void depdb_dyndep_options::
+ target_cwd (const dir_path& x)
+ {
+ this->target_cwd_ = x;
+ }
+
+ inline bool depdb_dyndep_options::
+ target_cwd_specified () const
+ {
+ return this->target_cwd_specified_;
+ }
+
+ inline void depdb_dyndep_options::
+ target_cwd_specified (bool x)
+ {
+ this->target_cwd_specified_ = x;
+ }
}
}
}
diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli
index 7d0936f..2fba0b0 100644
--- a/libbuild2/build/script/builtin.cli
+++ b/libbuild2/build/script/builtin.cli
@@ -17,8 +17,8 @@ namespace build2
//
class depdb_dyndep_options
{
- // Note that --byproduct, if any, must be the first option and is
- // handled ad hoc, kind of as a sub-command.
+ // Note that --byproduct or --dyn-target, if any, must be the first
+ // option and is handled ad hoc.
//
// Similarly, --update-{include,exclude} are handled ad hoc and must
// be literals, similar to the -- separator. They specify prerequisite
@@ -44,16 +44,19 @@ namespace build2
// and the other for prerequisite, we omit "prerequisite" as that's
// what we extract by default and most commonly. For example:
//
- // --what --what-target
- // --default-type --default-target-type
+ // --what --target-what
+ // --default-type --target-default-type
//
path --file; // Read from file rather than stdin.
string --format; // Dependency format: make (default).
- string --what; // Dependency kind, e.g., "header".
+ // Dynamic dependency extraction options.
+ //
+ string --what; // Prerequisite kind, e.g., "header".
- dir_paths --include-path|-I; // Search paths for generated files.
+ dir_paths --include-path|-I; // Search paths for generated
+ // prerequisites.
string --default-type; // Default prerequisite type to use
// if none could be derived from ext.
@@ -64,14 +67,36 @@ namespace build2
// normal mode).
dir_path --cwd; // Builtin's working directory used
- // to complete relative paths (only
- // in --byproduct mode).
+ // to complete relative paths of
+ // prerequisites (only in --byproduct
+ // mode).
bool --drop-cycles; // Drop prerequisites that are also
// targets. Only use if you are sure
// such cycles are harmless, that is,
// the output is not affected by such
// prerequisites' content.
+
+ // Dynamic target extraction options.
+ //
+ // This functionality is enabled with the --dyn-target option. Only
+ // the make format is supported, where the listed targets are added as
+ // ad hoc group members (unless already specified as static members).
+ // This functionality is not available in the byproduct mode.
+ //
+ // @@ BTW, here what would likely be more useful than default target
+ // is the ability to specify custom extension-to-type mapping in
+ // order to resolve ambiguities. See also the issue with getting
+ // these options during clean.
+ //
+ string --target-what; // Target kind, e.g., "source".
+
+ string --target-default-type; // Default target type to use if none
+ // could be derived from ext.
+
+ dir_path --target-cwd; // Builtin's working directory used to
+ // complete relative paths of targets.
+
};
}
}
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index df0a419..e7268f9 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -158,6 +158,7 @@ namespace build2
{
s.depdb_dyndep = depdb_dyndep_->second;
s.depdb_dyndep_byproduct = depdb_dyndep_byproduct_;
+ s.depdb_dyndep_dyn_target = depdb_dyndep_dyn_target_;
}
s.depdb_preamble = move (depdb_preamble_);
@@ -830,8 +831,18 @@ namespace build2
fail (l) << "multiple 'depdb dyndep' calls" <<
info (depdb_dyndep_->first) << "previous call is here";
- if (peek () == type::word && peeked ().value == "--byproduct")
- depdb_dyndep_byproduct_ = true;
+ if (peek () == type::word)
+ {
+ const string& v (peeked ().value);
+
+ // Note: --byproduct and --dyn-target are mutually
+ // exclusive.
+ //
+ if (v == "--byproduct")
+ depdb_dyndep_byproduct_ = true;
+ else if (v == "--dyn-target")
+ depdb_dyndep_dyn_target_ = true;
+ }
}
else
{
@@ -1280,6 +1291,7 @@ namespace build2
environment& e, const script& s, runner& r,
lines_iterator begin, lines_iterator end,
depdb& dd,
+ paths* dyn_targets,
bool* update,
optional<timestamp> mt,
bool* deferred_failure,
@@ -1308,12 +1320,17 @@ namespace build2
const script& scr;
depdb& dd;
+ paths* dyn_targets;
bool* update;
bool* deferred_failure;
optional<timestamp> mt;
dyndep_byproduct* byp;
- } data {trace, a, bs, t, e, s, dd, update, deferred_failure, mt, byp};
+ } data {
+ trace,
+ a, bs, t,
+ e, s,
+ dd, dyn_targets, update, deferred_failure, mt, byp};
auto exec_cmd = [this, &data] (token& t,
build2::script::token_type& tt,
@@ -1345,6 +1362,7 @@ namespace build2
li, ll,
data.a, data.bs, const_cast<file&> (data.t),
data.dd,
+ *data.dyn_targets,
*data.update,
*data.mt,
*data.deferred_failure,
@@ -1354,35 +1372,29 @@ namespace build2
{
names ns (exec_special (t, tt, true /* skip <cmd> */));
+ string v;
+ const char* w (nullptr);
if (cmd == "hash")
{
sha256 cs;
for (const name& n: ns)
to_checksum (cs, n);
- if (data.dd.expect (cs.string ()) != nullptr)
- l4 ([&] {
- data.trace (ll)
- << "'depdb hash' argument change forcing update of "
- << data.t;});
+ v = cs.string ();
+ w = "argument";
}
else if (cmd == "string")
{
- string s;
try
{
- s = convert<string> (move (ns));
+ v = convert<string> (move (ns));
}
catch (const invalid_argument& e)
{
fail (ll) << "invalid 'depdb string' argument: " << e;
}
- if (data.dd.expect (s) != nullptr)
- l4 ([&] {
- data.trace (ll)
- << "'depdb string' argument change forcing update of "
- << data.t;});
+ w = "argument";
}
else if (cmd == "env")
{
@@ -1403,14 +1415,32 @@ namespace build2
fail (ll) << pf << e;
}
- if (data.dd.expect (cs.string ()) != nullptr)
- l4 ([&] {
- data.trace (ll)
- << "'depdb env' environment change forcing update of "
- << data.t;});
+ v = cs.string ();
+ w = "environment";
}
else
assert (false);
+
+ // Prefix the value with the type letter. This serves two
+ // purposes:
+ //
+ // 1. It makes sure the result is never a blank line. We use
+ // blank lines as anchors to skip directly to certain entries
+ // (e.g., dynamic targets).
+ //
+ // 2. It allows us to detect the beginning of prerequisites
+ // since an absolute path will be distinguishable from these
+ // entries (in the future we may want to add an explicit
+ // blank after such custom entries to make this easier).
+ //
+ v.insert (0, 1, ' ');
+ v.insert (0, 1, cmd[0]); // `h`, `s`, or `e`
+
+ if (data.dd.expect (v) != nullptr)
+ l4 ([&] {
+ data.trace (ll)
+ << "'depdb " << cmd << "' " << w << " change forcing "
+ << "update of " << data.t;});
}
}
else
@@ -1617,6 +1647,7 @@ namespace build2
size_t li, const location& ll,
action a, const scope& bs, file& t,
depdb& dd,
+ paths& dyn_targets,
bool& update,
timestamp mt,
bool& deferred_failure,
@@ -1629,6 +1660,7 @@ namespace build2
depdb_dyndep_options ops;
bool prog (false);
bool byprod (false);
+ bool dyn_tgt (false);
// Prerequisite update filter (--update-*).
//
@@ -1671,11 +1703,9 @@ namespace build2
next (t, tt); // Skip the 'dyndep' command.
- if (tt == type::word && t.value == "--byproduct")
- {
- byprod = true;
+ if (tt == type::word && ((byprod = (t.value == "--byproduct")) ||
+ (dyn_tgt = (t.value == "--dyn-target"))))
next (t, tt);
- }
assert (byprod == (byprod_result != nullptr));
@@ -1894,10 +1924,23 @@ namespace build2
continue;
}
- // Handle --byproduct in the wrong place.
+ // Handle --byproduct and --dyn-target in the wrong place.
//
if (strcmp (a, "--byproduct") == 0)
- fail (ll) << "depdb dyndep: --byproduct must be first option";
+ {
+ fail (ll) << "depdb dyndep: "
+ << (dyn_tgt
+ ? "--byproduct specified with --dyn-target"
+ : "--byproduct must be first option");
+ }
+
+ if (strcmp (a, "--dyn-target") == 0)
+ {
+ fail (ll) << "depdb dyndep: "
+ << (byprod
+ ? "--dyn-target specified with --byproduct"
+ : "--dyn-target must be first option");
+ }
// Handle non-literal --update-*.
//
@@ -1922,16 +1965,9 @@ namespace build2
}
}
- // --what
- //
- const char* what (ops.what_specified ()
- ? ops.what ().c_str ()
- : "file");
-
// --format
//
dyndep_format format (dyndep_format::make);
-
if (ops.format_specified ())
{
const string& f (ops.format ());
@@ -1941,10 +1977,18 @@ namespace build2
<< f << "'";
}
+ // Prerequisite-specific options.
+ //
+
+ // --what
+ //
+ const char* what (ops.what_specified ()
+ ? ops.what ().c_str ()
+ : "file");
+
// --cwd
//
optional<dir_path> cwd;
-
if (ops.cwd_specified ())
{
if (!byprod)
@@ -1964,28 +2008,6 @@ namespace build2
fail (ll) << "depdb dyndep: -I specified with --byproduct";
}
- // --file
- //
- // Note that if --file is specified without a program, then we assume
- // it is one of the static prerequisites.
- //
- optional<path> file;
-
- if (ops.file_specified ())
- {
- file = move (ops.file ());
-
- if (file->relative ())
- {
- if (!cwd)
- fail (ll) << "depdb dyndep: relative path specified with --file";
-
- *file = *cwd / *file;
- }
- }
- else if (!prog)
- fail (ll) << "depdb dyndep: program or --file expected";
-
// --default-type
//
// Get the default prerequisite type falling back to file{} if not
@@ -1997,7 +2019,7 @@ namespace build2
// translation unit would want to make sure it resolves extracted
// system headers to h{} targets analogous to the c module's rule.
//
- const target_type* def_pt;
+ const target_type* def_pt (&file::static_type);
if (ops.default_type_specified ())
{
const string& t (ops.default_type ());
@@ -2005,10 +2027,8 @@ namespace build2
def_pt = bs.find_target_type (t);
if (def_pt == nullptr)
fail (ll) << "depdb dyndep: unknown target type '" << t
- << "' specific with --default-type";
+ << "' specified with --default-type";
}
- else
- def_pt = &file::static_type;
// --adhoc
//
@@ -2018,6 +2038,76 @@ namespace build2
fail (ll) << "depdb dyndep: --adhoc specified with --byproduct";
}
+ // Target-specific options.
+ //
+
+ // --target-what
+ //
+ const char* what_tgt ("file");
+ if (ops.target_what_specified ())
+ {
+ if (!dyn_tgt)
+ fail (ll) << "depdb dyndep: --target-what specified without "
+ << "--dyn-target";
+
+ what_tgt = ops.target_what ().c_str ();
+ }
+
+ // --target-cwd
+ //
+ optional<dir_path> cwd_tgt;
+ if (ops.target_cwd_specified ())
+ {
+ if (!dyn_tgt)
+ fail (ll) << "depdb dyndep: --target-cwd specified without "
+ << "--dyn-target";
+
+ cwd_tgt = move (ops.target_cwd ());
+
+ if (cwd_tgt->relative ())
+ fail (ll) << "depdb dyndep: relative path specified with "
+ << "--target-cwd";
+ }
+
+ // --target-default-type
+ //
+ const target_type* def_tt (&file::static_type);
+ if (ops.target_default_type_specified ())
+ {
+ if (!dyn_tgt)
+ fail (ll) << "depdb dyndep: --target-default-type specified "
+ << "without --dyn-target";
+
+ const string& t (ops.target_default_type ());
+
+ def_tt = bs.find_target_type (t);
+ if (def_tt == nullptr)
+ fail (ll) << "depdb dyndep: unknown target type '" << t
+ << "' specified with --target-default-type";
+ }
+
+
+ // --file (last since need --*cwd)
+ //
+ // Note that if --file is specified without a program, then we assume
+ // it is one of the static prerequisites.
+ //
+ optional<path> file;
+ if (ops.file_specified ())
+ {
+ file = move (ops.file ());
+
+ if (file->relative ())
+ {
+ if (!cwd && !cwd_tgt)
+ fail (ll) << "depdb dyndep: relative path specified with --file";
+
+ *file = (cwd ? *cwd : *cwd_tgt) / *file;
+ }
+ }
+ else if (!prog)
+ fail (ll) << "depdb dyndep: program or --file expected";
+
// Update prerequisite targets.
//
using dyndep = dyndep_rule;
@@ -2148,6 +2238,8 @@ namespace build2
return;
}
+ const scope& rs (*bs.root_scope ());
+
// This code is based on the prior work in the cc module (specifically
// extract_headers()) where you can often find more detailed rationale
// for some of the steps performed.
@@ -2161,6 +2253,7 @@ namespace build2
[] (const scope& bs, const string& n, const string& e)
{
// NOTE: another version in adhoc_buildscript_rule::apply().
+ // NOTE: now also used for dynamic targets below!
// @@ TODO: allow specifying base target types.
//
@@ -2275,14 +2368,17 @@ namespace build2
// they include the line index in their names to avoid clashes
// between lines).
//
- // Cleanups are not an issue, they will simply replaced. And
+ // Cleanups are not an issue, they will simply be replaced. And
// overriding the contents of the special files seems harmless and
// consistent with what would happen if the command redirects its
// output to a non-special file.
//
+ // Note: make it a maybe-cleanup in case the command cleans it
+ // up itself.
+ //
if (file)
environment_->clean (
- {build2::script::cleanup_type::always, *file},
+ {build2::script::cleanup_type::maybe, *file},
true /* implicit */);
}
};
@@ -2429,13 +2525,27 @@ namespace build2
<< t;
});
+ // While in the make format targets come before prerequisites, in
+ // depdb we store them after since any change to prerequisites can
+ // invalidate the set of targets. So we save them first and process
+ // later.
+ //
+ // Note also that we need to return them to the caller in case we are
+ // updating.
+
// If nothing so far has invalidated the dependency database, then try
// the cached data before running the program.
//
bool cache (!update);
+ bool skip_blank (false);
for (bool restart (true), first_run (true); restart; cache = false)
{
+ // Clear the state in case we are restarting.
+ //
+ if (dyn_tgt)
+ dyn_targets.clear ();
+
restart = false;
if (cache)
@@ -2444,7 +2554,8 @@ namespace build2
//
assert (skip_count == 0);
- // We should always end with a blank line.
+ // We should always end with a blank line after the list of
+ // dynamic prerequisites.
//
for (;;)
{
@@ -2458,8 +2569,11 @@ namespace build2
break;
}
- if (l->empty ()) // Done, nothing changed.
- return;
+ if (l->empty ()) // Done with prerequisites, nothing changed.
+ {
+ skip_blank = true;
+ break;
+ }
if (optional<bool> r = add (path (move (*l)), nullptr, mt))
{
@@ -2481,6 +2595,36 @@ namespace build2
return;
}
}
+
+ if (!restart) // Nothing changed.
+ {
+ if (dyn_tgt)
+ {
+ // We should always end with a blank line after the list of
+ // dynamic targets.
+ //
+ for (;;)
+ {
+ string* l (dd.read ());
+
+ // If the line is invalid, run the compiler.
+ //
+ if (l == nullptr)
+ {
+ restart = true;
+ break;
+ }
+
+ if (l->empty ()) // Done with target.
+ break;
+
+ dyn_targets.push_back (path (move (*l)));
+ }
+ }
+
+ if (!restart) // Done, nothing changed.
+ break; // Break earliy to keep cache=true.
+ }
}
else
{
@@ -2614,31 +2758,63 @@ namespace build2
if (r.second.empty ())
continue;
- // @@ TODO: what should we do about targets?
+ // Skip targets unless requested to extract.
//
- // Note that if we take GCC as an example, things are
+ // BTW, if you are wondering why don't we extract targets
+ // by default, take GCC as an example, where things are
// quite messed up: by default it ignores -o and just
// takes the source file name and replaces the extension
// with a platform-appropriate object file extension. One
// can specify a custom target (or even multiple targets)
- // with -MT or with -MQ (quoting). Though MinGW GCC still
- // does not quote `:` with -MQ. So in this case it's
+ // with -MT or with -MQ (quoting). So in this case it's
// definitely easier for the user to ignore the targets
// and just specify everything in the buildfile.
//
- // On the other hand, other tools are likely to produce
- // more sensible output (except perhaps for quoting).
- //
- // @@ Maybe in the lax mode we should only recognize `:`
- // if it's separated on at least one side?
- //
- // Alternatively, we could detect Windows drives in
- // paths and "handle" them (I believe this is what GNU
- // make does). Maybe we should have three formats:
- // make-lax, make, make-strict?
- //
if (r.first == make_type::target)
+ {
+ if (dyn_tgt)
+ {
+ path& f (r.second);
+
+ if (f.relative ())
+ {
+ if (!cwd_tgt)
+ fail (il) << "relative target path '" << f
+ << "' in make dependency declaration" <<
+ info << "consider using --target-cwd to specify "
+ << "relative path base";
+
+ f = *cwd_tgt / f;
+ }
+
+ // Note that unlike prerequisites, here we don't need
+ // normalize_external() since we expect the targets to
+ // be within this project.
+ //
+ try
+ {
+ f.normalize ();
+ }
+ catch (const invalid_path&)
+ {
+ fail << "invalid " << what_tgt << " path '"
+ << f.string () << "'";
+ }
+
+ // The target must be within this project.
+ //
+ if (!f.sub (rs.out_path ()))
+ {
+ fail << what_tgt << " target path " << f
+ << " must be inside project output directory "
+ << rs.out_path ();
+ }
+
+ dyn_targets.push_back (move (f));
+ }
+
continue;
+ }
if (optional<bool> u = add (move (r.second), &skip, rmt))
{
@@ -2667,7 +2843,7 @@ namespace build2
break;
}
- break;
+ break; // case
}
}
@@ -2678,9 +2854,88 @@ namespace build2
}
}
- // Add the terminating blank line (we are updating depdb).
+ // Add the dynamic prerequisites terminating blank line if we are
+ // updating depdb and unless it's already there.
+ //
+ if (!cache && !skip_blank)
+ dd.expect ("");
+
+ // Handle dynamic targets.
//
- dd.expect ("");
+ if (dyn_tgt)
+ {
+ // There is one more level (at least that we know of) to this rabbit
+ // hole: if the set of dynamic targets changes between clean and
+ // update and we do a `clean update` batch, then we will end up with
+ // old targets (as entered by clean from old depdb information)
+ // being present during update. So we need to clean them out.
+ //
+ // Optimize this for a first/single batch (common case) by noticing
+ // that there are only real targets to start with.
+ //
+ optional<vector<const target*>> dts;
+ for (const target* m (&t); m != nullptr; m = m->adhoc_member)
+ {
+ if (m->decl != target_decl::real)
+ dts = vector<const target*> ();
+ }
+
+ for (const path& f: dyn_targets)
+ {
+ // Note that this logic should be consistent with what we have in
+ // adhoc_buildscript_rule::apply() for perform_clean.
+ //
+ pair<const build2::file&, bool> r (
+ dyndep::inject_adhoc_group_member (
+ what_tgt,
+ a, bs, t,
+ f, // Can't move since need to return dyn_targets.
+ map_ext, *def_tt));
+
+ // Note that we have to track the dynamic target even if it was
+ // already a member (think `b update && b clean update`).
+ //
+ if (r.second || r.first.decl != target_decl::real)
+ {
+ if (!cache)
+ dd.expect (f);
+
+ if (dts)
+ dts->push_back (&r.first);
+ }
+ }
+
+ // Add the dynamic targets terminating blank line.
+ //
+ if (!cache)
+ dd.expect ("");
+
+ // Clean out old dynamic targets (skip the primary member).
+ //
+ if (dts)
+ {
+ for (target* p (&t); p->adhoc_member != nullptr; )
+ {
+ target* m (p->adhoc_member);
+
+ if (m->decl != target_decl::real)
+ {
+ // While there could be quite a few dynamic targets (think
+ // something like Doxygen), this will hopefully be optimized
+ // down to a contiguous memory region scan for an integer and
+ // so should be fast.
+ //
+ if (find (dts->begin (), dts->end (), m) == dts->end ())
+ {
+ p->adhoc_member = m->adhoc_member; // Drop m.
+ continue;
+ }
+ }
+
+ p = m;
+ }
+ }
+ }
// Reload $< and $> to make sure they contain the newly discovered
// prerequisites and targets.
diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx
index 70e24aa..b615a90 100644
--- a/libbuild2/build/script/parser.hxx
+++ b/libbuild2/build/script/parser.hxx
@@ -123,14 +123,16 @@ namespace build2
execute_depdb_preamble_dyndep (
action a, const scope& base, file& t,
environment& e, const script& s, runner& r,
- depdb& dd, bool& update, timestamp mt, bool& deferred_failure)
+ depdb& dd,
+ paths& dyn_targets,
+ bool& update, timestamp mt, bool& deferred_failure)
{
exec_depdb_preamble (
a, base, t,
e, s, r,
s.depdb_preamble.begin () + *s.depdb_dyndep,
s.depdb_preamble.end (),
- dd, &update, mt, &deferred_failure);
+ dd, &dyn_targets, &update, mt, &deferred_failure);
}
// This version doesn't actually execute the depdb-dyndep builtin (but
@@ -161,6 +163,7 @@ namespace build2
// This is getting a bit ugly (we also don't really need to pass
// depdb here). One day we will find a better way...
//
+ paths dyn_targets;
bool deferred_failure; // Dymmy.
dyndep_byproduct v;
@@ -169,7 +172,7 @@ namespace build2
e, s, r,
s.depdb_preamble.begin () + *s.depdb_dyndep,
s.depdb_preamble.end (),
- dd, &update, mt, &deferred_failure, &v);
+ dd, &dyn_targets, &update, mt, &deferred_failure, &v);
return v;
}
@@ -221,6 +224,7 @@ namespace build2
environment&, const script&, runner&,
lines_iterator begin, lines_iterator end,
depdb&,
+ paths* dyn_targets = nullptr,
bool* update = nullptr,
optional<timestamp> mt = nullopt,
bool* deferred_failure = nullptr,
@@ -231,6 +235,7 @@ namespace build2
size_t line_index, const location&,
action, const scope& base, file&,
depdb&,
+ paths& dyn_targets,
bool& update,
timestamp,
bool& deferred_failure,
@@ -355,6 +360,7 @@ namespace build2
optional<pair<location, size_t>>
depdb_dyndep_; // depdb-dyndep location/position.
bool depdb_dyndep_byproduct_ = false; // --byproduct
+ bool depdb_dyndep_dyn_target_ = false; // --dyn-target
lines depdb_preamble_; // Note: excluding depdb-clear.
// If present, the first impure function called in the body of the
diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx
index 57a893e..08f1bf4 100644
--- a/libbuild2/build/script/script.hxx
+++ b/libbuild2/build/script/script.hxx
@@ -82,8 +82,9 @@ namespace build2
bool depdb_value; // String or hash.
optional<size_t> depdb_dyndep; // Pos of first dyndep.
bool depdb_dyndep_byproduct = false; // dyndep --byproduct
+ bool depdb_dyndep_dyn_target = false;// dyndep --dyn-target
lines depdb_preamble; // Note include vars.
- bool depdb_preamble_temp_dir = false; // True if refs $~.
+ bool depdb_preamble_temp_dir = false;// True if refs $~.
location start_loc;
location end_loc;
diff --git a/libbuild2/dyndep.cxx b/libbuild2/dyndep.cxx
index ace901b..b793de8 100644
--- a/libbuild2/dyndep.cxx
+++ b/libbuild2/dyndep.cxx
@@ -162,12 +162,6 @@ namespace build2
dr << info << "consider listing it as static prerequisite of " << t;
}
- // Reverse-lookup target type(s) from file name/extension.
- //
- // If the list of base target types is specified, then only these types and
- // those derived from them are considered. Otherwise, any file-based type is
- // considered but not the file type itself.
- //
small_vector<const target_type*, 2> dyndep_rule::
map_extension (const scope& bs,
const string& n, const string& e,
@@ -877,4 +871,99 @@ namespace build2
return t;
}
+
+ pair<const file&, bool> dyndep_rule::
+ inject_adhoc_group_member (const char* what,
+ action, const scope& bs, target& t,
+ path f,
+ const function<map_extension_func>& map_ext,
+ const target_type& fallback)
+ {
+ path n (f.leaf ());
+ string e (n.extension ());
+ n.make_base ();
+
+ // Map extension to the target type, falling back to def_tt.
+ //
+ small_vector<const target_type*, 2> tts;
+ if (map_ext != nullptr)
+ tts = map_ext (bs, n.string (), e);
+
+ // Not sure what else we can do in this case.
+ //
+ if (tts.size () > 1)
+ {
+ diag_record dr (fail);
+
+ dr << "mapping of " << what << " target file " << f
+ << " to target type is ambiguous";
+
+ for (const target_type* tt: tts)
+ dr << info << "can be " << tt->name << "{}";
+ }
+
+ const target_type& tt (tts.empty () ? fallback : *tts.front ());
+
+ if (!tt.is_a<file> ())
+ {
+ fail << what << " target file " << f << " mapped to non-file-based "
+ << "target type " << tt.name << "{}";
+ }
+
+ // Assume nobody else can insert these members (seems reasonable seeing
+ // that their names are dynamically discovered).
+ //
+ auto l (search_new_locked (
+ bs.ctx,
+ tt,
+ f.directory (),
+ dir_path (), // Always in out.
+ move (n).string (),
+ &e,
+ &bs));
+
+ file* ft (&l.first.as<file> ()); // Note: non-const only if locked.
+
+ // Skip if this is one of the static targets (or a duplicate of the
+ // dynamic target).
+ //
+ // In particular, we expect to skip all the targets that we could not lock
+ // (e.g., in case all of this has already been done for the previous
+ // operation in a batch; make sure to test `update update update` and
+ // `update clean update ...` batches if changing anything here).
+ //
+ // While at it also find the ad hoc members list tail.
+ //
+ const_ptr<target>* tail (&t.adhoc_member);
+ for (target* m (&t); m != nullptr; m = m->adhoc_member)
+ {
+ if (ft == m)
+ {
+ tail = nullptr;
+ break;
+ }
+
+ tail = &m->adhoc_member;
+ }
+
+ if (tail == nullptr)
+ return pair<const file&, bool> (*ft, false);
+
+ if (!l.second)
+ fail << "dynamic " << what << " target " << *ft << " already exists "
+ << "and cannot be made ad hoc member of group " << t;
+
+ ft->group = &t;
+ l.second.unlock ();
+
+ // We need to be able to distinguish static targets from dynamic (see the
+ // static set hashing in adhoc_buildscript_rule::apply() for details).
+ //
+ assert (ft->decl != target_decl::real);
+
+ *tail = ft;
+ ft->path (move (f));
+
+ return pair<const file&, bool> (*ft, true);
+ }
}
diff --git a/libbuild2/dyndep.hxx b/libbuild2/dyndep.hxx
index a6f800e..6ae585e 100644
--- a/libbuild2/dyndep.hxx
+++ b/libbuild2/dyndep.hxx
@@ -94,6 +94,10 @@ namespace build2
// and those derived from them are considered. Otherwise, any file-based
// type is considered but not the file type itself.
//
+ // It's possible the extension-to-target type mapping is ambiguous (for
+ // example, because both C and C++-language headers use the same .h
+ // extension). So this function can return multiple target types.
+ //
static small_vector<const target_type*, 2>
map_extension (const scope& base,
const string& name, const string& ext,
@@ -244,6 +248,23 @@ namespace build2
return inject_group_member (
a, bs, g, move (p), T::static_type).template as<T> ();
}
+
+ // Find or insert a target file path as a target, make it a member of the
+ // specified ad hoc group unless it already is, and set its path. Return
+ // the target and an indication of whether it was added as a member.
+ //
+ // The target type is determined using the map_extension function if
+ // specified, falling back to the fallback type if unable to.
+ //
+ // The file path must be absolute and normalized. Note that this function
+ // assumes that this target can only be known as a member of this group.
+ //
+ static pair<const file&, bool>
+ inject_adhoc_group_member (const char* what,
+ action, const scope& base, target& g,
+ path,
+ const function<map_extension_func>&,
+ const target_type& fallback);
};
}
diff --git a/libbuild2/script/script.cxx b/libbuild2/script/script.cxx
index b8dfc68..4a6ca33 100644
--- a/libbuild2/script/script.cxx
+++ b/libbuild2/script/script.cxx
@@ -768,7 +768,9 @@ namespace build2
{
using script::cleanup;
- assert (!implicit || c.type == cleanup_type::always);
+ // Implicit never-cleanup doesn't make sense.
+ //
+ assert (!implicit || c.type != cleanup_type::never);
const path& p (c.path);
diff --git a/libbuild2/target.hxx b/libbuild2/target.hxx
index 037b18c..e935477 100644
--- a/libbuild2/target.hxx
+++ b/libbuild2/target.hxx
@@ -272,6 +272,10 @@ namespace build2
// fuzzy: they feel more `real` than `implied`. Maybe introduce
// `synthesized` in-between?
//
+ // @@ There are also now dynamically-discovered targets (ad hoc group
+ // members; see depdb-dyndep --dyn-target) which currently end up
+ // with prereq_new.
+ //
enum class target_decl: uint8_t
{
prereq_new = 1, // Created from prerequisite (create_new_target()).