aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2021-11-11 13:20:30 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2021-11-23 13:07:58 +0200
commit939beb11a5ccf58d7fe79a809a1b592c5c9143c0 (patch)
tree2aff4e52f277ecac62ce1cb8bf302ffd0884666a
parent189a1c2a8fad0716e0bc4132e21f664c80d7574b (diff)
Add support for dynamic dependencies in ad hoc Buildscript recipes
Specifically, add the new `depdb dyndep` builtin that can be used to extract dynamic dependencies from a program run or a file. For example: obje{hello.o}: cxx{hello} {{ s = $path($<[0]) depdb dyndep $cxx.poptions $cc.poptions --what=header --default-prereq-type=h -- $cxx.path $cxx.poptions $cc.poptions $cxx.mode -M -MG $s diag c++ ($<[0]) o = $path($>) $cxx.path $cxx.poptions $cc.poptions $cc.coptions $cxx.coptions $cxx.mode -o $o -c $s }} Currently only the `make` dependency format is supported.
-rw-r--r--libbuild2/adhoc-rule-buildscript.cxx743
-rw-r--r--libbuild2/adhoc-rule-buildscript.hxx12
-rw-r--r--libbuild2/build/script/builtin-options.cxx701
-rw-r--r--libbuild2/build/script/builtin-options.hxx456
-rw-r--r--libbuild2/build/script/builtin-options.ixx338
-rw-r--r--libbuild2/build/script/builtin.cli32
-rw-r--r--libbuild2/build/script/parser.cxx871
-rw-r--r--libbuild2/build/script/parser.hxx79
-rw-r--r--libbuild2/build/script/runner.hxx2
-rw-r--r--libbuild2/build/script/script.hxx11
-rw-r--r--libbuild2/build/script/types-parsers.cxx56
-rw-r--r--libbuild2/build/script/types-parsers.hxx49
-rw-r--r--libbuild2/buildfile79
-rw-r--r--libbuild2/cc/compile-rule.cxx848
-rw-r--r--libbuild2/cc/compile-rule.hxx44
-rw-r--r--libbuild2/cc/utility.cxx57
-rw-r--r--libbuild2/cc/utility.hxx29
-rw-r--r--libbuild2/dyndep.cxx667
-rw-r--r--libbuild2/dyndep.hxx168
-rw-r--r--libbuild2/filesystem.cxx55
-rw-r--r--libbuild2/filesystem.hxx29
-rw-r--r--libbuild2/make-parser.cxx137
-rw-r--r--libbuild2/make-parser.hxx85
-rw-r--r--libbuild2/make-parser.test.cxx90
-rw-r--r--libbuild2/make-parser.test.testscript98
-rw-r--r--libbuild2/script/parser.cxx2
-rw-r--r--libbuild2/script/run.cxx225
-rw-r--r--libbuild2/script/run.hxx15
-rw-r--r--libbuild2/script/script.hxx3
29 files changed, 4718 insertions, 1263 deletions
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx
index 61b4cb2..78f87ae 100644
--- a/libbuild2/adhoc-rule-buildscript.cxx
+++ b/libbuild2/adhoc-rule-buildscript.cxx
@@ -8,6 +8,7 @@
#include <libbuild2/depdb.hxx>
#include <libbuild2/scope.hxx>
#include <libbuild2/target.hxx>
+#include <libbuild2/dyndep.hxx>
#include <libbuild2/context.hxx>
#include <libbuild2/algorithm.hxx>
#include <libbuild2/filesystem.hxx> // path_perms(), auto_rmfile
@@ -22,6 +23,110 @@ using namespace std;
namespace build2
{
+ static inline void
+ hash_script_vars (sha256& cs,
+ const build::script::script& s,
+ const target& t,
+ names& storage)
+ {
+ context& ctx (t.ctx);
+
+ for (const string& n: s.vars)
+ {
+ cs.append (n);
+
+ lookup l;
+
+ if (const variable* var = ctx.var_pool.find (n))
+ l = t[var];
+
+ cs.append (!l.defined () ? '\x1' : l->null ? '\x2' : '\x3');
+
+ if (l)
+ {
+ storage.clear ();
+ names_view ns (reverse (*l, storage));
+
+ for (const name& n: ns)
+ to_checksum (cs, n);
+ }
+ }
+ }
+
+ // How should we hash target and prerequisite sets ($> and $<)? We could
+ // hash them as target names (i.e., the same as the $>/< content) or as
+ // paths (only for path-based targets). While names feel more general, they
+ // are also more expensive to compute. And for path-based targets, path is
+ // generally a good proxy for the target name. Since the bulk of the ad hoc
+ // recipes will presumably be operating exclusively on path-based targets,
+ // let's do it both ways.
+ //
+ static inline void
+ hash_target (sha256& cs, const target& t, names& storage)
+ {
+ if (const path_target* pt = t.is_a<path_target> ())
+ cs.append (pt->path ().string ());
+ else
+ {
+ storage.clear ();
+ t.as_name (storage);
+ for (const name& n: storage)
+ to_checksum (cs, n);
+ }
+ };
+
+ // The script can reference a program in one of four ways:
+ //
+ // 1. As an (imported) target (e.g., $cli)
+ //
+ // 2. As a process_path_ex (e.g., $cxx.path).
+ //
+ // 3. As a builtin (e.g., sed)
+ //
+ // 4. As a program path/name.
+ //
+ // When it comes to change tracking, there is nothing we can do for (4) (the
+ // user can track its environment manually with depdb-env) and there is
+ // nothing to do for (3) (assuming builtin semantics is stable/backwards-
+ // compatible). The (2) case is handled automatically by hashing all the
+ // variable values referenced by the script (see below), which in case of
+ // process_path_ex includes the checksums (both executable and environment),
+ // if available.
+ //
+ // This leaves the (1) case, which itself splits into two sub-cases: the
+ // target comes with the dependency information (e.g., imported from a
+ // project via an export stub) or it does not (e.g., imported as installed).
+ // We don't need to do anything extra for the first sub-case since the
+ // target's state/mtime can be relied upon like any other prerequisite.
+ // Which cannot be said about the second sub-case, where we reply on
+ // checksum that may be included as part of the target metadata.
+ //
+ // So what we are going to do is hash checksum metadata of every executable
+ // prerequisite target that has it (we do it here in order to include ad hoc
+ // prerequisites, which feels like the right thing to do; the user may mark
+ // tools as ad hoc in order to omit them from $<).
+ //
+ static inline void
+ hash_prerequisite_target (sha256& cs, sha256& exe_cs, sha256& env_cs,
+ const target& pt,
+ names& storage)
+ {
+ hash_target (cs, pt, storage);
+
+ if (const exe* et = pt.is_a<exe> ())
+ {
+ if (const string* c = et->lookup_metadata<string> ("checksum"))
+ {
+ exe_cs.append (*c);
+ }
+
+ if (const strings* e = et->lookup_metadata<strings> ("environment"))
+ {
+ hash_environment (env_cs, *e);
+ }
+ }
+ }
+
bool adhoc_buildscript_rule::
recipe_text (const scope& s,
const target_type& tt,
@@ -113,6 +218,20 @@ namespace build2
perform_update_id) != actions.end ();
}
+ struct adhoc_buildscript_rule::match_data
+ {
+ match_data (action a, const target& t, bool temp_dir)
+ : env (a, t, temp_dir) {}
+
+ build::script::environment env;
+ build::script::default_runner run;
+
+ path dd;
+ const scope* bs;
+ timestamp mt;
+ bool deferred_failure;
+ };
+
bool adhoc_buildscript_rule::
match (action a, target& t, const string& h, match_extra& me) const
{
@@ -140,15 +259,17 @@ namespace build2
recipe adhoc_buildscript_rule::
apply (action a,
- target& t,
+ target& xt,
match_extra& me,
const optional<timestamp>& d) const
{
+ tracer trace ("adhoc_buildscript_rule::apply");
+
// We don't support deadlines for any of these cases (see below).
//
if (d && (a.outer () ||
me.fallback ||
- (a == perform_update_id && t.is_a<file> ())))
+ (a == perform_update_id && xt.is_a<file> ())))
return empty_recipe;
// If this is an outer operation (e.g., update-for-test), then delegate to
@@ -156,20 +277,20 @@ namespace build2
//
if (a.outer ())
{
- match_inner (a, t);
+ match_inner (a, xt);
return execute_inner;
}
// Inject pattern's ad hoc group members, if any.
//
if (pattern != nullptr)
- pattern->apply_adhoc_members (a, t, me);
+ pattern->apply_adhoc_members (a, xt, me);
// Derive file names for the target and its ad hoc group members, if any.
//
if (a == perform_update_id || a == perform_clean_id)
{
- for (target* m (&t); m != nullptr; m = m->adhoc_member)
+ for (target* m (&xt); m != nullptr; m = m->adhoc_member)
{
if (auto* p = m->is_a<path_target> ())
p->derive_path ();
@@ -181,68 +302,306 @@ namespace build2
// We do it always instead of only if one of the targets is path-based in
// case the recipe creates temporary files or some such.
//
- inject_fsdir (a, t);
+ const fsdir* dir (inject_fsdir (a, xt));
// Match prerequisites.
//
- match_prerequisite_members (a, t);
+ match_prerequisite_members (a, xt);
// Inject pattern's prerequisites, if any.
//
if (pattern != nullptr)
- pattern->apply_prerequisites (a, t, me);
+ pattern->apply_prerequisites (a, xt, me);
// See if we are providing the standard clean as a fallback.
//
if (me.fallback)
return &perform_clean_depdb;
- if (a == perform_update_id && t.is_a<file> ())
+ // See if this is not update or not on a file-based target.
+ //
+ if (a != perform_update_id || !xt.is_a<file> ())
+ {
+ return [d, this] (action a, const target& t)
+ {
+ return default_action (a, t, d);
+ };
+ }
+
+ // See if this is the simple case with only static dependencies.
+ //
+ if (!script.depdb_dyndep)
{
return [this] (action a, const target& t)
{
return perform_update_file (a, t);
};
}
+
+ // This is a perform update on a file target with extraction of dynamic
+ // dependency information in the depdb preamble (depdb-dyndep).
+ //
+ // This means we may need to add additional prerequisites (or even target
+ // group members). We also have to save any such additional prerequisites
+ // in depdb so that we can check if any of them have changed on subsequent
+ // updates. So all this means that have to take care of depdb here in
+ // apply() instead of perform_*() like we normally do. We also do things
+ // in slightly different order due to the restrictions impose by the match
+ // phase.
+ //
+ // Note that the C/C++ header dependency extraction is the canonical
+ // example and all this logic is based on the prior work in the cc module
+ // where you can often find more detailed rationale for some of the steps
+ // performed (like the fsdir update below).
+ //
+ context& ctx (xt.ctx);
+
+ file& t (xt.as<file> ());
+ const path& tp (t.path ());
+
+ if (dir != nullptr)
+ fsdir_rule::perform_update_direct (a, t);
+
+ // Because the depdb preamble can access $<, we have to blank out all the
+ // ad hoc prerequisites. Since we will still need them later, we "move"
+ // them to the auxiliary data member in prerequisite_target (which also
+ // means we cannot use the standard execute_prerequisites()).
+ //
+ auto& pts (t.prerequisite_targets[a]);
+ for (prerequisite_target& p: pts)
+ {
+ // Note that fsdir{} injected above is adhoc.
+ //
+ if (p.target != nullptr && p.adhoc)
+ {
+ p.data = reinterpret_cast<uintptr_t> (p.target);
+ p.target = nullptr;
+ }
+ }
+
+ // NOTE: see the "static dependencies" version (with comments) below.
+ //
+ depdb dd (tp + ".d");
+
+ if (dd.expect ("<ad hoc buildscript recipe> 1") != nullptr)
+ l4 ([&]{trace << "rule mismatch forcing update of " << t;});
+
+ if (dd.expect (checksum) != nullptr)
+ l4 ([&]{trace << "recipe text change forcing update of " << t;});
+
+ if (!script.depdb_clear)
+ {
+ names storage;
+
+ sha256 prq_cs, exe_cs, env_cs;
+
+ for (const prerequisite_target& p: pts)
+ {
+ if (const target* pt =
+ (p.target != nullptr ? p.target :
+ p.data != 0 ? reinterpret_cast<target*> (p.data) :
+ nullptr))
+ {
+ hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage);
+ }
+ }
+
+ {
+ sha256 cs;
+ hash_script_vars (cs, script, t, storage);
+
+ if (dd.expect (cs.string ()) != nullptr)
+ l4 ([&]{trace << "recipe variable change forcing update of " << t;});
+ }
+
+ {
+ sha256 tcs;
+ for (const target* m (&t); m != nullptr; m = m->adhoc_member)
+ hash_target (tcs, *m, storage);
+
+ if (dd.expect (tcs.string ()) != nullptr)
+ l4 ([&]{trace << "target set change forcing update of " << t;});
+
+ if (dd.expect (prq_cs.string ()) != nullptr)
+ l4 ([&]{trace << "prerequisite set change forcing update of " << t;});
+ }
+
+ {
+ if (dd.expect (exe_cs.string ()) != nullptr)
+ l4 ([&]{trace << "program checksum change forcing update of " << t;});
+
+ if (dd.expect (env_cs.string ()) != nullptr)
+ l4 ([&]{trace << "environment change forcing update of " << t;});
+ }
+ }
+
+ const scope& bs (t.base_scope ());
+
+ unique_ptr<match_data> md (
+ new match_data (a, t, script.depdb_preamble_temp_dir));
+
+ build::script::environment& env (md->env);
+ build::script::default_runner& run (md->run);
+
+ run.enter (env, script.start_loc);
+
+ // Run the first half of the preamble (before depdb-dyndep).
+ //
+ {
+ build::script::parser p (ctx);
+ p.execute_depdb_preamble (a, bs, t, env, script, run, dd);
+ }
+
+ // Determine if we need to do an update based on the above checks.
+ //
+ bool update;
+ timestamp mt;
+
+ if (dd.writing ())
+ update = true;
else
{
- return [d, this] (action a, const target& t)
+ if ((mt = t.mtime ()) == timestamp_unknown)
+ t.mtime (mt = mtime (tp)); // Cache.
+
+ update = dd.mtime > mt;
+ }
+
+ if (update)
+ mt = timestamp_nonexistent;
+
+ // Update our prerequisite targets. While strictly speaking we only need
+ // to update those that are referenced by depdb-dyndep, communicating
+ // this is both tedious and error-prone. So we update them all.
+ //
+ for (const prerequisite_target& p: pts)
+ {
+ if (const target* pt =
+ (p.target != nullptr ? p.target :
+ p.data != 0 ? reinterpret_cast<target*> (p.data) : nullptr))
{
- return default_action (a, t, d);
- };
+ update = dyndep_rule::update (
+ trace, a, *pt, update ? timestamp_unknown : mt) || update;
+ }
+ }
+
+ // Run the second half of the preamble (depdb-dyndep commands) to extract
+ // dynamic dependencies.
+ //
+ // Note that this should be the last update to depdb (the invalidation
+ // order semantics).
+ //
+ bool deferred_failure (false);
+ {
+ build::script::parser p (ctx);
+ p.execute_depdb_preamble_dyndep (a, bs, t,
+ env, script, run,
+ dd,
+ update,
+ deferred_failure,
+ mt);
}
+
+ if (update && dd.reading () && !ctx.dry_run)
+ dd.touch = true;
+
+ dd.close ();
+ md->dd = move (dd.path);
+
+ // Pass on base scope and update/mtime.
+ //
+ md->bs = &bs;
+ md->mt = update ? timestamp_nonexistent : mt;
+ md->deferred_failure = deferred_failure;
+
+ // @@ TMP: re-enable once recipe becomes move_only_function.
+ //
+#if 0
+ return [this, md = move (md)] (action a, const target& t) mutable
+ {
+ auto r (perform_update_file_dyndep (a, t, *md));
+ md.reset (); // @@ TMP: is this really necessary (+mutable)?
+ return r;
+ };
+#else
+ t.data (move (md));
+ return recipe ([this] (action a, const target& t) mutable
+ {
+ auto md (move (t.data<unique_ptr<match_data>> ()));
+ return perform_update_file_dyndep (a, t, *md);
+ });
+#endif
}
target_state adhoc_buildscript_rule::
- perform_update_file (action a, const target& xt) const
+ perform_update_file_dyndep (action a, const target& xt, match_data& md) const
{
- tracer trace ("adhoc_buildscript_rule::perform_update_file");
+ tracer trace ("adhoc_buildscript_rule::perform_update_file_dyndep");
context& ctx (xt.ctx);
const file& t (xt.as<file> ());
const path& tp (t.path ());
- // How should we hash target and prerequisite sets ($> and $<)? We could
- // hash them as target names (i.e., the same as the $>/< content) or as
- // paths (only for path-based targets). While names feel more general,
- // they are also more expensive to compute. And for path-based targets,
- // path is generally a good proxy for the target name. Since the bulk of
- // the ad hoc recipes will presumably be operating exclusively on
- // path-based targets, let's do it both ways.
+ // While we've updated all our prerequisites in apply(), we still need to
+ // execute them here to keep the dependency counts straight.
//
- auto hash_target = [ns = names ()] (sha256& cs, const target& t) mutable
+ for (const prerequisite_target& p: t.prerequisite_targets[a])
{
- if (const path_target* pt = t.is_a<path_target> ())
- cs.append (pt->path ().string ());
- else
+ if (const target* pt =
+ (p.target != nullptr ? p.target :
+ p.data != 0 ? reinterpret_cast<target*> (p.data) : nullptr))
{
- ns.clear ();
- t.as_name (ns);
- for (const name& n: ns)
- to_checksum (cs, n);
+ target_state ts (execute_wait (a, *pt));
+ assert (ts == target_state::unchanged || ts == target_state::changed);
}
- };
+ }
+
+ build::script::environment& env (md.env);
+ build::script::default_runner& run (md.run);
+
+ // Force update in case of a deferred failure even if nothing changed.
+ //
+ if (md.mt != timestamp_nonexistent && !md.deferred_failure)
+ {
+ run.leave (env, script.end_loc);
+ return target_state::unchanged;
+ }
+
+ // Sequence start time for mtime checks below.
+ //
+ timestamp start (!ctx.dry_run && depdb::mtime_check ()
+ ? system_clock::now ()
+ : timestamp_unknown);
+
+ if (!ctx.dry_run || verb != 0)
+ {
+ if (execute_update_file (*md.bs, a, t, env, run, md.deferred_failure))
+ ;
+ else
+ run.leave (env, script.end_loc);
+ }
+ else
+ run.leave (env, script.end_loc);
+
+ timestamp now (system_clock::now ());
+
+ if (!ctx.dry_run)
+ depdb::check_mtime (start, md.dd, tp, now);
+
+ t.mtime (now);
+ return target_state::changed;
+ }
+
+ target_state adhoc_buildscript_rule::
+ perform_update_file (action a, const target& xt) const
+ {
+ tracer trace ("adhoc_buildscript_rule::perform_update_file");
+
+ context& ctx (xt.ctx);
+
+ const file& t (xt.as<file> ());
+ const path& tp (t.path ());
// Update prerequisites and determine if any of them render this target
// out-of-date.
@@ -250,6 +609,8 @@ namespace build2
timestamp mt (t.load_mtime ());
optional<target_state> ps;
+ names storage;
+
sha256 prq_cs, exe_cs, env_cs;
{
// This is essentially ps=execute_prerequisites(a, t, mt) which we
@@ -262,7 +623,9 @@ namespace build2
wait_guard wg (ctx, busy, t[a].task_count);
- for (const target*& pt: t.prerequisite_targets[a])
+ auto& pts (t.prerequisite_targets[a]);
+
+ for (const target*& pt: pts)
{
if (pt == nullptr) // Skipped.
continue;
@@ -279,7 +642,7 @@ namespace build2
wg.wait ();
bool e (mt == timestamp_nonexistent);
- for (prerequisite_target& p: t.prerequisite_targets[a])
+ for (prerequisite_target& p: pts)
{
if (p == nullptr)
continue;
@@ -318,56 +681,8 @@ namespace build2
// As part of this loop calculate checksums that need to include ad
// hoc prerequisites (unless the script tracks changes itself).
//
- if (script.depdb_clear)
- continue;
-
- hash_target (prq_cs, pt);
-
- // The script can reference a program in one of four ways:
- //
- // 1. As an (imported) target (e.g., $cli)
- //
- // 2. As a process_path_ex (e.g., $cxx.path).
- //
- // 3. As a builtin (e.g., sed)
- //
- // 4. As a program path/name.
- //
- // When it comes to change tracking, there is nothing we can do for
- // (4) (the user can track its environment manually with depdb-env)
- // and there is nothing to do for (3) (assuming builtin semantics is
- // stable/backwards-compatible). The (2) case is handled automatically
- // by hashing all the variable values referenced by the script (see
- // below), which in case of process_path_ex includes the checksums
- // (both executable and environment), if available.
- //
- // This leaves the (1) case, which itself splits into two sub-cases:
- // the target comes with the dependency information (e.g., imported
- // from a project via an export stub) or it does not (e.g., imported
- // as installed). We don't need to do anything extra for the first
- // sub-case since the target's state/mtime can be relied upon like any
- // other prerequisite. Which cannot be said about the second sub-case,
- // where we reply on checksum that may be included as part of the
- // target metadata.
- //
- // So what we are going to do is hash checksum metadata of every
- // executable prerequisite target that has it (we do it here in order
- // to include ad hoc prerequisites, which feels like the right thing
- // to do; the user may mark tools as ad hoc in order to omit them from
- // $<).
- //
- if (auto* et = pt.is_a<exe> ())
- {
- if (auto* c = et->lookup_metadata<string> ("checksum"))
- {
- exe_cs.append (*c);
- }
-
- if (auto* e = et->lookup_metadata<strings> ("environment"))
- {
- hash_environment (env_cs, *e);
- }
- }
+ if (!script.depdb_clear)
+ hash_prerequisite_target (prq_cs, exe_cs, env_cs, pt, storage);
}
if (!e)
@@ -379,6 +694,8 @@ namespace build2
// We use depdb to track changes to the script itself, input/output file
// names, tools, etc.
//
+ // NOTE: see the "dynamic dependencies" version above.
+ //
depdb dd (tp + ".d");
// First should come the rule name/version.
@@ -411,76 +728,53 @@ namespace build2
l4 ([&]{trace << "recipe text change forcing update of " << t;});
// Track the variables, targets, and prerequisites changes, unless the
- // script doesn't track the dependency changes itself.
- //
-
- // For each variable hash its name, undefined/null/non-null indicator,
- // and the value if non-null.
- //
- // Note that this excludes the special $< and $> variables which we
- // handle below.
- //
- // @@ TODO: maybe detect and decompose process_path_ex in order to
- // properly attribute checksum and environment changes?
+ // script tracks the dependency changes itself.
//
if (!script.depdb_clear)
{
- sha256 cs;
- names storage;
-
- for (const string& n: script.vars)
+ // For each variable hash its name, undefined/null/non-null indicator,
+ // and the value if non-null.
+ //
+ // Note that this excludes the special $< and $> variables which we
+ // handle below.
+ //
+ // @@ TODO: maybe detect and decompose process_path_ex in order to
+ // properly attribute checksum and environment changes?
+ //
{
- cs.append (n);
-
- lookup l;
-
- if (const variable* var = ctx.var_pool.find (n))
- l = t[var];
-
- cs.append (!l.defined () ? '\x1' : l->null ? '\x2' : '\x3');
+ sha256 cs;
+ hash_script_vars (cs, script, t, storage);
- if (l)
- {
- storage.clear ();
- names_view ns (reverse (*l, storage));
-
- for (const name& n: ns)
- to_checksum (cs, n);
- }
+ if (dd.expect (cs.string ()) != nullptr)
+ l4 ([&]{trace << "recipe variable change forcing update of " << t;});
}
- if (dd.expect (cs.string ()) != nullptr)
- l4 ([&]{trace << "recipe variable change forcing update of " << t;});
- }
-
- // Target and prerequisite sets ($> and $<).
- //
- if (!script.depdb_clear)
- {
- sha256 tcs;
- for (const target* m (&t); m != nullptr; m = m->adhoc_member)
- hash_target (tcs, *m);
+ // Target and prerequisite sets ($> and $<).
+ //
+ {
+ sha256 tcs;
+ for (const target* m (&t); m != nullptr; m = m->adhoc_member)
+ hash_target (tcs, *m, storage);
- if (dd.expect (tcs.string ()) != nullptr)
- l4 ([&]{trace << "target set change forcing update of " << t;});
+ if (dd.expect (tcs.string ()) != nullptr)
+ l4 ([&]{trace << "target set change forcing update of " << t;});
- if (dd.expect (prq_cs.string ()) != nullptr)
- l4 ([&]{trace << "prerequisite set change forcing update of " << t;});
- }
+ if (dd.expect (prq_cs.string ()) != nullptr)
+ l4 ([&]{trace << "prerequisite set change forcing update of " << t;});
+ }
- // Finally the programs and environment checksums.
- //
- if (!script.depdb_clear)
- {
- if (dd.expect (exe_cs.string ()) != nullptr)
- l4 ([&]{trace << "program checksum change forcing update of " << t;});
+ // Finally the programs and environment checksums.
+ //
+ {
+ if (dd.expect (exe_cs.string ()) != nullptr)
+ l4 ([&]{trace << "program checksum change forcing update of " << t;});
- if (dd.expect (env_cs.string ()) != nullptr)
- l4 ([&]{trace << "environment change forcing update of " << t;});
+ if (dd.expect (env_cs.string ()) != nullptr)
+ l4 ([&]{trace << "environment change forcing update of " << t;});
+ }
}
const scope* bs (nullptr);
- const scope* rs (nullptr);
// Execute the custom dependency change tracking commands, if present.
//
@@ -507,20 +801,19 @@ namespace build2
}
build::script::environment env (a, t, false /* temp_dir */);
- build::script::default_runner r;
+ build::script::default_runner run;
if (depdb_preamble)
{
bs = &t.base_scope ();
- rs = bs->root_scope ();
if (script.depdb_preamble_temp_dir)
env.set_temp_dir_variable ();
build::script::parser p (ctx);
- r.enter (env, script.start_loc);
- p.execute_depdb_preamble (*rs, *bs, env, script, r, dd);
+ run.enter (env, script.start_loc);
+ p.execute_depdb_preamble (a, *bs, t, env, script, run, dd);
}
// Update if depdb mismatch.
@@ -539,104 +832,124 @@ namespace build2
// below).
//
if (depdb_preamble)
- r.leave (env, script.end_loc);
+ run.leave (env, script.end_loc);
return *ps;
}
if (!ctx.dry_run || verb != 0)
{
- // Prepare to executing the script diag line and/or body.
- //
- // Note that it doesn't make much sense to use the temporary directory
- // variable ($~) in the 'diag' builtin call, so we postpone setting it
- // until the script body execution, that can potentially be omitted.
+ // Prepare to execute the script diag line and/or body.
//
if (bs == nullptr)
- {
bs = &t.base_scope ();
- rs = bs->root_scope ();
+
+ if (execute_update_file (*bs, a, t, env, run))
+ {
+ if (!ctx.dry_run)
+ dd.check_mtime (tp);
}
+ else if (depdb_preamble)
+ run.leave (env, script.end_loc);
+ }
+ else if (depdb_preamble)
+ run.leave (env, script.end_loc);
- build::script::parser p (ctx);
+ t.mtime (system_clock::now ());
+ return target_state::changed;
+ }
- if (verb == 1)
+ bool adhoc_buildscript_rule::
+ execute_update_file (const scope& bs,
+ action, const file& t,
+ build::script::environment& env,
+ build::script::default_runner& run,
+ bool deferred_failure) const
+ {
+ context& ctx (t.ctx);
+
+ const scope& rs (*bs.root_scope ());
+
+ // Note that it doesn't make much sense to use the temporary directory
+ // variable ($~) in the 'diag' builtin call, so we postpone setting it
+ // until the script body execution, that can potentially be omitted.
+ //
+ build::script::parser p (ctx);
+
+ if (verb == 1)
+ {
+ if (script.diag_line)
{
- if (script.diag_line)
- {
- text << p.execute_special (*rs, *bs, env, *script.diag_line);
- }
- else
- {
- // @@ TODO (and below):
- //
- // - we are printing target, not source (like in most other places)
- //
- // - printing of ad hoc target group (the {hxx cxx}{foo} idea)
- //
- // - if we are printing prerequisites, should we print all of them
- // (including tools)?
- //
- text << *script.diag_name << ' ' << t;
- }
+ text << p.execute_special (rs, bs, env, *script.diag_line);
}
-
- if (!ctx.dry_run || verb >= 2)
+ else
{
- // On failure remove the target files that may potentially exist but
- // be invalid.
+ // @@ TODO (and in default_action() below):
//
- small_vector<auto_rmfile, 8> rms;
+ // - we are printing target, not source (like in most other places)
+ //
+ // - printing of ad hoc target group (the {hxx cxx}{foo} idea)
+ //
+ // - if we are printing prerequisites, should we print all of them
+ // (including tools)?
+ //
+ text << *script.diag_name << ' ' << t;
+ }
+ }
- if (!ctx.dry_run)
+ if (!ctx.dry_run || verb >= 2)
+ {
+ // On failure remove the target files that may potentially exist but
+ // be invalid.
+ //
+ small_vector<auto_rmfile, 8> rms;
+
+ if (!ctx.dry_run)
+ {
+ for (const target* m (&t); m != nullptr; m = m->adhoc_member)
{
- for (const target* m (&t); m != nullptr; m = m->adhoc_member)
- {
- if (auto* f = m->is_a<file> ())
- rms.emplace_back (f->path ());
- }
+ if (auto* f = m->is_a<file> ())
+ rms.emplace_back (f->path ());
}
+ }
- if (script.body_temp_dir && !script.depdb_preamble_temp_dir)
- env.set_temp_dir_variable ();
+ if (script.body_temp_dir && !script.depdb_preamble_temp_dir)
+ env.set_temp_dir_variable ();
- p.execute_body (*rs, *bs, env, script, r, !depdb_preamble);
+ p.execute_body (rs, bs, env, script, run, script.depdb_preamble.empty ());
- if (!ctx.dry_run)
- {
- // If this is an executable, let's be helpful to the user and set
- // the executable bit on POSIX.
- //
-#ifndef _WIN32
- auto chmod = [] (const path& p)
- {
- path_perms (p,
- (path_perms (p) |
- permissions::xu |
- permissions::xg |
- permissions::xo));
- };
-
- for (const target* m (&t); m != nullptr; m = m->adhoc_member)
- {
- if (auto* p = m->is_a<exe> ())
- chmod (p->path ());
- }
-#endif
- dd.check_mtime (tp);
+ if (!ctx.dry_run)
+ {
+ if (deferred_failure)
+ fail << "expected error exit status from recipe body";
- for (auto& rm: rms)
- rm.cancel ();
+ // If this is an executable, let's be helpful to the user and set
+ // the executable bit on POSIX.
+ //
+#ifndef _WIN32
+ auto chmod = [] (const path& p)
+ {
+ path_perms (p,
+ (path_perms (p) |
+ permissions::xu |
+ permissions::xg |
+ permissions::xo));
+ };
+
+ for (const target* m (&t); m != nullptr; m = m->adhoc_member)
+ {
+ if (auto* p = m->is_a<exe> ())
+ chmod (p->path ());
}
+#endif
+ for (auto& rm: rms)
+ rm.cancel ();
}
- else if (depdb_preamble)
- r.leave (env, script.end_loc);
- }
- else if (depdb_preamble)
- r.leave (env, script.end_loc);
- t.mtime (system_clock::now ());
- return target_state::changed;
+ return true;
+ }
+ else
+ return false;
}
target_state adhoc_buildscript_rule::
@@ -666,7 +979,7 @@ namespace build2
}
else
{
- // @@ TODO: as above
+ // @@ TODO: as above (execute_update_file()).
//
text << *script.diag_name << ' ' << t;
}
diff --git a/libbuild2/adhoc-rule-buildscript.hxx b/libbuild2/adhoc-rule-buildscript.hxx
index 7f9c10a..51d37d4 100644
--- a/libbuild2/adhoc-rule-buildscript.hxx
+++ b/libbuild2/adhoc-rule-buildscript.hxx
@@ -38,6 +38,18 @@ namespace build2
target_state
perform_update_file (action, const target&) const;
+ struct match_data;
+
+ target_state
+ perform_update_file_dyndep (action, const target&, match_data&) const;
+
+ bool
+ execute_update_file (const scope&,
+ action a, const file&,
+ build::script::environment&,
+ build::script::default_runner&,
+ bool deferred_failure = false) const;
+
target_state
default_action (action, const target&, const optional<timestamp>&) const;
diff --git a/libbuild2/build/script/builtin-options.cxx b/libbuild2/build/script/builtin-options.cxx
new file mode 100644
index 0000000..cf99b12
--- /dev/null
+++ b/libbuild2/build/script/builtin-options.cxx
@@ -0,0 +1,701 @@
+// -*- C++ -*-
+//
+// This file was generated by CLI, a command line interface
+// compiler for C++.
+//
+
+// Begin prologue.
+//
+#include <libbuild2/build/script/types-parsers.hxx>
+//
+// End prologue.
+
+#include <libbuild2/build/script/builtin-options.hxx>
+
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+#include <utility>
+#include <ostream>
+#include <sstream>
+
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ namespace cli
+ {
+ // unknown_option
+ //
+ unknown_option::
+ ~unknown_option () throw ()
+ {
+ }
+
+ void unknown_option::
+ print (::std::ostream& os) const
+ {
+ os << "unknown option '" << option ().c_str () << "'";
+ }
+
+ const char* unknown_option::
+ what () const throw ()
+ {
+ return "unknown option";
+ }
+
+ // unknown_argument
+ //
+ unknown_argument::
+ ~unknown_argument () throw ()
+ {
+ }
+
+ void unknown_argument::
+ print (::std::ostream& os) const
+ {
+ os << "unknown argument '" << argument ().c_str () << "'";
+ }
+
+ const char* unknown_argument::
+ what () const throw ()
+ {
+ return "unknown argument";
+ }
+
+ // missing_value
+ //
+ missing_value::
+ ~missing_value () throw ()
+ {
+ }
+
+ void missing_value::
+ print (::std::ostream& os) const
+ {
+ os << "missing value for option '" << option ().c_str () << "'";
+ }
+
+ const char* missing_value::
+ what () const throw ()
+ {
+ return "missing option value";
+ }
+
+ // invalid_value
+ //
+ invalid_value::
+ ~invalid_value () throw ()
+ {
+ }
+
+ void invalid_value::
+ print (::std::ostream& os) const
+ {
+ os << "invalid value '" << value ().c_str () << "' for option '"
+ << option ().c_str () << "'";
+
+ if (!message ().empty ())
+ os << ": " << message ().c_str ();
+ }
+
+ const char* invalid_value::
+ what () const throw ()
+ {
+ return "invalid option value";
+ }
+
+ // eos_reached
+ //
+ void eos_reached::
+ print (::std::ostream& os) const
+ {
+ os << what ();
+ }
+
+ const char* eos_reached::
+ what () const throw ()
+ {
+ return "end of argument stream reached";
+ }
+
+ // scanner
+ //
+ scanner::
+ ~scanner ()
+ {
+ }
+
+ // argv_scanner
+ //
+ bool argv_scanner::
+ more ()
+ {
+ return i_ < argc_;
+ }
+
+ const char* argv_scanner::
+ peek ()
+ {
+ if (i_ < argc_)
+ return argv_[i_];
+ else
+ throw eos_reached ();
+ }
+
+ const char* argv_scanner::
+ next ()
+ {
+ if (i_ < argc_)
+ {
+ const char* r (argv_[i_]);
+
+ if (erase_)
+ {
+ for (int i (i_ + 1); i < argc_; ++i)
+ argv_[i - 1] = argv_[i];
+
+ --argc_;
+ argv_[argc_] = 0;
+ }
+ else
+ ++i_;
+
+ ++start_position_;
+ return r;
+ }
+ else
+ throw eos_reached ();
+ }
+
+ void argv_scanner::
+ skip ()
+ {
+ if (i_ < argc_)
+ {
+ ++i_;
+ ++start_position_;
+ }
+ else
+ throw eos_reached ();
+ }
+
+ std::size_t argv_scanner::
+ position ()
+ {
+ return start_position_;
+ }
+
+ // vector_scanner
+ //
+ bool vector_scanner::
+ more ()
+ {
+ return i_ < v_.size ();
+ }
+
+ const char* vector_scanner::
+ peek ()
+ {
+ if (i_ < v_.size ())
+ return v_[i_].c_str ();
+ else
+ throw eos_reached ();
+ }
+
+ const char* vector_scanner::
+ next ()
+ {
+ if (i_ < v_.size ())
+ return v_[i_++].c_str ();
+ else
+ throw eos_reached ();
+ }
+
+ void vector_scanner::
+ skip ()
+ {
+ if (i_ < v_.size ())
+ ++i_;
+ else
+ throw eos_reached ();
+ }
+
+ std::size_t vector_scanner::
+ position ()
+ {
+ return start_position_ + i_;
+ }
+
+ template <typename X>
+ struct parser
+ {
+ static void
+ parse (X& x, bool& xs, scanner& s)
+ {
+ using namespace std;
+
+ const char* o (s.next ());
+ if (s.more ())
+ {
+ string v (s.next ());
+ istringstream is (v);
+ if (!(is >> x && is.peek () == istringstream::traits_type::eof ()))
+ throw invalid_value (o, v);
+ }
+ else
+ throw missing_value (o);
+
+ xs = true;
+ }
+ };
+
+ template <>
+ struct parser<bool>
+ {
+ static void
+ parse (bool& x, scanner& s)
+ {
+ s.next ();
+ x = true;
+ }
+ };
+
+ template <>
+ struct parser<std::string>
+ {
+ static void
+ parse (std::string& x, bool& xs, scanner& s)
+ {
+ const char* o (s.next ());
+
+ if (s.more ())
+ x = s.next ();
+ else
+ throw missing_value (o);
+
+ xs = true;
+ }
+ };
+
+ template <typename X>
+ struct parser<std::pair<X, std::size_t> >
+ {
+ static void
+ parse (std::pair<X, std::size_t>& x, bool& xs, scanner& s)
+ {
+ x.second = s.position ();
+ parser<X>::parse (x.first, xs, s);
+ }
+ };
+
+ template <typename X>
+ struct parser<std::vector<X> >
+ {
+ static void
+ parse (std::vector<X>& c, bool& xs, scanner& s)
+ {
+ X x;
+ bool dummy;
+ parser<X>::parse (x, dummy, s);
+ c.push_back (x);
+ xs = true;
+ }
+ };
+
+ template <typename X, typename C>
+ struct parser<std::set<X, C> >
+ {
+ static void
+ parse (std::set<X, C>& c, bool& xs, scanner& s)
+ {
+ X x;
+ bool dummy;
+ parser<X>::parse (x, dummy, s);
+ c.insert (x);
+ xs = true;
+ }
+ };
+
+ template <typename K, typename V, typename C>
+ struct parser<std::map<K, V, C> >
+ {
+ static void
+ parse (std::map<K, V, C>& m, bool& xs, scanner& s)
+ {
+ const char* o (s.next ());
+
+ if (s.more ())
+ {
+ std::size_t pos (s.position ());
+ std::string ov (s.next ());
+ std::string::size_type p = ov.find ('=');
+
+ K k = K ();
+ V v = V ();
+ std::string kstr (ov, 0, p);
+ std::string vstr (ov, (p != std::string::npos ? p + 1 : ov.size ()));
+
+ int ac (2);
+ char* av[] =
+ {
+ const_cast<char*> (o),
+ 0
+ };
+
+ bool dummy;
+ if (!kstr.empty ())
+ {
+ av[1] = const_cast<char*> (kstr.c_str ());
+ argv_scanner s (0, ac, av, false, pos);
+ parser<K>::parse (k, dummy, s);
+ }
+
+ if (!vstr.empty ())
+ {
+ av[1] = const_cast<char*> (vstr.c_str ());
+ argv_scanner s (0, ac, av, false, pos);
+ parser<V>::parse (v, dummy, s);
+ }
+
+ m[k] = v;
+ }
+ else
+ throw missing_value (o);
+
+ xs = true;
+ }
+ };
+
+ template <typename X, typename T, T X::*M>
+ void
+ thunk (X& x, scanner& s)
+ {
+ parser<T>::parse (x.*M, s);
+ }
+
+ template <typename X, typename T, T X::*M, bool X::*S>
+ void
+ thunk (X& x, scanner& s)
+ {
+ parser<T>::parse (x.*M, x.*S, s);
+ }
+ }
+ }
+ }
+}
+
+#include <map>
+#include <cstring>
+
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ // depdb_dep_options
+ //
+
+ depdb_dep_options::
+ depdb_dep_options ()
+ : file_ (),
+ file_specified_ (false),
+ format_ (),
+ format_specified_ (false),
+ what_ (),
+ what_specified_ (false),
+ include_path_ (),
+ include_path_specified_ (false),
+ default_prereq_type_ (),
+ default_prereq_type_specified_ (false)
+ {
+ }
+
+ bool depdb_dep_options::
+ parse (int& argc,
+ char** argv,
+ bool erase,
+ ::build2::build::script::cli::unknown_mode opt,
+ ::build2::build::script::cli::unknown_mode arg)
+ {
+ ::build2::build::script::cli::argv_scanner s (argc, argv, erase);
+ bool r = _parse (s, opt, arg);
+ return r;
+ }
+
+ bool depdb_dep_options::
+ parse (int start,
+ int& argc,
+ char** argv,
+ bool erase,
+ ::build2::build::script::cli::unknown_mode opt,
+ ::build2::build::script::cli::unknown_mode arg)
+ {
+ ::build2::build::script::cli::argv_scanner s (start, argc, argv, erase);
+ bool r = _parse (s, opt, arg);
+ return r;
+ }
+
+ bool depdb_dep_options::
+ parse (int& argc,
+ char** argv,
+ int& end,
+ bool erase,
+ ::build2::build::script::cli::unknown_mode opt,
+ ::build2::build::script::cli::unknown_mode arg)
+ {
+ ::build2::build::script::cli::argv_scanner s (argc, argv, erase);
+ bool r = _parse (s, opt, arg);
+ end = s.end ();
+ return r;
+ }
+
+ bool depdb_dep_options::
+ parse (int start,
+ int& argc,
+ char** argv,
+ int& end,
+ bool erase,
+ ::build2::build::script::cli::unknown_mode opt,
+ ::build2::build::script::cli::unknown_mode arg)
+ {
+ ::build2::build::script::cli::argv_scanner s (start, argc, argv, erase);
+ bool r = _parse (s, opt, arg);
+ end = s.end ();
+ return r;
+ }
+
+ bool depdb_dep_options::
+ parse (::build2::build::script::cli::scanner& s,
+ ::build2::build::script::cli::unknown_mode opt,
+ ::build2::build::script::cli::unknown_mode arg)
+ {
+ bool r = _parse (s, opt, arg);
+ return r;
+ }
+
+ typedef
+ std::map<std::string, void (*) (depdb_dep_options&, ::build2::build::script::cli::scanner&)>
+ _cli_depdb_dep_options_map;
+
+ static _cli_depdb_dep_options_map _cli_depdb_dep_options_map_;
+
+ struct _cli_depdb_dep_options_map_init
+ {
+ _cli_depdb_dep_options_map_init ()
+ {
+ _cli_depdb_dep_options_map_["--file"] =
+ &::build2::build::script::cli::thunk< depdb_dep_options, path, &depdb_dep_options::file_,
+ &depdb_dep_options::file_specified_ >;
+ _cli_depdb_dep_options_map_["--format"] =
+ &::build2::build::script::cli::thunk< depdb_dep_options, string, &depdb_dep_options::format_,
+ &depdb_dep_options::format_specified_ >;
+ _cli_depdb_dep_options_map_["--what"] =
+ &::build2::build::script::cli::thunk< depdb_dep_options, string, &depdb_dep_options::what_,
+ &depdb_dep_options::what_specified_ >;
+ _cli_depdb_dep_options_map_["--include-path"] =
+ &::build2::build::script::cli::thunk< depdb_dep_options, dir_paths, &depdb_dep_options::include_path_,
+ &depdb_dep_options::include_path_specified_ >;
+ _cli_depdb_dep_options_map_["-I"] =
+ &::build2::build::script::cli::thunk< depdb_dep_options, dir_paths, &depdb_dep_options::include_path_,
+ &depdb_dep_options::include_path_specified_ >;
+ _cli_depdb_dep_options_map_["--default-prereq-type"] =
+ &::build2::build::script::cli::thunk< depdb_dep_options, string, &depdb_dep_options::default_prereq_type_,
+ &depdb_dep_options::default_prereq_type_specified_ >;
+ }
+ };
+
+ static _cli_depdb_dep_options_map_init _cli_depdb_dep_options_map_init_;
+
+ bool depdb_dep_options::
+ _parse (const char* o, ::build2::build::script::cli::scanner& s)
+ {
+ _cli_depdb_dep_options_map::const_iterator i (_cli_depdb_dep_options_map_.find (o));
+
+ if (i != _cli_depdb_dep_options_map_.end ())
+ {
+ (*(i->second)) (*this, s);
+ return true;
+ }
+
+ return false;
+ }
+
+ bool depdb_dep_options::
+ _parse (::build2::build::script::cli::scanner& s,
+ ::build2::build::script::cli::unknown_mode opt_mode,
+ ::build2::build::script::cli::unknown_mode arg_mode)
+ {
+ // Can't skip combined flags (--no-combined-flags).
+ //
+ assert (opt_mode != ::build2::build::script::cli::unknown_mode::skip);
+
+ bool r = false;
+ bool opt = true;
+
+ while (s.more ())
+ {
+ const char* o = s.peek ();
+
+ if (std::strcmp (o, "--") == 0)
+ {
+ opt = false;
+ s.skip ();
+ r = true;
+ continue;
+ }
+
+ if (opt)
+ {
+ if (_parse (o, s))
+ {
+ r = true;
+ continue;
+ }
+
+ if (std::strncmp (o, "-", 1) == 0 && o[1] != '\0')
+ {
+ // Handle combined option values.
+ //
+ std::string co;
+ if (const char* v = std::strchr (o, '='))
+ {
+ co.assign (o, 0, v - o);
+ ++v;
+
+ int ac (2);
+ char* av[] =
+ {
+ const_cast<char*> (co.c_str ()),
+ const_cast<char*> (v)
+ };
+
+ ::build2::build::script::cli::argv_scanner ns (0, ac, av);
+
+ if (_parse (co.c_str (), ns))
+ {
+ // Parsed the option but not its value?
+ //
+ if (ns.end () != 2)
+ throw ::build2::build::script::cli::invalid_value (co, v);
+
+ s.next ();
+ r = true;
+ continue;
+ }
+ else
+ {
+ // Set the unknown option and fall through.
+ //
+ o = co.c_str ();
+ }
+ }
+
+ // Handle combined flags.
+ //
+ char cf[3];
+ {
+ const char* p = o + 1;
+ for (; *p != '\0'; ++p)
+ {
+ if (!((*p >= 'a' && *p <= 'z') ||
+ (*p >= 'A' && *p <= 'Z') ||
+ (*p >= '0' && *p <= '9')))
+ break;
+ }
+
+ if (*p == '\0')
+ {
+ for (p = o + 1; *p != '\0'; ++p)
+ {
+ std::strcpy (cf, "-");
+ cf[1] = *p;
+ cf[2] = '\0';
+
+ int ac (1);
+ char* av[] =
+ {
+ cf
+ };
+
+ ::build2::build::script::cli::argv_scanner ns (0, ac, av);
+
+ if (!_parse (cf, ns))
+ break;
+ }
+
+ if (*p == '\0')
+ {
+ // All handled.
+ //
+ s.next ();
+ r = true;
+ continue;
+ }
+ else
+ {
+ // Set the unknown option and fall through.
+ //
+ o = cf;
+ }
+ }
+ }
+
+ switch (opt_mode)
+ {
+ case ::build2::build::script::cli::unknown_mode::skip:
+ {
+ s.skip ();
+ r = true;
+ continue;
+ }
+ case ::build2::build::script::cli::unknown_mode::stop:
+ {
+ break;
+ }
+ case ::build2::build::script::cli::unknown_mode::fail:
+ {
+ throw ::build2::build::script::cli::unknown_option (o);
+ }
+ }
+
+ break;
+ }
+ }
+
+ switch (arg_mode)
+ {
+ case ::build2::build::script::cli::unknown_mode::skip:
+ {
+ s.skip ();
+ r = true;
+ continue;
+ }
+ case ::build2::build::script::cli::unknown_mode::stop:
+ {
+ break;
+ }
+ case ::build2::build::script::cli::unknown_mode::fail:
+ {
+ throw ::build2::build::script::cli::unknown_argument (o);
+ }
+ }
+
+ break;
+ }
+
+ return r;
+ }
+ }
+ }
+}
+
+// Begin epilogue.
+//
+//
+// End epilogue.
+
diff --git a/libbuild2/build/script/builtin-options.hxx b/libbuild2/build/script/builtin-options.hxx
new file mode 100644
index 0000000..85d67b9
--- /dev/null
+++ b/libbuild2/build/script/builtin-options.hxx
@@ -0,0 +1,456 @@
+// -*- C++ -*-
+//
+// This file was generated by CLI, a command line interface
+// compiler for C++.
+//
+
+#ifndef LIBBUILD2_BUILD_SCRIPT_BUILTIN_OPTIONS_HXX
+#define LIBBUILD2_BUILD_SCRIPT_BUILTIN_OPTIONS_HXX
+
+// Begin prologue.
+//
+//
+// End prologue.
+
+#include <vector>
+#include <iosfwd>
+#include <string>
+#include <cstddef>
+#include <exception>
+
+#ifndef CLI_POTENTIALLY_UNUSED
+# if defined(_MSC_VER) || defined(__xlC__)
+# define CLI_POTENTIALLY_UNUSED(x) (void*)&x
+# else
+# define CLI_POTENTIALLY_UNUSED(x) (void)x
+# endif
+#endif
+
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ namespace cli
+ {
+ class unknown_mode
+ {
+ public:
+ enum value
+ {
+ skip,
+ stop,
+ fail
+ };
+
+ unknown_mode (value);
+
+ operator value () const
+ {
+ return v_;
+ }
+
+ private:
+ value v_;
+ };
+
+ // Exceptions.
+ //
+
+ class exception: public std::exception
+ {
+ public:
+ virtual void
+ print (::std::ostream&) const = 0;
+ };
+
+ ::std::ostream&
+ operator<< (::std::ostream&, const exception&);
+
+ class unknown_option: public exception
+ {
+ public:
+ virtual
+ ~unknown_option () throw ();
+
+ unknown_option (const std::string& option);
+
+ const std::string&
+ option () const;
+
+ virtual void
+ print (::std::ostream&) const;
+
+ virtual const char*
+ what () const throw ();
+
+ private:
+ std::string option_;
+ };
+
+ class unknown_argument: public exception
+ {
+ public:
+ virtual
+ ~unknown_argument () throw ();
+
+ unknown_argument (const std::string& argument);
+
+ const std::string&
+ argument () const;
+
+ virtual void
+ print (::std::ostream&) const;
+
+ virtual const char*
+ what () const throw ();
+
+ private:
+ std::string argument_;
+ };
+
+ class missing_value: public exception
+ {
+ public:
+ virtual
+ ~missing_value () throw ();
+
+ missing_value (const std::string& option);
+
+ const std::string&
+ option () const;
+
+ virtual void
+ print (::std::ostream&) const;
+
+ virtual const char*
+ what () const throw ();
+
+ private:
+ std::string option_;
+ };
+
+ class invalid_value: public exception
+ {
+ public:
+ virtual
+ ~invalid_value () throw ();
+
+ invalid_value (const std::string& option,
+ const std::string& value,
+ const std::string& message = std::string ());
+
+ const std::string&
+ option () const;
+
+ const std::string&
+ value () const;
+
+ const std::string&
+ message () const;
+
+ virtual void
+ print (::std::ostream&) const;
+
+ virtual const char*
+ what () const throw ();
+
+ private:
+ std::string option_;
+ std::string value_;
+ std::string message_;
+ };
+
+ class eos_reached: public exception
+ {
+ public:
+ virtual void
+ print (::std::ostream&) const;
+
+ virtual const char*
+ what () const throw ();
+ };
+
+ // Command line argument scanner interface.
+ //
+ // The values returned by next() are guaranteed to be valid
+ // for the two previous arguments up until a call to a third
+ // peek() or next().
+ //
+ // The position() function returns a monotonically-increasing
+ // number which, if stored, can later be used to determine the
+ // relative position of the argument returned by the following
+ // call to next(). Note that if multiple scanners are used to
+ // extract arguments from multiple sources, then the end
+ // position of the previous scanner should be used as the
+ // start position of the next.
+ //
+ class scanner
+ {
+ public:
+ virtual
+ ~scanner ();
+
+ virtual bool
+ more () = 0;
+
+ virtual const char*
+ peek () = 0;
+
+ virtual const char*
+ next () = 0;
+
+ virtual void
+ skip () = 0;
+
+ virtual std::size_t
+ position () = 0;
+ };
+
+ class argv_scanner: public scanner
+ {
+ public:
+ argv_scanner (int& argc,
+ char** argv,
+ bool erase = false,
+ std::size_t start_position = 0);
+
+ argv_scanner (int start,
+ int& argc,
+ char** argv,
+ bool erase = false,
+ std::size_t start_position = 0);
+
+ int
+ end () const;
+
+ virtual bool
+ more ();
+
+ virtual const char*
+ peek ();
+
+ virtual const char*
+ next ();
+
+ virtual void
+ skip ();
+
+ virtual std::size_t
+ position ();
+
+ protected:
+ std::size_t start_position_;
+ int i_;
+ int& argc_;
+ char** argv_;
+ bool erase_;
+ };
+
+ class vector_scanner: public scanner
+ {
+ public:
+ vector_scanner (const std::vector<std::string>&,
+ std::size_t start = 0,
+ std::size_t start_position = 0);
+
+ std::size_t
+ end () const;
+
+ void
+ reset (std::size_t start = 0, std::size_t start_position = 0);
+
+ virtual bool
+ more ();
+
+ virtual const char*
+ peek ();
+
+ virtual const char*
+ next ();
+
+ virtual void
+ skip ();
+
+ virtual std::size_t
+ position ();
+
+ private:
+ std::size_t start_position_;
+ const std::vector<std::string>& v_;
+ std::size_t i_;
+ };
+
+ template <typename X>
+ struct parser;
+ }
+ }
+ }
+}
+
+#include <libbuild2/types.hxx>
+
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ class depdb_dep_options
+ {
+ public:
+ depdb_dep_options ();
+
+ // Return true if anything has been parsed.
+ //
+ bool
+ parse (int& argc,
+ char** argv,
+ bool erase = false,
+ ::build2::build::script::cli::unknown_mode option = ::build2::build::script::cli::unknown_mode::fail,
+ ::build2::build::script::cli::unknown_mode argument = ::build2::build::script::cli::unknown_mode::stop);
+
+ bool
+ parse (int start,
+ int& argc,
+ char** argv,
+ bool erase = false,
+ ::build2::build::script::cli::unknown_mode option = ::build2::build::script::cli::unknown_mode::fail,
+ ::build2::build::script::cli::unknown_mode argument = ::build2::build::script::cli::unknown_mode::stop);
+
+ bool
+ parse (int& argc,
+ char** argv,
+ int& end,
+ bool erase = false,
+ ::build2::build::script::cli::unknown_mode option = ::build2::build::script::cli::unknown_mode::fail,
+ ::build2::build::script::cli::unknown_mode argument = ::build2::build::script::cli::unknown_mode::stop);
+
+ bool
+ parse (int start,
+ int& argc,
+ char** argv,
+ int& end,
+ bool erase = false,
+ ::build2::build::script::cli::unknown_mode option = ::build2::build::script::cli::unknown_mode::fail,
+ ::build2::build::script::cli::unknown_mode argument = ::build2::build::script::cli::unknown_mode::stop);
+
+ bool
+ parse (::build2::build::script::cli::scanner&,
+ ::build2::build::script::cli::unknown_mode option = ::build2::build::script::cli::unknown_mode::fail,
+ ::build2::build::script::cli::unknown_mode argument = ::build2::build::script::cli::unknown_mode::stop);
+
+ // Option accessors and modifiers.
+ //
+ const path&
+ file () const;
+
+ path&
+ file ();
+
+ void
+ file (const path&);
+
+ bool
+ file_specified () const;
+
+ void
+ file_specified (bool);
+
+ const string&
+ format () const;
+
+ string&
+ format ();
+
+ void
+ format (const string&);
+
+ bool
+ format_specified () const;
+
+ void
+ format_specified (bool);
+
+ const string&
+ what () const;
+
+ string&
+ what ();
+
+ void
+ what (const string&);
+
+ bool
+ what_specified () const;
+
+ void
+ what_specified (bool);
+
+ const dir_paths&
+ include_path () const;
+
+ dir_paths&
+ include_path ();
+
+ void
+ include_path (const dir_paths&);
+
+ bool
+ include_path_specified () const;
+
+ void
+ include_path_specified (bool);
+
+ const string&
+ default_prereq_type () const;
+
+ string&
+ default_prereq_type ();
+
+ void
+ default_prereq_type (const string&);
+
+ bool
+ default_prereq_type_specified () const;
+
+ void
+ default_prereq_type_specified (bool);
+
+ // Implementation details.
+ //
+ protected:
+ bool
+ _parse (const char*, ::build2::build::script::cli::scanner&);
+
+ private:
+ bool
+ _parse (::build2::build::script::cli::scanner&,
+ ::build2::build::script::cli::unknown_mode option,
+ ::build2::build::script::cli::unknown_mode argument);
+
+ public:
+ path file_;
+ bool file_specified_;
+ string format_;
+ bool format_specified_;
+ string what_;
+ bool what_specified_;
+ dir_paths include_path_;
+ bool include_path_specified_;
+ string default_prereq_type_;
+ bool default_prereq_type_specified_;
+ };
+ }
+ }
+}
+
+#include <libbuild2/build/script/builtin-options.ixx>
+
+// Begin epilogue.
+//
+//
+// End epilogue.
+
+#endif // LIBBUILD2_BUILD_SCRIPT_BUILTIN_OPTIONS_HXX
diff --git a/libbuild2/build/script/builtin-options.ixx b/libbuild2/build/script/builtin-options.ixx
new file mode 100644
index 0000000..06575c8
--- /dev/null
+++ b/libbuild2/build/script/builtin-options.ixx
@@ -0,0 +1,338 @@
+// -*- C++ -*-
+//
+// This file was generated by CLI, a command line interface
+// compiler for C++.
+//
+
+// Begin prologue.
+//
+//
+// End prologue.
+
+#include <cassert>
+
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ namespace cli
+ {
+ // unknown_mode
+ //
+ inline unknown_mode::
+ unknown_mode (value v)
+ : v_ (v)
+ {
+ }
+
+ // exception
+ //
+ inline ::std::ostream&
+ operator<< (::std::ostream& os, const exception& e)
+ {
+ e.print (os);
+ return os;
+ }
+
+ // unknown_option
+ //
+ inline unknown_option::
+ unknown_option (const std::string& option)
+ : option_ (option)
+ {
+ }
+
+ inline const std::string& unknown_option::
+ option () const
+ {
+ return option_;
+ }
+
+ // unknown_argument
+ //
+ inline unknown_argument::
+ unknown_argument (const std::string& argument)
+ : argument_ (argument)
+ {
+ }
+
+ inline const std::string& unknown_argument::
+ argument () const
+ {
+ return argument_;
+ }
+
+ // missing_value
+ //
+ inline missing_value::
+ missing_value (const std::string& option)
+ : option_ (option)
+ {
+ }
+
+ inline const std::string& missing_value::
+ option () const
+ {
+ return option_;
+ }
+
+ // invalid_value
+ //
+ inline invalid_value::
+ invalid_value (const std::string& option,
+ const std::string& value,
+ const std::string& message)
+ : option_ (option),
+ value_ (value),
+ message_ (message)
+ {
+ }
+
+ inline const std::string& invalid_value::
+ option () const
+ {
+ return option_;
+ }
+
+ inline const std::string& invalid_value::
+ value () const
+ {
+ return value_;
+ }
+
+ inline const std::string& invalid_value::
+ message () const
+ {
+ return message_;
+ }
+
+ // argv_scanner
+ //
+ inline argv_scanner::
+ argv_scanner (int& argc,
+ char** argv,
+ bool erase,
+ std::size_t sp)
+ : start_position_ (sp + 1),
+ i_ (1),
+ argc_ (argc),
+ argv_ (argv),
+ erase_ (erase)
+ {
+ }
+
+ inline argv_scanner::
+ argv_scanner (int start,
+ int& argc,
+ char** argv,
+ bool erase,
+ std::size_t sp)
+ : start_position_ (sp + static_cast<std::size_t> (start)),
+ i_ (start),
+ argc_ (argc),
+ argv_ (argv),
+ erase_ (erase)
+ {
+ }
+
+ inline int argv_scanner::
+ end () const
+ {
+ return i_;
+ }
+
+ // vector_scanner
+ //
+ inline vector_scanner::
+ vector_scanner (const std::vector<std::string>& v,
+ std::size_t i,
+ std::size_t sp)
+ : start_position_ (sp), v_ (v), i_ (i)
+ {
+ }
+
+ inline std::size_t vector_scanner::
+ end () const
+ {
+ return i_;
+ }
+
+ inline void vector_scanner::
+ reset (std::size_t i, std::size_t sp)
+ {
+ i_ = i;
+ start_position_ = sp;
+ }
+ }
+ }
+ }
+}
+
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ // depdb_dep_options
+ //
+
+ inline const path& depdb_dep_options::
+ file () const
+ {
+ return this->file_;
+ }
+
+ inline path& depdb_dep_options::
+ file ()
+ {
+ return this->file_;
+ }
+
+ inline void depdb_dep_options::
+ file (const path& x)
+ {
+ this->file_ = x;
+ }
+
+ inline bool depdb_dep_options::
+ file_specified () const
+ {
+ return this->file_specified_;
+ }
+
+ inline void depdb_dep_options::
+ file_specified (bool x)
+ {
+ this->file_specified_ = x;
+ }
+
+ inline const string& depdb_dep_options::
+ format () const
+ {
+ return this->format_;
+ }
+
+ inline string& depdb_dep_options::
+ format ()
+ {
+ return this->format_;
+ }
+
+ inline void depdb_dep_options::
+ format (const string& x)
+ {
+ this->format_ = x;
+ }
+
+ inline bool depdb_dep_options::
+ format_specified () const
+ {
+ return this->format_specified_;
+ }
+
+ inline void depdb_dep_options::
+ format_specified (bool x)
+ {
+ this->format_specified_ = x;
+ }
+
+ inline const string& depdb_dep_options::
+ what () const
+ {
+ return this->what_;
+ }
+
+ inline string& depdb_dep_options::
+ what ()
+ {
+ return this->what_;
+ }
+
+ inline void depdb_dep_options::
+ what (const string& x)
+ {
+ this->what_ = x;
+ }
+
+ inline bool depdb_dep_options::
+ what_specified () const
+ {
+ return this->what_specified_;
+ }
+
+ inline void depdb_dep_options::
+ what_specified (bool x)
+ {
+ this->what_specified_ = x;
+ }
+
+ inline const dir_paths& depdb_dep_options::
+ include_path () const
+ {
+ return this->include_path_;
+ }
+
+ inline dir_paths& depdb_dep_options::
+ include_path ()
+ {
+ return this->include_path_;
+ }
+
+ inline void depdb_dep_options::
+ include_path (const dir_paths& x)
+ {
+ this->include_path_ = x;
+ }
+
+ inline bool depdb_dep_options::
+ include_path_specified () const
+ {
+ return this->include_path_specified_;
+ }
+
+ inline void depdb_dep_options::
+ include_path_specified (bool x)
+ {
+ this->include_path_specified_ = x;
+ }
+
+ inline const string& depdb_dep_options::
+ default_prereq_type () const
+ {
+ return this->default_prereq_type_;
+ }
+
+ inline string& depdb_dep_options::
+ default_prereq_type ()
+ {
+ return this->default_prereq_type_;
+ }
+
+ inline void depdb_dep_options::
+ default_prereq_type (const string& x)
+ {
+ this->default_prereq_type_ = x;
+ }
+
+ inline bool depdb_dep_options::
+ default_prereq_type_specified () const
+ {
+ return this->default_prereq_type_specified_;
+ }
+
+ inline void depdb_dep_options::
+ default_prereq_type_specified (bool x)
+ {
+ this->default_prereq_type_specified_ = x;
+ }
+ }
+ }
+}
+
+// Begin epilogue.
+//
+//
+// End epilogue.
diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli
new file mode 100644
index 0000000..3ed3659
--- /dev/null
+++ b/libbuild2/build/script/builtin.cli
@@ -0,0 +1,32 @@
+// file : libbuild2/build/script/builtin.cli
+// license : MIT; see accompanying LICENSE file
+
+include <libbuild2/types.hxx>;
+
+// Note that options in this file are undocumented because we generate neither
+// the usage printing code nor man pages. Instead, they are documented in the
+// manual.
+//
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ // Pseudo-builtin options.
+ //
+ class depdb_dep_options
+ {
+ // Note that --byproduct, if any, must be the first option and is
+ // handled ad hoc, kind of as a sub-command.
+ //
+ path --file; // Read from file rather than stdin.
+ string --format; // Dependency format: make (default).
+ string --what; // Dependency kind, e.g., "header".
+ dir_paths --include-path|-I; // Search paths for generated files.
+ string --default-prereq-type; // Default prerequisite type to use
+ // if none could be derived from ext.
+ };
+ }
+ }
+}
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index 217fa11..67dbf69 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -3,13 +3,22 @@
#include <libbuild2/build/script/parser.hxx>
+#include <cstring> // strcmp()
+#include <sstream>
+
#include <libbutl/builtin.hxx>
+#include <libbuild2/depdb.hxx>
+#include <libbuild2/dyndep.hxx>
#include <libbuild2/function.hxx>
#include <libbuild2/algorithm.hxx>
+#include <libbuild2/make-parser.hxx>
+
+#include <libbuild2/script/run.hxx>
#include <libbuild2/build/script/lexer.hxx>
#include <libbuild2/build/script/runner.hxx>
+#include <libbuild2/build/script/builtin-options.hxx>
using namespace std;
using namespace butl;
@@ -125,6 +134,8 @@ namespace build2
// Save the custom dependency change tracking lines, if present.
//
s.depdb_clear = depdb_clear_.has_value ();
+ if (depdb_dyndep_)
+ s.depdb_dyndep = depdb_dyndep_->second;
s.depdb_preamble = move (depdb_preamble_);
return s;
@@ -487,7 +498,11 @@ namespace build2
next (t, tt);
if (tt != type::word ||
- (v != "clear" && v != "hash" && v != "string" && v != "env"))
+ (v != "clear" &&
+ v != "hash" &&
+ v != "string" &&
+ v != "env" &&
+ v != "dyndep"))
{
fail (get_location (t))
<< "expected 'depdb' builtin command instead of " << t;
@@ -527,12 +542,39 @@ namespace build2
// the referenced variable list, since it won't be used.
//
depdb_clear_ = l;
- save_line_ = nullptr;
+ save_line_ = nullptr;
script_->vars.clear ();
}
else
{
+ // Verify depdb-dyndep is last.
+ //
+ if (v == "dyndep")
+ {
+ // Note that for now we do not allow multiple dyndep calls.
+ // But we may wan to relax this later (though alternating
+ // targets with prerequisites in depdb may be tricky -- maybe
+ // still only allow additional targets in the first call).
+ //
+ if (!depdb_dyndep_)
+ depdb_dyndep_ = make_pair (l, depdb_preamble_.size ());
+ else
+ fail (l) << "multiple 'depdb dyndep' calls" <<
+ info (depdb_dyndep_->first) << "previous call is here";
+
+#if 0
+ if (peek () == type::word && peeked ().value == "--byproduct")
+ ;
+#endif
+ }
+ else
+ {
+ if (depdb_dyndep_)
+ fail (l) << "'depdb " << v << "' after 'depdb dyndep'" <<
+ info (depdb_dyndep_->first) << "'depdb dyndep' call is here";
+ }
+
// Move the script body to the end of the depdb preamble.
//
// Note that at this (pre-parsing) stage we cannot evaluate if
@@ -885,114 +927,140 @@ namespace build2
}
void parser::
- execute_depdb_preamble (const scope& rs, const scope& bs,
- environment& e, const script& s, runner& r,
- depdb& dd)
+ exec_depdb_preamble (action a, const scope& bs, const file& t,
+ environment& e, const script& s, runner& r,
+ lines_iterator begin, lines_iterator end,
+ depdb& dd,
+ bool* update,
+ bool* deferred_failure,
+ optional<timestamp> mt)
{
- tracer trace ("execute_depdb_preamble");
+ tracer trace ("exec_depdb_preamble");
// The only valid lines in the depdb preamble are the depdb builtin
// itself as well as the variable assignments, including via the set
// builtin.
- pre_exec (rs, bs, e, &s, &r);
+ pre_exec (*bs.root_scope (), bs, e, &s, &r);
// Let's "wrap up" the objects we operate upon into the single object
// to rely on "small function object" optimization.
//
struct
{
+ tracer& trace;
+
+ action a;
+ const scope& bs;
+ const file& t;
+
environment& env;
const script& scr;
+
depdb& dd;
- tracer& trace;
- } ctx {e, s, dd, trace};
-
- auto exec_cmd = [&ctx, this]
- (token& t,
- build2::script::token_type& tt,
- size_t li,
- bool /* single */,
- const location& ll)
+ bool* update;
+ bool* deferred_failure;
+ optional<timestamp> mt;
+
+ } data {trace, a, bs, t, e, s, dd, update, deferred_failure, mt};
+
+ auto exec_cmd = [this, &data] (token& t,
+ build2::script::token_type& tt,
+ size_t li,
+ bool /* single */,
+ const location& ll)
{
+ // Note that we never reset the line index to zero (as we do in
+ // execute_body()) assuming that there are some script body
+ // commands to follow.
+ //
if (tt == type::word && t.value == "depdb")
{
- names ns (exec_special (t, tt));
+ next (t, tt);
// This should have been enforced during pre-parsing.
//
- assert (!ns.empty ()); // <cmd> ... <newline>
+ assert (tt == type::word); // <cmd> ... <newline>
- const string& cmd (ns[0].value);
+ string cmd (move (t.value));
- if (cmd == "hash")
+ if (cmd == "dyndep")
{
- sha256 cs;
- for (auto i (ns.begin () + 1); i != ns.end (); ++i) // Skip <cmd>.
- to_checksum (cs, *i);
-
- if (ctx.dd.expect (cs.string ()) != nullptr)
- l4 ([&] {
- ctx.trace (ll)
- << "'depdb hash' argument change forcing update of "
- << ctx.env.target;});
+ // Note: cast is safe since this is always executed in apply().
+ //
+ exec_depdb_dyndep (t, tt,
+ li, ll,
+ data.a, data.bs, const_cast<file&> (data.t),
+ data.dd,
+ *data.update,
+ *data.deferred_failure,
+ *data.mt);
}
- else if (cmd == "string")
+ else
{
- string s;
- try
- {
- s = convert<string> (
- names (make_move_iterator (ns.begin () + 1),
- make_move_iterator (ns.end ())));
- }
- catch (const invalid_argument& e)
+ names ns (exec_special (t, tt, true /* skip <cmd> */));
+
+ if (cmd == "hash")
{
- fail (ll) << "invalid 'depdb string' argument: " << e;
+ sha256 cs;
+ for (const name& n: ns)
+ to_checksum (cs, n);
+
+ if (data.dd.expect (cs.string ()) != nullptr)
+ l4 ([&] {
+ data.trace (ll)
+ << "'depdb hash' argument change forcing update of "
+ << data.t;});
}
-
- if (ctx.dd.expect (s) != nullptr)
- l4 ([&] {
- ctx.trace (ll)
- << "'depdb string' argument change forcing update of "
- << ctx.env.target;});
- }
- else if (cmd == "env")
- {
- sha256 cs;
- const char* pf ("invalid 'depdb env' argument: ");
-
- try
+ else if (cmd == "string")
{
- // Skip <cmd>.
- //
- for (auto i (ns.begin () + 1); i != ns.end (); ++i)
+ string s;
+ try
+ {
+ s = convert<string> (move (ns));
+ }
+ catch (const invalid_argument& e)
{
- string vn (convert<string> (move (*i)));
- build2::script::verify_environment_var_name (vn, pf, ll);
- hash_environment (cs, vn);
+ fail (ll) << "invalid 'depdb string' argument: " << e;
}
+
+ if (data.dd.expect (s) != nullptr)
+ l4 ([&] {
+ data.trace (ll)
+ << "'depdb string' argument change forcing update of "
+ << data.t;});
}
- catch (const invalid_argument& e)
+ else if (cmd == "env")
{
- fail (ll) << pf << e;
- }
+ sha256 cs;
+ const char* pf ("invalid 'depdb env' argument: ");
- if (ctx.dd.expect (cs.string ()) != nullptr)
- l4 ([&] {
- ctx.trace (ll)
- << "'depdb env' environment change forcing update of "
- << ctx.env.target;});
+ try
+ {
+ for (name& n: ns)
+ {
+ string vn (convert<string> (move (n)));
+ build2::script::verify_environment_var_name (vn, pf, ll);
+ hash_environment (cs, vn);
+ }
+ }
+ catch (const invalid_argument& e)
+ {
+ fail (ll) << pf << e;
+ }
+
+ if (data.dd.expect (cs.string ()) != nullptr)
+ l4 ([&] {
+ data.trace (ll)
+ << "'depdb env' environment change forcing update of "
+ << data.t;});
+ }
+ else
+ assert (false);
}
- else
- assert (false);
}
else
{
- // Note that we don't reset the line index to zero (as we do in
- // execute_body()) assuming that there are some script body
- // commands to follow.
- //
command_expr ce (
parse_command_line (t, static_cast<token_type&> (tt)));
@@ -1006,7 +1074,7 @@ namespace build2
p.recall.string () == "set";
}) == ce.end ())
{
- const replay_tokens& rt (ctx.scr.depdb_preamble.back ().tokens);
+ const replay_tokens& rt (data.scr.depdb_preamble.back ().tokens);
assert (!rt.empty ());
fail (ll) << "disallowed command in depdb preamble" <<
@@ -1019,7 +1087,7 @@ namespace build2
}
};
- exec_lines (s.depdb_preamble, exec_cmd);
+ exec_lines (begin, end, exec_cmd);
}
void parser::
@@ -1051,7 +1119,7 @@ namespace build2
}
void parser::
- exec_lines (const lines& lns,
+ exec_lines (lines_iterator begin, lines_iterator end,
const function<exec_cmd_function>& exec_cmd)
{
// Note that we rely on "small function object" optimization for the
@@ -1090,25 +1158,23 @@ namespace build2
return runner_->run_if (*environment_, ce, li, ll);
};
- build2::script::parser::exec_lines (lns.begin (), lns.end (),
+ build2::script::parser::exec_lines (begin, end,
exec_set, exec_cmd, exec_if,
environment_->exec_line,
&environment_->var_pool);
}
names parser::
- exec_special (token& t, build2::script::token_type& tt,
- bool omit_builtin)
+ exec_special (token& t, build2::script::token_type& tt, bool skip_first)
{
- if (omit_builtin)
+ if (skip_first)
{
assert (tt != type::newline && tt != type::eos);
-
next (t, tt);
}
return tt != type::newline && tt != type::eos
- ? parse_names (t, tt, pattern_mode::expand)
+ ? parse_names (t, tt, pattern_mode::ignore)
: names ();
}
@@ -1134,6 +1200,649 @@ namespace build2
return r;
}
+ void parser::
+ exec_depdb_dyndep (token& lt, build2::script::token_type& ltt,
+ size_t li, const location& ll,
+ action a, const scope& bs, file& t,
+ depdb& dd,
+ bool& update,
+ bool& deferred_failure,
+ timestamp mt)
+ {
+ tracer trace ("exec_depdb_dyndep");
+
+ context& ctx (t.ctx);
+
+ // Similar approach to parse_env_builtin().
+ //
+ depdb_dep_options ops;
+ bool prog (false);
+ {
+ auto& t (lt);
+ auto& tt (ltt);
+
+ next (t, tt); // Skip 'dep' command.
+
+ // Note that an option name and value can belong to different name
+ // chunks. That's why we parse the arguments in the chunking mode
+ // into the list up to the `--` separator and parse this list into
+ // options afterwards. Note that the `--` separator should be
+ // omitted if there is no program (i.e., additional dependency info
+ // is being read from one of the prerequisites).
+ //
+ strings args;
+
+ names ns; // Reuse to reduce allocations.
+ while (tt != type::newline && tt != type::eos)
+ {
+ if (tt == type::word && t.value == "--")
+ {
+ prog = true;
+ break;
+ }
+
+ location l (get_location (t));
+
+ if (!start_names (tt))
+ fail (l) << "depdb dyndep: expected option or '--' separator "
+ << "instead of " << t;
+
+ parse_names (t, tt,
+ ns,
+ pattern_mode::ignore,
+ true /* chunk */,
+ "depdb dyndep builtin argument",
+ nullptr);
+
+ for (name& n: ns)
+ {
+ try
+ {
+ args.push_back (convert<string> (move (n)));
+ }
+ catch (const invalid_argument&)
+ {
+ diag_record dr (fail (l));
+ dr << "invalid string value ";
+ to_stream (dr.os, n, true /* quote */);
+ }
+ }
+
+ ns.clear ();
+ }
+
+ if (prog)
+ {
+ next (t, tt); // Skip '--'.
+
+ if (tt == type::newline || tt == type::eos)
+ fail (t) << "depdb dyndep: expected program name instead of "
+ << t;
+ }
+
+ // Parse the options.
+ //
+ // We would like to support both -I <dir> as well as -I<dir> forms
+ // for better compatibility. The latter requires manual parsing.
+ //
+ try
+ {
+ for (cli::vector_scanner scan (args); scan.more (); )
+ {
+ if (ops.parse (scan, cli::unknown_mode::stop) && !scan.more ())
+ break;
+
+ const char* a (scan.peek ());
+
+ // Handle -I<dir>
+ //
+ if (a[0] == '-' && a[1] == 'I')
+ {
+ try
+ {
+ ops.include_path ().push_back (dir_path (a + 2));
+ }
+ catch (const invalid_path&)
+ {
+ throw cli::invalid_value ("-I", a + 2);
+ }
+
+ scan.next ();
+ continue;
+ }
+
+#if 0
+ // Handle --byproduct in the wrong place.
+ //
+ if (strcmp (a, "--byproduct") == 0)
+ fail (ll) << "depdb dyndep: --byproduct must be first option";
+#endif
+
+ // Handle unknown option.
+ //
+ if (a[0] == '-')
+ throw cli::unknown_option (a);
+
+ // Handle unexpected argument.
+ //
+ fail (ll) << "depdb dyndep: unexpected argument '" << a << "'";
+ }
+ }
+ catch (const cli::exception& e)
+ {
+ fail (ll) << "depdb dyndep: " << e;
+ }
+ }
+
+ // Get the default prerequisite type falling back to file{} if not
+ // specified.
+ //
+ // The reason one would want to specify it is to make sure different
+ // rules "resolve" the same dynamic prerequisites to the same targets.
+ // For example, a rule that implements custom C compilation for some
+ // translation unit would want to make sure it resolves extracted
+ // system headers to h{} targets analogous to the c module's rule.
+ //
+ const target_type* def_pt;
+ if (ops.default_prereq_type_specified ())
+ {
+ const string& t (ops.default_prereq_type ());
+
+ def_pt = bs.find_target_type (t);
+ if (def_pt == nullptr)
+ fail (ll) << "unknown target type '" << t << "'";
+ }
+ else
+ def_pt = &file::static_type;
+
+ // This code is based on the prior work in the cc module (specifically
+ // extract_headers()) where you can often find more detailed rationale
+ // for some of the steps performed.
+
+ using dyndep = dyndep_rule;
+
+ // Build the maps lazily, only if/when needed.
+ //
+ using prefix_map = dyndep::prefix_map;
+ using srcout_map = dyndep::srcout_map;
+
+ function<dyndep::map_extension_func> map_ext (
+ [] (const scope& bs, const string& n, const string& e)
+ {
+ // @@ TODO: allow specifying base target types.
+ //
+ // Feels like the only reason one would want to specify base types
+ // is to tighten things up (as opposed to making some setup work)
+ // since it essentially restricts the set of registered target
+ // types that we will consider.
+ //
+ // Note also that these would be this project's target types while
+ // the file could be from another project.
+ //
+ return dyndep::map_extension (bs, n, e, nullptr);
+
+ // @@ TODO: should we return something as fallback (file{},
+ // def_pt)? Note: not the same semantics as enter_file()'s
+ // fallback. Feels like it could conceivably be different
+ // (e.g., h{} for fallback and hxx{} for some "unmappable" gen
+ // header). It looks like the "best" way currently is to define
+ // a custom target types for it (see moc{} in libQt5Core).
+ //
+ // Note also that we should only do this if bs is in our
+ // project.
+ });
+
+ // Don't we want to insert a "local"/prefixless mapping in case the
+ // user did not specify any -I's? But then will also need src-out
+ // remapping. So it will be equivalent to -I$out_base -I$src_base? But
+ // then it's not hard to add explicitly...
+ //
+ function<dyndep::prefix_map_func> pfx_map;
+
+ struct
+ {
+ tracer& trace;
+ const location& ll;
+ const depdb_dep_options& ops;
+ optional<prefix_map> map;
+ } pfx_data {trace, ll, ops, nullopt};
+
+ if (!ops.include_path ().empty ())
+ {
+ pfx_map = [this, &pfx_data] (action,
+ const scope& bs,
+ const target& t) -> const prefix_map&
+ {
+ if (!pfx_data.map)
+ {
+ pfx_data.map = prefix_map ();
+
+ const scope& rs (*bs.root_scope ());
+
+ for (dir_path d: pfx_data.ops.include_path ())
+ {
+ if (d.relative ())
+ fail (pfx_data.ll) << "depdb dyndep: relative include "
+ << "search path " << d;
+
+ if (!d.normalized (false /* canonical dir seperators */))
+ d.normalize ();
+
+ // If we are not inside our project root, then ignore.
+ //
+ if (d.sub (rs.out_path ()))
+ dyndep::append_prefix (
+ pfx_data.trace, *pfx_data.map, t, move (d));
+ }
+ }
+
+ return *pfx_data.map;
+ };
+ }
+
+ optional<path> file;
+ enum class format {make} fmt (format::make);
+ command_expr cmd;
+ srcout_map so_map;
+
+ // Parse the remainder of the command line as a program (which can be
+ // a pipe). If file is absent, then we save the command's stdout to a
+ // pipe. Otherwise, assume the command writes to file and add it to
+ // the cleanups.
+ //
+ // Note that MSVC /showInclude sends its output to stderr (and so
+ // could do other broken tools). However, the user can always merge
+ // stderr to stdout (2>&1).
+ //
+ auto init_run = [this, &ctx,
+ &lt, &ltt, &ll,
+ &ops, prog, &file, &cmd, &so_map] ()
+ {
+ // --format
+ //
+ if (ops.format_specified ())
+ {
+ const string& f (ops.format ());
+
+ if (f != "make")
+ fail (ll) << "depdb dyndep: invalid --format option value '"
+ << f << "'";
+ }
+
+ // --file
+ //
+ if (ops.file_specified ())
+ {
+ file = move (ops.file ());
+
+ if (file->relative ())
+ fail (ll) << "depdb dyndep: relative path specified with --file";
+ }
+
+ // Populate the srcout map with the -I$out_base -I$src_base pairs.
+ //
+ {
+ dyndep::srcout_builder builder (ctx, so_map);
+
+ for (dir_path d: ops.include_path ())
+ builder.next (move (d));
+ }
+
+ if (prog)
+ {
+ cmd = parse_command_line (lt, static_cast<token_type&> (ltt));
+
+ // If the output goes to stdout, then this should be a single
+ // pipeline without any logical operators (&& or ||).
+ //
+ if (!file && cmd.size () != 1)
+ fail (ll) << "depdb dyndep: command with stdout output cannot "
+ << "contain logical operators";
+
+ // Note that we may need to run this command multiple times. The
+ // two potential issues here are the re-registration of the
+ // clenups and re-use of the special files (stdin, stdout, etc;
+ // they include the line index in their names to avoid clashes
+ // between lines).
+ //
+ // Cleanups are not an issue, they will simply replaced. And
+ // overriding the contents of the special files seems harmless and
+ // consistent with what would happen if the command redirects its
+ // output to a non-special file.
+ //
+ if (file)
+ environment_->clean (
+ {build2::script::cleanup_type::always, *file},
+ true /* implicit */);
+ }
+ else
+ {
+ // Assume file is one of the prerequisites.
+ //
+ if (!file)
+ fail (ll) << "depdb dyndep: program or --file expected";
+ }
+ };
+
+ // Enter as a target, update, and add to the list of prerequisite
+ // targets a file.
+ //
+ const char* what (ops.what_specified ()
+ ? ops.what ().c_str ()
+ : "file");
+
+ size_t skip_count (0);
+ auto add = [this, &trace, what,
+ a, &bs, &t,
+ &map_ext, def_pt, &pfx_map, &so_map,
+ &dd, &skip_count] (path fp,
+ bool cache,
+ timestamp mt) -> optional<bool>
+ {
+ context& ctx (t.ctx);
+
+ // We can only defer the failure if we will be running the recipe
+ // body.
+ //
+ auto fail = [this, what, &ctx] (const auto& f) -> optional<bool>
+ {
+ bool df (!ctx.match_only && !ctx.dry_run_option);
+
+ diag_record dr;
+ dr << error << what << ' ' << f << " not found and no rule to "
+ << "generate it";
+
+ if (df)
+ dr << info << "failure deferred to recipe body diagnostics";
+
+ if (verb < 4)
+ dr << info << "re-run with --verbose=4 for more information";
+
+ if (df)
+ return nullopt;
+ else
+ dr << endf;
+ };
+
+ if (const build2::file* ft = dyndep::enter_file (
+ trace, what,
+ a, bs, t,
+ move (fp), cache, false /* normalize */,
+ map_ext, *def_pt, pfx_map, so_map).first)
+ {
+ if (optional<bool> u = dyndep::inject_file (
+ trace, what,
+ a, t,
+ *ft, mt, false /* fail */))
+ {
+ if (!cache)
+ dd.expect (ft->path ());
+
+ skip_count++;
+ return *u;
+ }
+ else if (cache)
+ {
+ dd.write (); // Invalidate this line.
+ return true;
+ }
+ else
+ return fail (*ft);
+ }
+ else
+ return fail (fp);
+ };
+
+ // If things go wrong (and they often do in this area), give the user
+ // a bit extra context.
+ //
+ auto df = make_diag_frame (
+ [this, &ll, &t] (const diag_record& dr)
+ {
+ if (verb != 0)
+ dr << info (ll) << "while extracting dynamic dependencies for "
+ << t;
+ });
+
+ // If nothing so far has invalidated the dependency database, then try
+ // the cached data before running the program.
+ //
+ bool cache (!update);
+
+ for (bool restart (true), first_run (true); restart; cache = false)
+ {
+ restart = false;
+
+ if (cache)
+ {
+ // If any, this is always the first run.
+ //
+ assert (skip_count == 0);
+
+ // We should always end with a blank line.
+ //
+ for (;;)
+ {
+ string* l (dd.read ());
+
+ // If the line is invalid, run the compiler.
+ //
+ if (l == nullptr)
+ {
+ restart = true;
+ break;
+ }
+
+ if (l->empty ()) // Done, nothing changed.
+ return;
+
+ if (optional<bool> r = add (path (move (*l)), true /*cache*/, mt))
+ {
+ restart = *r;
+
+ if (restart)
+ {
+ update = true;
+ l6 ([&]{trace << "restarting (cache)";});
+ break;
+ }
+ }
+ else
+ {
+ // Trigger rebuild and mark as expected to fail.
+ //
+ update = true;
+ deferred_failure = true;
+ return;
+ }
+ }
+ }
+ else
+ {
+ if (first_run)
+ {
+ init_run ();
+ first_run = false;
+ }
+ else if (!prog)
+ {
+ fail (ll) << "generated " << what << " without program to retry";
+ }
+
+ // Save the timestamp just before we run the command. If we depend
+ // on any file that has been updated since, then we should assume
+ // we have "seen" the old copy and restart.
+ //
+ timestamp rmt (prog ? system_clock::now () : mt);
+
+ // Run the command if any and reduce outputs to common istream.
+ //
+ // Note that the resulting stream should tolerate partial read.
+ //
+ // While reading the entire stdout into a string is not the most
+ // efficient way to do it, this does simplify things quite a bit,
+ // not least of which is not having to parse the output before
+ // knowing the program exist status.
+ //
+ istringstream iss;
+ if (prog)
+ {
+ string s;
+ build2::script::run (*environment_,
+ cmd,
+ li,
+ ll,
+ !file ? &s : nullptr);
+
+ if (!file)
+ {
+ iss.str (move (s));
+ iss.exceptions (istream::badbit);
+ }
+ }
+
+ ifdstream ifs (ifdstream::badbit);
+ if (file)
+ try
+ {
+ ifs.open (*file);
+ }
+ catch (const io_error& e)
+ {
+ fail (ll) << "unable to open file " << *file << ": " << e;
+ }
+
+ istream& is (file
+ ? static_cast<istream&> (ifs)
+ : static_cast<istream&> (iss));
+
+ const path_name& in (file
+ ? path_name (*file)
+ : path_name ("<stdin>"));
+
+ location il (in, 1);
+
+ // The way we parse things is format-specific.
+ //
+ size_t skip (skip_count);
+
+ switch (fmt)
+ {
+ case format::make:
+ {
+ using make_state = make_parser;
+ using make_type = make_parser::type;
+
+ make_parser make;
+
+ for (string l; !restart; ++il.line) // Reuse the buffer.
+ {
+ if (eof (getline (is, l)))
+ {
+ if (make.state != make_state::end)
+ fail (il) << "incomplete make dependency declaration";
+
+ break;
+ }
+
+ size_t pos (0);
+ do
+ {
+ pair<make_type, string> r;
+ {
+ auto df = make_diag_frame (
+ [this, &l] (const diag_record& dr)
+ {
+ if (verb != 0)
+ dr << info << "while parsing make dependency "
+ << "declaration line '" << l << "'";
+ });
+
+ r = make.next (l, pos, il, false /* strict */);
+ }
+
+ if (r.second.empty ())
+ continue;
+
+ // @@ TODO: what should we do about targets?
+ //
+ // Note that if we take GCC as an example, things are
+ // quite messed up: by default it ignores -o and just
+ // takes the source file name and replaces the extension
+ // with a platform-appropriate object file extension. One
+ // can specify a custom target (or even multiple targets)
+ // with -MT or with -MQ (quoting). Though MinGW GCC still
+ // does not quote `:` with -MQ. So in this case it's
+ // definitely easier for the user to ignore the targets
+ // and just specify everything in the buildfile.
+ //
+ // On the other hand, other tools are likely to produce
+ // more sensible output (except perhaps for quoting).
+ //
+ // @@ Maybe in the lax mode we should only recognize `:`
+ // if it's separated on at least one side?
+ //
+ // Alternatively, we could detect Windows drives in
+ // paths and "handle" them (I believe this is what GNU
+ // make does). Maybe we should have three formats:
+ // make-lax, make, make-strict?
+ //
+ if (r.first == make_type::target)
+ continue;
+
+ // Skip until where we left off.
+ //
+ if (skip != 0)
+ {
+ skip--;
+ continue;
+ }
+
+ if (optional<bool> u = add (path (move (r.second)),
+ false /* cache */,
+ rmt))
+ {
+ restart = *u;
+
+ if (restart)
+ {
+ update = true;
+ l6 ([&]{trace << "restarting";});
+ break;
+ }
+ }
+ else
+ {
+ // Trigger recompilation, mark as expected to fail, and
+ // bail out.
+ //
+ update = true;
+ deferred_failure = true;
+ break;
+ }
+ }
+ while (pos != l.size ());
+
+ if (make.state == make_state::end || deferred_failure)
+ break;
+ }
+
+ break;
+ }
+ }
+
+ // Bail out early if we have deferred a failure.
+ //
+ if (deferred_failure)
+ return;
+ }
+ }
+
+ // Add the terminating blank line (we are updating depdb).
+ //
+ dd.expect ("");
+ }
+
// When add a special variable don't forget to update lexer::word().
//
bool parser::
diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx
index b737a13..da15509 100644
--- a/libbuild2/build/script/parser.hxx
+++ b/libbuild2/build/script/parser.hxx
@@ -8,7 +8,6 @@
#include <libbuild2/forward.hxx>
#include <libbuild2/utility.hxx>
-#include <libbuild2/depdb.hxx>
#include <libbuild2/diagnostics.hxx>
#include <libbuild2/script/parser.hxx>
@@ -82,21 +81,51 @@ namespace build2
// initialize/clean up the environment before/after the script
// execution.
//
+ // Note: having both root and base scopes for testing (where we pass
+ // global scope for both).
+ //
void
execute_body (const scope& root, const scope& base,
environment&, const script&, runner&,
bool enter = true, bool leave = true);
+ // Execute the first or the second (dyndep) half of the depdb
+ // preamble.
+ //
// Note that it's the caller's responsibility to make sure that the
// runner's enter() function is called before the first preamble/body
// command execution and leave() -- after the last command.
//
void
- execute_depdb_preamble (const scope& root, const scope& base,
- environment&, const script&, runner&,
- depdb&);
+ execute_depdb_preamble (action a, const scope& base, const file& t,
+ environment& e, const script& s, runner& r,
+ depdb& dd)
+ {
+ auto b (s.depdb_preamble.begin ());
+ exec_depdb_preamble (
+ a, base, t,
+ e, s, r,
+ b,
+ (s.depdb_dyndep
+ ? b + *s.depdb_dyndep
+ : s.depdb_preamble.end ()),
+ dd);
+ }
+ void
+ execute_depdb_preamble_dyndep (
+ action a, const scope& base, file& t,
+ environment& e, const script& s, runner& r,
+ depdb& dd, bool& update, bool& deferred_failure, timestamp mt)
+ {
+ exec_depdb_preamble (
+ a, base, t,
+ e, s, r,
+ s.depdb_preamble.begin () + *s.depdb_dyndep,
+ s.depdb_preamble.end (),
+ dd, &update, &deferred_failure, mt);
+ }
// Parse a special builtin line into names, performing the variable
// and pattern expansions. If omit_builtin is true, then omit the
@@ -115,12 +144,38 @@ namespace build2
pre_exec (const scope& root, const scope& base,
environment&, const script*, runner*);
+ using lines_iterator = lines::const_iterator;
+
+ void
+ exec_lines (lines_iterator, lines_iterator,
+ const function<exec_cmd_function>&);
+
void
- exec_lines (const lines&, const function<exec_cmd_function>&);
+ exec_lines (const lines& l, const function<exec_cmd_function>& c)
+ {
+ exec_lines (l.begin (), l.end (), c);
+ }
names
- exec_special (token& t, build2::script::token_type& tt,
- bool omit_builtin = true);
+ exec_special (token&, build2::script::token_type&, bool skip_first);
+
+ void
+ exec_depdb_preamble (action, const scope& base, const file&,
+ environment&, const script&, runner&,
+ lines_iterator begin, lines_iterator end,
+ depdb&,
+ bool* update = nullptr,
+ bool* deferred_failure = nullptr,
+ optional<timestamp> mt = nullopt);
+
+ void
+ exec_depdb_dyndep (token&, build2::script::token_type&,
+ size_t line_index, const location&,
+ action, const scope& base, file&,
+ depdb&,
+ bool& update,
+ bool& deferred_failure,
+ timestamp);
// Helpers.
//
@@ -219,8 +274,14 @@ namespace build2
// depdb env <var-names> - Track the environment variables change as a
// hash.
//
- optional<location> depdb_clear_; // 'depdb clear' location if any.
- lines depdb_preamble_; // Note: excludes 'depdb clear'.
+ // depdb dyndep ... - Extract dynamic dependency information.
+ // Can only be the last depdb builtin call
+ // in the preamble.
+ //
+ optional<location> depdb_clear_; // depdb-clear location.
+ optional<pair<location, size_t>>
+ depdb_dyndep_; // depdb-dyndep location/position.
+ lines depdb_preamble_; // Note: excluding depdb-clear.
// If present, the first impure function called in the body of the
// script that performs update of a file-based target.
diff --git a/libbuild2/build/script/runner.hxx b/libbuild2/build/script/runner.hxx
index 431c446..558de9b 100644
--- a/libbuild2/build/script/runner.hxx
+++ b/libbuild2/build/script/runner.hxx
@@ -53,7 +53,7 @@ namespace build2
// Run command expressions.
//
// In dry-run mode don't run the expressions unless they are if-
- // conditions or execute the set or exit builtins, but prints them at
+ // conditions or execute the set or exit builtins, but print them at
// verbosity level 2 and up.
//
class default_runner: public runner
diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx
index e11cb45..9d7567c 100644
--- a/libbuild2/build/script/script.hxx
+++ b/libbuild2/build/script/script.hxx
@@ -29,6 +29,10 @@ namespace build2
using build2::script::deadline;
using build2::script::timeout;
+ // Forward declarations.
+ //
+ class default_runner;
+
// Notes:
//
// - Once parsed, the script can be executed in multiple threads with
@@ -70,9 +74,10 @@ namespace build2
// The script's custom dependency change tracking lines (see the
// script parser for details).
//
- bool depdb_clear;
- lines_type depdb_preamble;
- bool depdb_preamble_temp_dir = false; // True if references $~.
+ bool depdb_clear;
+ optional<size_t> depdb_dyndep; // Position of first depdb-dyndep.
+ lines_type depdb_preamble;
+ bool depdb_preamble_temp_dir = false; // True if refs $~.
location start_loc;
location end_loc;
diff --git a/libbuild2/build/script/types-parsers.cxx b/libbuild2/build/script/types-parsers.cxx
new file mode 100644
index 0000000..9ecfa13
--- /dev/null
+++ b/libbuild2/build/script/types-parsers.cxx
@@ -0,0 +1,56 @@
+// file : libbuild2/build/script/types-parsers.cxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#include <libbuild2/build/script/types-parsers.hxx>
+
+#include <libbuild2/build/script/builtin-options.hxx> // cli namespace
+
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ namespace cli
+ {
+ template <typename T>
+ static void
+ parse_path (T& x, scanner& s)
+ {
+ const char* o (s.next ());
+
+ if (!s.more ())
+ throw missing_value (o);
+
+ const char* v (s.next ());
+
+ try
+ {
+ x = T (v);
+
+ if (x.empty ())
+ throw invalid_value (o, v);
+ }
+ catch (const invalid_path&)
+ {
+ throw invalid_value (o, v);
+ }
+ }
+
+ void parser<path>::
+ parse (path& x, bool& xs, scanner& s)
+ {
+ xs = true;
+ parse_path (x, s);
+ }
+
+ void parser<dir_path>::
+ parse (dir_path& x, bool& xs, scanner& s)
+ {
+ xs = true;
+ parse_path (x, s);
+ }
+ }
+ }
+ }
+}
diff --git a/libbuild2/build/script/types-parsers.hxx b/libbuild2/build/script/types-parsers.hxx
new file mode 100644
index 0000000..a42dab7
--- /dev/null
+++ b/libbuild2/build/script/types-parsers.hxx
@@ -0,0 +1,49 @@
+// file : libbuild2/build/script/types-parsers.hxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+// CLI parsers, included into the generated source files.
+//
+
+#ifndef LIBBUILD2_BUILD_SCRIPT_TYPES_PARSERS_HXX
+#define LIBBUILD2_BUILD_SCRIPT_TYPES_PARSERS_HXX
+
+#include <libbuild2/types.hxx>
+
+namespace build2
+{
+ namespace build
+ {
+ namespace script
+ {
+ namespace cli
+ {
+ class scanner;
+
+ template <typename T>
+ struct parser;
+
+ template <>
+ struct parser<path>
+ {
+ static void
+ parse (path&, bool&, scanner&);
+
+ static void
+ merge (path& b, const path& a) {b = a;}
+ };
+
+ template <>
+ struct parser<dir_path>
+ {
+ static void
+ parse (dir_path&, bool&, scanner&);
+
+ static void
+ merge (dir_path& b, const dir_path& a) {b = a;}
+ };
+ }
+ }
+ }
+}
+
+#endif // LIBBUILD2_BUILD_SCRIPT_TYPES_PARSERS_HXX
diff --git a/libbuild2/buildfile b/libbuild2/buildfile
index 17003b5..f21be7b 100644
--- a/libbuild2/buildfile
+++ b/libbuild2/buildfile
@@ -32,7 +32,8 @@ lib{build2}: libul{build2}: \
libul{build2}: script/{hxx ixx txx cxx}{** -*-options -**.test...} \
script/{hxx ixx cxx}{builtin-options}
-libul{build2}: build/{hxx ixx txx cxx}{** -**.test...}
+libul{build2}: build/script/{hxx ixx txx cxx}{** -*-options -**.test...} \
+ build/script/{hxx ixx cxx}{builtin-options}
# Note that this won't work in libul{} since it's not installed.
#
@@ -216,38 +217,56 @@ else
# Generated options parser.
#
-script/
+# @@ Consider generating common cli runtime namespace if adding more option
+# files. Plus sommon types-parser.?xx (which could also potentially be
+# reused by the driver).
+#
+if $cli.configured
{
- if $cli.configured
+ cli.options += --std c++11 -I $src_root --include-with-brackets \
+--generate-vector-scanner --generate-modifier --generate-specifier \
+--suppress-usage
+
+ cli.cxx{*}:
{
- cli.cxx{builtin-options}: cli{builtin}
-
- cli.options += --std c++11 -I $src_root --include-with-brackets \
---include-prefix libbuild2/script --guard-prefix LIBBUILD2_SCRIPT \
---cli-namespace build2::script::cli --generate-vector-scanner \
---generate-modifier --generate-specifier --suppress-usage
-
- cli.cxx{*}:
- {
- # Include the generated cli files into the distribution and don't remove
- # them when cleaning in src (so that clean results in a state identical
- # to distributed). But don't install their headers since they are only
- # used internally in the testscript implementation.
- #
- dist = true
- clean = ($src_root != $out_root)
- install = false
-
- # We keep the generated code in the repository so copy it back to src in
- # case of a forwarded configuration.
- #
- backlink = overwrite
- }
- }
- else
- # No install for the pre-generated case.
+ # Include the generated cli files into the distribution and don't remove
+ # them when cleaning in src (so that clean results in a state identical
+ # to distributed). But don't install their headers since they are only
+ # used internally in the testscript implementation.
+ #
+ dist = true
+ clean = ($src_root != $out_root)
+ install = false
+
+ # We keep the generated code in the repository so copy it back to src in
+ # case of a forwarded configuration.
#
- hxx{builtin-options}@./ ixx{builtin-options}@./: install = false
+ backlink = overwrite
+ }
+
+ script/cli.cxx{builtin-options}: script/cli{builtin}
+ {
+ cli.options += --cli-namespace build2::script::cli \
+--include-prefix libbuild2/script --guard-prefix LIBBUILD2_SCRIPT
+ }
+
+ build/script/cli.cxx{builtin-options}: build/script/cli{builtin}
+ {
+ cli.options += --cli-namespace build2::build::script::cli \
+--include-prefix libbuild2/build/script --guard-prefix LIBBUILD2_BUILD_SCRIPT \
+--cxx-prologue "#include <libbuild2/build/script/types-parsers.hxx>" \
+--generate-parse
+ }
+}
+else
+{
+ # No install for the pre-generated case.
+ #
+ script/hxx{builtin-options}@./ \
+ script/ixx{builtin-options}@./: install = false
+
+ build/script/hxx{builtin-options}@./ \
+ build/script/ixx{builtin-options}@./: install = false
}
# Install into the libbuild2/ subdirectory of, say, /usr/include/
diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx
index 87fce90..525821f 100644
--- a/libbuild2/cc/compile-rule.cxx
+++ b/libbuild2/cc/compile-rule.cxx
@@ -16,6 +16,7 @@
#include <libbuild2/algorithm.hxx>
#include <libbuild2/filesystem.hxx> // mtime()
#include <libbuild2/diagnostics.hxx>
+#include <libbuild2/make-parser.hxx>
#include <libbuild2/bin/target.hxx>
@@ -707,7 +708,7 @@ namespace build2
void compile_rule::
append_library_prefixes (appended_libraries& ls, prefix_map& pm,
const scope& bs,
- action a, target& t, linfo li) const
+ action a, const target& t, linfo li) const
{
struct data
{
@@ -730,14 +731,21 @@ namespace build2
if (find (d.ls.begin (), d.ls.end (), &l) != d.ls.end ())
return false;
- const variable& var (
- com
- ? c_export_poptions
- : (t == x
- ? x_export_poptions
- : l.ctx.var_pool[t + ".export.poptions"]));
-
- append_prefixes (d.pm, l, var);
+ // If this target does not belong to any project (e.g, an "imported as
+ // installed" library), then it can't possibly generate any headers
+ // for us.
+ //
+ if (const scope* rs = l.base_scope ().root_scope ())
+ {
+ const variable& var (
+ com
+ ? c_export_poptions
+ : (t == x
+ ? x_export_poptions
+ : l.ctx.var_pool[t + ".export.poptions"]));
+
+ append_prefixes (d.pm, *rs, l, var);
+ }
if (com)
d.ls.push_back (&l);
@@ -775,69 +783,6 @@ namespace build2
}
}
- // Update the target during the match phase. Return true if it has changed
- // or if the passed timestamp is not timestamp_unknown and is older than
- // the target.
- //
- // This function is used to make sure header dependencies are up to date.
- //
- // There would normally be a lot of headers for every source file (think
- // all the system headers) and just calling execute_direct() on all of
- // them can get expensive. At the same time, most of these headers are
- // existing files that we will never be updating (again, system headers,
- // for example) and the rule that will match them is the fallback
- // file_rule. That rule has an optimization: it returns noop_recipe (which
- // causes the target state to be automatically set to unchanged) if the
- // file is known to be up to date. So we do the update "smartly".
- //
- static bool
- update (tracer& trace, action a, const target& t, timestamp ts)
- {
- const path_target* pt (t.is_a<path_target> ());
-
- if (pt == nullptr)
- ts = timestamp_unknown;
-
- target_state os (t.matched_state (a));
-
- if (os == target_state::unchanged)
- {
- if (ts == timestamp_unknown)
- return false;
- else
- {
- // We expect the timestamp to be known (i.e., existing file).
- //
- timestamp mt (pt->mtime ());
- assert (mt != timestamp_unknown);
- return mt > ts;
- }
- }
- else
- {
- // We only want to return true if our call to execute() actually
- // caused an update. In particular, the target could already have been
- // in target_state::changed because of a dependency extraction run for
- // some other source file.
- //
- // @@ MT perf: so we are going to switch the phase and execute for
- // any generated header.
- //
- phase_switch ps (t.ctx, run_phase::execute);
- target_state ns (execute_direct (a, t));
-
- if (ns != os && ns != target_state::unchanged)
- {
- l6 ([&]{trace << "updated " << t
- << "; old state " << os
- << "; new state " << ns;});
- return true;
- }
- else
- return ts != timestamp_unknown ? pt->newer (ts, ns) : false;
- }
- }
-
recipe compile_rule::
apply (action a, target& xt) const
{
@@ -1097,6 +1042,8 @@ namespace build2
md.symexport = l ? cast<bool> (l) : symexport;
}
+ // NOTE: see similar code in adhoc_buildscript_rule::apply().
+
// Make sure the output directory exists.
//
// Is this the right thing to do? It does smell a bit, but then we do
@@ -1547,77 +1494,13 @@ namespace build2
}
}
- // Reverse-lookup target type(s) from extension.
- //
- small_vector<const target_type*, 2> compile_rule::
- map_extension (const scope& bs, const string& n, const string& e) const
- {
- // We will just have to try all of the possible ones, in the "most
- // likely to match" order.
- //
- auto test = [&bs, &n, &e] (const target_type& tt) -> bool
- {
- // Call the extension derivation function. Here we know that it will
- // only use the target type and name from the target key so we can
- // pass bogus values for the rest.
- //
- target_key tk {&tt, nullptr, nullptr, &n, nullopt};
-
- // This is like prerequisite search.
- //
- optional<string> de (tt.default_extension (tk, bs, nullptr, true));
-
- return de && *de == e;
- };
-
- small_vector<const target_type*, 2> r;
-
- for (const target_type* const* p (x_inc); *p != nullptr; ++p)
- if (test (**p))
- r.push_back (*p);
-
- // Next try target types derived from any of the C-source types.
- //
- const target_type_map& ttm (bs.root_scope ()->root_extra->target_types);
-
- for (auto i (ttm.type_begin ()), e (ttm.type_end ()); i != e; ++i)
- {
- const target_type& dt (i->second);
-
- for (const target_type* const* p (x_inc); *p != nullptr; ++p)
- {
- const target_type& bt (**p);
-
- if (dt.is_a (bt))
- {
- if (dt != bt && test (dt))
- r.push_back (&dt);
-
- break;
- }
- }
- }
-
- return r;
- }
-
void compile_rule::
- append_prefixes (prefix_map& m, const target& t, const variable& var) const
+ append_prefixes (prefix_map& m,
+ const scope& rs, const target& t,
+ const variable& var) const
{
tracer trace (x, "compile_rule::append_prefixes");
- // If this target does not belong to any project (e.g, an "imported as
- // installed" library), then it can't possibly generate any headers for
- // us.
- //
- const scope& bs (t.base_scope ());
- const scope* rs (bs.root_scope ());
- if (rs == nullptr)
- return;
-
- const dir_path& out_base (t.dir);
- const dir_path& out_root (rs->out_path ());
-
if (auto l = t[var])
{
const auto& v (cast<strings> (l));
@@ -1675,136 +1558,8 @@ namespace build2
// If we are not inside our project root, then ignore.
//
- if (!d.sub (out_root))
- continue;
-
- // If the target directory is a sub-directory of the include
- // directory, then the prefix is the difference between the
- // two. Otherwise, leave it empty.
- //
- // The idea here is to make this "canonical" setup work auto-
- // magically:
- //
- // 1. We include all files with a prefix, e.g., <foo/bar>.
- // 2. The library target is in the foo/ sub-directory, e.g.,
- // /tmp/foo/.
- // 3. The poptions variable contains -I/tmp.
- //
- dir_path p (out_base.sub (d) ? out_base.leaf (d) : dir_path ());
-
- // We use the target's directory as out_base but that doesn't work
- // well for targets that are stashed in subdirectories. So as a
- // heuristics we are going to also enter the outer directories of
- // the original prefix. It is, however, possible, that another -I
- // option after this one will produce one of these outer prefixes as
- // its original prefix in which case we should override it.
- //
- // So we are going to assign the original prefix priority value 0
- // (highest) and then increment it for each outer prefix.
- //
- auto enter = [&trace, &m] (dir_path p, dir_path d, size_t prio)
- {
- auto j (m.lower_bound (p)), e (m.end ());
-
- if (j != e && j->first != p)
- j = e;
-
- if (j == m.end ())
- {
- if (verb >= 4)
- trace << "new mapping for prefix '" << p << "'\n"
- << " new mapping to " << d << " priority " << prio;
-
- m.emplace (move (p), prefix_value {move (d), prio});
- }
- else if (p.empty ())
- {
- // For prefixless we keep all the entries since for them we have
- // an extra check (target must be explicitly spelled out in a
- // buildfile).
- //
- if (verb >= 4)
- trace << "additional mapping for prefix '" << p << "'\n"
- << " new mapping to " << d << " priority " << prio;
-
- // Find the position where to insert according to the priority.
- // For equal priorities we use the insertion order.
- //
- do
- {
- if (j->second.priority > prio)
- break;
- }
- while (++j != e && j->first == p);
-
- m.emplace_hint (j, move (p), prefix_value {move (d), prio});
- }
- else
- {
- prefix_value& v (j->second);
-
- // We used to reject duplicates but it seems this can be
- // reasonably expected to work according to the order of the
- // -I options.
- //
- // Seeing that we normally have more "specific" -I paths first,
- // (so that we don't pick up installed headers, etc), we ignore
- // it.
- //
- if (v.directory == d)
- {
- if (v.priority > prio)
- v.priority = prio;
- }
- else if (v.priority <= prio)
- {
- if (verb >= 4)
- trace << "ignoring mapping for prefix '" << p << "'\n"
- << " existing mapping to " << v.directory
- << " priority " << v.priority << '\n'
- << " another mapping to " << d
- << " priority " << prio;
- }
- else
- {
- if (verb >= 4)
- trace << "overriding mapping for prefix '" << p << "'\n"
- << " existing mapping to " << v.directory
- << " priority " << v.priority << '\n'
- << " new mapping to " << d
- << " priority " << prio;
-
- v.directory = move (d);
- v.priority = prio;
- }
- }
- };
-
-#if 1
- // Enter all outer prefixes, including prefixless.
- //
- // The prefixless part is fuzzy but seems to be doing the right
- // thing ignoring/overriding-wise, at least in cases where one of
- // the competing -I paths is a subdirectory of another.
- //
- for (size_t prio (0);; ++prio)
- {
- bool e (p.empty ());
- enter ((e ? move (p) : p), (e ? move (d) : d), prio);
- if (e)
- break;
- p = p.directory ();
- }
-#else
- size_t prio (0);
- for (bool e (false); !e; ++prio)
- {
- dir_path n (p.directory ());
- e = n.empty ();
- enter ((e ? move (p) : p), (e ? move (d) : d), prio);
- p = move (n);
- }
-#endif
+ if (d.sub (rs.out_path ()))
+ append_prefix (trace, m, t, move (d));
}
}
}
@@ -1812,15 +1567,16 @@ namespace build2
auto compile_rule::
build_prefix_map (const scope& bs,
action a,
- target& t,
+ const target& t,
linfo li) const -> prefix_map
{
prefix_map pm;
// First process our own.
//
- append_prefixes (pm, t, x_poptions);
- append_prefixes (pm, t, c_poptions);
+ const scope& rs (*bs.root_scope ());
+ append_prefixes (pm, rs, t, x_poptions);
+ append_prefixes (pm, rs, t, c_poptions);
// Then process the include directories from prerequisite libraries.
//
@@ -1830,6 +1586,9 @@ namespace build2
return pm;
}
+ // @@ TMP
+ //
+#if 0
// Return the next make prerequisite starting from the specified
// position and update position to point to the start of the
// following prerequisite or l.size() if there are none left.
@@ -1891,6 +1650,7 @@ namespace build2
return r;
}
+#endif
// VC /showIncludes output. The first line is the file being compiled
// (unless clang-cl; handled by our caller). Then we have the list of
@@ -2312,7 +2072,7 @@ namespace build2
if (verb > 2)
{
diag_record dr;
- dr << error << "header '" << f << "' not found and no "
+ dr << error << "header " << f << " not found and no "
<< "rule to generate it";
if (verb < 4)
@@ -3031,355 +2791,56 @@ namespace build2
}
#endif
- // Enter as a target a header file. Depending on the cache flag, the file
- // is assumed to either have come from the depdb cache or from the
- // compiler run.
- //
- // Return the header target and an indication of whether it was remapped
- // or NULL if the header does not exist and cannot be generated. In the
- // latter case the passed header path is guaranteed to be still valid but
- // might have been adjusted (e.g., normalized, etc).
- //
// Note: this used to be a lambda inside extract_headers() so refer to the
// body of that function for the overall picture.
//
pair<const file*, bool> compile_rule::
enter_header (action a, const scope& bs, file& t, linfo li,
path&& f, bool cache, bool norm,
- optional<prefix_map>& pfx_map, srcout_map& so_map) const
+ optional<prefix_map>& pfx_map,
+ const srcout_map& so_map) const
{
tracer trace (x, "compile_rule::enter_header");
- // Find or maybe insert the target. The directory is only moved from if
- // insert is true. Note that it must be normalized.
- //
- auto find = [&trace, &t, this] (dir_path&& d,
- path&& f,
- bool insert) -> const file*
- {
- // Split the file into its name part and extension. Here we can assume
- // the name part is a valid filesystem name.
- //
- // Note that if the file has no extension, we record an empty
- // extension rather than NULL (which would signify that the default
- // extension should be added).
- //
- string e (f.extension ());
- string n (move (f).string ());
-
- if (!e.empty ())
- n.resize (n.size () - e.size () - 1); // One for the dot.
-
- // See if this directory is part of any project and if so determine
- // the target type.
- //
- // While at it also determine if this target is from the src or out
- // tree of said project.
- //
- dir_path out;
-
- // It's possible the extension-to-target type mapping is ambiguous
- // (usually because both C and X-language headers use the same .h
- // extension). In this case we will first try to find one that matches
- // an explicit target (similar logic to when insert is false).
- //
- small_vector<const target_type*, 2> tts;
-
- // Note that the path can be in out or src directory and the latter
- // can be associated with multiple scopes. So strictly speaking we
- // need to pick one that is "associated" with us. But that is still a
- // TODO (see scope_map::find() for details) and so for now we just
- // pick the first one (it's highly unlikely the source file extension
- // mapping will differ based on the configuration).
- //
- {
- const scope& bs (**t.ctx.scopes.find (d).first);
- if (const scope* rs = bs.root_scope ())
- {
- tts = map_extension (bs, n, e);
-
- if (!bs.out_eq_src () && d.sub (bs.src_path ()))
- out = out_src (d, *rs);
- }
- }
-
- // If it is outside any project, or the project doesn't have such an
- // extension, assume it is a plain old C header.
- //
- if (tts.empty ())
- {
- // If the project doesn't "know" this extension then we can't
- // possibly find an explicit target of this type.
- //
- if (!insert)
- {
- l6 ([&]{trace << "unknown header " << n << " extension '"
- << e << "'";});
- return nullptr;
- }
-
- tts.push_back (&h::static_type);
- }
-
- // Find or insert target.
- //
- // Note that in case of the target type ambiguity we first try to find
- // an explicit target that resolves this ambiguity.
- //
- const target* r (nullptr);
-
- if (!insert || tts.size () > 1)
- {
- // Note that we skip any target type-specific searches (like for an
- // existing file) and go straight for the target object since we
- // need to find the target explicitly spelled out.
- //
- // Also, it doesn't feel like we should be able to resolve an
- // absolute path with a spelled-out extension to multiple targets.
- //
- for (const target_type* tt: tts)
- {
- if ((r = t.ctx.targets.find (*tt, d, out, n, e, trace)) != nullptr)
- break;
- else
- l6 ([&]{trace << "no targe with target type " << tt->name;});
- }
-
- // Note: we can't do this because of the in-source builds where
- // there won't be explicit targets for non-generated headers.
- //
- // This should be harmless, however, since in our world generated
- // headers are normally spelled-out as explicit targets. And if not,
- // we will still get an error, just a bit less specific.
- //
-#if 0
- if (r == nullptr && insert)
- {
- f = d / n;
- if (!e.empty ())
- {
- f += '.';
- f += e;
- }
-
- diag_record dr (fail);
- dr << "mapping of header " << f << " to target type is ambiguous";
- for (const target_type* tt: tts)
- dr << info << "could be " << tt->name << "{}";
- dr << info << "spell-out its target to resolve this ambiguity";
- }
-#endif
- }
-
- // @@ OPT: move d, out, n
- //
- if (r == nullptr && insert)
- r = &search (t, *tts[0], d, out, n, &e, nullptr);
-
- return static_cast<const file*> (r);
- };
-
- // If it's not absolute then it either does not (yet) exist or is a
- // relative ""-include (see init_args() for details). Reduce the second
- // case to absolute.
- //
- // Note: we now always use absolute path to the translation unit so this
- // no longer applies. But let's keep it for posterity.
- //
-#if 0
- if (f.relative () && rels.relative ())
- {
- // If the relative source path has a directory component, make sure
- // it matches since ""-include will always start with that (none of
- // the compilers we support try to normalize this path). Failed that
- // we may end up searching for a generated header in a random
- // (working) directory.
- //
- const string& fs (f.string ());
- const string& ss (rels.string ());
-
- size_t p (path::traits::rfind_separator (ss));
-
- if (p == string::npos || // No directory.
- (fs.size () > p + 1 &&
- path::traits::compare (fs.c_str (), p, ss.c_str (), p) == 0))
- {
- path t (work / f); // The rels path is relative to work.
-
- if (exists (t))
- f = move (t);
- }
- }
-#endif
-
- const file* pt (nullptr);
- bool remapped (false);
-
- // If still relative then it does not exist.
- //
- if (f.relative ())
- {
- // This is probably as often an error as an auto-generated file, so
- // trace at level 4.
- //
- l4 ([&]{trace << "non-existent header '" << f << "'";});
-
- f.normalize ();
-
- // The relative path might still contain '..' (e.g., ../foo.hxx;
- // presumably ""-include'ed). We don't attempt to support auto-
- // generated headers with such inclusion styles.
- //
- if (f.normalized ())
- {
- if (!pfx_map)
- pfx_map = build_prefix_map (bs, a, t, li);
-
- // First try the whole file. Then just the directory.
- //
- // @@ Has to be a separate map since the prefix can be the same as
- // the file name.
- //
- // auto i (pfx_map->find (f));
-
- // Find the most qualified prefix of which we are a sub-path.
- //
- if (!pfx_map->empty ())
- {
- dir_path d (f.directory ());
- auto p (pfx_map->sup_range (d));
-
- if (p.first != p.second)
- {
- // Note that we can only have multiple entries for the
- // prefixless mapping.
- //
- dir_path pd; // Reuse.
- for (auto i (p.first); i != p.second; ++i)
- {
- // Note: value in pfx_map is not necessarily canonical.
- //
- pd = i->second.directory;
- pd.canonicalize ();
-
- l4 ([&]{trace << "try prefix '" << d << "' mapped to " << pd;});
-
- // If this is a prefixless mapping, then only use it if we can
- // resolve it to an existing target (i.e., it is explicitly
- // spelled out in a buildfile). @@ Hm, I wonder why, it's not
- // like we can generate any header without an explicit target.
- // Maybe for diagnostics (i.e., we will actually try to build
- // something there instead of just saying no mapping).
- //
- pt = find (pd / d, f.leaf (), !i->first.empty ());
- if (pt != nullptr)
- {
- f = pd / f;
- l4 ([&]{trace << "mapped as auto-generated " << f;});
- break;
- }
- else
- l4 ([&]{trace << "no explicit target in " << pd;});
- }
- }
- else
- l4 ([&]{trace << "no prefix map entry for '" << d << "'";});
- }
- else
- l4 ([&]{trace << "prefix map is empty";});
- }
- }
- else
+ struct data
{
- // Normalize the path unless it comes from the depdb, in which case
- // we've already done that (normally). This is also where we handle
- // src-out remap (again, not needed if cached).
- //
- if (!cache || norm)
- normalize_header (f);
-
- if (!cache)
+ linfo li;
+ optional<prefix_map>& pfx_map;
+ } d {li, pfx_map};
+
+ // If it is outside any project, or the project doesn't have such an
+ // extension, assume it is a plain old C header.
+ //
+ return enter_file (
+ trace, "header",
+ a, bs, t,
+ move (f), cache, norm,
+ [this] (const scope& bs, const string& n, const string& e)
{
- if (!so_map.empty ())
- {
- // Find the most qualified prefix of which we are a sub-path.
- //
- auto i (so_map.find_sup (f));
- if (i != so_map.end ())
- {
- // Ok, there is an out tree for this headers. Remap to a path
- // from the out tree and see if there is a target for it. Note
- // that the value in so_map is not necessarily canonical.
- //
- dir_path d (i->second);
- d /= f.leaf (i->first).directory ();
- d.canonicalize ();
-
- pt = find (move (d), f.leaf (), false); // d is not moved from.
-
- if (pt != nullptr)
- {
- path p (d / f.leaf ());
- l4 ([&]{trace << "remapping " << f << " to " << p;});
- f = move (p);
- remapped = true;
- }
- }
- }
- }
-
- if (pt == nullptr)
+ return map_extension (bs, n, e, x_inc);
+ },
+ h::static_type,
+ [this, &d] (action a, const scope& bs, const target& t)
+ -> const prefix_map&
{
- l6 ([&]{trace << "entering " << f;});
- pt = find (f.directory (), f.leaf (), true);
- }
- }
+ if (!d.pfx_map)
+ d.pfx_map = build_prefix_map (bs, a, t, d.li);
- return make_pair (pt, remapped);
+ return *d.pfx_map;
+ },
+ so_map);
}
- // Update and add to the list of prerequisite targets a header or header
- // unit target.
- //
- // Return the indication of whether it has changed or, if the passed
- // timestamp is not timestamp_unknown, is older than the target. If the
- // header does not exists nor can be generated (no rule), then issue
- // diagnostics and fail if the fail argument is true and return nullopt
- // otherwise.
- //
// Note: this used to be a lambda inside extract_headers() so refer to the
// body of that function for the overall picture.
//
optional<bool> compile_rule::
inject_header (action a, file& t,
- const file& pt, timestamp mt, bool f /* fail */) const
+ const file& pt, timestamp mt, bool fail) const
{
tracer trace (x, "compile_rule::inject_header");
- // Even if failing we still use try_match() in order to issue consistent
- // (with extract_headers() below) diagnostics (rather than the generic
- // "not rule to update ...").
- //
- if (!try_match (a, pt).first)
- {
- if (!f)
- return nullopt;
-
- diag_record dr;
- dr << fail << "header " << pt << " not found and no rule to "
- << "generate it";
-
- if (verb < 4)
- dr << info << "re-run with --verbose=4 for more information";
- }
-
- bool r (update (trace, a, pt, mt));
-
- // Add to our prerequisite target list.
- //
- t.prerequisite_targets[a].push_back (&pt);
-
- return r;
+ return inject_file (trace, "header", a, t, pt, mt, fail);
}
// Extract and inject header dependencies. Return the preprocessed source
@@ -3414,16 +2875,6 @@ namespace build2
file_cache::entry psrc;
bool puse (true);
- // If things go wrong (and they often do in this area), give the user a
- // bit extra context.
- //
- auto df = make_diag_frame (
- [&src](const diag_record& dr)
- {
- if (verb != 0)
- dr << info << "while extracting header dependencies from " << src;
- });
-
// Preprocesor mode that preserves as much information as possible while
// still performing inclusions. Also serves as a flag indicating whether
// this compiler uses the separate preprocess and compile setup.
@@ -3579,9 +3030,9 @@ namespace build2
// generator by end-users optional by shipping pre-generated headers.
//
// This is a nasty problem that doesn't seem to have a perfect solution
- // (except, perhaps, C++ modules). So what we are going to do is try to
- // rectify the situation by detecting and automatically remapping such
- // mis-inclusions. It works as follows.
+ // (except, perhaps, C++ modules and/or module mapper). So what we are
+ // going to do is try to rectify the situation by detecting and
+ // automatically remapping such mis-inclusions. It works as follows.
//
// First we will build a map of src/out pairs that were specified with
// -I. Here, for performance and simplicity, we will assume that they
@@ -3594,10 +3045,7 @@ namespace build2
// case, then we calculate a corresponding header in the out tree and,
// (this is the most important part), check if there is a target for
// this header in the out tree. This should be fairly accurate and not
- // require anything explicit from the user except perhaps for a case
- // where the header is generated out of nothing (so there is no need to
- // explicitly mention its target in the buildfile). But this probably
- // won't be very common.
+ // require anything explicit from the user.
//
// One tricky area in this setup are target groups: if the generated
// sources are mentioned in the buildfile as a group, then there might
@@ -3607,10 +3055,7 @@ namespace build2
// generated depending on the options (e.g., inline files might be
// suppressed), headers are usually non-optional.
//
- // Note that we use path_map instead of dir_path_map to allow searching
- // using path (file path).
- //
- srcout_map so_map; // path_map<dir_path>
+ srcout_map so_map;
// Dynamic module mapper.
//
@@ -3690,17 +3135,13 @@ namespace build2
// Populate the src-out with the -I$out_base -I$src_base pairs.
//
{
+ srcout_builder builder (ctx, so_map);
+
// Try to be fast and efficient by reusing buffers as much as
// possible.
//
string ds;
- // Previous -I innermost scope if out_base plus the difference
- // between the scope path and the -I path (normally empty).
- //
- const scope* s (nullptr);
- dir_path p;
-
for (auto i (args.begin ()), e (args.end ()); i != e; ++i)
{
const char* o (*i);
@@ -3725,7 +3166,7 @@ namespace build2
if (p == 0)
{
- s = nullptr;
+ builder.skip ();
continue;
}
@@ -3758,68 +3199,14 @@ namespace build2
//
if (!d.empty ())
{
- // Ignore any paths containing '.', '..' components. Allow
- // any directory separators though (think -I$src_root/foo
- // on Windows).
- //
- if (d.absolute () && d.normalized (false))
- {
- // If we have a candidate out_base, see if this is its
- // src_base.
- //
- if (s != nullptr)
- {
- const dir_path& bp (s->src_path ());
-
- if (d.sub (bp))
- {
- if (p.empty () || d.leaf (bp) == p)
- {
- // We've got a pair.
- //
- so_map.emplace (move (d), s->out_path () / p);
- s = nullptr; // Taken.
- continue;
- }
- }
-
- // Not a pair. Fall through to consider as out_base.
- //
- s = nullptr;
- }
-
- // See if this path is inside a project with an out-of-
- // tree build and is in the out directory tree.
- //
- const scope& bs (ctx.scopes.find_out (d));
- if (bs.root_scope () != nullptr)
- {
- if (!bs.out_eq_src ())
- {
- const dir_path& bp (bs.out_path ());
-
- bool e;
- if ((e = (d == bp)) || d.sub (bp))
- {
- s = &bs;
- if (e)
- p.clear ();
- else
- p = d.leaf (bp);
- }
- }
- }
- }
- else
- s = nullptr;
-
- ds = move (d).string (); // Move the buffer out.
+ if (!builder.next (move (d)))
+ ds = move (d).string (); // Move the buffer back out.
}
else
- s = nullptr;
+ builder.skip ();
}
else
- s = nullptr;
+ builder.skip ();
}
}
@@ -4159,15 +3546,12 @@ namespace build2
// to be inconvenient: some users like to re-run a failed build with
// -s not to get "swamped" with errors.
//
- bool df (!ctx.match_only && !ctx.dry_run_option);
-
- const file* ht (enter_header (a, bs, t, li,
- move (hp), cache, false /* norm */,
- pfx_map, so_map).first);
- if (ht == nullptr)
+ auto fail = [&ctx] (const auto& h) -> optional<bool>
{
+ bool df (!ctx.match_only && !ctx.dry_run_option);
+
diag_record dr;
- dr << error << "header '" << hp << "' not found and no rule to "
+ dr << error << "header " << h << " not found and no rule to "
<< "generate it";
if (df)
@@ -4176,41 +3560,42 @@ namespace build2
if (verb < 4)
dr << info << "re-run with --verbose=4 for more information";
- if (df) return nullopt; else dr << endf;
- }
+ if (df)
+ return nullopt;
+ else
+ dr << endf;
+ };
- // If we are reading the cache, then it is possible the file has since
- // been removed (think of a header in /usr/local/include that has been
- // uninstalled and now we need to use one from /usr/include). This
- // will lead to the match failure which we translate to a restart.
- //
- if (optional<bool> u = inject_header (a, t, *ht, mt, false /* fail */))
+ if (const file* ht = enter_header (a, bs, t, li,
+ move (hp), cache, false /* norm */,
+ pfx_map, so_map).first)
{
- // Verify/add it to the dependency database.
+ // If we are reading the cache, then it is possible the file has
+ // since been removed (think of a header in /usr/local/include that
+ // has been uninstalled and now we need to use one from
+ // /usr/include). This will lead to the match failure which we
+ // translate to a restart.
//
- if (!cache)
- dd.expect (ht->path ());
-
- skip_count++;
- return *u;
- }
- else if (!cache)
- {
- diag_record dr;
- dr << error << "header " << *ht << " not found and no rule to "
- << "generate it";
-
- if (df)
- dr << info << "failure deferred to compiler diagnostics";
-
- if (verb < 4)
- dr << info << "re-run with --verbose=4 for more information";
+ if (optional<bool> u = inject_header (a, t, *ht, mt, false /*fail*/))
+ {
+ // Verify/add it to the dependency database.
+ //
+ if (!cache)
+ dd.expect (ht->path ());
- if (df) return nullopt; else dr << endf;
+ skip_count++;
+ return *u;
+ }
+ else if (cache)
+ {
+ dd.write (); // Invalidate this line.
+ return true;
+ }
+ else
+ return fail (*ht);
}
-
- dd.write (); // Invalidate this line.
- return true;
+ else
+ return fail (hp);
};
// As above but for a header unit. Note that currently it is only used
@@ -4233,7 +3618,7 @@ namespace build2
if (ht == nullptr)
{
diag_record dr;
- dr << error << "header '" << hp << "' not found and no rule to "
+ dr << error << "header " << hp << " not found and no rule to "
<< "generate it";
if (df)
@@ -4279,6 +3664,16 @@ namespace build2
const path* drmp (nullptr); // Points to drm.path () if active.
+ // If things go wrong (and they often do in this area), give the user a
+ // bit extra context.
+ //
+ auto df = make_diag_frame (
+ [&src](const diag_record& dr)
+ {
+ if (verb != 0)
+ dr << info << "while extracting header dependencies from " << src;
+ });
+
// If nothing so far has invalidated the dependency database, then try
// the cached data before running the compiler.
//
@@ -4843,13 +4238,16 @@ namespace build2
if (second)
{
+ // Skip the source file.
+ //
+ make_parser::next (l, pos, true /* prereq */);
second = false;
- next_make (l, pos); // Skip the source file.
}
while (pos != l.size ())
{
- string f (next_make (l, pos));
+ string f (
+ make_parser::next (l, pos, true /* prereq */).first);
// Skip until where we left off.
//
diff --git a/libbuild2/cc/compile-rule.hxx b/libbuild2/cc/compile-rule.hxx
index f573968..568c04b 100644
--- a/libbuild2/cc/compile-rule.hxx
+++ b/libbuild2/cc/compile-rule.hxx
@@ -8,6 +8,7 @@
#include <libbuild2/utility.hxx>
#include <libbuild2/rule.hxx>
+#include <libbuild2/dyndep.hxx>
#include <libbuild2/file-cache.hxx>
#include <libbuild2/cc/types.hxx>
@@ -37,7 +38,8 @@ namespace build2
};
class LIBBUILD2_CC_SYMEXPORT compile_rule: public simple_rule,
- virtual common
+ virtual common,
+ dyndep_rule
{
public:
compile_rule (data&&);
@@ -91,45 +93,21 @@ namespace build2
const scope&,
action, const target&, linfo) const;
- // Mapping of include prefixes (e.g., foo in <foo/bar>) for auto-
- // generated headers to directories where they will be generated.
- //
- // We are using a prefix map of directories (dir_path_map) instead of
- // just a map in order to also cover sub-paths (e.g., <foo/more/bar> if
- // we continue with the example). Specifically, we need to make sure we
- // don't treat foobar as a sub-directory of foo.
- //
- // The priority is used to decide who should override whom. Lesser
- // values are considered higher priority. Note that we can have multiple
- // prefixless mapping (where priority is used to determine the order).
- // See append_prefixes() for details.
- //
- // @@ The keys should be normalized.
- //
- struct prefix_value
- {
- dir_path directory;
- size_t priority;
- };
- using prefix_map = dir_path_multimap<prefix_value>;
+ using prefix_map = dyndep_rule::prefix_map;
+ using srcout_map = dyndep_rule::srcout_map;
void
- append_prefixes (prefix_map&, const target&, const variable&) const;
+ append_prefixes (prefix_map&,
+ const scope&, const target&,
+ const variable&) const;
void
append_library_prefixes (appended_libraries&, prefix_map&,
const scope&,
- action, target&, linfo) const;
+ action, const target&, linfo) const;
prefix_map
- build_prefix_map (const scope&, action, target&, linfo) const;
-
- small_vector<const target_type*, 2>
- map_extension (const scope&, const string&, const string&) const;
-
- // Src-to-out re-mapping. See extract_headers() for details.
- //
- using srcout_map = path_map<dir_path>;
+ build_prefix_map (const scope&, action, const target&, linfo) const;
struct module_mapper_state;
@@ -143,7 +121,7 @@ namespace build2
pair<const file*, bool>
enter_header (action, const scope&, file&, linfo,
path&&, bool, bool,
- optional<prefix_map>&, srcout_map&) const;
+ optional<prefix_map>&, const srcout_map&) const;
optional<bool>
inject_header (action, file&, const file&, timestamp, bool) const;
diff --git a/libbuild2/cc/utility.cxx b/libbuild2/cc/utility.cxx
index ffe3e03..e02f85a 100644
--- a/libbuild2/cc/utility.cxx
+++ b/libbuild2/cc/utility.cxx
@@ -3,10 +3,6 @@
#include <libbuild2/cc/utility.hxx>
-#include <libbuild2/file.hxx>
-
-using namespace std;
-
namespace build2
{
namespace cc
@@ -17,58 +13,5 @@ namespace build2
const dir_path module_build_dir (dir_path (module_dir) /= "build");
const dir_path module_build_modules_dir (
dir_path (module_build_dir) /= "modules");
-
- void
- normalize_header (path& f)
- {
- // Interestingly, on most paltforms and with most compilers (Clang on
- // Linux being a notable exception) most system/compiler headers are
- // already normalized.
- //
- path_abnormality a (f.abnormalities ());
- if (a != path_abnormality::none)
- {
- // While we can reasonably expect this path to exit, things do go
- // south from time to time (like compiling under wine with file
- // wlantypes.h included as WlanTypes.h).
- //
- try
- {
- // If we have any parent components, then we have to verify the
- // normalized path matches realized.
- //
- path r;
- if ((a & path_abnormality::parent) == path_abnormality::parent)
- {
- r = f;
- r.realize ();
- }
-
- try
- {
- f.normalize ();
-
- // Note that we might still need to resolve symlinks in the
- // normalized path.
- //
- if (!r.empty () && f != r && path (f).realize () != r)
- f = move (r);
- }
- catch (const invalid_path&)
- {
- assert (!r.empty ()); // Shouldn't have failed if no `..`.
- f = move (r); // Fallback to realize.
- }
- }
- catch (const invalid_path&)
- {
- fail << "invalid header path '" << f.string () << "'";
- }
- catch (const system_error& e)
- {
- fail << "invalid header path '" << f.string () << "': " << e;
- }
- }
- }
}
}
diff --git a/libbuild2/cc/utility.hxx b/libbuild2/cc/utility.hxx
index 42e53e3..6ba4a20 100644
--- a/libbuild2/cc/utility.hxx
+++ b/libbuild2/cc/utility.hxx
@@ -9,6 +9,7 @@
#include <libbuild2/utility.hxx>
#include <libbuild2/target.hxx>
+#include <libbuild2/filesystem.hxx>
#include <libbuild2/bin/target.hxx>
#include <libbuild2/bin/utility.hxx>
@@ -51,29 +52,11 @@ namespace build2
// Normalize an absolute path to an existing header.
//
- // We used to just normalize the path but that could result in an invalid
- // path (e.g., for some system/compiler headers on CentOS 7 with Clang
- // 3.4) because of the symlinks (if a directory component is a symlink,
- // then any following `..` are resolved relative to the target; see
- // path::normalize() for background).
- //
- // Initially, to fix this, we realized (i.e., realpath(3)) it instead.
- // But that turned out also not to be quite right since now we have all
- // the symlinks resolved: conceptually it feels correct to keep the
- // original header names since that's how the user chose to arrange things
- // and practically this is how the compilers see/report them (e.g., the
- // GCC module mapper).
- //
- // So now we have a pretty elaborate scheme where we try to use the
- // normalized path if possible and fallback to realized. Normalized paths
- // will work for situations where `..` does not cross symlink boundaries,
- // which is the sane case. And for the insane case we only really care
- // about out-of-project files (i.e., system/compiler headers). In other
- // words, if you have the insane case inside your project, then you are on
- // your own.
- //
- void
- normalize_header (path&);
+ inline void
+ normalize_header (path& f)
+ {
+ normalize_external (f, "header");
+ }
}
}
diff --git a/libbuild2/dyndep.cxx b/libbuild2/dyndep.cxx
new file mode 100644
index 0000000..51fa7bc
--- /dev/null
+++ b/libbuild2/dyndep.cxx
@@ -0,0 +1,667 @@
+// file : libbuild2/dyndep.cxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#include <libbuild2/dyndep.hxx>
+
+#include <libbuild2/scope.hxx>
+#include <libbuild2/target.hxx>
+#include <libbuild2/context.hxx>
+#include <libbuild2/algorithm.hxx>
+#include <libbuild2/filesystem.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace build2
+{
+ bool dyndep_rule::
+ update (tracer& trace, action a, const target& t, timestamp ts)
+ {
+ // In particular, this function is used to make sure header dependencies
+ // are up to date.
+ //
+ // There would normally be a lot of headers for every source file (think
+ // all the system headers) and just calling execute_direct() on all of
+ // them can get expensive. At the same time, most of these headers are
+ // existing files that we will never be updating (again, system headers,
+ // for example) and the rule that will match them is the fallback
+ // file_rule. That rule has an optimization: it returns noop_recipe (which
+ // causes the target state to be automatically set to unchanged) if the
+ // file is known to be up to date. So we do the update "smartly".
+ //
+ const path_target* pt (t.is_a<path_target> ());
+
+ if (pt == nullptr)
+ ts = timestamp_unknown;
+
+ target_state os (t.matched_state (a));
+
+ if (os == target_state::unchanged)
+ {
+ if (ts == timestamp_unknown)
+ return false;
+ else
+ {
+ // We expect the timestamp to be known (i.e., existing file).
+ //
+ timestamp mt (pt->mtime ());
+ assert (mt != timestamp_unknown);
+ return mt > ts;
+ }
+ }
+ else
+ {
+ // We only want to return true if our call to execute() actually caused
+ // an update. In particular, the target could already have been in
+ // target_state::changed because of the dynamic dependency extraction
+ // run for some other target.
+ //
+ // @@ MT perf: so we are going to switch the phase and execute for
+ // any generated header.
+ //
+ phase_switch ps (t.ctx, run_phase::execute);
+ target_state ns (execute_direct (a, t));
+
+ if (ns != os && ns != target_state::unchanged)
+ {
+ l6 ([&]{trace << "updated " << t
+ << "; old state " << os
+ << "; new state " << ns;});
+ return true;
+ }
+ else
+ return ts != timestamp_unknown ? pt->newer (ts, ns) : false;
+ }
+ }
+
+ optional<bool> dyndep_rule::
+ inject_file (tracer& trace, const char* what,
+ action a, target& t,
+ const file& pt,
+ timestamp mt,
+ bool f)
+ {
+ // Even if failing we still use try_match() in order to issue consistent
+ // (with other places) diagnostics (rather than the generic "not rule to
+ // update ...").
+ //
+ if (!try_match (a, pt).first)
+ {
+ if (!f)
+ return nullopt;
+
+ diag_record dr;
+ dr << fail << what << ' ' << pt << " not found and no rule to "
+ << "generate it";
+
+ if (verb < 4)
+ dr << info << "re-run with --verbose=4 for more information";
+ }
+
+ bool r (update (trace, a, pt, mt));
+
+ // Add to our prerequisite target list.
+ //
+ t.prerequisite_targets[a].push_back (&pt);
+
+ return r;
+ }
+
+ // Reverse-lookup target type(s) from file name/extension.
+ //
+ // If the list of base target types is specified, then only these types and
+ // those derived from them are considered. Otherwise, any file-based type is
+ // considered but not the file type itself.
+ //
+ small_vector<const target_type*, 2> dyndep_rule::
+ map_extension (const scope& bs,
+ const string& n, const string& e,
+ const target_type* const* tts)
+ {
+ // We will just have to try all of the possible ones, in the "most
+ // likely to match" order.
+ //
+ auto test = [&bs, &n, &e] (const target_type& tt) -> bool
+ {
+ if (tt.default_extension != nullptr)
+ {
+ // Call the extension derivation function. Here we know that it will
+ // only use the target type and name from the target key so we can
+ // pass bogus values for the rest.
+ //
+ target_key tk {&tt, nullptr, nullptr, &n, nullopt};
+
+ // This is like prerequisite search.
+ //
+ optional<string> de (tt.default_extension (tk, bs, nullptr, true));
+
+ return de && *de == e;
+ }
+
+ return false;
+ };
+
+ small_vector<const target_type*, 2> r;
+
+ if (tts != nullptr)
+ {
+ // @@ What if these types are not known by this project? Maybe this
+ // should just be unified with the below loop? Need to make sure
+ // we don't rely on the order in which they are returned.
+ //
+ for (const target_type* const* p (tts); *p != nullptr; ++p)
+ if (test (**p))
+ r.push_back (*p);
+ }
+
+ // Next try target types derived from any of the base types (or file if
+ // there are no base types).
+ //
+ const target_type_map& ttm (bs.root_scope ()->root_extra->target_types);
+
+ for (auto i (ttm.type_begin ()), e (ttm.type_end ()); i != e; ++i)
+ {
+ const target_type& dt (i->second);
+
+ if (tts != nullptr)
+ {
+ for (const target_type* const* p (tts); *p != nullptr; ++p)
+ {
+ const target_type& bt (**p);
+
+ if (dt.is_a (bt))
+ {
+ if (dt != bt && test (dt))
+ r.push_back (&dt);
+
+ break;
+ }
+ }
+ }
+ else
+ {
+ // Anything file-derived but not the file itself.
+ //
+ if (dt.is_a<file> () && dt != file::static_type && test (dt))
+ r.push_back (&dt);
+ }
+ }
+
+ return r;
+ }
+
+ void dyndep_rule::
+ append_prefix (tracer& trace, prefix_map& m, const target& t, dir_path d)
+ {
+ // If the target directory is a sub-directory of the include directory,
+ // then the prefix is the difference between the two. Otherwise, leave it
+ // empty.
+ //
+ // The idea here is to make this "canonical" setup work auto-magically
+ // (using C/C++ #include's as an example):
+ //
+ // 1. We include all headers with a prefix, e.g., <foo/bar>.
+ //
+ // 2. The library target is in the foo/ sub-directory, e.g., /tmp/foo/.
+ //
+ // 3. The poptions variable contains -I/tmp.
+ //
+ dir_path p (t.dir.sub (d) ? t.dir.leaf (d) : dir_path ());
+
+ // We use the target's directory as out_base but that doesn't work well
+ // for targets that are stashed in subdirectories. So as a heuristics we
+ // are going to also enter the outer directories of the original prefix.
+ // It is, however, possible, that another directory after this one will
+ // produce one of these outer prefixes as its original prefix in which
+ // case we should override it.
+ //
+ // So we are going to assign the original prefix priority value 0
+ // (highest) and then increment it for each outer prefix.
+ //
+ auto enter = [&trace, &m] (dir_path p, dir_path d, size_t prio)
+ {
+ auto j (m.lower_bound (p)), e (m.end ());
+
+ if (j != e && j->first != p)
+ j = e;
+
+ if (j == m.end ())
+ {
+ if (verb >= 4)
+ trace << "new mapping for prefix '" << p << "'\n"
+ << " new mapping to " << d << " priority " << prio;
+
+ m.emplace (move (p), prefix_value {move (d), prio});
+ }
+ else if (p.empty ())
+ {
+ // For prefixless we keep all the entries since for them we have an
+ // extra check (target must be explicitly spelled out in a buildfile).
+ //
+ if (verb >= 4)
+ trace << "additional mapping for prefix '" << p << "'\n"
+ << " new mapping to " << d << " priority " << prio;
+
+ // Find the position where to insert according to the priority.
+ // For equal priorities we use the insertion order.
+ //
+ do
+ {
+ if (j->second.priority > prio)
+ break;
+ }
+ while (++j != e && j->first == p);
+
+ m.emplace_hint (j, move (p), prefix_value {move (d), prio});
+ }
+ else
+ {
+ prefix_value& v (j->second);
+
+ // We used to reject duplicates but it seems this can be reasonably
+ // expected to work according to the order of, say, -I options.
+ //
+ // Seeing that we normally have more "specific" -I paths first, (so
+ // that we don't pick up installed headers, etc), we ignore it.
+ //
+ if (v.directory == d)
+ {
+ if (v.priority > prio)
+ v.priority = prio;
+ }
+ else if (v.priority <= prio)
+ {
+ if (verb >= 4)
+ trace << "ignoring mapping for prefix '" << p << "'\n"
+ << " existing mapping to " << v.directory
+ << " priority " << v.priority << '\n'
+ << " another mapping to " << d << " priority " << prio;
+ }
+ else
+ {
+ if (verb >= 4)
+ trace << "overriding mapping for prefix '" << p << "'\n"
+ << " existing mapping to " << v.directory
+ << " priority " << v.priority << '\n'
+ << " new mapping to " << d << " priority " << prio;
+
+ v.directory = move (d);
+ v.priority = prio;
+ }
+ }
+ };
+
+ // Enter all outer prefixes, including prefixless.
+ //
+ // The prefixless part is fuzzy but seems to be doing the right thing
+ // ignoring/overriding-wise, at least in cases where one of the competing
+ // include search paths is a subdirectory of another.
+ //
+ for (size_t prio (0);; ++prio)
+ {
+ bool e (p.empty ());
+ enter ((e ? move (p) : p), (e ? move (d) : d), prio);
+ if (e)
+ break;
+ p = p.directory ();
+ }
+ }
+
+ bool dyndep_rule::srcout_builder::
+ next (dir_path&& d)
+ {
+ // Ignore any paths containing '.', '..' components. Allow any directory
+ // separators though (think -I$src_root/foo on Windows).
+ //
+ if (d.absolute () && d.normalized (false))
+ {
+ // If we have a candidate out_base, see if this is its src_base.
+ //
+ if (prev_ != nullptr)
+ {
+ const dir_path& bp (prev_->src_path ());
+
+ if (d.sub (bp))
+ {
+ if (diff_.empty () || d.leaf (bp) == diff_)
+ {
+ // We've got a pair.
+ //
+ map_.emplace (move (d), prev_->out_path () / diff_);
+ prev_ = nullptr; // Taken.
+ return true;
+ }
+ }
+
+ // Not a pair. Fall through to consider as out_base.
+ //
+ prev_ = nullptr;
+ }
+
+ // See if this path is inside a project with an out-of-tree build and is
+ // in the out directory tree.
+ //
+ const scope& bs (ctx_.scopes.find_out (d));
+ if (bs.root_scope () != nullptr)
+ {
+ if (!bs.out_eq_src ())
+ {
+ const dir_path& bp (bs.out_path ());
+
+ bool e;
+ if ((e = (d == bp)) || d.sub (bp))
+ {
+ prev_ = &bs;
+ if (e)
+ diff_.clear ();
+ else
+ diff_ = d.leaf (bp);
+ }
+ }
+ }
+ }
+ else
+ prev_ = nullptr;
+
+ return false;
+ }
+
+ pair<const file*, bool> dyndep_rule::
+ enter_file (tracer& trace, const char* what,
+ action a, const scope& bs, target& t,
+ path&& f, bool cache, bool norm,
+ const function<map_extension_func>& map_extension,
+ const target_type& fallback,
+ const function<prefix_map_func>& get_pfx_map,
+ const srcout_map& so_map)
+ {
+ // Find or maybe insert the target. The directory is only moved from if
+ // insert is true. Note that it must be normalized.
+ //
+ auto find = [&trace, what, &t,
+ &map_extension, &fallback] (dir_path&& d,
+ path&& f,
+ bool insert) -> const file*
+ {
+ // Split the file into its name part and extension. Here we can assume
+ // the name part is a valid filesystem name.
+ //
+ // Note that if the file has no extension, we record an empty extension
+ // rather than NULL (which would signify that the default extension
+ // should be added).
+ //
+ string e (f.extension ());
+ string n (move (f).string ());
+
+ if (!e.empty ())
+ n.resize (n.size () - e.size () - 1); // One for the dot.
+
+ // See if this directory is part of any project and if so determine
+ // the target type.
+ //
+ // While at it also determine if this target is from the src or out
+ // tree of said project.
+ //
+ dir_path out;
+
+ // It's possible the extension-to-target type mapping is ambiguous (for
+ // example, because both C and X-language headers use the same .h
+ // extension). In this case we will first try to find one that matches
+ // an explicit target (similar logic to when insert is false).
+ //
+ small_vector<const target_type*, 2> tts;
+
+ // Note that the path can be in out or src directory and the latter
+ // can be associated with multiple scopes. So strictly speaking we
+ // need to pick one that is "associated" with us. But that is still a
+ // TODO (see scope_map::find() for details) and so for now we just
+ // pick the first one (it's highly unlikely the source file extension
+ // mapping will differ based on the configuration).
+ //
+ {
+ const scope& bs (**t.ctx.scopes.find (d).first);
+ if (const scope* rs = bs.root_scope ())
+ {
+ if (map_extension != nullptr)
+ tts = map_extension (bs, n, e);
+
+ if (!bs.out_eq_src () && d.sub (bs.src_path ()))
+ out = out_src (d, *rs);
+ }
+ }
+
+ // If it is outside any project, or the project doesn't have such an
+ // extension, use the fallback target type.
+ //
+ if (tts.empty ())
+ {
+ // If the project doesn't "know" this extension then we can't possibly
+ // find an explicit target of this type.
+ //
+ if (!insert)
+ {
+ l6 ([&]{trace << "unknown " << what << ' ' << n << " extension '"
+ << e << "'";});
+ return nullptr;
+ }
+
+ tts.push_back (&fallback);
+ }
+
+ // Find or insert target.
+ //
+ // Note that in case of the target type ambiguity we first try to find
+ // an explicit target that resolves this ambiguity.
+ //
+ const target* r (nullptr);
+
+ if (!insert || tts.size () > 1)
+ {
+ // Note that we skip any target type-specific searches (like for an
+ // existing file) and go straight for the target object since we
+ // need to find the target explicitly spelled out.
+ //
+ // Also, it doesn't feel like we should be able to resolve an
+ // absolute path with a spelled-out extension to multiple targets.
+ //
+ for (const target_type* tt: tts)
+ {
+ if ((r = t.ctx.targets.find (*tt, d, out, n, e, trace)) != nullptr)
+ break;
+ else
+ l6 ([&]{trace << "no targe with target type " << tt->name;});
+ }
+
+ // Note: we can't do this because of the in-source builds where there
+ // won't be explicit targets for non-generated files.
+ //
+ // This should be harmless, however, since in our world generated file
+ // are spelled-out as explicit targets. And if not, we will still get
+ // an error, just a bit less specific.
+ //
+#if 0
+ if (r == nullptr && insert)
+ {
+ f = d / n;
+ if (!e.empty ())
+ {
+ f += '.';
+ f += e;
+ }
+
+ diag_record dr (fail);
+ dr << "ambiguous mapping of " << what ' ' << f << " to target type";
+ for (const target_type* tt: tts)
+ dr << info << "could be " << tt->name << "{}";
+ dr << info << "spell-out its target to resolve this ambiguity";
+ }
+#endif
+ }
+
+ // @@ OPT: move d, out, n
+ //
+ if (r == nullptr && insert)
+ r = &search (t, *tts[0], d, out, n, &e, nullptr);
+
+ return static_cast<const file*> (r);
+ };
+
+ // If it's not absolute then it either does not (yet) exist or is a
+ // relative ""-include (see init_args() for details). Reduce the second
+ // case to absolute.
+ //
+ // Note: we now always use absolute path to the translation unit so this
+ // no longer applies. But let's keep it for posterity.
+ //
+#if 0
+ if (f.relative () && rels.relative ())
+ {
+ // If the relative source path has a directory component, make sure it
+ // matches since ""-include will always start with that (none of the
+ // compilers we support try to normalize this path). Failed that we may
+ // end up searching for a generated header in a random (working)
+ // directory.
+ //
+ const string& fs (f.string ());
+ const string& ss (rels.string ());
+
+ size_t p (path::traits::rfind_separator (ss));
+
+ if (p == string::npos || // No directory.
+ (fs.size () > p + 1 &&
+ path::traits::compare (fs.c_str (), p, ss.c_str (), p) == 0))
+ {
+ path t (work / f); // The rels path is relative to work.
+
+ if (exists (t))
+ f = move (t);
+ }
+ }
+#endif
+
+ const file* pt (nullptr);
+ bool remapped (false);
+
+ // If still relative then it does not exist.
+ //
+ if (f.relative ())
+ {
+ // This is probably as often an error as an auto-generated file, so
+ // trace at level 4.
+ //
+ l4 ([&]{trace << "non-existent " << what << " '" << f << "'";});
+
+ f.normalize ();
+
+ // The relative path might still contain '..' (e.g., ../foo.hxx;
+ // presumably ""-include'ed). We don't attempt to support auto-
+ // generated files with such inclusion styles.
+ //
+ if (get_pfx_map != nullptr && f.normalized ())
+ {
+ const prefix_map& pfx_map (get_pfx_map (a, bs, t));
+
+ // First try the whole file. Then just the directory.
+ //
+ // @@ Has to be a separate map since the prefix can be the same as
+ // the file name.
+ //
+ // auto i (pfx_map->find (f));
+
+ // Find the most qualified prefix of which we are a sub-path.
+ //
+ if (!pfx_map.empty ())
+ {
+ dir_path d (f.directory ());
+ auto p (pfx_map.sup_range (d));
+
+ if (p.first != p.second)
+ {
+ // Note that we can only have multiple entries for the
+ // prefixless mapping.
+ //
+ dir_path pd; // Reuse.
+ for (auto i (p.first); i != p.second; ++i)
+ {
+ // Note: value in pfx_map is not necessarily canonical.
+ //
+ pd = i->second.directory;
+ pd.canonicalize ();
+
+ l4 ([&]{trace << "try prefix '" << d << "' mapped to " << pd;});
+
+ // If this is a prefixless mapping, then only use it if we can
+ // resolve it to an existing target (i.e., it is explicitly
+ // spelled out in a buildfile). @@ Hm, I wonder why, it's not
+ // like we can generate any file without an explicit target.
+ // Maybe for diagnostics (i.e., we will actually try to build
+ // something there instead of just saying no mapping).
+ //
+ pt = find (pd / d, f.leaf (), !i->first.empty ());
+ if (pt != nullptr)
+ {
+ f = pd / f;
+ l4 ([&]{trace << "mapped as auto-generated " << f;});
+ break;
+ }
+ else
+ l4 ([&]{trace << "no explicit target in " << pd;});
+ }
+ }
+ else
+ l4 ([&]{trace << "no prefix map entry for '" << d << "'";});
+ }
+ else
+ l4 ([&]{trace << "prefix map is empty";});
+ }
+ }
+ else
+ {
+ // Normalize the path unless it comes from the depdb, in which case
+ // we've already done that (normally). This is also where we handle
+ // src-out remap (again, not needed if cached).
+ //
+ if (!cache || norm)
+ normalize_external (f, what);
+
+ if (!cache)
+ {
+ if (!so_map.empty ())
+ {
+ // Find the most qualified prefix of which we are a sub-path.
+ //
+ auto i (so_map.find_sup (f));
+ if (i != so_map.end ())
+ {
+ // Ok, there is an out tree for this file. Remap to a path from
+ // the out tree and see if there is a target for it. Note that the
+ // value in so_map is not necessarily canonical.
+ //
+ dir_path d (i->second);
+ d /= f.leaf (i->first).directory ();
+ d.canonicalize ();
+
+ pt = find (move (d), f.leaf (), false); // d is not moved from.
+
+ if (pt != nullptr)
+ {
+ path p (d / f.leaf ());
+ l4 ([&]{trace << "remapping " << f << " to " << p;});
+ f = move (p);
+ remapped = true;
+ }
+ }
+ }
+ }
+
+ if (pt == nullptr)
+ {
+ l6 ([&]{trace << "entering " << f;});
+ pt = find (f.directory (), f.leaf (), true);
+ }
+ }
+
+ return make_pair (pt, remapped);
+ }
+}
diff --git a/libbuild2/dyndep.hxx b/libbuild2/dyndep.hxx
new file mode 100644
index 0000000..3ba0c09
--- /dev/null
+++ b/libbuild2/dyndep.hxx
@@ -0,0 +1,168 @@
+// file : libbuild2/dyndep.hxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_DYNDEP_HXX
+#define LIBBUILD2_DYNDEP_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/forward.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/action.hxx>
+#include <libbuild2/target.hxx>
+
+#include <libbuild2/export.hxx>
+
+// Additional functionality that is normally only useful for implementing
+// rules with dynamic dependencies.
+//
+namespace build2
+{
+ class LIBBUILD2_SYMEXPORT dyndep_rule
+ {
+ public:
+ // Update the target during the match phase. Return true if it has changed
+ // or if the passed timestamp is not timestamp_unknown and is older than
+ // the target.
+ //
+ static bool
+ update (tracer&, action, const target&, timestamp);
+
+ // Update and add to the list of prerequisite targets a prerequisite file
+ // target.
+ //
+ // Return the indication of whether it has changed or, if the passed
+ // timestamp is not timestamp_unknown, is older than this timestamp. If
+ // the prerequisite target does not exists nor can be generated (no rule),
+ // then issue diagnostics and fail if the fail argument is true and return
+ // nullopt otherwise.
+ //
+ static optional<bool>
+ inject_file (tracer&, const char* what,
+ action, target&,
+ const file& prerequiste,
+ timestamp,
+ bool fail);
+
+ // Reverse-lookup target type(s) from file name/extension.
+ //
+ // If the list of base target types is specified, then only these types
+ // and those derived from them are considered. Otherwise, any file-based
+ // type is considered but not the file type itself.
+ //
+ static small_vector<const target_type*, 2>
+ map_extension (const scope& base,
+ const string& name, const string& ext,
+ const target_type* const* bases);
+
+ // Mapping of inclusion prefixes (e.g., foo in #include <foo/bar>) for
+ // auto-generated files to inclusion search paths (e.g. -I) where they
+ // will be generated.
+ //
+ // We are using a prefix map of directories (dir_path_map) instead of just
+ // a map in order to also cover sub-paths (e.g., #include <foo/more/bar>
+ // if we continue with the example). Specifically, we need to make sure we
+ // don't treat foobar as a sub-directory of foo.
+ //
+ // The priority is used to decide who should override whom. Lesser values
+ // are considered higher priority. Note that we allow multiple prefixless
+ // mapping (where priority is used to determine the order). For details,
+ // see append_prefix().
+ //
+ // Note that the keys should be normalized.
+ //
+ struct prefix_value
+ {
+ dir_path directory;
+ size_t priority;
+ };
+
+ using prefix_map = dir_path_multimap<prefix_value>;
+
+ // Add the specified absolute and normalized inclusion search path into
+ // the prefix map of the specified target.
+ //
+ static void
+ append_prefix (tracer&, prefix_map&, const target&, dir_path);
+
+ // Mapping of src inclusion search paths to the corresponding out paths
+ // for auto-generated files re-mapping. See cc::extract_headers() for
+ // background.
+ //
+ // Note that we use path_map instead of dir_path_map to allow searching
+ // using path (file path).
+ //
+ using srcout_map = path_map<dir_path>;
+
+ class LIBBUILD2_SYMEXPORT srcout_builder
+ {
+ public:
+ srcout_builder (context& ctx, srcout_map& map): ctx_ (ctx), map_ (map) {}
+
+ // Process next -I path. Return true if an entry was added to the map,
+ // in which case the passed path is moved from.
+ //
+ bool
+ next (dir_path&&);
+
+ // Skip the previously cached first half.
+ //
+ void
+ skip ()
+ {
+ prev_ = nullptr;
+ }
+
+ private:
+ context& ctx_;
+ srcout_map& map_;
+
+ // Previous -I's innermost scope if out_base plus the difference between
+ // the scope path and the -I path (normally empty).
+ //
+ const scope* prev_ = nullptr;
+ dir_path diff_;
+ };
+
+ // Enter a prerequisite file as a target. If the path is relative, then
+ // assume this a non-existent generated file.
+ //
+ // Depending on the cache flag, the path is assumed to either have come
+ // from the depdb cache or from the compiler run. In the former case
+ // assume the path is already normalized unless the normalize flag is
+ // true.
+ //
+ // Return the file target and an indication of whether it was remapped or
+ // NULL if the file does not exist and cannot be generated. In the latter
+ // case the passed file path is guaranteed to still be valid but might
+ // have been adjusted (e.g., normalized, etc).
+ //
+ // The map_extension function is used to reverse-map a file extension to
+ // the target type. The fallback target type is used if it's NULL or
+ // didn't return anything but only in situations where we are sure the
+ // file is (or should be there; see the implementation for details).
+ //
+ // The prefix map function is only called if this is a non-existent
+ // generated file (so it can be initialized lazily). If it's NULL, then
+ // generated files will not be supported. The srcout map is only consulted
+ // if cache is false (so its initialization can be delayed until the call
+ // with cache=false).
+ //
+ using map_extension_func = small_vector<const target_type*, 2> (
+ const scope& base, const string& name, const string& ext);
+
+ using prefix_map_func = const prefix_map& (
+ action, const scope& base, const target&);
+
+ static pair<const file*, bool>
+ enter_file (tracer&, const char* what,
+ action, const scope& base, target&,
+ path&& prerequisite, bool cache, bool norm,
+ const function<map_extension_func>&,
+ const target_type& fallback,
+ const function<prefix_map_func>&,
+ const srcout_map&);
+ };
+}
+
+#endif // LIBBUILD2_DYNDEP_HXX
diff --git a/libbuild2/filesystem.cxx b/libbuild2/filesystem.cxx
index fbe145c..2e3309d 100644
--- a/libbuild2/filesystem.cxx
+++ b/libbuild2/filesystem.cxx
@@ -323,4 +323,59 @@ namespace build2
fail << "unable to set path " << p << " permissions: " << e;
}
}
+
+ void
+ normalize_external (path& f, const char* what)
+ {
+ // The main motivating case for this logic are C/C++ headers.
+ //
+ // Interestingly, on most paltforms and with most compilers (Clang on
+ // Linux being a notable exception) most system/compiler headers are
+ // already normalized.
+ //
+ path_abnormality a (f.abnormalities ());
+ if (a != path_abnormality::none)
+ {
+ // While we can reasonably expect this path to exit, things do go south
+ // from time to time (like compiling under wine with file wlantypes.h
+ // included as WlanTypes.h).
+ //
+ try
+ {
+ // If we have any parent components, then we have to verify the
+ // normalized path matches realized.
+ //
+ path r;
+ if ((a & path_abnormality::parent) == path_abnormality::parent)
+ {
+ r = f;
+ r.realize ();
+ }
+
+ try
+ {
+ f.normalize ();
+
+ // Note that we might still need to resolve symlinks in the
+ // normalized path.
+ //
+ if (!r.empty () && f != r && path (f).realize () != r)
+ f = move (r);
+ }
+ catch (const invalid_path&)
+ {
+ assert (!r.empty ()); // Shouldn't have failed if no `..`.
+ f = move (r); // Fallback to realize.
+ }
+ }
+ catch (const invalid_path&)
+ {
+ fail << "invalid " << what << " path '" << f.string () << "'";
+ }
+ catch (const system_error& e)
+ {
+ fail << "invalid " << what << " path '" << f.string () << "': " << e;
+ }
+ }
+ }
}
diff --git a/libbuild2/filesystem.hxx b/libbuild2/filesystem.hxx
index ee7ba9a..565e832 100644
--- a/libbuild2/filesystem.hxx
+++ b/libbuild2/filesystem.hxx
@@ -189,6 +189,35 @@ namespace build2
LIBBUILD2_SYMEXPORT void
path_perms (const path&, permissions);
+
+ // Normalize an absolute path to an existing file that may reside outside of
+ // any project and could involve funny filesystem business (e.g., relative
+ // directory symlinks). For example, a C/C++ header path returned by a
+ // compiler which could be a system header.
+ //
+ // We used to just normalize such a path but that could result in an invalid
+ // path (e.g., for some system/compiler headers on CentOS 7 with Clang 3.4)
+ // because of the symlinks (if a directory component is a symlink, then any
+ // following `..` are resolved relative to the target; see path::normalize()
+ // for background).
+ //
+ // Initially, to fix this, we realized (i.e., realpath(3)) it instead. But
+ // that turned out also not to be quite right since now we have all the
+ // symlinks resolved: conceptually it feels correct to keep the original
+ // header names since that's how the user chose to arrange things and
+ // practically this is how compilers see/report them (e.g., the GCC module
+ // mapper).
+ //
+ // So now we have a pretty elaborate scheme where we try to use the
+ // normalized path if possible and fallback to realized. Normalized paths
+ // will work for situations where `..` does not cross symlink boundaries,
+ // which is the sane case. And for the insane case we only really care
+ // about out-of-project files (i.e., system/compiler headers). In other
+ // words, if you have the insane case inside your project, then you are on
+ // your own.
+ //
+ LIBBUILD2_SYMEXPORT void
+ normalize_external (path&, const char* what);
}
#include <libbuild2/filesystem.ixx>
diff --git a/libbuild2/make-parser.cxx b/libbuild2/make-parser.cxx
new file mode 100644
index 0000000..d076a0a
--- /dev/null
+++ b/libbuild2/make-parser.cxx
@@ -0,0 +1,137 @@
+// file : libbuild2/make-parser.cxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#include <libbuild2/make-parser.hxx>
+
+#include <libbuild2/diagnostics.hxx>
+
+namespace build2
+{
+ auto make_parser::
+ next (const string& l,
+ size_t& p,
+ const location& ll,
+ bool strict) -> pair<type, string>
+ {
+ assert (state != end);
+
+ pair<string, bool> r (
+ next (l, p, !strict ? state == prereqs : optional<bool> ()));
+
+ type t (state == prereqs ? type::prereq : type::target);
+
+ // Deal with the end.
+ //
+ if (r.second)
+ {
+ if (state == begin && r.first.empty ())
+ ; // Skip leading blank line.
+ else
+ {
+ if (state != prereqs)
+ fail (ll) << "end of make dependency declaration before ':'";
+
+ state = end;
+ }
+ }
+ // Deal with the first target.
+ //
+ else if (state == begin && !r.first.empty ())
+ state = targets;
+
+ // Deal with `:`.
+ //
+ if (p != l.size () && l[p] == ':')
+ {
+ switch (state)
+ {
+ case begin: fail (ll) << "':' before make target"; break;
+ case targets: state = prereqs; break;
+ case prereqs: fail (ll) << "':' after make prerequisite"; break;
+ case end: break;
+ }
+
+ if (++p == l.size ())
+ state = end; // Not a mere optimization: the caller will get next line.
+ }
+
+ return pair<type, string> (t, move (r.first));
+ }
+
+ pair<string, bool> make_parser::
+ next (const string& l, size_t& p, optional<bool> prereq)
+ {
+ size_t n (l.size ());
+
+ // Skip leading spaces.
+ //
+ for (; p != n && l[p] == ' '; p++) ;
+
+ // Lines containing multiple targets/prerequisites are customarily 80
+ // characters max.
+ //
+ string r;
+ r.reserve (n - p);
+
+ // Scan the next target/prerequisite while watching out for escape
+ // sequences.
+ //
+ // @@ Can't we do better for the (common) case where nothing is escaped?
+ //
+ for (char c, q (prereq && *prereq ? '\0' : ':');
+ p != n && (c = l[p]) != ' ' && c != q; )
+ {
+ // If we have another character, then handle the escapes.
+ //
+ if (++p != n)
+ {
+ if (c == '\\')
+ {
+ // This may or may not be an escape sequence depending on whether
+ // what follows is "escapable".
+ //
+ switch (c = l[p])
+ {
+ case '\\':
+ case ' ':
+ case ':': ++p; break;
+ default: c = '\\'; // Restore.
+ }
+ }
+ else if (c == '$')
+ {
+ // Got to be another (escaped) '$'.
+ //
+ if (l[p] == '$')
+ ++p;
+ }
+ }
+ // Note that the newline escape is not necessarily separated with space.
+ //
+ else if (c == '\\')
+ {
+ --p;
+ break;
+ }
+
+ r += c;
+ }
+
+ // Skip trailing spaces.
+ //
+ for (; p != n && l[p] == ' '; p++) ;
+
+ // Skip final '\' and determine if this is the end.
+ //
+ bool e (false);
+ if (p == n - 1)
+ {
+ if (l[p] == '\\')
+ p++;
+ }
+ else if (p == n)
+ e = true;
+
+ return pair<string, bool> (move (r), e);
+ }
+}
diff --git a/libbuild2/make-parser.hxx b/libbuild2/make-parser.hxx
new file mode 100644
index 0000000..fac2215
--- /dev/null
+++ b/libbuild2/make-parser.hxx
@@ -0,0 +1,85 @@
+// file : libbuild2/make-parser.hxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_MAKE_PARSER_HXX
+#define LIBBUILD2_MAKE_PARSER_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/export.hxx>
+
+namespace build2
+{
+ // Make dependency declaration parser.
+ //
+ // The format is line-based (but with potential line continuations) so we
+ // parse one line at a time. This allows the caller to bail out early (for
+ // example, on encountering a non-existent generated file).
+ //
+ class LIBBUILD2_SYMEXPORT make_parser
+ {
+ public:
+ enum {begin, targets, prereqs, end} state = begin;
+
+ // Parse next target/prerequisite on a line starting from the specified
+ // position. Update the position to point to the start of the following
+ // target/prerequisite or l.size() if there is nothing left on this
+ // line. May return an empty string for a valid if unlikely dependency
+ // declarations (see below) or if passing leading blank lines (both of
+ // which should normally be just skipped). Issue diagnostics and throw
+ // failed if the declaration is invalid.
+ //
+ // If strict is false, then allow unescaped `:` in prerequisites.
+ //
+ // Note that the (p != l.size) should be in the do-while rather than in a
+ // while loop. In other words, except for the leading blank lines, the
+ // parser needs to see the blank line to correctly identify the end of the
+ // declaration. See make-parser.test.cxx for a recommended usage.
+ //
+ // To parse more than one declaration, reset the state to begin after
+ // reaching end.
+ //
+ enum class type {target, prereq};
+
+ pair<type, string>
+ next (const string&, size_t&, const location&, bool strict);
+
+ // Lower-level stateless API.
+ //
+ public:
+ // Parse next target/prerequisite on a line starting from the specified
+ // position. Return the target/prerequisite as well as an indication of
+ // whether the end of the dependency declaration was reached. Update the
+ // position to point to the start of the following target/prerequisite,
+ // `:`, or l.size() if there is nothing left on this line.
+ //
+ // Note that some broken tools (notably MinGW GCC) do not escape `:`
+ // properly. To tolerate such cases the caller may specify that what's
+ // being parsed is the prerequisite list in which case unescaped `:` will
+ // be treated literally.
+ //
+ // Note also that this function may return an empty string (with
+ // end=false) for a valid if unlikely dependency declaration, for example
+ // (using | to represent backslash):
+ //
+ // foo:|
+ // |
+ // bar
+ //
+ // It would also return an empty string (with end=true) if passed and
+ // empty or whitespace-only line.
+ //
+ // Note also that in the make language line continuations introduce a
+ // whitespace rather than just being remove. For example, the following
+ // declaration has two prerequisites:
+ //
+ // foo: bar|
+ // baz
+ //
+ static pair<string, bool>
+ next (const string&, size_t&, optional<bool> prereq = nullopt);
+ };
+}
+
+#endif // LIBBUILD2_MAKE_PARSER_HXX
diff --git a/libbuild2/make-parser.test.cxx b/libbuild2/make-parser.test.cxx
new file mode 100644
index 0000000..189407a
--- /dev/null
+++ b/libbuild2/make-parser.test.cxx
@@ -0,0 +1,90 @@
+// file : libbuild2/make-parser.test.cxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#include <iostream>
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/make-parser.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+#undef NDEBUG
+#include <cassert>
+
+using namespace std;
+
+namespace build2
+{
+ int
+ main (int, char* argv[])
+ {
+ bool strict (false);
+
+ // Fake build system driver, default verbosity.
+ //
+ init_diag (1);
+ init (nullptr, argv[0]);
+
+ path_name in ("<stdin>");
+
+ try
+ {
+ cin.exceptions (istream::badbit);
+
+ using make_state = make_parser;
+ using make_type = make_parser::type;
+
+ make_parser make;
+
+ location ll (in, 1);
+ for (string l; !eof (getline (cin, l)); ++ll.line)
+ {
+ if (make.state == make_state::end)
+ {
+ cout << endl;
+ make.state = make_state::begin;
+ }
+
+ // Skip leading blank lines to reduce output noise.
+ //
+ if (make.state == make_state::begin && l.empty ())
+ continue;
+
+ size_t pos (0);
+ do
+ {
+ pair<make_type, string> r (make.next (l, pos, ll, strict));
+
+ cout << (r.first == make_type::target ? 'T' : 'P');
+
+ if (!r.second.empty ())
+ cout << ' ' << r.second;
+
+ cout << endl;
+ }
+ while (pos != l.size ());
+ }
+
+ if (make.state != make_state::end && make.state != make_state::begin)
+ fail (ll) << "incomplete make dependency declaration";
+ }
+ catch (const io_error& e)
+ {
+ cerr << "unable to read stdin: " << e << endl;
+ return 1;
+ }
+ catch (const failed&)
+ {
+ return 1;
+ }
+
+ return 0;
+ }
+}
+
+int
+main (int argc, char* argv[])
+{
+ return build2::main (argc, argv);
+}
diff --git a/libbuild2/make-parser.test.testscript b/libbuild2/make-parser.test.testscript
new file mode 100644
index 0000000..6db00eb
--- /dev/null
+++ b/libbuild2/make-parser.test.testscript
@@ -0,0 +1,98 @@
+# file : libbuild2/make-parser.test.testscript
+# license : MIT; see accompanying LICENSE file
+
+: valid
+:
+$* <<EOI >>EOO
+ foo:
+
+ foo: bar
+
+ foo: bar baz
+
+ foo: bar \
+ baz
+
+ foo: bar\
+ baz
+
+ foo:\
+ bar baz\
+ fox
+
+ foo: bar \
+ \
+ baz
+
+ foo: bar\
+
+ foo bar: baz
+
+ foo \
+ bar: baz
+
+ foo \
+ bar \
+ : baz
+
+ \
+ foo: bar
+ EOI
+ T foo
+
+ T foo
+ P bar
+
+ T foo
+ P bar
+ P baz
+
+ T foo
+ P bar
+ P baz
+
+ T foo
+ P bar
+ P baz
+
+ T foo
+ P
+ P bar
+ P baz
+ P fox
+
+ T foo
+ P bar
+ P
+ P baz
+
+ T foo
+ P bar
+ P
+
+ T foo
+ T bar
+ P baz
+
+ T foo
+ T bar
+ P baz
+
+ T foo
+ T bar
+ T
+ P baz
+
+ T
+ T foo
+ P bar
+ EOO
+
+: lax
+:
+$* <<EOI >>EOO
+ foo: c:\tmp\bar
+ EOI
+ T foo
+ P c:\tmp\bar
+ EOO
diff --git a/libbuild2/script/parser.cxx b/libbuild2/script/parser.cxx
index fffe7bb..7722002 100644
--- a/libbuild2/script/parser.cxx
+++ b/libbuild2/script/parser.cxx
@@ -1313,7 +1313,7 @@ namespace build2
// Note that an option name and value can belong to different name
// chunks. That's why we parse the env builtin arguments in the chunking
// mode into the argument/location pair list up to the '--' separator
- // and parse this list into the variable sets/unsets afterwords.
+ // and parse this list into the variable sets/unsets afterwards.
//
// Align the size with environment_vars (double because of -u <var>
// which is two arguments).
diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx
index f3b5cad..12078c4 100644
--- a/libbuild2/script/run.cxx
+++ b/libbuild2/script/run.cxx
@@ -944,6 +944,83 @@ namespace build2
: path (c.program.recall_string ());
}
+ // Read out the stream content into a string. Throw io_error on the
+ // underlying OS error.
+ //
+ // If the execution deadline is specified, then turn the stream into the
+ // non-blocking mode reading its content in chunks and with a single
+ // operation otherwise. If the specified deadline is reached while
+ // reading the stream, then bail out for the successful deadline and
+ // fail otherwise. Note that in the former case the result will be
+ // incomplete, but we leave it to the caller to handle that.
+ //
+ // Note that on Windows we can only turn pipe file descriptors into the
+ // non-blocking mode. Thus, we have no choice but to read from
+ // descriptors of other types synchronously there. That implies that we
+ // can potentially block indefinitely reading a file and missing the
+ // deadline on Windows. Note though, that the user can normally rewrite
+ // the command, for example, `set foo <<<file` with `cat file | set foo`
+ // to avoid this problem.
+ //
+ static string
+ read (auto_fd in,
+#ifndef _WIN32
+ bool,
+#else
+ bool pipe,
+#endif
+ const optional<deadline>& dl,
+ const command& deadline_cmd,
+ const location& ll)
+ {
+ string r;
+ ifdstream cin;
+
+#ifndef _WIN32
+ if (dl)
+#else
+ if (dl && pipe)
+#endif
+ {
+ fdselect_set fds {in.get ()};
+ cin.open (move (in), fdstream_mode::non_blocking);
+
+ const timestamp& dlt (dl->value);
+
+ for (char buf[4096];; )
+ {
+ timestamp now (system_clock::now ());
+
+ if (dlt <= now || ifdselect (fds, dlt - now) == 0)
+ {
+ if (!dl->success)
+ fail (ll) << cmd_path (deadline_cmd)
+ << " terminated: execution timeout expired";
+ else
+ break;
+ }
+
+ streamsize n (cin.readsome (buf, sizeof (buf)));
+
+ // Bail out if eos is reached.
+ //
+ if (n == 0)
+ break;
+
+ r.append (buf, n);
+ }
+ }
+ else
+ {
+ cin.open (move (in));
+ r = cin.read_text ();
+ }
+
+ cin.close ();
+
+ return r;
+ }
+
// The set pseudo-builtin: set variable from the stdin input.
//
// set [-e|--exact] [(-n|--newline)|(-w|--whitespace)] [<attr>] <var>
@@ -952,11 +1029,7 @@ namespace build2
set_builtin (environment& env,
const strings& args,
auto_fd in,
-#ifndef _WIN32
- bool,
-#else
bool pipe,
-#endif
const optional<deadline>& dl,
const command& deadline_cmd,
const location& ll)
@@ -988,70 +1061,9 @@ namespace build2
fail (ll) << "set: empty variable name";
// Read out the stream content into a string while keeping an eye on
- // the deadline. Then parse it according to the split mode.
+ // the deadline.
//
- string s;
- {
- ifdstream cin;
-
- // If the execution deadline is specified, then turn the stream into
- // the non-blocking mode reading its content in chunks and with a
- // single operation otherwise. If the specified deadline is reached
- // while reading the stream, then bail out for the successful
- // deadline and fail otherwise. Note that in the former case the
- // variable value will be incomplete, but we leave it to the caller
- // to handle that.
- //
- // Note that on Windows we can only turn pipe file descriptors into
- // the non-blocking mode. Thus, we have no choice but to read from
- // descriptors of other types synchronously there. That implies that
- // we can potentially block indefinitely reading a file and missing
- // the deadline on Windows. Note though, that the user can always
- // rewrite `set foo <<<file` with `cat file | set foo` to avoid this
- // problem.
- //
-#ifndef _WIN32
- if (dl)
-#else
- if (dl && pipe)
-#endif
- {
- fdselect_set fds {in.get ()};
- cin.open (move (in), fdstream_mode::non_blocking);
-
- const timestamp& dlt (dl->value);
-
- for (char buf[4096];; )
- {
- timestamp now (system_clock::now ());
-
- if (dlt <= now || ifdselect (fds, dlt - now) == 0)
- {
- if (!dl->success)
- fail (ll) << cmd_path (deadline_cmd)
- << " terminated: execution timeout expired";
- else
- break;
- }
-
- streamsize n (cin.readsome (buf, sizeof (buf)));
-
- // Bail out if eos is reached.
- //
- if (n == 0)
- break;
-
- s.append (buf, n);
- }
- }
- else
- {
- cin.open (move (in));
- s = cin.read_text ();
- }
-
- cin.close ();
- }
+ string s (read (move (in), pipe, dl, deadline_cmd, ll));
// Parse the stream content into the variable value.
//
@@ -1137,7 +1149,7 @@ namespace build2
}
catch (const io_error& e)
{
- fail (ll) << "set: " << e;
+ fail (ll) << "set: unable to read from stdin: " << e;
}
catch (const cli::exception& e)
{
@@ -1202,14 +1214,42 @@ namespace build2
auto_fd ifd,
size_t ci, size_t li, const location& ll,
bool diag,
+ string* output,
optional<deadline> dl = nullopt,
const command* dl_cmd = nullptr, // env -t <cmd>
pipe_command* prev_cmd = nullptr)
{
tracer trace ("script::run_pipe");
- if (bc == ec) // End of the pipeline.
+ // At the end of the pipeline read out its stdout, if requested.
+ //
+ if (bc == ec)
+ {
+ if (output != nullptr)
+ {
+ // The pipeline can't be empty.
+ //
+ assert (ifd != nullfd && prev_cmd != nullptr);
+
+ const command& c (prev_cmd->cmd);
+
+ try
+ {
+ *output = read (move (ifd),
+ true /* pipe */,
+ dl,
+ dl_cmd != nullptr ? *dl_cmd : c,
+ ll);
+ }
+ catch (const io_error& e)
+ {
+ fail (ll) << "io error reading " << cmd_path (c) << " output: "
+ << e;
+ }
+ }
+
return true;
+ }
// The overall plan is to run the first command in the pipe, reading its
// input from the file descriptor passed (or, for the first command,
@@ -1261,6 +1301,11 @@ namespace build2
command_pipe::const_iterator nc (bc + 1);
bool last (nc == ec);
+ // Make sure that stdout is not redirected if meant to be read.
+ //
+ if (last && output != nullptr && c.out)
+ fail (ll) << "stdout cannot be redirected";
+
// True if the process path is not pre-searched and the program path
// still needs to be resolved.
//
@@ -1272,7 +1317,7 @@ namespace build2
const redirect& in ((c.in ? *c.in : env.in).effective ());
- const redirect* out (!last
+ const redirect* out (!last || output != nullptr
? nullptr // stdout is piped.
: &(c.out ? *c.out : env.out).effective ());
@@ -1340,6 +1385,9 @@ namespace build2
if (c.out)
fail (ll) << program << " builtin stdout cannot be redirected";
+ if (output != nullptr)
+ fail (ll) << program << " builtin stdout cannot be read";
+
if (c.err)
fail (ll) << program << " builtin stderr cannot be redirected";
@@ -1529,6 +1577,9 @@ namespace build2
if (c.out)
fail (ll) << "set builtin stdout cannot be redirected";
+ if (output != nullptr)
+ fail (ll) << "set builtin stdout cannot be read";
+
if (c.err)
fail (ll) << "set builtin stderr cannot be redirected";
@@ -1661,7 +1712,7 @@ namespace build2
// script failures investigation and, for example, for validation
// "tightening".
//
- if (last)
+ if (last && out != nullptr)
ofd.out = open (*out, 1, osp);
else
{
@@ -1690,7 +1741,7 @@ namespace build2
fail (ll) << "stdout and stderr redirected to each other";
auto_fd& self (mo ? ofd.out : efd);
- auto_fd& other (mo ? efd : ofd.out);
+ auto_fd& other (mo ? efd : ofd.out);
try
{
@@ -1704,9 +1755,9 @@ namespace build2
}
}
- // All descriptors should be open to the date.
+ // By now all descriptors should be open.
//
- assert (ofd.out.get () != -1 && efd.get () != -1);
+ assert (ofd.out != nullfd && efd != nullfd);
// Wait for a process/builtin to complete until the deadline is reached
// and return the underlying wait function result (optional<something>).
@@ -1756,7 +1807,7 @@ namespace build2
// is exiting on Windows, etc) then just ignore this, postponing
// the potential failure till the kill() call.
//
- l5 ([&]{trace (c->loc) <<"unable to terminate " << prog (c)
+ l5 ([&]{trace (c->loc) << "unable to terminate " << prog (c)
<< ": " << e;});
}
@@ -2123,6 +2174,7 @@ namespace build2
nc, ec,
move (ofd.in),
ci + 1, li, ll, diag,
+ output,
dl, dl_cmd,
&pc);
@@ -2249,6 +2301,7 @@ namespace build2
nc, ec,
move (ofd.in),
ci + 1, li, ll, diag,
+ output,
dl, dl_cmd,
&pc);
@@ -2376,7 +2429,7 @@ namespace build2
if (success)
success =
check_output (pr, esp, isp, err, ll, env, diag, "stderr") &&
- (!last ||
+ (out == nullptr ||
check_output (pr, osp, isp, *out, ll, env, diag, "stdout"));
return success;
@@ -2386,7 +2439,8 @@ namespace build2
run_expr (environment& env,
const command_expr& expr,
size_t li, const location& ll,
- bool diag)
+ bool diag,
+ string* output)
{
// Commands are numbered sequentially throughout the expression
// starting with 1. Number 0 means the command is a single one.
@@ -2424,10 +2478,15 @@ namespace build2
// with false.
//
if (!((or_op && r) || (!or_op && !r)))
+ {
+ assert (!p.empty ());
+
r = run_pipe (env,
p.begin (), p.end (),
auto_fd (),
- ci, li, ll, print);
+ ci, li, ll, print,
+ output);
+ }
ci += p.size ();
}
@@ -2438,24 +2497,26 @@ namespace build2
void
run (environment& env,
const command_expr& expr,
- size_t li, const location& ll)
+ size_t li, const location& ll,
+ string* output)
{
// Note that we don't print the expression at any verbosity level
// assuming that the caller does this, potentially providing some
// additional information (command type, etc).
//
- if (!run_expr (env, expr, li, ll, true /* diag */))
+ if (!run_expr (env, expr, li, ll, true /* diag */, output))
throw failed (); // Assume diagnostics is already printed.
}
bool
run_if (environment& env,
const command_expr& expr,
- size_t li, const location& ll)
+ size_t li, const location& ll,
+ string* output)
{
// Note that we don't print the expression here (see above).
//
- return run_expr (env, expr, li, ll, false /* diag */);
+ return run_expr (env, expr, li, ll, false /* diag */, output);
}
void
diff --git a/libbuild2/script/run.hxx b/libbuild2/script/run.hxx
index 477dd88..8bc246c 100644
--- a/libbuild2/script/run.hxx
+++ b/libbuild2/script/run.hxx
@@ -38,11 +38,22 @@ namespace build2
// Location is the start position of this command line in the script. It
// can be used in diagnostics.
//
+ // Optionally, save the command output into the referenced variable. In
+ // this case assume that the expression contains a single pipline.
+ //
void
- run (environment&, const command_expr&, size_t index, const location&);
+ run (environment&,
+ const command_expr&,
+ size_t index,
+ const location&,
+ string* output = nullptr);
bool
- run_if (environment&, const command_expr&, size_t, const location&);
+ run_if (environment&,
+ const command_expr&,
+ size_t index,
+ const location&,
+ string* output = nullptr);
// Perform the registered special file cleanups in the direct order and
// then the regular cleanups in the reverse order.
diff --git a/libbuild2/script/script.hxx b/libbuild2/script/script.hxx
index d162900..81bc13c 100644
--- a/libbuild2/script/script.hxx
+++ b/libbuild2/script/script.hxx
@@ -495,7 +495,8 @@ namespace build2
// Register a cleanup. If the cleanup is explicit, then override the
// cleanup type if this path is already registered. Ignore implicit
- // registration of a path outside root directory (see below).
+ // registration of a path outside sandbox directory, if specified (see
+ // above).
//
void
clean (cleanup, bool implicit);