aboutsummaryrefslogtreecommitdiff
path: root/libbuild2
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2020-11-20 22:07:37 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2020-12-02 17:31:04 +0300
commit0ff39fd77b3127c7a250e7f817e34dfaecbcc208 (patch)
treeedb20351f3d44558201b5668823c191a8722d3a5 /libbuild2
parent41a6f8b7d3036708f36ea1b5bd5b8d4289428fe5 (diff)
Add support for buildscript depdb preamble
Diffstat (limited to 'libbuild2')
-rw-r--r--libbuild2/adhoc-rule-buildscript.cxx125
-rw-r--r--libbuild2/adhoc-rule-buildscript.hxx3
-rw-r--r--libbuild2/adhoc-rule-cxx.cxx3
-rw-r--r--libbuild2/adhoc-rule-cxx.hxx3
-rw-r--r--libbuild2/build/script/parser+body.test.testscript (renamed from libbuild2/build/script/parser+line.test.testscript)59
-rw-r--r--libbuild2/build/script/parser+depdb.test.testscript90
-rw-r--r--libbuild2/build/script/parser+diag.test.testscript37
-rw-r--r--libbuild2/build/script/parser.cxx340
-rw-r--r--libbuild2/build/script/parser.hxx41
-rw-r--r--libbuild2/build/script/parser.test.cxx75
-rw-r--r--libbuild2/build/script/script.cxx17
-rw-r--r--libbuild2/build/script/script.hxx22
-rw-r--r--libbuild2/parser.cxx25
-rw-r--r--libbuild2/recipe.hxx5
-rw-r--r--libbuild2/rule.hxx3
-rw-r--r--libbuild2/script/run.cxx135
16 files changed, 734 insertions, 249 deletions
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx
index 9470d3c..dad638e 100644
--- a/libbuild2/adhoc-rule-buildscript.cxx
+++ b/libbuild2/adhoc-rule-buildscript.cxx
@@ -23,7 +23,8 @@ using namespace std;
namespace build2
{
bool adhoc_buildscript_rule::
- recipe_text (context& ctx, const target& tg, string&& t, attributes& as)
+ recipe_text (context& ctx, const target& tg, const adhoc_actions& acts,
+ string&& t, attributes& as)
{
// Handle and erase recipe-specific attributes.
//
@@ -56,7 +57,7 @@ namespace build2
istringstream is (move (t));
build::script::parser p (ctx);
- script = p.pre_parse (tg,
+ script = p.pre_parse (tg, acts,
is, loc.file, loc.line + 1,
move (diag), as.loc);
@@ -87,14 +88,14 @@ namespace build2
if (script.depdb_clear)
os << ind << "depdb clear" << endl;
- script::dump (os, ind, script.depdb_lines);
+ script::dump (os, ind, script.depdb_preamble);
if (script.diag_line)
{
os << ind; script::dump (os, *script.diag_line, true /* newline */);
}
- script::dump (os, ind, script.lines);
+ script::dump (os, ind, script.body);
ind.resize (ind.size () - 2);
os << ind << string (braces, '}');
}
@@ -452,69 +453,43 @@ namespace build2
// Execute the custom dependency change tracking commands, if present.
//
- if (!script.depdb_lines.empty ())
- {
- bs = &t.base_scope ();
- rs = bs->root_scope ();
+ // Note that we share the environment between the execute_depdb_preamble()
+ // and execute_body() calls, which is not merely an optimization since
+ // variables set in the preamble must be available in the body.
+ //
+ // Creating the environment instance is not cheap so optimize for the
+ // common case where we don't have the depdb preamble and nothing to
+ // update.
+ //
+ bool depdb_preamble (!script.depdb_preamble.empty ());
- // While it would have been nice to reuse the environment for both
- // dependency tracking and execution, there are complications (creating
- // temporary directory, etc).
- //
- build::script::environment e (a, t, false /* temp_dir */);
- build::script::parser p (ctx);
+ if (!depdb_preamble)
+ {
+ if (dd.writing () || dd.mtime > mt)
+ update = true;
- for (const script::line& l: script.depdb_lines)
+ if (!update)
{
- names ns (p.execute_special (*rs, *bs, e, l));
+ dd.close ();
+ return *ps;
+ }
+ }
- // These should have been enforced during pre-parsing.
- //
- assert (!ns.empty ()); // <cmd> ... <newline>
- assert (l.tokens.size () > 2); // 'depdb' <cmd> ... <newline>
+ build::script::environment env (a, t, false /* temp_dir */);
+ build::script::default_runner r;
- const string& cmd (ns[0].value);
+ if (depdb_preamble)
+ {
+ bs = &t.base_scope ();
+ rs = bs->root_scope ();
- location loc (l.tokens[0].location ());
+ if (script.depdb_preamble_temp_dir)
+ env.set_temp_dir_variable ();
- if (cmd == "hash")
- {
- sha256 cs;
- for (auto i (ns.begin () + 1); i != ns.end (); ++i) // Skip <cmd>.
- to_checksum (cs, *i);
-
- if (dd.expect (cs.string ()) != nullptr)
- l4 ([&] {
- diag_record dr (trace);
- dr << "'depdb hash' argument change forcing update of " << t <<
- info (loc); script::dump (dr.os, l);
- });
- }
- else if (cmd == "string")
- {
- string s;
- try
- {
- s = convert<string> (names (make_move_iterator (ns.begin () + 1),
- make_move_iterator (ns.end ())));
- }
- catch (const invalid_argument& e)
- {
- fail (l.tokens[2].location ())
- << "invalid 'depdb string' argument: " << e;
- }
+ build::script::parser p (ctx);
- if (dd.expect (s) != nullptr)
- l4 ([&] {
- diag_record dr (trace);
- dr << "'depdb string' argument change forcing update of "
- << t <<
- info (loc); script::dump (dr.os, l);
- });
- }
- else
- assert (false);
- }
+ r.enter (env, script.start_loc);
+ p.execute_depdb_preamble (*rs, *bs, env, script, r, dd);
}
// Update if depdb mismatch.
@@ -527,24 +502,38 @@ namespace build2
// If nothing changed, then we are done.
//
if (!update)
+ {
+ // Note that if we execute the depdb preamble but not the script body,
+ // we need to call the runner's leave() function explicitly (here and
+ // below).
+ //
+ if (depdb_preamble)
+ r.leave (env, script.end_loc);
+
return *ps;
+ }
if (!ctx.dry_run || verb != 0)
{
+ // Prepare to executing the script diag line and/or body.
+ //
+ // Note that it doesn't make much sense to use the temporary directory
+ // variable ($~) in the 'diag' builtin call, so we postpone setting it
+ // until the script body execution, that can potentially be omitted.
+ //
if (bs == nullptr)
{
bs = &t.base_scope ();
rs = bs->root_scope ();
}
- build::script::environment e (a, t, script.temp_dir);
build::script::parser p (ctx);
if (verb == 1)
{
if (script.diag_line)
{
- text << p.execute_special (*rs, *bs, e, *script.diag_line);
+ text << p.execute_special (*rs, *bs, env, *script.diag_line);
}
else
{
@@ -577,8 +566,10 @@ namespace build2
}
}
- build::script::default_runner r;
- p.execute (*rs, *bs, e, script, r);
+ if (script.body_temp_dir && !script.depdb_preamble_temp_dir)
+ env.set_temp_dir_variable ();
+
+ p.execute_body (*rs, *bs, env, script, r, !depdb_preamble);
if (!ctx.dry_run)
{
@@ -607,7 +598,11 @@ namespace build2
rm.cancel ();
}
}
+ else if (depdb_preamble)
+ r.leave (env, script.end_loc);
}
+ else if (depdb_preamble)
+ r.leave (env, script.end_loc);
t.mtime (system_clock::now ());
return target_state::changed;
@@ -629,7 +624,7 @@ namespace build2
const scope& bs (t.base_scope ());
const scope& rs (*bs.root_scope ());
- build::script::environment e (a, t, script.temp_dir, deadline);
+ build::script::environment e (a, t, script.body_temp_dir, deadline);
build::script::parser p (ctx);
if (verb == 1)
@@ -649,7 +644,7 @@ namespace build2
if (!ctx.dry_run || verb >= 2)
{
build::script::default_runner r;
- p.execute (rs, bs, e, script, r);
+ p.execute_body (rs, bs, e, script, r);
}
}
diff --git a/libbuild2/adhoc-rule-buildscript.hxx b/libbuild2/adhoc-rule-buildscript.hxx
index 89515eb..d98ffcf 100644
--- a/libbuild2/adhoc-rule-buildscript.hxx
+++ b/libbuild2/adhoc-rule-buildscript.hxx
@@ -43,7 +43,8 @@ namespace build2
: adhoc_rule ("<ad hoc buildscript recipe>", l, b) {}
virtual bool
- recipe_text (context&, const target&, string&&, attributes&) override;
+ recipe_text (context&, const target&, const adhoc_actions&,
+ string&&, attributes&) override;
virtual void
dump_attributes (ostream&) const override;
diff --git a/libbuild2/adhoc-rule-cxx.cxx b/libbuild2/adhoc-rule-cxx.cxx
index 3f52b53..55f8ccb 100644
--- a/libbuild2/adhoc-rule-cxx.cxx
+++ b/libbuild2/adhoc-rule-cxx.cxx
@@ -38,7 +38,8 @@ namespace build2
}
bool adhoc_cxx_rule::
- recipe_text (context&, const target&, string&& t, attributes&)
+ recipe_text (context&, const target&, const adhoc_actions&,
+ string&& t, attributes&)
{
code = move (t);
return true;
diff --git a/libbuild2/adhoc-rule-cxx.hxx b/libbuild2/adhoc-rule-cxx.hxx
index 038d2e1..dfcdf3f 100644
--- a/libbuild2/adhoc-rule-cxx.hxx
+++ b/libbuild2/adhoc-rule-cxx.hxx
@@ -61,7 +61,8 @@ namespace build2
optional<string> sep);
virtual bool
- recipe_text (context&, const target&, string&& t, attributes&) override;
+ recipe_text (context&, const target&, const adhoc_actions&,
+ string&&, attributes&) override;
virtual
~adhoc_cxx_rule () override;
diff --git a/libbuild2/build/script/parser+line.test.testscript b/libbuild2/build/script/parser+body.test.testscript
index 1b39265..11c2609 100644
--- a/libbuild2/build/script/parser+line.test.testscript
+++ b/libbuild2/build/script/parser+body.test.testscript
@@ -1,7 +1,62 @@
-# file : libbuild2/build/script/parser+line.test.testscript
+# file : libbuild2/build/script/parser+body.test.testscript
# license : MIT; see accompanying LICENSE file
-test.options += -d
+test.options += -b
+
+: lines
+:
+$* <<EOI >>EOO
+ s = 'foo'
+ if echo "$s" | sed 's/o/a/p' >>>? 'bar'
+ f = 'baz'
+ else
+ f = 'fox'
+ end
+ depdb clear
+ depdb string "$s"
+ depdb hash "$f"
+ foo "$s" "$f"
+ EOI
+ foo "$s" "$f"
+ EOO
+
+: temp_dir
+:
+{
+ test.options += -t
+
+ : no
+ :
+ $* <<EOI >false
+ foo
+ EOI
+
+ : yes
+ :
+ $* <<EOI >true
+ foo
+ f = $~/f
+ EOI
+
+ : preamble-no
+ :
+ $* <<EOI >false
+ f1 = $~/f2
+ depdb string "$f1"
+ f2 = $~/f3
+ depdb string "$f2"
+ foo "$f1" "$f2"
+ EOI
+
+ : preamble-yes
+ :
+ $* <<EOI >true
+ f1 = $~/f1
+ depdb string "$f1"
+ f2 = $~/f2
+ foo "$f2"
+ EOI
+}
: command
:
diff --git a/libbuild2/build/script/parser+depdb.test.testscript b/libbuild2/build/script/parser+depdb.test.testscript
new file mode 100644
index 0000000..38c4236
--- /dev/null
+++ b/libbuild2/build/script/parser+depdb.test.testscript
@@ -0,0 +1,90 @@
+# file : libbuild2/build/script/parser+depdb.test.testscript
+# license : MIT; see accompanying LICENSE file
+
+test.options += -d
+
+: clear
+:
+{
+ : multiple
+ :
+ $* <<EOI 2>>EOE != 0
+ depdb clear
+ depdb clear
+ EOI
+ buildfile:12:1: error: multiple 'depdb clear' builtin calls
+ buildfile:11:1: info: previous call is here
+ EOE
+
+ : after-string
+ :
+ $* <<EOI 2>>EOE != 0
+ a = b
+ depdb string "$a"
+ depdb clear
+ EOI
+ buildfile:13:1: error: 'depdb clear' should be the first 'depdb' builtin call
+ buildfile:12:1: info: first 'depdb' call is here
+ EOE
+}
+
+: preamble
+:
+{
+ : no-body
+ :
+ $* <<EOI >>EOO
+ s = 'foo'
+ if echo "$s" | sed 's/o/a/p' >>>? 'bar'
+ f = 'baz'
+ else
+ f = 'fox'
+ end
+ depdb clear
+ depdb string "$s"
+ depdb hash "$f"
+ foo "$s" "$f"
+ EOI
+ s = 'foo'
+ if echo "$s" | sed 's/o/a/p' >>>? 'bar'
+ f = 'baz'
+ else
+ f = 'fox'
+ end
+ depdb string "$s"
+ depdb hash "$f"
+ EOO
+
+ : temp_dir
+ :
+ {
+ test.options += -t
+
+ : no
+ :
+ $* <<EOI >false
+ f = foo
+ depdb hash "$f"
+ f = $~/f
+ foo "$f"
+ EOI
+
+ : yes
+ :
+ $* <<EOI >true
+ f = $~/f
+ depdb hash "$f"
+ foo "$f"
+ EOI
+
+ : yes-mult
+ :
+ $* <<EOI >true
+ f = $~/f
+ depdb hash "$f"
+ s = "abc"
+ depdb string "$s"
+ foo "$f"
+ EOI
+ }
+}
diff --git a/libbuild2/build/script/parser+diag.test.testscript b/libbuild2/build/script/parser+diag.test.testscript
index 5b4e64a..30eb859 100644
--- a/libbuild2/build/script/parser+diag.test.testscript
+++ b/libbuild2/build/script/parser+diag.test.testscript
@@ -33,8 +33,8 @@ $* <<EOI >>~%EOO%
: ambiguity
:
{
-: name
-:
+ : name
+ :
$* test <<EOI 2>>EOE != 0
echo abc
diag xyz
@@ -54,6 +54,29 @@ $* <<EOI >>~%EOO%
buildfile:14:1: error: multiple 'diag' builtin calls
buildfile:12:1: info: previous call is here
EOE
+
+ : names
+ :
+ $* <<EOI 2>>EOE != 0
+ cp abc xyz
+ cat xyz
+ EOI
+ buildfile:11:1: error: low-verbosity script diagnostics name is ambiguous
+ buildfile:11:1: info: could be 'cp'
+ buildfile:12:1: info: could be 'cat'
+ info: consider specifying it explicitly with the 'diag' recipe attribute
+ info: or provide custom low-verbosity diagnostics with the 'diag' builtin
+ EOE
+
+ : none
+ :
+ $* <<EOI 2>>EOE != 0
+ a = 'b'
+ EOI
+ buildfile:11:1: error: unable to deduce low-verbosity script diagnostics name
+ info: consider specifying it explicitly with the 'diag' recipe attribute
+ info: or provide custom low-verbosity diagnostics with the 'diag' builtin
+ EOE
}
: inside-if
@@ -91,3 +114,13 @@ $* <<EOI 2>>EOE != 0
EOI
buildfile:11:8: error: 'diag' call via 'env' builtin
EOE
+
+: before-depdb
+:
+$* <<EOI 2>>EOE != 0
+ diag test
+ depdb clear
+ EOI
+ buildfile:11:1: error: 'diag' builtin call before 'depdb' call
+ buildfile:12:1: info: 'depdb' call is here
+ EOE
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index 8f2c46d..9021da1 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -26,7 +26,7 @@ namespace build2
//
script parser::
- pre_parse (const target& tg,
+ pre_parse (const target& tg, const adhoc_actions& acts,
istream& is, const path_name& pn, uint64_t line,
optional<string> diag, const location& diag_loc)
{
@@ -39,9 +39,10 @@ namespace build2
// The script shouldn't be able to modify the target/scopes.
//
- target_ = const_cast<target*> (&tg);
- scope_ = const_cast<scope*> (&tg.base_scope ());
- root_ = scope_->root_scope ();
+ target_ = const_cast<target*> (&tg);
+ actions_ = &acts;
+ scope_ = const_cast<scope*> (&tg.base_scope ());
+ root_ = scope_->root_scope ();
pbase_ = scope_->src_path_;
@@ -108,7 +109,7 @@ namespace build2
// Save the custom dependency change tracking lines, if present.
//
s.depdb_clear = depdb_clear_.has_value ();
- s.depdb_lines = move (depdb_lines_);
+ s.depdb_preamble = move (depdb_preamble_);
return s;
}
@@ -232,7 +233,7 @@ namespace build2
if (save_line_ != nullptr)
{
if (save_line_ == &ln)
- script_->lines.push_back (move (ln));
+ script_->body.push_back (move (ln));
else
*save_line_ = move (ln);
}
@@ -268,12 +269,12 @@ namespace build2
// cmd_if, not cmd_end. Thus remember the start position of the
// next logical line.
//
- size_t i (script_->lines.size ());
+ size_t i (script_->body.size ());
pre_parse_line (t, tt, true /* if_line */);
assert (tt == type::newline);
- line_type lt (script_->lines[i].type);
+ line_type lt (script_->body[i].type);
// First take care of 'end'.
//
@@ -407,8 +408,8 @@ namespace build2
// Instruct the parser to save the diag builtin line separately
// from the script lines, when it is fully parsed. Note that it
- // will be executed prior to the script execution to obtain the
- // custom diagnostics.
+ // will be executed prior to the script body execution to obtain
+ // the custom diagnostics.
//
diag_line_ = make_pair (line (), l);
save_line_ = &diag_line_->first;
@@ -428,9 +429,27 @@ namespace build2
{
verify ();
+ // Verify that depdb is not used for anything other than
+ // performing update.
+ //
+ assert (actions_ != nullptr);
+
+ for (const action& a: *actions_)
+ {
+ if (a != perform_update_id)
+ fail (l) << "'depdb' builtin cannot be used to "
+ << ctx.meta_operation_table[a.meta_operation ()].name
+ << ' ' << ctx.operation_table[a.operation ()];
+ }
+
+ if (diag_line_)
+ fail (diag_line_->second)
+ << "'diag' builtin call before 'depdb' call" <<
+ info (l) << "'depdb' call is here";
+
// Note that the rest of the line contains the builtin command
- // name, potentially followed by the arguments to be
- // hashed/saved. Thus, we parse it in the value lexer mode.
+ // name, potentially followed by the arguments to be hashed/saved.
+ // Thus, we parse it in the value lexer mode.
//
mode (lexer_mode::value);
@@ -453,11 +472,27 @@ namespace build2
fail (l) << "multiple 'depdb clear' builtin calls" <<
info (*depdb_clear_) << "previous call is here";
- if (!depdb_lines_.empty ())
- fail (l) << "'depdb clear' should be the first 'depdb' "
- << "builtin call" <<
- info (depdb_lines_[0].tokens[0].location ())
- << "first 'depdb' call is here";
+ if (!depdb_preamble_.empty ())
+ {
+ diag_record dr (fail (l));
+ dr << "'depdb clear' should be the first 'depdb' builtin call";
+
+ // Print the first depdb call location.
+ //
+ for (const line& l: depdb_preamble_)
+ {
+ const replay_tokens& rt (l.tokens);
+ assert (!rt.empty ());
+
+ const token& t (rt[0].token);
+ if (t.type == type::word && t.value == "depdb")
+ {
+ dr << info (rt[0].location ())
+ << "first 'depdb' call is here";
+ break;
+ }
+ }
+ }
// Save the builtin location, cancel the line saving, and clear
// the referenced variable list, since it won't be used.
@@ -469,13 +504,37 @@ namespace build2
}
else
{
+ // Move the script body to the end of the depdb preamble.
+ //
+ // Note that at this (pre-parsing) stage we cannot evaluate if
+ // all the script body lines are allowed for depdb preamble.
+ // That, in particular, would require to analyze pipelines to
+ // see if they are terminated with the set builtin, but this
+ // information is only available at the execution stage. Thus,
+ // we move into the preamble whatever is there and delay the
+ // check until the execution.
+ //
+ lines& ls (script_->body);
+ depdb_preamble_.insert (depdb_preamble_.end (),
+ make_move_iterator (ls.begin ()),
+ make_move_iterator (ls.end ()));
+ ls.clear ();
+
+ // Also move the body_temp_dir flag, if it is true.
+ //
+ if (script_->body_temp_dir)
+ {
+ script_->depdb_preamble_temp_dir = true;
+ script_->body_temp_dir = false;
+ }
+
// Instruct the parser to save the depdb builtin line separately
// from the script lines, when it is fully parsed. Note that the
// builtin command arguments will be validated during execution,
// when expanded.
//
- depdb_lines_.push_back (line ());
- save_line_ = &depdb_lines_.back ();
+ depdb_preamble_.push_back (line ());
+ save_line_ = &depdb_preamble_.back ();
}
// Parse the rest of the line and bail out.
@@ -531,8 +590,8 @@ namespace build2
{
if (pre_parse_suspended_)
{
- dr << info (l) << "while deducing low-verbosity script "
- << "diagnostics name";
+ dr << info (l)
+ << "while deducing low-verbosity script diagnostics name";
suggest_diag (dr);
}
@@ -757,8 +816,153 @@ namespace build2
}
void parser::
- execute (const scope& rs, const scope& bs,
- environment& e, const script& s, runner& r)
+ execute_body (const scope& rs, const scope& bs,
+ environment& e, const script& s, runner& r,
+ bool enter, bool leave)
+ {
+ pre_exec (rs, bs, e, &s, &r);
+
+ if (enter)
+ runner_->enter (e, s.start_loc);
+
+ // Note that we rely on "small function object" optimization here.
+ //
+ auto exec_cmd = [this] (token& t, build2::script::token_type& tt,
+ size_t li,
+ bool single,
+ const location& ll)
+ {
+ // We use the 0 index to signal that this is the only command.
+ //
+ if (single)
+ li = 0;
+
+ command_expr ce (
+ parse_command_line (t, static_cast<token_type&> (tt)));
+
+ runner_->run (*environment_, ce, li, ll);
+ };
+
+ exec_lines (s.body, exec_cmd);
+
+ if (leave)
+ runner_->leave (e, s.end_loc);
+ }
+
+ void parser::
+ execute_depdb_preamble (const scope& rs, const scope& bs,
+ environment& e, const script& s, runner& r,
+ depdb& dd)
+ {
+ tracer trace ("execute_depdb_preamble");
+
+ // The only valid lines in the depdb preamble are the depdb builtin
+ // itself as well as the variable assignments, including via the set
+ // builtin.
+
+ pre_exec (rs, bs, e, &s, &r);
+
+ // Let's "wrap up" the objects we operate upon into the single object
+ // to rely on "small function object" optimization.
+ //
+ struct
+ {
+ environment& env;
+ const script& scr;
+ depdb& dd;
+ tracer& trace;
+ } ctx {e, s, dd, trace};
+
+ auto exec_cmd = [&ctx, this]
+ (token& t,
+ build2::script::token_type& tt,
+ size_t li,
+ bool /* single */,
+ const location& ll)
+ {
+ if (tt == type::word && t.value == "depdb")
+ {
+ names ns (exec_special (t, tt));
+
+ // This should have been enforced during pre-parsing.
+ //
+ assert (!ns.empty ()); // <cmd> ... <newline>
+
+ const string& cmd (ns[0].value);
+
+ if (cmd == "hash")
+ {
+ sha256 cs;
+ for (auto i (ns.begin () + 1); i != ns.end (); ++i) // Skip <cmd>.
+ to_checksum (cs, *i);
+
+ if (ctx.dd.expect (cs.string ()) != nullptr)
+ l4 ([&] {
+ ctx.trace (ll)
+ << "'depdb hash' argument change forcing update of "
+ << ctx.env.target;});
+ }
+ else if (cmd == "string")
+ {
+ string s;
+ try
+ {
+ s = convert<string> (
+ names (make_move_iterator (ns.begin () + 1),
+ make_move_iterator (ns.end ())));
+ }
+ catch (const invalid_argument& e)
+ {
+ fail (ll) << "invalid 'depdb string' argument: " << e;
+ }
+
+ if (ctx.dd.expect (s) != nullptr)
+ l4 ([&] {
+ ctx.trace (ll)
+ << "'depdb string' argument change forcing update of "
+ << ctx.env.target;});
+ }
+ else
+ assert (false);
+ }
+ else
+ {
+ // Note that we don't reset the line index to zero (as we do in
+ // execute_body()) assuming that there are some script body
+ // commands to follow.
+ //
+ command_expr ce (
+ parse_command_line (t, static_cast<token_type&> (tt)));
+
+ // Verify that this expression executes the set builtin.
+ //
+ if (find_if (ce.begin (), ce.end (),
+ [] (const expr_term& et)
+ {
+ const process_path& p (et.pipe.back ().program);
+ return p.initial == nullptr &&
+ p.recall.string () == "set";
+ }) == ce.end ())
+ {
+ const replay_tokens& rt (ctx.scr.depdb_preamble.back ().tokens);
+ assert (!rt.empty ());
+
+ fail (ll) << "disallowed command in depdb preamble" <<
+ info << "only variable assignments are allowed in "
+ << "depdb preamble" <<
+ info (rt[0].location ()) << "depdb preamble ends here";
+ }
+
+ runner_->run (*environment_, ce, li, ll);
+ }
+ };
+
+ exec_lines (s.depdb_preamble, exec_cmd);
+ }
+
+ void parser::
+ pre_exec (const scope& rs, const scope& bs,
+ environment& e, const script* s, runner* r)
{
path_ = nullptr; // Set by replays.
@@ -766,6 +970,8 @@ namespace build2
set_lexer (nullptr);
+ actions_ = nullptr;
+
// The script shouldn't be able to modify the scopes.
//
// Note that for now we don't set target_ since it's not clear what
@@ -776,20 +982,15 @@ namespace build2
scope_ = const_cast<scope*> (&bs);
pbase_ = scope_->src_path_;
- script_ = const_cast<script*> (&s);
- runner_ = &r;
+ script_ = const_cast<script*> (s);
+ runner_ = r;
environment_ = &e;
-
- exec_script ();
}
void parser::
- exec_script ()
+ exec_lines (const lines& lns,
+ const function<exec_cmd_function>& exec_cmd)
{
- const script& s (*script_);
-
- runner_->enter (*environment_, s.start_loc);
-
// Note that we rely on "small function object" optimization for the
// exec_*() lambdas.
//
@@ -814,22 +1015,6 @@ namespace build2
apply_value_attributes (&var, lhs, move (rhs), kind);
};
- auto exec_cmd = [this] (token& t, build2::script::token_type& tt,
- size_t li,
- bool single,
- const location& ll)
- {
- // We use the 0 index to signal that this is the only command.
- //
- if (single)
- li = 0;
-
- command_expr ce (
- parse_command_line (t, static_cast<token_type&> (tt)));
-
- runner_->run (*environment_, ce, li, ll);
- };
-
auto exec_if = [this] (token& t, build2::script::token_type& tt,
size_t li,
const location& ll)
@@ -842,14 +1027,26 @@ namespace build2
return runner_->run_if (*environment_, ce, li, ll);
};
- size_t li (1);
+ build2::script::parser::exec_lines (lns.begin (), lns.end (),
+ exec_set, exec_cmd, exec_if,
+ environment_->exec_line,
+ &environment_->var_pool);
+ }
- exec_lines (s.lines.begin (), s.lines.end (),
- exec_set, exec_cmd, exec_if,
- li,
- &environment_->var_pool);
+ names parser::
+ exec_special (token& t, build2::script::token_type& tt,
+ bool omit_builtin)
+ {
+ if (omit_builtin)
+ {
+ assert (tt != type::newline && tt != type::eos);
- runner_->leave (*environment_, s.end_loc);
+ next (t, tt);
+ }
+
+ return tt != type::newline && tt != type::eos
+ ? parse_names (t, tt, pattern_mode::expand)
+ : names ();
}
names parser::
@@ -858,25 +1055,7 @@ namespace build2
const line& ln,
bool omit_builtin)
{
- path_ = nullptr; // Set by replays.
-
- pre_parse_ = false;
-
- set_lexer (nullptr);
-
- // The script shouldn't be able to modify the scopes.
- //
- // Note that for now we don't set target_ since it's not clear what
- // it could be used for (we need scope_ for calling functions such as
- // $target.path()).
- //
- root_ = const_cast<scope*> (&rs);
- scope_ = const_cast<scope*> (&bs);
- pbase_ = scope_->src_path_;
-
- script_ = nullptr;
- runner_ = nullptr;
- environment_ = &e;
+ pre_exec (rs, bs, e, nullptr /* script */, nullptr /* runner */);
// Copy the tokens and start playing.
//
@@ -886,16 +1065,7 @@ namespace build2
build2::script::token_type tt;
next (t, tt);
- if (omit_builtin)
- {
- assert (tt != type::newline && tt != type::eos);
-
- next (t, tt);
- }
-
- names r (tt != type::newline && tt != type::eos
- ? parse_names (t, tt, pattern_mode::expand)
- : names ());
+ names r (exec_special (t, tt, omit_builtin));
replay_stop ();
return r;
@@ -928,7 +1098,7 @@ namespace build2
if (special_variable (name))
{
if (name == "~")
- script_->temp_dir = true;
+ script_->body_temp_dir = true;
}
else if (!name.empty ())
{
@@ -970,7 +1140,7 @@ namespace build2
//
if (script_ != nullptr &&
!script_->depdb_clear &&
- script_->depdb_lines.empty ())
+ script_->depdb_preamble.empty ())
{
if (r.defined () && !r.belongs (*environment_))
{
diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx
index 5ada8be..e96a806 100644
--- a/libbuild2/build/script/parser.hxx
+++ b/libbuild2/build/script/parser.hxx
@@ -8,6 +8,7 @@
#include <libbuild2/forward.hxx>
#include <libbuild2/utility.hxx>
+#include <libbuild2/depdb.hxx>
#include <libbuild2/diagnostics.hxx>
#include <libbuild2/script/parser.hxx>
@@ -34,7 +35,7 @@ namespace build2
// name.
//
script
- pre_parse (const target&,
+ pre_parse (const target&, const adhoc_actions& acts,
istream&, const path_name&, uint64_t line,
optional<string> diag_name, const location& diag_loc);
@@ -63,9 +64,26 @@ namespace build2
// Execute. Issue diagnostics and throw failed in case of an error.
//
public:
+
+ // By default call the runner's enter() and leave() functions that
+ // initialize/clean up the environment before/after the script
+ // execution.
+ //
+ void
+ execute_body (const scope& root, const scope& base,
+ environment&, const script&, runner&,
+ bool enter = true, bool leave = true);
+
+
+ // Note that it's the caller's responsibility to make sure that the
+ // runner's enter() function is called before the first preamble/body
+ // command execution and leave() -- after the last command.
+ //
void
- execute (const scope& root, const scope& base,
- environment&, const script&, runner&);
+ execute_depdb_preamble (const scope& root, const scope& base,
+ environment&, const script&, runner&,
+ depdb&);
+
// Parse a special builtin line into names, performing the variable
// and pattern expansions. If omit_builtin is true, then omit the
@@ -78,8 +96,18 @@ namespace build2
bool omit_builtin = true);
protected:
+ // Setup the parser for subsequent exec_*() function calls.
+ //
+ void
+ pre_exec (const scope& root, const scope& base,
+ environment&, const script*, runner*);
+
void
- exec_script ();
+ exec_lines (const lines&, const function<exec_cmd_function>&);
+
+ names
+ exec_special (token& t, build2::script::token_type& tt,
+ bool omit_builtin = true);
// Helpers.
//
@@ -112,6 +140,7 @@ namespace build2
protected:
script* script_;
+ const adhoc_actions* actions_; // Non-NULL during pre-parsing.
// Current low-verbosity script diagnostics and its weight.
//
@@ -165,8 +194,8 @@ namespace build2
//
// depdb string <arg> - Track the argument (single) change as string.
//
- optional<location> depdb_clear_; // 'depdb clear' location if any.
- lines depdb_lines_; // Note: excludes 'depdb clear'.
+ optional<location> depdb_clear_; // 'depdb clear' location if any.
+ lines depdb_preamble_; // Note: excludes 'depdb clear'.
// True during pre-parsing when the pre-parse mode is temporarily
// suspended to perform expansion.
diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx
index 7f2840d..4a3e8cc 100644
--- a/libbuild2/build/script/parser.test.cxx
+++ b/libbuild2/build/script/parser.test.cxx
@@ -71,20 +71,24 @@ namespace build2
// Usages:
//
// argv[0] [-l]
- // argv[0] -d
- // argv[0] -p
+ // argv[0] -b [-t]
+ // argv[0] -d [-t]
+ // argv[0] -q
// argv[0] -g [<diag-name>]
//
// In the first form read the script from stdin and trace the script
- // execution to stdout using the custom print runner.
+ // body execution to stdout using the custom print runner.
//
// In the second form read the script from stdin, parse it and dump the
- // resulting lines to stdout.
+ // script body lines to stdout.
//
- // In the third form read the script from stdin, parse it and print
+ // In the third form read the script from stdin, parse it and dump the
+ // depdb preamble lines to stdout.
+ //
+ // In the forth form read the script from stdin, parse it and print
// line tokens quoting information to stdout.
//
- // In the forth form read the script from stdin, parse it and print the
+ // In the fifth form read the script from stdin, parse it and print the
// low-verbosity script diagnostics name or custom low-verbosity
// diagnostics to stdout. If the script doesn't deduce any of them, then
// print the diagnostics and exit with non-zero code.
@@ -92,10 +96,17 @@ namespace build2
// -l
// Print the script line number for each executed expression.
//
+ // -b
+ // Dump the parsed script body to stdout.
+ //
// -d
- // Dump the parsed script to sdout.
+ // Dump the parsed script depdb preamble to stdout.
//
- // -p
+ // -t
+ // Print true if the body (-b) or depdb preamble (-d) references the
+ // temporary directory and false otherwise.
+ //
+ // -q
// Print the parsed script tokens quoting information to sdout. If a
// token is quoted follow its representation with its quoting
// information in the [<quoting>/<completeness>] form, where:
@@ -115,13 +126,15 @@ namespace build2
enum class mode
{
run,
- dump,
- print,
+ body,
+ depdb_preamble,
+ quoting,
diag
} m (mode::run);
bool print_line (false);
optional<string> diag_name;
+ bool temp_dir (false);
for (int i (1); i != argc; ++i)
{
@@ -129,10 +142,17 @@ namespace build2
if (a == "-l")
print_line = true;
+ else if (a == "-b")
+ m = mode::body;
else if (a == "-d")
- m = mode::dump;
- else if (a == "-p")
- m = mode::print;
+ m = mode::depdb_preamble;
+ else if (a == "-t")
+ {
+ assert (m == mode::body || m == mode::depdb_preamble);
+ temp_dir = true;
+ }
+ else if (a == "-q")
+ m = mode::quoting;
else if (a == "-g")
m = mode::diag;
else
@@ -179,12 +199,14 @@ namespace build2
tt.path (path ("driver"));
+ adhoc_actions acts {perform_update_id};
+
// Parse and run.
//
parser p (ctx);
path_name nm ("buildfile");
- script s (p.pre_parse (tt,
+ script s (p.pre_parse (tt, acts,
cin, nm,
11 /* line */,
(m != mode::diag
@@ -196,9 +218,9 @@ namespace build2
{
case mode::run:
{
- environment e (perform_update_id, tt, false /* temp_dir */);
+ environment e (perform_update_id, tt, s.body_temp_dir);
print_runner r (print_line);
- p.execute (ctx.global_scope, ctx.global_scope, e, s, r);
+ p.execute_body (ctx.global_scope, ctx.global_scope, e, s, r);
break;
}
case mode::diag:
@@ -221,14 +243,27 @@ namespace build2
break;
}
- case mode::dump:
+ case mode::body:
{
- dump (cout, "", s.lines);
+ if (!temp_dir)
+ dump (cout, "", s.body);
+ else
+ cout << (s.body_temp_dir ? "true" : "false") << endl;
+
+ break;
+ }
+ case mode::depdb_preamble:
+ {
+ if (!temp_dir)
+ dump (cout, "", s.depdb_preamble);
+ else
+ cout << (s.depdb_preamble_temp_dir ? "true" : "false") << endl;
+
break;
}
- case mode::print:
+ case mode::quoting:
{
- for (const line& l: s.lines)
+ for (const line& l: s.body)
{
for (const replay_token& rt: l.tokens)
{
diff --git a/libbuild2/build/script/script.cxx b/libbuild2/build/script/script.cxx
index c6b57c3..e003b6a 100644
--- a/libbuild2/build/script/script.cxx
+++ b/libbuild2/build/script/script.cxx
@@ -74,13 +74,20 @@ namespace build2
assign (var_pool.insert ("<")) = move (ns);
}
- // Set the $~ special variable.
- //
if (temp)
- {
+ set_temp_dir_variable ();
+ }
+
+ void environment::
+ set_temp_dir_variable ()
+ {
+ // Note that the temporary directory could have been created
+ // implicitly by the runner.
+ //
+ if (temp_dir.path.empty ())
create_temp_dir ();
- assign (var_pool.insert<dir_path> ("~")) = temp_dir.path;
- }
+
+ assign (var_pool.insert<dir_path> ("~")) = temp_dir.path;
}
void environment::
diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx
index 9284813..e11cb45 100644
--- a/libbuild2/build/script/script.hxx
+++ b/libbuild2/build/script/script.hxx
@@ -45,7 +45,8 @@ namespace build2
// Note that the variables are not pre-entered into a pool during the
// parsing phase, so the line variable pointers are NULL.
//
- lines_type lines;
+ lines_type body;
+ bool body_temp_dir = false; // True if the body references $~.
// Referenced ordinary (non-special) variables.
//
@@ -59,10 +60,6 @@ namespace build2
//
small_vector<string, 2> vars; // 2 for command and options.
- // True if script references the $~ special variable.
- //
- bool temp_dir = false;
-
// Command name for low-verbosity diagnostics and custom low-verbosity
// diagnostics line. Note: cannot be both (see the script parser for
// details).
@@ -74,7 +71,8 @@ namespace build2
// script parser for details).
//
bool depdb_clear;
- lines_type depdb_lines;
+ lines_type depdb_preamble;
+ bool depdb_preamble_temp_dir = false; // True if references $~.
location start_loc;
location end_loc;
@@ -136,6 +134,18 @@ namespace build2
optional<deadline> script_deadline;
optional<deadline> fragment_deadline;
+ // Index of the next script line to be executed. Used and incremented
+ // by the parser's execute_depdb_preamble() and execute_body()
+ // function calls to produce special file names, etc.
+ //
+ size_t exec_line = 1;
+
+ // Create the temporary directory (if it doesn't exist yet) and set
+ // the $~ special variable to its path.
+ //
+ void
+ set_temp_dir_variable ();
+
virtual void
set_variable (string&& name,
names&&,
diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx
index 8676c9d..92e0090 100644
--- a/libbuild2/parser.cxx
+++ b/libbuild2/parser.cxx
@@ -1236,17 +1236,6 @@ namespace build2
if (!skip)
{
- if (d.first)
- {
- if (ar->recipe_text (ctx, *target_, move (t.value), d.as))
- d.clean = true;
-
- // Verify we have no unhandled attributes.
- //
- for (attribute& a: d.as)
- fail (d.as.loc) << "unknown recipe attribute " << a << endf;
- }
-
auto& ars (target_->adhoc_recipes);
ars.push_back (adhoc_recipe {{}, move (ar)});
@@ -1315,6 +1304,20 @@ namespace build2
ars.back ().actions.push_back (a);
}
}
+
+ if (d.first)
+ {
+ adhoc_recipe& ar (ars.back ());
+
+ if (ar.rule->recipe_text (
+ ctx, *target_, ar.actions, move (t.value), d.as))
+ d.clean = true;
+
+ // Verify we have no unhandled attributes.
+ //
+ for (attribute& a: d.as)
+ fail (d.as.loc) << "unknown recipe attribute " << a << endf;
+ }
}
next (t, tt);
diff --git a/libbuild2/recipe.hxx b/libbuild2/recipe.hxx
index b88311c..7eca34a 100644
--- a/libbuild2/recipe.hxx
+++ b/libbuild2/recipe.hxx
@@ -54,10 +54,11 @@ namespace build2
// A recipe is a fragment of a rule so we handle ad hoc recipies by
// "completing" them to rules.
//
+ using adhoc_actions = small_vector<action, 1>;
struct adhoc_recipe
{
- small_vector<action, 1> actions;
- shared_ptr<adhoc_rule> rule;
+ adhoc_actions actions;
+ shared_ptr<adhoc_rule> rule;
};
}
diff --git a/libbuild2/rule.hxx b/libbuild2/rule.hxx
index 8796659..e29c0b8 100644
--- a/libbuild2/rule.hxx
+++ b/libbuild2/rule.hxx
@@ -160,7 +160,8 @@ namespace build2
// therefore requires cleanup.
//
virtual bool
- recipe_text (context&, const target&, string&&, attributes&) = 0;
+ recipe_text (context&, const target&, const adhoc_actions&,
+ string&&, attributes&) = 0;
public:
// Some of the operations come in compensating pairs, such as update and
diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx
index 3daa789..182696b 100644
--- a/libbuild2/script/run.cxx
+++ b/libbuild2/script/run.cxx
@@ -2299,19 +2299,82 @@ namespace build2
void
clean (environment& env, const location& ll)
{
- context& ctx (env.context);
+ // We don't use the build2 filesystem utilities here in order to remove
+ // the filesystem entries regardless of the dry-run mode and also to add
+ // the location info to diagnostics. Other than that, these lambdas
+ // implement the respective utility functions semantics.
+ //
+ auto rmfile = [&ll] (const path& f)
+ {
+ try
+ {
+ rmfile_status r (try_rmfile (f));
+
+ if (r == rmfile_status::success && verb >= 3)
+ text << "rm " << f;
+
+ return r;
+ }
+ catch (const system_error& e)
+ {
+ fail (ll) << "unable to remove file " << f << ": " << e << endf;
+ }
+ };
+
+ auto rmdir = [&ll] (const dir_path& d)
+ {
+ try
+ {
+ rmdir_status r (!work.sub (d)
+ ? try_rmdir (d)
+ : rmdir_status::not_empty);
+
+ if (r == rmdir_status::success && verb >= 3)
+ text << "rmdir " << d;
+
+ return r;
+ }
+ catch (const system_error& e)
+ {
+ fail (ll) << "unable to remove directory " << d << ": " << e << endf;
+ }
+ };
+
+ auto rmdir_r = [&ll] (const dir_path& d, bool dir)
+ {
+ if (work.sub (d)) // Don't try to remove working directory.
+ return rmdir_status::not_empty;
+
+ if (!build2::entry_exists (d))
+ return rmdir_status::not_exist;
+
+ try
+ {
+ butl::rmdir_r (d, dir);
+ }
+ catch (const system_error& e)
+ {
+ fail (ll) << "unable to remove directory " << d << ": " << e << endf;
+ }
+
+ if (verb >= 3)
+ text << "rmdir -r " << d;
+
+ return rmdir_status::success;
+ };
+
const dir_path& wdir (*env.work_dir.path);
// Note that we operate with normalized paths here.
//
- // Remove special files. The order is not important as we don't
- // expect directories here.
+ // Remove special files. The order is not important as we don't expect
+ // directories here.
//
for (const path& p: env.special_cleanups)
{
// Remove the file if exists. Fail otherwise.
//
- if (rmfile (ctx, p, 3) == rmfile_status::not_exist)
+ if (rmfile (p) == rmfile_status::not_exist)
fail (ll) << "registered for cleanup special file " << p
<< " does not exist";
}
@@ -2332,9 +2395,9 @@ namespace build2
// Wildcard with the last component being '***' (without trailing
// separator) matches all files and sub-directories recursively as
- // well as the start directories itself. So we will recursively
- // remove the directories that match the parent (for the original
- // path) directory wildcard.
+ // well as the start directories itself. So we will recursively remove
+ // the directories that match the parent (for the original path)
+ // directory wildcard.
//
bool recursive (cp.leaf ().representation () == "***");
const path& p (!recursive ? cp : cp.directory ());
@@ -2345,15 +2408,19 @@ namespace build2
{
bool removed (false);
- auto rm = [&cp, recursive, &removed, &ll, &ctx, &wdir]
+ auto rm = [&cp,
+ recursive,
+ &removed,
+ &ll,
+ &wdir,
+ &rmfile, &rmdir, &rmdir_r]
(path&& pe, const string&, bool interm)
{
if (!interm)
{
- // While removing the entry we can get not_exist due to
- // racing conditions, but that's ok if somebody did our job.
- // Note that we still set the removed flag to true in this
- // case.
+ // While removing the entry we can get not_exist due to racing
+ // conditions, but that's ok if somebody did our job. Note that
+ // we still set the removed flag to true in this case.
//
removed = true; // Will be meaningless on failure.
@@ -2363,7 +2430,7 @@ namespace build2
if (!recursive)
{
- rmdir_status r (rmdir (ctx, d, 3));
+ rmdir_status r (rmdir (d));
if (r != rmdir_status::not_empty)
return true;
@@ -2377,13 +2444,10 @@ namespace build2
}
else
{
- // Don't remove the working directory (it will be removed
- // by the dedicated cleanup).
- //
- // Cast to uint16_t to avoid ambiguity with
- // libbutl::rmdir_r().
+ // Don't remove the working directory (it will be removed by
+ // the dedicated cleanup).
//
- rmdir_status r (rmdir_r (ctx, d, d != wdir, 3));
+ rmdir_status r (rmdir_r (d, d != wdir));
if (r != rmdir_status::not_empty)
return true;
@@ -2396,14 +2460,14 @@ namespace build2
}
}
else
- rmfile (ctx, pe, 3);
+ rmfile (pe);
}
return true;
};
- // Note that here we rely on the fact that recursive iterating
- // goes depth-first (which make sense for the cleanup).
+ // Note that here we rely on the fact that recursive iterating goes
+ // depth-first (which make sense for the cleanup).
//
try
{
@@ -2434,9 +2498,8 @@ namespace build2
: "file");
}
- // Remove the directory if exists and empty. Fail otherwise.
- // Removal of non-existing directory is not an error for 'maybe'
- // cleanup type.
+ // Remove the directory if exists and empty. Fail otherwise. Removal
+ // of non-existing directory is not an error for 'maybe' cleanup type.
//
if (p.to_directory ())
{
@@ -2444,20 +2507,10 @@ namespace build2
bool wd (d == wdir);
// Don't remove the working directory for the recursive cleanup
- // (it will be removed by the dedicated one).
- //
- // Note that the root working directory contains the
- // .buildignore file (see above).
- //
- // @@ If 'd' is a file then will fail with a diagnostics having
- // no location info. Probably need to add an optional location
- // parameter to rmdir() function. The same problem exists for
- // a file cleanup when try to rmfile() directory instead of
- // file.
+ // since it needs to be removed by the caller (can contain
+ // .buildignore file, etc).
//
- rmdir_status r (recursive
- ? rmdir_r (ctx, d, !wd, static_cast <uint16_t> (3))
- : rmdir (ctx, d, 3));
+ rmdir_status r (recursive ? rmdir_r (d, !wd) : rmdir (d));
if (r == rmdir_status::success ||
(r == rmdir_status::not_exist && t == cleanup_type::maybe))
@@ -2473,10 +2526,10 @@ namespace build2
print_dir (dr, d, ll);
}
- // Remove the file if exists. Fail otherwise. Removal of
- // non-existing file is not an error for 'maybe' cleanup type.
+ // Remove the file if exists. Fail otherwise. Removal of non-existing
+ // file is not an error for 'maybe' cleanup type.
//
- if (rmfile (ctx, p, 3) == rmfile_status::not_exist &&
+ if (rmfile (p) == rmfile_status::not_exist &&
t == cleanup_type::always)
fail (ll) << "registered for cleanup file " << p
<< " does not exist";