diff options
Diffstat (limited to 'libbuild2')
| -rw-r--r-- | libbuild2/adhoc-rule-buildscript.cxx | 125 | ||||
| -rw-r--r-- | libbuild2/adhoc-rule-buildscript.hxx | 3 | ||||
| -rw-r--r-- | libbuild2/adhoc-rule-cxx.cxx | 3 | ||||
| -rw-r--r-- | libbuild2/adhoc-rule-cxx.hxx | 3 | ||||
| -rw-r--r-- | libbuild2/build/script/parser+body.test.testscript (renamed from libbuild2/build/script/parser+line.test.testscript) | 59 | ||||
| -rw-r--r-- | libbuild2/build/script/parser+depdb.test.testscript | 90 | ||||
| -rw-r--r-- | libbuild2/build/script/parser+diag.test.testscript | 37 | ||||
| -rw-r--r-- | libbuild2/build/script/parser.cxx | 340 | ||||
| -rw-r--r-- | libbuild2/build/script/parser.hxx | 41 | ||||
| -rw-r--r-- | libbuild2/build/script/parser.test.cxx | 75 | ||||
| -rw-r--r-- | libbuild2/build/script/script.cxx | 17 | ||||
| -rw-r--r-- | libbuild2/build/script/script.hxx | 22 | ||||
| -rw-r--r-- | libbuild2/parser.cxx | 25 | ||||
| -rw-r--r-- | libbuild2/recipe.hxx | 5 | ||||
| -rw-r--r-- | libbuild2/rule.hxx | 3 | ||||
| -rw-r--r-- | libbuild2/script/run.cxx | 135 |
16 files changed, 734 insertions, 249 deletions
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx index 9470d3c..dad638e 100644 --- a/libbuild2/adhoc-rule-buildscript.cxx +++ b/libbuild2/adhoc-rule-buildscript.cxx @@ -23,7 +23,8 @@ using namespace std; namespace build2 { bool adhoc_buildscript_rule:: - recipe_text (context& ctx, const target& tg, string&& t, attributes& as) + recipe_text (context& ctx, const target& tg, const adhoc_actions& acts, + string&& t, attributes& as) { // Handle and erase recipe-specific attributes. // @@ -56,7 +57,7 @@ namespace build2 istringstream is (move (t)); build::script::parser p (ctx); - script = p.pre_parse (tg, + script = p.pre_parse (tg, acts, is, loc.file, loc.line + 1, move (diag), as.loc); @@ -87,14 +88,14 @@ namespace build2 if (script.depdb_clear) os << ind << "depdb clear" << endl; - script::dump (os, ind, script.depdb_lines); + script::dump (os, ind, script.depdb_preamble); if (script.diag_line) { os << ind; script::dump (os, *script.diag_line, true /* newline */); } - script::dump (os, ind, script.lines); + script::dump (os, ind, script.body); ind.resize (ind.size () - 2); os << ind << string (braces, '}'); } @@ -452,69 +453,43 @@ namespace build2 // Execute the custom dependency change tracking commands, if present. // - if (!script.depdb_lines.empty ()) - { - bs = &t.base_scope (); - rs = bs->root_scope (); + // Note that we share the environment between the execute_depdb_preamble() + // and execute_body() calls, which is not merely an optimization since + // variables set in the preamble must be available in the body. + // + // Creating the environment instance is not cheap so optimize for the + // common case where we don't have the depdb preamble and nothing to + // update. + // + bool depdb_preamble (!script.depdb_preamble.empty ()); - // While it would have been nice to reuse the environment for both - // dependency tracking and execution, there are complications (creating - // temporary directory, etc). - // - build::script::environment e (a, t, false /* temp_dir */); - build::script::parser p (ctx); + if (!depdb_preamble) + { + if (dd.writing () || dd.mtime > mt) + update = true; - for (const script::line& l: script.depdb_lines) + if (!update) { - names ns (p.execute_special (*rs, *bs, e, l)); + dd.close (); + return *ps; + } + } - // These should have been enforced during pre-parsing. - // - assert (!ns.empty ()); // <cmd> ... <newline> - assert (l.tokens.size () > 2); // 'depdb' <cmd> ... <newline> + build::script::environment env (a, t, false /* temp_dir */); + build::script::default_runner r; - const string& cmd (ns[0].value); + if (depdb_preamble) + { + bs = &t.base_scope (); + rs = bs->root_scope (); - location loc (l.tokens[0].location ()); + if (script.depdb_preamble_temp_dir) + env.set_temp_dir_variable (); - if (cmd == "hash") - { - sha256 cs; - for (auto i (ns.begin () + 1); i != ns.end (); ++i) // Skip <cmd>. - to_checksum (cs, *i); - - if (dd.expect (cs.string ()) != nullptr) - l4 ([&] { - diag_record dr (trace); - dr << "'depdb hash' argument change forcing update of " << t << - info (loc); script::dump (dr.os, l); - }); - } - else if (cmd == "string") - { - string s; - try - { - s = convert<string> (names (make_move_iterator (ns.begin () + 1), - make_move_iterator (ns.end ()))); - } - catch (const invalid_argument& e) - { - fail (l.tokens[2].location ()) - << "invalid 'depdb string' argument: " << e; - } + build::script::parser p (ctx); - if (dd.expect (s) != nullptr) - l4 ([&] { - diag_record dr (trace); - dr << "'depdb string' argument change forcing update of " - << t << - info (loc); script::dump (dr.os, l); - }); - } - else - assert (false); - } + r.enter (env, script.start_loc); + p.execute_depdb_preamble (*rs, *bs, env, script, r, dd); } // Update if depdb mismatch. @@ -527,24 +502,38 @@ namespace build2 // If nothing changed, then we are done. // if (!update) + { + // Note that if we execute the depdb preamble but not the script body, + // we need to call the runner's leave() function explicitly (here and + // below). + // + if (depdb_preamble) + r.leave (env, script.end_loc); + return *ps; + } if (!ctx.dry_run || verb != 0) { + // Prepare to executing the script diag line and/or body. + // + // Note that it doesn't make much sense to use the temporary directory + // variable ($~) in the 'diag' builtin call, so we postpone setting it + // until the script body execution, that can potentially be omitted. + // if (bs == nullptr) { bs = &t.base_scope (); rs = bs->root_scope (); } - build::script::environment e (a, t, script.temp_dir); build::script::parser p (ctx); if (verb == 1) { if (script.diag_line) { - text << p.execute_special (*rs, *bs, e, *script.diag_line); + text << p.execute_special (*rs, *bs, env, *script.diag_line); } else { @@ -577,8 +566,10 @@ namespace build2 } } - build::script::default_runner r; - p.execute (*rs, *bs, e, script, r); + if (script.body_temp_dir && !script.depdb_preamble_temp_dir) + env.set_temp_dir_variable (); + + p.execute_body (*rs, *bs, env, script, r, !depdb_preamble); if (!ctx.dry_run) { @@ -607,7 +598,11 @@ namespace build2 rm.cancel (); } } + else if (depdb_preamble) + r.leave (env, script.end_loc); } + else if (depdb_preamble) + r.leave (env, script.end_loc); t.mtime (system_clock::now ()); return target_state::changed; @@ -629,7 +624,7 @@ namespace build2 const scope& bs (t.base_scope ()); const scope& rs (*bs.root_scope ()); - build::script::environment e (a, t, script.temp_dir, deadline); + build::script::environment e (a, t, script.body_temp_dir, deadline); build::script::parser p (ctx); if (verb == 1) @@ -649,7 +644,7 @@ namespace build2 if (!ctx.dry_run || verb >= 2) { build::script::default_runner r; - p.execute (rs, bs, e, script, r); + p.execute_body (rs, bs, e, script, r); } } diff --git a/libbuild2/adhoc-rule-buildscript.hxx b/libbuild2/adhoc-rule-buildscript.hxx index 89515eb..d98ffcf 100644 --- a/libbuild2/adhoc-rule-buildscript.hxx +++ b/libbuild2/adhoc-rule-buildscript.hxx @@ -43,7 +43,8 @@ namespace build2 : adhoc_rule ("<ad hoc buildscript recipe>", l, b) {} virtual bool - recipe_text (context&, const target&, string&&, attributes&) override; + recipe_text (context&, const target&, const adhoc_actions&, + string&&, attributes&) override; virtual void dump_attributes (ostream&) const override; diff --git a/libbuild2/adhoc-rule-cxx.cxx b/libbuild2/adhoc-rule-cxx.cxx index 3f52b53..55f8ccb 100644 --- a/libbuild2/adhoc-rule-cxx.cxx +++ b/libbuild2/adhoc-rule-cxx.cxx @@ -38,7 +38,8 @@ namespace build2 } bool adhoc_cxx_rule:: - recipe_text (context&, const target&, string&& t, attributes&) + recipe_text (context&, const target&, const adhoc_actions&, + string&& t, attributes&) { code = move (t); return true; diff --git a/libbuild2/adhoc-rule-cxx.hxx b/libbuild2/adhoc-rule-cxx.hxx index 038d2e1..dfcdf3f 100644 --- a/libbuild2/adhoc-rule-cxx.hxx +++ b/libbuild2/adhoc-rule-cxx.hxx @@ -61,7 +61,8 @@ namespace build2 optional<string> sep); virtual bool - recipe_text (context&, const target&, string&& t, attributes&) override; + recipe_text (context&, const target&, const adhoc_actions&, + string&&, attributes&) override; virtual ~adhoc_cxx_rule () override; diff --git a/libbuild2/build/script/parser+line.test.testscript b/libbuild2/build/script/parser+body.test.testscript index 1b39265..11c2609 100644 --- a/libbuild2/build/script/parser+line.test.testscript +++ b/libbuild2/build/script/parser+body.test.testscript @@ -1,7 +1,62 @@ -# file : libbuild2/build/script/parser+line.test.testscript +# file : libbuild2/build/script/parser+body.test.testscript # license : MIT; see accompanying LICENSE file -test.options += -d +test.options += -b + +: lines +: +$* <<EOI >>EOO + s = 'foo' + if echo "$s" | sed 's/o/a/p' >>>? 'bar' + f = 'baz' + else + f = 'fox' + end + depdb clear + depdb string "$s" + depdb hash "$f" + foo "$s" "$f" + EOI + foo "$s" "$f" + EOO + +: temp_dir +: +{ + test.options += -t + + : no + : + $* <<EOI >false + foo + EOI + + : yes + : + $* <<EOI >true + foo + f = $~/f + EOI + + : preamble-no + : + $* <<EOI >false + f1 = $~/f2 + depdb string "$f1" + f2 = $~/f3 + depdb string "$f2" + foo "$f1" "$f2" + EOI + + : preamble-yes + : + $* <<EOI >true + f1 = $~/f1 + depdb string "$f1" + f2 = $~/f2 + foo "$f2" + EOI +} : command : diff --git a/libbuild2/build/script/parser+depdb.test.testscript b/libbuild2/build/script/parser+depdb.test.testscript new file mode 100644 index 0000000..38c4236 --- /dev/null +++ b/libbuild2/build/script/parser+depdb.test.testscript @@ -0,0 +1,90 @@ +# file : libbuild2/build/script/parser+depdb.test.testscript +# license : MIT; see accompanying LICENSE file + +test.options += -d + +: clear +: +{ + : multiple + : + $* <<EOI 2>>EOE != 0 + depdb clear + depdb clear + EOI + buildfile:12:1: error: multiple 'depdb clear' builtin calls + buildfile:11:1: info: previous call is here + EOE + + : after-string + : + $* <<EOI 2>>EOE != 0 + a = b + depdb string "$a" + depdb clear + EOI + buildfile:13:1: error: 'depdb clear' should be the first 'depdb' builtin call + buildfile:12:1: info: first 'depdb' call is here + EOE +} + +: preamble +: +{ + : no-body + : + $* <<EOI >>EOO + s = 'foo' + if echo "$s" | sed 's/o/a/p' >>>? 'bar' + f = 'baz' + else + f = 'fox' + end + depdb clear + depdb string "$s" + depdb hash "$f" + foo "$s" "$f" + EOI + s = 'foo' + if echo "$s" | sed 's/o/a/p' >>>? 'bar' + f = 'baz' + else + f = 'fox' + end + depdb string "$s" + depdb hash "$f" + EOO + + : temp_dir + : + { + test.options += -t + + : no + : + $* <<EOI >false + f = foo + depdb hash "$f" + f = $~/f + foo "$f" + EOI + + : yes + : + $* <<EOI >true + f = $~/f + depdb hash "$f" + foo "$f" + EOI + + : yes-mult + : + $* <<EOI >true + f = $~/f + depdb hash "$f" + s = "abc" + depdb string "$s" + foo "$f" + EOI + } +} diff --git a/libbuild2/build/script/parser+diag.test.testscript b/libbuild2/build/script/parser+diag.test.testscript index 5b4e64a..30eb859 100644 --- a/libbuild2/build/script/parser+diag.test.testscript +++ b/libbuild2/build/script/parser+diag.test.testscript @@ -33,8 +33,8 @@ $* <<EOI >>~%EOO% : ambiguity : { -: name -: + : name + : $* test <<EOI 2>>EOE != 0 echo abc diag xyz @@ -54,6 +54,29 @@ $* <<EOI >>~%EOO% buildfile:14:1: error: multiple 'diag' builtin calls buildfile:12:1: info: previous call is here EOE + + : names + : + $* <<EOI 2>>EOE != 0 + cp abc xyz + cat xyz + EOI + buildfile:11:1: error: low-verbosity script diagnostics name is ambiguous + buildfile:11:1: info: could be 'cp' + buildfile:12:1: info: could be 'cat' + info: consider specifying it explicitly with the 'diag' recipe attribute + info: or provide custom low-verbosity diagnostics with the 'diag' builtin + EOE + + : none + : + $* <<EOI 2>>EOE != 0 + a = 'b' + EOI + buildfile:11:1: error: unable to deduce low-verbosity script diagnostics name + info: consider specifying it explicitly with the 'diag' recipe attribute + info: or provide custom low-verbosity diagnostics with the 'diag' builtin + EOE } : inside-if @@ -91,3 +114,13 @@ $* <<EOI 2>>EOE != 0 EOI buildfile:11:8: error: 'diag' call via 'env' builtin EOE + +: before-depdb +: +$* <<EOI 2>>EOE != 0 + diag test + depdb clear + EOI + buildfile:11:1: error: 'diag' builtin call before 'depdb' call + buildfile:12:1: info: 'depdb' call is here + EOE diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index 8f2c46d..9021da1 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -26,7 +26,7 @@ namespace build2 // script parser:: - pre_parse (const target& tg, + pre_parse (const target& tg, const adhoc_actions& acts, istream& is, const path_name& pn, uint64_t line, optional<string> diag, const location& diag_loc) { @@ -39,9 +39,10 @@ namespace build2 // The script shouldn't be able to modify the target/scopes. // - target_ = const_cast<target*> (&tg); - scope_ = const_cast<scope*> (&tg.base_scope ()); - root_ = scope_->root_scope (); + target_ = const_cast<target*> (&tg); + actions_ = &acts; + scope_ = const_cast<scope*> (&tg.base_scope ()); + root_ = scope_->root_scope (); pbase_ = scope_->src_path_; @@ -108,7 +109,7 @@ namespace build2 // Save the custom dependency change tracking lines, if present. // s.depdb_clear = depdb_clear_.has_value (); - s.depdb_lines = move (depdb_lines_); + s.depdb_preamble = move (depdb_preamble_); return s; } @@ -232,7 +233,7 @@ namespace build2 if (save_line_ != nullptr) { if (save_line_ == &ln) - script_->lines.push_back (move (ln)); + script_->body.push_back (move (ln)); else *save_line_ = move (ln); } @@ -268,12 +269,12 @@ namespace build2 // cmd_if, not cmd_end. Thus remember the start position of the // next logical line. // - size_t i (script_->lines.size ()); + size_t i (script_->body.size ()); pre_parse_line (t, tt, true /* if_line */); assert (tt == type::newline); - line_type lt (script_->lines[i].type); + line_type lt (script_->body[i].type); // First take care of 'end'. // @@ -407,8 +408,8 @@ namespace build2 // Instruct the parser to save the diag builtin line separately // from the script lines, when it is fully parsed. Note that it - // will be executed prior to the script execution to obtain the - // custom diagnostics. + // will be executed prior to the script body execution to obtain + // the custom diagnostics. // diag_line_ = make_pair (line (), l); save_line_ = &diag_line_->first; @@ -428,9 +429,27 @@ namespace build2 { verify (); + // Verify that depdb is not used for anything other than + // performing update. + // + assert (actions_ != nullptr); + + for (const action& a: *actions_) + { + if (a != perform_update_id) + fail (l) << "'depdb' builtin cannot be used to " + << ctx.meta_operation_table[a.meta_operation ()].name + << ' ' << ctx.operation_table[a.operation ()]; + } + + if (diag_line_) + fail (diag_line_->second) + << "'diag' builtin call before 'depdb' call" << + info (l) << "'depdb' call is here"; + // Note that the rest of the line contains the builtin command - // name, potentially followed by the arguments to be - // hashed/saved. Thus, we parse it in the value lexer mode. + // name, potentially followed by the arguments to be hashed/saved. + // Thus, we parse it in the value lexer mode. // mode (lexer_mode::value); @@ -453,11 +472,27 @@ namespace build2 fail (l) << "multiple 'depdb clear' builtin calls" << info (*depdb_clear_) << "previous call is here"; - if (!depdb_lines_.empty ()) - fail (l) << "'depdb clear' should be the first 'depdb' " - << "builtin call" << - info (depdb_lines_[0].tokens[0].location ()) - << "first 'depdb' call is here"; + if (!depdb_preamble_.empty ()) + { + diag_record dr (fail (l)); + dr << "'depdb clear' should be the first 'depdb' builtin call"; + + // Print the first depdb call location. + // + for (const line& l: depdb_preamble_) + { + const replay_tokens& rt (l.tokens); + assert (!rt.empty ()); + + const token& t (rt[0].token); + if (t.type == type::word && t.value == "depdb") + { + dr << info (rt[0].location ()) + << "first 'depdb' call is here"; + break; + } + } + } // Save the builtin location, cancel the line saving, and clear // the referenced variable list, since it won't be used. @@ -469,13 +504,37 @@ namespace build2 } else { + // Move the script body to the end of the depdb preamble. + // + // Note that at this (pre-parsing) stage we cannot evaluate if + // all the script body lines are allowed for depdb preamble. + // That, in particular, would require to analyze pipelines to + // see if they are terminated with the set builtin, but this + // information is only available at the execution stage. Thus, + // we move into the preamble whatever is there and delay the + // check until the execution. + // + lines& ls (script_->body); + depdb_preamble_.insert (depdb_preamble_.end (), + make_move_iterator (ls.begin ()), + make_move_iterator (ls.end ())); + ls.clear (); + + // Also move the body_temp_dir flag, if it is true. + // + if (script_->body_temp_dir) + { + script_->depdb_preamble_temp_dir = true; + script_->body_temp_dir = false; + } + // Instruct the parser to save the depdb builtin line separately // from the script lines, when it is fully parsed. Note that the // builtin command arguments will be validated during execution, // when expanded. // - depdb_lines_.push_back (line ()); - save_line_ = &depdb_lines_.back (); + depdb_preamble_.push_back (line ()); + save_line_ = &depdb_preamble_.back (); } // Parse the rest of the line and bail out. @@ -531,8 +590,8 @@ namespace build2 { if (pre_parse_suspended_) { - dr << info (l) << "while deducing low-verbosity script " - << "diagnostics name"; + dr << info (l) + << "while deducing low-verbosity script diagnostics name"; suggest_diag (dr); } @@ -757,8 +816,153 @@ namespace build2 } void parser:: - execute (const scope& rs, const scope& bs, - environment& e, const script& s, runner& r) + execute_body (const scope& rs, const scope& bs, + environment& e, const script& s, runner& r, + bool enter, bool leave) + { + pre_exec (rs, bs, e, &s, &r); + + if (enter) + runner_->enter (e, s.start_loc); + + // Note that we rely on "small function object" optimization here. + // + auto exec_cmd = [this] (token& t, build2::script::token_type& tt, + size_t li, + bool single, + const location& ll) + { + // We use the 0 index to signal that this is the only command. + // + if (single) + li = 0; + + command_expr ce ( + parse_command_line (t, static_cast<token_type&> (tt))); + + runner_->run (*environment_, ce, li, ll); + }; + + exec_lines (s.body, exec_cmd); + + if (leave) + runner_->leave (e, s.end_loc); + } + + void parser:: + execute_depdb_preamble (const scope& rs, const scope& bs, + environment& e, const script& s, runner& r, + depdb& dd) + { + tracer trace ("execute_depdb_preamble"); + + // The only valid lines in the depdb preamble are the depdb builtin + // itself as well as the variable assignments, including via the set + // builtin. + + pre_exec (rs, bs, e, &s, &r); + + // Let's "wrap up" the objects we operate upon into the single object + // to rely on "small function object" optimization. + // + struct + { + environment& env; + const script& scr; + depdb& dd; + tracer& trace; + } ctx {e, s, dd, trace}; + + auto exec_cmd = [&ctx, this] + (token& t, + build2::script::token_type& tt, + size_t li, + bool /* single */, + const location& ll) + { + if (tt == type::word && t.value == "depdb") + { + names ns (exec_special (t, tt)); + + // This should have been enforced during pre-parsing. + // + assert (!ns.empty ()); // <cmd> ... <newline> + + const string& cmd (ns[0].value); + + if (cmd == "hash") + { + sha256 cs; + for (auto i (ns.begin () + 1); i != ns.end (); ++i) // Skip <cmd>. + to_checksum (cs, *i); + + if (ctx.dd.expect (cs.string ()) != nullptr) + l4 ([&] { + ctx.trace (ll) + << "'depdb hash' argument change forcing update of " + << ctx.env.target;}); + } + else if (cmd == "string") + { + string s; + try + { + s = convert<string> ( + names (make_move_iterator (ns.begin () + 1), + make_move_iterator (ns.end ()))); + } + catch (const invalid_argument& e) + { + fail (ll) << "invalid 'depdb string' argument: " << e; + } + + if (ctx.dd.expect (s) != nullptr) + l4 ([&] { + ctx.trace (ll) + << "'depdb string' argument change forcing update of " + << ctx.env.target;}); + } + else + assert (false); + } + else + { + // Note that we don't reset the line index to zero (as we do in + // execute_body()) assuming that there are some script body + // commands to follow. + // + command_expr ce ( + parse_command_line (t, static_cast<token_type&> (tt))); + + // Verify that this expression executes the set builtin. + // + if (find_if (ce.begin (), ce.end (), + [] (const expr_term& et) + { + const process_path& p (et.pipe.back ().program); + return p.initial == nullptr && + p.recall.string () == "set"; + }) == ce.end ()) + { + const replay_tokens& rt (ctx.scr.depdb_preamble.back ().tokens); + assert (!rt.empty ()); + + fail (ll) << "disallowed command in depdb preamble" << + info << "only variable assignments are allowed in " + << "depdb preamble" << + info (rt[0].location ()) << "depdb preamble ends here"; + } + + runner_->run (*environment_, ce, li, ll); + } + }; + + exec_lines (s.depdb_preamble, exec_cmd); + } + + void parser:: + pre_exec (const scope& rs, const scope& bs, + environment& e, const script* s, runner* r) { path_ = nullptr; // Set by replays. @@ -766,6 +970,8 @@ namespace build2 set_lexer (nullptr); + actions_ = nullptr; + // The script shouldn't be able to modify the scopes. // // Note that for now we don't set target_ since it's not clear what @@ -776,20 +982,15 @@ namespace build2 scope_ = const_cast<scope*> (&bs); pbase_ = scope_->src_path_; - script_ = const_cast<script*> (&s); - runner_ = &r; + script_ = const_cast<script*> (s); + runner_ = r; environment_ = &e; - - exec_script (); } void parser:: - exec_script () + exec_lines (const lines& lns, + const function<exec_cmd_function>& exec_cmd) { - const script& s (*script_); - - runner_->enter (*environment_, s.start_loc); - // Note that we rely on "small function object" optimization for the // exec_*() lambdas. // @@ -814,22 +1015,6 @@ namespace build2 apply_value_attributes (&var, lhs, move (rhs), kind); }; - auto exec_cmd = [this] (token& t, build2::script::token_type& tt, - size_t li, - bool single, - const location& ll) - { - // We use the 0 index to signal that this is the only command. - // - if (single) - li = 0; - - command_expr ce ( - parse_command_line (t, static_cast<token_type&> (tt))); - - runner_->run (*environment_, ce, li, ll); - }; - auto exec_if = [this] (token& t, build2::script::token_type& tt, size_t li, const location& ll) @@ -842,14 +1027,26 @@ namespace build2 return runner_->run_if (*environment_, ce, li, ll); }; - size_t li (1); + build2::script::parser::exec_lines (lns.begin (), lns.end (), + exec_set, exec_cmd, exec_if, + environment_->exec_line, + &environment_->var_pool); + } - exec_lines (s.lines.begin (), s.lines.end (), - exec_set, exec_cmd, exec_if, - li, - &environment_->var_pool); + names parser:: + exec_special (token& t, build2::script::token_type& tt, + bool omit_builtin) + { + if (omit_builtin) + { + assert (tt != type::newline && tt != type::eos); - runner_->leave (*environment_, s.end_loc); + next (t, tt); + } + + return tt != type::newline && tt != type::eos + ? parse_names (t, tt, pattern_mode::expand) + : names (); } names parser:: @@ -858,25 +1055,7 @@ namespace build2 const line& ln, bool omit_builtin) { - path_ = nullptr; // Set by replays. - - pre_parse_ = false; - - set_lexer (nullptr); - - // The script shouldn't be able to modify the scopes. - // - // Note that for now we don't set target_ since it's not clear what - // it could be used for (we need scope_ for calling functions such as - // $target.path()). - // - root_ = const_cast<scope*> (&rs); - scope_ = const_cast<scope*> (&bs); - pbase_ = scope_->src_path_; - - script_ = nullptr; - runner_ = nullptr; - environment_ = &e; + pre_exec (rs, bs, e, nullptr /* script */, nullptr /* runner */); // Copy the tokens and start playing. // @@ -886,16 +1065,7 @@ namespace build2 build2::script::token_type tt; next (t, tt); - if (omit_builtin) - { - assert (tt != type::newline && tt != type::eos); - - next (t, tt); - } - - names r (tt != type::newline && tt != type::eos - ? parse_names (t, tt, pattern_mode::expand) - : names ()); + names r (exec_special (t, tt, omit_builtin)); replay_stop (); return r; @@ -928,7 +1098,7 @@ namespace build2 if (special_variable (name)) { if (name == "~") - script_->temp_dir = true; + script_->body_temp_dir = true; } else if (!name.empty ()) { @@ -970,7 +1140,7 @@ namespace build2 // if (script_ != nullptr && !script_->depdb_clear && - script_->depdb_lines.empty ()) + script_->depdb_preamble.empty ()) { if (r.defined () && !r.belongs (*environment_)) { diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx index 5ada8be..e96a806 100644 --- a/libbuild2/build/script/parser.hxx +++ b/libbuild2/build/script/parser.hxx @@ -8,6 +8,7 @@ #include <libbuild2/forward.hxx> #include <libbuild2/utility.hxx> +#include <libbuild2/depdb.hxx> #include <libbuild2/diagnostics.hxx> #include <libbuild2/script/parser.hxx> @@ -34,7 +35,7 @@ namespace build2 // name. // script - pre_parse (const target&, + pre_parse (const target&, const adhoc_actions& acts, istream&, const path_name&, uint64_t line, optional<string> diag_name, const location& diag_loc); @@ -63,9 +64,26 @@ namespace build2 // Execute. Issue diagnostics and throw failed in case of an error. // public: + + // By default call the runner's enter() and leave() functions that + // initialize/clean up the environment before/after the script + // execution. + // + void + execute_body (const scope& root, const scope& base, + environment&, const script&, runner&, + bool enter = true, bool leave = true); + + + // Note that it's the caller's responsibility to make sure that the + // runner's enter() function is called before the first preamble/body + // command execution and leave() -- after the last command. + // void - execute (const scope& root, const scope& base, - environment&, const script&, runner&); + execute_depdb_preamble (const scope& root, const scope& base, + environment&, const script&, runner&, + depdb&); + // Parse a special builtin line into names, performing the variable // and pattern expansions. If omit_builtin is true, then omit the @@ -78,8 +96,18 @@ namespace build2 bool omit_builtin = true); protected: + // Setup the parser for subsequent exec_*() function calls. + // + void + pre_exec (const scope& root, const scope& base, + environment&, const script*, runner*); + void - exec_script (); + exec_lines (const lines&, const function<exec_cmd_function>&); + + names + exec_special (token& t, build2::script::token_type& tt, + bool omit_builtin = true); // Helpers. // @@ -112,6 +140,7 @@ namespace build2 protected: script* script_; + const adhoc_actions* actions_; // Non-NULL during pre-parsing. // Current low-verbosity script diagnostics and its weight. // @@ -165,8 +194,8 @@ namespace build2 // // depdb string <arg> - Track the argument (single) change as string. // - optional<location> depdb_clear_; // 'depdb clear' location if any. - lines depdb_lines_; // Note: excludes 'depdb clear'. + optional<location> depdb_clear_; // 'depdb clear' location if any. + lines depdb_preamble_; // Note: excludes 'depdb clear'. // True during pre-parsing when the pre-parse mode is temporarily // suspended to perform expansion. diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx index 7f2840d..4a3e8cc 100644 --- a/libbuild2/build/script/parser.test.cxx +++ b/libbuild2/build/script/parser.test.cxx @@ -71,20 +71,24 @@ namespace build2 // Usages: // // argv[0] [-l] - // argv[0] -d - // argv[0] -p + // argv[0] -b [-t] + // argv[0] -d [-t] + // argv[0] -q // argv[0] -g [<diag-name>] // // In the first form read the script from stdin and trace the script - // execution to stdout using the custom print runner. + // body execution to stdout using the custom print runner. // // In the second form read the script from stdin, parse it and dump the - // resulting lines to stdout. + // script body lines to stdout. // - // In the third form read the script from stdin, parse it and print + // In the third form read the script from stdin, parse it and dump the + // depdb preamble lines to stdout. + // + // In the forth form read the script from stdin, parse it and print // line tokens quoting information to stdout. // - // In the forth form read the script from stdin, parse it and print the + // In the fifth form read the script from stdin, parse it and print the // low-verbosity script diagnostics name or custom low-verbosity // diagnostics to stdout. If the script doesn't deduce any of them, then // print the diagnostics and exit with non-zero code. @@ -92,10 +96,17 @@ namespace build2 // -l // Print the script line number for each executed expression. // + // -b + // Dump the parsed script body to stdout. + // // -d - // Dump the parsed script to sdout. + // Dump the parsed script depdb preamble to stdout. // - // -p + // -t + // Print true if the body (-b) or depdb preamble (-d) references the + // temporary directory and false otherwise. + // + // -q // Print the parsed script tokens quoting information to sdout. If a // token is quoted follow its representation with its quoting // information in the [<quoting>/<completeness>] form, where: @@ -115,13 +126,15 @@ namespace build2 enum class mode { run, - dump, - print, + body, + depdb_preamble, + quoting, diag } m (mode::run); bool print_line (false); optional<string> diag_name; + bool temp_dir (false); for (int i (1); i != argc; ++i) { @@ -129,10 +142,17 @@ namespace build2 if (a == "-l") print_line = true; + else if (a == "-b") + m = mode::body; else if (a == "-d") - m = mode::dump; - else if (a == "-p") - m = mode::print; + m = mode::depdb_preamble; + else if (a == "-t") + { + assert (m == mode::body || m == mode::depdb_preamble); + temp_dir = true; + } + else if (a == "-q") + m = mode::quoting; else if (a == "-g") m = mode::diag; else @@ -179,12 +199,14 @@ namespace build2 tt.path (path ("driver")); + adhoc_actions acts {perform_update_id}; + // Parse and run. // parser p (ctx); path_name nm ("buildfile"); - script s (p.pre_parse (tt, + script s (p.pre_parse (tt, acts, cin, nm, 11 /* line */, (m != mode::diag @@ -196,9 +218,9 @@ namespace build2 { case mode::run: { - environment e (perform_update_id, tt, false /* temp_dir */); + environment e (perform_update_id, tt, s.body_temp_dir); print_runner r (print_line); - p.execute (ctx.global_scope, ctx.global_scope, e, s, r); + p.execute_body (ctx.global_scope, ctx.global_scope, e, s, r); break; } case mode::diag: @@ -221,14 +243,27 @@ namespace build2 break; } - case mode::dump: + case mode::body: { - dump (cout, "", s.lines); + if (!temp_dir) + dump (cout, "", s.body); + else + cout << (s.body_temp_dir ? "true" : "false") << endl; + + break; + } + case mode::depdb_preamble: + { + if (!temp_dir) + dump (cout, "", s.depdb_preamble); + else + cout << (s.depdb_preamble_temp_dir ? "true" : "false") << endl; + break; } - case mode::print: + case mode::quoting: { - for (const line& l: s.lines) + for (const line& l: s.body) { for (const replay_token& rt: l.tokens) { diff --git a/libbuild2/build/script/script.cxx b/libbuild2/build/script/script.cxx index c6b57c3..e003b6a 100644 --- a/libbuild2/build/script/script.cxx +++ b/libbuild2/build/script/script.cxx @@ -74,13 +74,20 @@ namespace build2 assign (var_pool.insert ("<")) = move (ns); } - // Set the $~ special variable. - // if (temp) - { + set_temp_dir_variable (); + } + + void environment:: + set_temp_dir_variable () + { + // Note that the temporary directory could have been created + // implicitly by the runner. + // + if (temp_dir.path.empty ()) create_temp_dir (); - assign (var_pool.insert<dir_path> ("~")) = temp_dir.path; - } + + assign (var_pool.insert<dir_path> ("~")) = temp_dir.path; } void environment:: diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx index 9284813..e11cb45 100644 --- a/libbuild2/build/script/script.hxx +++ b/libbuild2/build/script/script.hxx @@ -45,7 +45,8 @@ namespace build2 // Note that the variables are not pre-entered into a pool during the // parsing phase, so the line variable pointers are NULL. // - lines_type lines; + lines_type body; + bool body_temp_dir = false; // True if the body references $~. // Referenced ordinary (non-special) variables. // @@ -59,10 +60,6 @@ namespace build2 // small_vector<string, 2> vars; // 2 for command and options. - // True if script references the $~ special variable. - // - bool temp_dir = false; - // Command name for low-verbosity diagnostics and custom low-verbosity // diagnostics line. Note: cannot be both (see the script parser for // details). @@ -74,7 +71,8 @@ namespace build2 // script parser for details). // bool depdb_clear; - lines_type depdb_lines; + lines_type depdb_preamble; + bool depdb_preamble_temp_dir = false; // True if references $~. location start_loc; location end_loc; @@ -136,6 +134,18 @@ namespace build2 optional<deadline> script_deadline; optional<deadline> fragment_deadline; + // Index of the next script line to be executed. Used and incremented + // by the parser's execute_depdb_preamble() and execute_body() + // function calls to produce special file names, etc. + // + size_t exec_line = 1; + + // Create the temporary directory (if it doesn't exist yet) and set + // the $~ special variable to its path. + // + void + set_temp_dir_variable (); + virtual void set_variable (string&& name, names&&, diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index 8676c9d..92e0090 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -1236,17 +1236,6 @@ namespace build2 if (!skip) { - if (d.first) - { - if (ar->recipe_text (ctx, *target_, move (t.value), d.as)) - d.clean = true; - - // Verify we have no unhandled attributes. - // - for (attribute& a: d.as) - fail (d.as.loc) << "unknown recipe attribute " << a << endf; - } - auto& ars (target_->adhoc_recipes); ars.push_back (adhoc_recipe {{}, move (ar)}); @@ -1315,6 +1304,20 @@ namespace build2 ars.back ().actions.push_back (a); } } + + if (d.first) + { + adhoc_recipe& ar (ars.back ()); + + if (ar.rule->recipe_text ( + ctx, *target_, ar.actions, move (t.value), d.as)) + d.clean = true; + + // Verify we have no unhandled attributes. + // + for (attribute& a: d.as) + fail (d.as.loc) << "unknown recipe attribute " << a << endf; + } } next (t, tt); diff --git a/libbuild2/recipe.hxx b/libbuild2/recipe.hxx index b88311c..7eca34a 100644 --- a/libbuild2/recipe.hxx +++ b/libbuild2/recipe.hxx @@ -54,10 +54,11 @@ namespace build2 // A recipe is a fragment of a rule so we handle ad hoc recipies by // "completing" them to rules. // + using adhoc_actions = small_vector<action, 1>; struct adhoc_recipe { - small_vector<action, 1> actions; - shared_ptr<adhoc_rule> rule; + adhoc_actions actions; + shared_ptr<adhoc_rule> rule; }; } diff --git a/libbuild2/rule.hxx b/libbuild2/rule.hxx index 8796659..e29c0b8 100644 --- a/libbuild2/rule.hxx +++ b/libbuild2/rule.hxx @@ -160,7 +160,8 @@ namespace build2 // therefore requires cleanup. // virtual bool - recipe_text (context&, const target&, string&&, attributes&) = 0; + recipe_text (context&, const target&, const adhoc_actions&, + string&&, attributes&) = 0; public: // Some of the operations come in compensating pairs, such as update and diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx index 3daa789..182696b 100644 --- a/libbuild2/script/run.cxx +++ b/libbuild2/script/run.cxx @@ -2299,19 +2299,82 @@ namespace build2 void clean (environment& env, const location& ll) { - context& ctx (env.context); + // We don't use the build2 filesystem utilities here in order to remove + // the filesystem entries regardless of the dry-run mode and also to add + // the location info to diagnostics. Other than that, these lambdas + // implement the respective utility functions semantics. + // + auto rmfile = [&ll] (const path& f) + { + try + { + rmfile_status r (try_rmfile (f)); + + if (r == rmfile_status::success && verb >= 3) + text << "rm " << f; + + return r; + } + catch (const system_error& e) + { + fail (ll) << "unable to remove file " << f << ": " << e << endf; + } + }; + + auto rmdir = [&ll] (const dir_path& d) + { + try + { + rmdir_status r (!work.sub (d) + ? try_rmdir (d) + : rmdir_status::not_empty); + + if (r == rmdir_status::success && verb >= 3) + text << "rmdir " << d; + + return r; + } + catch (const system_error& e) + { + fail (ll) << "unable to remove directory " << d << ": " << e << endf; + } + }; + + auto rmdir_r = [&ll] (const dir_path& d, bool dir) + { + if (work.sub (d)) // Don't try to remove working directory. + return rmdir_status::not_empty; + + if (!build2::entry_exists (d)) + return rmdir_status::not_exist; + + try + { + butl::rmdir_r (d, dir); + } + catch (const system_error& e) + { + fail (ll) << "unable to remove directory " << d << ": " << e << endf; + } + + if (verb >= 3) + text << "rmdir -r " << d; + + return rmdir_status::success; + }; + const dir_path& wdir (*env.work_dir.path); // Note that we operate with normalized paths here. // - // Remove special files. The order is not important as we don't - // expect directories here. + // Remove special files. The order is not important as we don't expect + // directories here. // for (const path& p: env.special_cleanups) { // Remove the file if exists. Fail otherwise. // - if (rmfile (ctx, p, 3) == rmfile_status::not_exist) + if (rmfile (p) == rmfile_status::not_exist) fail (ll) << "registered for cleanup special file " << p << " does not exist"; } @@ -2332,9 +2395,9 @@ namespace build2 // Wildcard with the last component being '***' (without trailing // separator) matches all files and sub-directories recursively as - // well as the start directories itself. So we will recursively - // remove the directories that match the parent (for the original - // path) directory wildcard. + // well as the start directories itself. So we will recursively remove + // the directories that match the parent (for the original path) + // directory wildcard. // bool recursive (cp.leaf ().representation () == "***"); const path& p (!recursive ? cp : cp.directory ()); @@ -2345,15 +2408,19 @@ namespace build2 { bool removed (false); - auto rm = [&cp, recursive, &removed, &ll, &ctx, &wdir] + auto rm = [&cp, + recursive, + &removed, + &ll, + &wdir, + &rmfile, &rmdir, &rmdir_r] (path&& pe, const string&, bool interm) { if (!interm) { - // While removing the entry we can get not_exist due to - // racing conditions, but that's ok if somebody did our job. - // Note that we still set the removed flag to true in this - // case. + // While removing the entry we can get not_exist due to racing + // conditions, but that's ok if somebody did our job. Note that + // we still set the removed flag to true in this case. // removed = true; // Will be meaningless on failure. @@ -2363,7 +2430,7 @@ namespace build2 if (!recursive) { - rmdir_status r (rmdir (ctx, d, 3)); + rmdir_status r (rmdir (d)); if (r != rmdir_status::not_empty) return true; @@ -2377,13 +2444,10 @@ namespace build2 } else { - // Don't remove the working directory (it will be removed - // by the dedicated cleanup). - // - // Cast to uint16_t to avoid ambiguity with - // libbutl::rmdir_r(). + // Don't remove the working directory (it will be removed by + // the dedicated cleanup). // - rmdir_status r (rmdir_r (ctx, d, d != wdir, 3)); + rmdir_status r (rmdir_r (d, d != wdir)); if (r != rmdir_status::not_empty) return true; @@ -2396,14 +2460,14 @@ namespace build2 } } else - rmfile (ctx, pe, 3); + rmfile (pe); } return true; }; - // Note that here we rely on the fact that recursive iterating - // goes depth-first (which make sense for the cleanup). + // Note that here we rely on the fact that recursive iterating goes + // depth-first (which make sense for the cleanup). // try { @@ -2434,9 +2498,8 @@ namespace build2 : "file"); } - // Remove the directory if exists and empty. Fail otherwise. - // Removal of non-existing directory is not an error for 'maybe' - // cleanup type. + // Remove the directory if exists and empty. Fail otherwise. Removal + // of non-existing directory is not an error for 'maybe' cleanup type. // if (p.to_directory ()) { @@ -2444,20 +2507,10 @@ namespace build2 bool wd (d == wdir); // Don't remove the working directory for the recursive cleanup - // (it will be removed by the dedicated one). - // - // Note that the root working directory contains the - // .buildignore file (see above). - // - // @@ If 'd' is a file then will fail with a diagnostics having - // no location info. Probably need to add an optional location - // parameter to rmdir() function. The same problem exists for - // a file cleanup when try to rmfile() directory instead of - // file. + // since it needs to be removed by the caller (can contain + // .buildignore file, etc). // - rmdir_status r (recursive - ? rmdir_r (ctx, d, !wd, static_cast <uint16_t> (3)) - : rmdir (ctx, d, 3)); + rmdir_status r (recursive ? rmdir_r (d, !wd) : rmdir (d)); if (r == rmdir_status::success || (r == rmdir_status::not_exist && t == cleanup_type::maybe)) @@ -2473,10 +2526,10 @@ namespace build2 print_dir (dr, d, ll); } - // Remove the file if exists. Fail otherwise. Removal of - // non-existing file is not an error for 'maybe' cleanup type. + // Remove the file if exists. Fail otherwise. Removal of non-existing + // file is not an error for 'maybe' cleanup type. // - if (rmfile (ctx, p, 3) == rmfile_status::not_exist && + if (rmfile (p) == rmfile_status::not_exist && t == cleanup_type::always) fail (ll) << "registered for cleanup file " << p << " does not exist"; |