From 920ed11a433b0e292a18adb8c68829a00e8c70cc Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Tue, 26 May 2020 21:35:59 +0300 Subject: Allow process path values and targets as buildscript program names Also deduce the recipe name. --- libbuild2/build/script/parser+line.test.testscript | 4 +- libbuild2/build/script/parser.cxx | 423 ++++++++++++++++++++- libbuild2/build/script/parser.hxx | 47 ++- libbuild2/build/script/parser.test.cxx | 9 +- libbuild2/build/script/runner.cxx | 10 +- libbuild2/build/script/script.hxx | 1 - 6 files changed, 471 insertions(+), 23 deletions(-) (limited to 'libbuild2/build') diff --git a/libbuild2/build/script/parser+line.test.testscript b/libbuild2/build/script/parser+line.test.testscript index 6401d91..45b07b7 100644 --- a/libbuild2/build/script/parser+line.test.testscript +++ b/libbuild2/build/script/parser+line.test.testscript @@ -38,7 +38,7 @@ $* <>EOF $* <>EOO foo 'bar' "baz" '' "" "$foo" - "foo$" + "foo$bar" "fo"o "foo"\" "foo\\" @@ -55,7 +55,7 @@ $* <>EOO EOI foo 'bar' "baz" '' "" "$foo" - "foo$" + "foo$bar" "foo" "foo\"" "foo\\" diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index 81e78f3..2a0555e 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -3,10 +3,15 @@ #include +#include + +#include + #include #include using namespace std; +using namespace butl; namespace build2 { @@ -21,8 +26,9 @@ namespace build2 // script parser:: - pre_parse (istream& is, const path_name& pn, uint64_t line, - optional diag) + pre_parse (const target& tg, + istream& is, const path_name& pn, uint64_t line, + optional diag_name, const location& diag_loc) { path_ = &pn; @@ -31,13 +37,25 @@ namespace build2 lexer l (is, *path_, line, lexer_mode::command_line); set_lexer (&l); - script s; - s.diag = move (diag); + // The script shouldn't be able to modify the target/scopes. + // + target_ = const_cast (&tg); + scope_ = const_cast (&tg.base_scope ()); + root_ = scope_->root_scope (); + + pbase_ = scope_->src_path_; + script s; script_ = &s; runner_ = nullptr; environment_ = nullptr; + if (diag_name) + { + diag = make_pair (move (*diag_name), diag_loc); + diag_weight = 4; + } + s.start_loc = location (*path_, line, 1); token t (pre_parse_script ()); @@ -46,6 +64,36 @@ namespace build2 s.end_loc = get_location (t); + // Diagnose absent/ambigous script name. + // + { + diag_record dr; + + if (!diag) + { + dr << fail (s.start_loc) + << "unable to deduce low-verbosity script diagnostics name"; + } + else if (diag2) + { + dr << fail (s.start_loc) + << "low-verbosity script diagnostics name is ambiguous" << + info (diag->second) << "could be '" << diag->first << "'" << + info (diag2->second) << "could be '" << diag2->first << "'"; + } + + if (!dr.empty ()) + { + dr << info << "consider specifying it explicitly with the 'diag' " + << "recipe attribute"; + + dr << info << "or provide custom low-verbosity diagnostics with " + << "the 'diag' builtin"; + } + } + + s.diag = move (diag->first); + return s; } @@ -150,16 +198,26 @@ namespace build2 assert (tt == type::newline); - ln.type = lt; - ln.tokens = replay_data (); - script_->lines.push_back (move (ln)); + //@@ TODO: we need to make sure special builtin is the first command. - if (lt == line_type::cmd_if || lt == line_type::cmd_ifn) + // Save the script line, unless this is a special builtin (indicated + // by the replay::stop mode). + // + if (replay_ == replay::save) { - tt = peek (lexer_mode::first_token); + ln.type = lt; + ln.tokens = replay_data (); + script_->lines.push_back (move (ln)); + + if (lt == line_type::cmd_if || lt == line_type::cmd_ifn) + { + tt = peek (lexer_mode::first_token); - pre_parse_if_else (t, tt); + pre_parse_if_else (t, tt); + } } + else + assert (replay_ == replay::stop && lt == line_type::cmd); } void parser:: @@ -244,6 +302,333 @@ namespace build2 // Execute. // + optional parser:: + parse_program (token& t, build2::script::token_type& tt, names& ns) + { + const location l (get_location (t)); + + // Set the current script name if it is not set or its weight is less + // than the new name weight, skipping names with the zero weight. If + // the weight is the same but the name is different then record this + // ambiguity, unless one is already recorded. This ambiguity will be + // reported at the end of the script pre-parsing, unless discarded by + // the name with a greater weight. + // + auto set_diag = [&l, this] (string d, uint8_t w) + { + if (diag_weight < w) + { + diag = make_pair (move (d), l); + diag_weight = w; + diag2 = nullopt; + } + else if (w != 0 && w == diag_weight && d != diag->first && !diag2) + diag2 = make_pair (move (d), l); + }; + + // Handle special builtins. + // + if (pre_parse_) + { + if (tt == type::word && t.value == "diag") + { + // @@ Redo the diag directive handling (save line separately and + // execute later, before script execution). + // + if (diag_weight == 4) + { + fail (script_->start_loc) + << "low-verbosity script diagnostics name is ambiguous" << + info (diag->second) << "could be '" << diag->first << "'" << + info (l) << "could be ''"; + } + + set_diag ("", 4); + + build2::script::parser::parse_program (t, tt, ns); + replay_stop (); + + return nullopt; + } + } + + auto suggest_diag = [this] (const diag_record& dr) + { + dr << info << "consider specifying it explicitly with " + << "the 'diag' recipe attribute"; + + dr << info << "or provide custom low-verbosity diagnostics " + << " with the 'diag' builtin"; + }; + + parse_names_result pr; + { + // During pre-parse, if the script name is not set manually, we + // suspend pre-parse, parse the command names for real and try to + // deduce the script name from the result. Otherwise, we continue to + // pre-parse and bail out after parsing the names. + // + // Note that the later is not just an optimization since expansion + // that wouldn't fail during execution may fail in this special + // mode, for example: + // + // ... + // {{ + // x = true + // ba($x ? r : z) + // }} + // + // v = a b + // ... + // {{ + // v = o + // fo$v + // }} + // + // This is also the reason why we add a diag frame. + // + if (pre_parse_ && diag_weight != 4) + { + pre_parse_ = false; // Make parse_names() perform expansions. + pre_parse_suspended_ = true; + } + + auto df = make_diag_frame ( + [&l, &suggest_diag, this] (const diag_record& dr) + { + if (pre_parse_suspended_) + { + dr << info (l) << "while deducing low-verbosity script " + << "diagnostics name"; + + suggest_diag (dr); + } + }); + + pr = parse_names (t, tt, + ns, + pattern_mode::ignore, + true /* chunk */, + "command line", + nullptr); + + if (pre_parse_suspended_) + { + pre_parse_suspended_ = false; + pre_parse_ = true; + } + + if (pre_parse_ && diag_weight == 4) + return nullopt; + } + + // Try to translate names into a process path, unless there is nothing + // to translate. + // + // We only end up here in the pre-parse mode if we are still searching + // for the script name. + // + if (!pr.not_null || ns.empty ()) + { + if (pre_parse_) + { + diag_record dr (fail (l)); + dr << "unable to deduce low-verbosity script diagnostics name"; + suggest_diag (dr); + } + + return nullopt; + } + + // We have to handle process_path[_ex] and executable target. The + // process_path[_ex] we may have to recognize syntactically because + // of the loss of type, for example: + // + // c = $cxx.path --version + // + // {{ + // $c ... + // }} + // + // This is further complicated by the fact that the first name in + // process_path[_ex] may or may not be a pair (it's not a pair if + // recall and effective paths are the same). If it's not a pair and we + // are dealing with process_path, then we don't need to do anything + // extra -- it will just be treated as normal program path. However, + // if it's process_path_ex, then we may end up with something along + // these lines: + // + // /usr/bin/g++ name@c++ checksum@deadbeef --version + // + // Which is a bit harder to recognize syntactically. So what we are + // going to do is have a separate first pass which reduces the + // syntactic cases to the typed ones. + // + names pp_ns; + if (pr.type == &value_traits::value_type || + pr.type == &value_traits::value_type) + { + pp_ns = move (ns); + ns.clear (); + } + else if (ns[0].file ()) + { + // Find the end of the value. + // + auto b (ns.begin ()), i (b), e (ns.end ()); + for (i += i->pair ? 2 : 1; i != e && i->pair; i += 2) + { + if (!i->simple () || + (i->value != "name" && i->value != "checksum")) + break; + } + + if (b->pair || i != b + 1) // First is a pair or pairs after. + { + pp_ns.insert (pp_ns.end (), + make_move_iterator (b), make_move_iterator (i)); + + ns.erase (b, i); + + pr.type = i != b + 1 + ? &value_traits::value_type + : &value_traits::value_type; + } + } + + // Handle process_path[_ex], for example: + // + // {{ + // $cxx.path ... + // }} + // + if (pr.type == &value_traits::value_type) + { + auto pp (convert (move (pp_ns))); + + if (pre_parse_) + { + diag_record dr (fail (l)); + dr << "unable to deduce low-verbosity script diagnostics name " + << "from process path " << pp; + suggest_diag (dr); + } + else + return optional (move (pp)); + } + else if (pr.type == &value_traits::value_type) + { + auto pp (convert (move (pp_ns))); + + if (pre_parse_) + { + if (pp.name) + { + set_diag (move (*pp.name), 3); + return nullopt; + } + + diag_record dr (fail (l)); + dr << "unable to deduce low-verbosity script diagnostics name " + << "from process path " << pp; + suggest_diag (dr); + } + else + return optional (move (pp)); + } + // + // Handle the executable target, for example: + // + // import! [metadata] cli = cli%exe{cli} + // ... + // {{ + // $cli ... + // }} + // + else if (!ns[0].simple ()) + { + if (const target* t = search_existing ( + ns[0], *scope_, ns[0].pair ? ns[1].dir : empty_dir_path)) + { + if (const auto* et = t->is_a ()) + { + if (pre_parse_) + { + if (auto* n = et->lookup_metadata ("name")) + { + set_diag (*n, 3); + return nullopt; + } + // Fall through. + } + else + { + process_path pp (et->process_path ()); + + if (pp.empty ()) + fail (l) << "target " << *et << " is out of date" << + info << "consider specifying it as a prerequisite of " + << environment_->target; + + ns.erase (ns.begin (), ns.begin () + (ns[0].pair ? 2 : 1)); + return optional (move (pp)); + } + } + + if (pre_parse_) + { + diag_record dr (fail (l)); + dr << "unable to deduce low-verbosity script diagnostics name " + << "from target " << *t; + suggest_diag (dr); + } + } + + if (pre_parse_) + { + diag_record dr (fail (l)); + dr << "unable to deduce low-verbosity script diagnostics name " + << "from " << ns; + suggest_diag (dr); + } + else + return nullopt; + } + else if (pre_parse_) + { + // If we are here, the name is simple and is not part of a pair. + // + string& v (ns[0].value); + + // Try to interpret the name as a builtin. + // + const builtin_info* bi (builtins.find (v)); + + if (bi != nullptr) + { + set_diag (move (v), bi->weight); + return nullopt; + } + // + // Try to interpret the name as a pseudo-builtin. + // + // Note that both of them has the zero weight and cannot be picked + // up as a script name. + // + else if (v == "set" || v == "exit") + { + return nullopt; + } + + diag_record dr (fail (l)); + dr << "unable to deduce low-verbosity script diagnostics name " + << "for program " << ns[0]; + suggest_diag (dr); + } + + return nullopt; + } + void parser:: execute (const scope& rs, const scope& bs, environment& e, const script& s, runner& r) @@ -256,6 +641,10 @@ namespace build2 // The script shouldn't be able to modify the scopes. // + // Note that for now we don't set target_ since it's not clear what + // it could be used for (we need scope_ for calling functions such as + // $target.path()). + // root_ = const_cast (&rs); scope_ = const_cast (&bs); pbase_ = scope_->src_path_; @@ -350,8 +739,10 @@ namespace build2 // In the pre-parse mode collect the referenced variable names for the // script semantics change tracking. // - if (pre_parse_) + if (pre_parse_ || pre_parse_suspended_) { + lookup r; + // Add the variable name skipping special variables and suppressing // duplicates. While at it, check if the script temporary directory // is referenced and set the flag, if that's the case. @@ -363,13 +754,21 @@ namespace build2 } else if (!name.empty ()) { + if (pre_parse_suspended_) + { + const variable* pvar (scope_->ctx.var_pool.find (name)); + + if (pvar != nullptr) + r = (*target_)[*pvar]; + } + auto& vars (script_->vars); if (find (vars.begin (), vars.end (), name) == vars.end ()) vars.push_back (move (name)); } - return lookup (); + return r; } if (!qual.empty ()) diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx index 5f08209..15d4ede 100644 --- a/libbuild2/build/script/parser.hxx +++ b/libbuild2/build/script/parser.hxx @@ -34,8 +34,9 @@ namespace build2 // name. // script - pre_parse (istream&, const path_name&, uint64_t line, - optional diag); + pre_parse (const target&, + istream&, const path_name&, uint64_t line, + optional diag, const location& diag_loc); // Recursive descent parser. // @@ -82,9 +83,51 @@ namespace build2 virtual lookup lookup_variable (name&&, string&&, const location&) override; + // During execution translate the process path and executable targets + // leaving the rest for the base parser to handle. + // + // During pre-parsing try to deduce the low-verbosity script + // diagnostics name. + // + virtual optional + parse_program (token&, build2::script::token_type&, names&) override; + + void + parse_program_diag (token&, build2::script::token_type&, names&); + protected: script* script_; + // Current low-verbosity script diagnostics name and weight. + // + // During pre-parsing each command leading names are translated into a + // potential script name, unless it is set manually (with the diag + // directive or via the constructor). The potential script name has a + // weight associated with it, so script names with greater weights + // override names with lesser weights. The possible weights are: + // + // 0 - builtins that do not add to the script semantics (exit, + // true, etc) and are never picked up as a script name + // + // [1 2] - other builtins + // + // 3 - process path or executable target + // + // 4 - manually set names + // + // If two potential script names with the same weights are encountered + // then this ambiguity is reported unless a higher-weighted name is + // encountered later. + // + optional> diag; + optional> diag2; + uint8_t diag_weight = 0; + + // True during pre-parsing when the pre-parse mode is temporarily + // suspended to perform expansion. + // + bool pre_parse_suspended_ = false; + // Execute state. // runner* runner_; diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx index 10e1701..de3e839 100644 --- a/libbuild2/build/script/parser.test.cxx +++ b/libbuild2/build/script/parser.test.cxx @@ -58,7 +58,7 @@ namespace build2 cout << endl; - return e.back ().pipe.back ().program.string () == "true"; + return e.back ().pipe.back ().program.recall.string () == "true"; } virtual void @@ -160,7 +160,12 @@ namespace build2 // parser p (ctx); path_name nm ("buildfile"); - script s (p.pre_parse (cin, nm, 11 /* line */, nullopt)); + + script s (p.pre_parse (tt, + cin, nm, + 11 /* line */, + string ("test"), + location (nm, 10))); switch (m) { diff --git a/libbuild2/build/script/runner.cxx b/libbuild2/build/script/runner.cxx index 315a248..2a59505 100644 --- a/libbuild2/build/script/runner.cxx +++ b/libbuild2/build/script/runner.cxx @@ -61,14 +61,14 @@ namespace build2 try { - // Note that the temporary directory must be empty to date. + // Note that the temporary directory must be empty. // rmdir_status r (try_rmdir (td)); if (r != rmdir_status::success) { // While there can be no fault of the script being currently - // executed let's add the location anyway to ease the + // executed let's add the location anyway to help with // troubleshooting. And let's stick to that principle down the // road. // @@ -110,8 +110,10 @@ namespace build2 find_if (expr.begin (), expr.end (), [] (const expr_term& et) { - const string& p (et.pipe.back ().program.string ()); - return p == "set" || p == "exit"; + const process_path& p (et.pipe.back ().program); + return p.initial == nullptr && + (p.recall.string () == "set" || + p.recall.string () == "exit"); }) != expr.end ()) build2::script::run (env, expr, li, ll); else if (verb >= 2) diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx index 08cdbd2..de759de 100644 --- a/libbuild2/build/script/script.hxx +++ b/libbuild2/build/script/script.hxx @@ -37,7 +37,6 @@ namespace build2 class script { public: - // Note that the variables are not pre-entered into a pool during the // parsing phase, so the line variable pointers are NULL. // -- cgit v1.1