diff options
Diffstat (limited to 'libbuild2/parser.cxx')
| -rw-r--r-- | libbuild2/parser.cxx | 5380 |
1 files changed, 4236 insertions, 1144 deletions
diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index ab4fa48..5321cd5 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -6,8 +6,7 @@ #include <sstream> #include <iostream> // cout -#include <libbutl/filesystem.mxx> // path_search -#include <libbutl/path-pattern.mxx> +#include <libbutl/filesystem.hxx> // path_search #include <libbuild2/rule.hxx> #include <libbuild2/dump.hxx> @@ -23,6 +22,10 @@ #include <libbuild2/adhoc-rule-cxx.hxx> #include <libbuild2/adhoc-rule-buildscript.hxx> +#include <libbuild2/adhoc-rule-regex-pattern.hxx> + +#include <libbuild2/dist/module.hxx> // module + #include <libbuild2/config/utility.hxx> // lookup_config using namespace std; @@ -41,7 +44,10 @@ namespace build2 { o << '='; names storage; - to_stream (o, reverse (a.value, storage), true /* quote */, '@'); + to_stream (o, + reverse (a.value, storage, true /* reduce */), + quote_mode::normal, + '@'); } return o; @@ -50,33 +56,22 @@ namespace build2 class parser::enter_scope { public: - enter_scope (): p_ (nullptr), r_ (nullptr), s_ (nullptr), b_ (nullptr) {} + enter_scope () + : p_ (nullptr), r_ (nullptr), s_ (nullptr), b_ (nullptr) {} enter_scope (parser& p, dir_path&& d) : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_) { - // Try hard not to call normalize(). Most of the time we will go just - // one level deeper. - // - bool n (true); - - if (d.relative ()) - { - // Relative scopes are opened relative to out, not src. - // - if (d.simple () && !d.current () && !d.parent ()) - { - d = dir_path (p.scope_->out_path ()) /= d.string (); - n = false; - } - else - d = p.scope_->out_path () / d; - } - - if (n) - d.normalize (); + complete_normalize (*p.scope_, d); + e_ = p.switch_scope (d); + } - p.switch_scope (d); + // As above but for already absolute and normalized directory. + // + enter_scope (parser& p, const dir_path& d, bool) + : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_) + { + e_ = p.switch_scope (d); } ~enter_scope () @@ -93,8 +88,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_scope (enter_scope&& x) {*this = move (x);} - enter_scope& operator= (enter_scope&& x) + enter_scope (enter_scope&& x) noexcept {*this = move (x);} + enter_scope& operator= (enter_scope&& x) noexcept { if (this != &x) { @@ -102,6 +97,7 @@ namespace build2 r_ = x.r_; s_ = x.s_; b_ = x.b_; + e_ = move (x.e_); x.p_ = nullptr; } return *this; @@ -110,11 +106,37 @@ namespace build2 enter_scope (const enter_scope&) = delete; enter_scope& operator= (const enter_scope&) = delete; + static void + complete_normalize (scope& s, dir_path& d) + { + // Try hard not to call normalize(). Most of the time we will go just + // one level deeper. + // + bool n (true); + + if (d.relative ()) + { + // Relative scopes are opened relative to out, not src. + // + if (d.simple () && !d.current () && !d.parent ()) + { + d = dir_path (s.out_path ()) /= d.string (); + n = false; + } + else + d = s.out_path () / d; + } + + if (n) + d.normalize (); + } + private: parser* p_; scope* r_; scope* s_; const dir_path* b_; // Pattern base. + auto_project_env e_; }; class parser::enter_target @@ -150,7 +172,11 @@ namespace build2 tracer& tr) { auto r (p.scope_->find_target_type (n, o, loc)); - return p.ctx.targets.insert ( + + if (r.first.factory == nullptr) + p.fail (loc) << "abstract target type " << r.first.name << "{}"; + + return p.ctx->targets.insert ( r.first, // target type move (n.dir), move (o.dir), @@ -170,12 +196,16 @@ namespace build2 tracer& tr) { auto r (p.scope_->find_target_type (n, o, loc)); - return p.ctx.targets.find (r.first, // target type - n.dir, - o.dir, - n.value, - r.second, // extension - tr); + + if (r.first.factory == nullptr) + p.fail (loc) << "abstract target type " << r.first.name << "{}"; + + return p.ctx->targets.find (r.first, // target type + n.dir, + o.dir, + n.value, + r.second, // extension + tr); } ~enter_target () @@ -186,8 +216,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_target (enter_target&& x) {*this = move (x);} - enter_target& operator= (enter_target&& x) { + enter_target (enter_target&& x) noexcept {*this = move (x);} + enter_target& operator= (enter_target&& x) noexcept { p_ = x.p_; t_ = x.t_; x.p_ = nullptr; return *this;} enter_target (const enter_target&) = delete; @@ -218,8 +248,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_prerequisite (enter_prerequisite&& x) {*this = move (x);} - enter_prerequisite& operator= (enter_prerequisite&& x) { + enter_prerequisite (enter_prerequisite&& x) noexcept {*this = move (x);} + enter_prerequisite& operator= (enter_prerequisite&& x) noexcept { p_ = x.p_; r_ = x.r_; x.p_ = nullptr; return *this;} enter_prerequisite (const enter_prerequisite&) = delete; @@ -235,6 +265,7 @@ namespace build2 { pre_parse_ = false; attributes_.clear (); + condition_ = nullopt; default_target_ = nullptr; peeked_ = false; replay_ = replay::stop; @@ -247,10 +278,11 @@ namespace build2 scope* root, scope& base, target* tgt, - prerequisite* prq) + prerequisite* prq, + bool enter) { lexer l (is, in); - parse_buildfile (l, root, base, tgt, prq); + parse_buildfile (l, root, base, tgt, prq, enter); } void parser:: @@ -258,7 +290,8 @@ namespace build2 scope* root, scope& base, target* tgt, - prerequisite* prq) + prerequisite* prq, + bool enter) { path_ = &l.name (); lexer_ = &l; @@ -270,9 +303,16 @@ namespace build2 pbase_ = scope_->src_path_; - if (path_->path != nullptr) - enter_buildfile (*path_->path); // Note: needs scope_. + // Note that root_ may not be a project root (see parse_export_stub()). + // + auto_project_env penv ( + stage_ != stage::boot && root_ != nullptr && root_->root_extra != nullptr + ? auto_project_env (*root_) + : auto_project_env ()); + const buildfile* bf (enter && path_->path != nullptr + ? &enter_buildfile<buildfile> (*path_->path) + : nullptr); token t; type tt; next (t, tt); @@ -284,13 +324,34 @@ namespace build2 else { parse_clause (t, tt); - process_default_target (t); + + if (stage_ != stage::boot && stage_ != stage::root) + process_default_target (t, bf); } if (tt != type::eos) fail (t) << "unexpected " << t; } + names parser:: + parse_export_stub (istream& is, const path_name& name, + const scope& rs, scope& gs, scope& ts) + { + // Enter the export stub manually with correct out. + // + if (name.path != nullptr) + { + dir_path out (!rs.out_eq_src () + ? out_src (name.path->directory (), rs) + : dir_path ()); + + enter_buildfile<buildfile> (*name.path, move (out)); + } + + parse_buildfile (is, name, &gs, ts, nullptr, nullptr, false /* enter */); + return move (export_value); + } + token parser:: parse_variable (lexer& l, scope& s, const variable& var, type kind) { @@ -336,6 +397,81 @@ namespace build2 return make_pair (move (lhs), move (t)); } + names parser:: + parse_names (lexer& l, + const dir_path* b, + pattern_mode pmode, + const char* what, + const string* separators) + { + path_ = &l.name (); + lexer_ = &l; + + root_ = nullptr; + scope_ = nullptr; + target_ = nullptr; + prerequisite_ = nullptr; + + pbase_ = b; + + token t; + type tt; + + mode (lexer_mode::value, '@'); + next (t, tt); + + names r (parse_names (t, tt, pmode, what, separators)); + + if (tt != type::eos) + fail (t) << "unexpected " << t; + + return r; + } + + value parser:: + parse_eval (lexer& l, scope& rs, scope& bs, pattern_mode pmode) + { + path_ = &l.name (); + lexer_ = &l; + + root_ = &rs; + scope_ = &bs; + target_ = nullptr; + prerequisite_ = nullptr; + + pbase_ = scope_->src_path_; + + // Note that root_ may not be a project root. + // + auto_project_env penv ( + stage_ != stage::boot && root_ != nullptr && root_->root_extra != nullptr + ? auto_project_env (*root_) + : auto_project_env ()); + + token t; + type tt; + next (t, tt); + + if (tt != type::lparen) + fail (t) << "expected '(' instead of " << t; + + location loc (get_location (t)); + mode (lexer_mode::eval, '@'); + next_with_attributes (t, tt); + + values vs (parse_eval (t, tt, pmode)); + + if (next (t, tt) != type::eos) + fail (t) << "unexpected " << t; + + switch (vs.size ()) + { + case 0: return value (names ()); + case 1: return move (vs[0]); + default: fail (loc) << "expected single value" << endf; + } + } + bool parser:: parse_clause (token& t, type& tt, bool one) { @@ -477,6 +613,16 @@ namespace build2 { f = &parser::parse_config; } + else if (n == "config.environment") + { + f = &parser::parse_config_environment; + } + else if (n == "recipe") + { + // Valid only after recipe header (%). + // + fail (t) << n << " directive without % recipe header"; + } if (f != nullptr) { @@ -493,9 +639,39 @@ namespace build2 location nloc (get_location (t)); names ns; - if (tt != type::labrace) + // We have to parse names in chunks to detect invalid cases of the + // group{foo}<...> syntax. + // + // Consider (1): + // + // x = + // group{foo} $x<...>: + // + // And (2): + // + // x = group{foo} group{bar} + // $x<...>: + // + // As well as (3): + // + // <...><...>: + // + struct chunk { - ns = parse_names (t, tt, pattern_mode::ignore); + size_t pos; // Index in ns of the beginning of the last chunk. + location loc; // Position of the beginning of the last chunk. + }; + optional<chunk> ns_last; + + bool labrace_first (tt == type::labrace); + if (!labrace_first) + { + do + { + ns_last = chunk {ns.size (), get_location (t)}; + parse_names (t, tt, ns, pattern_mode::preserve, true /* chunk */); + } + while (start_names (tt)); // Allow things like function calls that don't result in anything. // @@ -511,45 +687,89 @@ namespace build2 } } - // Handle ad hoc target group specification (<...>). + // Handle target group specification (<...>). // - // We keep an "optional" (empty) vector of names parallel to ns. + // We keep an "optional" (empty) vector of names parallel to ns that + // contains the group members. Note that when we "catch" gns up to ns, + // we populate it with ad hoc (as opposed to explicit) groups with no + // members. // - adhoc_names ans; + group_names gns; if (tt == type::labrace) { - while (tt == type::labrace) + for (; tt == type::labrace; labrace_first = false) { - // Parse target names inside < >. + // Detect explicit group (group{foo}<...>). + // + // Note that `<` first thing on the line is not seperated thus the + // labrace_first complication. + // + bool expl (!t.separated && !labrace_first); + if (expl) + { + // Note: (N) refers to the example in the above comment. + // + if (!ns_last /* (3) */ || ns_last->pos == ns.size () /* (1) */) + { + fail (t) << "group name or whitespace expected before '<'"; + } + else + { + size_t n (ns.size () - ns_last->pos); + + // Note: could be a pair. + // + if ((n > 2 || (n == 2 && !ns[ns_last->pos].pair)) /* (2) */) + { + fail (t) << "single group name or whitespace expected before " + << "'<' instead of '" + << names_view (ns.data () + ns_last->pos, n) << "'"; + } + } + } + + // Parse target names inside <>. // // We "reserve" the right to have attributes inside <> though what // exactly that would mean is unclear. One potentially useful - // semantics would be the ability to specify attributes for ad hoc - // members though the fact that the primary target is listed first - // would make it rather unintuitive. Maybe attributes that change - // the group semantics itself? + // semantics would be the ability to specify attributes for group + // members though the fact that the primary target for ad hoc groups + // is listed first would make it rather unintuitive. Maybe + // attributes that change the group semantics itself? // next_with_attributes (t, tt); auto at (attributes_push (t, tt)); if (at.first) - fail (at.second) << "attributes before ad hoc target"; + fail (at.second) << "attributes before group member"; else attributes_pop (); - // Allow empty case (<>). + // For explicit groups, the group target is already in ns and all + // the members should go straight to gns. // - if (tt != type::rabrace) + // For ad hoc groups, the first name (or a pair) is the primary + // target which we need to keep in ns. The rest, if any, are ad + // hoc members that we should move to gns. + // + if (expl) { - location aloc (get_location (t)); + gns.resize (ns.size ()); // Catch up with the names vector. + group_names_loc& g (gns.back ()); + g.expl = true; + g.group_loc = move (ns_last->loc); + g.member_loc = get_location (t); // Start of members. + + if (tt != type::rabrace) // Handle empty case (<>) + parse_names (t, tt, g.ns, pattern_mode::preserve); + } + else if (tt != type::rabrace) // Allow and ignore empty case (<>). + { + location mloc (get_location (t)); // Start of members. - // The first name (or a pair) is the primary target which we need - // to keep in ns. The rest, if any, are ad hoc members that we - // should move to ans. - // size_t m (ns.size ()); - parse_names (t, tt, ns, pattern_mode::ignore); + parse_names (t, tt, ns, pattern_mode::preserve); size_t n (ns.size ()); // Another empty case (<$empty>). @@ -564,11 +784,10 @@ namespace build2 { n -= m; // Number of names in ns we should end up with. - ans.resize (n); // Catch up with the names vector. - adhoc_names_loc& a (ans.back ()); - - a.loc = move (aloc); - a.ns.insert (a.ns.end (), + gns.resize (n); // Catch up with the names vector. + group_names_loc& g (gns.back ()); + g.group_loc = g.member_loc = move (mloc); + g.ns.insert (g.ns.end (), make_move_iterator (ns.begin () + n), make_move_iterator (ns.end ())); ns.resize (n); @@ -582,12 +801,16 @@ namespace build2 // Parse the next chunk of target names after >, if any. // next (t, tt); - if (start_names (tt)) - parse_names (t, tt, ns, pattern_mode::ignore); + ns_last = nullopt; // To detect <...><...>. + while (start_names (tt)) + { + ns_last = chunk {ns.size (), get_location (t)}; + parse_names (t, tt, ns, pattern_mode::preserve, true /* chunk */); + } } - if (!ans.empty ()) - ans.resize (ns.size ()); // Catch up with the final chunk. + if (!gns.empty ()) + gns.resize (ns.size ()); // Catch up with the final chunk. if (tt != type::colon) fail (t) << "expected ':' instead of " << t; @@ -606,29 +829,134 @@ namespace build2 if (ns.empty ()) fail (t) << "expected target before ':'"; - if (at.first) - fail (at.second) << "attributes before target"; - else - attributes_pop (); + attributes as (attributes_pop ()); - // Call the specified parsing function (either variable or block) for - // each target. We handle multiple targets by replaying the tokens - // since the value/block may contain variable expansions that would be - // sensitive to the target context in which they are evaluated. The - // function signature is: + // Call the specified parsing function (variable value/block) for + // one/each pattern/target. We handle multiple targets by replaying + // the tokens since the value/block may contain variable expansions + // that would be sensitive to the target context in which they are + // evaluated. The function signature is: // - // void (token& t, type& tt, const target_type* type, string pat) + // void (token& t, type& tt, + // optional<bool> member, // true -- explict, false -- ad hoc + // optional<pattern_type>, const target_type* pat_tt, string pat, + // const location& pat_loc) // - // Note that the target and its ad hoc members are inserted implied + // Note that the target and its group members are inserted implied // but this flag can be cleared and default_target logic applied if // appropriate. // - auto for_each = [this, &trace, &t, &tt, &ns, &nloc, &ans] (auto&& f) + auto for_one_pat = [this, &t, &tt] (auto&& f, + name&& n, + const location& nloc) { + // Reduce the various directory/value combinations to the scope + // directory (if any) and the pattern. Here are more interesting + // examples of patterns: + // + // */ -- */{} + // dir{*} -- dir{*} + // dir{*/} -- */dir{} + // + // foo/*/ -- foo/*/{} + // foo/dir{*/} -- foo/*/dir{} + // + // Note that these are not patterns: + // + // foo*/file{bar} + // foo*/dir{bar/} + // + // While these are: + // + // file{foo*/bar} + // dir{foo*/bar/} + // + // And this is a half-pattern (foo* should no be treated as a + // pattern but that's unfortunately indistinguishable): + // + // foo*/dir{*/} -- foo*/*/dir{} + // + // Note also that none of this applies to regex patterns (see + // the parsing code for details). + // + if (*n.pattern == pattern_type::path) + { + if (n.value.empty () && !n.dir.empty ()) + { + // Note that we use string and not the representation: in a + // sense the trailing slash in the pattern is subsumed by + // the target type. + // + if (n.dir.simple ()) + n.value = move (n.dir).string (); + else + { + n.value = n.dir.leaf ().string (); + n.dir.make_directory (); + } + + // Treat directory as type dir{} similar to other places. + // + if (n.untyped ()) + n.type = "dir"; + } + else + { + // Move the directory part, if any, from value to dir. + // + try + { + n.canonicalize (); + } + catch (const invalid_path& e) + { + fail (nloc) << "invalid path '" << e.path << "'"; + } + catch (const invalid_argument&) + { + fail (nloc) << "invalid pattern '" << n.value << "'"; + } + } + } + + // If we have the directory, then it is the scope. + // + enter_scope sg; + if (!n.dir.empty ()) + { + if (path_pattern (n.dir)) + fail (nloc) << "pattern in directory " << n.dir.representation (); + + sg = enter_scope (*this, move (n.dir)); + } + + // Resolve target type. If none is specified, then it's file{}. + // + // Note: abstract target type is ok here. + // + const target_type* ttype (n.untyped () + ? &file::static_type + : scope_->find_target_type (n.type)); + + if (ttype == nullptr) + fail (nloc) << "unknown target type " << n.type << + info << "perhaps the module that defines this target type is " + << "not loaded by project " << *scope_->root_scope (); + + f (t, tt, nullopt, n.pattern, ttype, move (n.value), nloc); + }; + + auto for_each = [this, &trace, &for_one_pat, + &t, &tt, &as, &ns, &nloc, &gns] (auto&& f) + { + // We need replay if we have multiple targets or group members. + // // Note: watch out for an out-qualified single target (two names). // replay_guard rg (*this, - ns.size () > 2 || (ns.size () == 2 && !ns[0].pair)); + ns.size () > 2 || + (ns.size () == 2 && !ns[0].pair) || + !gns.empty ()); for (size_t i (0), e (ns.size ()); i != e; ) { @@ -640,57 +968,67 @@ namespace build2 // Figure out if this is a target or a target type/pattern (yeah, // it can be a mixture). // - if (path_pattern (n.value)) + if (n.pattern) { + if (!as.empty ()) + fail (as.loc) << "attributes before target type/pattern"; + if (n.pair) fail (nloc) << "out-qualified target type/pattern"; - if (!ans.empty () && !ans[i].ns.empty ()) - fail (ans[i].loc) << "ad hoc member in target type/pattern"; + if (!gns.empty () && !gns[i].ns.empty ()) + fail (gns[i].member_loc) + << "group member in target type/pattern"; - // If we have the directory, then it is the scope. - // - enter_scope sg; - if (!n.dir.empty ()) - sg = enter_scope (*this, move (n.dir)); + if (*n.pattern == pattern_type::regex_substitution) + fail (nloc) << "regex substitution " << n << " without " + << "regex pattern"; - // Resolve target type. If none is specified or if it is '*', - // use the root of the hierarchy. So these are all equivalent: - // - // *: foo = bar - // {*}: foo = bar - // *{*}: foo = bar - // - const target_type* ti ( - n.untyped () || n.type == "*" - ? &target::static_type - : scope_->find_target_type (n.type)); - - if (ti == nullptr) - fail (nloc) << "unknown target type " << n.type; - - f (t, tt, ti, move (n.value)); + for_one_pat (forward<decltype (f)> (f), move (n), nloc); } else { - name o (n.pair ? move (ns[++i]) : name ()); - enter_target tg (*this, - move (n), - move (o), - true /* implied */, - nloc, - trace); - - // Enter ad hoc members. - // - if (!ans.empty ()) + bool expl; + vector<reference_wrapper<target>> gms; { - // Note: index after the pair increment. + name o (n.pair ? move (ns[++i]) : name ()); + enter_target tg (*this, + move (n), + move (o), + true /* implied */, + nloc, + trace); + + if (!as.empty ()) + apply_target_attributes (*target_, as); + + // Enter group members. // - enter_adhoc_members (move (ans[i]), true /* implied */); + if (!gns.empty ()) + { + // Note: index after the pair increment. + // + group_names_loc& g (gns[i]); + expl = g.expl; + + if (expl && !target_->is_a<group> ()) + fail (g.group_loc) << *target_ << " is not group target"; + + gms = expl + ? enter_explicit_members (move (g), true /* implied */) + : enter_adhoc_members (move (g), true /* implied */); + } + + f (t, tt, nullopt, nullopt, nullptr, string (), location ()); } - f (t, tt, nullptr, string ()); + for (target& gm: gms) + { + rg.play (); // Replay. + + enter_target tg (*this, gm); + f (t, tt, expl, nullopt, nullptr, string (), location ()); + } } if (++i != e) @@ -700,6 +1038,473 @@ namespace build2 next_with_attributes (t, tt); // Recognize attributes after `:`. + // See if this could be an ad hoc pattern rule. It's a pattern rule if + // the primary target is a pattern and it has (1) prerequisites and/or + // (2) recipes. Only one primary target per pattern rule declaration + // is allowed. + // + // Note, however, that what looks like a pattern may turn out to be + // just a pattern-specific variable assignment or variable block, + // which both can appear with multiple targets/patterns on the left + // hand side, or even a mixture of them. Still, instead of trying to + // weave the pattern rule logic into the already hairy code below, we + // are going to handle it separately and deal with the "degenerate" + // cases (variable assignment/block) both here and below. + // + if (ns[0].pattern && ns.size () == (ns[0].pair ? 2 : 1)) + { + name& n (ns[0]); + + if (n.qualified ()) + fail (nloc) << "project name in target pattern " << n; + + if (n.pair) + fail (nloc) << "out-qualified target pattern"; + + if (*n.pattern == pattern_type::regex_substitution) + fail (nloc) << "regex substitution " << n << " without " + << "regex pattern"; + + // Parse prerequisites, if any. + // + location ploc; + names pns; + if (tt != type::newline) + { + auto at (attributes_push (t, tt)); + + if (!start_names (tt)) + fail (t) << "unexpected " << t; + + // Note that unlike below, here we preserve the pattern in the + // prerequisites. + // + ploc = get_location (t); + pns = parse_names (t, tt, pattern_mode::preserve); + + // Target type/pattern-specific variable assignment. + // + if (tt == type::assign || tt == type::prepend || tt == type::append) + { + // Note: ns contains single target name. + // + if (!gns.empty ()) + fail (gns[0].member_loc) + << "group member in target type/pattern"; + + // Note: see the same code below if changing anything here. + // + type akind (tt); + const location aloc (get_location (t)); + + const variable& var (parse_variable_name (move (pns), ploc)); + apply_variable_attributes (var); + + if (var.visibility > variable_visibility::target) + { + fail (nloc) << "variable " << var << " has " << var.visibility + << " visibility but is assigned on a target"; + } + + for_one_pat ( + [this, &var, akind, &aloc] ( + token& t, type& tt, + optional<bool>, + optional<pattern_type> pt, const target_type* ptt, + string pat, const location& ploc) + { + + parse_type_pattern_variable (t, tt, + *pt, *ptt, move (pat), ploc, + var, akind, aloc); + }, + move (n), + nloc); + + next_after_newline (t, tt); + + if (!as.empty ()) + fail (as.loc) << "attributes before target type/pattern"; + + continue; // Just a target type/pattern-specific var assignment. + } + + if (at.first) + fail (at.second) << "attributes before prerequisite pattern"; + else + attributes_pop (); + + // @@ TODO + // + if (tt == type::colon) + fail (t) << "prerequisite type/pattern-specific variables " + << "not yet supported"; + } + + // Next we may have a target type/pattern specific variable block + // potentially followed by recipes. + // + next_after_newline (t, tt); + if (tt == type::lcbrace && peek () == type::newline) + { + // Note: see the same code below if changing anything here. + // + next (t, tt); // Newline. + next (t, tt); // First token inside the variable block. + + for_one_pat ( + [this] ( + token& t, type& tt, + optional<bool>, + optional<pattern_type> pt, const target_type* ptt, + string pat, const location& ploc) + { + parse_variable_block (t, tt, pt, ptt, move (pat), ploc); + }, + name (n), // Note: can't move (could still be a rule). + nloc); + + if (tt != type::rcbrace) + fail (t) << "expected '}' instead of " << t; + + next (t, tt); // Newline. + next_after_newline (t, tt, '}'); // Should be on its own line. + + // See if this is just a target type/pattern-specific var block. + // + if (pns.empty () && + tt != type::percent && tt != type::multi_lcbrace) + { + // Note: ns contains single target name. + // + if (!gns.empty ()) + fail (gns[0].member_loc) + << "group member in target type/pattern"; + + if (!as.empty ()) + fail (as.loc) << "attributes before target type/pattern"; + + continue; + } + } + + // Ok, this is an ad hoc pattern rule. + // + // First process the attributes. + // + string rn; + { + const location& l (as.loc); + + for (auto& a: as) + { + const string& n (a.name); + value& v (a.value); + + // rule_name= + // + if (n == "rule_name") + { + try + { + rn = convert<string> (move (v)); + + if (rn.empty ()) + throw invalid_argument ("empty name"); + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else + fail (l) << "unknown ad hoc pattern rule attribute " << a; + } + } + + // What should we do if we have neither prerequisites nor recipes? + // While such a declaration doesn't make much sense, it can happen, + // for example, with an empty variable expansion: + // + // file{*.txt}: $extra + // + // So let's silently ignore it. + // + if (pns.empty () && tt != type::percent && tt != type::multi_lcbrace) + continue; + + // Process and verify the pattern. + // + pattern_type pt (*n.pattern); + optional<pattern_type> st; + const char* pn; + + switch (pt) + { + case pattern_type::path: + pn = "path"; + break; + case pattern_type::regex_pattern: + pn = "regex"; + st = pattern_type::regex_substitution; + break; + case pattern_type::regex_substitution: + // Unreachable. + break; + } + + // Make sure patterns have no directory components. While we may + // decide to support this in the future, currently the appropriate + // semantics is not immediately obvious. Whatever we decide, it + // should be consistent with the target type/pattern-specific + // variables where it is interpreted as a scope (and which doesn't + // feel like the best option for pattern rules). See also depdb + // dyndep --update-* patterns. + // + auto check_pattern = [this] (name& n, const location& loc) + { + try + { + // Move the directory component for path patterns. + // + if (*n.pattern == pattern_type::path) + n.canonicalize (); + + if (n.dir.empty ()) + return; + } + catch (const invalid_path&) + { + // Fall through. + } + + fail (loc) << "directory in pattern " << n; + }; + + check_pattern (n, nloc); + + // If we have group members, verify all the members are patterns or + // substitutions (ad hoc) or subsitutions (explicit) and of the + // correct pattern type. A rule for an explicit group that wishes to + // match based on some of its members feels far fetched. + // + // For explicit groups the use-case is to inject static members + // which could otherwise be tedious to specify for each group. + // + const location& mloc (gns.empty () ? location () : gns[0].member_loc); + names ns (gns.empty () ? names () : move (gns[0].ns)); + bool expl (gns.empty () ? false : gns[0].expl); + + for (name& n: ns) + { + if (!n.pattern || !(*n.pattern == pt || (st && *n.pattern == *st))) + { + fail (mloc) << "expected " << pn << " pattern or substitution " + << "instead of " << n; + } + + if (*n.pattern != pattern_type::regex_substitution) + { + if (expl) + fail (mloc) << "explicit group member pattern " << n; + + check_pattern (n, mloc); + } + } + + // The same for prerequisites except here we can have non-patterns. + // + for (name& n: pns) + { + if (n.pattern) + { + if (!(*n.pattern == pt || (st && *n.pattern == *st))) + { + fail (ploc) << "expected " << pn << " pattern or substitution " + << "instead of " << n; + } + + if (*n.pattern != pattern_type::regex_substitution) + check_pattern (n, ploc); + } + } + + // Derive the rule name unless specified explicitly. It must be + // unique in this scope. + // + // It would have been nice to include the location but unless we + // include the absolute path to the buildfile (which would be + // unwieldy), it could be ambigous. + // + // NOTE: we rely on the <...> format in dump. + // + if (rn.empty ()) + rn = "<ad hoc pattern rule #" + + to_string (scope_->adhoc_rules.size () + 1) + '>'; + + auto& ars (scope_->adhoc_rules); + + auto i (find_if (ars.begin (), ars.end (), + [&rn] (const unique_ptr<adhoc_rule_pattern>& rp) + { + return rp->rule_name == rn; + })); + + const target_type* ttype (nullptr); + if (i != ars.end ()) + { + // @@ TODO: append ad hoc members, prereqs (we now have + // [rule_name=] which we can use to reference the same + // rule). + // + ttype = &(*i)->type; + assert (false); + } + else + { + // Resolve target type (same as in for_one_pat()). + // + ttype = n.untyped () + ? &file::static_type + : scope_->find_target_type (n.type); + + if (ttype == nullptr) + fail (nloc) << "unknown target type " << n.type << + info << "perhaps the module that defines this target type is " + << "not loaded by project " << *scope_->root_scope (); + + if (!gns.empty ()) + { + if (ttype->is_a<group> () != expl) + fail (nloc) << "group type and target type mismatch"; + } + + unique_ptr<adhoc_rule_pattern> rp; + switch (pt) + { + case pattern_type::path: + // @@ TODO + fail (nloc) << "path pattern rules not yet supported"; + break; + case pattern_type::regex_pattern: + rp.reset (new adhoc_rule_regex_pattern ( + *scope_, rn, *ttype, + move (n), nloc, + move (ns), mloc, + move (pns), ploc)); + break; + case pattern_type::regex_substitution: + // Unreachable. + break; + } + + ars.push_back (move (rp)); + i = --ars.end (); + } + + adhoc_rule_pattern& rp (**i); + + // Parse the recipe chain if any. + // + if (tt == type::percent || tt == type::multi_lcbrace) + { + small_vector<shared_ptr<adhoc_rule>, 1> recipes; + parse_recipe (t, tt, token (t), recipes, ttype, rn); + + for (shared_ptr<adhoc_rule>& pr: recipes) + { + // Can be NULL if the recipe is disabled with a condition. + // + if (pr != nullptr) + { + pr->pattern = &rp; // Connect recipe to pattern. + rp.rules.push_back (move (pr)); + } + } + + // Register this adhoc rule for all its actions. + // + for (shared_ptr<adhoc_rule>& pr: rp.rules) + { + adhoc_rule& r (*pr); + + for (action a: r.actions) + { + // This covers both duplicate recipe actions within the rule + // pattern (similar to parse_recipe()) as well as conflicts + // with other rules (ad hoc or not). + // + if (!scope_->rules.insert (a, *ttype, rp.rule_name, r)) + { + const meta_operation_info* mf ( + root_->root_extra->meta_operations[a.meta_operation ()]); + + const operation_info* of ( + root_->root_extra->operations[a.operation ()]); + + fail (r.loc) + << "duplicate " << mf->name << '(' << of->name << ") rule " + << rp.rule_name << " for target type " << ttype->name + << "{}"; + } + + // We also register for a wildcard operation in order to get + // called to provide the reverse operation fallback (see + // match_impl() for the gory details). + // + // Note that we may end up trying to insert a duplicate of the + // same rule (e.g., for the same meta-operation). Feels like + // we should never try to insert for a different rule since + // for ad hoc rules names are unique. + // + scope_->rules.insert ( + a.meta_operation (), 0, + *ttype, rp.rule_name, rp.fallback_rule_); + + // We also register for the dist meta-operation in order to + // inject additional prerequisites which may "pull" additional + // sources into the distribution. Unless there is an explicit + // recipe for dist. + // + // And the same for the configure meta-operation to, for + // example, make sure a hinted ad hoc rule matches. @@ Hm, + // maybe we fixed this with action-specific hints? But the + // injection part above may still apply. BTW, this is also + // required for see-through groups in order to resolve their + // member. + // + // Note also that the equivalent semantics for ad hoc recipes + // is provided by match_adhoc_recipe(). + // + if (a.meta_operation () == perform_id) + { + auto reg = [this, ttype, &rp, &r] (action ea) + { + for (shared_ptr<adhoc_rule>& pr: rp.rules) + for (action a: pr->actions) + if (ea == a) + return; + + scope_->rules.insert (ea, *ttype, rp.rule_name, r); + }; + + reg (action (dist_id, a.operation ())); + reg (action (configure_id, a.operation ())); + } + + // @@ TODO: if this rule does dynamic member discovery of a + // see-through target group, then we may also need to + // register update for other meta-operations (see, for + // example, wildcard update registration in the cli + // module). BTW, we can now detect such a target via + // its target type flags. + } + } + } + + continue; + } + if (tt == type::newline) { // See if this is a target-specific variable and/or recipe block(s). @@ -717,11 +1522,6 @@ namespace build2 // x = y x = y // } } // - // @@ This might change a bit once we support ad hoc rules (where we - // may have prerequisites for a pattern; but perhaps this should be - // handled separately since the parse_dependency() is already too - // complex and there will be no chains in this case). - // next (t, tt); if (tt == type::percent || tt == type::multi_lcbrace || @@ -741,7 +1541,9 @@ namespace build2 st = token (t), // Save start token (will be gone on replay). recipes = small_vector<shared_ptr<adhoc_rule>, 1> ()] (token& t, type& tt, - const target_type* type, string pat) mutable + optional<bool> gm, // true -- explicit, false -- ad hoc + optional<pattern_type> pt, const target_type* ptt, string pat, + const location& ploc) mutable { token rt; // Recipe start token. @@ -749,9 +1551,18 @@ namespace build2 // if (st.type == type::lcbrace) { + // Note: see the same code above if changing anything here. + // next (t, tt); // Newline. next (t, tt); // First token inside the variable block. - parse_variable_block (t, tt, type, move (pat)); + + // For explicit groups we only assign variables on the group + // omitting the members. + // + if (!gm || !*gm) + parse_variable_block (t, tt, pt, ptt, move (pat), ploc); + else + skip_block (t, tt); if (tt != type::rcbrace) fail (t) << "expected '}' instead of " << t; @@ -767,8 +1578,20 @@ namespace build2 else rt = st; - if (type != nullptr) - fail (rt) << "recipe in target type/pattern"; + // If this is a group member then we know we are replaying and + // can skip the recipe. + // + if (gm) + { + replay_skip (); + next (t, tt); + return; + } + + if (pt) + fail (rt) << "unexpected recipe after target type/pattern" << + info << "ad hoc pattern rule may not be combined with other " + << "targets or patterns"; parse_recipe (t, tt, rt, recipes); }; @@ -785,7 +1608,7 @@ namespace build2 // Note also that we treat this as an explicit dependency // declaration (i.e., not implied). // - enter_targets (move (ns), nloc, move (ans), 0); + enter_targets (move (ns), nloc, move (gns), 0, as); } continue; @@ -800,7 +1623,8 @@ namespace build2 if (!start_names (tt)) fail (t) << "unexpected " << t; - // @@ PAT: currently we pattern-expand target-specific vars. + // @@ PAT: currently we pattern-expand target-specific var names (see + // also parse_import()). // const location ploc (get_location (t)); names pns (parse_names (t, tt, pattern_mode::expand)); @@ -813,6 +1637,8 @@ namespace build2 // if (tt == type::assign || tt == type::prepend || tt == type::append) { + // Note: see the same code above if changing anything here. + // type akind (tt); const location aloc (get_location (t)); @@ -830,17 +1656,31 @@ namespace build2 // Parse the assignment for each target. // - for_each ([this, &var, akind, &aloc] (token& t, type& tt, - const target_type* type, - string pat) - { - if (type == nullptr) - parse_variable (t, tt, var, akind); - else - parse_type_pattern_variable (t, tt, - *type, move (pat), - var, akind, aloc); - }); + for_each ( + [this, &var, akind, &aloc] ( + token& t, type& tt, + optional<bool> gm, + optional<pattern_type> pt, const target_type* ptt, string pat, + const location& ploc) + { + if (pt) + parse_type_pattern_variable (t, tt, + *pt, *ptt, move (pat), ploc, + var, akind, aloc); + else + { + // Skip explicit group members (see the block case above for + // background). + // + if (!gm || !*gm) + parse_variable (t, tt, var, akind); + else + { + next (t, tt); + skip_line (t, tt); + } + } + }); next_after_newline (t, tt); } @@ -857,8 +1697,9 @@ namespace build2 parse_dependency (t, tt, move (ns), nloc, - move (ans), - move (pns), ploc); + move (gns), + move (pns), ploc, + as); } continue; @@ -870,6 +1711,7 @@ namespace build2 // // x = y // foo/ x = y (ns will have two elements) + // foo/x = y (ns will have one element) // // And in the future we may also want to support: // @@ -894,19 +1736,33 @@ namespace build2 // let parse_variable_name() complain. // dir_path d; - if (ns.size () == 2 && ns[0].directory ()) + size_t p; + if ((ns.size () == 2 && ns[0].directory ()) || + (ns.size () == 1 && ns[0].simple () && + (p = path_traits::rfind_separator (ns[0].value)) != string::npos)) { if (at.first) fail (at.second) << "attributes before scope directory"; - d = move (ns[0].dir); - ns.erase (ns.begin ()); - // Make sure it's not a pattern (see also the target case above and // scope below). // - if (path_pattern (d)) - fail (nloc) << "pattern in directory " << d.representation (); + if (ns[0].pattern) + fail (nloc) << "pattern in " << ns[0]; + + if (ns.size () == 2) + { + d = move (ns[0].dir); + ns.erase (ns.begin ()); + } + else + { + // Note that p cannot point to the last character since then it + // would have been a directory, not a simple name. + // + d = dir_path (ns[0].value, 0, p + 1); + ns[0].value.erase (0, p + 1); + } } const variable& var (parse_variable_name (move (ns), nloc)); @@ -944,13 +1800,11 @@ namespace build2 if (next (t, tt) == type::lcbrace && peek () == type::newline) { - dir_path&& d (move (ns[0].dir)); - // Make sure not a pattern (see also the target and directory cases // above). // - if (path_pattern (d)) - fail (nloc) << "pattern in directory " << d.representation (); + if (ns[0].pattern) + fail (nloc) << "pattern in " << ns[0]; next (t, tt); // Newline. next (t, tt); // First token inside the block. @@ -963,7 +1817,7 @@ namespace build2 // Can contain anything that a top level can. // { - enter_scope sg (*this, move (d)); + enter_scope sg (*this, move (ns[0].dir)); parse_clause (t, tt); } @@ -1006,7 +1860,8 @@ namespace build2 void parser:: parse_variable_block (token& t, type& tt, - const target_type* type, string pat) + optional<pattern_type> pt, const target_type* ptt, + string pat, const location& ploc) { // Parse a target or prerequisite-specific variable block. If type is not // NULL, then this is a target type/pattern-specific block. @@ -1023,8 +1878,11 @@ namespace build2 { attributes_push (t, tt); + // Variable names should not contain patterns so we preserve them here + // and diagnose in parse_variable_name(). + // location nloc (get_location (t)); - names ns (parse_names (t, tt, pattern_mode::ignore, "variable name")); + names ns (parse_names (t, tt, pattern_mode::preserve, "variable name")); if (tt != type::assign && tt != type::prepend && @@ -1041,12 +1899,12 @@ namespace build2 << " visibility but is assigned on a target"; } - if (type == nullptr) - parse_variable (t, tt, var, tt); - else + if (pt) parse_type_pattern_variable (t, tt, - *type, pat, // Note: can't move. + *pt, *ptt, pat, ploc, // Note: can't move. var, tt, get_location (t)); + else + parse_variable (t, tt, var, tt); if (tt != type::newline) fail (t) << "expected newline instead of " << t; @@ -1058,12 +1916,14 @@ namespace build2 void parser:: parse_recipe (token& t, type& tt, const token& start, - small_vector<shared_ptr<adhoc_rule>, 1>& recipes) + small_vector<shared_ptr<adhoc_rule>, 1>& recipes, + const target_type* ttype, + const string& name) { // Parse a recipe chain. // // % [<attrs>] [<buildspec>] - // [if|switch ...] + // [if|if!|switch|recipe ...] // {{ [<lang> ...] // ... // }} @@ -1071,19 +1931,42 @@ namespace build2 // // enter: start is percent or openining multi-curly-brace // leave: token past newline after last closing multi-curly-brace + // + // If target_ is not NULL, then add the recipe to its adhoc_recipes. + // Otherwise, return it in recipes (used for pattern rules). if (stage_ == stage::boot) fail (t) << "ad hoc recipe specified during bootstrap"; // If we have a recipe, the target is not implied. // - if (target_->decl != target_decl::real) + if (target_ != nullptr) { - for (target* m (target_); m != nullptr; m = m->adhoc_member) - m->decl = target_decl::real; + // @@ What if some members are added later? + // + // @@ Also, what happends if redeclared as real dependency, do we + // upgrade the members? + // + if (target_->decl != target_decl::real) + { + target_->decl = target_decl::real; - if (default_target_ == nullptr) - default_target_ = target_; + if (group* g = target_->is_a<group> ()) + { + for (const target& m: g->static_members) + const_cast<target&> (m).decl = target_decl::real; // During load. + } + else + { + for (target* m (target_->adhoc_member); + m != nullptr; + m = m->adhoc_member) + m->decl = target_decl::real; + } + + if (default_target_ == nullptr) + default_target_ = target_; + } } bool first (replay_ != replay::play); // First target. @@ -1092,7 +1975,15 @@ namespace build2 t = start; tt = t.type; for (size_t i (0); tt == type::percent || tt == type::multi_lcbrace; ++i) { - recipes.push_back (nullptr); // For missing else/default (see below). + // For missing else/default (see below). + // + // Note that it may remain NULL if we have, say, an if-condition that + // evaluates to false and no else. While it may be tempting to get rid + // of such "holes", it's not easy due to the replay semantics (see the + // target_ != nullptr block below). So we expect the caller to be + // prepared to handle this. + // + recipes.push_back (nullptr); attributes as; buildspec bs; @@ -1100,6 +1991,8 @@ namespace build2 struct data { + const target_type* ttype; + const string& name; small_vector<shared_ptr<adhoc_rule>, 1>& recipes; bool first; bool& clean; @@ -1107,7 +2000,131 @@ namespace build2 attributes& as; buildspec& bs; const location& bsloc; - } d {recipes, first, clean, i, as, bs, bsloc}; + function<void (string&&)> parse_trailer; + } d {ttype, name, recipes, first, clean, i, as, bs, bsloc, {}}; + + d.parse_trailer = [this, &d] (string&& text) + { + if (d.first) + { + adhoc_rule& ar (*d.recipes.back ()); + + // Translate each buildspec entry into action and add it to the + // recipe entry. + // + const location& l (d.bsloc); + + for (metaopspec& m: d.bs) + { + meta_operation_id mi (ctx->meta_operation_table.find (m.name)); + + if (mi == 0) + fail (l) << "unknown meta-operation " << m.name; + + const meta_operation_info* mf ( + root_->root_extra->meta_operations[mi]); + + if (mf == nullptr) + fail (l) << "project " << *root_ << " does not support meta-" + << "operation " << ctx->meta_operation_table[mi].name; + + for (opspec& o: m) + { + operation_id oi; + if (o.name.empty ()) + { + if (mf->operation_pre == nullptr) + oi = update_id; + else + // Calling operation_pre() to translate doesn't feel + // appropriate here. + // + fail (l) << "default operation in recipe action" << endf; + } + else + oi = ctx->operation_table.find (o.name); + + if (oi == 0) + fail (l) << "unknown operation " << o.name; + + const operation_info* of (root_->root_extra->operations[oi]); + + if (of == nullptr) + fail (l) << "project " << *root_ << " does not support " + << "operation " << ctx->operation_table[oi]; + + // Note: for now always inner (see match_rule_impl() for + // details). + // + action a (mi, oi); + + // Check for duplicates (local). + // + if (find_if ( + d.recipes.begin (), d.recipes.end (), + [a] (const shared_ptr<adhoc_rule>& r) + { + auto& as (r->actions); + return find (as.begin (), as.end (), a) != as.end (); + }) != d.recipes.end ()) + { + fail (l) << "duplicate " << mf->name << '(' << of->name + << ") recipe"; + } + + ar.actions.push_back (a); + } + } + + // Set the recipe text. + // + if (ar.recipe_text ( + *scope_, + d.ttype != nullptr ? *d.ttype : target_->type (), + move (text), + d.as)) + d.clean = true; + + // Verify we have no unhandled attributes. + // + for (attribute& a: d.as) + fail (d.as.loc) << "unknown recipe attribute " << a << endf; + } + + // Copy the recipe over to the target verifying there are no + // duplicates (global). + // + if (target_ != nullptr) + { + const shared_ptr<adhoc_rule>& r (d.recipes[d.i]); + + for (const shared_ptr<adhoc_rule>& er: target_->adhoc_recipes) + { + auto& as (er->actions); + + for (action a: r->actions) + { + if (find (as.begin (), as.end (), a) != as.end ()) + { + const meta_operation_info* mf ( + root_->root_extra->meta_operations[a.meta_operation ()]); + + const operation_info* of ( + root_->root_extra->operations[a.operation ()]); + + fail (d.bsloc) + << "duplicate " << mf->name << '(' << of->name + << ") recipe for target " << *target_; + } + } + } + + target_->adhoc_recipes.push_back (r); + + // Note that "registration" of configure_* and dist_* actions + // (similar to ad hoc rules) is provided by match_adhoc_recipe(). + } + }; // Note that this function must be called at most once per iteration. // @@ -1134,7 +2151,6 @@ namespace build2 else fail (t) << "expected recipe language instead of " << t; - shared_ptr<adhoc_rule> ar; if (!skip) { if (d.first) @@ -1146,11 +2162,20 @@ namespace build2 // location loc (get_location (st)); - if (!lang) + // @@ We could add an attribute (name= or recipe_name=) to allow + // the user specify a friendly name for diagnostics, similar + // to rule_name. + + shared_ptr<adhoc_rule> ar; + if (!lang || icasecmp (*lang, "buildscript") == 0) { // Buildscript // - ar.reset (new adhoc_buildscript_rule (loc, st.value.size ())); + ar.reset ( + new adhoc_buildscript_rule ( + d.name.empty () ? "<ad hoc buildscript recipe>" : d.name, + loc, + st.value.size ())); } else if (icasecmp (*lang, "c++") == 0) { @@ -1200,20 +2225,23 @@ namespace build2 } ar.reset ( - new adhoc_cxx_rule (loc, st.value.size (), ver, move (sep))); + new adhoc_cxx_rule ( + d.name.empty () ? "<ad hoc c++ recipe>" : d.name, + loc, + st.value.size (), + ver, + move (sep))); } else fail (lloc) << "unknown recipe language '" << *lang << "'"; assert (d.recipes[d.i] == nullptr); - d.recipes[d.i] = ar; + d.recipes[d.i] = move (ar); } else { skip_line (t, tt); - assert (d.recipes[d.i] != nullptr); - ar = d.recipes[d.i]; } } else @@ -1235,98 +2263,204 @@ namespace build2 } if (!skip) + d.parse_trailer (move (t.value)); + + next (t, tt); + assert (tt == type::multi_rcbrace); + + next (t, tt); // Newline. + next_after_newline (t, tt, token (t)); // Should be on its own line. + }; + + auto parse_recipe_directive = [this, &d] (token& t, type& tt, + const string&) + { + // Parse recipe directive: + // + // recipe <lang> <file> + // + // Note that here <lang> is not optional. + // + // @@ We could guess <lang> from the extension. + + // Use value mode to minimize the number of special characters. + // + mode (lexer_mode::value, '@'); + + // Parse <lang>. + // + if (next (t, tt) != type::word) + fail (t) << "expected recipe language instead of " << t; + + location lloc (get_location (t)); + string lang (t.value); + next (t, tt); + + // Parse <file> as names to get variable expansion, etc. + // + location nloc (get_location (t)); + names ns (parse_names (t, tt, pattern_mode::ignore, "file name")); + + path file; + try + { + file = convert<path> (move (ns)); + } + catch (const invalid_argument& e) { - auto& ars (target_->adhoc_recipes); - ars.push_back (adhoc_recipe {{}, move (ar)}); + fail (nloc) << "invalid recipe file path: " << e; + } - // Translate each buildspec entry into action and add it into the - // target's ad hoc recipes entry. + string text; + if (d.first) + { + // Source relative to the buildfile rather than src scope. In + // particular, this make sourcing from exported buildfiles work. // - const location& l (d.bsloc); - - for (metaopspec& m: d.bs) + if (file.relative () && path_->path != nullptr) { - meta_operation_id mi (ctx.meta_operation_table.find (m.name)); + // Note: all sourced/included/imported paths are absolute and + // normalized. + // + file = path_->path->directory () / file; + } - if (mi == 0) - fail (l) << "unknown meta-operation " << m.name; + file.normalize (); - const meta_operation_info* mf ( - root_->root_extra->meta_operations[mi]); + try + { + ifdstream ifs (file); + text = ifs.read_text (); + } + catch (const io_error& e) + { + fail (nloc) << "unable to read recipe file " << file << ": " << e; + } - if (mf == nullptr) - fail (l) << "target " << *target_ << " does not support meta-" - << "operation " << ctx.meta_operation_table[mi].name; + shared_ptr<adhoc_rule> ar; + { + // This is expected to be the location of the opening multi-curly + // with the recipe body starting from the following line. So we + // need to fudge the line number a bit. + // + location loc (file, 0, 1); - for (opspec& o: m) + if (icasecmp (lang, "buildscript") == 0) { - operation_id oi; - if (o.name.empty ()) + // Buildscript + // + ar.reset ( + new adhoc_buildscript_rule ( + d.name.empty () ? "<ad hoc buildscript recipe>" : d.name, + loc, + 2)); // Use `{{` and `}}` for dump. + + // Enter as buildfile-like so that it gets automatically + // distributed. Note: must be consistent with build/export/ + // handling in process_default_target(). + // + enter_buildfile<buildscript> (file); + } + else if (icasecmp (lang, "c++") == 0) + { + // C++ + // + // We expect to find a C++ comment line with version and + // optional fragment separator before the first non-comment, + // non-blank line: + // + // // c++ <ver> [<sep>] + // + string s; + location sloc (file, 1, 1); { - if (mf->operation_pre == nullptr) - oi = update_id; - else - // Calling operation_pre() to translate doesn't feel - // appropriate here. - // - fail (l) << "default operation in recipe action" << endf; - } - else - oi = ctx.operation_table.find (o.name); - - if (oi == 0) - fail (l) << "unknown operation " << o.name; + // Note: observe blank lines for accurate line count. + // + size_t b (0), e (0); + for (size_t m (0), n (text.size ()); + next_word (text, n, b, e, m, '\n', '\r'), b != n; + sloc.line++) + { + s.assign (text, b, e - b); - const operation_info* of (root_->root_extra->operations[oi]); + if (!trim (s).empty ()) + { + if (icasecmp (s, "// c++ ", 7) == 0) + break; - if (of == nullptr) - fail (l) << "target " << *target_ << " does not support " - << "operation " << ctx.operation_table[oi]; + if (s[0] != '/' || s[1] != '/') + { + b = e; + break; + } + } + } - // Note: for now always inner (see match_rule() for details). - // - action a (mi, oi); + if (b == e) + fail (sloc) << "no '// c++ <version> [<separator>]' line"; + } - // Check for duplicates. - // - if (find_if ( - ars.begin (), ars.end (), - [a] (const adhoc_recipe& r) - { - auto& as (r.actions); - return find (as.begin (), as.end (), a) != as.end (); - }) != ars.end ()) + uint64_t ver; + optional<string> sep; { - fail (l) << "duplicate recipe for " << mf->name << '(' - << of->name << ')'; - } + size_t b (7), e (7); + if (next_word (s, b, e, ' ', '\t') == 0) + fail (sloc) << "missing c++ recipe version" << endf; - ars.back ().actions.push_back (a); - } - } + try + { + ver = convert<uint64_t> (build2::name (string (s, b, e - b))); + } + catch (const invalid_argument& e) + { + fail (sloc) << "invalid c++ recipe version: " << e << endf; + } - if (d.first) - { - adhoc_recipe& ar (ars.back ()); + if (next_word (s, b, e, ' ', '\t') != 0) + { + sep = string (s, b, e - b); - if (ar.rule->recipe_text ( - ctx, *target_, ar.actions, move (t.value), d.as)) - d.clean = true; + if (next_word (s, b, e, ' ', '\t') != 0) + fail (sloc) << "junk after fragment separator"; + } + } - // Verify we have no unhandled attributes. - // - for (attribute& a: d.as) - fail (d.as.loc) << "unknown recipe attribute " << a << endf; + ar.reset ( + new adhoc_cxx_rule ( + d.name.empty () ? "<ad hoc c++ recipe>" : d.name, + loc, + 2, // Use `{{` and `}}` for dump. + ver, + move (sep))); + + // Enter as buildfile-like so that it gets automatically + // distributed. Note: must be consistent with build/export/ + // handling in process_default_target(). + // + // While ideally we would want to use the cxx{} target type, + // it's defined in a seperate build system module (which may not + // even be loaded by this project, so even runtime lookup won't + // work). So we use file{} instead. + // + enter_buildfile<build2::file> (file); + } + else + fail (lloc) << "unknown recipe language '" << lang << "'"; } + + assert (d.recipes[d.i] == nullptr); + d.recipes[d.i] = move (ar); } + else + assert (d.recipes[d.i] != nullptr); - next (t, tt); - assert (tt == type::multi_rcbrace); + d.parse_trailer (move (text)); - next (t, tt); // Newline. - next_after_newline (t, tt, token (t)); // Should be on its own line. + next_after_newline (t, tt); }; + bsloc = get_location (t); // Fallback location. + if (tt == type::percent) { // Similar code to parse_buildspec() except here we recognize @@ -1342,8 +2476,7 @@ namespace build2 // // TODO: handle and erase common attributes if/when we have any. // - as = move (attributes_top ()); - attributes_pop (); + as = attributes_pop (); // Handle the buildspec. // @@ -1383,7 +2516,7 @@ namespace build2 expire_mode (); next_after_newline (t, tt, "recipe action"); - // See if this is if-else or switch. + // See if this is if-else/switch or `recipe`. // // We want the keyword test similar to parse_clause() but we cannot do // it if replaying. So we skip it with understanding that if it's not @@ -1399,14 +2532,21 @@ namespace build2 // handy if we want to provide a custom recipe but only on certain // platforms or some such). - if (n == "if") + if (n == "if" || n == "if!") { - parse_if_else (t, tt, true /* multi */, parse_block); + parse_if_else (t, tt, true /* multi */, + parse_block, parse_recipe_directive); continue; } else if (n == "switch") { - parse_switch (t, tt, true /* multi */, parse_block); + parse_switch (t, tt, true /* multi */, + parse_block, parse_recipe_directive); + continue; + } + else if (n == "recipe") + { + parse_recipe_directive (t, tt, "" /* kind */); continue; } @@ -1414,7 +2554,7 @@ namespace build2 } if (tt != type::multi_lcbrace) - fail (t) << "expected recipe block instead of " << t; + fail (t) << "expected recipe block or 'recipe' instead of " << t; // Fall through. } @@ -1459,13 +2599,97 @@ namespace build2 } } - void parser:: - enter_adhoc_members (adhoc_names_loc&& ans, bool implied) + vector<reference_wrapper<target>> parser:: + enter_explicit_members (group_names_loc&& gns, bool implied) + { + tracer trace ("parser::enter_explicit_members", &path_); + + names& ns (gns.ns); + const location& loc (gns.member_loc); + + vector<reference_wrapper<target>> r; + r.reserve (ns.size ()); + + group& g (target_->as<group> ()); + auto& ms (g.static_members); + + for (size_t i (0); i != ns.size (); ++i) + { + name&& n (move (ns[i])); + name&& o (n.pair ? move (ns[++i]) : name ()); + + if (n.qualified ()) + fail (loc) << "project name in target " << n; + + // We derive the path unless the target name ends with the '...' escape + // which here we treat as the "let the rule derive the path" indicator + // (see target::split_name() for details). This will only be useful for + // referring to group members that are managed by the group's matching + // rule. Note also that omitting '...' for such a member could be used + // to override the file name, provided the rule checks if the path has + // already been derived before doing it itself. + // + // @@ What can the ad hoc recipe/rule do differently here? Maybe get + // path from dynamic targets? Maybe we will have custom path + // derivation support in buildscript in the future? + // + bool escaped; + { + const string& v (n.value); + size_t p (v.size ()); + + escaped = (p > 3 && + v[--p] == '.' && v[--p] == '.' && v[--p] == '.' && + v[--p] != '.'); + } + + target& m (enter_target::insert_target (*this, + move (n), move (o), + implied, + loc, trace)); + + if (g == m) + fail (loc) << "explicit group member " << m << " is group itself"; + + // Add as static member skipping duplicates. + // + if (find (ms.begin (), ms.end (), m) == ms.end ()) + { + if (m.group == nullptr) + m.group = &g; + else if (m.group != &g) + fail (loc) << g << " group member " << m << " already belongs to " + << "group " << *m.group; + + ms.push_back (m); + } + + if (!escaped) + { + if (file* ft = m.is_a<file> ()) + ft->derive_path (); + } + + r.push_back (m); + } + + return r; + } + + vector<reference_wrapper<target>> parser:: + enter_adhoc_members (group_names_loc&& gns, bool implied) { tracer trace ("parser::enter_adhoc_members", &path_); - names& ns (ans.ns); - const location& loc (ans.loc); + names& ns (gns.ns); + const location& loc (gns.member_loc); + + if (target_->is_a<group> ()) + fail (loc) << "ad hoc group primary member " << *target_ + << " is explicit group"; + + vector<reference_wrapper<target>> r; + r.reserve (ns.size ()); for (size_t i (0); i != ns.size (); ++i) { @@ -1493,14 +2717,16 @@ namespace build2 v[--p] != '.'); } - target& at ( - enter_target::insert_target (*this, - move (n), move (o), - implied, - loc, trace)); + target& m (enter_target::insert_target (*this, + move (n), move (o), + implied, + loc, trace)); + + if (target_ == &m) + fail (loc) << "ad hoc group member " << m << " is primary target"; - if (target_ == &at) - fail (loc) << "ad hoc group member " << at << " is primary target"; + if (m.is_a<group> ()) + fail (loc) << "ad hoc group member " << m << " is explicit group"; // Add as an ad hoc member at the end of the chain skipping duplicates. // @@ -1508,7 +2734,7 @@ namespace build2 const_ptr<target>* mp (&target_->adhoc_member); for (; *mp != nullptr; mp = &(*mp)->adhoc_member) { - if (*mp == &at) + if (*mp == &m) { mp = nullptr; break; @@ -1517,30 +2743,41 @@ namespace build2 if (mp != nullptr) { - *mp = &at; - at.group = target_; + if (m.group == nullptr) + m.group = target_; + else if (m.group != target_) + fail (loc) << *target_ << " ad hoc group member " << m + << " already belongs to group " << *m.group; + *mp = &m; } } if (!escaped) { - if (file* ft = at.is_a<file> ()) + if (file* ft = m.is_a<file> ()) ft->derive_path (); } + + r.push_back (m); } + + return r; } - small_vector<reference_wrapper<target>, 1> parser:: + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> parser:: enter_targets (names&& tns, const location& tloc, // Target names. - adhoc_names&& ans, // Ad hoc target names. - size_t prereq_size) + group_names&& gns, // Group member names. + size_t prereq_size, + const attributes& tas) // Target attributes. { - // Enter all the targets (normally we will have just one) and their ad hoc - // groups. + // Enter all the targets (normally we will have just one) and their group + // members. // tracer trace ("parser::enter_targets", &path_); - small_vector<reference_wrapper<target>, 1> tgs; + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> tgs; for (size_t i (0); i != tns.size (); ++i) { @@ -1550,24 +2787,36 @@ namespace build2 if (n.qualified ()) fail (tloc) << "project name in target " << n; - // Make sure none of our targets are patterns (maybe we will allow - // quoting later). + // Make sure none of our targets are patterns. // - if (path_pattern (n.value)) - fail (tloc) << "pattern in target " << n; + if (n.pattern) + fail (tloc) << "unexpected pattern in target " << n << + info << "ad hoc pattern rule may not be combined with other " + << "targets or patterns"; enter_target tg (*this, move (n), move (o), false /* implied */, tloc, trace); - // Enter ad hoc members. + if (!tas.empty ()) + apply_target_attributes (*target_, tas); + + // Enter group members. // - if (!ans.empty ()) + vector<reference_wrapper<target>> gms; + if (!gns.empty ()) { // Note: index after the pair increment. // - enter_adhoc_members (move (ans[i]), false /* implied */); + group_names_loc& g (gns[i]); + + if (g.expl && !target_->is_a<group> ()) + fail (g.group_loc) << *target_ << " is not group target"; + + gms = g.expl + ? enter_explicit_members (move (g), false /* implied */) + : enter_adhoc_members (move (g), false /* implied */); } if (default_target_ == nullptr) @@ -1575,17 +2824,97 @@ namespace build2 target_->prerequisites_state_.store (2, memory_order_relaxed); target_->prerequisites_.reserve (prereq_size); - tgs.push_back (*target_); + tgs.emplace_back (*target_, move (gms)); } return tgs; } void parser:: + apply_target_attributes (target& t, const attributes& as) + { + const location& l (as.loc); + + for (auto& a: as) + { + const string& n (a.name); + const value& v (a.value); + + // rule_hint= + // liba@rule_hint= + // + size_t p (string::npos); + if (n == "rule_hint" || + ((p = n.find ('@')) != string::npos && + n.compare (p + 1, string::npos, "rule_hint") == 0)) + { + // Resolve target type, if specified. + // + const target_type* tt (nullptr); + if (p != string::npos) + { + string t (n, 0, p); + tt = scope_->find_target_type (t); + + if (tt == nullptr) + fail (l) << "unknown target type " << t << " in rule_hint " + << "attribute"; + } + + // The rule hint value is vector<pair<optional<string>, string>> where + // the first half is the operation and the second half is the hint. + // Absent operation is used as a fallback for update/clean. + // + const names& ns (v.as<names> ()); + + for (auto i (ns.begin ()); i != ns.end (); ++i) + { + operation_id oi (default_id); + if (i->pair) + { + const name& n (*i++); + + if (!n.simple ()) + fail (l) << "expected operation name instead of " << n + << " in rule_hint attribute"; + + const string& v (n.value); + + if (!v.empty ()) + { + oi = ctx->operation_table.find (v); + + if (oi == 0) + fail (l) << "unknown operation " << v << " in rule_hint " + << "attribute"; + + if (root_->root_extra->operations[oi] == nullptr) + fail (l) << "project " << *root_ << " does not support " + << "operation " << ctx->operation_table[oi] + << " specified in rule_hint attribute"; + } + } + + const name& n (*i); + + if (!n.simple () || n.empty ()) + fail (l) << "expected hint instead of " << n << " in rule_hint " + << "attribute"; + + t.rule_hints.insert (tt, oi, n.value); + } + } + else + fail (l) << "unknown target attribute " << a; + } + } + + void parser:: parse_dependency (token& t, token_type& tt, names&& tns, const location& tloc, // Target names. - adhoc_names&& ans, // Ad hoc target names. - names&& pns, const location& ploc) // Prereq names. + group_names&& gns, // Group member names. + names&& pns, const location& ploc, // Prereq names. + const attributes& tas) // Target attributes. { // Parse a dependency chain and/or a target/prerequisite-specific variable // assignment/block and/or recipe block(s). @@ -1595,33 +2924,92 @@ namespace build2 // tracer trace ("parser::parse_dependency", &path_); + // Diagnose conditional prerequisites. Note that we want to diagnose this + // even if pns is empty (think empty variable expansion; the literal "no + // prerequisites" case is handled elsewhere). + // + // @@ TMP For now we only do it during the dist meta-operation. In the + // future we should tighten this to any meta-operation provided + // the dist module is loaded. + // + // @@ TMP For now it's a warning because we have dependencies like + // cli.cxx{foo}: cli{foo} which are not currently possible to + // rewrite (cli.cxx{} is not always registered). + // + if (condition_ && + ctx->current_mif != nullptr && + ctx->current_mif->id == dist_id) + { + // Only issue the warning for the projects being distributed. In + // particular, this makes sure we don't complain about imported + // projects. Note: use amalgamation to cover bundled subprojects. + // + auto* dm (root_->bundle_scope ()->find_module<dist::module> ( + dist::module::name)); + + if (dm != nullptr && dm->distributed) + { + warn (tloc) << "conditional dependency declaration may result in " + << "incomplete distribution" << + info (ploc) << "prerequisite declared here" << + info (*condition_) << "conditional buildfile fragment starts here" << + info << "instead use 'include' prerequisite-specific variable to " + << "conditionally include prerequisites" << + info << "for example: " + << "<target>: <prerequisite>: include = (<condition>)" << + info << "for details, see https://github.com/build2/HOWTO/blob/" + << "master/entries/keep-build-graph-config-independent.md"; + } + } + // First enter all the targets. // - small_vector<reference_wrapper<target>, 1> tgs ( - enter_targets (move (tns), tloc, move (ans), pns.size ())); + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> + tgs (enter_targets (move (tns), tloc, move (gns), pns.size (), tas)); // Now enter each prerequisite into each target. // - for (name& pn: pns) + for (auto i (pns.begin ()); i != pns.end (); ++i) { // We cannot reuse the names if we (potentially) may need to pass them // as targets in case of a chain (see below). // - name n (tt != type::colon ? move (pn) : pn); + name n (tt != type::colon ? move (*i) : *i); // See also scope::find_prerequisite_key(). // auto rp (scope_->find_target_type (n, ploc)); - const target_type* tt (rp.first); + const target_type* t (rp.first); optional<string>& e (rp.second); - if (tt == nullptr) - fail (ploc) << "unknown target type " << n.type; + if (t == nullptr) + { + if (n.proj) + { + // If the target type is unknown then no phase 2 import (like + // rule-specific search) can possibly succeed so we can fail now and + // with a more accurate reason. See import2(names) for background. + // + diag_record dr; + dr << fail (ploc) << "unable to import target " << n; + import_suggest (dr, *n.proj, nullptr, string (), false); + } + else + { + fail (ploc) << "unknown target type " << n.type << + info << "perhaps the module that defines this target type is " + << "not loaded by project " << *scope_->root_scope (); + } + } + + if (t->factory == nullptr) + fail (ploc) << "abstract target type " << t->name << "{}"; // Current dir collapses to an empty one. // if (!n.dir.empty ()) - n.dir.normalize (false, true); + n.dir.normalize (false /* actual */, true); // @@ OUT: for now we assume the prerequisite's out is undetermined. The // only way to specify an src prerequisite will be with the explicit @@ -1632,10 +3020,47 @@ namespace build2 // a special indicator. Also, one can easily and natually suppress any // searches by specifying the absolute path. // + name o; + if (n.pair) + { + assert (n.pair == '@'); + + ++i; + o = tt != type::colon ? move (*i) : *i; + + if (!o.directory ()) + fail (ploc) << "expected directory after '@'"; + + o.dir.normalize (); // Note: don't collapse current to empty. + + // Make sure out and src are parallel unless both were specified as + // absolute. We make an exception for this case because out may be + // used to "tag" imported targets (see cc::search_library()). So it's + // sort of the "I know what I am doing" escape hatch (it would have + // been even better to verify such a target is outside any project + // but that won't be cheap). + // + // For now we require that both are either relative or absolute. + // + // See similar code for targets in scope::find_target_type(). + // + if (n.dir.absolute () && o.dir.absolute ()) + ; + else if (n.dir.empty () && o.dir.current ()) + ; + else if (o.dir.relative () && + n.dir.relative () && + o.dir == n.dir) + ; + else + fail (ploc) << "prerequisite output directory " << o.dir + << " must be parallel to source directory " << n.dir; + } + prerequisite p (move (n.proj), - *tt, + *t, move (n.dir), - dir_path (), + move (o.dir), move (n.value), move (e), *scope_); @@ -1644,7 +3069,7 @@ namespace build2 { // Move last prerequisite (which will normally be the only one). // - target& t (*i); + target& t (i->first); t.prerequisites_.push_back (++i == e ? move (p) : prerequisite (p, memory_order_relaxed)); @@ -1657,20 +3082,42 @@ namespace build2 // // We handle multiple targets and/or prerequisites by replaying the tokens // (see the target-specific case comments for details). The function - // signature is: + // signature for for_each_t (see for_each on the gm argument semantics): + // + // void (token& t, type& tt, optional<bool> gm) + // + // And for for_each_p: // // void (token& t, type& tt) // auto for_each_t = [this, &t, &tt, &tgs] (auto&& f) { - replay_guard rg (*this, tgs.size () > 1); + // We need replay if we have multiple targets or group members. + // + replay_guard rg (*this, tgs.size () > 1 || !tgs[0].second.empty ()); for (auto ti (tgs.begin ()), te (tgs.end ()); ti != te; ) { - target& tg (*ti); - enter_target tgg (*this, tg); + target& tg (ti->first); + const vector<reference_wrapper<target>>& gms (ti->second); + + { + enter_target g (*this, tg); + f (t, tt, nullopt); + } + + if (!gms.empty ()) + { + bool expl (tg.is_a<group> ()); - f (t, tt); + for (target& gm: gms) + { + rg.play (); // Replay. + + enter_target g (*this, gm); + f (t, tt, expl); + } + } if (++ti != te) rg.play (); // Replay. @@ -1683,8 +3130,8 @@ namespace build2 for (auto ti (tgs.begin ()), te (tgs.end ()); ti != te; ) { - target& tg (*ti); - enter_target tgg (*this, tg); + target& tg (ti->first); + enter_target g (*this, tg); for (size_t pn (tg.prerequisites_.size ()), pi (pn - pns.size ()); pi != pn; ) @@ -1727,7 +3174,7 @@ namespace build2 this, st = token (t), // Save start token (will be gone on replay). recipes = small_vector<shared_ptr<adhoc_rule>, 1> ()] - (token& t, type& tt) mutable + (token& t, type& tt, optional<bool> gm) mutable { token rt; // Recipe start token. @@ -1737,7 +3184,14 @@ namespace build2 { next (t, tt); // Newline. next (t, tt); // First token inside the variable block. - parse_variable_block (t, tt); + + // Skip explicit group members (see the block case above for + // background). + // + if (!gm || !*gm) + parse_variable_block (t, tt); + else + skip_block (t, tt); if (tt != type::rcbrace) fail (t) << "expected '}' instead of " << t; @@ -1753,6 +3207,16 @@ namespace build2 else rt = st; + // If this is a group member then we know we are replaying and can + // skip the recipe. + // + if (gm) + { + replay_skip (); + next (t, tt); + return; + } + parse_recipe (t, tt, rt, recipes); }; @@ -1762,21 +3226,6 @@ namespace build2 return; } - // What should we do if there are no prerequisites (for example, because - // of an empty wildcard result)? We can fail or we can ignore. In most - // cases, however, this is probably an error (for example, forgetting to - // checkout a git submodule) so let's not confuse the user and fail (one - // can always handle the optional prerequisites case with a variable and - // an if). - // - if (pns.empty ()) - fail (ploc) << "no prerequisites in dependency chain or prerequisite-" - << "specific variable assignment"; - - next_with_attributes (t, tt); // Recognize attributes after `:`. - - auto at (attributes_push (t, tt)); - // If we are here, then this can be one of three things: // // 1. A prerequisite-specific variable bloc: @@ -1790,10 +3239,37 @@ namespace build2 // // foo: bar: x = y // - // 3. A further dependency chain : + // 3. A further dependency chain: // // foo: bar: baz ... // + // What should we do if there are no prerequisites, for example, because + // of an empty wildcard result or empty variable expansion? We can fail or + // we can ignore. In most cases, however, this is probably an error (for + // example, forgetting to checkout a git submodule) so let's not confuse + // the user and fail (one can always handle the optional prerequisites + // case with a variable and an if). + // + // On the other hand, we allow just empty prerequisites (which is also the + // more common case by far) and so it's strange that we don't allow the + // same with, say, `include = false`: + // + // exe{foo}: cxx{$empty} # Ok. + // exe{foo}: cxx{$empty}: include = false # Not Ok? + // + // So let's ignore in the first two cases (variable block and assignment) + // for consistency. The dependency chain is iffy both conceptually and + // implementation-wise (it could be followed by a variable block). So + // let's keep it an error for now. + // + // Note that the syntactically-empty prerequisite list is still an error: + // + // exe{foo}: : include = false # Error. + // + next_with_attributes (t, tt); // Recognize attributes after `:`. + + auto at (attributes_push (t, tt)); + if (tt == type::newline || tt == type::eos) { attributes_pop (); // Must be none since can't be standalone. @@ -1808,15 +3284,22 @@ namespace build2 // Parse the block for each prerequisites of each target. // - for_each_p ([this] (token& t, token_type& tt) - { - next (t, tt); // First token inside the block. + if (!pns.empty ()) + for_each_p ([this] (token& t, token_type& tt) + { + next (t, tt); // First token inside the block. - parse_variable_block (t, tt); + parse_variable_block (t, tt); - if (tt != type::rcbrace) - fail (t) << "expected '}' instead of " << t; - }); + if (tt != type::rcbrace) + fail (t) << "expected '}' instead of " << t; + }); + else + { + skip_block (t, tt); + if (tt != type::rcbrace) + fail (t) << "expected '}' instead of " << t; + } next (t, tt); // Presumably newline after '}'. next_after_newline (t, tt, '}'); // Should be on its own line. @@ -1839,10 +3322,13 @@ namespace build2 // Parse the assignment for each prerequisites of each target. // - for_each_p ([this, &var, at] (token& t, token_type& tt) - { - parse_variable (t, tt, var, at); - }); + if (!pns.empty ()) + for_each_p ([this, &var, at] (token& t, token_type& tt) + { + parse_variable (t, tt, var, at); + }); + else + skip_line (t, tt); next_after_newline (t, tt); @@ -1861,6 +3347,13 @@ namespace build2 // else { + if (pns.empty ()) + fail (ploc) << "no prerequisites in dependency chain"; + + // @@ This is actually ambiguous: prerequisite or target attributes + // (or both or neither)? Perhaps this should be prerequisites for + // the same reason as below (these are prerequsites first). + // if (at.first) fail (at.second) << "attributes before prerequisites"; else @@ -1872,30 +3365,35 @@ namespace build2 // we just say that the dependency chain is equivalent to specifying // each dependency separately. // - // Also note that supporting ad hoc target group specification in - // chains will be complicated. For example, what if prerequisites that - // have ad hoc targets don't end up being chained? Do we just silently - // drop them? Also, these are prerequsites first that happened to be - // reused as target names so perhaps it is the right thing not to - // support, conceptually. + // Also note that supporting target group specification in chains will + // be complicated. For example, what if prerequisites that have group + // members don't end up being chained? Do we just silently drop them? + // Also, these are prerequsites first that happened to be reused as + // target names so perhaps it is the right thing not to support, + // conceptually. // parse_dependency (t, tt, move (pns), ploc, - {} /* ad hoc target name */, - move (ns), loc); + {} /* group names */, + move (ns), loc, + attributes () /* target attributes */); } } } void parser:: - source (istream& is, const path_name& in, const location& loc, bool deft) + source_buildfile (istream& is, + const path_name& in, + const location& loc, + optional<bool> deft) { - tracer trace ("parser::source", &path_); + tracer trace ("parser::source_buildfile", &path_); l5 ([&]{trace (loc) << "entering " << in;}); - if (in.path != nullptr) - enter_buildfile (*in.path); + const buildfile* bf (in.path != nullptr + ? &enter_buildfile<buildfile> (*in.path) + : nullptr); const path_name* op (path_); path_ = ∈ @@ -1905,11 +3403,11 @@ namespace build2 lexer_ = &l; target* odt; - if (deft) - { + if (!deft || *deft) odt = default_target_; + + if (deft && *deft) default_target_ = nullptr; - } token t; type tt; @@ -1919,12 +3417,15 @@ namespace build2 if (tt != type::eos) fail (t) << "unexpected " << t; - if (deft) + if (deft && *deft) { - process_default_target (t); - default_target_ = odt; + if (stage_ != stage::boot && stage_ != stage::root) + process_default_target (t, bf); } + if (!deft || *deft) + default_target_ = odt; + lexer_ = ol; path_ = op; @@ -1934,11 +3435,35 @@ namespace build2 void parser:: parse_source (token& t, type& tt) { + // source [<attrs>] <path>+ + // + // The rest should be a list of buildfiles. Parse them as names in the - // value mode to get variable expansion and directory prefixes. + // value mode to get variable expansion and directory prefixes. Also + // handle optional attributes. // mode (lexer_mode::value, '@'); - next (t, tt); + next_with_attributes (t, tt); + attributes_push (t, tt); + + bool nodt (false); // Source buildfile without default target semantics. + { + attributes as (attributes_pop ()); + const location& l (as.loc); + + for (const attribute& a: as) + { + const string& n (a.name); + + if (n == "no_default_target") + { + nodt = true; + } + else + fail (l) << "unknown source directive attribute " << a; + } + } + const location l (get_location (t)); names ns (tt != type::newline && tt != type::eos ? parse_names (t, tt, pattern_mode::expand, "path", nullptr) @@ -1965,10 +3490,10 @@ namespace build2 try { ifdstream ifs (p); - source (ifs, - path_name (p), - get_location (t), - false /* default_target */); + source_buildfile (ifs, + path_name (p), + get_location (t), + nodt ? optional<bool> {} : false); } catch (const io_error& e) { @@ -1982,6 +3507,9 @@ namespace build2 void parser:: parse_include (token& t, type& tt) { + // include <path>+ + // + tracer trace ("parser::parse_include", &path_); if (stage_ == stage::boot) @@ -2077,10 +3605,7 @@ namespace build2 // out the absolute buildfile path since we may switch the project // root and src_root with it (i.e., include into a sub-project). // - scope* ors (root_); - scope* ocs (scope_); - const dir_path* opb (pbase_); - switch_scope (out_base); + enter_scope sg (*this, out_base, true /* absolute & normalized */); if (root_ == nullptr) fail (l) << "out of project include from " << out_base; @@ -2093,31 +3618,40 @@ namespace build2 l6 ([&]{trace (l) << "absolute path " << p;}); - if (!root_->buildfiles.insert (p).second) // Note: may be "new" root. + // Note: may be "new" root. + // + if (!root_->root_extra->insert_buildfile (p)) { l5 ([&]{trace (l) << "skipping already included " << p;}); - pbase_ = opb; - scope_ = ocs; - root_ = ors; continue; } + // Note: see a variant of this in parse_import(). + // + // Clear/restore if/switch location. + // + // We do it here but not in parse_source since the included buildfile is + // in a sense expected to be a standalone entity (think a file included + // from an export stub). + // + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = nullopt; + try { ifdstream ifs (p); - source (ifs, - path_name (p), - get_location (t), - true /* default_target */); + source_buildfile (ifs, + path_name (p), + get_location (t), + true /* default_target */); } catch (const io_error& e) { fail (l) << "unable to read buildfile " << p << ": " << e; } - - pbase_ = opb; - scope_ = ocs; - root_ = ors; } next_after_newline (t, tt); @@ -2128,6 +3662,10 @@ namespace build2 { // run <name> [<arg>...] // + // Note that if the result of executing the program can be affected by + // environment variables and this result can in turn affect the build + // result, then such variables should be reported with the + // config.environment directive. // Parse the command line as names in the value mode to get variable // expansion, etc. @@ -2141,7 +3679,7 @@ namespace build2 { args = convert<strings> ( tt != type::newline && tt != type::eos - ? parse_names (t, tt, pattern_mode::ignore, "argument", nullptr) + ? parse_names (t, tt, pattern_mode::expand, "argument", nullptr) : names ()); } catch (const invalid_argument& e) @@ -2160,13 +3698,16 @@ namespace build2 [] (const string& s) {return s.c_str ();}); cargs.push_back (nullptr); + // Note: we are in the serial load phase and so no diagnostics buffering + // is needed. + // process pr (run_start (3 /* verbosity */, cargs, 0 /* stdin */, -1 /* stdout */, - true /* error */, - dir_path () /* cwd */, + 2 /* stderr */, nullptr /* env */, + dir_path () /* cwd */, l)); try { @@ -2188,10 +3729,10 @@ namespace build2 dr << info (l) << "while parsing " << args[0] << " output"; }); - source (is, - path_name ("<stdout>"), - l, - false /* default_target */); + source_buildfile (is, + path_name ("<stdout>"), + l, + false /* default_target */); } is.close (); // Detect errors. @@ -2205,7 +3746,7 @@ namespace build2 // caused by that and let run_finish() deal with it. } - run_finish (cargs, pr, l); + run_finish (cargs, pr, 2 /* verbosity */, false /* omit_normal */, l); next_after_newline (t, tt); } @@ -2251,33 +3792,45 @@ namespace build2 // which case it will be duplicating them in its root.build file). So // for now we allow this trusting the user knows what they are doing. // - string proj; - { - const project_name& n (named_project (*root_)); - - if (!n.empty ()) - proj = n.variable (); - } + // There is another special case: a buildfile imported from another + // project. In this case we also allow <project> to be the imported + // project name in addition to importing. The thinking here is that an + // imported buildfile is in a sense like a module (may provide rules which + // may require configuration, etc) and should be able to use its own + // project name (which is often the corresponding tool name) in the + // configuration variables, just like modules. In this case we use the + // imported project name as the reporting module name (but which can + // be overridden with config.report.module attribute). + // + const location loc (get_location (t)); - // We are now in the normal lexing mode. Since we always have <var> we - // don't have to resort to manual parsing (as in import) and can just let - // the lexer handle `?=`. + // We are now in the normal lexing mode and we let the lexer handle `?=`. // next_with_attributes (t, tt); // Get variable attributes, if any, and deal with the special config.* - // attributes. Since currently they can only appear in the config - // directive, we handle them in an ad hoc manner. + // attributes as well as null. Since currently they can only appear in the + // config directive, we handle them in an ad hoc manner. // attributes_push (t, tt); attributes& as (attributes_top ()); + bool nullable (false); optional<string> report; string report_var; + // Reporting module name. Empty means the config module reporting + // project's own configuration. + // + project_name report_module; + for (auto i (as.begin ()); i != as.end (); ) { - if (i->name == "config.report") + if (i->name == "null") + { + nullable = true; + } + else if (i->name == "config.report") { try { @@ -2289,7 +3842,7 @@ namespace build2 report = move (v); else throw invalid_argument ( - "expected 'false' or format name instead of '" + v + "'"); + "expected 'false' or format name instead of '" + v + '\''); } catch (const invalid_argument& e) { @@ -2301,6 +3854,23 @@ namespace build2 try { report_var = convert<string> (move (i->value)); + + if (!report) + report = string ("true"); + } + catch (const invalid_argument& e) + { + fail (as.loc) << "invalid " << i->name << " attribute value: " << e; + } + } + else if (i->name == "config.report.module") + { + try + { + report_module = convert<project_name> (move (i->value)); + + if (!report) + report = string ("true"); } catch (const invalid_argument& e) { @@ -2320,19 +3890,22 @@ namespace build2 fail (t) << "expected configuration variable name instead of " << t; string name (move (t.value)); + bool config (name.compare (0, 7, "config.") == 0); // As a way to print custom (discovered, computed, etc) configuration // information we allow specifying a non config.* variable provided it is - // explicitly marked with the config.report attribute. + // explicitly marked with the config.report attribute (or another + // attribute that implies it). // bool new_val (false); + string org_var; // Original variable if config.report.variable specified. + + const variable* var (nullptr); // config.* variable. lookup l; - if (report && - *report != "false" && - name.compare (0, 7, "config.") != 0) + if (report && *report != "false" && !config) { - if (!as.empty ()) + if (!as.empty () || nullable) fail (as.loc) << "unexpected attributes for report-only variable"; attributes_pop (); @@ -2346,7 +3919,14 @@ namespace build2 // philosophical question. In either case it doesn't seem useful for it // to unconditionally force reporting at level 2. // - report_var = move (name); + if (!report_var.empty ()) + { + // For example, config [config.report.variable=multi] multi_database + // + org_var = move (name); + } + else + report_var = move (name); next (t, tt); // We shouldn't have the default value part. } @@ -2359,119 +3939,315 @@ namespace build2 // config prefix and the project substring. // { - diag_record dr; + string proj; + { + const project_name& n (named_project (*root_)); - if (name.compare (0, 7, "config.") != 0) - dr << fail (t) << "configuration variable '" << name - << "' does not start with 'config.'"; + if (!n.empty ()) + proj = n.variable (); + } - if (!proj.empty ()) + diag_record dr; + do // Breakout loop. { - size_t p (name.find ('.' + proj)); + if (!config) + { + dr << fail (t) << "configuration variable '" << name + << "' does not start with 'config.'"; + break; + } + + auto match = [&name] (const string& proj) + { + size_t p (name.find ('.' + proj)); + return (p != string::npos && + ((p += proj.size () + 1) == name.size () || // config.<proj> + name[p] == '.')); // config.<proj>. + }; + + if (!proj.empty () && match (proj)) + break; - if (p == string::npos || - ((p += proj.size () + 1) != name.size () && // config.<proj> - name[p] != '.')) // config.<proj>. + // See if this buildfile belongs to a different project. If so, use + // the project name as the reporting module name. + // + if (path_->path != nullptr) { + // Note: all sourced/included/imported paths are absolute and + // normalized. + // + const path& f (*path_->path); + dir_path d (f.directory ()); + + auto p (ctx->scopes.find (d)); // Note: never empty. + if (*p.first != &ctx->global_scope) + { + // The buildfile will most likely be in src which means we may + // end up with multiple scopes (see scope_map for background). + // First check if one of them is us. If not, then we can extract + // the project name from any one of them. + // + const scope& bs (**p.first); // Save. + + for (; p.first != p.second; ++p.first) + { + if (root_ == (*p.first)->root_scope ()) + break; + } + + if (p.first == p.second) + { + // Note: we expect the project itself to be named. + // + const project_name& n (project (*bs.root_scope ())); + + if (!n.empty ()) + { + // If the buildfile comes from a different project, then + // it's more likely to use the imported project's config + // variables. So replace proj with that for diagnostics + // below. + // + proj = n.variable (); + + if (*report != "false" && verb >= 2) + report_module = n; + } + } + } + else + { + // If the buildfile is not in any project, then it could be + // installed. + // + // Per import2_buildfile(), exported buildfiles are installed + // into $install.buildfile/<proj>/.... + // + const dir_path& id (build_install_buildfile); + + if (!id.empty () && d.sub (id)) + { + dir_path l (d.leaf (id)); + if (!l.empty ()) + { + project_name n (*l.begin ()); + proj = n.variable (); + + if (*report != "false" && verb >= 2) + report_module = move (n); + } + } + } + } + + if (!proj.empty () && match (proj)) + break; + + // Note: only if proj not empty (see above). + // + if (!proj.empty ()) dr << fail (t) << "configuration variable '" << name << "' does not include project name"; - } } + while (false); if (!dr.empty ()) dr << info << "expected variable name in the 'config[.**]." << (proj.empty () ? "<project>" : proj.c_str ()) << ".**' form"; } - const variable& var ( - scope_->var_pool ().insert (move (name), true /* overridable */)); - - apply_variable_attributes (var); + var = &parse_variable_name (move (name), get_location (t)); + apply_variable_attributes (*var); // Note that even though we are relying on the config.** variable // pattern to set global visibility, let's make sure as a sanity check. // - if (var.visibility != variable_visibility::global) + if (var->visibility != variable_visibility::global) { - fail (t) << "configuration variable " << var << " has " - << var.visibility << " visibility"; + fail (t) << "configuration variable " << *var << " has " + << var->visibility << " visibility"; } - // We have to lookup the value whether we have the default part or not - // in order to mark it as saved. We also have to do this to get the new - // value status. + // See if we have the default value part. // - using config::lookup_config; + next (t, tt); + bool def_val (tt != type::newline && tt != type::eos); - l = lookup_config (new_val, *root_, var); + if (def_val && tt != type::default_assign) + fail (t) << "expected '?=' instead of " << t << " after " + << "configuration variable name"; - // See if we have the default value part. + // If this is the special config.<project>.develop variable, verify it + // is of type bool and has false as the default value. We also only save + // it in config.build if it's true and suppress any unused warnings in + // config::save_config() if specified but not used by the project. // - next (t, tt); + // Here we also have the unnamed project issues (see above for details) + // and so we actually recognize any config.**.develop. + // + bool dev; + { + size_t p (var->name.rfind ('.')); + dev = p != 6 && var->name.compare (p + 1, string::npos, "develop") == 0; + } - if (tt != type::newline && tt != type::eos) + uint64_t sflags (0); + if (dev) { - if (tt != type::default_assign) - fail (t) << "expected '?=' instead of " << t << " after " - << "configuration variable name"; + if (var->type != &value_traits<bool>::value_type) + fail (loc) << *var << " variable must be of type bool"; + + // This is quite messy: below we don't always parse the value (plus it + // may be computed) so here we just peek at the next token. But we + // have to do this in the same mode as parse_variable_value(). + // + if (!def_val || + peek (lexer_mode::value, '@') != type::word || + peeked ().value != "false") + fail (loc) << *var << " variable default value must be literal false"; + + if (nullable) + fail (loc) << *var << " variable must not be nullable"; + + sflags |= config::save_false_omitted; + } + // We have to lookup the value whether we have the default part or not + // in order to mark it as saved. We also have to do this to get the new + // value status. + // + l = config::lookup_config (new_val, *root_, *var, sflags); + + // Handle the default value. + // + if (def_val) + { // The rest is the default value which we should parse in the value // mode. But before switching check whether we need to evaluate it at // all. // if (l.defined ()) + { + // Peek at the attributes to detect whether the value is NULL. + // + if (!dev && !nullable) + { + // Essentially a prefix of parse_variable_value(). + // + mode (lexer_mode::value, '@'); + next_with_attributes (t, tt); + attributes_push (t, tt, true); + for (const attribute& a: attributes_pop ()) + { + if (a.name == "null") + { + nullable = true; + break; + } + } + } + skip_line (t, tt); + } else { - value lhs, rhs (parse_variable_value (t, tt)); - apply_value_attributes (&var, lhs, move (rhs), type::assign); - l = lookup_config (new_val, *root_, var, move (lhs)); + value lhs, rhs (parse_variable_value (t, tt, !dev /* mode */)); + apply_value_attributes (var, lhs, move (rhs), type::assign); + + if (!nullable) + nullable = lhs.null; + + l = config::lookup_config (new_val, *root_, *var, move (lhs), sflags); } } + + // If the variable is not nullable, verify the value is not NULL. + // + // Note that undefined is not the same as NULL (if it is undefined, we + // should either see the default value or if there is no default value, + // then the user is expected to handle the undefined case). + // + if (!nullable && l.defined () && l->null) + fail (loc) << "null value in non-nullable variable " << *var; } // We will be printing the report at either level 2 (-v) or 3 (-V) - // depending on the final value of config_report_new. + // depending on the final value of config_report::new_value. // - // Note that for the config_report_new calculation we only incorporate - // variables that we are actually reporting. + // Note that for the config_report::new_value calculation we only + // incorporate variables that we are actually reporting. // if (*report != "false" && verb >= 2) { + // Find existing or insert new config_report entry for this module. + // + auto i (find_if (config_reports.begin (), + config_reports.end (), + [&report_module] (const config_report& r) + { + return r.module == report_module; + })); + + if (i == config_reports.end ()) + { + config_reports.push_back ( + config_report {move (report_module), {}, false}); + i = config_reports.end () - 1; + } + + auto& report_values (i->values); + bool& report_new_value (i->new_value); + // We don't want to lookup the report variable value here since it's // most likely not set yet. // if (!report_var.empty ()) { + if (org_var.empty () && var != nullptr) + org_var = var->name; + // In a somewhat hackish way we pass the variable in an undefined // lookup. // + // Note: consistent with parse_variable_name() wrt overridability. + // l = lookup (); l.var = &root_->var_pool ().insert ( - move (report_var), true /* overridable */); + move (report_var), + report_var.find ('.') != string::npos /* overridable */); } if (l.var != nullptr) { - auto r (make_pair (l, move (*report))); - // If we have a duplicate, update it (it could be useful to have // multiple config directives to "probe" the value before calculating // the default; see lookup_config() for details). // - auto i (find_if (config_report.begin (), - config_report.end (), - [&l] (const pair<lookup, string>& p) + // Since the original variable is what the user will see in the + // report, we prefer that as a key. + // + auto i (find_if (report_values.begin (), + report_values.end (), + [&org_var, &l] (const config_report::value& v) { - return p.first.var == l.var; + return (v.org.empty () && org_var.empty () + ? v.val.var == l.var + : (v.org.empty () + ? v.val.var->name == org_var + : v.org == l.var->name)); })); - if (i == config_report.end ()) - config_report.push_back (move (r)); + if (i == report_values.end ()) + report_values.push_back ( + config_report::value {l, move (*report), move (org_var)}); else - *i = move (r); + { + i->val = l; + i->fmt = move (*report); + if (i->org.empty ()) i->org = move (org_var); + } - config_report_new = config_report_new || new_val; + report_new_value = report_new_value || new_val; } } @@ -2479,6 +4255,48 @@ namespace build2 } void parser:: + parse_config_environment (token& t, type& tt) + { + // config.environment <name>... + // + + // While we could allow this directive during bootstrap, it would have to + // be after loading the config module, which can be error prone. So we + // disallow it for now (it's also not clear "configuring" bootstrap with + // environment variables is a good idea; think of info, etc). + // + if (stage_ == stage::boot) + fail (t) << "config.environment during bootstrap"; + + // Parse the rest as names in the value mode to get variable expansion, + // etc. + // + mode (lexer_mode::value); + next (t, tt); + const location l (get_location (t)); + + strings ns; + try + { + ns = convert<strings> ( + tt != type::newline && tt != type::eos + ? parse_names (t, tt, + pattern_mode::ignore, + "environment variable name", + nullptr) + : names ()); + } + catch (const invalid_argument& e) + { + fail (l) << "invalid environment variable name: " << e.what (); + } + + config::save_environment (*root_, ns); + + next_after_newline (t, tt); + } + + void parser:: parse_import (token& t, type& tt) { tracer trace ("parser::parse_import", &path_); @@ -2486,170 +4304,215 @@ namespace build2 if (stage_ == stage::boot) fail (t) << "import during bootstrap"; - // General import format: + // General import form: + // + // import[?!] [<attrs>] <var> = [<attrs>] (<target>|<project>%<target>])+ + // + // Special form for importing buildfiles: // - // import[?!] [<attrs>] [<var>=](<target>|<project>%<target>])+ + // import[?!] [<attrs>] (<target>|<project>%<target>])+ // bool opt (t.value.back () == '?'); - bool ph2 (opt || t.value.back () == '!'); + optional<string> ph2 (opt || t.value.back () == '!' + ? optional<string> (string ()) + : nullopt); - type atype; // Assignment type. - value* val (nullptr); - const variable* var (nullptr); - - // We are now in the normal lexing mode and here is the problem: we need - // to switch to the value mode so that we don't treat certain characters - // as separators (e.g., + in 'libstdc++'). But at the same time we need - // to detect if we have the <var>= part. So what we are going to do is - // switch to the value mode, get the first token, and then re-parse it - // manually looking for =/=+/+=. + // We are now in the normal lexing mode and we let the lexer handle `=`. // - // Note that if we ever wanted to support value attributes, that would be - // non-trivial. - // - mode (lexer_mode::value, '@'); next_with_attributes (t, tt); - // Get variable (or value) attributes, if any, and deal with the special - // metadata attribute. Since currently it can only appear in the import - // directive, we handle it in an ad hoc manner. + // Get variable (or value, in the second form) attributes, if any, and + // deal with the special metadata and rule_hint attributes. Since + // currently they can only appear in the import directive, we handle them + // in an ad hoc manner. // attributes_push (t, tt); - attributes& as (attributes_top ()); - bool meta (false); - for (auto i (as.begin ()); i != as.end (); ) + bool meta (false); // Import with metadata. + bool once (false); // Import buildfile once. + bool nodt (false); // Import buildfile without default target semantics. { - if (i->name == "metadata") - { - if (!ph2) - fail (as.loc) << "loading metadata requires immediate import" << - info << "consider using the import! directive instead"; + attributes& as (attributes_top ()); + const location& l (as.loc); - meta = true; - } - else - { - ++i; - continue; - } - - i = as.erase (i); - } - - const location vloc (get_location (t)); - - if (tt == type::word) - { - // Split the token into the variable name and value at position (p) of - // '=', taking into account leading/trailing '+'. The variable name is - // returned while the token is set to the value part. If the resulting - // token value is empty, get the next token. Also set assignment type - // (at). - // - auto split = [&atype, &t, &tt, this] (size_t p) -> string + for (auto i (as.begin ()); i != as.end (); ) { - string& v (t.value); - size_t e; + const string& n (i->name); + value& v (i->value); - if (p != 0 && v[p - 1] == '+') // += + if (n == "metadata") { - e = p--; - atype = type::append; + if (!ph2) + fail (l) << "loading metadata requires immediate import" << + info << "consider using the import! directive instead"; + + meta = true; } - else if (p + 1 != v.size () && v[p + 1] == '+') // =+ + else if (n == "no_default_target") { - e = p + 1; - atype = type::prepend; + nodt = true; } - else // = + else if (n == "once") { - e = p; - atype = type::assign; + once = true; } + else if (n == "rule_hint") + { + if (!ph2) + fail (l) << "rule hint can only be used with immediate import" << + info << "consider using the import! directive instead"; - string nv (v, e + 1); // value - v.resize (p); // var name - v.swap (nv); - - if (v.empty ()) - next (t, tt); - - return nv; - }; - - // Is this the 'foo=...' case? - // - size_t p (t.value.find ('=')); - auto& vp (scope_->var_pool ()); - - if (p != string::npos) - var = &vp.insert (split (p), true /* overridable */); - // - // This could still be the 'foo =...' case. - // - else if (peek () == type::word) - { - const string& v (peeked ().value); - size_t n (v.size ()); + // Here we only allow a single name. + // + try + { + ph2 = convert<string> (move (v)); - // We should start with =/+=/=+. - // - if (n > 0 && - (v[p = 0] == '=' || - (n > 1 && v[0] == '+' && v[p = 1] == '='))) + if (ph2->empty ()) + throw invalid_argument ("empty name"); + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else { - var = &vp.insert (move (t.value), true /* overridable */); - next (t, tt); // Get the peeked token. - split (p); // Returned name should be empty. + ++i; + continue; } + + i = as.erase (i); } } - if (var != nullptr) + // Note that before supporting the second form (without <var>) we used to + // parse the value after assignment in the value mode. However, we don't + // really need to since what we should have is a bunch of target names. + // In other words, whatever the value mode does not treat as special + // compared to the normal mode (like `:`) would be illegal here. + // + // Note that we expant patterns for the ad hoc import case: + // + // import sub = */ + // + // @@ PAT: the only issue here is that we currently pattern-expand var + // name (same assue as with target-specific var names). + // + if (!start_names (tt)) + fail (t) << "expected variable name or buildfile target instead of " << t; + + location loc (get_location (t)); + names ns (parse_names (t, tt, pattern_mode::expand)); + + // Next could come the assignment operator. Note that we don't support + // default assignment (?=) yet (could make sense when attempting to import + // alternatives or some such). + // + type atype; + const variable* var (nullptr); + if (tt == type::assign || tt == type::append || tt == type::prepend) { + var = &parse_variable_name (move (ns), loc); apply_variable_attributes (*var); if (var->visibility > variable_visibility::scope) { - fail (vloc) << "variable " << *var << " has " << var->visibility - << " visibility but is assigned in import"; + fail (loc) << "variable " << *var << " has " << var->visibility + << " visibility but is assigned in import"; } - val = atype == type::assign - ? &scope_->assign (*var) - : &scope_->append (*var); + atype = tt; + next_with_attributes (t, tt); + attributes_push (t, tt, true /* standalone */); + + if (!start_names (tt)) + fail (t) << "expected target to import instead of " << t; + + loc = get_location (t); + ns = parse_names (t, tt, pattern_mode::expand); } - else + else if (tt == type::default_assign) + fail (t) << "default assignment not yet supported"; + + + // If there are any value attributes, roundtrip the names through the + // value applying the attributes. + // + if (!attributes_top ().empty ()) { - if (!as.empty ()) - fail (as.loc) << "attributes without variable"; + value lhs, rhs (move (ns)); + apply_value_attributes (nullptr, lhs, move (rhs), type::assign); - attributes_pop (); + if (!lhs) + fail (loc) << "expected target to import instead of null value"; + + untypify (lhs, true /* reduce */); + ns = move (lhs.as<names> ()); } + else + attributes_pop (); - // The rest should be a list of projects and/or targets. Parse them as - // names to get variable expansion and directory prefixes. - // - // Note: that we expant patterns for the ad hoc import case: - // - // import sub = */ - // - const location l (get_location (t)); - names ns (tt != type::newline && tt != type::eos - ? parse_names (t, tt, pattern_mode::expand) - : names ()); + value* val (var != nullptr ? + &(atype == type::assign + ? scope_->assign (*var) + : scope_->append (*var)) + : nullptr); for (name& n: ns) { - // @@ Could this be an out-qualified ad hoc import? + // @@ Could this be an out-qualified ad hoc import? Yes, see comment + // about buildfile import in import_load(). // if (n.pair) - fail (l) << "unexpected pair in import"; + fail (loc) << "unexpected pair in import"; + + // See if we are importing a buildfile target. Such an import is always + // immediate. + // + bool bf (n.type == "buildfile"); + if (bf) + { + if (meta) + fail (loc) << "metadata requested for buildfile target " << n; + + if (var != nullptr) + { + if (once) + fail (loc) << "once importation requested with variable assignment"; + + if (nodt) + fail (loc) << "no_default_target importation requested with " + << "variable assignment"; + } + + if (ph2 && !ph2->empty ()) + fail (loc) << "rule hint specified for buildfile target " << n; + } + else + { + if (once) + fail (loc) << "once importation requested for target " << n; + + if (nodt) + fail (loc) << "no_default_target importation requested for target " + << n; + + if (var == nullptr) + fail (loc) << "variable assignment required to import target " << n; + } // import() will check the name, if required. // - names r (import (*scope_, move (n), ph2, opt, meta, l).first); + import_result<scope> ir ( + import (*scope_, + move (n), + ph2 ? ph2 : bf ? optional<string> (string ()) : nullopt, + opt, + meta, + loc)); + + names& r (ir.name); if (val != nullptr) { @@ -2660,17 +4523,87 @@ namespace build2 } else { - if (atype == type::assign) - val->assign (move (r), var); - else if (atype == type::prepend) - val->prepend (move (r), var); - else - val->append (move (r), var); + // Import (more precisely, alias) the target type into this project + // if not known. + // + // Note that if the result is ignored (val is NULL), then it's fair + // to assume this is not necessary. + // + if (const scope* iroot = ir.target) + { + const name& n (r.front ()); + if (n.typed ()) + import_target_type (*root_, *iroot, n.type, loc); + } + + if (atype == type::assign) val->assign (move (r), var); + else if (atype == type::prepend) val->prepend (move (r), var); + else val->append (move (r), var); } if (atype == type::assign) atype = type::append; // Append subsequent values. } + else + { + assert (bf); + + if (r.empty ()) // Optional not found. + { + assert (opt); + continue; + } + + // Note: see also import_buildfile(). + // + assert (r.size () == 1); // See import_load() for details. + name& n (r.front ()); + path p (n.dir / n.value); // Should already include extension. + + // Note: similar to parse_include(). + // + // Nuance: we insert this buildfile even with once=false in case it + // gets imported with once=true from another place. + // + if (!root_->root_extra->insert_buildfile (p) && once) + { + l5 ([&]{trace (loc) << "skipping already imported " << p;}); + continue; + } + + // Clear/restore if/switch location. + // + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = nullopt; + + try + { + ifdstream ifs (p); + + auto df = make_diag_frame ( + [this, &p, &loc] (const diag_record& dr) + { + dr << info (loc) << p << " imported from here"; + }); + + // @@ Do we want to enter this buildfile? What's the harm (one + // benefit is that it will be in dump). But, we currently don't + // out-qualify them, though feels like there is nothing fatal + // in that, just inaccurate. + // + source_buildfile (ifs, + path_name (p), + loc, + nodt ? optional<bool> {} : false); + } + catch (const io_error& e) + { + fail (loc) << "unable to read imported buildfile " << p << ": " << e; + } + } } next_after_newline (t, tt); @@ -2712,7 +4645,12 @@ namespace build2 fail (l) << "null value in export"; if (val.type != nullptr) - untypify (val); + { + // While feels far-fetched, let's preserve empty typed values in the + // result. + // + untypify (val, false /* reduce */); + } export_value = move (val).as<names> (); @@ -2752,6 +4690,9 @@ namespace build2 n = move (i->value); + if (n[0] == '_') + fail (l) << "module name '" << n << "' starts with underscore"; + if (i->pair) try { @@ -2796,41 +4737,160 @@ namespace build2 void parser:: parse_define (token& t, type& tt) { - // define <derived>: <base> + // define [<attrs>] <derived>: <base> + // define <alias> = <scope>/<type> // // See tests/define. // - if (next (t, tt) != type::word) - fail (t) << "expected name instead of " << t << " in target type " - << "definition"; + next_with_attributes (t, tt); - string dn (move (t.value)); - const location dnl (get_location (t)); + attributes_push (t, tt); + attributes as (attributes_pop ()); - if (next (t, tt) != type::colon) - fail (t) << "expected ':' instead of " << t << " in target type " + if (tt != type::word) + fail (t) << "expected name instead of " << t << " in target type " << "definition"; + string n (move (t.value)); + const location nl (get_location (t)); + next (t, tt); - if (tt == type::word) + if (tt == type::colon) { + // Handle attributes. + // + target_type::flag fs (target_type::flag::none); + { + const location& l (as.loc); + + for (attribute& a: as) + { + const string& n (a.name); + value& v (a.value); + + if (n == "see_through") fs |= target_type::flag::see_through; + else if (n == "member_hint") fs |= target_type::flag::member_hint; + else + fail (l) << "unknown target type definition attribute " << n; + + if (!v.null) + fail (l) << "unexpected value in attribute " << n; + } + } + + if (next (t, tt) != type::word) + fail (t) << "expected name instead of " << t << " in target type " + << "definition"; + // Target. // const string& bn (t.value); const target_type* bt (scope_->find_target_type (bn)); if (bt == nullptr) - fail (t) << "unknown target type " << bn; + fail (t) << "unknown target type " << bn << + info << "perhaps the module that defines this target type is " + << "not loaded by project " << *scope_->root_scope (); - if (!root_->derive_target_type (move (dn), *bt).second) - fail (dnl) << "target type " << dn << " already defined in this " - << "project"; + // The derive_target_type() call below does not produce a non-abstract + // type if passed an abstract base. So we ban this for now (it's unclear + // why would someone want to do this). + // + if (bt->factory == nullptr) + fail (t) << "abstract base target type " << bt->name << "{}"; + + // Note that the group{foo}<...> syntax is only recognized for group- + // based targets and ad hoc buildscript recipes/rules only match group. + // (We may want to relax this for member_hint in the future since its + // currently also used on non-mtime-based targets, though what exactly + // we will do in ad hoc recipes/rules in this case is fuzzy). + // + if ((fs & target_type::flag::group) == target_type::flag::group && + !bt->is_a<group> ()) + fail (t) << "base target type " << bn << " must be group for " + << "group-related attribute"; + + if (!root_->derive_target_type (move (n), *bt, fs).second) + fail (nl) << "target type " << n << " already defined in project " + << *root_; next (t, tt); // Get newline. } + else if (tt == type::assign) + { + if (!as.empty ()) + fail (as.loc) << "unexpected target type alias attribute"; + + // The rest should be a path-like target type. Parse it as names in + // the value mode to get variable expansion, etc. + // + mode (lexer_mode::value, '@'); + next (t, tt); + const location tl (get_location (t)); + names ns ( + parse_names (t, tt, pattern_mode::ignore, "target type", nullptr)); + + name* tn (nullptr); + if (ns.size () == 1) + { + tn = &ns.front (); + + if (tn->file ()) + { + try + { + tn->canonicalize (); + + if (tn->dir.absolute ()) + tn->dir.normalize (); + else + tn = nullptr; + } + catch (const invalid_path&) {tn = nullptr;} + catch (const invalid_argument&) {tn = nullptr;} + } + else + tn = nullptr; + } + + if (tn == nullptr) + fail (tl) << "expected scope-qualified target type instead of " << ns; + + // If we got here, then tn->dir is the scope and tn->value is the target + // type. + // + // NOTE: see similar code in import_target_type(). + // + const target_type* tt (nullptr); + if (const scope* rs = ctx->scopes.find_out (tn->dir).root_scope ()) + { + tt = rs->find_target_type (tn->value); + + if (tt == nullptr) + fail (tl) << "unknown target type " << tn->value << " in scope " + << *rs; + } + else + fail (tl) << "unknown project scope " << tn->dir << " in scope" + << "-qualified target type" << + info << "did you forget to import the corresponding project?"; + + if (n != tn->value) + fail (nl) << "alias target type name " << n << " does not match " + << tn->value; + + // Note that this is potentially a shallow reference to a user-derived + // target type. Seeing that we only ever destory the entire graph, this + // should be ok. + // + auto p (root_->root_extra->target_types.insert (*tt)); + + if (!p.second && &p.first.get () != tt) + fail (nl) << "target type " << n << " already defined in this project"; + } else - fail (t) << "expected name instead of " << t << " in target type " + fail (t) << "expected ':' or '=' instead of " << t << " in target type " << "definition"; next_after_newline (t, tt); @@ -2839,19 +4899,28 @@ namespace build2 void parser:: parse_if_else (token& t, type& tt) { + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = get_location (t); + parse_if_else (t, tt, false /* multi */, [this] (token& t, type& tt, bool s, const string& k) { return parse_clause_block (t, tt, s, k); - }); + }, + {}); } void parser:: parse_if_else (token& t, type& tt, bool multi, const function<void ( - token&, type&, bool, const string&)>& parse_block) + token&, type&, bool, const string&)>& parse_block, + const function<void ( + token&, token_type&, const string&)>& parse_recipe_directive) { // Handle the whole if-else chain. See tests/if-else. // @@ -2876,7 +4945,7 @@ namespace build2 // is not an option. So let's skip it. // if (taken) - skip_line (t, tt); + skip_line (t, tt); // Skip expression. else { if (tt == type::newline || tt == type::eos) @@ -2936,31 +5005,65 @@ namespace build2 parse_block (t, tt, !take, k); taken = taken || take; } - else if (!multi) // No lines in multi-curly if-else. + else { - if (take) + // The only valid line in multi-curly if-else is `recipe`. + // + if (multi) { - if (!parse_clause (t, tt, true)) - fail (t) << "expected " << k << "-line instead of " << t; + // Note that we cannot do the keyword test if we are replaying. So + // we skip it with the understanding that if it's not a keywords, + // then we wouldn't have gotten here on the replay. + // + if (tt == type::word && + (replay_ == replay::play || keyword (t)) && + t.value == "recipe") + { + if (take) + { + parse_recipe_directive (t, tt, k); + taken = true; + } + else + { + skip_line (t, tt); - taken = true; + if (tt == type::newline) + next (t, tt); + } + } + else + fail (t) << "expected " << k << "-block or 'recipe' instead of " + << t; } else { - skip_line (t, tt); + if (tt == type::multi_lcbrace) + fail (t) << "expected " << k << "-line instead of " << t << + info << "did you forget to specify % recipe header?"; - if (tt == type::newline) - next (t, tt); + if (take) + { + if (!parse_clause (t, tt, true)) + fail (t) << "expected " << k << "-line instead of " << t; + + taken = true; + } + else + { + skip_line (t, tt); + + if (tt == type::newline) + next (t, tt); + } } } - else - fail (t) << "expected " << k << "-block instead of " << t; // See if we have another el* keyword. // // Note that we cannot do the keyword test if we are replaying. So we // skip it with the understanding that if it's not a keywords, then we - // wouldn't have gotten here on the reply (see parse_recipe() for + // wouldn't have gotten here on the replay (see parse_recipe() for // details). // if (k != "else" && @@ -2980,19 +5083,28 @@ namespace build2 void parser:: parse_switch (token& t, type& tt) { + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = get_location (t); + parse_switch (t, tt, false /* multi */, [this] (token& t, type& tt, bool s, const string& k) { return parse_clause_block (t, tt, s, k); - }); + }, + {}); } void parser:: parse_switch (token& t, type& tt, bool multi, const function<void ( - token&, type&, bool, const string&)>& parse_block) + token&, type&, bool, const string&)>& parse_block, + const function<void ( + token&, token_type&, const string&)>& parse_recipe_directive) { // switch <value> [: <func> [<arg>]] [, <value>...] // { @@ -3049,12 +5161,12 @@ namespace build2 { next (t, tt); const location l (get_location (t)); - names ns (parse_names (t, tt, pattern_mode::ignore, "function name")); + names ns (parse_names (t, tt, pattern_mode::preserve, "function name")); if (ns.empty () || ns[0].empty ()) fail (l) << "function name expected after ':'"; - if (!ns[0].simple ()) + if (ns[0].pattern || !ns[0].simple ()) fail (l) << "function name expected instead of " << ns[0]; e.func = move (ns[0].value); @@ -3087,7 +5199,7 @@ namespace build2 { // Note that we cannot do the keyword test if we are replaying. So we // skip it with the understanding that if it's not a keywords, then we - // wouldn't have gotten here on the reply (see parse_recipe() for + // wouldn't have gotten here on the replay (see parse_recipe() for // details). Note that this appears to mean that replay cannot be used // if we allow lines, only blocks. Consider: // @@ -3190,7 +5302,7 @@ namespace build2 if (!e.arg.empty ()) args.push_back (value (e.arg)); - value r (ctx.functions.call (scope_, *e.func, args, l)); + value r (ctx->functions.call (scope_, *e.func, args, l)); // We support two types of functions: matchers and extractors: // a matcher returns a statically-typed bool value while an @@ -3293,25 +5405,49 @@ namespace build2 parse_block (t, tt, !take, k); taken = taken || take; } - else if (!multi) // No lines in multi-curly if-else. + else { - if (take) + if (multi) { - if (!parse_clause (t, tt, true)) - fail (t) << "expected " << k << "-line instead of " << t; + if (tt == type::word && + (replay_ == replay::play || keyword (t)) && + t.value == "recipe") + { + if (take) + { + parse_recipe_directive (t, tt, k); + taken = true; + } + else + { + skip_line (t, tt); - taken = true; + if (tt == type::newline) + next (t, tt); + } + } + else + fail (t) << "expected " << k << "-block or 'recipe' instead of " + << t; } else { - skip_line (t, tt); + if (take) + { + if (!parse_clause (t, tt, true)) + fail (t) << "expected " << k << "-line instead of " << t; - if (tt == type::newline) - next (t, tt); + taken = true; + } + else + { + skip_line (t, tt); + + if (tt == type::newline) + next (t, tt); + } } } - else - fail (t) << "expected " << k << "-block instead of " << t; } if (tt != type::rcbrace) @@ -3324,10 +5460,10 @@ namespace build2 void parser:: parse_for (token& t, type& tt) { - // for <varname>: <value> + // for [<var-attrs>] <varname> [<elem-attrs>]: [<val-attrs>] <value> // <line> // - // for <varname>: <value> + // for [<var-attrs>] <varname> [<elem-attrs>]: [<val-attrs>] <value> // { // <block> // } @@ -3338,13 +5474,12 @@ namespace build2 next_with_attributes (t, tt); attributes_push (t, tt); - // @@ PAT: currently we pattern-expand for var. + // Enable list element attributes. // - const location vloc (get_location (t)); - names vns (parse_names (t, tt, pattern_mode::expand)); + enable_attributes (); - if (tt != type::colon) - fail (t) << "expected ':' instead of " << t << " after variable name"; + const location vloc (get_location (t)); + names vns (parse_names (t, tt, pattern_mode::preserve)); const variable& var (parse_variable_name (move (vns), vloc)); apply_variable_attributes (var); @@ -3355,6 +5490,17 @@ namespace build2 << " visibility but is assigned in for-loop"; } + // Parse the list element attributes, if present. + // + attributes_push (t, tt); + + if (tt != type::colon) + fail (t) << "expected ':' instead of " << t << " after variable name"; + + // Save element attributes so that we can inject them on each iteration. + // + attributes val_attrs (attributes_pop ()); + // Now the value (list of names) to iterate over. Parse it similar to a // value on the RHS of an assignment (expansion, attributes). // @@ -3363,15 +5509,24 @@ namespace build2 value val (parse_value_with_attributes (t, tt, pattern_mode::expand)); - // If this value is a vector, then save its element type so that we + // If the value type provides custom iterate function, then use that (see + // value_type::iterate for details). + // + auto iterate (val.type != nullptr ? val.type->iterate : nullptr); + + // If this value is a container, then save its element type so that we // can typify each element below. // const value_type* etype (nullptr); - if (val && val.type != nullptr) + if (!iterate && val && val.type != nullptr) { etype = val.type->element_type; - untypify (val); + + // Note that here we don't want to be reducing empty simple values to + // empty lists. + // + untypify (val, false /* reduce */); } if (tt != type::newline) @@ -3419,32 +5574,50 @@ namespace build2 // Iterate. // - value& v (scope_->assign (var)); // Assign even if no iterations. + value& lhs (scope_->assign (var)); // Assign even if no iterations. if (!val) return; - names& ns (val.as<names> ()); - - if (ns.empty ()) - return; + names* ns (nullptr); + if (!iterate) + { + ns = &val.as<names> (); + if (ns->empty ()) + return; + } istringstream is (move (body)); - for (auto i (ns.begin ()), e (ns.end ());; ) + struct data + { + const variable& var; + const attributes& val_attrs; + uint64_t line; + bool block; + value& lhs; + istringstream& is; + + } d {var, val_attrs, line, block, lhs, is}; + + function<void (value&&, bool first)> iteration = + [this, &d] (value&& v, bool first) { - // Set the variable value. + // Rewind the stream. // - bool pair (i->pair); - names n; - n.push_back (move (*i)); - if (pair) n.push_back (move (*++i)); - v = value (move (n)); + if (!first) + { + d.is.clear (); + d.is.seekg (0); + } + + // Inject element attributes. + // + attributes_.push_back (d.val_attrs); - if (etype != nullptr) - typify (v, *etype, &var); + apply_value_attributes (&d.var, d.lhs, move (v), type::assign); - lexer l (is, *path_, line); + lexer l (d.is, *path_, d.line); lexer* ol (lexer_); lexer_ = &l; @@ -3452,7 +5625,7 @@ namespace build2 type tt; next (t, tt); - if (block) + if (d.block) { next (t, tt); // { next (t, tt); // <newline> @@ -3460,20 +5633,33 @@ namespace build2 parse_clause (t, tt); - if (tt != (block ? type::rcbrace : type::eos)) - fail (t) << "expected name " << (block ? "or '}' " : "") + if (tt != (d.block ? type::rcbrace : type::eos)) + fail (t) << "expected name " << (d.block ? "or '}' " : "") << "instead of " << t; lexer_ = ol; + }; - if (++i == e) - break; + if (!iterate) + { + for (auto b (ns->begin ()), i (b), e (ns->end ()); i != e; ++i) + { + // Set the variable value. + // + bool pair (i->pair); + names n; + n.push_back (move (*i)); + if (pair) n.push_back (move (*++i)); + value v (move (n)); - // Rewind the stream. - // - is.clear (); - is.seekg (0); + if (etype != nullptr) + typify (v, *etype, &var); + + iteration (move (v), i == b); + } } + else + iterate (val, iteration); } void parser:: @@ -3546,7 +5732,7 @@ namespace build2 if (value v = parse_value_with_attributes (t, tt, pattern_mode::expand)) { names storage; - cout << reverse (v, storage) << endl; + cout << reverse (v, storage, true /* reduce */) << endl; } else cout << "[null]" << endl; @@ -3579,7 +5765,7 @@ namespace build2 if (value v = parse_value_with_attributes (t, tt, pattern_mode::expand)) { names storage; - dr << reverse (v, storage); + dr << reverse (v, storage, true /* reduce */); } if (tt != type::eos) @@ -3598,7 +5784,7 @@ namespace build2 const location l (get_location (t)); next (t, tt); names ns (tt != type::newline && tt != type::eos - ? parse_names (t, tt, pattern_mode::ignore) + ? parse_names (t, tt, pattern_mode::preserve) : names ()); text (l) << "dump:"; @@ -3609,8 +5795,10 @@ namespace build2 if (ns.empty ()) { + // Indent two spaces. + // if (scope_ != nullptr) - dump (*scope_, " "); // Indent two spaces. + dump (scope_, nullopt /* action */, dump_format::buildfile, " "); else os << " <no current scope>" << endl; } @@ -3621,10 +5809,17 @@ namespace build2 name& n (*i++); name o (n.pair ? move (*i++) : name ()); + // @@ TODO + // + if (n.pattern) + fail (l) << "dumping target patterns no yet supported"; + const target* t (enter_target::find_target (*this, n, o, l, trace)); + // Indent two spaces. + // if (t != nullptr) - dump (*t, " "); // Indent two spaces. + dump (t, nullopt /* action */, dump_format::buildfile, " "); else { os << " <no target " << n; @@ -3642,20 +5837,59 @@ namespace build2 } const variable& parser:: + parse_variable_name (string&& on, const location& l) + { + // Enter a variable name for assignment (as opposed to lookup). + + // If the variable is qualified (and thus public), make it overridable. + // + // Note that the overridability can still be restricted (e.g., by a module + // that enters this variable or by a pattern). + // + bool ovr (on.find ('.') != string::npos); + auto r (scope_->var_pool ().insert (move (on), nullptr, nullptr, &ovr)); + + if (!r.second) + return r.first; + + // If it's newly entered, verify it's not reserved for the build2 core. + // We reserve: + // + // - Variable components that start with underscore (_x, x._y). + // + // - Variables in the `build`, `import`, and `export` namespaces. + // + const string& n (r.first.name); + + const char* w ( + n[0] == '_' ? "name starts with underscore" : + n.find ("._") != string::npos ? "component starts with underscore" : + n.compare (0, 6, "build.") == 0 ? "is in 'build' namespace" : + n.compare (0, 7, "import.") == 0 ? "is in 'import' namespace" : + n.compare (0, 7, "export.") == 0 ? "is in 'export' namespace" : nullptr); + + if (w != nullptr) + fail (l) << "variable name '" << n << "' is reserved" << + info << "variable " << w; + + return r.first; + } + + const variable& parser:: parse_variable_name (names&& ns, const location& l) { // Parse and enter a variable name for assignment (as opposed to lookup). - // The list should contain a single, simple name. + // The list should contain a single, simple name. Go an extra mile to + // issue less confusing diagnostics. // - if (ns.size () != 1 || !ns[0].simple () || ns[0].empty ()) + size_t n (ns.size ()); + if (n == 0 || (n == 1 && ns[0].empty ())) + fail (l) << "empty variable name"; + else if (n != 1 || ns[0].pattern || !ns[0].simple ()) fail (l) << "expected variable name instead of " << ns; - // Note that the overridability can still be restricted (e.g., by a module - // that enters this variable or by a pattern). - // - return scope_->var_pool ().insert ( - move (ns[0].value), true /* overridable */); + return parse_variable_name (move (ns[0].value), l); } void parser:: @@ -3688,31 +5922,46 @@ namespace build2 } void parser:: - parse_type_pattern_variable (token& t, token_type& tt, - const target_type& type, string pat, - const variable& var, token_type kind, - const location& loc) + parse_type_pattern_variable ( + token& t, token_type& tt, + pattern_type pt, const target_type& ptt, string pat, const location& ploc, + const variable& var, token_type kind, const location& loc) { // Parse target type/pattern-specific variable assignment. // - // See old-tests/variable/type-pattern. // Note: expanding the value in the current scope context. // value rhs (parse_variable_value (t, tt)); - // Leave the value untyped unless we are assigning. - // - pair<reference_wrapper<value>, bool> p ( - scope_->target_vars[type][move (pat)].insert ( - var, kind == type::assign)); + pair<reference_wrapper<value>, bool> p (rhs /* dummy */, false); + try + { + // Leave the value untyped unless we are assigning. + // + // Note that the pattern is preserved if insert fails with regex_error. + // + p = scope_->target_vars[ptt].insert (pt, move (pat)).insert ( + var, kind == type::assign, false /* reset_extra */); + } + catch (const regex_error& e) + { + // Print regex_error description if meaningful (no space). + // + fail (ploc) << "invalid regex pattern '" << pat << "'" << e; + } value& lhs (p.first); // We store prepend/append values untyped (similar to overrides). // if (rhs.type != nullptr && kind != type::assign) - untypify (rhs); + { + // Our heuristics for prepend/append of a typed value is to preserve + // empty (see apply_value_attributes() for details) so do not reduce. + // + untypify (rhs, false /* reduce */); + } if (p.second) { @@ -3779,10 +6028,15 @@ namespace build2 } value parser:: - parse_variable_value (token& t, type& tt) + parse_variable_value (token& t, type& tt, bool m) { - mode (lexer_mode::value, '@'); - next_with_attributes (t, tt); + if (m) + { + mode (lexer_mode::value, '@'); + next_with_attributes (t, tt); + } + else + next (t, tt); // Parse value attributes if any. Note that it's ok not to have anything // after the attributes (e.g., foo=[null]). @@ -3794,32 +6048,119 @@ namespace build2 : value (names ()); } - static const value_type* - map_type (const string& n) + const value_type* parser:: + find_value_type (const scope*, const string& n) { - auto ptr = [] (const value_type& vt) {return &vt;}; - - return - n == "bool" ? ptr (value_traits<bool>::value_type) : - n == "int64" ? ptr (value_traits<int64_t>::value_type) : - n == "uint64" ? ptr (value_traits<uint64_t>::value_type) : - n == "string" ? ptr (value_traits<string>::value_type) : - n == "path" ? ptr (value_traits<path>::value_type) : - n == "dir_path" ? ptr (value_traits<dir_path>::value_type) : - n == "abs_dir_path" ? ptr (value_traits<abs_dir_path>::value_type) : - n == "name" ? ptr (value_traits<name>::value_type) : - n == "name_pair" ? ptr (value_traits<name_pair>::value_type) : - n == "target_triplet" ? ptr (value_traits<target_triplet>::value_type) : - n == "project_name" ? ptr (value_traits<project_name>::value_type) : - - n == "int64s" ? ptr (value_traits<int64s>::value_type) : - n == "uint64s" ? ptr (value_traits<uint64s>::value_type) : - n == "strings" ? ptr (value_traits<strings>::value_type) : - n == "paths" ? ptr (value_traits<paths>::value_type) : - n == "dir_paths" ? ptr (value_traits<dir_paths>::value_type) : - n == "names" ? ptr (value_traits<vector<name>>::value_type) : - - nullptr; + switch (n[0]) + { + case 'a': + { + if (n == "abs_dir_path") return &value_traits<abs_dir_path>::value_type; + break; + } + case 'b': + { + if (n == "bool") return &value_traits<bool>::value_type; + break; + } + case 'c': + { + if (n == "cmdline") return &value_traits<cmdline>::value_type; + break; + } + case 'd': + { + if (n.compare (0, 8, "dir_path") == 0) + { + if (n[8] == '\0') return &value_traits<dir_path>::value_type; + if (n[8] == 's' && + n[9] == '\0') return &value_traits<dir_paths>::value_type; + } + break; + } + case 'i': + { + if (n.compare (0, 5, "int64") == 0) + { + if (n[5] == '\0') return &value_traits<int64_t>::value_type; + if (n[5] == 's' && + n[6] == '\0') return &value_traits<int64s>::value_type; + } + break; + } + case 'j': + { + if (n.compare (0, 4, "json") == 0) + { + if (n[4] == '\0') return &value_traits<json_value>::value_type; + if (n == "json_array") return &value_traits<json_array>::value_type; + if (n == "json_object") + return &value_traits<json_object>::value_type; + if (n == "json_set") + return &value_traits<set<json_value>>::value_type; + if (n == "json_map") + return &value_traits<map<json_value, json_value>>::value_type; + } + break; + } + case 'n': + { + if (n.compare (0, 4, "name") == 0) + { + if (n[4] == '\0') return &value_traits<name>::value_type; + if (n[4] == 's' && + n[5] == '\0') return &value_traits<vector<name>>::value_type; + if (n == "name_pair") return &value_traits<name_pair>::value_type; + } + break; + } + + case 'p': + { + if (n.compare (0, 4, "path") == 0) + { + if (n[4] == '\0') return &value_traits<path>::value_type; + if (n[4] == 's' && + n[5] == '\0') return &value_traits<paths>::value_type; + } + else if (n == "project_name") + return &value_traits<project_name>::value_type; + break; + } + case 's': + { + if (n.compare (0, 6, "string") == 0) + { + if (n[6] == '\0') return &value_traits<string>::value_type; + if (n[6] == 's' && + n[7] == '\0') return &value_traits<strings>::value_type; + if (n == "string_set") return &value_traits<set<string>>::value_type; + if (n == "string_map") + return &value_traits<map<string,string>>::value_type; + } + break; + } + case 't': + { + if (n == "target_triplet") + return &value_traits<target_triplet>::value_type; + break; + } + case 'u': + { + if (n.compare (0, 6, "uint64") == 0) + { + if (n[6] == '\0') return &value_traits<uint64_t>::value_type; + if (n[6] == 's' && + n[7] == '\0') return &value_traits<uint64s>::value_type; + } + break; + } + default: + break; + } + + return nullptr; } void parser:: @@ -3841,19 +6182,62 @@ namespace build2 string& n (a.name); value& v (a.value); - if (const value_type* t = map_type (n)) + if (n == "visibility") { + try + { + string s (convert<string> (move (v))); + + variable_visibility r; + if (s == "global") r = variable_visibility::global; + else if (s == "project") r = variable_visibility::project; + else if (s == "scope") r = variable_visibility::scope; + else if (s == "target") r = variable_visibility::target; + else if (s == "prerequisite") r = variable_visibility::prereq; + else throw invalid_argument ("unknown visibility name"); + + if (vis && r != *vis) + fail (l) << "conflicting variable visibilities: " << s << ", " + << *vis; + + vis = r; + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else if (n == "overridable") + { + try + { + // Treat absent value (represented as NULL) as true. + // + bool r (v.null || convert<bool> (move (v))); + + if (ovr && r != *ovr) + fail (l) << "conflicting variable overridabilities"; + + ovr = r; + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else if (const value_type* t = find_value_type (root_, n)) + { + if (!v.null) + fail (l) << "unexpected value in attribute " << a; + if (type != nullptr && t != type) - fail (l) << "multiple variable types: " << n << ", " << type->name; + fail (l) << "conflicting variable types: " << n << ", " + << type->name; type = t; - // Fall through. } else fail (l) << "unknown variable attribute " << a; - - if (!v.null) - fail (l) << "unexpected value in attribute " << a; } if (type != nullptr && var.type != nullptr) @@ -3865,15 +6249,33 @@ namespace build2 << var.type->name << " to " << type->name; } - //@@ TODO: the same checks for vis and ovr (when we have the corresponding - // attributes). + if (vis) + { + // Note that this logic naturally makes sure that a project-private + // variable doesn't have global visibility (since it would have been + // entered with the project visibility). + // + if (var.visibility == *vis) + vis = nullopt; + else if (var.visibility > *vis) // See variable_pool::update(). + fail (l) << "changing variable " << var << " visibility from " + << var.visibility << " to " << *vis; + } - if (type || vis || ovr) - ctx.var_pool.update (const_cast<variable&> (var), - type, - vis ? &*vis : nullptr, - ovr ? &*ovr : nullptr); + if (ovr) + { + // Note that the overridability incompatibilities are diagnosed by + // update(). So we just need to diagnose the project-private case. + // + if (*ovr && var.owner != &ctx->var_pool) + fail (l) << "private variable " << var << " cannot be overridable"; + } + if (type || vis || ovr) + var.owner->update (const_cast<variable&> (var), + type, + vis ? &*vis : nullptr, + ovr ? &*ovr : nullptr); } void parser:: @@ -3883,7 +6285,7 @@ namespace build2 type kind) { attributes as (attributes_pop ()); - const location& l (as.loc); + const location& l (as.loc); // This points to value if no attributes. // Essentially this is an attribute-augmented assign/append/prepend. // @@ -3897,16 +6299,18 @@ namespace build2 if (n == "null") { + // @@ Looks like here we assume representationally empty? + // if (rhs && !rhs.empty ()) // Note: null means we had an expansion. fail (l) << "value with null attribute"; null = true; // Fall through. } - else if (const value_type* t = map_type (n)) + else if (const value_type* t = find_value_type (root_, n)) { if (type != nullptr && t != type) - fail (l) << "multiple value types: " << n << ", " << type->name; + fail (l) << "conflicting value types: " << n << ", " << type->name; type = t; // Fall through. @@ -3954,6 +6358,13 @@ namespace build2 bool rhs_type (false); if (rhs.type != nullptr) { + // Our heuristics is to not reduce typed RHS empty simple values for + // prepend/append and additionally for assign provided LHS is a + // container. + // + bool reduce (kind == type::assign && + (type == nullptr || !type->container)); + // Only consider RHS type if there is no explicit or variable type. // if (type == nullptr) @@ -3964,7 +6375,7 @@ namespace build2 // Reduce this to the untyped value case for simplicity. // - untypify (rhs); + untypify (rhs, reduce); } if (kind == type::assign) @@ -3993,6 +6404,17 @@ namespace build2 } else { + auto df = make_diag_frame ( + [this, var, &l](const diag_record& dr) + { + if (!l.empty ()) + { + dr << info (l); + if (var != nullptr) dr << "variable " << var->name << ' '; + dr << "value is assigned here"; + } + }); + if (kind == type::assign) { if (rhs) @@ -4339,7 +6761,7 @@ namespace build2 const location nl (get_location (t)); next (t, tt); - value n (parse_value (t, tt, pattern_mode::ignore)); + value n (parse_value (t, tt, pattern_mode::preserve)); if (tt != type::rparen) fail (t) << "expected ')' after variable name"; @@ -4347,16 +6769,38 @@ namespace build2 if (pre_parse_) return v; // Empty. - if (v.type != nullptr || !v || v.as<names> ().size () != 1) - fail (l) << "expected target before ':'"; - - if (n.type != nullptr || !n || n.as<names> ().size () != 1) + // We used to return this as a <target>:<name> pair but that meant we + // could not handle an out-qualified target (which is represented as + // <target>@<out> pair). As a somewhat of a hack, we deal with this by + // changing the order of the name and target to be <name>:<target> with + // the qualified case becoming a "tripple pair" <name>:<target>@<out>. + // + // @@ This is actually not great since it's possible to observe such a + // tripple pair, for example with `print (file{x}@./:y)`. + // + if (n.type != nullptr || !n || n.as<names> ().size () != 1 || + n.as<names> ()[0].pattern) fail (nl) << "expected variable name after ':'"; - names& ns (v.as<names> ()); + names& ns (n.as<names> ()); ns.back ().pair = ':'; - ns.push_back (move (n.as<names> ().back ())); - return v; + + if (v.type == nullptr && v) + { + names& ts (v.as<names> ()); + + size_t s (ts.size ()); + if (s == 1 || (s == 2 && ts.front ().pair == '@')) + { + ns.push_back (move (ts.front ())); + if (s == 2) + ns.push_back (move (ts.back ())); + + return n; + } + } + + fail (l) << "expected target before ':'" << endf; } else { @@ -4425,8 +6869,13 @@ namespace build2 } pair<bool, location> parser:: - attributes_push (token& t, type& tt, bool standalone) + attributes_push (token& t, type& tt, bool standalone, bool next_token) { + // To make sure that the attributes are not standalone we need to read the + // token which follows ']'. + // + assert (standalone || next_token); + location l (get_location (t)); bool has (tt == type::lsbrace); @@ -4449,6 +6898,10 @@ namespace build2 // Parse the attribute name with expansion (we rely on this in some // old and hairy tests). // + // Note that the attributes lexer mode does not recognize `{}@` as + // special and we rely on that in the rule hint attributes + // (libs@rule_hint=cxx). + // const location l (get_location (t)); names ns ( @@ -4490,32 +6943,33 @@ namespace build2 } while (tt != type::rsbrace); } + else + has = false; // `[]` doesn't count. if (tt != type::rsbrace) fail (t) << "expected ']' instead of " << t; - next (t, tt); - - if (tt == type::newline || tt == type::eos) + if (next_token) { - if (!standalone) - fail (t) << "standalone attributes"; + next (t, tt); + + if (tt == type::newline || tt == type::eos) + { + if (!standalone) + fail (t) << "standalone attributes"; + } + // + // Verify that the attributes are separated from the following word or + // "word-producing" token. + // + else if (!t.separated && (tt == type::word || + tt == type::dollar || + tt == type::lparen || + tt == type::lcbrace)) + fail (t) << "whitespace required after attributes" << + info (l) << "use the '\\[' escape sequence if this is a wildcard " + << "pattern"; } - // - // We require attributes to be separated from the following word or - // "word-producing" tokens (`$` for variable expansions/function calls, - // `(` for eval contexts, and `{` for name generation) to reduce the - // possibility of confusing them with wildcard patterns. Consider: - // - // ./: [abc]-foo.txt - // - else if (!t.separated && (tt == type::word || - tt == type::dollar || - tt == type::lparen || - tt == type::lcbrace)) - fail (t) << "whitespace required after attributes" << - info (l) << "use the '\\[' escape sequence if this is a wildcard " - << "pattern"; return make_pair (has, l); } @@ -4524,7 +6978,11 @@ namespace build2 // static inline name& append_name (names& ns, - optional<project_name> p, dir_path d, string t, string v, + optional<project_name> p, + dir_path d, + string t, + string v, + optional<name::pattern_type> pat, const location& loc) { // The directory/value must not be empty if we have a type. @@ -4532,7 +6990,7 @@ namespace build2 if (d.empty () && v.empty () && !t.empty ()) fail (loc) << "typed empty name"; - ns.emplace_back (move (p), move (d), move (t), move (v)); + ns.emplace_back (move (p), move (d), move (t), move (v), pat); return ns.back (); } @@ -4639,7 +7097,10 @@ namespace build2 ns.push_back (ns[pairn - 1]); } - name& r (append_name (ns, move (p), move (d), move (t), move (v), loc)); + name& r ( + append_name (ns, + move (p), move (d), move (t), move (v), cn.pattern, + loc)); r.pair = cn.pair; } @@ -4743,9 +7204,11 @@ namespace build2 // May throw invalid_path. // auto include_pattern = - [&r, &append, &include_match, sp, &l, this] (string&& p, - optional<string>&& e, - bool a) + [this, + &append, &include_match, + &r, sp, &l, &dir] (string&& p, + optional<string>&& e, + bool a) { // If we don't already have any matches and our pattern doesn't contain // multiple recursive wildcards, then the result will be unique and we @@ -4754,21 +7217,26 @@ namespace build2 // bool unique (r.empty () && path_pattern_recursive (path (p)) <= 1); - function<void (string&&, optional<string>&&)> appf; + struct data + { + const optional<string>& e; + const dir_path& sp; + function<void (string&&, optional<string>&&)> appf; + + } d {e, *sp, nullptr}; + if (unique) - appf = [a, &append] (string&& v, optional<string>&& e) + d.appf = [a, &append] (string&& v, optional<string>&& e) { append (move (v), move (e), a); }; else - appf = [a, &include_match] (string&& v, optional<string>&& e) + d.appf = [a, &include_match] (string&& v, optional<string>&& e) { include_match (move (v), move (e), a); }; - auto process = [this, &e, &appf, sp] (path&& m, - const string& p, - bool interm) + auto process = [&d, this] (path&& m, const string& p, bool interm) { // Ignore entries that start with a dot unless the pattern that // matched them also starts with a dot. Also ignore directories @@ -4780,21 +7248,69 @@ namespace build2 (root_ != nullptr && root_->root_extra != nullptr && m.to_directory () && - exists (*sp / m / root_->root_extra->buildignore_file))) + exists (d.sp / m / root_->root_extra->buildignore_file))) return !interm; // Note that we have to make copies of the extension since there will // multiple entries for each pattern. // if (!interm) - appf (move (m).representation (), optional<string> (e)); + { + // If the extension is empty (meaning there should be no extension, + // for example hxx{Q*.}), skip entries with extensions. + // + if (!d.e || !d.e->empty () || m.extension_cstring () == nullptr) + d.appf (move (m).representation (), optional<string> (d.e)); + } return true; }; + const function<bool (const dir_entry&)> dangling ( + [&dir] (const dir_entry& de) + { + bool sl (de.ltype () == entry_type::symlink); + + const path& n (de.path ()); + + // One case where this turned out to be not worth it practically + // (too much noise) is the backlinks to executables (and the + // associated DLL assemblies for Windows). So we now have this + // heuristics that if this looks like an executable (or DLL for + // Windows), then we omit the warning. On POSIX, where executables + // don't have extensions, we will consider it an executable only if + // we are not looking for directories (which also normally don't + // have extension). + // + // @@ PEDANTIC: re-enable if --pedantic. + // + if (sl) + { + string e (n.extension ()); + + if ((e.empty () && !dir) || + path_traits::compare (e, "exe") == 0 || + path_traits::compare (e, "dll") == 0 || + path_traits::compare (e, "pdb") == 0 || // .{exe,dll}.pdb + (path_traits::compare (e, "dlls") == 0 && // .exe.dlls assembly + path_traits::compare (n.base ().extension (), "exe") == 0)) + return true; + } + + warn << "skipping " + << (sl ? "dangling symlink" : "inaccessible entry") + << ' ' << de.base () / n; + + return true; + }); + try { - path_search (path (move (p)), process, *sp); + path_search (path (move (p)), + process, + *sp, + path_match_flags::follow_symlinks, + dangling); } catch (const system_error& e) { @@ -4962,6 +7478,7 @@ namespace build2 if ((n.pair & 0x02) != 0) { e = move (n.type); + n.type.clear (); // Remove non-empty extension from the name (it got to be there, see // above). @@ -5010,7 +7527,7 @@ namespace build2 bool cross) { if (pp) - pmode = pattern_mode::ignore; + pmode = pattern_mode::preserve; next (t, tt); // Get what's after '{'. const location loc (get_location (t)); // Start of names. @@ -5036,7 +7553,7 @@ namespace build2 // This can be an ordinary name group or a pattern (with inclusions and // exclusions). We want to detect which one it is since for patterns we // want just the list of simple names without pair/dir/type added (those - // are added after the pattern expansion in parse_names_pattern()). + // are added after the pattern expansion in expand_name_pattern()). // // Detecting which one it is is tricky. We cannot just peek at the token // and look for some wildcards since the pattern can be the result of an @@ -5217,10 +7734,11 @@ namespace build2 tracer trace ("parser::parse_names", &path_); if (pp) - pmode = pattern_mode::ignore; + pmode = pattern_mode::preserve; // Returned value NULL/type and pattern (see below). // + bool rvalue (false); bool vnull (false); const value_type* vtype (nullptr); optional<const target_type*> rpat; @@ -5235,10 +7753,38 @@ namespace build2 // bool concat (false); bool concat_quoted (false); + bool concat_quoted_first (false); name concat_data; - auto concat_typed = [&vnull, &vtype, &concat, &concat_data, this] - (value&& rhs, const location& loc) + auto concat_diag_multiple = [this] (const location& loc, + const char* what_expansion) + { + diag_record dr (fail (loc)); + + dr << "concatenating " << what_expansion << " contains multiple values"; + + // See if this looks like a subscript without an evaluation context and + // help the user out. + // + if (mode () != lexer_mode::eval) + { + const token& t (peeked ()); // Should be peeked at. + + if (t.type == type::word && + t.qtype == quote_type::unquoted && + t.value[0] == '[') + { + dr << info << "wrap it in (...) evaluation context if this " + << "is value subscript"; + } + } + }; + + auto concat_typed = [this, what, &vnull, &vtype, + &concat, &concat_data, + &concat_diag_multiple] (value&& rhs, + const location& loc, + const char* what_expansion) { // If we have no LHS yet, then simply copy value/type. // @@ -5255,6 +7801,10 @@ namespace build2 // RHS. // + // Note that if RHS contains multiple values then we expect the result + // to be a single value somehow or, more likely, there to be no + // suitable $builtin.concat() overload. + // a.push_back (move (rhs)); const char* l ((a[0].type != nullptr ? a[0].type->name : "<untyped>")); @@ -5271,7 +7821,10 @@ namespace build2 dr << info << "use quoting to force untyped concatenation"; }); - p = ctx.functions.try_call ( + if (ctx == nullptr) + fail << "literal " << what << " expected"; + + p = ctx->functions.try_call ( scope_, "builtin.concat", vector_view<value> (a), loc); } @@ -5293,29 +7846,33 @@ namespace build2 if (!vnull) { if (vtype != nullptr) - untypify (rhs); + untypify (rhs, true /* reduce */); names& d (rhs.as<names> ()); - // If the value is empty, then untypify() will (typically; no pun - // intended) represent it as an empty sequence of names rather than - // a sequence of one empty name. This is usually what we need (see - // simple_reverse() for details) but not in this case. + // If the value is empty, then we asked untypify() to reduce it to + // an empty sequence of names rather than a sequence of one empty + // name. // - if (!d.empty ()) + if (size_t n = d.size ()) { - assert (d.size () == 1); // Must be a single value. + if (n != 1) + { + assert (what_expansion != nullptr); + concat_diag_multiple (loc, what_expansion); + } + concat_data = move (d[0]); } } }; - // Set the result pattern target type and switch to the ignore mode. + // Set the result pattern target type and switch to the preserve mode. // // The goal of the detect mode is to assemble the "raw" list (the pattern // itself plus inclusions/exclusions) that will then be passed to - // parse_names_pattern(). So clear pair, directory, and type (they will be - // added during pattern expansion) and change the mode to ignore (to + // expand_name_pattern(). So clear pair, directory, and type (they will be + // added during pattern expansion) and change the mode to preserve (to // prevent any expansions in inclusions/exclusions). // auto pattern_detected = @@ -5326,28 +7883,21 @@ namespace build2 pairn = 0; dp = nullptr; tp = nullptr; - pmode = pattern_mode::ignore; + pmode = pattern_mode::preserve; rpat = ttp; }; // Return '+' or '-' if a token can start an inclusion or exclusion // (pattern or group), '\0' otherwise. The result can be used as bool. - // - // @@ Note that we only need to make sure that the leading '+' or '-' - // characters are unquoted. We could consider some partially quoted - // tokens as starting inclusion or exclusion as well, for example - // +'foo*'. However, currently we can not determine which part of a - // token is quoted, and so can't distinguish the above token from - // '+'foo*. This is why we end up with a criteria that is stricter than - // is really required. + // Note that token::qfirst covers both quoting and escaping. // auto pattern_prefix = [] (const token& t) -> char { char c; - return t.type == type::word && ((c = t.value[0]) == '+' || c == '-') && - t.qtype == quote_type::unquoted - ? c - : '\0'; + return (t.type == type::word && !t.qfirst && + ((c = t.value[0]) == '+' || c == '-') + ? c + : '\0'); }; // A name sequence potentially starts with a pattern if it starts with a @@ -5418,6 +7968,8 @@ namespace build2 // continue accumulating or inject. We inject if the next token is not a // word, var expansion, or eval context or if it is separated. // + optional<pair<const value_type*, name>> path_concat; // Backup. + if (concat && last_concat ()) { // Concatenation does not affect the tokens we get, only what we do @@ -5427,9 +7979,11 @@ namespace build2 assert (!pre_parse_); bool quoted (concat_quoted); + bool quoted_first (concat_quoted_first); concat = false; concat_quoted = false; + concat_quoted_first = false; // If this is a result of typed concatenation, then don't inject. For // one we don't want any of the "interpretations" performed in the @@ -5455,19 +8009,32 @@ namespace build2 // dir/{$str} // file{$str} // - vnull = false; // A concatenation cannot produce NULL. + // And yet another exception: if the type is path or dir_path and the + // pattern mode is not ignore, then we will inject to try our luck in + // interpreting the concatenation result as a path pattern. This makes + // sure patterns like `$src_base/*.txt` work, naturally. Failed that, + // we will handle this concatenation as we do for other types (via the + // path_concat backup). + // + + // A concatenation cannot produce value/NULL. + // + vnull = false; + rvalue = false; if (vtype != nullptr) { bool e1 (tt == type::lcbrace && !peeked ().separated); bool e2 (pp || dp != nullptr || tp != nullptr); + const value_type* pt (&value_traits<path>::value_type); + const value_type* dt (&value_traits<dir_path>::value_type); + if (e1 || e2) { - if (vtype == &value_traits<path>::value_type || - vtype == &value_traits<string>::value_type) + if (vtype == pt || vtype == &value_traits<string>::value_type) ; // Representation is already in concat_data.value. - else if (vtype == &value_traits<dir_path>::value_type) + else if (vtype == dt) concat_data.value = move (concat_data.dir).representation (); else { @@ -5482,6 +8049,20 @@ namespace build2 vtype = nullptr; // Fall through to injection. } + else if (pmode != pattern_mode::ignore && + (vtype == pt || vtype == dt)) + { + path_concat = make_pair (vtype, concat_data); + + // Note: for path the representation is already in + // concat_data.value. + // + if (vtype == dt) + concat_data.value = move (concat_data.dir).representation (); + + vtype = nullptr; + // Fall through to injection. + } else { // This is either a simple name (untyped concatenation; in which @@ -5512,7 +8093,7 @@ namespace build2 t = token (move (concat_data.value), true, quoted ? quote_type::mixed : quote_type::unquoted, - false, + false, quoted_first, t.line, t.column); } else if (!first) @@ -5554,6 +8135,7 @@ namespace build2 string val (move (t.value)); const location loc (get_location (t)); bool quoted (t.qtype != quote_type::unquoted); + bool quoted_first (t.qfirst); // Should we accumulate? If the buffer is not empty, then we continue // accumulating (the case where we are separated should have been @@ -5564,6 +8146,8 @@ namespace build2 if (concat || // Continue. !last_concat ()) // Start. { + bool e (val.empty ()); + // If LHS is typed then do typed concatenation. // if (concat && vtype != nullptr) @@ -5572,7 +8156,7 @@ namespace build2 // names ns; ns.push_back (name (move (val))); - concat_typed (value (move (ns)), get_location (t)); + concat_typed (value (move (ns)), get_location (t), nullptr); } else { @@ -5584,17 +8168,26 @@ namespace build2 v += val; } - concat = true; + // Consider something like this: ""$foo where foo='+foo'. Should we + // treat the plus as a first (unquoted) character? Feels like we + // should not. The way we achieve this is a bit hackish: we make it + // look like a quoted first character. Note that there is a second + // half of this in expansion case which deals with $empty+foo. + // + if (!concat) // First. + concat_quoted_first = quoted_first || e; + concat_quoted = quoted || concat_quoted; + concat = true; continue; } // Find a separator (slash or %). // - string::size_type p (separators != nullptr - ? val.find_last_of (*separators) - : string::npos); + string::size_type pos (separators != nullptr + ? val.find_last_of (*separators) + : string::npos); // First take care of project. A project-qualified name is not very // common, so we can afford some copying for the sake of simplicity. @@ -5602,10 +8195,10 @@ namespace build2 optional<project_name> p1; const optional<project_name>* pp1 (&pp); - if (p != string::npos) + if (pos != string::npos) { - bool last (val[p] == '%'); - string::size_type q (last ? p : val.rfind ('%', p - 1)); + bool last (val[pos] == '%'); + string::size_type q (last ? pos : val.rfind ('%', pos - 1)); for (; q != string::npos; ) // Breakout loop. { @@ -5635,13 +8228,13 @@ namespace build2 // Now fix the rest of the name. // val.erase (0, q + 1); - p = last ? string::npos : p - (q + 1); + pos = last ? string::npos : pos - (q + 1); break; } } - string::size_type n (p != string::npos ? val.size () - 1 : 0); + size_t size (pos != string::npos ? val.size () - 1 : 0); // See if this is a type name, directory prefix, or both. That // is, it is followed by an un-separated '{'. @@ -5665,10 +8258,12 @@ namespace build2 if (ttp == nullptr) ppat = pinc = false; + else if (ttp->factory == nullptr) + fail (loc) << "abstract target type " << ttp->name << "{}"; } } - if (p != n && tp != nullptr && !pinc) + if (pos != size && tp != nullptr && !pinc) fail (loc) << "nested type name " << val; dir_path d1; @@ -5679,9 +8274,9 @@ namespace build2 try { - if (p == string::npos) // type + if (pos == string::npos) // type tp1 = &val; - else if (p == n) // directory + else if (pos == size) // directory { if (dp == nullptr) d1 = dir_path (val); @@ -5692,12 +8287,12 @@ namespace build2 } else // both { - t1.assign (val, p + 1, n - p); + t1.assign (val, pos + 1, size - pos); if (dp == nullptr) - d1 = dir_path (val, 0, p + 1); + d1 = dir_path (val, 0, pos + 1); else - d1 = *dp / dir_path (val, 0, p + 1); + d1 = *dp / dir_path (val, 0, pos + 1); dp1 = &d1; tp1 = &t1; @@ -5727,87 +8322,232 @@ namespace build2 continue; } - // See if this is a wildcard pattern. + // See if this is a pattern, path or regex. + // + // A path pattern either contains an unquoted wildcard character or, + // in the curly context, starts with unquoted/unescaped `+`. + // + // A regex pattern starts with unquoted/unescaped `~` followed by a + // non-alphanumeric delimiter and has the following form: + // + // ~/<pat>/[<flags>] // - // It should either contain a wildcard character or, in a curly - // context, start with unquoted '+'. + // A regex substitution starts with unquoted/unescaped '^' followed by + // a non-alphanumeric delimiter and has the follwing form: // - // Note that in the general case we need to convert it to a path prior - // to testing for being a pattern (think of b[a/r] that is not a - // pattern). If the conversion fails then this is not a path pattern. + // ^/<sub>/[<flags>] // - auto pattern = [&val, &loc, this] () + // Any non-alphanumeric character other that `/` can be used as a + // delimiter but escaping of the delimiter character is not supported + // (one benefit of this is that we can store and print the pattern as + // is without worrying about escaping; the non-alphanumeric part is to + // allow values like ~host and ^cat). + // + // The following pattern flags are recognized: + // + // i -- match ignoring case + // e -- match including extension + // + // Note that we cannot express certain path patterns that start with + // the regex introducer using quoting (for example, `~*`) since + // quoting prevents the whole from being recognized as a path + // pattern. However, we can achieve this with escaping (for example, + // \~*). This works automatically since we treat (at the lexer level) + // escaped first characters as quoted without treating the whole thing + // as quoted. Note that there is also the corresponding logic in + // to_stream(name). + // + // A pattern cannot be project-qualified. + // + optional<pattern_type> pat; + + if (pmode != pattern_mode::ignore && !*pp1) { - // Let's optimize it a bit for the common cases. + // Note that in the general case we need to convert it to a path + // prior to testing for being a pattern (think of b[a/r] that is not + // a pattern). // - if (val.find_first_of ("*?[") == string::npos) - return false; + auto path_pattern = [&val, &loc, this] () + { + // Let's optimize it a bit for the common cases. + // + if (val.find_first_of ("*?[") == string::npos) + return false; - if (path::traits_type::find_separator (val) == string::npos) - return path_pattern (val); + if (path_traits::find_separator (val) == string::npos) + return build2::path_pattern (val); - try + try + { + return build2::path_pattern (path (val)); + } + catch (const invalid_path& e) + { + fail (loc) << "invalid path '" << e.path << "'" << endf; + } + }; + + auto regex_pattern = [&val] () { - return path_pattern (path (val)); - } - catch (const invalid_path& e) + return ((val[0] == '~' || val[0] == '^') && + val[1] != '\0' && !alnum (val[1])); + }; + + if (pmode != pattern_mode::preserve) { - fail (loc) << "invalid path '" << e.path << "'" << endf; - } - }; + // Note that if we have no base directory or cannot resolve the + // target type, then this affectively becomes the ignore mode. + // + if (pbase_ != nullptr || (dp != nullptr && dp->absolute ())) + { + // Note that we have to check for regex patterns first since + // they may also be detected as path patterns. + // + if (!quoted_first && !path_concat && regex_pattern ()) + { + // Note: we may decide to support regex-based name generation + // some day (though a substitution won't make sense here). + // + fail (loc) << "regex pattern-based name generation" << + info << "quote '" << val << "' (or escape first character) " + << "to treat it as literal name (or path pattern)"; + } + else if ((!quoted && path_pattern ()) || + (!quoted_first && curly && val[0] == '+')) + { + // Resolve the target type if there is one. + // + const target_type* ttp (tp != nullptr && scope_ != nullptr + ? scope_->find_target_type (*tp) + : nullptr); - if (pmode != pattern_mode::ignore && - !*pp1 && // Cannot be project-qualified. - !quoted && // Cannot be quoted. - ((dp != nullptr && dp->absolute ()) || pbase_ != nullptr) && - (pattern () || (curly && val[0] == '+'))) - { - // Resolve the target if there is one. If we fail, then this is not - // a pattern. - // - const target_type* ttp (tp != nullptr && scope_ != nullptr - ? scope_->find_target_type (*tp) - : nullptr); + if (ttp != nullptr && ttp->factory == nullptr) + fail (loc) << "abstract target type " << ttp->name << "{}"; - if (tp == nullptr || ttp != nullptr) + if (tp == nullptr || ttp != nullptr) + { + if (pmode == pattern_mode::detect) + { + // Strip the literal unquoted plus character for the first + // pattern in the group. + // + if (ppat) + { + assert (val[0] == '+'); + val.erase (0, 1); + ppat = pinc = false; + } + + // Set the detect pattern mode to expand if the pattern is + // not followed by the inclusion/exclusion pattern/match. + // Note that if it is '}' (i.e., the end of the group), + // then it is a single pattern and the expansion is what + // we want. + // + if (!pattern_prefix (peeked ())) + pmode = pattern_mode::expand; + } + + if (pmode == pattern_mode::expand) + { + count = expand_name_pattern (get_location (t), + names {name (move (val))}, + ns, + what, + pairn, + dp, tp, ttp); + continue; + } + + pattern_detected (ttp); + + // Fall through. + } + } + } + } + else { - if (pmode == pattern_mode::detect) + // For the preserve mode we treat it as a pattern if it look like + // one syntactically. For now we also don't treat leading `+` in + // the curly context as an indication of a path pattern (since + // there isn't any good reason to; see also to_stream(name) for + // the corresponding serialization logic). + // + if (!quoted_first && !path_concat && regex_pattern ()) { - // Strip the literal unquoted plus character for the first - // pattern in the group. + const char* w; + if (val[0] == '~') + { + w = "regex pattern"; + pat = pattern_type::regex_pattern; + } + else + { + w = "regex substitution"; + pat = pattern_type::regex_substitution; + } + + size_t n (val.size ()); + + // Verify delimiters and find the position of the flags. // - if (ppat) + char d (val[1]); + size_t p (val.rfind (d)); + + if (p == 1) { - assert (val[0] == '+'); + fail (loc) << "no trailing delimiter '" << d << "' in " + << w << " '" << val << "'" << + info << "quote '" << val << "' (or escape first character) " + << "to treat it as literal name (or path pattern)"; + } - val.erase (0, 1); - ppat = pinc = false; + // Verify flags. + // + for (size_t i (++p); i != n; ++i) + { + char f (val[i]); + + if (*pat == pattern_type::regex_pattern) + { + if (f == 'i' || f == 'e') + continue; + } + + fail (loc) << "unknown flag '" << f << "' in " << w << " '" + << val << "'"; } - // Reset the detect pattern mode to expand if the pattern is not - // followed by the inclusion/exclusion pattern/match. Note that - // if it is '}' (i.e., the end of the group), then it is a single - // pattern and the expansion is what we want. + val.erase (0, 1); // Remove `~` or `^`. + + // Make sure we don't treat something like `~/.../` as a + // directory. // - if (!pattern_prefix (peeked ())) - pmode = pattern_mode::expand; + pos = string::npos; + size = 0; } + else if (!quoted && path_pattern ()) + pat = pattern_type::path; + } + } - if (pmode == pattern_mode::expand) - { - count = expand_name_pattern (get_location (t), - names {name (move (val))}, - ns, - what, - pairn, - dp, tp, ttp); - continue; - } + // If this is a concatenation of the path or dir_path type and it is + // not a pattern, then handle it in the same way as concatenations of + // other types (see above). + // + if (path_concat && !pat) + { + ns.push_back (move (path_concat->second)); - pattern_detected (ttp); + // Restore the type information if that's the only name. + // + if (start == ns.size () && last_token ()) + vtype = path_concat->first; - // Fall through. - } + // Restart the loop. + // + continue; } // If we are a second half of a pair, add another first half @@ -5826,7 +8566,9 @@ namespace build2 // in scope::find_target_type(). This would also mess up // reversibility to simple name. // - if (p == n) + // Note: a regex pattern cannot be a directory (see above). + // + if (pos == size) { // For reversibility to simple name, only treat it as a directory // if the string is an exact representation. @@ -5841,7 +8583,7 @@ namespace build2 append_name ( ns, *pp1, move (dir), (tp != nullptr ? *tp : string ()), string (), - loc); + pat, loc); continue; } @@ -5852,6 +8594,7 @@ namespace build2 (dp != nullptr ? *dp : dir_path ()), (tp != nullptr ? *tp : string ()), move (val), + pat, loc); continue; @@ -5861,6 +8604,9 @@ namespace build2 // if (tt == type::dollar || tt == type::lparen) { + if (ctx == nullptr) + fail << "literal " << what << " expected"; + // These cases are pretty similar in that in both we quickly end up // with a list of names that we need to splice into the result. // @@ -5882,11 +8628,15 @@ namespace build2 // token is a paren or a word, we turn it on and switch to the eval // mode if what we get next is a paren. // - // Also sniff out the special variables string from mode data for - // the ad hoc $() handling below. - // mode (lexer_mode::variable); + // Sniff out the special variables string from mode data and use + // that to recognize special variables in the ad hoc $() handling + // below. + // + // Note: must be done before calling next() which may expire the + // mode. + // auto special = [s = reinterpret_cast<const char*> (mode_data ())] (const token& t) -> char { @@ -5925,136 +8675,202 @@ namespace build2 next (t, tt); loc = get_location (t); - name qual; - string name; - - if (t.separated) - ; // Leave the name empty to fail below. - else if (tt == type::word) + if (tt == type::escape) { - name = move (t.value); + // For now we only support all the simple C/C++ escape sequences + // plus \0 (which in C/C++ is an octal escape sequence). See the + // lexer part for details. + // + // Note: cannot be subscripted. + // + if (!pre_parse_) + { + string s; + switch (char c = t.value[0]) + { + case '\'': + case '"': + case '?': + case '\\': s = c; break; + case '0': s = '\0'; break; + case 'a': s = '\a'; break; + case 'b': s = '\b'; break; + case 'f': s = '\f'; break; + case 'n': s = '\n'; break; + case 'r': s = '\r'; break; + case 't': s = '\t'; break; + case 'v': s = '\v'; break; + default: + assert (false); + } + + result_data = name (move (s)); + what = "escape sequence expansion"; + } + + tt = peek (); } - else if (tt == type::lparen) + else { - expire_mode (); - mode (lexer_mode::eval, '@'); - next_with_attributes (t, tt); + names qual; + string name; - // Handle the $(x) case ad hoc. We do it this way in order to get - // the variable name even during pre-parse. It should also be - // faster. - // - char c; - if ((tt == type::word || (c = special (t))) && - peek () == type::rparen) + if (t.separated) + ; // Leave the name empty to fail below. + else if (tt == type::word) { - name = (tt == type::word ? move (t.value) : string (1, c)); - next (t, tt); // Get `)`. + name = move (t.value); } - else + else if (tt == type::lparen) { - //@@ OUT will parse @-pair and do well? - // - values vs (parse_eval (t, tt, pmode)); + expire_mode (); + mode (lexer_mode::eval, '@'); + next_with_attributes (t, tt); - if (!pre_parse_) + // Handle the $(x) case ad hoc. We do it this way in order to + // get the variable name even during pre-parse. It should also + // be faster. + // + char c ('\0'); + if ((tt == type::word + ? path_traits::rfind_separator (t.value) == string::npos + : (c = special (t))) && + peek () == type::rparen) + { + name = (tt == type::word ? move (t.value) : string (1, c)); + next (t, tt); // Get `)`. + } + else { - if (vs.size () != 1) - fail (loc) << "expected single variable/function name"; + using name_type = build2::name; - value& v (vs[0]); + values vs (parse_eval (t, tt, pmode)); - if (!v) - fail (loc) << "null variable/function name"; + if (!pre_parse_) + { + if (vs.size () != 1) + fail (loc) << "expected single variable/function name"; - names storage; - vector_view<build2::name> ns (reverse (v, storage)); // Movable. - size_t n (ns.size ()); + value& v (vs[0]); - // We cannot handle scope-qualification in the eval context as - // we do for target-qualification (see eval-qual) since then - // we would be treating all paths as qualified variables. So - // we have to do it here. - // - if (n == 2 && ns[0].pair == ':') // $(foo: x) - { - qual = move (ns[0]); + if (!v) + fail (loc) << "null variable/function name"; - if (qual.empty ()) - fail (loc) << "empty variable/function qualification"; - } - else if (n == 2 && ns[0].directory ()) // $(foo/ x) - { - qual = move (ns[0]); - qual.pair = '/'; - } - else if (n > 1) - fail (loc) << "expected variable/function name instead of '" - << ns << "'"; + names storage; + vector_view<name_type> ns ( + reverse (v, storage, true /* reduce */)); // Movable. + size_t n (ns.size ()); - // Note: checked for empty below. - // - if (!ns[n - 1].simple ()) - fail (loc) << "expected variable/function name instead of '" - << ns[n - 1] << "'"; + // We cannot handle scope-qualification in the eval context + // as we do for target-qualification (see eval-qual) since + // then we would be treating all paths as qualified + // variables. So we have to do it here. + // + if (n >= 2 && ns[0].pair == ':') // $(foo: x) + { + // Note: name is first (see eval for details). + // + qual.push_back (move (ns[1])); - name = move (ns[n - 1].value); - } - } - } - else - fail (t) << "expected variable/function name instead of " << t; + if (qual.back ().empty ()) + fail (loc) << "empty variable/function qualification"; - if (!pre_parse_ && name.empty ()) - fail (loc) << "empty variable/function name"; + if (n > 2) + qual.push_back (move (ns[2])); - // Figure out whether this is a variable expansion with potential - // subscript or a function call. - // - if (sub) enable_subscript (); - tt = peek (); + // Move name to the last position (see below). + // + swap (ns[0], ns[n - 1]); + } + else if (n == 2 && ns[0].directory ()) // $(foo/ x) + { + qual.push_back (move (ns[0])); + qual.back ().pair = '/'; + } + else if (n > 1) + fail (loc) << "expected variable/function name instead of '" + << ns << "'"; + + // Note: checked for empty below. + // + if (!ns[n - 1].simple ()) + fail (loc) << "expected variable/function name instead of '" + << ns[n - 1] << "'"; + + size_t p; + if (n == 1 && // $(foo/x) + (p = path_traits::rfind_separator (ns[0].value)) != + string::npos) + { + // Note that p cannot point to the last character since + // then it would have been a directory, not a simple name. + // + string& s (ns[0].value); + + name = string (s, p + 1); + s.resize (p + 1); + qual.push_back (name_type (dir_path (move (s)))); + qual.back ().pair = '/'; + } + else + name = move (ns[n - 1].value); + } + } + } + else + fail (t) << "expected variable/function name instead of " << t; - // Note that we require function call opening paren to be - // unseparated; consider: $x ($x == 'foo' ? 'FOO' : 'BAR'). - // - if (tt == type::lparen && !peeked ().separated) - { - // Function call. - // - next (t, tt); // Get '('. - mode (lexer_mode::eval, '@'); - next_with_attributes (t, tt); + if (!pre_parse_ && name.empty ()) + fail (loc) << "empty variable/function name"; - // @@ Should we use (target/scope) qualification (of name) as the - // context in which to call the function? Hm, interesting... + // Figure out whether this is a variable expansion with potential + // subscript or a function call. // - values args (parse_eval (t, tt, pmode)); - if (sub) enable_subscript (); tt = peek (); - // Note that we "move" args to call(). + // Note that we require function call opening paren to be + // unseparated; consider: $x ($x == 'foo' ? 'FOO' : 'BAR'). // - if (!pre_parse_) + if (tt == type::lparen && !peeked ().separated) { - result_data = ctx.functions.call (scope_, name, args, loc); - what = "function call"; + // Function call. + // + next (t, tt); // Get '('. + mode (lexer_mode::eval, '@'); + next_with_attributes (t, tt); + + // @@ Should we use (target/scope) qualification (of name) as + // the context in which to call the function? Hm, interesting... + // + values args (parse_eval (t, tt, pmode)); + + if (sub) enable_subscript (); + tt = peek (); + + // Note that we "move" args to call(). + // + if (!pre_parse_) + { + result_data = ctx->functions.call (scope_, name, args, loc); + what = "function call"; + } + else + lookup_function (move (name), loc); } else - lookup_function (move (name), loc); - } - else - { - // Variable expansion. - // - lookup l (lookup_variable (move (qual), move (name), loc)); - - if (!pre_parse_) { - if (l.defined ()) - result = l.value; // Otherwise leave as NULL result_data. + // Variable expansion. + // + lookup l (lookup_variable (move (qual), move (name), loc)); - what = "variable expansion"; + if (!pre_parse_) + { + if (l.defined ()) + result = l.value; // Otherwise leave as NULL result_data. + + what = "variable expansion"; + } } } } @@ -6086,85 +8902,132 @@ namespace build2 // Handle value subscript. // - if (tt == type::lsbrace) + if (mode () == lexer_mode::eval) // Note: not if(sub)! { - location bl (get_location (t)); - next (t, tt); // `[` - mode (lexer_mode::subscript, '\0' /* pair */); - next (t, tt); - - location l (get_location (t)); - value v ( - tt != type::rsbrace - ? parse_value (t, tt, pattern_mode::ignore, "value subscript") - : value (names ())); - - if (tt != type::rsbrace) + while (tt == type::lsbrace) { - // Note: wildcard pattern should have `]` as well so no escaping - // suggestion. - // - fail (t) << "expected ']' instead of " << t; - } + location bl (get_location (t)); + next (t, tt); // `[` + mode (lexer_mode::subscript, '\0' /* pair */); + next (t, tt); - if (!pre_parse_) - { - uint64_t j; - try - { - j = convert<uint64_t> (move (v)); - } - catch (const invalid_argument& e) + location l (get_location (t)); + value v ( + tt != type::rsbrace + ? parse_value (t, tt, pattern_mode::ignore, "value subscript") + : value (names ())); + + if (tt != type::rsbrace) { - fail (l) << "invalid value subscript: " << e << - info (bl) << "use the '\\[' escape sequence if this is a " - << "wildcard pattern" << endf; + // Note: wildcard pattern should have `]` as well so no escaping + // suggestion. + // + fail (t) << "expected ']' instead of " << t; } - // Similar to expanding an undefined variable, we return NULL if - // the index is out of bounds. - // - // Note that result may or may not point to result_data. - // - if (result->null) - result_data = value (); - else if (result->type == nullptr) + if (!pre_parse_) { - const names& ns (result->as<names> ()); - - // Pair-aware subscript. + // For type-specific subscript implementations we pass the + // subscript value as is. // - names r; - for (auto i (ns.begin ()); i != ns.end (); ++i, --j) + if (auto f = (result->type != nullptr + ? result->type->subscript + : nullptr)) { - if (j == 0) + result_data = f (*result, &result_data, move (v), l, bl); + } + else + { + uint64_t j; + try { - r.push_back (*i); - if (i->pair) - r.push_back (*++i); - break; + j = convert<uint64_t> (move (v)); } + catch (const invalid_argument& e) + { + fail (l) << "invalid value subscript: " << e << + info (bl) << "use the '\\[' escape sequence if this is a " + << "wildcard pattern" << endf; + } + + // Similar to expanding an undefined variable, we return NULL + // if the index is out of bounds. + // + // Note that result may or may not point to result_data. + // + if (result->null) + result_data = value (); + else if (result->type == nullptr) + { + const names& ns (result->as<names> ()); + + // Pair-aware subscript. + // + names r; + for (auto i (ns.begin ()); i != ns.end (); ++i, --j) + { + if (j == 0) + { + r.push_back (*i); + if (i->pair) + r.push_back (*++i); + break; + } + + if (i->pair) + ++i; + } + + result_data = r.empty () ? value () : value (move (r)); + } + else + { + // Similar logic to parse_for(). + // + const value_type* etype (result->type->element_type); + + value val (result == &result_data + ? value (move (result_data)) + : value (*result)); - if (i->pair) - ++i; + untypify (val, false /* reduce */); + + names& ns (val.as<names> ()); + + // Pair-aware subscript. + // + names r; + for (auto i (ns.begin ()); i != ns.end (); ++i, --j) + { + bool p (i->pair); + + if (j == 0) + { + r.push_back (move (*i)); + if (p) + r.push_back (move (*++i)); + break; + } + + if (p) + ++i; + } + + result_data = r.empty () ? value () : value (move (r)); + + if (etype != nullptr) + typify (result_data, *etype, nullptr /* var */); + } } - result_data = r.empty () ? value () : value (move (r)); - } - else - { - // @@ TODO: we would want to return a value with element type. - // - //result_data = ... - fail (l) << "typed value subscript not yet supported" << - info (bl) << "use the '\\[' escape sequence if this is a " - << "wildcard pattern"; + result = &result_data; } - result = &result_data; + // See if we have chained subscript. + // + enable_subscript (); + tt = peek (); } - - tt = peek (); } if (pre_parse_) @@ -6208,7 +9071,8 @@ namespace build2 // then it should not be overloaded for a type). In a quoted // context we use $string() which returns a "canonical // representation" (e.g., a directory path without a trailing - // slash). + // slash). Note: looks like we use typed $concat() now in the + // unquoted context. // if (result->type != nullptr && quoted) { @@ -6231,7 +9095,10 @@ namespace build2 dr << info (loc) << "while converting " << t << " to string"; }); - p = ctx.functions.try_call ( + if (ctx == nullptr) + fail << "literal " << what << " expected"; + + p = ctx->functions.try_call ( scope_, "string", vector_view<value> (&result_data, 1), loc); } @@ -6239,7 +9106,11 @@ namespace build2 fail (loc) << "no string conversion for " << t; result_data = move (p.first); - untypify (result_data); // Convert to untyped simple name. + + // Convert to untyped simple name reducing empty string to empty + // names as an optimization. + // + untypify (result_data, true /* reduce */); } if ((concat && vtype != nullptr) || // LHS typed. @@ -6248,52 +9119,59 @@ namespace build2 if (result != &result_data) // Same reason as above. result = &(result_data = *result); - concat_typed (move (result_data), loc); + concat_typed (move (result_data), loc, what); } // // Untyped concatenation. Note that if RHS is NULL/empty, we still // set the concat flag. // - else if (!result->null && !result->empty ()) + else if (!result->null) { - // This can only an untyped value. + // This can only be an untyped value. // // @@ Could move if result == &result_data. // const names& lv (cast<names> (*result)); - // This should be a simple value or a simple directory. - // - if (lv.size () > 1) - fail (loc) << "concatenating " << what << " contains multiple " - << "values"; + if (size_t s = lv.size ()) + { + // This should be a simple value or a simple directory. + // + if (s > 1) + concat_diag_multiple (loc, what); - const name& n (lv[0]); + const name& n (lv[0]); - if (n.qualified ()) - fail (loc) << "concatenating " << what << " contains project " - << "name"; + if (n.qualified ()) + fail (loc) << "concatenating " << what << " contains project " + << "name"; - if (n.typed ()) - fail (loc) << "concatenating " << what << " contains type"; + if (n.typed ()) + fail (loc) << "concatenating " << what << " contains target type"; - if (!n.dir.empty ()) - { - if (!n.value.empty ()) - fail (loc) << "concatenating " << what << " contains " - << "directory"; + if (!n.dir.empty ()) + { + if (!n.value.empty ()) + fail (loc) << "concatenating " << what << " contains " + << "directory"; - // Note that here we cannot assume what's in dir is really a - // path (think s/foo/bar/) so we have to reverse it exactly. - // - concat_data.value += n.dir.representation (); + // Note that here we cannot assume what's in dir is really a + // path (think s/foo/bar/) so we have to reverse it exactly. + // + concat_data.value += n.dir.representation (); + } + else + concat_data.value += n.value; } - else - concat_data.value += n.value; } - concat = true; + // The same little hack as in the word case ($empty+foo). + // + if (!concat) // First. + concat_quoted_first = true; + concat_quoted = quoted || concat_quoted; + concat = true; } else { @@ -6305,20 +9183,32 @@ namespace build2 { vnull = result->null; vtype = result->type; + rvalue = true; } // Nothing else to do here if the result is NULL or empty. // - if (result->null || result->empty ()) - continue; - - // @@ Could move if nv is result_data; see untypify(). + // Note that we cannot use value::empty() here since we are + // interested in representationally empty. // - names nv_storage; - names_view nv (reverse (*result, nv_storage)); + if (!result->null) + { + // @@ Could move if nv is result_data; see untypify(). + // + // Nuance: we should only be reducing empty simple value to empty + // list if we are not a second half of a pair. + // + bool pair (!ns.empty () && ns.back ().pair); - count = splice_names ( - loc, nv, move (nv_storage), ns, what, pairn, pp, dp, tp); + names nv_storage; + names_view nv (reverse (*result, nv_storage, !pair /* reduce */)); + + if (!nv.empty ()) + { + count = splice_names ( + loc, nv, move (nv_storage), ns, what, pairn, pp, dp, tp); + } + } } continue; @@ -6361,6 +9251,7 @@ namespace build2 (dp != nullptr ? *dp : dir_path ()), (tp != nullptr ? *tp : string ()), string (), + nullopt, /* pattern */ get_location (t)); count = 1; } @@ -6381,6 +9272,7 @@ namespace build2 (dp != nullptr ? *dp : dir_path ()), (tp != nullptr ? *tp : string ()), string (), + nullopt, /* pattern */ get_location (t)); count = 0; } @@ -6408,6 +9300,7 @@ namespace build2 (dp != nullptr ? *dp : dir_path ()), (tp != nullptr ? *tp : string ()), string (), + nullopt, /* pattern */ get_location (t)); break; } @@ -6426,13 +9319,14 @@ namespace build2 (dp != nullptr ? *dp : dir_path ()), (tp != nullptr ? *tp : string ()), string (), + nullopt, /* pattern */ get_location (t)); } if (pre_parse_) - assert (!vnull && vtype == nullptr && !rpat); + assert (!rvalue && !vnull && vtype == nullptr && !rpat); - return parse_names_result {!vnull, vtype, rpat}; + return parse_names_result {rvalue, !vnull, vtype, rpat}; } void parser:: @@ -6504,8 +9398,8 @@ namespace build2 // // print +foo // - // So wepeek at one more character since what we expect next ('=') can't - // be whitespace-separated. + // So we peek at one more character since what we expect next ('=') + // can't be whitespace-separated. // return c0 == '\n' || c0 == '\0' || c0 == '(' || (p.second && @@ -6549,14 +9443,16 @@ namespace build2 buildspec parser:: parse_buildspec (istream& is, const path_name& in) { - // We do "effective escaping" and only for ['"\$(] (basically what's - // necessary inside a double-quoted literal plus the single quote). + // We do "effective escaping" of the special `'"\$(` characters (basically + // what's escapable inside a double-quoted literal plus the single quote; + // note, however, that we exclude line continuations and `)` since they + // would make directory paths on Windows unusable). // path_ = ∈ lexer l (is, *path_, 1 /* line */, "\'\"\\$("); lexer_ = &l; - root_ = &ctx.global_scope.rw (); + root_ = &ctx->global_scope.rw (); scope_ = root_; target_ = nullptr; prerequisite_ = nullptr; @@ -6791,8 +9687,11 @@ namespace build2 } lookup parser:: - lookup_variable (name&& qual, string&& name, const location& loc) + lookup_variable (names&& qual, string&& name, const location& loc) { + // Note that this function can be called during execute (for example, from + // scripts). In particular, this means we cannot use enter_{scope,target}. + if (pre_parse_) return lookup (); @@ -6804,9 +9703,6 @@ namespace build2 // If we are qualified, it can be a scope or a target. // - enter_scope sg; - enter_target tg; - if (qual.empty ()) { s = scope_; @@ -6815,36 +9711,70 @@ namespace build2 } else { - switch (qual.pair) + // What should we do if we cannot find the qualification (scope or + // target)? We can "fall through" to an outer scope (there is always the + // global scope backstop), we can return NULL straight away, or we can + // fail. It feels like in most cases unknown scope or target is a + // mistake and doing anything other than failing is just making things + // harder to debug. + // + switch (qual.front ().pair) { case '/': { - assert (qual.directory ()); - sg = enter_scope (*this, move (qual.dir)); - s = scope_; + assert (qual.front ().directory ()); + + dir_path& d (qual.front ().dir); + enter_scope::complete_normalize (*scope_, d); + + s = &ctx->scopes.find_out (d); + + if (s->out_path () != d) + fail (loc) << "unknown scope " << d << " in scope-qualified " + << "variable " << name << " expansion" << + info << "did you forget to include the corresponding buildfile?"; + break; } - case ':': + default: { - qual.pair = '\0'; + build2::name n (move (qual.front ())), o; + + if (n.pair) + o = move (qual.back ()); + + t = enter_target::find_target (*this, n, o, loc, trace); + + if (t == nullptr || !operator>= (t->decl, target_decl::implied)) // VC14 + { + diag_record dr (fail (loc)); + + dr << "unknown target " << n; + + if (n.pair && !o.dir.empty ()) + dr << '@' << o.dir; + + dr << " in target-qualified variable " << name << " expansion"; + } - // @@ OUT TODO + // Use the target's var_pool for good measure. // - tg = enter_target ( - *this, move (qual), build2::name (), true, loc, trace); - t = target_; + s = &t->base_scope (); + break; } - default: assert (false); } } // Lookup. // - if (const variable* pvar = scope_->var_pool ().find (name)) + if (const variable* pvar = + (s != nullptr ? s : scope_)->var_pool ().find (name)) { auto& var (*pvar); + // Note: the order of the following blocks is important. + if (p != nullptr) { // The lookup depth is a bit of a hack but should be harmless since @@ -6892,11 +9822,13 @@ namespace build2 assert (pre_parse_); } - void parser:: + auto_project_env parser:: switch_scope (const dir_path& d) { tracer trace ("parser::switch_scope", &path_); + auto_project_env r; + // Switching the project during bootstrap can result in bizarre nesting // with unexpected loading order (e.g., config.build are loaded from inner // to outter rather than the expected reverse). On the other hand, it can @@ -6913,6 +9845,10 @@ namespace build2 if (proj && p.second != root_) { root_ = p.second; + + if (root_ != nullptr) + r = auto_project_env (*root_); + l5 ([&] { if (root_ != nullptr) @@ -6921,64 +9857,217 @@ namespace build2 trace << "switching to out of project scope"; }); } + + return r; } + // file.cxx + // + extern const dir_path std_export_dir; + extern const dir_path alt_export_dir; + void parser:: - process_default_target (token& t) + process_default_target (token& t, const buildfile* bf) { tracer trace ("parser::process_default_target", &path_); // The logic is as follows: if we have an explicit current directory - // target, then that's the default target. Otherwise, we take the - // first target and use it as a prerequisite to create an implicit - // current directory target, effectively making it the default - // target via an alias. If there are no targets in this buildfile, - // then we don't do anything. + // target, then that's the default target. Otherwise, we take the first + // target and use it as a prerequisite to create an implicit current + // directory target, effectively making it the default target via an + // alias. If this is a project root buildfile, then also add exported + // buildfiles. And if there are no targets in this buildfile, then we + // don't do anything (reasonably assuming it's not root). // if (default_target_ == nullptr) // No targets in this buildfile. return; - target& dt (*default_target_); - target* ct ( - const_cast<target*> ( // Ok (serial execution). - ctx.targets.find (dir::static_type, // Explicit current dir target. - scope_->out_path (), - dir_path (), // Out tree target. - string (), - nullopt, - trace))); - - if (ct == nullptr) - { - l5 ([&]{trace (t) << "creating current directory alias for " << dt;}); - - // While this target is not explicitly mentioned in the buildfile, we - // say that we behave as if it were. Thus not implied. - // - ct = &ctx.targets.insert (dir::static_type, - scope_->out_path (), - dir_path (), - string (), - nullopt, - target_decl::real, - trace).first; - // Fall through. - } - else if (ct->decl != target_decl::real) + const_cast<target*> ( // Ok (serial execution). + ctx->targets.find (dir::static_type, // Explicit current dir target. + scope_->out_path (), + dir_path (), // Out tree target. + string (), + nullopt, + trace))); + + if (ct != nullptr && ct->decl == target_decl::real) + ; // Existing and not implied. + else { - ct->decl = target_decl::real; - // Fall through. + target& dt (*default_target_); + + if (ct == nullptr) + { + l5 ([&]{trace (t) << "creating current directory alias for " << dt;}); + + // While this target is not explicitly mentioned in the buildfile, we + // say that we behave as if it were. Thus not implied. + // + ct = &ctx->targets.insert (dir::static_type, + scope_->out_path (), + dir_path (), + string (), + nullopt, + target_decl::real, + trace).first; + } + else + ct->decl = target_decl::real; + + ct->prerequisites_state_.store (2, memory_order_relaxed); + ct->prerequisites_.push_back (prerequisite (dt)); } - else - return; // Existing and not implied. - ct->prerequisites_state_.store (2, memory_order_relaxed); - ct->prerequisites_.emplace_back (prerequisite (dt)); + // See if this is a root buildfile and not in a simple project. + // + if (bf != nullptr && + root_ != nullptr && + root_->root_extra != nullptr && + root_->root_extra->loaded && + *root_->root_extra->project != nullptr && + bf->dir == root_->src_path () && + bf->name == root_->root_extra->buildfile_file.string ()) + { + // See if we have any exported buildfiles. + // + const dir_path& export_dir ( + root_->root_extra->altn ? alt_export_dir : std_export_dir); + + dir_path d (root_->src_path () / export_dir); + if (exists (d)) + { + // Make sure prerequisites are set. + // + ct->prerequisites_state_.store (2, memory_order_relaxed); + + const string& build_ext (root_->root_extra->build_ext); + + // Return true if entered any exported buildfiles. + // + // Note: recursive lambda. + // + auto iterate = [this, &trace, + ct, &build_ext] (const dir_path& d, + const auto& iterate) -> bool + { + bool r (false); + + try + { + for (const dir_entry& e: + dir_iterator (d, dir_iterator::detect_dangling)) + { + switch (e.type ()) + { + case entry_type::directory: + { + r = iterate (d / path_cast<dir_path> (e.path ()), iterate) || r; + break; + } + case entry_type::regular: + { + const path& n (e.path ()); + + // Besides the buildfile also export buildscript and C++ files + // that are used to provide recipe implementations (see + // parse_recipe() for details). + // + string e (n.extension ()); + if (const target_type* tt = ( + e == build_ext ? &buildfile::static_type : + e == "buildscript" ? &buildscript::static_type : + e == "cxx" || + e == "cpp" || + e == "cc" ? &file::static_type : nullptr)) + { + // Enter as if found by search_existing_file(). Note that + // entering it as real would cause file_rule not to match + // for clean. + // + // Note that these targets may already be entered (for + // example, if already imported). + // + const target& bf ( + ctx->targets.insert (*tt, + d, + (root_->out_eq_src () + ? dir_path () + : out_src (d, *root_)), + n.base ().string (), + move (e), + target_decl::prereq_file, + trace).first); + + ct->prerequisites_.push_back (prerequisite (bf)); + r = true; + } + + break; + } + case entry_type::unknown: + { + bool sl (e.ltype () == entry_type::symlink); + + fail << (sl ? "dangling symlink" : "inaccessible entry") + << ' ' << d / e.path (); + + break; + } + default: + break; + } + } + } + catch (const system_error& e) + { + fail << "unable to iterate over " << d << ": " << e; + } + + return r; + }; + + if (iterate (d, iterate)) + { + // Arrange for the exported buildfiles to be installed, recreating + // subdirectories inside export/. Essentially, we are arranging for + // this: + // + // build/export/file{*}: + // { + // install = buildfile/ + // install.subdirs = true + // } + // + if (cast_false<bool> (root_->vars["install.loaded"])) + { + enter_scope es (*this, dir_path (export_dir)); + auto& vars (scope_->target_vars[file::static_type]["*"]); + + // @@ TODO: get cached variables from the module once we have one. + // + { + auto r (vars.insert (*root_->var_pool ().find ("install"))); + + if (r.second) // Already set by the user? + r.first = path_cast<path> (dir_path ("buildfile")); + } + + { + auto r (vars.insert ( + *root_->var_pool (true).find ("install.subdirs"))); + if (r.second) + r.first = true; + } + } + } + } + } } - void parser:: - enter_buildfile (const path& p) + template <typename T> + const T& parser:: + enter_buildfile (const path& p, optional<dir_path> out) { tracer trace ("parser::enter_buildfile", &path_); @@ -6986,17 +10075,20 @@ namespace build2 // Figure out if we need out. // - dir_path out; - if (scope_->src_path_ != nullptr && - scope_->src_path () != scope_->out_path () && - d.sub (scope_->src_path ())) + dir_path o; + if (out) + o = move (*out); + else if (root_ != nullptr && + root_->src_path_ != nullptr && + !root_->out_eq_src () && + d.sub (*root_->src_path_)) { - out = out_src (d, *root_); + o = out_src (d, *root_); } - ctx.targets.insert<buildfile> ( + return ctx->targets.insert<T> ( move (d), - move (out), + move (o), p.leaf ().base ().string (), p.extension (), // Always specified. trace); |