diff options
Diffstat (limited to 'libbuild2/parser.cxx')
-rw-r--r-- | libbuild2/parser.cxx | 2736 |
1 files changed, 2139 insertions, 597 deletions
diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index d50f86f..5d77e2b 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -24,6 +24,8 @@ #include <libbuild2/adhoc-rule-regex-pattern.hxx> +#include <libbuild2/dist/module.hxx> // module + #include <libbuild2/config/utility.hxx> // lookup_config using namespace std; @@ -42,7 +44,10 @@ namespace build2 { o << '='; names storage; - to_stream (o, reverse (a.value, storage), true /* quote */, '@'); + to_stream (o, + reverse (a.value, storage, true /* reduce */), + quote_mode::normal, + '@'); } return o; @@ -57,27 +62,7 @@ namespace build2 enter_scope (parser& p, dir_path&& d) : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_) { - // Try hard not to call normalize(). Most of the time we will go just - // one level deeper. - // - bool n (true); - - if (d.relative ()) - { - // Relative scopes are opened relative to out, not src. - // - if (d.simple () && !d.current () && !d.parent ()) - { - d = dir_path (p.scope_->out_path ()) /= d.string (); - n = false; - } - else - d = p.scope_->out_path () / d; - } - - if (n) - d.normalize (); - + complete_normalize (*p.scope_, d); e_ = p.switch_scope (d); } @@ -103,8 +88,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_scope (enter_scope&& x) {*this = move (x);} - enter_scope& operator= (enter_scope&& x) + enter_scope (enter_scope&& x) noexcept {*this = move (x);} + enter_scope& operator= (enter_scope&& x) noexcept { if (this != &x) { @@ -121,6 +106,31 @@ namespace build2 enter_scope (const enter_scope&) = delete; enter_scope& operator= (const enter_scope&) = delete; + static void + complete_normalize (scope& s, dir_path& d) + { + // Try hard not to call normalize(). Most of the time we will go just + // one level deeper. + // + bool n (true); + + if (d.relative ()) + { + // Relative scopes are opened relative to out, not src. + // + if (d.simple () && !d.current () && !d.parent ()) + { + d = dir_path (s.out_path ()) /= d.string (); + n = false; + } + else + d = s.out_path () / d; + } + + if (n) + d.normalize (); + } + private: parser* p_; scope* r_; @@ -162,7 +172,7 @@ namespace build2 tracer& tr) { auto r (p.scope_->find_target_type (n, o, loc)); - return p.ctx.targets.insert ( + return p.ctx->targets.insert ( r.first, // target type move (n.dir), move (o.dir), @@ -182,12 +192,12 @@ namespace build2 tracer& tr) { auto r (p.scope_->find_target_type (n, o, loc)); - return p.ctx.targets.find (r.first, // target type - n.dir, - o.dir, - n.value, - r.second, // extension - tr); + return p.ctx->targets.find (r.first, // target type + n.dir, + o.dir, + n.value, + r.second, // extension + tr); } ~enter_target () @@ -198,8 +208,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_target (enter_target&& x) {*this = move (x);} - enter_target& operator= (enter_target&& x) { + enter_target (enter_target&& x) noexcept {*this = move (x);} + enter_target& operator= (enter_target&& x) noexcept { p_ = x.p_; t_ = x.t_; x.p_ = nullptr; return *this;} enter_target (const enter_target&) = delete; @@ -230,8 +240,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_prerequisite (enter_prerequisite&& x) {*this = move (x);} - enter_prerequisite& operator= (enter_prerequisite&& x) { + enter_prerequisite (enter_prerequisite&& x) noexcept {*this = move (x);} + enter_prerequisite& operator= (enter_prerequisite&& x) noexcept { p_ = x.p_; r_ = x.r_; x.p_ = nullptr; return *this;} enter_prerequisite (const enter_prerequisite&) = delete; @@ -247,6 +257,7 @@ namespace build2 { pre_parse_ = false; attributes_.clear (); + condition_ = nullopt; default_target_ = nullptr; peeked_ = false; replay_ = replay::stop; @@ -259,10 +270,11 @@ namespace build2 scope* root, scope& base, target* tgt, - prerequisite* prq) + prerequisite* prq, + bool enter) { lexer l (is, in); - parse_buildfile (l, root, base, tgt, prq); + parse_buildfile (l, root, base, tgt, prq, enter); } void parser:: @@ -270,7 +282,8 @@ namespace build2 scope* root, scope& base, target* tgt, - prerequisite* prq) + prerequisite* prq, + bool enter) { path_ = &l.name (); lexer_ = &l; @@ -289,9 +302,9 @@ namespace build2 ? auto_project_env (*root_) : auto_project_env ()); - if (path_->path != nullptr) - enter_buildfile (*path_->path); // Note: needs scope_. - + const buildfile* bf (enter && path_->path != nullptr + ? &enter_buildfile (*path_->path) + : nullptr); token t; type tt; next (t, tt); @@ -303,13 +316,32 @@ namespace build2 else { parse_clause (t, tt); - process_default_target (t); + process_default_target (t, bf); } if (tt != type::eos) fail (t) << "unexpected " << t; } + names parser:: + parse_export_stub (istream& is, const path_name& name, + const scope& rs, scope& gs, scope& ts) + { + // Enter the export stub manually with correct out. + // + if (name.path != nullptr) + { + dir_path out (!rs.out_eq_src () + ? out_src (name.path->directory (), rs) + : dir_path ()); + + enter_buildfile (*name.path, move (out)); + } + + parse_buildfile (is, name, &gs, ts, nullptr, nullptr, false /* enter */); + return move (export_value); + } + token parser:: parse_variable (lexer& l, scope& s, const variable& var, type kind) { @@ -355,6 +387,81 @@ namespace build2 return make_pair (move (lhs), move (t)); } + names parser:: + parse_names (lexer& l, + const dir_path* b, + pattern_mode pmode, + const char* what, + const string* separators) + { + path_ = &l.name (); + lexer_ = &l; + + root_ = nullptr; + scope_ = nullptr; + target_ = nullptr; + prerequisite_ = nullptr; + + pbase_ = b; + + token t; + type tt; + + mode (lexer_mode::value, '@'); + next (t, tt); + + names r (parse_names (t, tt, pmode, what, separators)); + + if (tt != type::eos) + fail (t) << "unexpected " << t; + + return r; + } + + value parser:: + parse_eval (lexer& l, scope& rs, scope& bs, pattern_mode pmode) + { + path_ = &l.name (); + lexer_ = &l; + + root_ = &rs; + scope_ = &bs; + target_ = nullptr; + prerequisite_ = nullptr; + + pbase_ = scope_->src_path_; + + // Note that root_ may not be a project root. + // + auto_project_env penv ( + stage_ != stage::boot && root_ != nullptr && root_->root_extra != nullptr + ? auto_project_env (*root_) + : auto_project_env ()); + + token t; + type tt; + next (t, tt); + + if (tt != type::lparen) + fail (t) << "expected '(' instead of " << t; + + location loc (get_location (t)); + mode (lexer_mode::eval, '@'); + next_with_attributes (t, tt); + + values vs (parse_eval (t, tt, pmode)); + + if (next (t, tt) != type::eos) + fail (t) << "unexpected " << t; + + switch (vs.size ()) + { + case 0: return value (names ()); + case 1: return move (vs[0]); + default: fail (loc) << "expected single value" << endf; + } + } + bool parser:: parse_clause (token& t, type& tt, bool one) { @@ -516,9 +623,39 @@ namespace build2 location nloc (get_location (t)); names ns; - if (tt != type::labrace) + // We have to parse names in chunks to detect invalid cases of the + // group{foo}<...> syntax. + // + // Consider (1): + // + // x = + // group{foo} $x<...>: + // + // And (2): + // + // x = group{foo} group{bar} + // $x<...>: + // + // As well as (3): + // + // <...><...>: + // + struct chunk { - ns = parse_names (t, tt, pattern_mode::preserve); + size_t pos; // Index in ns of the beginning of the last chunk. + location loc; // Position of the beginning of the last chunk. + }; + optional<chunk> ns_last; + + bool labrace_first (tt == type::labrace); + if (!labrace_first) + { + do + { + ns_last = chunk {ns.size (), get_location (t)}; + parse_names (t, tt, ns, pattern_mode::preserve, true /* chunk */); + } + while (start_names (tt)); // Allow things like function calls that don't result in anything. // @@ -534,44 +671,87 @@ namespace build2 } } - // Handle ad hoc target group specification (<...>). + // Handle target group specification (<...>). // // We keep an "optional" (empty) vector of names parallel to ns that - // contains the ad hoc group members. + // contains the group members. Note that when we "catch" gns up to ns, + // we populate it with ad hoc (as opposed to explicit) groups with no + // members. // - adhoc_names ans; + group_names gns; if (tt == type::labrace) { - while (tt == type::labrace) + for (; tt == type::labrace; labrace_first = false) { - // Parse target names inside < >. + // Detect explicit group (group{foo}<...>). + // + // Note that `<` first thing on the line is not seperated thus the + // labrace_first complication. + // + bool expl (!t.separated && !labrace_first); + if (expl) + { + // Note: (N) refers to the example in the above comment. + // + if (!ns_last /* (3) */ || ns_last->pos == ns.size () /* (1) */) + { + fail (t) << "group name or whitespace expected before '<'"; + } + else + { + size_t n (ns.size () - ns_last->pos); + + // Note: could be a pair. + // + if ((n > 2 || (n == 2 && !ns[ns_last->pos].pair)) /* (2) */) + { + fail (t) << "single group name or whitespace expected before " + << "'<' instead of '" + << names_view (ns.data () + ns_last->pos, n) << "'"; + } + } + } + + // Parse target names inside <>. // // We "reserve" the right to have attributes inside <> though what // exactly that would mean is unclear. One potentially useful - // semantics would be the ability to specify attributes for ad hoc - // members though the fact that the primary target is listed first - // would make it rather unintuitive. Maybe attributes that change - // the group semantics itself? + // semantics would be the ability to specify attributes for group + // members though the fact that the primary target for ad hoc groups + // is listed first would make it rather unintuitive. Maybe + // attributes that change the group semantics itself? // next_with_attributes (t, tt); auto at (attributes_push (t, tt)); if (at.first) - fail (at.second) << "attributes before ad hoc target"; + fail (at.second) << "attributes before group member"; else attributes_pop (); - // Allow empty case (<>). + // For explicit groups, the group target is already in ns and all + // the members should go straight to gns. // - if (tt != type::rabrace) + // For ad hoc groups, the first name (or a pair) is the primary + // target which we need to keep in ns. The rest, if any, are ad + // hoc members that we should move to gns. + // + if (expl) + { + gns.resize (ns.size ()); // Catch up with the names vector. + group_names_loc& g (gns.back ()); + g.expl = true; + g.group_loc = move (ns_last->loc); + g.member_loc = get_location (t); // Start of members. + + if (tt != type::rabrace) // Handle empty case (<>) + parse_names (t, tt, g.ns, pattern_mode::preserve); + } + else if (tt != type::rabrace) // Allow and ignore empty case (<>). { - location aloc (get_location (t)); + location mloc (get_location (t)); // Start of members. - // The first name (or a pair) is the primary target which we need - // to keep in ns. The rest, if any, are ad hoc members that we - // should move to ans. - // size_t m (ns.size ()); parse_names (t, tt, ns, pattern_mode::preserve); size_t n (ns.size ()); @@ -588,11 +768,10 @@ namespace build2 { n -= m; // Number of names in ns we should end up with. - ans.resize (n); // Catch up with the names vector. - adhoc_names_loc& a (ans.back ()); - - a.loc = move (aloc); - a.ns.insert (a.ns.end (), + gns.resize (n); // Catch up with the names vector. + group_names_loc& g (gns.back ()); + g.group_loc = g.member_loc = move (mloc); + g.ns.insert (g.ns.end (), make_move_iterator (ns.begin () + n), make_move_iterator (ns.end ())); ns.resize (n); @@ -606,12 +785,16 @@ namespace build2 // Parse the next chunk of target names after >, if any. // next (t, tt); - if (start_names (tt)) - parse_names (t, tt, ns, pattern_mode::preserve); + ns_last = nullopt; // To detect <...><...>. + while (start_names (tt)) + { + ns_last = chunk {ns.size (), get_location (t)}; + parse_names (t, tt, ns, pattern_mode::preserve, true /* chunk */); + } } - if (!ans.empty ()) - ans.resize (ns.size ()); // Catch up with the final chunk. + if (!gns.empty ()) + gns.resize (ns.size ()); // Catch up with the final chunk. if (tt != type::colon) fail (t) << "expected ':' instead of " << t; @@ -630,10 +813,7 @@ namespace build2 if (ns.empty ()) fail (t) << "expected target before ':'"; - if (at.first) - fail (at.second) << "attributes before target"; - else - attributes_pop (); + attributes as (attributes_pop ()); // Call the specified parsing function (variable value/block) for // one/each pattern/target. We handle multiple targets by replaying @@ -642,10 +822,11 @@ namespace build2 // evaluated. The function signature is: // // void (token& t, type& tt, + // optional<bool> member, // true -- explict, false -- ad hoc // optional<pattern_type>, const target_type* pat_tt, string pat, // const location& pat_loc) // - // Note that the target and its ad hoc members are inserted implied + // Note that the target and its group members are inserted implied // but this flag can be cleared and default_target logic applied if // appropriate. // @@ -742,16 +923,20 @@ namespace build2 if (ttype == nullptr) fail (nloc) << "unknown target type " << n.type; - f (t, tt, n.pattern, ttype, move (n.value), nloc); + f (t, tt, nullopt, n.pattern, ttype, move (n.value), nloc); }; auto for_each = [this, &trace, &for_one_pat, - &t, &tt, &ns, &nloc, &ans] (auto&& f) + &t, &tt, &as, &ns, &nloc, &gns] (auto&& f) { + // We need replay if we have multiple targets or group members. + // // Note: watch out for an out-qualified single target (two names). // replay_guard rg (*this, - ns.size () > 2 || (ns.size () == 2 && !ns[0].pair)); + ns.size () > 2 || + (ns.size () == 2 && !ns[0].pair) || + !gns.empty ()); for (size_t i (0), e (ns.size ()); i != e; ) { @@ -765,11 +950,15 @@ namespace build2 // if (n.pattern) { + if (!as.empty ()) + fail (as.loc) << "attributes before target type/pattern"; + if (n.pair) fail (nloc) << "out-qualified target type/pattern"; - if (!ans.empty () && !ans[i].ns.empty ()) - fail (ans[i].loc) << "ad hoc member in target type/pattern"; + if (!gns.empty () && !gns[i].ns.empty ()) + fail (gns[i].member_loc) + << "group member in target type/pattern"; if (*n.pattern == pattern_type::regex_substitution) fail (nloc) << "regex substitution " << n << " without " @@ -779,24 +968,47 @@ namespace build2 } else { - name o (n.pair ? move (ns[++i]) : name ()); - enter_target tg (*this, - move (n), - move (o), - true /* implied */, - nloc, - trace); - - // Enter ad hoc members. - // - if (!ans.empty ()) + bool expl; + vector<reference_wrapper<target>> gms; { - // Note: index after the pair increment. + name o (n.pair ? move (ns[++i]) : name ()); + enter_target tg (*this, + move (n), + move (o), + true /* implied */, + nloc, + trace); + + if (!as.empty ()) + apply_target_attributes (*target_, as); + + // Enter group members. // - enter_adhoc_members (move (ans[i]), true /* implied */); + if (!gns.empty ()) + { + // Note: index after the pair increment. + // + group_names_loc& g (gns[i]); + expl = g.expl; + + if (expl && !target_->is_a<group> ()) + fail (g.group_loc) << *target_ << " is not group target"; + + gms = expl + ? enter_explicit_members (move (g), true /* implied */) + : enter_adhoc_members (move (g), true /* implied */); + } + + f (t, tt, nullopt, nullopt, nullptr, string (), location ()); } - f (t, tt, nullopt, nullptr, string (), location ()); + for (target& gm: gms) + { + rg.play (); // Replay. + + enter_target tg (*this, gm); + f (t, tt, expl, nullopt, nullptr, string (), location ()); + } } if (++i != e) @@ -850,12 +1062,15 @@ namespace build2 ploc = get_location (t); pns = parse_names (t, tt, pattern_mode::preserve); - // Target-specific variable assignment. + // Target type/pattern-specific variable assignment. // if (tt == type::assign || tt == type::prepend || tt == type::append) { - if (!ans.empty ()) - fail (ans[0].loc) << "ad hoc member in target type/pattern"; + // Note: ns contains single target name. + // + if (!gns.empty ()) + fail (gns[0].member_loc) + << "group member in target type/pattern"; // Note: see the same code below if changing anything here. // @@ -874,6 +1089,7 @@ namespace build2 for_one_pat ( [this, &var, akind, &aloc] ( token& t, type& tt, + optional<bool>, optional<pattern_type> pt, const target_type* ptt, string pat, const location& ploc) { @@ -886,6 +1102,10 @@ namespace build2 nloc); next_after_newline (t, tt); + + if (!as.empty ()) + fail (as.loc) << "attributes before target type/pattern"; + continue; // Just a target type/pattern-specific var assignment. } @@ -915,6 +1135,7 @@ namespace build2 for_one_pat ( [this] ( token& t, type& tt, + optional<bool>, optional<pattern_type> pt, const target_type* ptt, string pat, const location& ploc) { @@ -934,8 +1155,14 @@ namespace build2 if (pns.empty () && tt != type::percent && tt != type::multi_lcbrace) { - if (!ans.empty ()) - fail (ans[0].loc) << "ad hoc member in target type/pattern"; + // Note: ns contains single target name. + // + if (!gns.empty ()) + fail (gns[0].member_loc) + << "group member in target type/pattern"; + + if (!as.empty ()) + fail (as.loc) << "attributes before target type/pattern"; continue; } @@ -943,6 +1170,38 @@ namespace build2 // Ok, this is an ad hoc pattern rule. // + // First process the attributes. + // + string rn; + { + const location& l (as.loc); + + for (auto& a: as) + { + const string& n (a.name); + value& v (a.value); + + // rule_name= + // + if (n == "rule_name") + { + try + { + rn = convert<string> (move (v)); + + if (rn.empty ()) + throw invalid_argument ("empty name"); + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else + fail (l) << "unknown ad hoc pattern rule attribute " << a; + } + } + // What should we do if we have neither prerequisites nor recipes? // While such a declaration doesn't make much sense, it can happen, // for example, with an empty variable expansion: @@ -979,7 +1238,8 @@ namespace build2 // semantics is not immediately obvious. Whatever we decide, it // should be consistent with the target type/pattern-specific // variables where it is interpreted as a scope (and which doesn't - // feel like the best option for pattern rules). + // feel like the best option for pattern rules). See also depdb + // dyndep --update-* patterns. // auto check_pattern = [this] (name& n, const location& loc) { @@ -1003,22 +1263,33 @@ namespace build2 check_pattern (n, nloc); - // Verify all the ad hoc members are patterns or substitutions and - // of the correct type. + // If we have group members, verify all the members are patterns or + // substitutions (ad hoc) or subsitutions (explicit) and of the + // correct pattern type. A rule for an explicit group that wishes to + // match based on some of its members feels far fetched. + // + // For explicit groups the use-case is to inject static members + // which could otherwise be tedious to specify for each group. // - names ns (ans.empty () ? names () : move (ans[0].ns)); - const location& aloc (ans.empty () ? location () : ans[0].loc); + const location& mloc (gns.empty () ? location () : gns[0].member_loc); + names ns (gns.empty () ? names () : move (gns[0].ns)); + bool expl (gns.empty () ? false : gns[0].expl); for (name& n: ns) { if (!n.pattern || !(*n.pattern == pt || (st && *n.pattern == *st))) { - fail (aloc) << "expected " << pn << " pattern or substitution " + fail (mloc) << "expected " << pn << " pattern or substitution " << "instead of " << n; } if (*n.pattern != pattern_type::regex_substitution) - check_pattern (n, aloc); + { + if (expl) + fail (mloc) << "explicit group member pattern " << n; + + check_pattern (n, mloc); + } } // The same for prerequisites except here we can have non-patterns. @@ -1038,14 +1309,18 @@ namespace build2 } } - // Derive the rule name. It must be unique in this scope. + // Derive the rule name unless specified explicitly. It must be + // unique in this scope. // // It would have been nice to include the location but unless we // include the absolute path to the buildfile (which would be // unwieldy), it could be ambigous. // - string rn ("<ad hoc pattern rule #" + - to_string (scope_->adhoc_rules.size () + 1) + '>'); + // NOTE: we rely on the <...> format in dump. + // + if (rn.empty ()) + rn = "<ad hoc pattern rule #" + + to_string (scope_->adhoc_rules.size () + 1) + '>'; auto& ars (scope_->adhoc_rules); @@ -1058,7 +1333,9 @@ namespace build2 const target_type* ttype (nullptr); if (i != ars.end ()) { - // @@ TODO: append ad hoc members, prereqs. + // @@ TODO: append ad hoc members, prereqs (we now have + // [rule_name=] which we can use to reference the same + // rule). // ttype = &(*i)->type; assert (false); @@ -1074,6 +1351,12 @@ namespace build2 if (ttype == nullptr) fail (nloc) << "unknown target type " << n.type; + if (!gns.empty ()) + { + if (ttype->is_a<group> () != expl) + fail (nloc) << "group type and target type mismatch"; + } + unique_ptr<adhoc_rule_pattern> rp; switch (pt) { @@ -1085,7 +1368,7 @@ namespace build2 rp.reset (new adhoc_rule_regex_pattern ( *scope_, rn, *ttype, move (n), nloc, - move (ns), aloc, + move (ns), mloc, move (pns), ploc)); break; case pattern_type::regex_substitution: @@ -1108,15 +1391,24 @@ namespace build2 for (shared_ptr<adhoc_rule>& pr: recipes) { + // Can be NULL if the recipe is disabled with a condition. + // + if (pr != nullptr) + { + pr->pattern = &rp; // Connect recipe to pattern. + rp.rules.push_back (move (pr)); + } + } + + // Register this adhoc rule for all its actions. + // + for (shared_ptr<adhoc_rule>& pr: rp.rules) + { adhoc_rule& r (*pr); - r.pattern = &rp; // Connect recipe to pattern. - rp.rules.push_back (move (pr)); - // Register this adhoc rule for all its actions. - // for (action a: r.actions) { - // This covers both duplicate recipe actions withing the rule + // This covers both duplicate recipe actions within the rule // pattern (similar to parse_recipe()) as well as conflicts // with other rules (ad hoc or not). // @@ -1146,6 +1438,44 @@ namespace build2 scope_->rules.insert ( a.meta_operation (), 0, *ttype, rp.rule_name, rp.fallback_rule_); + + // We also register for the dist meta-operation in order to + // inject additional prerequisites which may "pull" additional + // sources into the distribution. Unless there is an explicit + // recipe for dist. + // + // And the same for the configure meta-operation to, for + // example, make sure a hinted ad hoc rule matches. @@ Hm, + // maybe we fixed this with action-specific hints? But the + // injection part above may still apply. BTW, this is also + // required for see-through groups in order to resolve their + // member. + // + // Note also that the equivalent semantics for ad hoc recipes + // is provided by match_adhoc_recipe(). + // + if (a.meta_operation () == perform_id) + { + auto reg = [this, ttype, &rp, &r] (action ea) + { + for (shared_ptr<adhoc_rule>& pr: rp.rules) + for (action a: pr->actions) + if (ea == a) + return; + + scope_->rules.insert (ea, *ttype, rp.rule_name, r); + }; + + reg (action (dist_id, a.operation ())); + reg (action (configure_id, a.operation ())); + } + + // @@ TODO: if this rule does dynamic member discovery of a + // see-through target group, then we may also need to + // register update for other meta-operations (see, for + // example, wildcard update registration in the cli + // module). BTW, we can now detect such a target via + // its target type flags. } } } @@ -1189,6 +1519,7 @@ namespace build2 st = token (t), // Save start token (will be gone on replay). recipes = small_vector<shared_ptr<adhoc_rule>, 1> ()] (token& t, type& tt, + optional<bool> gm, // true -- explicit, false -- ad hoc optional<pattern_type> pt, const target_type* ptt, string pat, const location& ploc) mutable { @@ -1202,7 +1533,14 @@ namespace build2 // next (t, tt); // Newline. next (t, tt); // First token inside the variable block. - parse_variable_block (t, tt, pt, ptt, move (pat), ploc); + + // For explicit groups we only assign variables on the group + // omitting the members. + // + if (!gm || !*gm) + parse_variable_block (t, tt, pt, ptt, move (pat), ploc); + else + skip_block (t, tt); if (tt != type::rcbrace) fail (t) << "expected '}' instead of " << t; @@ -1218,6 +1556,16 @@ namespace build2 else rt = st; + // If this is a group member then we know we are replaying and + // can skip the recipe. + // + if (gm) + { + replay_skip (); + next (t, tt); + return; + } + if (pt) fail (rt) << "unexpected recipe after target type/pattern" << info << "ad hoc pattern rule may not be combined with other " @@ -1238,7 +1586,7 @@ namespace build2 // Note also that we treat this as an explicit dependency // declaration (i.e., not implied). // - enter_targets (move (ns), nloc, move (ans), 0); + enter_targets (move (ns), nloc, move (gns), 0, as); } continue; @@ -1253,7 +1601,8 @@ namespace build2 if (!start_names (tt)) fail (t) << "unexpected " << t; - // @@ PAT: currently we pattern-expand target-specific var names. + // @@ PAT: currently we pattern-expand target-specific var names (see + // also parse_import()). // const location ploc (get_location (t)); names pns (parse_names (t, tt, pattern_mode::expand)); @@ -1288,6 +1637,7 @@ namespace build2 for_each ( [this, &var, akind, &aloc] ( token& t, type& tt, + optional<bool> gm, optional<pattern_type> pt, const target_type* ptt, string pat, const location& ploc) { @@ -1296,7 +1646,18 @@ namespace build2 *pt, *ptt, move (pat), ploc, var, akind, aloc); else - parse_variable (t, tt, var, akind); + { + // Skip explicit group members (see the block case above for + // background). + // + if (!gm || !*gm) + parse_variable (t, tt, var, akind); + else + { + next (t, tt); + skip_line (t, tt); + } + } }); next_after_newline (t, tt); @@ -1314,8 +1675,9 @@ namespace build2 parse_dependency (t, tt, move (ns), nloc, - move (ans), - move (pns), ploc); + move (gns), + move (pns), ploc, + as); } continue; @@ -1539,7 +1901,7 @@ namespace build2 // Parse a recipe chain. // // % [<attrs>] [<buildspec>] - // [if|switch ...] + // [if|if!|switch ...] // {{ [<lang> ...] // ... // }} @@ -1558,10 +1920,27 @@ namespace build2 // if (target_ != nullptr) { + // @@ What if some members are added later? + // + // @@ Also, what happends if redeclared as real dependency, do we + // upgrade the members? + // if (target_->decl != target_decl::real) { - for (target* m (target_); m != nullptr; m = m->adhoc_member) - m->decl = target_decl::real; + target_->decl = target_decl::real; + + if (group* g = target_->is_a<group> ()) + { + for (const target& m: g->static_members) + const_cast<target&> (m).decl = target_decl::real; // During load. + } + else + { + for (target* m (target_->adhoc_member); + m != nullptr; + m = m->adhoc_member) + m->decl = target_decl::real; + } if (default_target_ == nullptr) default_target_ = target_; @@ -1574,7 +1953,15 @@ namespace build2 t = start; tt = t.type; for (size_t i (0); tt == type::percent || tt == type::multi_lcbrace; ++i) { - recipes.push_back (nullptr); // For missing else/default (see below). + // For missing else/default (see below). + // + // Note that it may remain NULL if we have, say, an if-condition that + // evaluates to false and no else. While it may be tempting to get rid + // of such "holes", it's not easy due to the replay semantics (see the + // target_ != nullptr block below). So we expect the caller to be + // prepared to handle this. + // + recipes.push_back (nullptr); attributes as; buildspec bs; @@ -1629,6 +2016,10 @@ namespace build2 // location loc (get_location (st)); + // @@ We could add an attribute (name= or recipe_name=) to allow + // the user specify a friendly name for diagnostics, similar + // to rule_name. + shared_ptr<adhoc_rule> ar; if (!lang) { @@ -1738,7 +2129,7 @@ namespace build2 for (metaopspec& m: d.bs) { - meta_operation_id mi (ctx.meta_operation_table.find (m.name)); + meta_operation_id mi (ctx->meta_operation_table.find (m.name)); if (mi == 0) fail (l) << "unknown meta-operation " << m.name; @@ -1748,7 +2139,7 @@ namespace build2 if (mf == nullptr) fail (l) << "project " << *root_ << " does not support meta-" - << "operation " << ctx.meta_operation_table[mi].name; + << "operation " << ctx->meta_operation_table[mi].name; for (opspec& o: m) { @@ -1764,7 +2155,7 @@ namespace build2 fail (l) << "default operation in recipe action" << endf; } else - oi = ctx.operation_table.find (o.name); + oi = ctx->operation_table.find (o.name); if (oi == 0) fail (l) << "unknown operation " << o.name; @@ -1773,7 +2164,7 @@ namespace build2 if (of == nullptr) fail (l) << "project " << *root_ << " does not support " - << "operation " << ctx.operation_table[oi]; + << "operation " << ctx->operation_table[oi]; // Note: for now always inner (see match_rule() for details). // @@ -1841,6 +2232,9 @@ namespace build2 } target_->adhoc_recipes.push_back (r); + + // Note that "registration" of configure_* and dist_* actions + // (similar to ad hoc rules) is provided by match_adhoc_recipe(). } } @@ -1868,8 +2262,7 @@ namespace build2 // // TODO: handle and erase common attributes if/when we have any. // - as = move (attributes_top ()); - attributes_pop (); + as = attributes_pop (); // Handle the buildspec. // @@ -1925,7 +2318,7 @@ namespace build2 // handy if we want to provide a custom recipe but only on certain // platforms or some such). - if (n == "if") + if (n == "if" || n == "if!") { parse_if_else (t, tt, true /* multi */, parse_block); continue; @@ -1985,13 +2378,97 @@ namespace build2 } } - void parser:: - enter_adhoc_members (adhoc_names_loc&& ans, bool implied) + vector<reference_wrapper<target>> parser:: + enter_explicit_members (group_names_loc&& gns, bool implied) + { + tracer trace ("parser::enter_explicit_members", &path_); + + names& ns (gns.ns); + const location& loc (gns.member_loc); + + vector<reference_wrapper<target>> r; + r.reserve (ns.size ()); + + group& g (target_->as<group> ()); + auto& ms (g.static_members); + + for (size_t i (0); i != ns.size (); ++i) + { + name&& n (move (ns[i])); + name&& o (n.pair ? move (ns[++i]) : name ()); + + if (n.qualified ()) + fail (loc) << "project name in target " << n; + + // We derive the path unless the target name ends with the '...' escape + // which here we treat as the "let the rule derive the path" indicator + // (see target::split_name() for details). This will only be useful for + // referring to group members that are managed by the group's matching + // rule. Note also that omitting '...' for such a member could be used + // to override the file name, provided the rule checks if the path has + // already been derived before doing it itself. + // + // @@ What can the ad hoc recipe/rule do differently here? Maybe get + // path from dynamic targets? Maybe we will have custom path + // derivation support in buildscript in the future? + // + bool escaped; + { + const string& v (n.value); + size_t p (v.size ()); + + escaped = (p > 3 && + v[--p] == '.' && v[--p] == '.' && v[--p] == '.' && + v[--p] != '.'); + } + + target& m (enter_target::insert_target (*this, + move (n), move (o), + implied, + loc, trace)); + + if (g == m) + fail (loc) << "explicit group member " << m << " is group itself"; + + // Add as static member skipping duplicates. + // + if (find (ms.begin (), ms.end (), m) == ms.end ()) + { + if (m.group == nullptr) + m.group = &g; + else if (m.group != &g) + fail (loc) << g << " group member " << m << " already belongs to " + << "group " << *m.group; + + ms.push_back (m); + } + + if (!escaped) + { + if (file* ft = m.is_a<file> ()) + ft->derive_path (); + } + + r.push_back (m); + } + + return r; + } + + vector<reference_wrapper<target>> parser:: + enter_adhoc_members (group_names_loc&& gns, bool implied) { tracer trace ("parser::enter_adhoc_members", &path_); - names& ns (ans.ns); - const location& loc (ans.loc); + names& ns (gns.ns); + const location& loc (gns.member_loc); + + if (target_->is_a<group> ()) + fail (loc) << "ad hoc group primary member " << *target_ + << " is explicit group"; + + vector<reference_wrapper<target>> r; + r.reserve (ns.size ()); for (size_t i (0); i != ns.size (); ++i) { @@ -2019,14 +2496,16 @@ namespace build2 v[--p] != '.'); } - target& at ( - enter_target::insert_target (*this, - move (n), move (o), - implied, - loc, trace)); + target& m (enter_target::insert_target (*this, + move (n), move (o), + implied, + loc, trace)); - if (target_ == &at) - fail (loc) << "ad hoc group member " << at << " is primary target"; + if (target_ == &m) + fail (loc) << "ad hoc group member " << m << " is primary target"; + + if (m.is_a<group> ()) + fail (loc) << "ad hoc group member " << m << " is explicit group"; // Add as an ad hoc member at the end of the chain skipping duplicates. // @@ -2034,7 +2513,7 @@ namespace build2 const_ptr<target>* mp (&target_->adhoc_member); for (; *mp != nullptr; mp = &(*mp)->adhoc_member) { - if (*mp == &at) + if (*mp == &m) { mp = nullptr; break; @@ -2043,30 +2522,41 @@ namespace build2 if (mp != nullptr) { - *mp = &at; - at.group = target_; + if (m.group == nullptr) + m.group = target_; + else if (m.group != target_) + fail (loc) << *target_ << " ad hoc group member " << m + << " already belongs to group " << *m.group; + *mp = &m; } } if (!escaped) { - if (file* ft = at.is_a<file> ()) + if (file* ft = m.is_a<file> ()) ft->derive_path (); } + + r.push_back (m); } + + return r; } - small_vector<reference_wrapper<target>, 1> parser:: + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> parser:: enter_targets (names&& tns, const location& tloc, // Target names. - adhoc_names&& ans, // Ad hoc target names. - size_t prereq_size) + group_names&& gns, // Group member names. + size_t prereq_size, + const attributes& tas) // Target attributes. { - // Enter all the targets (normally we will have just one) and their ad hoc - // groups. + // Enter all the targets (normally we will have just one) and their group + // members. // tracer trace ("parser::enter_targets", &path_); - small_vector<reference_wrapper<target>, 1> tgs; + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> tgs; for (size_t i (0); i != tns.size (); ++i) { @@ -2088,13 +2578,24 @@ namespace build2 false /* implied */, tloc, trace); - // Enter ad hoc members. + if (!tas.empty ()) + apply_target_attributes (*target_, tas); + + // Enter group members. // - if (!ans.empty ()) + vector<reference_wrapper<target>> gms; + if (!gns.empty ()) { // Note: index after the pair increment. // - enter_adhoc_members (move (ans[i]), false /* implied */); + group_names_loc& g (gns[i]); + + if (g.expl && !target_->is_a<group> ()) + fail (g.group_loc) << *target_ << " is not group target"; + + gms = g.expl + ? enter_explicit_members (move (g), false /* implied */) + : enter_adhoc_members (move (g), false /* implied */); } if (default_target_ == nullptr) @@ -2102,17 +2603,97 @@ namespace build2 target_->prerequisites_state_.store (2, memory_order_relaxed); target_->prerequisites_.reserve (prereq_size); - tgs.push_back (*target_); + tgs.emplace_back (*target_, move (gms)); } return tgs; } void parser:: + apply_target_attributes (target& t, const attributes& as) + { + const location& l (as.loc); + + for (auto& a: as) + { + const string& n (a.name); + const value& v (a.value); + + // rule_hint= + // liba@rule_hint= + // + size_t p (string::npos); + if (n == "rule_hint" || + ((p = n.find ('@')) != string::npos && + n.compare (p + 1, string::npos, "rule_hint") == 0)) + { + // Resolve target type, if specified. + // + const target_type* tt (nullptr); + if (p != string::npos) + { + string t (n, 0, p); + tt = scope_->find_target_type (t); + + if (tt == nullptr) + fail (l) << "unknown target type " << t << " in rule_hint " + << "attribute"; + } + + // The rule hint value is vector<pair<optional<string>, string>> where + // the first half is the operation and the second half is the hint. + // Absent operation is used as a fallback for update/clean. + // + const names& ns (v.as<names> ()); + + for (auto i (ns.begin ()); i != ns.end (); ++i) + { + operation_id oi (default_id); + if (i->pair) + { + const name& n (*i++); + + if (!n.simple ()) + fail (l) << "expected operation name instead of " << n + << " in rule_hint attribute"; + + const string& v (n.value); + + if (!v.empty ()) + { + oi = ctx->operation_table.find (v); + + if (oi == 0) + fail (l) << "unknown operation " << v << " in rule_hint " + << "attribute"; + + if (root_->root_extra->operations[oi] == nullptr) + fail (l) << "project " << *root_ << " does not support " + << "operation " << ctx->operation_table[oi] + << " specified in rule_hint attribute"; + } + } + + const name& n (*i); + + if (!n.simple () || n.empty ()) + fail (l) << "expected hint instead of " << n << " in rule_hint " + << "attribute"; + + t.rule_hints.insert (tt, oi, n.value); + } + } + else + fail (l) << "unknown target attribute " << a; + } + } + + void parser:: parse_dependency (token& t, token_type& tt, names&& tns, const location& tloc, // Target names. - adhoc_names&& ans, // Ad hoc target names. - names&& pns, const location& ploc) // Prereq names. + group_names&& gns, // Group member names. + names&& pns, const location& ploc, // Prereq names. + const attributes& tas) // Target attributes. { // Parse a dependency chain and/or a target/prerequisite-specific variable // assignment/block and/or recipe block(s). @@ -2122,33 +2703,72 @@ namespace build2 // tracer trace ("parser::parse_dependency", &path_); + // Diagnose conditional prerequisites. Note that we want to diagnose this + // even if pns is empty (think empty variable expansion; the literal "no + // prerequisites" case is handled elsewhere). + // + // @@ TMP For now we only do it during the dist meta-operation. In the + // future we should tighten this to any meta-operation provided + // the dist module is loaded. + // + // @@ TMP For now it's a warning because we have dependencies like + // cli.cxx{foo}: cli{foo} which are not currently possible to + // rewrite (cli.cxx{} is not always registered). + // + if (condition_ && + ctx->current_mif != nullptr && + ctx->current_mif->id == dist_id) + { + // Only issue the warning for the projects being distributed. In + // particular, this makes sure we don't complain about imported + // projects. Note: use amalgamation to cover bundled subprojects. + // + auto* dm (root_->bundle_scope ()->find_module<dist::module> ( + dist::module::name)); + + if (dm != nullptr && dm->distributed) + { + warn (tloc) << "conditional dependency declaration may result in " + << "incomplete distribution" << + info (ploc) << "prerequisite declared here" << + info (*condition_) << "conditional buildfile fragment starts here" << + info << "instead use 'include' prerequisite-specific variable to " + << "conditionally include prerequisites" << + info << "for example: " + << "<target>: <prerequisite>: include = (<condition>)" << + info << "for details, see https://github.com/build2/HOWTO/blob/" + << "master/entries/keep-build-graph-config-independent.md"; + } + } + // First enter all the targets. // - small_vector<reference_wrapper<target>, 1> tgs ( - enter_targets (move (tns), tloc, move (ans), pns.size ())); + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> + tgs (enter_targets (move (tns), tloc, move (gns), pns.size (), tas)); // Now enter each prerequisite into each target. // - for (name& pn: pns) + for (auto i (pns.begin ()); i != pns.end (); ++i) { // We cannot reuse the names if we (potentially) may need to pass them // as targets in case of a chain (see below). // - name n (tt != type::colon ? move (pn) : pn); + name n (tt != type::colon ? move (*i) : *i); // See also scope::find_prerequisite_key(). // auto rp (scope_->find_target_type (n, ploc)); - const target_type* tt (rp.first); + const target_type* t (rp.first); optional<string>& e (rp.second); - if (tt == nullptr) + if (t == nullptr) fail (ploc) << "unknown target type " << n.type; // Current dir collapses to an empty one. // if (!n.dir.empty ()) - n.dir.normalize (false, true); + n.dir.normalize (false /* actual */, true); // @@ OUT: for now we assume the prerequisite's out is undetermined. The // only way to specify an src prerequisite will be with the explicit @@ -2159,10 +2779,47 @@ namespace build2 // a special indicator. Also, one can easily and natually suppress any // searches by specifying the absolute path. // + name o; + if (n.pair) + { + assert (n.pair == '@'); + + ++i; + o = tt != type::colon ? move (*i) : *i; + + if (!o.directory ()) + fail (ploc) << "expected directory after '@'"; + + o.dir.normalize (); // Note: don't collapse current to empty. + + // Make sure out and src are parallel unless both were specified as + // absolute. We make an exception for this case because out may be + // used to "tag" imported targets (see cc::search_library()). So it's + // sort of the "I know what I am doing" escape hatch (it would have + // been even better to verify such a target is outside any project + // but that won't be cheap). + // + // For now we require that both are either relative or absolute. + // + // See similar code for targets in scope::find_target_type(). + // + if (n.dir.absolute () && o.dir.absolute ()) + ; + else if (n.dir.empty () && o.dir.current ()) + ; + else if (o.dir.relative () && + n.dir.relative () && + o.dir == n.dir) + ; + else + fail (ploc) << "prerequisite output directory " << o.dir + << " must be parallel to source directory " << n.dir; + } + prerequisite p (move (n.proj), - *tt, + *t, move (n.dir), - dir_path (), + move (o.dir), move (n.value), move (e), *scope_); @@ -2171,7 +2828,7 @@ namespace build2 { // Move last prerequisite (which will normally be the only one). // - target& t (*i); + target& t (i->first); t.prerequisites_.push_back (++i == e ? move (p) : prerequisite (p, memory_order_relaxed)); @@ -2184,20 +2841,42 @@ namespace build2 // // We handle multiple targets and/or prerequisites by replaying the tokens // (see the target-specific case comments for details). The function - // signature is: + // signature for for_each_t (see for_each on the gm argument semantics): + // + // void (token& t, type& tt, optional<bool> gm) + // + // And for for_each_p: // // void (token& t, type& tt) // auto for_each_t = [this, &t, &tt, &tgs] (auto&& f) { - replay_guard rg (*this, tgs.size () > 1); + // We need replay if we have multiple targets or group members. + // + replay_guard rg (*this, tgs.size () > 1 || !tgs[0].second.empty ()); for (auto ti (tgs.begin ()), te (tgs.end ()); ti != te; ) { - target& tg (*ti); - enter_target tgg (*this, tg); + target& tg (ti->first); + const vector<reference_wrapper<target>>& gms (ti->second); - f (t, tt); + { + enter_target g (*this, tg); + f (t, tt, nullopt); + } + + if (!gms.empty ()) + { + bool expl (tg.is_a<group> ()); + + for (target& gm: gms) + { + rg.play (); // Replay. + + enter_target g (*this, gm); + f (t, tt, expl); + } + } if (++ti != te) rg.play (); // Replay. @@ -2210,8 +2889,8 @@ namespace build2 for (auto ti (tgs.begin ()), te (tgs.end ()); ti != te; ) { - target& tg (*ti); - enter_target tgg (*this, tg); + target& tg (ti->first); + enter_target g (*this, tg); for (size_t pn (tg.prerequisites_.size ()), pi (pn - pns.size ()); pi != pn; ) @@ -2254,7 +2933,7 @@ namespace build2 this, st = token (t), // Save start token (will be gone on replay). recipes = small_vector<shared_ptr<adhoc_rule>, 1> ()] - (token& t, type& tt) mutable + (token& t, type& tt, optional<bool> gm) mutable { token rt; // Recipe start token. @@ -2264,7 +2943,14 @@ namespace build2 { next (t, tt); // Newline. next (t, tt); // First token inside the variable block. - parse_variable_block (t, tt); + + // Skip explicit group members (see the block case above for + // background). + // + if (!gm || !*gm) + parse_variable_block (t, tt); + else + skip_block (t, tt); if (tt != type::rcbrace) fail (t) << "expected '}' instead of " << t; @@ -2280,6 +2966,16 @@ namespace build2 else rt = st; + // If this is a group member then we know we are replaying and can + // skip the recipe. + // + if (gm) + { + replay_skip (); + next (t, tt); + return; + } + parse_recipe (t, tt, rt, recipes); }; @@ -2289,21 +2985,6 @@ namespace build2 return; } - // What should we do if there are no prerequisites (for example, because - // of an empty wildcard result)? We can fail or we can ignore. In most - // cases, however, this is probably an error (for example, forgetting to - // checkout a git submodule) so let's not confuse the user and fail (one - // can always handle the optional prerequisites case with a variable and - // an if). - // - if (pns.empty ()) - fail (ploc) << "no prerequisites in dependency chain or prerequisite-" - << "specific variable assignment"; - - next_with_attributes (t, tt); // Recognize attributes after `:`. - - auto at (attributes_push (t, tt)); - // If we are here, then this can be one of three things: // // 1. A prerequisite-specific variable bloc: @@ -2317,10 +2998,37 @@ namespace build2 // // foo: bar: x = y // - // 3. A further dependency chain : + // 3. A further dependency chain: // // foo: bar: baz ... // + // What should we do if there are no prerequisites, for example, because + // of an empty wildcard result or empty variable expansion? We can fail or + // we can ignore. In most cases, however, this is probably an error (for + // example, forgetting to checkout a git submodule) so let's not confuse + // the user and fail (one can always handle the optional prerequisites + // case with a variable and an if). + // + // On the other hand, we allow just empty prerequisites (which is also the + // more common case by far) and so it's strange that we don't allow the + // same with, say, `include = false`: + // + // exe{foo}: cxx{$empty} # Ok. + // exe{foo}: cxx{$empty}: include = false # Not Ok? + // + // So let's ignore in the first two cases (variable block and assignment) + // for consistency. The dependency chain is iffy both conceptually and + // implementation-wise (it could be followed by a variable block). So + // let's keep it an error for now. + // + // Note that the syntactically-empty prerequisite list is still an error: + // + // exe{foo}: : include = false # Error. + // + next_with_attributes (t, tt); // Recognize attributes after `:`. + + auto at (attributes_push (t, tt)); + if (tt == type::newline || tt == type::eos) { attributes_pop (); // Must be none since can't be standalone. @@ -2335,15 +3043,22 @@ namespace build2 // Parse the block for each prerequisites of each target. // - for_each_p ([this] (token& t, token_type& tt) - { - next (t, tt); // First token inside the block. + if (!pns.empty ()) + for_each_p ([this] (token& t, token_type& tt) + { + next (t, tt); // First token inside the block. - parse_variable_block (t, tt); + parse_variable_block (t, tt); - if (tt != type::rcbrace) - fail (t) << "expected '}' instead of " << t; - }); + if (tt != type::rcbrace) + fail (t) << "expected '}' instead of " << t; + }); + else + { + skip_block (t, tt); + if (tt != type::rcbrace) + fail (t) << "expected '}' instead of " << t; + } next (t, tt); // Presumably newline after '}'. next_after_newline (t, tt, '}'); // Should be on its own line. @@ -2366,10 +3081,13 @@ namespace build2 // Parse the assignment for each prerequisites of each target. // - for_each_p ([this, &var, at] (token& t, token_type& tt) - { - parse_variable (t, tt, var, at); - }); + if (!pns.empty ()) + for_each_p ([this, &var, at] (token& t, token_type& tt) + { + parse_variable (t, tt, var, at); + }); + else + skip_line (t, tt); next_after_newline (t, tt); @@ -2388,6 +3106,13 @@ namespace build2 // else { + if (pns.empty ()) + fail (ploc) << "no prerequisites in dependency chain"; + + // @@ This is actually ambiguous: prerequisite or target attributes + // (or both or neither)? Perhaps this should be prerequisites for + // the same reason as below (these are prerequsites first). + // if (at.first) fail (at.second) << "attributes before prerequisites"; else @@ -2399,30 +3124,35 @@ namespace build2 // we just say that the dependency chain is equivalent to specifying // each dependency separately. // - // Also note that supporting ad hoc target group specification in - // chains will be complicated. For example, what if prerequisites that - // have ad hoc targets don't end up being chained? Do we just silently - // drop them? Also, these are prerequsites first that happened to be - // reused as target names so perhaps it is the right thing not to - // support, conceptually. + // Also note that supporting target group specification in chains will + // be complicated. For example, what if prerequisites that have group + // members don't end up being chained? Do we just silently drop them? + // Also, these are prerequsites first that happened to be reused as + // target names so perhaps it is the right thing not to support, + // conceptually. // parse_dependency (t, tt, move (pns), ploc, - {} /* ad hoc target name */, - move (ns), loc); + {} /* group names */, + move (ns), loc, + attributes () /* target attributes */); } } } void parser:: - source (istream& is, const path_name& in, const location& loc, bool deft) + source_buildfile (istream& is, + const path_name& in, + const location& loc, + bool deft) { - tracer trace ("parser::source", &path_); + tracer trace ("parser::source_buildfile", &path_); l5 ([&]{trace (loc) << "entering " << in;}); - if (in.path != nullptr) - enter_buildfile (*in.path); + const buildfile* bf (in.path != nullptr + ? &enter_buildfile (*in.path) + : nullptr); const path_name* op (path_); path_ = ∈ @@ -2448,7 +3178,7 @@ namespace build2 if (deft) { - process_default_target (t); + process_default_target (t, bf); default_target_ = odt; } @@ -2492,10 +3222,10 @@ namespace build2 try { ifdstream ifs (p); - source (ifs, - path_name (p), - get_location (t), - false /* default_target */); + source_buildfile (ifs, + path_name (p), + get_location (t), + false /* default_target */); } catch (const io_error& e) { @@ -2617,19 +3347,35 @@ namespace build2 l6 ([&]{trace (l) << "absolute path " << p;}); - if (!root_->buildfiles.insert (p).second) // Note: may be "new" root. + // Note: may be "new" root. + // + if (!root_->root_extra->insert_buildfile (p)) { l5 ([&]{trace (l) << "skipping already included " << p;}); continue; } + // Note: see a variant of this in parse_import(). + // + // Clear/restore if/switch location. + // + // We do it here but not in parse_source since the included buildfile is + // in a sense expected to be a standalone entity (think a file included + // from an export stub). + // + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = nullopt; + try { ifdstream ifs (p); - source (ifs, - path_name (p), - get_location (t), - true /* default_target */); + source_buildfile (ifs, + path_name (p), + get_location (t), + true /* default_target */); } catch (const io_error& e) { @@ -2681,13 +3427,16 @@ namespace build2 [] (const string& s) {return s.c_str ();}); cargs.push_back (nullptr); + // Note: we are in the serial load phase and so no diagnostics buffering + // is needed. + // process pr (run_start (3 /* verbosity */, cargs, 0 /* stdin */, -1 /* stdout */, - true /* error */, - dir_path () /* cwd */, + 2 /* stderr */, nullptr /* env */, + dir_path () /* cwd */, l)); try { @@ -2709,10 +3458,10 @@ namespace build2 dr << info (l) << "while parsing " << args[0] << " output"; }); - source (is, - path_name ("<stdout>"), - l, - false /* default_target */); + source_buildfile (is, + path_name ("<stdout>"), + l, + false /* default_target */); } is.close (); // Detect errors. @@ -2726,7 +3475,7 @@ namespace build2 // caused by that and let run_finish() deal with it. } - run_finish (cargs, pr, l); + run_finish (cargs, pr, 2 /* verbosity */, false /* omit_normal */, l); next_after_newline (t, tt); } @@ -2787,18 +3536,23 @@ namespace build2 next_with_attributes (t, tt); // Get variable attributes, if any, and deal with the special config.* - // attributes. Since currently they can only appear in the config - // directive, we handle them in an ad hoc manner. + // attributes as well as null. Since currently they can only appear in the + // config directive, we handle them in an ad hoc manner. // attributes_push (t, tt); attributes& as (attributes_top ()); + bool nullable (false); optional<string> report; string report_var; for (auto i (as.begin ()); i != as.end (); ) { - if (i->name == "config.report") + if (i->name == "null") + { + nullable = true; + } + else if (i->name == "config.report") { try { @@ -2810,7 +3564,7 @@ namespace build2 report = move (v); else throw invalid_argument ( - "expected 'false' or format name instead of '" + v + "'"); + "expected 'false' or format name instead of '" + v + '\''); } catch (const invalid_argument& e) { @@ -2852,7 +3606,7 @@ namespace build2 if (report && *report != "false" && !config) { - if (!as.empty ()) + if (!as.empty () || nullable) fail (as.loc) << "unexpected attributes for report-only variable"; attributes_pop (); @@ -2954,6 +3708,9 @@ namespace build2 peeked ().value != "false") fail (loc) << var << " variable default value must be literal false"; + if (nullable) + fail (loc) << var << " variable must not be nullable"; + sflags |= config::save_false_omitted; } @@ -2972,14 +3729,48 @@ namespace build2 // all. // if (l.defined ()) + { + // Peek at the attributes to detect whether the value is NULL. + // + if (!dev && !nullable) + { + // Essentially a prefix of parse_variable_value(). + // + mode (lexer_mode::value, '@'); + next_with_attributes (t, tt); + attributes_push (t, tt, true); + for (const attribute& a: attributes_pop ()) + { + if (a.name == "null") + { + nullable = true; + break; + } + } + } + skip_line (t, tt); + } else { value lhs, rhs (parse_variable_value (t, tt, !dev /* mode */)); apply_value_attributes (&var, lhs, move (rhs), type::assign); + + if (!nullable) + nullable = lhs.null; + l = config::lookup_config (new_val, *root_, var, move (lhs), sflags); } } + + // If the variable is not nullable, verify the value is not NULL. + // + // Note that undefined is not the same as NULL (if it is undefined, we + // should either see the default value or if there is no default value, + // then the user is expected to handle the undefined case). + // + if (!nullable && l.defined () && l->null) + fail (loc) << "null value in non-nullable variable " << var; } // We will be printing the report at either level 2 (-v) or 3 (-V) @@ -2998,9 +3789,12 @@ namespace build2 // In a somewhat hackish way we pass the variable in an undefined // lookup. // + // Note: consistent with parse_variable_name() wrt overridability. + // l = lookup (); l.var = &root_->var_pool ().insert ( - move (report_var), true /* overridable */); + move (report_var), + report_var.find ('.') != string::npos /* overridable */); } if (l.var != nullptr) @@ -3080,118 +3874,274 @@ namespace build2 if (stage_ == stage::boot) fail (t) << "import during bootstrap"; - // General import format: + // General import form: // // import[?!] [<attrs>] <var> = [<attrs>] (<target>|<project>%<target>])+ // + // Special form for importing buildfiles: + // + // import[?!] [<attrs>] (<target>|<project>%<target>])+ + // bool opt (t.value.back () == '?'); - bool ph2 (opt || t.value.back () == '!'); + optional<string> ph2 (opt || t.value.back () == '!' + ? optional<string> (string ()) + : nullopt); // We are now in the normal lexing mode and we let the lexer handle `=`. // next_with_attributes (t, tt); - // Get variable attributes, if any, and deal with the special metadata - // attribute. Since currently it can only appear in the import directive, - // we handle it in an ad hoc manner. + // Get variable (or value, in the second form) attributes, if any, and + // deal with the special metadata and rule_hint attributes. Since + // currently they can only appear in the import directive, we handle them + // in an ad hoc manner. // attributes_push (t, tt); - attributes& as (attributes_top ()); - bool meta (false); - for (auto i (as.begin ()); i != as.end (); ) + bool meta (false); // Import with metadata. + bool once (false); // Import buildfile once. { - if (i->name == "metadata") - { - if (!ph2) - fail (as.loc) << "loading metadata requires immediate import" << - info << "consider using the import! directive instead"; + attributes& as (attributes_top ()); + const location& l (as.loc); - meta = true; - } - else + for (auto i (as.begin ()); i != as.end (); ) { - ++i; - continue; - } + const string& n (i->name); + value& v (i->value); - i = as.erase (i); - } + if (n == "metadata") + { + if (!ph2) + fail (l) << "loading metadata requires immediate import" << + info << "consider using the import! directive instead"; - if (tt != type::word) - fail (t) << "expected variable name instead of " << t; + meta = true; + } + else if (n == "once") + { + once = true; + } + else if (n == "rule_hint") + { + if (!ph2) + fail (l) << "rule hint can only be used with immediate import" << + info << "consider using the import! directive instead"; - const variable& var ( - parse_variable_name (move (t.value), get_location (t))); - apply_variable_attributes (var); + // Here we only allow a single name. + // + try + { + ph2 = convert<string> (move (v)); - if (var.visibility > variable_visibility::scope) - { - fail (t) << "variable " << var << " has " << var.visibility - << " visibility but is assigned in import"; + if (ph2->empty ()) + throw invalid_argument ("empty name"); + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else + { + ++i; + continue; + } + + i = as.erase (i); + } } - // Next should come the assignment operator. Note that we don't support + // Note that before supporting the second form (without <var>) we used to + // parse the value after assignment in the value mode. However, we don't + // really need to since what we should have is a bunch of target names. + // In other words, whatever the value mode does not treat as special + // compared to the normal mode (like `:`) would be illegal here. + // + // Note that we expant patterns for the ad hoc import case: + // + // import sub = */ + // + // @@ PAT: the only issue here is that we currently pattern-expand var + // name (same assue as with target-specific var names). + // + if (!start_names (tt)) + fail (t) << "expected variable name or buildfile target instead of " << t; + + location loc (get_location (t)); + names ns (parse_names (t, tt, pattern_mode::expand)); + + // Next could come the assignment operator. Note that we don't support // default assignment (?=) yet (could make sense when attempting to import // alternatives or some such). // - next (t, tt); + type atype; + const variable* var (nullptr); + if (tt == type::assign || tt == type::append || tt == type::prepend) + { + var = &parse_variable_name (move (ns), loc); + apply_variable_attributes (*var); + + if (var->visibility > variable_visibility::scope) + { + fail (loc) << "variable " << *var << " has " << var->visibility + << " visibility but is assigned in import"; + } - if (tt != type::assign && tt != type::append && tt != type::prepend) - fail (t) << "expected variable assignment instead of " << t; + atype = tt; + next_with_attributes (t, tt); + attributes_push (t, tt, true /* standalone */); - type atype (tt); - value& val (atype == type::assign - ? scope_->assign (var) - : scope_->append (var)); + if (!start_names (tt)) + fail (t) << "expected target to import instead of " << t; - // The rest should be a list of targets. Parse them similar to a value on - // the RHS of an assignment (attributes, etc). - // - // Note that we expant patterns for the ad hoc import case: - // - // import sub = */ + loc = get_location (t); + ns = parse_names (t, tt, pattern_mode::expand); + } + else if (tt == type::default_assign) + fail (t) << "default assignment not yet supported"; + + + // If there are any value attributes, roundtrip the names through the + // value applying the attributes. // - mode (lexer_mode::value, '@'); - next_with_attributes (t, tt); + if (!attributes_top ().empty ()) + { + value lhs, rhs (move (ns)); + apply_value_attributes (nullptr, lhs, move (rhs), type::assign); - if (tt == type::newline || tt == type::eos) - fail (t) << "expected target to import instead of " << t; + if (!lhs) + fail (loc) << "expected target to import instead of null value"; - const location loc (get_location (t)); + untypify (lhs, true /* reduce */); + ns = move (lhs.as<names> ()); + } + else + attributes_pop (); - if (value v = parse_value_with_attributes (t, tt, pattern_mode::expand)) + value* val (var != nullptr ? + &(atype == type::assign + ? scope_->assign (*var) + : scope_->append (*var)) + : nullptr); + + for (name& n: ns) { - names storage; - for (name& n: reverse (v, storage)) + // @@ Could this be an out-qualified ad hoc import? Yes, see comment + // about buildfile import in import_load(). + // + if (n.pair) + fail (loc) << "unexpected pair in import"; + + // See if we are importing a buildfile target. Such an import is always + // immediate. + // + bool bf (n.type == "buildfile"); + if (bf) { - // @@ Could this be an out-qualified ad hoc import? - // - if (n.pair) - fail (loc) << "unexpected pair in import"; + if (meta) + fail (loc) << "metadata requested for buildfile target " << n; - // import() will check the name, if required. - // - names r (import (*scope_, move (n), ph2, opt, meta, loc).first); + if (once && var != nullptr) + fail (loc) << "once importation requested with variable assignment"; + + if (ph2 && !ph2->empty ()) + fail (loc) << "rule hint specified for buildfile target " << n; + } + else + { + if (once) + fail (loc) << "once importation requested for target " << n; + + if (var == nullptr) + fail (loc) << "variable assignment required to import target " << n; + } + + // import() will check the name, if required. + // + names r (import (*scope_, + move (n), + ph2 ? ph2 : bf ? optional<string> (string ()) : nullopt, + opt, + meta, + loc).first); + if (val != nullptr) + { if (r.empty ()) // Optional not found. { if (atype == type::assign) - val = nullptr; + *val = nullptr; } else { - if (atype == type::assign) - val.assign (move (r), &var); - else if (atype == type::prepend) - val.prepend (move (r), &var); - else - val.append (move (r), &var); + if (atype == type::assign) val->assign (move (r), var); + else if (atype == type::prepend) val->prepend (move (r), var); + else val->append (move (r), var); } if (atype == type::assign) atype = type::append; // Append subsequent values. } + else + { + assert (bf); + + if (r.empty ()) // Optional not found. + { + assert (opt); + continue; + } + + // Note: see also import_buildfile(). + // + assert (r.size () == 1); // See import_load() for details. + name& n (r.front ()); + path p (n.dir / n.value); // Should already include extension. + + // Note: similar to parse_include(). + // + // Nuance: we insert this buildfile even with once=false in case it + // gets imported with once=true from another place. + // + if (!root_->root_extra->insert_buildfile (p) && once) + { + l5 ([&]{trace (loc) << "skipping already imported " << p;}); + continue; + } + + // Clear/restore if/switch location. + // + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = nullopt; + + try + { + ifdstream ifs (p); + + auto df = make_diag_frame ( + [this, &loc] (const diag_record& dr) + { + dr << info (loc) << "imported from here"; + }); + + // @@ Do we want to enter this buildfile? What's the harm (one + // benefit is that it will be in dump). But, we currently don't + // out-qualify them, though feels like there is nothing fatal + // in that, just inaccurate. + // + source_buildfile (ifs, + path_name (p), + loc, + false /* default_target */); + } + catch (const io_error& e) + { + fail (loc) << "unable to read imported buildfile " << p << ": " << e; + } + } } next_after_newline (t, tt); @@ -3233,7 +4183,12 @@ namespace build2 fail (l) << "null value in export"; if (val.type != nullptr) - untypify (val); + { + // While feels far-fetched, let's preserve empty typed values in the + // result. + // + untypify (val, false /* reduce */); + } export_value = move (val).as<names> (); @@ -3273,6 +4228,9 @@ namespace build2 n = move (i->value); + if (n[0] == '_') + fail (l) << "module name '" << n << "' starts with underscore"; + if (i->pair) try { @@ -3317,11 +4275,37 @@ namespace build2 void parser:: parse_define (token& t, type& tt) { - // define <derived>: <base> + // define [<attrs>] <derived>: <base> // // See tests/define. // - if (next (t, tt) != type::word) + next_with_attributes (t, tt); + + // Handle attributes. + // + attributes_push (t, tt); + + target_type::flag flags (target_type::flag::none); + { + attributes as (attributes_pop ()); + const location& l (as.loc); + + for (attribute& a: as) + { + const string& n (a.name); + value& v (a.value); + + if (n == "see_through") flags |= target_type::flag::see_through; + else if (n == "member_hint") flags |= target_type::flag::member_hint; + else + fail (l) << "unknown target type definition attribute " << n; + + if (!v.null) + fail (l) << "unexpected value in attribute " << n; + } + } + + if (tt != type::word) fail (t) << "expected name instead of " << t << " in target type " << "definition"; @@ -3344,7 +4328,18 @@ namespace build2 if (bt == nullptr) fail (t) << "unknown target type " << bn; - if (!root_->derive_target_type (move (dn), *bt).second) + // Note that the group{foo}<...> syntax is only recognized for group- + // based targets and ad hoc buildscript recipes/rules only match group. + // (We may want to relax this for member_hint in the future since its + // currently also used on non-mtime-based targets, though what exactly + // we will do in ad hoc recipes/rules in this case is fuzzy). + // + if ((flags & target_type::flag::group) == target_type::flag::group && + !bt->is_a<group> ()) + fail (t) << "base target type " << bn << " must be group for " + << "group-related attribute"; + + if (!root_->derive_target_type (move (dn), *bt, flags).second) fail (dnl) << "target type " << dn << " already defined in this " << "project"; @@ -3360,6 +4355,12 @@ namespace build2 void parser:: parse_if_else (token& t, type& tt) { + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = get_location (t); + parse_if_else (t, tt, false /* multi */, [this] (token& t, type& tt, bool s, const string& k) @@ -3501,6 +4502,12 @@ namespace build2 void parser:: parse_switch (token& t, type& tt) { + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = get_location (t); + parse_switch (t, tt, false /* multi */, [this] (token& t, type& tt, bool s, const string& k) @@ -3711,7 +4718,7 @@ namespace build2 if (!e.arg.empty ()) args.push_back (value (e.arg)); - value r (ctx.functions.call (scope_, *e.func, args, l)); + value r (ctx->functions.call (scope_, *e.func, args, l)); // We support two types of functions: matchers and extractors: // a matcher returns a statically-typed bool value while an @@ -3845,10 +4852,10 @@ namespace build2 void parser:: parse_for (token& t, type& tt) { - // for <varname>: <value> + // for [<var-attrs>] <varname> [<elem-attrs>]: [<val-attrs>] <value> // <line> // - // for <varname>: <value> + // for [<var-attrs>] <varname> [<elem-attrs>]: [<val-attrs>] <value> // { // <block> // } @@ -3859,13 +4866,12 @@ namespace build2 next_with_attributes (t, tt); attributes_push (t, tt); - // @@ PAT: currently we pattern-expand for var. + // Enable list element attributes. // - const location vloc (get_location (t)); - names vns (parse_names (t, tt, pattern_mode::expand)); + enable_attributes (); - if (tt != type::colon) - fail (t) << "expected ':' instead of " << t << " after variable name"; + const location vloc (get_location (t)); + names vns (parse_names (t, tt, pattern_mode::preserve)); const variable& var (parse_variable_name (move (vns), vloc)); apply_variable_attributes (var); @@ -3876,6 +4882,17 @@ namespace build2 << " visibility but is assigned in for-loop"; } + // Parse the list element attributes, if present. + // + attributes_push (t, tt); + + if (tt != type::colon) + fail (t) << "expected ':' instead of " << t << " after variable name"; + + // Save element attributes so that we can inject them on each iteration. + // + attributes val_attrs (attributes_pop ()); + // Now the value (list of names) to iterate over. Parse it similar to a // value on the RHS of an assignment (expansion, attributes). // @@ -3892,7 +4909,11 @@ namespace build2 if (val && val.type != nullptr) { etype = val.type->element_type; - untypify (val); + + // Note that here we don't want to be reducing empty simple values to + // empty lists. + // + untypify (val, false /* reduce */); } if (tt != type::newline) @@ -3940,7 +4961,7 @@ namespace build2 // Iterate. // - value& v (scope_->assign (var)); // Assign even if no iterations. + value& lhs (scope_->assign (var)); // Assign even if no iterations. if (!val) return; @@ -3960,11 +4981,17 @@ namespace build2 names n; n.push_back (move (*i)); if (pair) n.push_back (move (*++i)); - v = value (move (n)); + value v (move (n)); if (etype != nullptr) typify (v, *etype, &var); + // Inject element attributes. + // + attributes_.push_back (val_attrs); + + apply_value_attributes (&var, lhs, move (v), type::assign); + lexer l (is, *path_, line); lexer* ol (lexer_); lexer_ = &l; @@ -4067,7 +5094,7 @@ namespace build2 if (value v = parse_value_with_attributes (t, tt, pattern_mode::expand)) { names storage; - cout << reverse (v, storage) << endl; + cout << reverse (v, storage, true /* reduce */) << endl; } else cout << "[null]" << endl; @@ -4100,7 +5127,7 @@ namespace build2 if (value v = parse_value_with_attributes (t, tt, pattern_mode::expand)) { names storage; - dr << reverse (v, storage); + dr << reverse (v, storage, true /* reduce */); } if (tt != type::eos) @@ -4130,8 +5157,10 @@ namespace build2 if (ns.empty ()) { + // Indent two spaces. + // if (scope_ != nullptr) - dump (*scope_, " "); // Indent two spaces. + dump (scope_, nullopt /* action */, dump_format::buildfile, " "); else os << " <no current scope>" << endl; } @@ -4149,8 +5178,10 @@ namespace build2 const target* t (enter_target::find_target (*this, n, o, l, trace)); + // Indent two spaces. + // if (t != nullptr) - dump (*t, " "); // Indent two spaces. + dump (t, nullopt /* action */, dump_format::buildfile, " "); else { os << " <no target " << n; @@ -4172,10 +5203,12 @@ namespace build2 { // Enter a variable name for assignment (as opposed to lookup). + // If the variable is qualified (and thus public), make it overridable. + // // Note that the overridability can still be restricted (e.g., by a module // that enters this variable or by a pattern). // - bool ovr (true); + bool ovr (on.find ('.') != string::npos); auto r (scope_->var_pool ().insert (move (on), nullptr, nullptr, &ovr)); if (!r.second) @@ -4209,9 +5242,13 @@ namespace build2 { // Parse and enter a variable name for assignment (as opposed to lookup). - // The list should contain a single, simple name. + // The list should contain a single, simple name. Go an extra mile to + // issue less confusing diagnostics. // - if (ns.size () != 1 || ns[0].pattern || !ns[0].simple () || ns[0].empty ()) + size_t n (ns.size ()); + if (n == 0 || (n == 1 && ns[0].empty ())) + fail (l) << "empty variable name"; + else if (n != 1 || ns[0].pattern || !ns[0].simple ()) fail (l) << "expected variable name instead of " << ns; return parse_variable_name (move (ns[0].value), l); @@ -4267,7 +5304,7 @@ namespace build2 // Note that the pattern is preserved if insert fails with regex_error. // p = scope_->target_vars[ptt].insert (pt, move (pat)).insert ( - var, kind == type::assign); + var, kind == type::assign, false /* reset_extra */); } catch (const regex_error& e) { @@ -4281,7 +5318,12 @@ namespace build2 // We store prepend/append values untyped (similar to overrides). // if (rhs.type != nullptr && kind != type::assign) - untypify (rhs); + { + // Our heuristics for prepend/append of a typed value is to preserve + // empty (see apply_value_attributes() for details) so do not reduce. + // + untypify (rhs, false /* reduce */); + } if (p.second) { @@ -4368,8 +5410,8 @@ namespace build2 : value (names ()); } - static const value_type* - map_type (const string& n) + const value_type* parser:: + find_value_type (const scope*, const string& n) { auto ptr = [] (const value_type& vt) {return &vt;}; @@ -4392,6 +5434,7 @@ namespace build2 n == "paths" ? ptr (value_traits<paths>::value_type) : n == "dir_paths" ? ptr (value_traits<dir_paths>::value_type) : n == "names" ? ptr (value_traits<vector<name>>::value_type) : + n == "cmdline" ? ptr (value_traits<cmdline>::value_type) : nullptr; } @@ -4415,19 +5458,62 @@ namespace build2 string& n (a.name); value& v (a.value); - if (const value_type* t = map_type (n)) + if (n == "visibility") { + try + { + string s (convert<string> (move (v))); + + variable_visibility r; + if (s == "global") r = variable_visibility::global; + else if (s == "project") r = variable_visibility::project; + else if (s == "scope") r = variable_visibility::scope; + else if (s == "target") r = variable_visibility::target; + else if (s == "prerequisite") r = variable_visibility::prereq; + else throw invalid_argument ("unknown visibility name"); + + if (vis && r != *vis) + fail (l) << "conflicting variable visibilities: " << s << ", " + << *vis; + + vis = r; + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else if (n == "overridable") + { + try + { + // Treat absent value (represented as NULL) as true. + // + bool r (v.null || convert<bool> (move (v))); + + if (ovr && r != *ovr) + fail (l) << "conflicting variable overridabilities"; + + ovr = r; + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else if (const value_type* t = find_value_type (root_, n)) + { + if (!v.null) + fail (l) << "unexpected value in attribute " << a; + if (type != nullptr && t != type) - fail (l) << "multiple variable types: " << n << ", " << type->name; + fail (l) << "conflicting variable types: " << n << ", " + << type->name; type = t; - // Fall through. } else fail (l) << "unknown variable attribute " << a; - - if (!v.null) - fail (l) << "unexpected value in attribute " << a; } if (type != nullptr && var.type != nullptr) @@ -4439,15 +5525,33 @@ namespace build2 << var.type->name << " to " << type->name; } - //@@ TODO: the same checks for vis and ovr (when we have the corresponding - // attributes). + if (vis) + { + // Note that this logic naturally makes sure that a project-private + // variable doesn't have global visibility (since it would have been + // entered with the project visibility). + // + if (var.visibility == *vis) + vis = nullopt; + else if (var.visibility > *vis) // See variable_pool::update(). + fail (l) << "changing variable " << var << " visibility from " + << var.visibility << " to " << *vis; + } - if (type || vis || ovr) - ctx.var_pool.update (const_cast<variable&> (var), - type, - vis ? &*vis : nullptr, - ovr ? &*ovr : nullptr); + if (ovr) + { + // Note that the overridability incompatibilities are diagnosed by + // update(). So we just need to diagnose the project-private case. + // + if (*ovr && var.owner != &ctx->var_pool) + fail (l) << "private variable " << var << " cannot be overridable"; + } + if (type || vis || ovr) + var.owner->update (const_cast<variable&> (var), + type, + vis ? &*vis : nullptr, + ovr ? &*ovr : nullptr); } void parser:: @@ -4477,10 +5581,10 @@ namespace build2 null = true; // Fall through. } - else if (const value_type* t = map_type (n)) + else if (const value_type* t = find_value_type (root_, n)) { if (type != nullptr && t != type) - fail (l) << "multiple value types: " << n << ", " << type->name; + fail (l) << "conflicting value types: " << n << ", " << type->name; type = t; // Fall through. @@ -4528,6 +5632,13 @@ namespace build2 bool rhs_type (false); if (rhs.type != nullptr) { + // Our heuristics is to not reduce typed RHS empty simple values for + // prepend/append and additionally for assign provided LHS is a + // container. + // + bool reduce (kind == type::assign && + (type == nullptr || !type->container)); + // Only consider RHS type if there is no explicit or variable type. // if (type == nullptr) @@ -4538,7 +5649,7 @@ namespace build2 // Reduce this to the untyped value case for simplicity. // - untypify (rhs); + untypify (rhs, reduce); } if (kind == type::assign) @@ -4921,17 +6032,38 @@ namespace build2 if (pre_parse_) return v; // Empty. - if (v.type != nullptr || !v || v.as<names> ().size () != 1) - fail (l) << "expected target before ':'"; - + // We used to return this as a <target>:<name> pair but that meant we + // could not handle an out-qualified target (which is represented as + // <target>@<out> pair). As a somewhat of a hack, we deal with this by + // changing the order of the name and target to be <name>:<target> with + // the qualified case becoming a "tripple pair" <name>:<target>@<out>. + // + // @@ This is actually not great since it's possible to observe such a + // tripple pair, for example with `print (file{x}@./:y)`. + // if (n.type != nullptr || !n || n.as<names> ().size () != 1 || n.as<names> ()[0].pattern) fail (nl) << "expected variable name after ':'"; - names& ns (v.as<names> ()); + names& ns (n.as<names> ()); ns.back ().pair = ':'; - ns.push_back (move (n.as<names> ().back ())); - return v; + + if (v.type == nullptr && v) + { + names& ts (v.as<names> ()); + + size_t s (ts.size ()); + if (s == 1 || (s == 2 && ts.front ().pair == '@')) + { + ns.push_back (move (ts.front ())); + if (s == 2) + ns.push_back (move (ts.back ())); + + return n; + } + } + + fail (l) << "expected target before ':'" << endf; } else { @@ -5000,8 +6132,13 @@ namespace build2 } pair<bool, location> parser:: - attributes_push (token& t, type& tt, bool standalone) + attributes_push (token& t, type& tt, bool standalone, bool next_token) { + // To make sure that the attributes are not standalone we need to read the + // token which follows ']'. + // + assert (standalone || next_token); + location l (get_location (t)); bool has (tt == type::lsbrace); @@ -5024,6 +6161,10 @@ namespace build2 // Parse the attribute name with expansion (we rely on this in some // old and hairy tests). // + // Note that the attributes lexer mode does not recognize `{}@` as + // special and we rely on that in the rule hint attributes + // (libs@rule_hint=cxx). + // const location l (get_location (t)); names ns ( @@ -5065,32 +6206,33 @@ namespace build2 } while (tt != type::rsbrace); } + else + has = false; // `[]` doesn't count. if (tt != type::rsbrace) fail (t) << "expected ']' instead of " << t; - next (t, tt); - - if (tt == type::newline || tt == type::eos) + if (next_token) { - if (!standalone) - fail (t) << "standalone attributes"; + next (t, tt); + + if (tt == type::newline || tt == type::eos) + { + if (!standalone) + fail (t) << "standalone attributes"; + } + // + // Verify that the attributes are separated from the following word or + // "word-producing" token. + // + else if (!t.separated && (tt == type::word || + tt == type::dollar || + tt == type::lparen || + tt == type::lcbrace)) + fail (t) << "whitespace required after attributes" << + info (l) << "use the '\\[' escape sequence if this is a wildcard " + << "pattern"; } - // - // We require attributes to be separated from the following word or - // "word-producing" tokens (`$` for variable expansions/function calls, - // `(` for eval contexts, and `{` for name generation) to reduce the - // possibility of confusing them with wildcard patterns. Consider: - // - // ./: [abc]-foo.txt - // - else if (!t.separated && (tt == type::word || - tt == type::dollar || - tt == type::lparen || - tt == type::lcbrace)) - fail (t) << "whitespace required after attributes" << - info (l) << "use the '\\[' escape sequence if this is a wildcard " - << "pattern"; return make_pair (has, l); } @@ -5325,9 +6467,11 @@ namespace build2 // May throw invalid_path. // auto include_pattern = - [&r, &append, &include_match, sp, &l, this] (string&& p, - optional<string>&& e, - bool a) + [this, + &append, &include_match, + &r, sp, &l, &dir] (string&& p, + optional<string>&& e, + bool a) { // If we don't already have any matches and our pattern doesn't contain // multiple recursive wildcards, then the result will be unique and we @@ -5374,14 +6518,62 @@ namespace build2 // multiple entries for each pattern. // if (!interm) - d.appf (move (m).representation (), optional<string> (d.e)); + { + // If the extension is empty (meaning there should be no extension, + // for example hxx{Q*.}), skip entries with extensions. + // + if (!d.e || !d.e->empty () || m.extension_cstring () == nullptr) + d.appf (move (m).representation (), optional<string> (d.e)); + } return true; }; + const function<bool (const dir_entry&)> dangling ( + [&dir] (const dir_entry& de) + { + bool sl (de.ltype () == entry_type::symlink); + + const path& n (de.path ()); + + // One case where this turned out to be not worth it practically + // (too much noise) is the backlinks to executables (and the + // associated DLL assemblies for Windows). So we now have this + // heuristics that if this looks like an executable (or DLL for + // Windows), then we omit the warning. On POSIX, where executables + // don't have extensions, we will consider it an executable only if + // we are not looking for directories (which also normally don't + // have extension). + // + // @@ PEDANTIC: re-enable if --pedantic. + // + if (sl) + { + string e (n.extension ()); + + if ((e.empty () && !dir) || + path_traits::compare (e, "exe") == 0 || + path_traits::compare (e, "dll") == 0 || + path_traits::compare (e, "pdb") == 0 || // .{exe,dll}.pdb + (path_traits::compare (e, "dlls") == 0 && // .exe.dlls assembly + path_traits::compare (n.base ().extension (), "exe") == 0)) + return true; + } + + warn << "skipping " + << (sl ? "dangling symlink" : "inaccessible entry") + << ' ' << de.base () / n; + + return true; + }); + try { - path_search (path (move (p)), process, *sp); + path_search (path (move (p)), + process, + *sp, + path_match_flags::follow_symlinks, + dangling); } catch (const system_error& e) { @@ -5549,6 +6741,7 @@ namespace build2 if ((n.pair & 0x02) != 0) { e = move (n.type); + n.type.clear (); // Remove non-empty extension from the name (it got to be there, see // above). @@ -5826,9 +7019,35 @@ namespace build2 bool concat_quoted_first (false); name concat_data; - auto concat_typed = [this, &vnull, &vtype, - &concat, &concat_data] (value&& rhs, - const location& loc) + auto concat_diag_multiple = [this] (const location& loc, + const char* what_expansion) + { + diag_record dr (fail (loc)); + + dr << "concatenating " << what_expansion << " contains multiple values"; + + // See if this looks like a subscript without an evaluation context and + // help the user out. + // + if (mode () != lexer_mode::eval) + { + const token& t (peeked ()); // Should be peeked at. + + if (t.type == type::word && + t.qtype == quote_type::unquoted && + t.value[0] == '[') + { + dr << info << "wrap it in (...) evaluation context if this " + << "is value subscript"; + } + } + }; + + auto concat_typed = [this, what, &vnull, &vtype, + &concat, &concat_data, + &concat_diag_multiple] (value&& rhs, + const location& loc, + const char* what_expansion) { // If we have no LHS yet, then simply copy value/type. // @@ -5845,6 +7064,10 @@ namespace build2 // RHS. // + // Note that if RHS contains multiple values then we expect the result + // to be a single value somehow or, more likely, there to be no + // suitable $builtin.concat() overload. + // a.push_back (move (rhs)); const char* l ((a[0].type != nullptr ? a[0].type->name : "<untyped>")); @@ -5861,7 +7084,10 @@ namespace build2 dr << info << "use quoting to force untyped concatenation"; }); - p = ctx.functions.try_call ( + if (ctx == nullptr) + fail << "literal " << what << " expected"; + + p = ctx->functions.try_call ( scope_, "builtin.concat", vector_view<value> (a), loc); } @@ -5883,18 +7109,22 @@ namespace build2 if (!vnull) { if (vtype != nullptr) - untypify (rhs); + untypify (rhs, true /* reduce */); names& d (rhs.as<names> ()); - // If the value is empty, then untypify() will (typically; no pun - // intended) represent it as an empty sequence of names rather than - // a sequence of one empty name. This is usually what we need (see - // simple_reverse() for details) but not in this case. + // If the value is empty, then we asked untypify() to reduce it to + // an empty sequence of names rather than a sequence of one empty + // name. // if (!d.empty ()) { - assert (d.size () == 1); // Must be a single value. + if (d.size () != 1) + { + assert (what_expansion != nullptr); + concat_diag_multiple (loc, what_expansion); + } + concat_data = move (d[0]); } } @@ -6001,6 +7231,8 @@ namespace build2 // continue accumulating or inject. We inject if the next token is not a // word, var expansion, or eval context or if it is separated. // + optional<pair<const value_type*, name>> path_concat; // Backup. + if (concat && last_concat ()) { // Concatenation does not affect the tokens we get, only what we do @@ -6040,6 +7272,13 @@ namespace build2 // dir/{$str} // file{$str} // + // And yet another exception: if the type is path or dir_path and the + // pattern mode is not ignore, then we will inject to try our luck in + // interpreting the concatenation result as a path pattern. This makes + // sure patterns like `$src_base/*.txt` work, naturally. Failed that, + // we will handle this concatenation as we do for other types (via the + // path_concat backup). + // // A concatenation cannot produce value/NULL. // @@ -6051,12 +7290,14 @@ namespace build2 bool e1 (tt == type::lcbrace && !peeked ().separated); bool e2 (pp || dp != nullptr || tp != nullptr); + const value_type* pt (&value_traits<path>::value_type); + const value_type* dt (&value_traits<dir_path>::value_type); + if (e1 || e2) { - if (vtype == &value_traits<path>::value_type || - vtype == &value_traits<string>::value_type) + if (vtype == pt || vtype == &value_traits<string>::value_type) ; // Representation is already in concat_data.value. - else if (vtype == &value_traits<dir_path>::value_type) + else if (vtype == dt) concat_data.value = move (concat_data.dir).representation (); else { @@ -6071,6 +7312,20 @@ namespace build2 vtype = nullptr; // Fall through to injection. } + else if (pmode != pattern_mode::ignore && + (vtype == pt || vtype == dt)) + { + path_concat = make_pair (vtype, concat_data); + + // Note: for path the representation is already in + // concat_data.value. + // + if (vtype == dt) + concat_data.value = move (concat_data.dir).representation (); + + vtype = nullptr; + // Fall through to injection. + } else { // This is either a simple name (untyped concatenation; in which @@ -6164,7 +7419,7 @@ namespace build2 // names ns; ns.push_back (name (move (val))); - concat_typed (value (move (ns)), get_location (t)); + concat_typed (value (move (ns)), get_location (t), nullptr); } else { @@ -6331,7 +7586,7 @@ namespace build2 // See if this is a pattern, path or regex. // // A path pattern either contains an unquoted wildcard character or, - // in the curly context, start with unquoted/unescaped `+`. + // in the curly context, starts with unquoted/unescaped `+`. // // A regex pattern starts with unquoted/unescaped `~` followed by a // non-alphanumeric delimiter and has the following form: @@ -6409,7 +7664,7 @@ namespace build2 // Note that we have to check for regex patterns first since // they may also be detected as path patterns. // - if (!quoted_first && regex_pattern ()) + if (!quoted_first && !path_concat && regex_pattern ()) { // Note: we may decide to support regex-based name generation // some day (though a substitution won't make sense here). @@ -6477,7 +7732,7 @@ namespace build2 // there isn't any good reason to; see also to_stream(name) for // the corresponding serialization logic). // - if (!quoted_first && regex_pattern ()) + if (!quoted_first && !path_concat && regex_pattern ()) { const char* w; if (val[0] == '~') @@ -6535,6 +7790,24 @@ namespace build2 } } + // If this is a concatenation of the path or dir_path type and it is + // not a pattern, then handle it in the same way as concatenations of + // other types (see above). + // + if (path_concat && !pat) + { + ns.push_back (move (path_concat->second)); + + // Restore the type information if that's the only name. + // + if (start == ns.size () && last_token ()) + vtype = path_concat->first; + + // Restart the loop. + // + continue; + } + // If we are a second half of a pair, add another first half // unless this is the first instance. // @@ -6589,6 +7862,9 @@ namespace build2 // if (tt == type::dollar || tt == type::lparen) { + if (ctx == nullptr) + fail << "literal " << what << " expected"; + // These cases are pretty similar in that in both we quickly end up // with a list of names that we need to splice into the result. // @@ -6610,11 +7886,15 @@ namespace build2 // token is a paren or a word, we turn it on and switch to the eval // mode if what we get next is a paren. // - // Also sniff out the special variables string from mode data for - // the ad hoc $() handling below. - // mode (lexer_mode::variable); + // Sniff out the special variables string from mode data and use + // that to recognize special variables in the ad hoc $() handling + // below. + // + // Note: must be done before calling next() which may expire the + // mode. + // auto special = [s = reinterpret_cast<const char*> (mode_data ())] (const token& t) -> char { @@ -6653,156 +7933,202 @@ namespace build2 next (t, tt); loc = get_location (t); - name qual; - string name; - - if (t.separated) - ; // Leave the name empty to fail below. - else if (tt == type::word) + if (tt == type::escape) { - name = move (t.value); + // For now we only support all the simple C/C++ escape sequences + // plus \0 (which in C/C++ is an octal escape sequence). See the + // lexer part for details. + // + // Note: cannot be subscripted. + // + if (!pre_parse_) + { + string s; + switch (char c = t.value[0]) + { + case '\'': + case '"': + case '?': + case '\\': s = c; break; + case '0': s = '\0'; break; + case 'a': s = '\a'; break; + case 'b': s = '\b'; break; + case 'f': s = '\f'; break; + case 'n': s = '\n'; break; + case 'r': s = '\r'; break; + case 't': s = '\t'; break; + case 'v': s = '\v'; break; + default: + assert (false); + } + + result_data = name (move (s)); + what = "escape sequence expansion"; + } + + tt = peek (); } - else if (tt == type::lparen) + else { - expire_mode (); - mode (lexer_mode::eval, '@'); - next_with_attributes (t, tt); + names qual; + string name; - // Handle the $(x) case ad hoc. We do it this way in order to get - // the variable name even during pre-parse. It should also be - // faster. - // - char c; - if ((tt == type::word - ? path_traits::rfind_separator (t.value) == string::npos - : (c = special (t))) && - peek () == type::rparen) + if (t.separated) + ; // Leave the name empty to fail below. + else if (tt == type::word) { - name = (tt == type::word ? move (t.value) : string (1, c)); - next (t, tt); // Get `)`. + name = move (t.value); } - else + else if (tt == type::lparen) { - using name_type = build2::name; + expire_mode (); + mode (lexer_mode::eval, '@'); + next_with_attributes (t, tt); - //@@ OUT will parse @-pair and do well? + // Handle the $(x) case ad hoc. We do it this way in order to + // get the variable name even during pre-parse. It should also + // be faster. // - values vs (parse_eval (t, tt, pmode)); - - if (!pre_parse_) + char c; + if ((tt == type::word + ? path_traits::rfind_separator (t.value) == string::npos + : (c = special (t))) && + peek () == type::rparen) + { + name = (tt == type::word ? move (t.value) : string (1, c)); + next (t, tt); // Get `)`. + } + else { - if (vs.size () != 1) - fail (loc) << "expected single variable/function name"; + using name_type = build2::name; - value& v (vs[0]); + values vs (parse_eval (t, tt, pmode)); - if (!v) - fail (loc) << "null variable/function name"; + if (!pre_parse_) + { + if (vs.size () != 1) + fail (loc) << "expected single variable/function name"; - names storage; - vector_view<name_type> ns (reverse (v, storage)); // Movable. - size_t n (ns.size ()); + value& v (vs[0]); - // We cannot handle scope-qualification in the eval context as - // we do for target-qualification (see eval-qual) since then - // we would be treating all paths as qualified variables. So - // we have to do it here. - // - if (n == 2 && ns[0].pair == ':') // $(foo: x) - { - qual = move (ns[0]); + if (!v) + fail (loc) << "null variable/function name"; - if (qual.empty ()) - fail (loc) << "empty variable/function qualification"; - } - else if (n == 2 && ns[0].directory ()) // $(foo/ x) - { - qual = move (ns[0]); - qual.pair = '/'; - } - else if (n > 1) - fail (loc) << "expected variable/function name instead of '" - << ns << "'"; + names storage; + vector_view<name_type> ns ( + reverse (v, storage, true /* reduce */)); // Movable. + size_t n (ns.size ()); - // Note: checked for empty below. - // - if (!ns[n - 1].simple ()) - fail (loc) << "expected variable/function name instead of '" - << ns[n - 1] << "'"; + // We cannot handle scope-qualification in the eval context + // as we do for target-qualification (see eval-qual) since + // then we would be treating all paths as qualified + // variables. So we have to do it here. + // + if (n >= 2 && ns[0].pair == ':') // $(foo: x) + { + // Note: name is first (see eval for details). + // + qual.push_back (move (ns[1])); - size_t p; - if (n == 1 && // $(foo/x) - (p = path_traits::rfind_separator (ns[0].value)) != - string::npos) - { - // Note that p cannot point to the last character since then - // it would have been a directory, not a simple name. + if (qual.back ().empty ()) + fail (loc) << "empty variable/function qualification"; + + if (n > 2) + qual.push_back (move (ns[2])); + + // Move name to the last position (see below). + // + swap (ns[0], ns[n - 1]); + } + else if (n == 2 && ns[0].directory ()) // $(foo/ x) + { + qual.push_back (move (ns[0])); + qual.back ().pair = '/'; + } + else if (n > 1) + fail (loc) << "expected variable/function name instead of '" + << ns << "'"; + + // Note: checked for empty below. // - string& s (ns[0].value); + if (!ns[n - 1].simple ()) + fail (loc) << "expected variable/function name instead of '" + << ns[n - 1] << "'"; - name = string (s, p + 1); - s.resize (p + 1); - qual = name_type (dir_path (move (s))); - qual.pair = '/'; + size_t p; + if (n == 1 && // $(foo/x) + (p = path_traits::rfind_separator (ns[0].value)) != + string::npos) + { + // Note that p cannot point to the last character since + // then it would have been a directory, not a simple name. + // + string& s (ns[0].value); + + name = string (s, p + 1); + s.resize (p + 1); + qual.push_back (name_type (dir_path (move (s)))); + qual.back ().pair = '/'; + } + else + name = move (ns[n - 1].value); } - else - name = move (ns[n - 1].value); } } - } - else - fail (t) << "expected variable/function name instead of " << t; - - if (!pre_parse_ && name.empty ()) - fail (loc) << "empty variable/function name"; + else + fail (t) << "expected variable/function name instead of " << t; - // Figure out whether this is a variable expansion with potential - // subscript or a function call. - // - if (sub) enable_subscript (); - tt = peek (); + if (!pre_parse_ && name.empty ()) + fail (loc) << "empty variable/function name"; - // Note that we require function call opening paren to be - // unseparated; consider: $x ($x == 'foo' ? 'FOO' : 'BAR'). - // - if (tt == type::lparen && !peeked ().separated) - { - // Function call. + // Figure out whether this is a variable expansion with potential + // subscript or a function call. // - next (t, tt); // Get '('. - mode (lexer_mode::eval, '@'); - next_with_attributes (t, tt); - - // @@ Should we use (target/scope) qualification (of name) as the - // context in which to call the function? Hm, interesting... - // - values args (parse_eval (t, tt, pmode)); - if (sub) enable_subscript (); tt = peek (); - // Note that we "move" args to call(). + // Note that we require function call opening paren to be + // unseparated; consider: $x ($x == 'foo' ? 'FOO' : 'BAR'). // - if (!pre_parse_) + if (tt == type::lparen && !peeked ().separated) { - result_data = ctx.functions.call (scope_, name, args, loc); - what = "function call"; + // Function call. + // + next (t, tt); // Get '('. + mode (lexer_mode::eval, '@'); + next_with_attributes (t, tt); + + // @@ Should we use (target/scope) qualification (of name) as + // the context in which to call the function? Hm, interesting... + // + values args (parse_eval (t, tt, pmode)); + + if (sub) enable_subscript (); + tt = peek (); + + // Note that we "move" args to call(). + // + if (!pre_parse_) + { + result_data = ctx->functions.call (scope_, name, args, loc); + what = "function call"; + } + else + lookup_function (move (name), loc); } else - lookup_function (move (name), loc); - } - else - { - // Variable expansion. - // - lookup l (lookup_variable (move (qual), move (name), loc)); - - if (!pre_parse_) { - if (l.defined ()) - result = l.value; // Otherwise leave as NULL result_data. + // Variable expansion. + // + lookup l (lookup_variable (move (qual), move (name), loc)); + + if (!pre_parse_) + { + if (l.defined ()) + result = l.value; // Otherwise leave as NULL result_data. - what = "variable expansion"; + what = "variable expansion"; + } } } } @@ -6834,7 +8160,7 @@ namespace build2 // Handle value subscript. // - if (tt == type::lsbrace) + if (tt == type::lsbrace && mode () == lexer_mode::eval) { location bl (get_location (t)); next (t, tt); // `[` @@ -6901,12 +8227,44 @@ namespace build2 } else { - // @@ TODO: we would want to return a value with element type. + // Similar logic to parse_for(). // - //result_data = ... - fail (l) << "typed value subscript not yet supported" << - info (bl) << "use the '\\[' escape sequence if this is a " - << "wildcard pattern"; + // @@ Maybe we should invent type-aware subscript? Could also + // be used for non-index subscripts (map keys etc). + // + const value_type* etype (result->type->element_type); + + value val (result == &result_data + ? value (move (result_data)) + : value (*result)); + + untypify (val, false /* reduce */); + + names& ns (val.as<names> ()); + + // Pair-aware subscript. + // + names r; + for (auto i (ns.begin ()); i != ns.end (); ++i, --j) + { + bool p (i->pair); + + if (j == 0) + { + r.push_back (move (*i)); + if (p) + r.push_back (move (*++i)); + break; + } + + if (p) + ++i; + } + + result_data = r.empty () ? value () : value (move (r)); + + if (etype != nullptr) + typify (result_data, *etype, nullptr /* var */); } result = &result_data; @@ -6956,7 +8314,8 @@ namespace build2 // then it should not be overloaded for a type). In a quoted // context we use $string() which returns a "canonical // representation" (e.g., a directory path without a trailing - // slash). + // slash). Note: looks like we use typed $concat() now in the + // unquoted context. // if (result->type != nullptr && quoted) { @@ -6979,7 +8338,10 @@ namespace build2 dr << info (loc) << "while converting " << t << " to string"; }); - p = ctx.functions.try_call ( + if (ctx == nullptr) + fail << "literal " << what << " expected"; + + p = ctx->functions.try_call ( scope_, "string", vector_view<value> (&result_data, 1), loc); } @@ -6987,7 +8349,11 @@ namespace build2 fail (loc) << "no string conversion for " << t; result_data = move (p.first); - untypify (result_data); // Convert to untyped simple name. + + // Convert to untyped simple name reducing empty string to empty + // names as an optimization. + // + untypify (result_data, true /* reduce */); } if ((concat && vtype != nullptr) || // LHS typed. @@ -6996,7 +8362,7 @@ namespace build2 if (result != &result_data) // Same reason as above. result = &(result_data = *result); - concat_typed (move (result_data), loc); + concat_typed (move (result_data), loc, what); } // // Untyped concatenation. Note that if RHS is NULL/empty, we still @@ -7013,8 +8379,7 @@ namespace build2 // This should be a simple value or a simple directory. // if (lv.size () > 1) - fail (loc) << "concatenating " << what << " contains multiple " - << "values"; + concat_diag_multiple (loc, what); const name& n (lv[0]); @@ -7069,7 +8434,7 @@ namespace build2 // @@ Could move if nv is result_data; see untypify(). // names nv_storage; - names_view nv (reverse (*result, nv_storage)); + names_view nv (reverse (*result, nv_storage, true /* reduce */)); count = splice_names ( loc, nv, move (nv_storage), ns, what, pairn, pp, dp, tp); @@ -7307,14 +8672,16 @@ namespace build2 buildspec parser:: parse_buildspec (istream& is, const path_name& in) { - // We do "effective escaping" and only for ['"\$(] (basically what's - // necessary inside a double-quoted literal plus the single quote). + // We do "effective escaping" of the special `'"\$(` characters (basically + // what's escapable inside a double-quoted literal plus the single quote; + // note, however, that we exclude line continuations and `)` since they + // would make directory paths on Windows unusable). // path_ = ∈ lexer l (is, *path_, 1 /* line */, "\'\"\\$("); lexer_ = &l; - root_ = &ctx.global_scope.rw (); + root_ = &ctx->global_scope.rw (); scope_ = root_; target_ = nullptr; prerequisite_ = nullptr; @@ -7549,8 +8916,11 @@ namespace build2 } lookup parser:: - lookup_variable (name&& qual, string&& name, const location& loc) + lookup_variable (names&& qual, string&& name, const location& loc) { + // Note that this function can be called during execute (for example, from + // scripts). In particular, this means we cannot use enter_{scope,target}. + if (pre_parse_) return lookup (); @@ -7562,9 +8932,6 @@ namespace build2 // If we are qualified, it can be a scope or a target. // - enter_scope sg; - enter_target tg; - if (qual.empty ()) { s = scope_; @@ -7573,36 +8940,70 @@ namespace build2 } else { - switch (qual.pair) + // What should we do if we cannot find the qualification (scope or + // target)? We can "fall through" to an outer scope (there is always the + // global scope backstop), we can return NULL straight away, or we can + // fail. It feels like in most cases unknown scope or target is a + // mistake and doing anything other than failing is just making things + // harder to debug. + // + switch (qual.front ().pair) { case '/': { - assert (qual.directory ()); - sg = enter_scope (*this, move (qual.dir)); - s = scope_; + assert (qual.front ().directory ()); + + dir_path& d (qual.front ().dir); + enter_scope::complete_normalize (*scope_, d); + + s = &ctx->scopes.find_out (d); + + if (s->out_path () != d) + fail (loc) << "unknown scope " << d << " in scope-qualified " + << "variable " << name << " expansion" << + info << "did you forget to include the corresponding buildfile?"; + break; } - case ':': + default: { - qual.pair = '\0'; + build2::name n (move (qual.front ())), o; + + if (n.pair) + o = move (qual.back ()); + + t = enter_target::find_target (*this, n, o, loc, trace); + + if (t == nullptr || !operator>= (t->decl, target_decl::implied)) // VC14 + { + diag_record dr (fail (loc)); + + dr << "unknown target " << n; - // @@ OUT TODO + if (n.pair && !o.dir.empty ()) + dr << '@' << o.dir; + + dr << " in target-qualified variable " << name << " expansion"; + } + + // Use the target's var_pool for good measure. // - tg = enter_target ( - *this, move (qual), build2::name (), true, loc, trace); - t = target_; + s = &t->base_scope (); + break; } - default: assert (false); } } // Lookup. // - if (const variable* pvar = scope_->var_pool ().find (name)) + if (const variable* pvar = + (s != nullptr ? s : scope_)->var_pool ().find (name)) { auto& var (*pvar); + // Note: the order of the following blocks is important. + if (p != nullptr) { // The lookup depth is a bit of a hack but should be harmless since @@ -7689,62 +9090,200 @@ namespace build2 return r; } + // file.cxx + // + extern const dir_path std_export_dir; + extern const dir_path alt_export_dir; + void parser:: - process_default_target (token& t) + process_default_target (token& t, const buildfile* bf) { tracer trace ("parser::process_default_target", &path_); // The logic is as follows: if we have an explicit current directory - // target, then that's the default target. Otherwise, we take the - // first target and use it as a prerequisite to create an implicit - // current directory target, effectively making it the default - // target via an alias. If there are no targets in this buildfile, - // then we don't do anything. + // target, then that's the default target. Otherwise, we take the first + // target and use it as a prerequisite to create an implicit current + // directory target, effectively making it the default target via an + // alias. If this is a project root buildfile, then also add exported + // buildfiles. And if there are no targets in this buildfile, then we + // don't do anything (reasonably assuming it's not root). // if (default_target_ == nullptr) // No targets in this buildfile. return; - target& dt (*default_target_); - target* ct ( - const_cast<target*> ( // Ok (serial execution). - ctx.targets.find (dir::static_type, // Explicit current dir target. - scope_->out_path (), - dir_path (), // Out tree target. - string (), - nullopt, - trace))); - - if (ct == nullptr) - { - l5 ([&]{trace (t) << "creating current directory alias for " << dt;}); - - // While this target is not explicitly mentioned in the buildfile, we - // say that we behave as if it were. Thus not implied. - // - ct = &ctx.targets.insert (dir::static_type, - scope_->out_path (), - dir_path (), - string (), - nullopt, - target_decl::real, - trace).first; - // Fall through. - } - else if (ct->decl != target_decl::real) + const_cast<target*> ( // Ok (serial execution). + ctx->targets.find (dir::static_type, // Explicit current dir target. + scope_->out_path (), + dir_path (), // Out tree target. + string (), + nullopt, + trace))); + + if (ct != nullptr && ct->decl == target_decl::real) + ; // Existing and not implied. + else { - ct->decl = target_decl::real; - // Fall through. + target& dt (*default_target_); + + if (ct == nullptr) + { + l5 ([&]{trace (t) << "creating current directory alias for " << dt;}); + + // While this target is not explicitly mentioned in the buildfile, we + // say that we behave as if it were. Thus not implied. + // + ct = &ctx->targets.insert (dir::static_type, + scope_->out_path (), + dir_path (), + string (), + nullopt, + target_decl::real, + trace).first; + } + else + ct->decl = target_decl::real; + + ct->prerequisites_state_.store (2, memory_order_relaxed); + ct->prerequisites_.push_back (prerequisite (dt)); } - else - return; // Existing and not implied. - ct->prerequisites_state_.store (2, memory_order_relaxed); - ct->prerequisites_.emplace_back (prerequisite (dt)); + // See if this is a root buildfile and not in a simple project. + // + if (bf != nullptr && + root_ != nullptr && + root_->root_extra != nullptr && + root_->root_extra->loaded && + *root_->root_extra->project != nullptr && + bf->dir == root_->src_path () && + bf->name == root_->root_extra->buildfile_file.string ()) + { + // See if we have any exported buildfiles. + // + const dir_path& export_dir ( + root_->root_extra->altn ? alt_export_dir : std_export_dir); + + dir_path d (root_->src_path () / export_dir); + if (exists (d)) + { + // Make sure prerequisites are set. + // + ct->prerequisites_state_.store (2, memory_order_relaxed); + + const string& build_ext (root_->root_extra->build_ext); + + // Return true if entered any exported buildfiles. + // + // Note: recursive lambda. + // + auto iterate = [this, &trace, + ct, &build_ext] (const dir_path& d, + const auto& iterate) -> bool + { + bool r (false); + + try + { + for (const dir_entry& e: + dir_iterator (d, dir_iterator::detect_dangling)) + { + switch (e.type ()) + { + case entry_type::directory: + { + r = iterate (d / path_cast<dir_path> (e.path ()), iterate) || r; + break; + } + case entry_type::regular: + { + const path& n (e.path ()); + + if (n.extension () == build_ext) + { + // Similar to above, enter as real. + // + // Note that these targets may already be entered (for + // example, if already imported). + // + const target& bf ( + ctx->targets.insert (buildfile::static_type, + d, + (root_->out_eq_src () + ? dir_path () + : out_src (d, *root_)), + n.base ().string (), + build_ext, + target_decl::real, + trace).first); + + ct->prerequisites_.push_back (prerequisite (bf)); + r = true; + } + + break; + } + case entry_type::unknown: + { + bool sl (e.ltype () == entry_type::symlink); + + fail << (sl ? "dangling symlink" : "inaccessible entry") + << ' ' << d / e.path (); + + break; + } + default: + break; + } + } + } + catch (const system_error& e) + { + fail << "unable to iterate over " << d << ": " << e; + } + + return r; + }; + + if (iterate (d, iterate)) + { + // Arrange for the exported buildfiles to be installed, recreating + // subdirectories inside export/. Essentially, we are arranging for + // this: + // + // build/export/buildfile{*}: + // { + // install = buildfile/ + // install.subdirs = true + // } + // + if (cast_false<bool> (root_->vars["install.loaded"])) + { + enter_scope es (*this, dir_path (export_dir)); + auto& vars (scope_->target_vars[buildfile::static_type]["*"]); + + // @@ TODO: get cached variables from the module once we have one. + // + { + auto r (vars.insert (*root_->var_pool ().find ("install"))); + + if (r.second) // Already set by the user? + r.first = path_cast<path> (dir_path ("buildfile")); + } + + { + auto r (vars.insert ( + *root_->var_pool (true).find ("install.subdirs"))); + if (r.second) + r.first = true; + } + } + } + } + } } - void parser:: - enter_buildfile (const path& p) + const buildfile& parser:: + enter_buildfile (const path& p, optional<dir_path> out) { tracer trace ("parser::enter_buildfile", &path_); @@ -7752,17 +9291,20 @@ namespace build2 // Figure out if we need out. // - dir_path out; - if (scope_->src_path_ != nullptr && - scope_->src_path () != scope_->out_path () && - d.sub (scope_->src_path ())) + dir_path o; + if (out) + o = move (*out); + else if (root_ != nullptr && + root_->src_path_ != nullptr && + !root_->out_eq_src () && + d.sub (*root_->src_path_)) { - out = out_src (d, *root_); + o = out_src (d, *root_); } - ctx.targets.insert<buildfile> ( + return ctx->targets.insert<buildfile> ( move (d), - move (out), + move (o), p.leaf ().base ().string (), p.extension (), // Always specified. trace); |