diff options
Diffstat (limited to 'libbuild2/parser.cxx')
| -rw-r--r-- | libbuild2/parser.cxx | 3873 |
1 files changed, 2964 insertions, 909 deletions
diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index 99e67a7..5321cd5 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -24,6 +24,8 @@ #include <libbuild2/adhoc-rule-regex-pattern.hxx> +#include <libbuild2/dist/module.hxx> // module + #include <libbuild2/config/utility.hxx> // lookup_config using namespace std; @@ -42,7 +44,10 @@ namespace build2 { o << '='; names storage; - to_stream (o, reverse (a.value, storage), quote_mode::normal, '@'); + to_stream (o, + reverse (a.value, storage, true /* reduce */), + quote_mode::normal, + '@'); } return o; @@ -57,27 +62,7 @@ namespace build2 enter_scope (parser& p, dir_path&& d) : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_) { - // Try hard not to call normalize(). Most of the time we will go just - // one level deeper. - // - bool n (true); - - if (d.relative ()) - { - // Relative scopes are opened relative to out, not src. - // - if (d.simple () && !d.current () && !d.parent ()) - { - d = dir_path (p.scope_->out_path ()) /= d.string (); - n = false; - } - else - d = p.scope_->out_path () / d; - } - - if (n) - d.normalize (); - + complete_normalize (*p.scope_, d); e_ = p.switch_scope (d); } @@ -103,8 +88,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_scope (enter_scope&& x) {*this = move (x);} - enter_scope& operator= (enter_scope&& x) + enter_scope (enter_scope&& x) noexcept {*this = move (x);} + enter_scope& operator= (enter_scope&& x) noexcept { if (this != &x) { @@ -121,6 +106,31 @@ namespace build2 enter_scope (const enter_scope&) = delete; enter_scope& operator= (const enter_scope&) = delete; + static void + complete_normalize (scope& s, dir_path& d) + { + // Try hard not to call normalize(). Most of the time we will go just + // one level deeper. + // + bool n (true); + + if (d.relative ()) + { + // Relative scopes are opened relative to out, not src. + // + if (d.simple () && !d.current () && !d.parent ()) + { + d = dir_path (s.out_path ()) /= d.string (); + n = false; + } + else + d = s.out_path () / d; + } + + if (n) + d.normalize (); + } + private: parser* p_; scope* r_; @@ -162,7 +172,11 @@ namespace build2 tracer& tr) { auto r (p.scope_->find_target_type (n, o, loc)); - return p.ctx.targets.insert ( + + if (r.first.factory == nullptr) + p.fail (loc) << "abstract target type " << r.first.name << "{}"; + + return p.ctx->targets.insert ( r.first, // target type move (n.dir), move (o.dir), @@ -182,12 +196,16 @@ namespace build2 tracer& tr) { auto r (p.scope_->find_target_type (n, o, loc)); - return p.ctx.targets.find (r.first, // target type - n.dir, - o.dir, - n.value, - r.second, // extension - tr); + + if (r.first.factory == nullptr) + p.fail (loc) << "abstract target type " << r.first.name << "{}"; + + return p.ctx->targets.find (r.first, // target type + n.dir, + o.dir, + n.value, + r.second, // extension + tr); } ~enter_target () @@ -198,8 +216,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_target (enter_target&& x) {*this = move (x);} - enter_target& operator= (enter_target&& x) { + enter_target (enter_target&& x) noexcept {*this = move (x);} + enter_target& operator= (enter_target&& x) noexcept { p_ = x.p_; t_ = x.t_; x.p_ = nullptr; return *this;} enter_target (const enter_target&) = delete; @@ -230,8 +248,8 @@ namespace build2 // Note: move-assignable to empty only. // - enter_prerequisite (enter_prerequisite&& x) {*this = move (x);} - enter_prerequisite& operator= (enter_prerequisite&& x) { + enter_prerequisite (enter_prerequisite&& x) noexcept {*this = move (x);} + enter_prerequisite& operator= (enter_prerequisite&& x) noexcept { p_ = x.p_; r_ = x.r_; x.p_ = nullptr; return *this;} enter_prerequisite (const enter_prerequisite&) = delete; @@ -247,6 +265,7 @@ namespace build2 { pre_parse_ = false; attributes_.clear (); + condition_ = nullopt; default_target_ = nullptr; peeked_ = false; replay_ = replay::stop; @@ -259,10 +278,11 @@ namespace build2 scope* root, scope& base, target* tgt, - prerequisite* prq) + prerequisite* prq, + bool enter) { lexer l (is, in); - parse_buildfile (l, root, base, tgt, prq); + parse_buildfile (l, root, base, tgt, prq, enter); } void parser:: @@ -270,7 +290,8 @@ namespace build2 scope* root, scope& base, target* tgt, - prerequisite* prq) + prerequisite* prq, + bool enter) { path_ = &l.name (); lexer_ = &l; @@ -289,9 +310,9 @@ namespace build2 ? auto_project_env (*root_) : auto_project_env ()); - if (path_->path != nullptr) - enter_buildfile (*path_->path); // Note: needs scope_. - + const buildfile* bf (enter && path_->path != nullptr + ? &enter_buildfile<buildfile> (*path_->path) + : nullptr); token t; type tt; next (t, tt); @@ -303,13 +324,34 @@ namespace build2 else { parse_clause (t, tt); - process_default_target (t); + + if (stage_ != stage::boot && stage_ != stage::root) + process_default_target (t, bf); } if (tt != type::eos) fail (t) << "unexpected " << t; } + names parser:: + parse_export_stub (istream& is, const path_name& name, + const scope& rs, scope& gs, scope& ts) + { + // Enter the export stub manually with correct out. + // + if (name.path != nullptr) + { + dir_path out (!rs.out_eq_src () + ? out_src (name.path->directory (), rs) + : dir_path ()); + + enter_buildfile<buildfile> (*name.path, move (out)); + } + + parse_buildfile (is, name, &gs, ts, nullptr, nullptr, false /* enter */); + return move (export_value); + } + token parser:: parse_variable (lexer& l, scope& s, const variable& var, type kind) { @@ -355,6 +397,37 @@ namespace build2 return make_pair (move (lhs), move (t)); } + names parser:: + parse_names (lexer& l, + const dir_path* b, + pattern_mode pmode, + const char* what, + const string* separators) + { + path_ = &l.name (); + lexer_ = &l; + + root_ = nullptr; + scope_ = nullptr; + target_ = nullptr; + prerequisite_ = nullptr; + + pbase_ = b; + + token t; + type tt; + + mode (lexer_mode::value, '@'); + next (t, tt); + + names r (parse_names (t, tt, pmode, what, separators)); + + if (tt != type::eos) + fail (t) << "unexpected " << t; + + return r; + } + value parser:: parse_eval (lexer& l, scope& rs, scope& bs, pattern_mode pmode) { @@ -544,6 +617,12 @@ namespace build2 { f = &parser::parse_config_environment; } + else if (n == "recipe") + { + // Valid only after recipe header (%). + // + fail (t) << n << " directive without % recipe header"; + } if (f != nullptr) { @@ -560,9 +639,39 @@ namespace build2 location nloc (get_location (t)); names ns; - if (tt != type::labrace) + // We have to parse names in chunks to detect invalid cases of the + // group{foo}<...> syntax. + // + // Consider (1): + // + // x = + // group{foo} $x<...>: + // + // And (2): + // + // x = group{foo} group{bar} + // $x<...>: + // + // As well as (3): + // + // <...><...>: + // + struct chunk { - ns = parse_names (t, tt, pattern_mode::preserve); + size_t pos; // Index in ns of the beginning of the last chunk. + location loc; // Position of the beginning of the last chunk. + }; + optional<chunk> ns_last; + + bool labrace_first (tt == type::labrace); + if (!labrace_first) + { + do + { + ns_last = chunk {ns.size (), get_location (t)}; + parse_names (t, tt, ns, pattern_mode::preserve, true /* chunk */); + } + while (start_names (tt)); // Allow things like function calls that don't result in anything. // @@ -578,44 +687,87 @@ namespace build2 } } - // Handle ad hoc target group specification (<...>). + // Handle target group specification (<...>). // // We keep an "optional" (empty) vector of names parallel to ns that - // contains the ad hoc group members. + // contains the group members. Note that when we "catch" gns up to ns, + // we populate it with ad hoc (as opposed to explicit) groups with no + // members. // - adhoc_names ans; + group_names gns; if (tt == type::labrace) { - while (tt == type::labrace) + for (; tt == type::labrace; labrace_first = false) { - // Parse target names inside < >. + // Detect explicit group (group{foo}<...>). + // + // Note that `<` first thing on the line is not seperated thus the + // labrace_first complication. + // + bool expl (!t.separated && !labrace_first); + if (expl) + { + // Note: (N) refers to the example in the above comment. + // + if (!ns_last /* (3) */ || ns_last->pos == ns.size () /* (1) */) + { + fail (t) << "group name or whitespace expected before '<'"; + } + else + { + size_t n (ns.size () - ns_last->pos); + + // Note: could be a pair. + // + if ((n > 2 || (n == 2 && !ns[ns_last->pos].pair)) /* (2) */) + { + fail (t) << "single group name or whitespace expected before " + << "'<' instead of '" + << names_view (ns.data () + ns_last->pos, n) << "'"; + } + } + } + + // Parse target names inside <>. // // We "reserve" the right to have attributes inside <> though what // exactly that would mean is unclear. One potentially useful - // semantics would be the ability to specify attributes for ad hoc - // members though the fact that the primary target is listed first - // would make it rather unintuitive. Maybe attributes that change - // the group semantics itself? + // semantics would be the ability to specify attributes for group + // members though the fact that the primary target for ad hoc groups + // is listed first would make it rather unintuitive. Maybe + // attributes that change the group semantics itself? // next_with_attributes (t, tt); auto at (attributes_push (t, tt)); if (at.first) - fail (at.second) << "attributes before ad hoc target"; + fail (at.second) << "attributes before group member"; else attributes_pop (); - // Allow empty case (<>). + // For explicit groups, the group target is already in ns and all + // the members should go straight to gns. // - if (tt != type::rabrace) + // For ad hoc groups, the first name (or a pair) is the primary + // target which we need to keep in ns. The rest, if any, are ad + // hoc members that we should move to gns. + // + if (expl) + { + gns.resize (ns.size ()); // Catch up with the names vector. + group_names_loc& g (gns.back ()); + g.expl = true; + g.group_loc = move (ns_last->loc); + g.member_loc = get_location (t); // Start of members. + + if (tt != type::rabrace) // Handle empty case (<>) + parse_names (t, tt, g.ns, pattern_mode::preserve); + } + else if (tt != type::rabrace) // Allow and ignore empty case (<>). { - location aloc (get_location (t)); + location mloc (get_location (t)); // Start of members. - // The first name (or a pair) is the primary target which we need - // to keep in ns. The rest, if any, are ad hoc members that we - // should move to ans. - // size_t m (ns.size ()); parse_names (t, tt, ns, pattern_mode::preserve); size_t n (ns.size ()); @@ -632,11 +784,10 @@ namespace build2 { n -= m; // Number of names in ns we should end up with. - ans.resize (n); // Catch up with the names vector. - adhoc_names_loc& a (ans.back ()); - - a.loc = move (aloc); - a.ns.insert (a.ns.end (), + gns.resize (n); // Catch up with the names vector. + group_names_loc& g (gns.back ()); + g.group_loc = g.member_loc = move (mloc); + g.ns.insert (g.ns.end (), make_move_iterator (ns.begin () + n), make_move_iterator (ns.end ())); ns.resize (n); @@ -650,12 +801,16 @@ namespace build2 // Parse the next chunk of target names after >, if any. // next (t, tt); - if (start_names (tt)) - parse_names (t, tt, ns, pattern_mode::preserve); + ns_last = nullopt; // To detect <...><...>. + while (start_names (tt)) + { + ns_last = chunk {ns.size (), get_location (t)}; + parse_names (t, tt, ns, pattern_mode::preserve, true /* chunk */); + } } - if (!ans.empty ()) - ans.resize (ns.size ()); // Catch up with the final chunk. + if (!gns.empty ()) + gns.resize (ns.size ()); // Catch up with the final chunk. if (tt != type::colon) fail (t) << "expected ':' instead of " << t; @@ -683,10 +838,11 @@ namespace build2 // evaluated. The function signature is: // // void (token& t, type& tt, + // optional<bool> member, // true -- explict, false -- ad hoc // optional<pattern_type>, const target_type* pat_tt, string pat, // const location& pat_loc) // - // Note that the target and its ad hoc members are inserted implied + // Note that the target and its group members are inserted implied // but this flag can be cleared and default_target logic applied if // appropriate. // @@ -776,23 +932,31 @@ namespace build2 // Resolve target type. If none is specified, then it's file{}. // + // Note: abstract target type is ok here. + // const target_type* ttype (n.untyped () ? &file::static_type : scope_->find_target_type (n.type)); if (ttype == nullptr) - fail (nloc) << "unknown target type " << n.type; + fail (nloc) << "unknown target type " << n.type << + info << "perhaps the module that defines this target type is " + << "not loaded by project " << *scope_->root_scope (); - f (t, tt, n.pattern, ttype, move (n.value), nloc); + f (t, tt, nullopt, n.pattern, ttype, move (n.value), nloc); }; auto for_each = [this, &trace, &for_one_pat, - &t, &tt, &as, &ns, &nloc, &ans] (auto&& f) + &t, &tt, &as, &ns, &nloc, &gns] (auto&& f) { + // We need replay if we have multiple targets or group members. + // // Note: watch out for an out-qualified single target (two names). // replay_guard rg (*this, - ns.size () > 2 || (ns.size () == 2 && !ns[0].pair)); + ns.size () > 2 || + (ns.size () == 2 && !ns[0].pair) || + !gns.empty ()); for (size_t i (0), e (ns.size ()); i != e; ) { @@ -812,8 +976,9 @@ namespace build2 if (n.pair) fail (nloc) << "out-qualified target type/pattern"; - if (!ans.empty () && !ans[i].ns.empty ()) - fail (ans[i].loc) << "ad hoc member in target type/pattern"; + if (!gns.empty () && !gns[i].ns.empty ()) + fail (gns[i].member_loc) + << "group member in target type/pattern"; if (*n.pattern == pattern_type::regex_substitution) fail (nloc) << "regex substitution " << n << " without " @@ -823,27 +988,47 @@ namespace build2 } else { - name o (n.pair ? move (ns[++i]) : name ()); - enter_target tg (*this, - move (n), - move (o), - true /* implied */, - nloc, - trace); - - if (!as.empty ()) - apply_target_attributes (*target_, as); - - // Enter ad hoc members. - // - if (!ans.empty ()) + bool expl; + vector<reference_wrapper<target>> gms; { - // Note: index after the pair increment. + name o (n.pair ? move (ns[++i]) : name ()); + enter_target tg (*this, + move (n), + move (o), + true /* implied */, + nloc, + trace); + + if (!as.empty ()) + apply_target_attributes (*target_, as); + + // Enter group members. // - enter_adhoc_members (move (ans[i]), true /* implied */); + if (!gns.empty ()) + { + // Note: index after the pair increment. + // + group_names_loc& g (gns[i]); + expl = g.expl; + + if (expl && !target_->is_a<group> ()) + fail (g.group_loc) << *target_ << " is not group target"; + + gms = expl + ? enter_explicit_members (move (g), true /* implied */) + : enter_adhoc_members (move (g), true /* implied */); + } + + f (t, tt, nullopt, nullopt, nullptr, string (), location ()); } - f (t, tt, nullopt, nullptr, string (), location ()); + for (target& gm: gms) + { + rg.play (); // Replay. + + enter_target tg (*this, gm); + f (t, tt, expl, nullopt, nullptr, string (), location ()); + } } if (++i != e) @@ -897,12 +1082,15 @@ namespace build2 ploc = get_location (t); pns = parse_names (t, tt, pattern_mode::preserve); - // Target-specific variable assignment. + // Target type/pattern-specific variable assignment. // if (tt == type::assign || tt == type::prepend || tt == type::append) { - if (!ans.empty ()) - fail (ans[0].loc) << "ad hoc member in target type/pattern"; + // Note: ns contains single target name. + // + if (!gns.empty ()) + fail (gns[0].member_loc) + << "group member in target type/pattern"; // Note: see the same code below if changing anything here. // @@ -921,6 +1109,7 @@ namespace build2 for_one_pat ( [this, &var, akind, &aloc] ( token& t, type& tt, + optional<bool>, optional<pattern_type> pt, const target_type* ptt, string pat, const location& ploc) { @@ -966,6 +1155,7 @@ namespace build2 for_one_pat ( [this] ( token& t, type& tt, + optional<bool>, optional<pattern_type> pt, const target_type* ptt, string pat, const location& ploc) { @@ -985,8 +1175,11 @@ namespace build2 if (pns.empty () && tt != type::percent && tt != type::multi_lcbrace) { - if (!ans.empty ()) - fail (ans[0].loc) << "ad hoc member in target type/pattern"; + // Note: ns contains single target name. + // + if (!gns.empty ()) + fail (gns[0].member_loc) + << "group member in target type/pattern"; if (!as.empty ()) fail (as.loc) << "attributes before target type/pattern"; @@ -1090,22 +1283,33 @@ namespace build2 check_pattern (n, nloc); - // Verify all the ad hoc members are patterns or substitutions and - // of the correct type. + // If we have group members, verify all the members are patterns or + // substitutions (ad hoc) or subsitutions (explicit) and of the + // correct pattern type. A rule for an explicit group that wishes to + // match based on some of its members feels far fetched. + // + // For explicit groups the use-case is to inject static members + // which could otherwise be tedious to specify for each group. // - names ns (ans.empty () ? names () : move (ans[0].ns)); - const location& aloc (ans.empty () ? location () : ans[0].loc); + const location& mloc (gns.empty () ? location () : gns[0].member_loc); + names ns (gns.empty () ? names () : move (gns[0].ns)); + bool expl (gns.empty () ? false : gns[0].expl); for (name& n: ns) { if (!n.pattern || !(*n.pattern == pt || (st && *n.pattern == *st))) { - fail (aloc) << "expected " << pn << " pattern or substitution " + fail (mloc) << "expected " << pn << " pattern or substitution " << "instead of " << n; } if (*n.pattern != pattern_type::regex_substitution) - check_pattern (n, aloc); + { + if (expl) + fail (mloc) << "explicit group member pattern " << n; + + check_pattern (n, mloc); + } } // The same for prerequisites except here we can have non-patterns. @@ -1165,7 +1369,15 @@ namespace build2 : scope_->find_target_type (n.type); if (ttype == nullptr) - fail (nloc) << "unknown target type " << n.type; + fail (nloc) << "unknown target type " << n.type << + info << "perhaps the module that defines this target type is " + << "not loaded by project " << *scope_->root_scope (); + + if (!gns.empty ()) + { + if (ttype->is_a<group> () != expl) + fail (nloc) << "group type and target type mismatch"; + } unique_ptr<adhoc_rule_pattern> rp; switch (pt) @@ -1178,7 +1390,7 @@ namespace build2 rp.reset (new adhoc_rule_regex_pattern ( *scope_, rn, *ttype, move (n), nloc, - move (ns), aloc, + move (ns), mloc, move (pns), ploc)); break; case pattern_type::regex_substitution: @@ -1201,8 +1413,13 @@ namespace build2 for (shared_ptr<adhoc_rule>& pr: recipes) { - pr->pattern = &rp; // Connect recipe to pattern. - rp.rules.push_back (move (pr)); + // Can be NULL if the recipe is disabled with a condition. + // + if (pr != nullptr) + { + pr->pattern = &rp; // Connect recipe to pattern. + rp.rules.push_back (move (pr)); + } } // Register this adhoc rule for all its actions. @@ -1250,7 +1467,14 @@ namespace build2 // recipe for dist. // // And the same for the configure meta-operation to, for - // example, make sure a hinted ad hoc rule matches. + // example, make sure a hinted ad hoc rule matches. @@ Hm, + // maybe we fixed this with action-specific hints? But the + // injection part above may still apply. BTW, this is also + // required for see-through groups in order to resolve their + // member. + // + // Note also that the equivalent semantics for ad hoc recipes + // is provided by match_adhoc_recipe(). // if (a.meta_operation () == perform_id) { @@ -1272,7 +1496,8 @@ namespace build2 // see-through target group, then we may also need to // register update for other meta-operations (see, for // example, wildcard update registration in the cli - // module). + // module). BTW, we can now detect such a target via + // its target type flags. } } } @@ -1316,6 +1541,7 @@ namespace build2 st = token (t), // Save start token (will be gone on replay). recipes = small_vector<shared_ptr<adhoc_rule>, 1> ()] (token& t, type& tt, + optional<bool> gm, // true -- explicit, false -- ad hoc optional<pattern_type> pt, const target_type* ptt, string pat, const location& ploc) mutable { @@ -1329,7 +1555,14 @@ namespace build2 // next (t, tt); // Newline. next (t, tt); // First token inside the variable block. - parse_variable_block (t, tt, pt, ptt, move (pat), ploc); + + // For explicit groups we only assign variables on the group + // omitting the members. + // + if (!gm || !*gm) + parse_variable_block (t, tt, pt, ptt, move (pat), ploc); + else + skip_block (t, tt); if (tt != type::rcbrace) fail (t) << "expected '}' instead of " << t; @@ -1345,6 +1578,16 @@ namespace build2 else rt = st; + // If this is a group member then we know we are replaying and + // can skip the recipe. + // + if (gm) + { + replay_skip (); + next (t, tt); + return; + } + if (pt) fail (rt) << "unexpected recipe after target type/pattern" << info << "ad hoc pattern rule may not be combined with other " @@ -1365,7 +1608,7 @@ namespace build2 // Note also that we treat this as an explicit dependency // declaration (i.e., not implied). // - enter_targets (move (ns), nloc, move (ans), 0, as); + enter_targets (move (ns), nloc, move (gns), 0, as); } continue; @@ -1380,7 +1623,8 @@ namespace build2 if (!start_names (tt)) fail (t) << "unexpected " << t; - // @@ PAT: currently we pattern-expand target-specific var names. + // @@ PAT: currently we pattern-expand target-specific var names (see + // also parse_import()). // const location ploc (get_location (t)); names pns (parse_names (t, tt, pattern_mode::expand)); @@ -1415,6 +1659,7 @@ namespace build2 for_each ( [this, &var, akind, &aloc] ( token& t, type& tt, + optional<bool> gm, optional<pattern_type> pt, const target_type* ptt, string pat, const location& ploc) { @@ -1423,7 +1668,18 @@ namespace build2 *pt, *ptt, move (pat), ploc, var, akind, aloc); else - parse_variable (t, tt, var, akind); + { + // Skip explicit group members (see the block case above for + // background). + // + if (!gm || !*gm) + parse_variable (t, tt, var, akind); + else + { + next (t, tt); + skip_line (t, tt); + } + } }); next_after_newline (t, tt); @@ -1441,7 +1697,7 @@ namespace build2 parse_dependency (t, tt, move (ns), nloc, - move (ans), + move (gns), move (pns), ploc, as); } @@ -1667,7 +1923,7 @@ namespace build2 // Parse a recipe chain. // // % [<attrs>] [<buildspec>] - // [if|switch ...] + // [if|if!|switch|recipe ...] // {{ [<lang> ...] // ... // }} @@ -1686,10 +1942,27 @@ namespace build2 // if (target_ != nullptr) { + // @@ What if some members are added later? + // + // @@ Also, what happends if redeclared as real dependency, do we + // upgrade the members? + // if (target_->decl != target_decl::real) { - for (target* m (target_); m != nullptr; m = m->adhoc_member) - m->decl = target_decl::real; + target_->decl = target_decl::real; + + if (group* g = target_->is_a<group> ()) + { + for (const target& m: g->static_members) + const_cast<target&> (m).decl = target_decl::real; // During load. + } + else + { + for (target* m (target_->adhoc_member); + m != nullptr; + m = m->adhoc_member) + m->decl = target_decl::real; + } if (default_target_ == nullptr) default_target_ = target_; @@ -1702,7 +1975,15 @@ namespace build2 t = start; tt = t.type; for (size_t i (0); tt == type::percent || tt == type::multi_lcbrace; ++i) { - recipes.push_back (nullptr); // For missing else/default (see below). + // For missing else/default (see below). + // + // Note that it may remain NULL if we have, say, an if-condition that + // evaluates to false and no else. While it may be tempting to get rid + // of such "holes", it's not easy due to the replay semantics (see the + // target_ != nullptr block below). So we expect the caller to be + // prepared to handle this. + // + recipes.push_back (nullptr); attributes as; buildspec bs; @@ -1719,7 +2000,131 @@ namespace build2 attributes& as; buildspec& bs; const location& bsloc; - } d {ttype, name, recipes, first, clean, i, as, bs, bsloc}; + function<void (string&&)> parse_trailer; + } d {ttype, name, recipes, first, clean, i, as, bs, bsloc, {}}; + + d.parse_trailer = [this, &d] (string&& text) + { + if (d.first) + { + adhoc_rule& ar (*d.recipes.back ()); + + // Translate each buildspec entry into action and add it to the + // recipe entry. + // + const location& l (d.bsloc); + + for (metaopspec& m: d.bs) + { + meta_operation_id mi (ctx->meta_operation_table.find (m.name)); + + if (mi == 0) + fail (l) << "unknown meta-operation " << m.name; + + const meta_operation_info* mf ( + root_->root_extra->meta_operations[mi]); + + if (mf == nullptr) + fail (l) << "project " << *root_ << " does not support meta-" + << "operation " << ctx->meta_operation_table[mi].name; + + for (opspec& o: m) + { + operation_id oi; + if (o.name.empty ()) + { + if (mf->operation_pre == nullptr) + oi = update_id; + else + // Calling operation_pre() to translate doesn't feel + // appropriate here. + // + fail (l) << "default operation in recipe action" << endf; + } + else + oi = ctx->operation_table.find (o.name); + + if (oi == 0) + fail (l) << "unknown operation " << o.name; + + const operation_info* of (root_->root_extra->operations[oi]); + + if (of == nullptr) + fail (l) << "project " << *root_ << " does not support " + << "operation " << ctx->operation_table[oi]; + + // Note: for now always inner (see match_rule_impl() for + // details). + // + action a (mi, oi); + + // Check for duplicates (local). + // + if (find_if ( + d.recipes.begin (), d.recipes.end (), + [a] (const shared_ptr<adhoc_rule>& r) + { + auto& as (r->actions); + return find (as.begin (), as.end (), a) != as.end (); + }) != d.recipes.end ()) + { + fail (l) << "duplicate " << mf->name << '(' << of->name + << ") recipe"; + } + + ar.actions.push_back (a); + } + } + + // Set the recipe text. + // + if (ar.recipe_text ( + *scope_, + d.ttype != nullptr ? *d.ttype : target_->type (), + move (text), + d.as)) + d.clean = true; + + // Verify we have no unhandled attributes. + // + for (attribute& a: d.as) + fail (d.as.loc) << "unknown recipe attribute " << a << endf; + } + + // Copy the recipe over to the target verifying there are no + // duplicates (global). + // + if (target_ != nullptr) + { + const shared_ptr<adhoc_rule>& r (d.recipes[d.i]); + + for (const shared_ptr<adhoc_rule>& er: target_->adhoc_recipes) + { + auto& as (er->actions); + + for (action a: r->actions) + { + if (find (as.begin (), as.end (), a) != as.end ()) + { + const meta_operation_info* mf ( + root_->root_extra->meta_operations[a.meta_operation ()]); + + const operation_info* of ( + root_->root_extra->operations[a.operation ()]); + + fail (d.bsloc) + << "duplicate " << mf->name << '(' << of->name + << ") recipe for target " << *target_; + } + } + } + + target_->adhoc_recipes.push_back (r); + + // Note that "registration" of configure_* and dist_* actions + // (similar to ad hoc rules) is provided by match_adhoc_recipe(). + } + }; // Note that this function must be called at most once per iteration. // @@ -1762,7 +2167,7 @@ namespace build2 // to rule_name. shared_ptr<adhoc_rule> ar; - if (!lang) + if (!lang || icasecmp (*lang, "buildscript") == 0) { // Buildscript // @@ -1858,129 +2263,200 @@ namespace build2 } if (!skip) - { - if (d.first) - { - adhoc_rule& ar (*d.recipes.back ()); - - // Translate each buildspec entry into action and add it to the - // recipe entry. - // - const location& l (d.bsloc); + d.parse_trailer (move (t.value)); - for (metaopspec& m: d.bs) - { - meta_operation_id mi (ctx.meta_operation_table.find (m.name)); - - if (mi == 0) - fail (l) << "unknown meta-operation " << m.name; + next (t, tt); + assert (tt == type::multi_rcbrace); - const meta_operation_info* mf ( - root_->root_extra->meta_operations[mi]); + next (t, tt); // Newline. + next_after_newline (t, tt, token (t)); // Should be on its own line. + }; - if (mf == nullptr) - fail (l) << "project " << *root_ << " does not support meta-" - << "operation " << ctx.meta_operation_table[mi].name; + auto parse_recipe_directive = [this, &d] (token& t, type& tt, + const string&) + { + // Parse recipe directive: + // + // recipe <lang> <file> + // + // Note that here <lang> is not optional. + // + // @@ We could guess <lang> from the extension. - for (opspec& o: m) - { - operation_id oi; - if (o.name.empty ()) - { - if (mf->operation_pre == nullptr) - oi = update_id; - else - // Calling operation_pre() to translate doesn't feel - // appropriate here. - // - fail (l) << "default operation in recipe action" << endf; - } - else - oi = ctx.operation_table.find (o.name); + // Use value mode to minimize the number of special characters. + // + mode (lexer_mode::value, '@'); - if (oi == 0) - fail (l) << "unknown operation " << o.name; + // Parse <lang>. + // + if (next (t, tt) != type::word) + fail (t) << "expected recipe language instead of " << t; - const operation_info* of (root_->root_extra->operations[oi]); + location lloc (get_location (t)); + string lang (t.value); + next (t, tt); - if (of == nullptr) - fail (l) << "project " << *root_ << " does not support " - << "operation " << ctx.operation_table[oi]; + // Parse <file> as names to get variable expansion, etc. + // + location nloc (get_location (t)); + names ns (parse_names (t, tt, pattern_mode::ignore, "file name")); - // Note: for now always inner (see match_rule() for details). - // - action a (mi, oi); + path file; + try + { + file = convert<path> (move (ns)); + } + catch (const invalid_argument& e) + { + fail (nloc) << "invalid recipe file path: " << e; + } - // Check for duplicates (local). - // - if (find_if ( - d.recipes.begin (), d.recipes.end (), - [a] (const shared_ptr<adhoc_rule>& r) - { - auto& as (r->actions); - return find (as.begin (), as.end (), a) != as.end (); - }) != d.recipes.end ()) - { - fail (l) << "duplicate " << mf->name << '(' << of->name - << ") recipe"; - } + string text; + if (d.first) + { + // Source relative to the buildfile rather than src scope. In + // particular, this make sourcing from exported buildfiles work. + // + if (file.relative () && path_->path != nullptr) + { + // Note: all sourced/included/imported paths are absolute and + // normalized. + // + file = path_->path->directory () / file; + } - ar.actions.push_back (a); - } - } + file.normalize (); - // Set the recipe text. - // - if (ar.recipe_text ( - *scope_, - d.ttype != nullptr ? *d.ttype : target_->type (), - move (t.value), - d.as)) - d.clean = true; - - // Verify we have no unhandled attributes. - // - for (attribute& a: d.as) - fail (d.as.loc) << "unknown recipe attribute " << a << endf; + try + { + ifdstream ifs (file); + text = ifs.read_text (); + } + catch (const io_error& e) + { + fail (nloc) << "unable to read recipe file " << file << ": " << e; } - // Copy the recipe over to the target verifying there are no - // duplicates (global). - // - if (target_ != nullptr) + shared_ptr<adhoc_rule> ar; { - const shared_ptr<adhoc_rule>& r (d.recipes[d.i]); + // This is expected to be the location of the opening multi-curly + // with the recipe body starting from the following line. So we + // need to fudge the line number a bit. + // + location loc (file, 0, 1); + + if (icasecmp (lang, "buildscript") == 0) + { + // Buildscript + // + ar.reset ( + new adhoc_buildscript_rule ( + d.name.empty () ? "<ad hoc buildscript recipe>" : d.name, + loc, + 2)); // Use `{{` and `}}` for dump. - for (const shared_ptr<adhoc_rule>& er: target_->adhoc_recipes) + // Enter as buildfile-like so that it gets automatically + // distributed. Note: must be consistent with build/export/ + // handling in process_default_target(). + // + enter_buildfile<buildscript> (file); + } + else if (icasecmp (lang, "c++") == 0) { - auto& as (er->actions); + // C++ + // + // We expect to find a C++ comment line with version and + // optional fragment separator before the first non-comment, + // non-blank line: + // + // // c++ <ver> [<sep>] + // + string s; + location sloc (file, 1, 1); + { + // Note: observe blank lines for accurate line count. + // + size_t b (0), e (0); + for (size_t m (0), n (text.size ()); + next_word (text, n, b, e, m, '\n', '\r'), b != n; + sloc.line++) + { + s.assign (text, b, e - b); - for (action a: r->actions) + if (!trim (s).empty ()) + { + if (icasecmp (s, "// c++ ", 7) == 0) + break; + + if (s[0] != '/' || s[1] != '/') + { + b = e; + break; + } + } + } + + if (b == e) + fail (sloc) << "no '// c++ <version> [<separator>]' line"; + } + + uint64_t ver; + optional<string> sep; { - if (find (as.begin (), as.end (), a) != as.end ()) + size_t b (7), e (7); + if (next_word (s, b, e, ' ', '\t') == 0) + fail (sloc) << "missing c++ recipe version" << endf; + + try { - const meta_operation_info* mf ( - root_->root_extra->meta_operations[a.meta_operation ()]); + ver = convert<uint64_t> (build2::name (string (s, b, e - b))); + } + catch (const invalid_argument& e) + { + fail (sloc) << "invalid c++ recipe version: " << e << endf; + } - const operation_info* of ( - root_->root_extra->operations[a.operation ()]); + if (next_word (s, b, e, ' ', '\t') != 0) + { + sep = string (s, b, e - b); - fail (d.bsloc) - << "duplicate " << mf->name << '(' << of->name - << ") recipe for target " << *target_; + if (next_word (s, b, e, ' ', '\t') != 0) + fail (sloc) << "junk after fragment separator"; } } - } - target_->adhoc_recipes.push_back (r); + ar.reset ( + new adhoc_cxx_rule ( + d.name.empty () ? "<ad hoc c++ recipe>" : d.name, + loc, + 2, // Use `{{` and `}}` for dump. + ver, + move (sep))); + + // Enter as buildfile-like so that it gets automatically + // distributed. Note: must be consistent with build/export/ + // handling in process_default_target(). + // + // While ideally we would want to use the cxx{} target type, + // it's defined in a seperate build system module (which may not + // even be loaded by this project, so even runtime lookup won't + // work). So we use file{} instead. + // + enter_buildfile<build2::file> (file); + } + else + fail (lloc) << "unknown recipe language '" << lang << "'"; } + + assert (d.recipes[d.i] == nullptr); + d.recipes[d.i] = move (ar); } + else + assert (d.recipes[d.i] != nullptr); - next (t, tt); - assert (tt == type::multi_rcbrace); + d.parse_trailer (move (text)); - next (t, tt); // Newline. - next_after_newline (t, tt, token (t)); // Should be on its own line. + next_after_newline (t, tt); }; bsloc = get_location (t); // Fallback location. @@ -2040,7 +2516,7 @@ namespace build2 expire_mode (); next_after_newline (t, tt, "recipe action"); - // See if this is if-else or switch. + // See if this is if-else/switch or `recipe`. // // We want the keyword test similar to parse_clause() but we cannot do // it if replaying. So we skip it with understanding that if it's not @@ -2056,14 +2532,21 @@ namespace build2 // handy if we want to provide a custom recipe but only on certain // platforms or some such). - if (n == "if") + if (n == "if" || n == "if!") { - parse_if_else (t, tt, true /* multi */, parse_block); + parse_if_else (t, tt, true /* multi */, + parse_block, parse_recipe_directive); continue; } else if (n == "switch") { - parse_switch (t, tt, true /* multi */, parse_block); + parse_switch (t, tt, true /* multi */, + parse_block, parse_recipe_directive); + continue; + } + else if (n == "recipe") + { + parse_recipe_directive (t, tt, "" /* kind */); continue; } @@ -2071,7 +2554,7 @@ namespace build2 } if (tt != type::multi_lcbrace) - fail (t) << "expected recipe block instead of " << t; + fail (t) << "expected recipe block or 'recipe' instead of " << t; // Fall through. } @@ -2116,13 +2599,97 @@ namespace build2 } } - void parser:: - enter_adhoc_members (adhoc_names_loc&& ans, bool implied) + vector<reference_wrapper<target>> parser:: + enter_explicit_members (group_names_loc&& gns, bool implied) + { + tracer trace ("parser::enter_explicit_members", &path_); + + names& ns (gns.ns); + const location& loc (gns.member_loc); + + vector<reference_wrapper<target>> r; + r.reserve (ns.size ()); + + group& g (target_->as<group> ()); + auto& ms (g.static_members); + + for (size_t i (0); i != ns.size (); ++i) + { + name&& n (move (ns[i])); + name&& o (n.pair ? move (ns[++i]) : name ()); + + if (n.qualified ()) + fail (loc) << "project name in target " << n; + + // We derive the path unless the target name ends with the '...' escape + // which here we treat as the "let the rule derive the path" indicator + // (see target::split_name() for details). This will only be useful for + // referring to group members that are managed by the group's matching + // rule. Note also that omitting '...' for such a member could be used + // to override the file name, provided the rule checks if the path has + // already been derived before doing it itself. + // + // @@ What can the ad hoc recipe/rule do differently here? Maybe get + // path from dynamic targets? Maybe we will have custom path + // derivation support in buildscript in the future? + // + bool escaped; + { + const string& v (n.value); + size_t p (v.size ()); + + escaped = (p > 3 && + v[--p] == '.' && v[--p] == '.' && v[--p] == '.' && + v[--p] != '.'); + } + + target& m (enter_target::insert_target (*this, + move (n), move (o), + implied, + loc, trace)); + + if (g == m) + fail (loc) << "explicit group member " << m << " is group itself"; + + // Add as static member skipping duplicates. + // + if (find (ms.begin (), ms.end (), m) == ms.end ()) + { + if (m.group == nullptr) + m.group = &g; + else if (m.group != &g) + fail (loc) << g << " group member " << m << " already belongs to " + << "group " << *m.group; + + ms.push_back (m); + } + + if (!escaped) + { + if (file* ft = m.is_a<file> ()) + ft->derive_path (); + } + + r.push_back (m); + } + + return r; + } + + vector<reference_wrapper<target>> parser:: + enter_adhoc_members (group_names_loc&& gns, bool implied) { tracer trace ("parser::enter_adhoc_members", &path_); - names& ns (ans.ns); - const location& loc (ans.loc); + names& ns (gns.ns); + const location& loc (gns.member_loc); + + if (target_->is_a<group> ()) + fail (loc) << "ad hoc group primary member " << *target_ + << " is explicit group"; + + vector<reference_wrapper<target>> r; + r.reserve (ns.size ()); for (size_t i (0); i != ns.size (); ++i) { @@ -2150,14 +2717,16 @@ namespace build2 v[--p] != '.'); } - target& at ( - enter_target::insert_target (*this, - move (n), move (o), - implied, - loc, trace)); + target& m (enter_target::insert_target (*this, + move (n), move (o), + implied, + loc, trace)); + + if (target_ == &m) + fail (loc) << "ad hoc group member " << m << " is primary target"; - if (target_ == &at) - fail (loc) << "ad hoc group member " << at << " is primary target"; + if (m.is_a<group> ()) + fail (loc) << "ad hoc group member " << m << " is explicit group"; // Add as an ad hoc member at the end of the chain skipping duplicates. // @@ -2165,7 +2734,7 @@ namespace build2 const_ptr<target>* mp (&target_->adhoc_member); for (; *mp != nullptr; mp = &(*mp)->adhoc_member) { - if (*mp == &at) + if (*mp == &m) { mp = nullptr; break; @@ -2174,31 +2743,41 @@ namespace build2 if (mp != nullptr) { - *mp = &at; - at.group = target_; + if (m.group == nullptr) + m.group = target_; + else if (m.group != target_) + fail (loc) << *target_ << " ad hoc group member " << m + << " already belongs to group " << *m.group; + *mp = &m; } } if (!escaped) { - if (file* ft = at.is_a<file> ()) + if (file* ft = m.is_a<file> ()) ft->derive_path (); } + + r.push_back (m); } + + return r; } - small_vector<reference_wrapper<target>, 1> parser:: + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> parser:: enter_targets (names&& tns, const location& tloc, // Target names. - adhoc_names&& ans, // Ad hoc target names. + group_names&& gns, // Group member names. size_t prereq_size, const attributes& tas) // Target attributes. { - // Enter all the targets (normally we will have just one) and their ad hoc - // groups. + // Enter all the targets (normally we will have just one) and their group + // members. // tracer trace ("parser::enter_targets", &path_); - small_vector<reference_wrapper<target>, 1> tgs; + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> tgs; for (size_t i (0); i != tns.size (); ++i) { @@ -2223,13 +2802,21 @@ namespace build2 if (!tas.empty ()) apply_target_attributes (*target_, tas); - // Enter ad hoc members. + // Enter group members. // - if (!ans.empty ()) + vector<reference_wrapper<target>> gms; + if (!gns.empty ()) { // Note: index after the pair increment. // - enter_adhoc_members (move (ans[i]), false /* implied */); + group_names_loc& g (gns[i]); + + if (g.expl && !target_->is_a<group> ()) + fail (g.group_loc) << *target_ << " is not group target"; + + gms = g.expl + ? enter_explicit_members (move (g), false /* implied */) + : enter_adhoc_members (move (g), false /* implied */); } if (default_target_ == nullptr) @@ -2237,7 +2824,7 @@ namespace build2 target_->prerequisites_state_.store (2, memory_order_relaxed); target_->prerequisites_.reserve (prereq_size); - tgs.push_back (*target_); + tgs.emplace_back (*target_, move (gms)); } return tgs; @@ -2295,7 +2882,7 @@ namespace build2 if (!v.empty ()) { - oi = ctx.operation_table.find (v); + oi = ctx->operation_table.find (v); if (oi == 0) fail (l) << "unknown operation " << v << " in rule_hint " @@ -2303,7 +2890,7 @@ namespace build2 if (root_->root_extra->operations[oi] == nullptr) fail (l) << "project " << *root_ << " does not support " - << "operation " << ctx.operation_table[oi] + << "operation " << ctx->operation_table[oi] << " specified in rule_hint attribute"; } } @@ -2325,7 +2912,7 @@ namespace build2 void parser:: parse_dependency (token& t, token_type& tt, names&& tns, const location& tloc, // Target names. - adhoc_names&& ans, // Ad hoc target names. + group_names&& gns, // Group member names. names&& pns, const location& ploc, // Prereq names. const attributes& tas) // Target attributes. { @@ -2337,33 +2924,92 @@ namespace build2 // tracer trace ("parser::parse_dependency", &path_); + // Diagnose conditional prerequisites. Note that we want to diagnose this + // even if pns is empty (think empty variable expansion; the literal "no + // prerequisites" case is handled elsewhere). + // + // @@ TMP For now we only do it during the dist meta-operation. In the + // future we should tighten this to any meta-operation provided + // the dist module is loaded. + // + // @@ TMP For now it's a warning because we have dependencies like + // cli.cxx{foo}: cli{foo} which are not currently possible to + // rewrite (cli.cxx{} is not always registered). + // + if (condition_ && + ctx->current_mif != nullptr && + ctx->current_mif->id == dist_id) + { + // Only issue the warning for the projects being distributed. In + // particular, this makes sure we don't complain about imported + // projects. Note: use amalgamation to cover bundled subprojects. + // + auto* dm (root_->bundle_scope ()->find_module<dist::module> ( + dist::module::name)); + + if (dm != nullptr && dm->distributed) + { + warn (tloc) << "conditional dependency declaration may result in " + << "incomplete distribution" << + info (ploc) << "prerequisite declared here" << + info (*condition_) << "conditional buildfile fragment starts here" << + info << "instead use 'include' prerequisite-specific variable to " + << "conditionally include prerequisites" << + info << "for example: " + << "<target>: <prerequisite>: include = (<condition>)" << + info << "for details, see https://github.com/build2/HOWTO/blob/" + << "master/entries/keep-build-graph-config-independent.md"; + } + } + // First enter all the targets. // - small_vector<reference_wrapper<target>, 1> tgs ( - enter_targets (move (tns), tloc, move (ans), pns.size (), tas)); + small_vector<pair<reference_wrapper<target>, + vector<reference_wrapper<target>>>, 1> + tgs (enter_targets (move (tns), tloc, move (gns), pns.size (), tas)); // Now enter each prerequisite into each target. // - for (name& pn: pns) + for (auto i (pns.begin ()); i != pns.end (); ++i) { // We cannot reuse the names if we (potentially) may need to pass them // as targets in case of a chain (see below). // - name n (tt != type::colon ? move (pn) : pn); + name n (tt != type::colon ? move (*i) : *i); // See also scope::find_prerequisite_key(). // auto rp (scope_->find_target_type (n, ploc)); - const target_type* tt (rp.first); + const target_type* t (rp.first); optional<string>& e (rp.second); - if (tt == nullptr) - fail (ploc) << "unknown target type " << n.type; + if (t == nullptr) + { + if (n.proj) + { + // If the target type is unknown then no phase 2 import (like + // rule-specific search) can possibly succeed so we can fail now and + // with a more accurate reason. See import2(names) for background. + // + diag_record dr; + dr << fail (ploc) << "unable to import target " << n; + import_suggest (dr, *n.proj, nullptr, string (), false); + } + else + { + fail (ploc) << "unknown target type " << n.type << + info << "perhaps the module that defines this target type is " + << "not loaded by project " << *scope_->root_scope (); + } + } + + if (t->factory == nullptr) + fail (ploc) << "abstract target type " << t->name << "{}"; // Current dir collapses to an empty one. // if (!n.dir.empty ()) - n.dir.normalize (false, true); + n.dir.normalize (false /* actual */, true); // @@ OUT: for now we assume the prerequisite's out is undetermined. The // only way to specify an src prerequisite will be with the explicit @@ -2374,10 +3020,47 @@ namespace build2 // a special indicator. Also, one can easily and natually suppress any // searches by specifying the absolute path. // + name o; + if (n.pair) + { + assert (n.pair == '@'); + + ++i; + o = tt != type::colon ? move (*i) : *i; + + if (!o.directory ()) + fail (ploc) << "expected directory after '@'"; + + o.dir.normalize (); // Note: don't collapse current to empty. + + // Make sure out and src are parallel unless both were specified as + // absolute. We make an exception for this case because out may be + // used to "tag" imported targets (see cc::search_library()). So it's + // sort of the "I know what I am doing" escape hatch (it would have + // been even better to verify such a target is outside any project + // but that won't be cheap). + // + // For now we require that both are either relative or absolute. + // + // See similar code for targets in scope::find_target_type(). + // + if (n.dir.absolute () && o.dir.absolute ()) + ; + else if (n.dir.empty () && o.dir.current ()) + ; + else if (o.dir.relative () && + n.dir.relative () && + o.dir == n.dir) + ; + else + fail (ploc) << "prerequisite output directory " << o.dir + << " must be parallel to source directory " << n.dir; + } + prerequisite p (move (n.proj), - *tt, + *t, move (n.dir), - dir_path (), + move (o.dir), move (n.value), move (e), *scope_); @@ -2386,7 +3069,7 @@ namespace build2 { // Move last prerequisite (which will normally be the only one). // - target& t (*i); + target& t (i->first); t.prerequisites_.push_back (++i == e ? move (p) : prerequisite (p, memory_order_relaxed)); @@ -2399,20 +3082,42 @@ namespace build2 // // We handle multiple targets and/or prerequisites by replaying the tokens // (see the target-specific case comments for details). The function - // signature is: + // signature for for_each_t (see for_each on the gm argument semantics): + // + // void (token& t, type& tt, optional<bool> gm) + // + // And for for_each_p: // // void (token& t, type& tt) // auto for_each_t = [this, &t, &tt, &tgs] (auto&& f) { - replay_guard rg (*this, tgs.size () > 1); + // We need replay if we have multiple targets or group members. + // + replay_guard rg (*this, tgs.size () > 1 || !tgs[0].second.empty ()); for (auto ti (tgs.begin ()), te (tgs.end ()); ti != te; ) { - target& tg (*ti); - enter_target tgg (*this, tg); + target& tg (ti->first); + const vector<reference_wrapper<target>>& gms (ti->second); - f (t, tt); + { + enter_target g (*this, tg); + f (t, tt, nullopt); + } + + if (!gms.empty ()) + { + bool expl (tg.is_a<group> ()); + + for (target& gm: gms) + { + rg.play (); // Replay. + + enter_target g (*this, gm); + f (t, tt, expl); + } + } if (++ti != te) rg.play (); // Replay. @@ -2425,8 +3130,8 @@ namespace build2 for (auto ti (tgs.begin ()), te (tgs.end ()); ti != te; ) { - target& tg (*ti); - enter_target tgg (*this, tg); + target& tg (ti->first); + enter_target g (*this, tg); for (size_t pn (tg.prerequisites_.size ()), pi (pn - pns.size ()); pi != pn; ) @@ -2469,7 +3174,7 @@ namespace build2 this, st = token (t), // Save start token (will be gone on replay). recipes = small_vector<shared_ptr<adhoc_rule>, 1> ()] - (token& t, type& tt) mutable + (token& t, type& tt, optional<bool> gm) mutable { token rt; // Recipe start token. @@ -2479,7 +3184,14 @@ namespace build2 { next (t, tt); // Newline. next (t, tt); // First token inside the variable block. - parse_variable_block (t, tt); + + // Skip explicit group members (see the block case above for + // background). + // + if (!gm || !*gm) + parse_variable_block (t, tt); + else + skip_block (t, tt); if (tt != type::rcbrace) fail (t) << "expected '}' instead of " << t; @@ -2495,6 +3207,16 @@ namespace build2 else rt = st; + // If this is a group member then we know we are replaying and can + // skip the recipe. + // + if (gm) + { + replay_skip (); + next (t, tt); + return; + } + parse_recipe (t, tt, rt, recipes); }; @@ -2504,21 +3226,6 @@ namespace build2 return; } - // What should we do if there are no prerequisites (for example, because - // of an empty wildcard result)? We can fail or we can ignore. In most - // cases, however, this is probably an error (for example, forgetting to - // checkout a git submodule) so let's not confuse the user and fail (one - // can always handle the optional prerequisites case with a variable and - // an if). - // - if (pns.empty ()) - fail (ploc) << "no prerequisites in dependency chain or prerequisite-" - << "specific variable assignment"; - - next_with_attributes (t, tt); // Recognize attributes after `:`. - - auto at (attributes_push (t, tt)); - // If we are here, then this can be one of three things: // // 1. A prerequisite-specific variable bloc: @@ -2532,10 +3239,37 @@ namespace build2 // // foo: bar: x = y // - // 3. A further dependency chain : + // 3. A further dependency chain: // // foo: bar: baz ... // + // What should we do if there are no prerequisites, for example, because + // of an empty wildcard result or empty variable expansion? We can fail or + // we can ignore. In most cases, however, this is probably an error (for + // example, forgetting to checkout a git submodule) so let's not confuse + // the user and fail (one can always handle the optional prerequisites + // case with a variable and an if). + // + // On the other hand, we allow just empty prerequisites (which is also the + // more common case by far) and so it's strange that we don't allow the + // same with, say, `include = false`: + // + // exe{foo}: cxx{$empty} # Ok. + // exe{foo}: cxx{$empty}: include = false # Not Ok? + // + // So let's ignore in the first two cases (variable block and assignment) + // for consistency. The dependency chain is iffy both conceptually and + // implementation-wise (it could be followed by a variable block). So + // let's keep it an error for now. + // + // Note that the syntactically-empty prerequisite list is still an error: + // + // exe{foo}: : include = false # Error. + // + next_with_attributes (t, tt); // Recognize attributes after `:`. + + auto at (attributes_push (t, tt)); + if (tt == type::newline || tt == type::eos) { attributes_pop (); // Must be none since can't be standalone. @@ -2550,15 +3284,22 @@ namespace build2 // Parse the block for each prerequisites of each target. // - for_each_p ([this] (token& t, token_type& tt) - { - next (t, tt); // First token inside the block. + if (!pns.empty ()) + for_each_p ([this] (token& t, token_type& tt) + { + next (t, tt); // First token inside the block. - parse_variable_block (t, tt); + parse_variable_block (t, tt); - if (tt != type::rcbrace) - fail (t) << "expected '}' instead of " << t; - }); + if (tt != type::rcbrace) + fail (t) << "expected '}' instead of " << t; + }); + else + { + skip_block (t, tt); + if (tt != type::rcbrace) + fail (t) << "expected '}' instead of " << t; + } next (t, tt); // Presumably newline after '}'. next_after_newline (t, tt, '}'); // Should be on its own line. @@ -2581,10 +3322,13 @@ namespace build2 // Parse the assignment for each prerequisites of each target. // - for_each_p ([this, &var, at] (token& t, token_type& tt) - { - parse_variable (t, tt, var, at); - }); + if (!pns.empty ()) + for_each_p ([this, &var, at] (token& t, token_type& tt) + { + parse_variable (t, tt, var, at); + }); + else + skip_line (t, tt); next_after_newline (t, tt); @@ -2603,6 +3347,9 @@ namespace build2 // else { + if (pns.empty ()) + fail (ploc) << "no prerequisites in dependency chain"; + // @@ This is actually ambiguous: prerequisite or target attributes // (or both or neither)? Perhaps this should be prerequisites for // the same reason as below (these are prerequsites first). @@ -2618,16 +3365,16 @@ namespace build2 // we just say that the dependency chain is equivalent to specifying // each dependency separately. // - // Also note that supporting ad hoc target group specification in - // chains will be complicated. For example, what if prerequisites that - // have ad hoc targets don't end up being chained? Do we just silently - // drop them? Also, these are prerequsites first that happened to be - // reused as target names so perhaps it is the right thing not to - // support, conceptually. + // Also note that supporting target group specification in chains will + // be complicated. For example, what if prerequisites that have group + // members don't end up being chained? Do we just silently drop them? + // Also, these are prerequsites first that happened to be reused as + // target names so perhaps it is the right thing not to support, + // conceptually. // parse_dependency (t, tt, move (pns), ploc, - {} /* ad hoc target name */, + {} /* group names */, move (ns), loc, attributes () /* target attributes */); } @@ -2635,14 +3382,18 @@ namespace build2 } void parser:: - source (istream& is, const path_name& in, const location& loc, bool deft) + source_buildfile (istream& is, + const path_name& in, + const location& loc, + optional<bool> deft) { - tracer trace ("parser::source", &path_); + tracer trace ("parser::source_buildfile", &path_); l5 ([&]{trace (loc) << "entering " << in;}); - if (in.path != nullptr) - enter_buildfile (*in.path); + const buildfile* bf (in.path != nullptr + ? &enter_buildfile<buildfile> (*in.path) + : nullptr); const path_name* op (path_); path_ = ∈ @@ -2652,11 +3403,11 @@ namespace build2 lexer_ = &l; target* odt; - if (deft) - { + if (!deft || *deft) odt = default_target_; + + if (deft && *deft) default_target_ = nullptr; - } token t; type tt; @@ -2666,12 +3417,15 @@ namespace build2 if (tt != type::eos) fail (t) << "unexpected " << t; - if (deft) + if (deft && *deft) { - process_default_target (t); - default_target_ = odt; + if (stage_ != stage::boot && stage_ != stage::root) + process_default_target (t, bf); } + if (!deft || *deft) + default_target_ = odt; + lexer_ = ol; path_ = op; @@ -2681,11 +3435,35 @@ namespace build2 void parser:: parse_source (token& t, type& tt) { + // source [<attrs>] <path>+ + // + // The rest should be a list of buildfiles. Parse them as names in the - // value mode to get variable expansion and directory prefixes. + // value mode to get variable expansion and directory prefixes. Also + // handle optional attributes. // mode (lexer_mode::value, '@'); - next (t, tt); + next_with_attributes (t, tt); + attributes_push (t, tt); + + bool nodt (false); // Source buildfile without default target semantics. + { + attributes as (attributes_pop ()); + const location& l (as.loc); + + for (const attribute& a: as) + { + const string& n (a.name); + + if (n == "no_default_target") + { + nodt = true; + } + else + fail (l) << "unknown source directive attribute " << a; + } + } + const location l (get_location (t)); names ns (tt != type::newline && tt != type::eos ? parse_names (t, tt, pattern_mode::expand, "path", nullptr) @@ -2712,10 +3490,10 @@ namespace build2 try { ifdstream ifs (p); - source (ifs, - path_name (p), - get_location (t), - false /* default_target */); + source_buildfile (ifs, + path_name (p), + get_location (t), + nodt ? optional<bool> {} : false); } catch (const io_error& e) { @@ -2729,6 +3507,9 @@ namespace build2 void parser:: parse_include (token& t, type& tt) { + // include <path>+ + // + tracer trace ("parser::parse_include", &path_); if (stage_ == stage::boot) @@ -2837,19 +3618,35 @@ namespace build2 l6 ([&]{trace (l) << "absolute path " << p;}); - if (!root_->buildfiles.insert (p).second) // Note: may be "new" root. + // Note: may be "new" root. + // + if (!root_->root_extra->insert_buildfile (p)) { l5 ([&]{trace (l) << "skipping already included " << p;}); continue; } + // Note: see a variant of this in parse_import(). + // + // Clear/restore if/switch location. + // + // We do it here but not in parse_source since the included buildfile is + // in a sense expected to be a standalone entity (think a file included + // from an export stub). + // + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = nullopt; + try { ifdstream ifs (p); - source (ifs, - path_name (p), - get_location (t), - true /* default_target */); + source_buildfile (ifs, + path_name (p), + get_location (t), + true /* default_target */); } catch (const io_error& e) { @@ -2901,13 +3698,16 @@ namespace build2 [] (const string& s) {return s.c_str ();}); cargs.push_back (nullptr); + // Note: we are in the serial load phase and so no diagnostics buffering + // is needed. + // process pr (run_start (3 /* verbosity */, cargs, 0 /* stdin */, -1 /* stdout */, - true /* error */, - dir_path () /* cwd */, + 2 /* stderr */, nullptr /* env */, + dir_path () /* cwd */, l)); try { @@ -2929,10 +3729,10 @@ namespace build2 dr << info (l) << "while parsing " << args[0] << " output"; }); - source (is, - path_name ("<stdout>"), - l, - false /* default_target */); + source_buildfile (is, + path_name ("<stdout>"), + l, + false /* default_target */); } is.close (); // Detect errors. @@ -2946,7 +3746,7 @@ namespace build2 // caused by that and let run_finish() deal with it. } - run_finish (cargs, pr, l); + run_finish (cargs, pr, 2 /* verbosity */, false /* omit_normal */, l); next_after_newline (t, tt); } @@ -2992,14 +3792,16 @@ namespace build2 // which case it will be duplicating them in its root.build file). So // for now we allow this trusting the user knows what they are doing. // - string proj; - { - const project_name& n (named_project (*root_)); - - if (!n.empty ()) - proj = n.variable (); - } - + // There is another special case: a buildfile imported from another + // project. In this case we also allow <project> to be the imported + // project name in addition to importing. The thinking here is that an + // imported buildfile is in a sense like a module (may provide rules which + // may require configuration, etc) and should be able to use its own + // project name (which is often the corresponding tool name) in the + // configuration variables, just like modules. In this case we use the + // imported project name as the reporting module name (but which can + // be overridden with config.report.module attribute). + // const location loc (get_location (t)); // We are now in the normal lexing mode and we let the lexer handle `?=`. @@ -3017,6 +3819,11 @@ namespace build2 optional<string> report; string report_var; + // Reporting module name. Empty means the config module reporting + // project's own configuration. + // + project_name report_module; + for (auto i (as.begin ()); i != as.end (); ) { if (i->name == "null") @@ -3035,7 +3842,7 @@ namespace build2 report = move (v); else throw invalid_argument ( - "expected 'false' or format name instead of '" + v + "'"); + "expected 'false' or format name instead of '" + v + '\''); } catch (const invalid_argument& e) { @@ -3047,6 +3854,23 @@ namespace build2 try { report_var = convert<string> (move (i->value)); + + if (!report) + report = string ("true"); + } + catch (const invalid_argument& e) + { + fail (as.loc) << "invalid " << i->name << " attribute value: " << e; + } + } + else if (i->name == "config.report.module") + { + try + { + report_module = convert<project_name> (move (i->value)); + + if (!report) + report = string ("true"); } catch (const invalid_argument& e) { @@ -3070,9 +3894,13 @@ namespace build2 // As a way to print custom (discovered, computed, etc) configuration // information we allow specifying a non config.* variable provided it is - // explicitly marked with the config.report attribute. + // explicitly marked with the config.report attribute (or another + // attribute that implies it). // bool new_val (false); + string org_var; // Original variable if config.report.variable specified. + + const variable* var (nullptr); // config.* variable. lookup l; if (report && *report != "false" && !config) @@ -3091,7 +3919,14 @@ namespace build2 // philosophical question. In either case it doesn't seem useful for it // to unconditionally force reporting at level 2. // - report_var = move (name); + if (!report_var.empty ()) + { + // For example, config [config.report.variable=multi] multi_database + // + org_var = move (name); + } + else + report_var = move (name); next (t, tt); // We shouldn't have the default value part. } @@ -3104,41 +3939,133 @@ namespace build2 // config prefix and the project substring. // { - diag_record dr; + string proj; + { + const project_name& n (named_project (*root_)); - if (!config) - dr << fail (t) << "configuration variable '" << name - << "' does not start with 'config.'"; + if (!n.empty ()) + proj = n.variable (); + } - if (!proj.empty ()) + diag_record dr; + do // Breakout loop. { - size_t p (name.find ('.' + proj)); + if (!config) + { + dr << fail (t) << "configuration variable '" << name + << "' does not start with 'config.'"; + break; + } - if (p == string::npos || - ((p += proj.size () + 1) != name.size () && // config.<proj> - name[p] != '.')) // config.<proj>. + auto match = [&name] (const string& proj) { + size_t p (name.find ('.' + proj)); + return (p != string::npos && + ((p += proj.size () + 1) == name.size () || // config.<proj> + name[p] == '.')); // config.<proj>. + }; + + if (!proj.empty () && match (proj)) + break; + + // See if this buildfile belongs to a different project. If so, use + // the project name as the reporting module name. + // + if (path_->path != nullptr) + { + // Note: all sourced/included/imported paths are absolute and + // normalized. + // + const path& f (*path_->path); + dir_path d (f.directory ()); + + auto p (ctx->scopes.find (d)); // Note: never empty. + if (*p.first != &ctx->global_scope) + { + // The buildfile will most likely be in src which means we may + // end up with multiple scopes (see scope_map for background). + // First check if one of them is us. If not, then we can extract + // the project name from any one of them. + // + const scope& bs (**p.first); // Save. + + for (; p.first != p.second; ++p.first) + { + if (root_ == (*p.first)->root_scope ()) + break; + } + + if (p.first == p.second) + { + // Note: we expect the project itself to be named. + // + const project_name& n (project (*bs.root_scope ())); + + if (!n.empty ()) + { + // If the buildfile comes from a different project, then + // it's more likely to use the imported project's config + // variables. So replace proj with that for diagnostics + // below. + // + proj = n.variable (); + + if (*report != "false" && verb >= 2) + report_module = n; + } + } + } + else + { + // If the buildfile is not in any project, then it could be + // installed. + // + // Per import2_buildfile(), exported buildfiles are installed + // into $install.buildfile/<proj>/.... + // + const dir_path& id (build_install_buildfile); + + if (!id.empty () && d.sub (id)) + { + dir_path l (d.leaf (id)); + if (!l.empty ()) + { + project_name n (*l.begin ()); + proj = n.variable (); + + if (*report != "false" && verb >= 2) + report_module = move (n); + } + } + } + } + + if (!proj.empty () && match (proj)) + break; + + // Note: only if proj not empty (see above). + // + if (!proj.empty ()) dr << fail (t) << "configuration variable '" << name << "' does not include project name"; - } } + while (false); if (!dr.empty ()) dr << info << "expected variable name in the 'config[.**]." << (proj.empty () ? "<project>" : proj.c_str ()) << ".**' form"; } - const variable& var ( - parse_variable_name (move (name), get_location (t))); - apply_variable_attributes (var); + var = &parse_variable_name (move (name), get_location (t)); + apply_variable_attributes (*var); // Note that even though we are relying on the config.** variable // pattern to set global visibility, let's make sure as a sanity check. // - if (var.visibility != variable_visibility::global) + if (var->visibility != variable_visibility::global) { - fail (t) << "configuration variable " << var << " has " - << var.visibility << " visibility"; + fail (t) << "configuration variable " << *var << " has " + << var->visibility << " visibility"; } // See if we have the default value part. @@ -3160,15 +4087,15 @@ namespace build2 // bool dev; { - size_t p (var.name.rfind ('.')); - dev = p != 6 && var.name.compare (p + 1, string::npos, "develop") == 0; + size_t p (var->name.rfind ('.')); + dev = p != 6 && var->name.compare (p + 1, string::npos, "develop") == 0; } uint64_t sflags (0); if (dev) { - if (var.type != &value_traits<bool>::value_type) - fail (loc) << var << " variable must be of type bool"; + if (var->type != &value_traits<bool>::value_type) + fail (loc) << *var << " variable must be of type bool"; // This is quite messy: below we don't always parse the value (plus it // may be computed) so here we just peek at the next token. But we @@ -3177,10 +4104,10 @@ namespace build2 if (!def_val || peek (lexer_mode::value, '@') != type::word || peeked ().value != "false") - fail (loc) << var << " variable default value must be literal false"; + fail (loc) << *var << " variable default value must be literal false"; if (nullable) - fail (loc) << var << " variable must not be nullable"; + fail (loc) << *var << " variable must not be nullable"; sflags |= config::save_false_omitted; } @@ -3189,7 +4116,7 @@ namespace build2 // in order to mark it as saved. We also have to do this to get the new // value status. // - l = config::lookup_config (new_val, *root_, var, sflags); + l = config::lookup_config (new_val, *root_, *var, sflags); // Handle the default value. // @@ -3225,12 +4152,12 @@ namespace build2 else { value lhs, rhs (parse_variable_value (t, tt, !dev /* mode */)); - apply_value_attributes (&var, lhs, move (rhs), type::assign); + apply_value_attributes (var, lhs, move (rhs), type::assign); if (!nullable) nullable = lhs.null; - l = config::lookup_config (new_val, *root_, var, move (lhs), sflags); + l = config::lookup_config (new_val, *root_, *var, move (lhs), sflags); } } @@ -3241,51 +4168,86 @@ namespace build2 // then the user is expected to handle the undefined case). // if (!nullable && l.defined () && l->null) - fail (loc) << "null value in non-nullable variable " << var; + fail (loc) << "null value in non-nullable variable " << *var; } // We will be printing the report at either level 2 (-v) or 3 (-V) - // depending on the final value of config_report_new. + // depending on the final value of config_report::new_value. // - // Note that for the config_report_new calculation we only incorporate - // variables that we are actually reporting. + // Note that for the config_report::new_value calculation we only + // incorporate variables that we are actually reporting. // if (*report != "false" && verb >= 2) { + // Find existing or insert new config_report entry for this module. + // + auto i (find_if (config_reports.begin (), + config_reports.end (), + [&report_module] (const config_report& r) + { + return r.module == report_module; + })); + + if (i == config_reports.end ()) + { + config_reports.push_back ( + config_report {move (report_module), {}, false}); + i = config_reports.end () - 1; + } + + auto& report_values (i->values); + bool& report_new_value (i->new_value); + // We don't want to lookup the report variable value here since it's // most likely not set yet. // if (!report_var.empty ()) { + if (org_var.empty () && var != nullptr) + org_var = var->name; + // In a somewhat hackish way we pass the variable in an undefined // lookup. // + // Note: consistent with parse_variable_name() wrt overridability. + // l = lookup (); l.var = &root_->var_pool ().insert ( - move (report_var), true /* overridable */); + move (report_var), + report_var.find ('.') != string::npos /* overridable */); } if (l.var != nullptr) { - auto r (make_pair (l, move (*report))); - // If we have a duplicate, update it (it could be useful to have // multiple config directives to "probe" the value before calculating // the default; see lookup_config() for details). // - auto i (find_if (config_report.begin (), - config_report.end (), - [&l] (const pair<lookup, string>& p) + // Since the original variable is what the user will see in the + // report, we prefer that as a key. + // + auto i (find_if (report_values.begin (), + report_values.end (), + [&org_var, &l] (const config_report::value& v) { - return p.first.var == l.var; + return (v.org.empty () && org_var.empty () + ? v.val.var == l.var + : (v.org.empty () + ? v.val.var->name == org_var + : v.org == l.var->name)); })); - if (i == config_report.end ()) - config_report.push_back (move (r)); + if (i == report_values.end ()) + report_values.push_back ( + config_report::value {l, move (*report), move (org_var)}); else - *i = move (r); + { + i->val = l; + i->fmt = move (*report); + if (i->org.empty ()) i->org = move (org_var); + } - config_report_new = config_report_new || new_val; + report_new_value = report_new_value || new_val; } } @@ -3342,118 +4304,306 @@ namespace build2 if (stage_ == stage::boot) fail (t) << "import during bootstrap"; - // General import format: + // General import form: // // import[?!] [<attrs>] <var> = [<attrs>] (<target>|<project>%<target>])+ // + // Special form for importing buildfiles: + // + // import[?!] [<attrs>] (<target>|<project>%<target>])+ + // bool opt (t.value.back () == '?'); - bool ph2 (opt || t.value.back () == '!'); + optional<string> ph2 (opt || t.value.back () == '!' + ? optional<string> (string ()) + : nullopt); // We are now in the normal lexing mode and we let the lexer handle `=`. // next_with_attributes (t, tt); - // Get variable attributes, if any, and deal with the special metadata - // attribute. Since currently it can only appear in the import directive, - // we handle it in an ad hoc manner. + // Get variable (or value, in the second form) attributes, if any, and + // deal with the special metadata and rule_hint attributes. Since + // currently they can only appear in the import directive, we handle them + // in an ad hoc manner. // attributes_push (t, tt); - attributes& as (attributes_top ()); - bool meta (false); - for (auto i (as.begin ()); i != as.end (); ) + bool meta (false); // Import with metadata. + bool once (false); // Import buildfile once. + bool nodt (false); // Import buildfile without default target semantics. { - if (i->name == "metadata") - { - if (!ph2) - fail (as.loc) << "loading metadata requires immediate import" << - info << "consider using the import! directive instead"; + attributes& as (attributes_top ()); + const location& l (as.loc); - meta = true; - } - else + for (auto i (as.begin ()); i != as.end (); ) { - ++i; - continue; - } + const string& n (i->name); + value& v (i->value); - i = as.erase (i); - } + if (n == "metadata") + { + if (!ph2) + fail (l) << "loading metadata requires immediate import" << + info << "consider using the import! directive instead"; - if (tt != type::word) - fail (t) << "expected variable name instead of " << t; + meta = true; + } + else if (n == "no_default_target") + { + nodt = true; + } + else if (n == "once") + { + once = true; + } + else if (n == "rule_hint") + { + if (!ph2) + fail (l) << "rule hint can only be used with immediate import" << + info << "consider using the import! directive instead"; - const variable& var ( - parse_variable_name (move (t.value), get_location (t))); - apply_variable_attributes (var); + // Here we only allow a single name. + // + try + { + ph2 = convert<string> (move (v)); - if (var.visibility > variable_visibility::scope) - { - fail (t) << "variable " << var << " has " << var.visibility - << " visibility but is assigned in import"; + if (ph2->empty ()) + throw invalid_argument ("empty name"); + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else + { + ++i; + continue; + } + + i = as.erase (i); + } } - // Next should come the assignment operator. Note that we don't support + // Note that before supporting the second form (without <var>) we used to + // parse the value after assignment in the value mode. However, we don't + // really need to since what we should have is a bunch of target names. + // In other words, whatever the value mode does not treat as special + // compared to the normal mode (like `:`) would be illegal here. + // + // Note that we expant patterns for the ad hoc import case: + // + // import sub = */ + // + // @@ PAT: the only issue here is that we currently pattern-expand var + // name (same assue as with target-specific var names). + // + if (!start_names (tt)) + fail (t) << "expected variable name or buildfile target instead of " << t; + + location loc (get_location (t)); + names ns (parse_names (t, tt, pattern_mode::expand)); + + // Next could come the assignment operator. Note that we don't support // default assignment (?=) yet (could make sense when attempting to import // alternatives or some such). // - next (t, tt); + type atype; + const variable* var (nullptr); + if (tt == type::assign || tt == type::append || tt == type::prepend) + { + var = &parse_variable_name (move (ns), loc); + apply_variable_attributes (*var); + + if (var->visibility > variable_visibility::scope) + { + fail (loc) << "variable " << *var << " has " << var->visibility + << " visibility but is assigned in import"; + } - if (tt != type::assign && tt != type::append && tt != type::prepend) - fail (t) << "expected variable assignment instead of " << t; + atype = tt; + next_with_attributes (t, tt); + attributes_push (t, tt, true /* standalone */); - type atype (tt); - value& val (atype == type::assign - ? scope_->assign (var) - : scope_->append (var)); + if (!start_names (tt)) + fail (t) << "expected target to import instead of " << t; - // The rest should be a list of targets. Parse them similar to a value on - // the RHS of an assignment (attributes, etc). - // - // Note that we expant patterns for the ad hoc import case: - // - // import sub = */ + loc = get_location (t); + ns = parse_names (t, tt, pattern_mode::expand); + } + else if (tt == type::default_assign) + fail (t) << "default assignment not yet supported"; + + + // If there are any value attributes, roundtrip the names through the + // value applying the attributes. // - mode (lexer_mode::value, '@'); - next_with_attributes (t, tt); + if (!attributes_top ().empty ()) + { + value lhs, rhs (move (ns)); + apply_value_attributes (nullptr, lhs, move (rhs), type::assign); - if (tt == type::newline || tt == type::eos) - fail (t) << "expected target to import instead of " << t; + if (!lhs) + fail (loc) << "expected target to import instead of null value"; - const location loc (get_location (t)); + untypify (lhs, true /* reduce */); + ns = move (lhs.as<names> ()); + } + else + attributes_pop (); - if (value v = parse_value_with_attributes (t, tt, pattern_mode::expand)) + value* val (var != nullptr ? + &(atype == type::assign + ? scope_->assign (*var) + : scope_->append (*var)) + : nullptr); + + for (name& n: ns) { - names storage; - for (name& n: reverse (v, storage)) + // @@ Could this be an out-qualified ad hoc import? Yes, see comment + // about buildfile import in import_load(). + // + if (n.pair) + fail (loc) << "unexpected pair in import"; + + // See if we are importing a buildfile target. Such an import is always + // immediate. + // + bool bf (n.type == "buildfile"); + if (bf) { - // @@ Could this be an out-qualified ad hoc import? - // - if (n.pair) - fail (loc) << "unexpected pair in import"; + if (meta) + fail (loc) << "metadata requested for buildfile target " << n; - // import() will check the name, if required. - // - names r (import (*scope_, move (n), ph2, opt, meta, loc).first); + if (var != nullptr) + { + if (once) + fail (loc) << "once importation requested with variable assignment"; + + if (nodt) + fail (loc) << "no_default_target importation requested with " + << "variable assignment"; + } + + if (ph2 && !ph2->empty ()) + fail (loc) << "rule hint specified for buildfile target " << n; + } + else + { + if (once) + fail (loc) << "once importation requested for target " << n; + + if (nodt) + fail (loc) << "no_default_target importation requested for target " + << n; + if (var == nullptr) + fail (loc) << "variable assignment required to import target " << n; + } + + // import() will check the name, if required. + // + import_result<scope> ir ( + import (*scope_, + move (n), + ph2 ? ph2 : bf ? optional<string> (string ()) : nullopt, + opt, + meta, + loc)); + + names& r (ir.name); + + if (val != nullptr) + { if (r.empty ()) // Optional not found. { if (atype == type::assign) - val = nullptr; + *val = nullptr; } else { - if (atype == type::assign) - val.assign (move (r), &var); - else if (atype == type::prepend) - val.prepend (move (r), &var); - else - val.append (move (r), &var); + // Import (more precisely, alias) the target type into this project + // if not known. + // + // Note that if the result is ignored (val is NULL), then it's fair + // to assume this is not necessary. + // + if (const scope* iroot = ir.target) + { + const name& n (r.front ()); + if (n.typed ()) + import_target_type (*root_, *iroot, n.type, loc); + } + + if (atype == type::assign) val->assign (move (r), var); + else if (atype == type::prepend) val->prepend (move (r), var); + else val->append (move (r), var); } if (atype == type::assign) atype = type::append; // Append subsequent values. } + else + { + assert (bf); + + if (r.empty ()) // Optional not found. + { + assert (opt); + continue; + } + + // Note: see also import_buildfile(). + // + assert (r.size () == 1); // See import_load() for details. + name& n (r.front ()); + path p (n.dir / n.value); // Should already include extension. + + // Note: similar to parse_include(). + // + // Nuance: we insert this buildfile even with once=false in case it + // gets imported with once=true from another place. + // + if (!root_->root_extra->insert_buildfile (p) && once) + { + l5 ([&]{trace (loc) << "skipping already imported " << p;}); + continue; + } + + // Clear/restore if/switch location. + // + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = nullopt; + + try + { + ifdstream ifs (p); + + auto df = make_diag_frame ( + [this, &p, &loc] (const diag_record& dr) + { + dr << info (loc) << p << " imported from here"; + }); + + // @@ Do we want to enter this buildfile? What's the harm (one + // benefit is that it will be in dump). But, we currently don't + // out-qualify them, though feels like there is nothing fatal + // in that, just inaccurate. + // + source_buildfile (ifs, + path_name (p), + loc, + nodt ? optional<bool> {} : false); + } + catch (const io_error& e) + { + fail (loc) << "unable to read imported buildfile " << p << ": " << e; + } + } } next_after_newline (t, tt); @@ -3495,7 +4645,12 @@ namespace build2 fail (l) << "null value in export"; if (val.type != nullptr) - untypify (val); + { + // While feels far-fetched, let's preserve empty typed values in the + // result. + // + untypify (val, false /* reduce */); + } export_value = move (val).as<names> (); @@ -3582,41 +4737,160 @@ namespace build2 void parser:: parse_define (token& t, type& tt) { - // define <derived>: <base> + // define [<attrs>] <derived>: <base> + // define <alias> = <scope>/<type> // // See tests/define. // - if (next (t, tt) != type::word) - fail (t) << "expected name instead of " << t << " in target type " - << "definition"; + next_with_attributes (t, tt); - string dn (move (t.value)); - const location dnl (get_location (t)); + attributes_push (t, tt); + attributes as (attributes_pop ()); - if (next (t, tt) != type::colon) - fail (t) << "expected ':' instead of " << t << " in target type " + if (tt != type::word) + fail (t) << "expected name instead of " << t << " in target type " << "definition"; + string n (move (t.value)); + const location nl (get_location (t)); + next (t, tt); - if (tt == type::word) + if (tt == type::colon) { + // Handle attributes. + // + target_type::flag fs (target_type::flag::none); + { + const location& l (as.loc); + + for (attribute& a: as) + { + const string& n (a.name); + value& v (a.value); + + if (n == "see_through") fs |= target_type::flag::see_through; + else if (n == "member_hint") fs |= target_type::flag::member_hint; + else + fail (l) << "unknown target type definition attribute " << n; + + if (!v.null) + fail (l) << "unexpected value in attribute " << n; + } + } + + if (next (t, tt) != type::word) + fail (t) << "expected name instead of " << t << " in target type " + << "definition"; + // Target. // const string& bn (t.value); const target_type* bt (scope_->find_target_type (bn)); if (bt == nullptr) - fail (t) << "unknown target type " << bn; + fail (t) << "unknown target type " << bn << + info << "perhaps the module that defines this target type is " + << "not loaded by project " << *scope_->root_scope (); - if (!root_->derive_target_type (move (dn), *bt).second) - fail (dnl) << "target type " << dn << " already defined in this " - << "project"; + // The derive_target_type() call below does not produce a non-abstract + // type if passed an abstract base. So we ban this for now (it's unclear + // why would someone want to do this). + // + if (bt->factory == nullptr) + fail (t) << "abstract base target type " << bt->name << "{}"; + + // Note that the group{foo}<...> syntax is only recognized for group- + // based targets and ad hoc buildscript recipes/rules only match group. + // (We may want to relax this for member_hint in the future since its + // currently also used on non-mtime-based targets, though what exactly + // we will do in ad hoc recipes/rules in this case is fuzzy). + // + if ((fs & target_type::flag::group) == target_type::flag::group && + !bt->is_a<group> ()) + fail (t) << "base target type " << bn << " must be group for " + << "group-related attribute"; + + if (!root_->derive_target_type (move (n), *bt, fs).second) + fail (nl) << "target type " << n << " already defined in project " + << *root_; next (t, tt); // Get newline. } + else if (tt == type::assign) + { + if (!as.empty ()) + fail (as.loc) << "unexpected target type alias attribute"; + + // The rest should be a path-like target type. Parse it as names in + // the value mode to get variable expansion, etc. + // + mode (lexer_mode::value, '@'); + next (t, tt); + const location tl (get_location (t)); + names ns ( + parse_names (t, tt, pattern_mode::ignore, "target type", nullptr)); + + name* tn (nullptr); + if (ns.size () == 1) + { + tn = &ns.front (); + + if (tn->file ()) + { + try + { + tn->canonicalize (); + + if (tn->dir.absolute ()) + tn->dir.normalize (); + else + tn = nullptr; + } + catch (const invalid_path&) {tn = nullptr;} + catch (const invalid_argument&) {tn = nullptr;} + } + else + tn = nullptr; + } + + if (tn == nullptr) + fail (tl) << "expected scope-qualified target type instead of " << ns; + + // If we got here, then tn->dir is the scope and tn->value is the target + // type. + // + // NOTE: see similar code in import_target_type(). + // + const target_type* tt (nullptr); + if (const scope* rs = ctx->scopes.find_out (tn->dir).root_scope ()) + { + tt = rs->find_target_type (tn->value); + + if (tt == nullptr) + fail (tl) << "unknown target type " << tn->value << " in scope " + << *rs; + } + else + fail (tl) << "unknown project scope " << tn->dir << " in scope" + << "-qualified target type" << + info << "did you forget to import the corresponding project?"; + + if (n != tn->value) + fail (nl) << "alias target type name " << n << " does not match " + << tn->value; + + // Note that this is potentially a shallow reference to a user-derived + // target type. Seeing that we only ever destory the entire graph, this + // should be ok. + // + auto p (root_->root_extra->target_types.insert (*tt)); + + if (!p.second && &p.first.get () != tt) + fail (nl) << "target type " << n << " already defined in this project"; + } else - fail (t) << "expected name instead of " << t << " in target type " + fail (t) << "expected ':' or '=' instead of " << t << " in target type " << "definition"; next_after_newline (t, tt); @@ -3625,19 +4899,28 @@ namespace build2 void parser:: parse_if_else (token& t, type& tt) { + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = get_location (t); + parse_if_else (t, tt, false /* multi */, [this] (token& t, type& tt, bool s, const string& k) { return parse_clause_block (t, tt, s, k); - }); + }, + {}); } void parser:: parse_if_else (token& t, type& tt, bool multi, const function<void ( - token&, type&, bool, const string&)>& parse_block) + token&, type&, bool, const string&)>& parse_block, + const function<void ( + token&, token_type&, const string&)>& parse_recipe_directive) { // Handle the whole if-else chain. See tests/if-else. // @@ -3662,7 +4945,7 @@ namespace build2 // is not an option. So let's skip it. // if (taken) - skip_line (t, tt); + skip_line (t, tt); // Skip expression. else { if (tt == type::newline || tt == type::eos) @@ -3722,31 +5005,65 @@ namespace build2 parse_block (t, tt, !take, k); taken = taken || take; } - else if (!multi) // No lines in multi-curly if-else. + else { - if (take) + // The only valid line in multi-curly if-else is `recipe`. + // + if (multi) { - if (!parse_clause (t, tt, true)) - fail (t) << "expected " << k << "-line instead of " << t; + // Note that we cannot do the keyword test if we are replaying. So + // we skip it with the understanding that if it's not a keywords, + // then we wouldn't have gotten here on the replay. + // + if (tt == type::word && + (replay_ == replay::play || keyword (t)) && + t.value == "recipe") + { + if (take) + { + parse_recipe_directive (t, tt, k); + taken = true; + } + else + { + skip_line (t, tt); - taken = true; + if (tt == type::newline) + next (t, tt); + } + } + else + fail (t) << "expected " << k << "-block or 'recipe' instead of " + << t; } else { - skip_line (t, tt); + if (tt == type::multi_lcbrace) + fail (t) << "expected " << k << "-line instead of " << t << + info << "did you forget to specify % recipe header?"; - if (tt == type::newline) - next (t, tt); + if (take) + { + if (!parse_clause (t, tt, true)) + fail (t) << "expected " << k << "-line instead of " << t; + + taken = true; + } + else + { + skip_line (t, tt); + + if (tt == type::newline) + next (t, tt); + } } } - else - fail (t) << "expected " << k << "-block instead of " << t; // See if we have another el* keyword. // // Note that we cannot do the keyword test if we are replaying. So we // skip it with the understanding that if it's not a keywords, then we - // wouldn't have gotten here on the reply (see parse_recipe() for + // wouldn't have gotten here on the replay (see parse_recipe() for // details). // if (k != "else" && @@ -3766,19 +5083,28 @@ namespace build2 void parser:: parse_switch (token& t, type& tt) { + auto g = make_guard ([this, old = condition_] () mutable + { + condition_ = old; + }); + condition_ = get_location (t); + parse_switch (t, tt, false /* multi */, [this] (token& t, type& tt, bool s, const string& k) { return parse_clause_block (t, tt, s, k); - }); + }, + {}); } void parser:: parse_switch (token& t, type& tt, bool multi, const function<void ( - token&, type&, bool, const string&)>& parse_block) + token&, type&, bool, const string&)>& parse_block, + const function<void ( + token&, token_type&, const string&)>& parse_recipe_directive) { // switch <value> [: <func> [<arg>]] [, <value>...] // { @@ -3873,7 +5199,7 @@ namespace build2 { // Note that we cannot do the keyword test if we are replaying. So we // skip it with the understanding that if it's not a keywords, then we - // wouldn't have gotten here on the reply (see parse_recipe() for + // wouldn't have gotten here on the replay (see parse_recipe() for // details). Note that this appears to mean that replay cannot be used // if we allow lines, only blocks. Consider: // @@ -3976,7 +5302,7 @@ namespace build2 if (!e.arg.empty ()) args.push_back (value (e.arg)); - value r (ctx.functions.call (scope_, *e.func, args, l)); + value r (ctx->functions.call (scope_, *e.func, args, l)); // We support two types of functions: matchers and extractors: // a matcher returns a statically-typed bool value while an @@ -4079,25 +5405,49 @@ namespace build2 parse_block (t, tt, !take, k); taken = taken || take; } - else if (!multi) // No lines in multi-curly if-else. + else { - if (take) + if (multi) { - if (!parse_clause (t, tt, true)) - fail (t) << "expected " << k << "-line instead of " << t; + if (tt == type::word && + (replay_ == replay::play || keyword (t)) && + t.value == "recipe") + { + if (take) + { + parse_recipe_directive (t, tt, k); + taken = true; + } + else + { + skip_line (t, tt); - taken = true; + if (tt == type::newline) + next (t, tt); + } + } + else + fail (t) << "expected " << k << "-block or 'recipe' instead of " + << t; } else { - skip_line (t, tt); + if (take) + { + if (!parse_clause (t, tt, true)) + fail (t) << "expected " << k << "-line instead of " << t; - if (tt == type::newline) - next (t, tt); + taken = true; + } + else + { + skip_line (t, tt); + + if (tt == type::newline) + next (t, tt); + } } } - else - fail (t) << "expected " << k << "-block instead of " << t; } if (tt != type::rcbrace) @@ -4110,10 +5460,10 @@ namespace build2 void parser:: parse_for (token& t, type& tt) { - // for <varname>: <value> + // for [<var-attrs>] <varname> [<elem-attrs>]: [<val-attrs>] <value> // <line> // - // for <varname>: <value> + // for [<var-attrs>] <varname> [<elem-attrs>]: [<val-attrs>] <value> // { // <block> // } @@ -4124,13 +5474,12 @@ namespace build2 next_with_attributes (t, tt); attributes_push (t, tt); - // @@ PAT: currently we pattern-expand for var. + // Enable list element attributes. // - const location vloc (get_location (t)); - names vns (parse_names (t, tt, pattern_mode::expand)); + enable_attributes (); - if (tt != type::colon) - fail (t) << "expected ':' instead of " << t << " after variable name"; + const location vloc (get_location (t)); + names vns (parse_names (t, tt, pattern_mode::preserve)); const variable& var (parse_variable_name (move (vns), vloc)); apply_variable_attributes (var); @@ -4141,6 +5490,17 @@ namespace build2 << " visibility but is assigned in for-loop"; } + // Parse the list element attributes, if present. + // + attributes_push (t, tt); + + if (tt != type::colon) + fail (t) << "expected ':' instead of " << t << " after variable name"; + + // Save element attributes so that we can inject them on each iteration. + // + attributes val_attrs (attributes_pop ()); + // Now the value (list of names) to iterate over. Parse it similar to a // value on the RHS of an assignment (expansion, attributes). // @@ -4149,15 +5509,24 @@ namespace build2 value val (parse_value_with_attributes (t, tt, pattern_mode::expand)); - // If this value is a vector, then save its element type so that we + // If the value type provides custom iterate function, then use that (see + // value_type::iterate for details). + // + auto iterate (val.type != nullptr ? val.type->iterate : nullptr); + + // If this value is a container, then save its element type so that we // can typify each element below. // const value_type* etype (nullptr); - if (val && val.type != nullptr) + if (!iterate && val && val.type != nullptr) { etype = val.type->element_type; - untypify (val); + + // Note that here we don't want to be reducing empty simple values to + // empty lists. + // + untypify (val, false /* reduce */); } if (tt != type::newline) @@ -4205,32 +5574,50 @@ namespace build2 // Iterate. // - value& v (scope_->assign (var)); // Assign even if no iterations. + value& lhs (scope_->assign (var)); // Assign even if no iterations. if (!val) return; - names& ns (val.as<names> ()); - - if (ns.empty ()) - return; + names* ns (nullptr); + if (!iterate) + { + ns = &val.as<names> (); + if (ns->empty ()) + return; + } istringstream is (move (body)); - for (auto i (ns.begin ()), e (ns.end ());; ) + struct data { - // Set the variable value. + const variable& var; + const attributes& val_attrs; + uint64_t line; + bool block; + value& lhs; + istringstream& is; + + } d {var, val_attrs, line, block, lhs, is}; + + function<void (value&&, bool first)> iteration = + [this, &d] (value&& v, bool first) + { + // Rewind the stream. + // + if (!first) + { + d.is.clear (); + d.is.seekg (0); + } + + // Inject element attributes. // - bool pair (i->pair); - names n; - n.push_back (move (*i)); - if (pair) n.push_back (move (*++i)); - v = value (move (n)); + attributes_.push_back (d.val_attrs); - if (etype != nullptr) - typify (v, *etype, &var); + apply_value_attributes (&d.var, d.lhs, move (v), type::assign); - lexer l (is, *path_, line); + lexer l (d.is, *path_, d.line); lexer* ol (lexer_); lexer_ = &l; @@ -4238,7 +5625,7 @@ namespace build2 type tt; next (t, tt); - if (block) + if (d.block) { next (t, tt); // { next (t, tt); // <newline> @@ -4246,20 +5633,33 @@ namespace build2 parse_clause (t, tt); - if (tt != (block ? type::rcbrace : type::eos)) - fail (t) << "expected name " << (block ? "or '}' " : "") + if (tt != (d.block ? type::rcbrace : type::eos)) + fail (t) << "expected name " << (d.block ? "or '}' " : "") << "instead of " << t; lexer_ = ol; + }; - if (++i == e) - break; + if (!iterate) + { + for (auto b (ns->begin ()), i (b), e (ns->end ()); i != e; ++i) + { + // Set the variable value. + // + bool pair (i->pair); + names n; + n.push_back (move (*i)); + if (pair) n.push_back (move (*++i)); + value v (move (n)); - // Rewind the stream. - // - is.clear (); - is.seekg (0); + if (etype != nullptr) + typify (v, *etype, &var); + + iteration (move (v), i == b); + } } + else + iterate (val, iteration); } void parser:: @@ -4332,7 +5732,7 @@ namespace build2 if (value v = parse_value_with_attributes (t, tt, pattern_mode::expand)) { names storage; - cout << reverse (v, storage) << endl; + cout << reverse (v, storage, true /* reduce */) << endl; } else cout << "[null]" << endl; @@ -4365,7 +5765,7 @@ namespace build2 if (value v = parse_value_with_attributes (t, tt, pattern_mode::expand)) { names storage; - dr << reverse (v, storage); + dr << reverse (v, storage, true /* reduce */); } if (tt != type::eos) @@ -4395,8 +5795,10 @@ namespace build2 if (ns.empty ()) { + // Indent two spaces. + // if (scope_ != nullptr) - dump (*scope_, " "); // Indent two spaces. + dump (scope_, nullopt /* action */, dump_format::buildfile, " "); else os << " <no current scope>" << endl; } @@ -4414,8 +5816,10 @@ namespace build2 const target* t (enter_target::find_target (*this, n, o, l, trace)); + // Indent two spaces. + // if (t != nullptr) - dump (*t, " "); // Indent two spaces. + dump (t, nullopt /* action */, dump_format::buildfile, " "); else { os << " <no target " << n; @@ -4437,10 +5841,12 @@ namespace build2 { // Enter a variable name for assignment (as opposed to lookup). + // If the variable is qualified (and thus public), make it overridable. + // // Note that the overridability can still be restricted (e.g., by a module // that enters this variable or by a pattern). // - bool ovr (true); + bool ovr (on.find ('.') != string::npos); auto r (scope_->var_pool ().insert (move (on), nullptr, nullptr, &ovr)); if (!r.second) @@ -4474,9 +5880,13 @@ namespace build2 { // Parse and enter a variable name for assignment (as opposed to lookup). - // The list should contain a single, simple name. + // The list should contain a single, simple name. Go an extra mile to + // issue less confusing diagnostics. // - if (ns.size () != 1 || ns[0].pattern || !ns[0].simple () || ns[0].empty ()) + size_t n (ns.size ()); + if (n == 0 || (n == 1 && ns[0].empty ())) + fail (l) << "empty variable name"; + else if (n != 1 || ns[0].pattern || !ns[0].simple ()) fail (l) << "expected variable name instead of " << ns; return parse_variable_name (move (ns[0].value), l); @@ -4532,7 +5942,7 @@ namespace build2 // Note that the pattern is preserved if insert fails with regex_error. // p = scope_->target_vars[ptt].insert (pt, move (pat)).insert ( - var, kind == type::assign); + var, kind == type::assign, false /* reset_extra */); } catch (const regex_error& e) { @@ -4546,7 +5956,12 @@ namespace build2 // We store prepend/append values untyped (similar to overrides). // if (rhs.type != nullptr && kind != type::assign) - untypify (rhs); + { + // Our heuristics for prepend/append of a typed value is to preserve + // empty (see apply_value_attributes() for details) so do not reduce. + // + untypify (rhs, false /* reduce */); + } if (p.second) { @@ -4633,32 +6048,119 @@ namespace build2 : value (names ()); } - static const value_type* - map_type (const string& n) + const value_type* parser:: + find_value_type (const scope*, const string& n) { - auto ptr = [] (const value_type& vt) {return &vt;}; - - return - n == "bool" ? ptr (value_traits<bool>::value_type) : - n == "int64" ? ptr (value_traits<int64_t>::value_type) : - n == "uint64" ? ptr (value_traits<uint64_t>::value_type) : - n == "string" ? ptr (value_traits<string>::value_type) : - n == "path" ? ptr (value_traits<path>::value_type) : - n == "dir_path" ? ptr (value_traits<dir_path>::value_type) : - n == "abs_dir_path" ? ptr (value_traits<abs_dir_path>::value_type) : - n == "name" ? ptr (value_traits<name>::value_type) : - n == "name_pair" ? ptr (value_traits<name_pair>::value_type) : - n == "target_triplet" ? ptr (value_traits<target_triplet>::value_type) : - n == "project_name" ? ptr (value_traits<project_name>::value_type) : - - n == "int64s" ? ptr (value_traits<int64s>::value_type) : - n == "uint64s" ? ptr (value_traits<uint64s>::value_type) : - n == "strings" ? ptr (value_traits<strings>::value_type) : - n == "paths" ? ptr (value_traits<paths>::value_type) : - n == "dir_paths" ? ptr (value_traits<dir_paths>::value_type) : - n == "names" ? ptr (value_traits<vector<name>>::value_type) : - - nullptr; + switch (n[0]) + { + case 'a': + { + if (n == "abs_dir_path") return &value_traits<abs_dir_path>::value_type; + break; + } + case 'b': + { + if (n == "bool") return &value_traits<bool>::value_type; + break; + } + case 'c': + { + if (n == "cmdline") return &value_traits<cmdline>::value_type; + break; + } + case 'd': + { + if (n.compare (0, 8, "dir_path") == 0) + { + if (n[8] == '\0') return &value_traits<dir_path>::value_type; + if (n[8] == 's' && + n[9] == '\0') return &value_traits<dir_paths>::value_type; + } + break; + } + case 'i': + { + if (n.compare (0, 5, "int64") == 0) + { + if (n[5] == '\0') return &value_traits<int64_t>::value_type; + if (n[5] == 's' && + n[6] == '\0') return &value_traits<int64s>::value_type; + } + break; + } + case 'j': + { + if (n.compare (0, 4, "json") == 0) + { + if (n[4] == '\0') return &value_traits<json_value>::value_type; + if (n == "json_array") return &value_traits<json_array>::value_type; + if (n == "json_object") + return &value_traits<json_object>::value_type; + if (n == "json_set") + return &value_traits<set<json_value>>::value_type; + if (n == "json_map") + return &value_traits<map<json_value, json_value>>::value_type; + } + break; + } + case 'n': + { + if (n.compare (0, 4, "name") == 0) + { + if (n[4] == '\0') return &value_traits<name>::value_type; + if (n[4] == 's' && + n[5] == '\0') return &value_traits<vector<name>>::value_type; + if (n == "name_pair") return &value_traits<name_pair>::value_type; + } + break; + } + + case 'p': + { + if (n.compare (0, 4, "path") == 0) + { + if (n[4] == '\0') return &value_traits<path>::value_type; + if (n[4] == 's' && + n[5] == '\0') return &value_traits<paths>::value_type; + } + else if (n == "project_name") + return &value_traits<project_name>::value_type; + break; + } + case 's': + { + if (n.compare (0, 6, "string") == 0) + { + if (n[6] == '\0') return &value_traits<string>::value_type; + if (n[6] == 's' && + n[7] == '\0') return &value_traits<strings>::value_type; + if (n == "string_set") return &value_traits<set<string>>::value_type; + if (n == "string_map") + return &value_traits<map<string,string>>::value_type; + } + break; + } + case 't': + { + if (n == "target_triplet") + return &value_traits<target_triplet>::value_type; + break; + } + case 'u': + { + if (n.compare (0, 6, "uint64") == 0) + { + if (n[6] == '\0') return &value_traits<uint64_t>::value_type; + if (n[6] == 's' && + n[7] == '\0') return &value_traits<uint64s>::value_type; + } + break; + } + default: + break; + } + + return nullptr; } void parser:: @@ -4680,19 +6182,62 @@ namespace build2 string& n (a.name); value& v (a.value); - if (const value_type* t = map_type (n)) + if (n == "visibility") + { + try + { + string s (convert<string> (move (v))); + + variable_visibility r; + if (s == "global") r = variable_visibility::global; + else if (s == "project") r = variable_visibility::project; + else if (s == "scope") r = variable_visibility::scope; + else if (s == "target") r = variable_visibility::target; + else if (s == "prerequisite") r = variable_visibility::prereq; + else throw invalid_argument ("unknown visibility name"); + + if (vis && r != *vis) + fail (l) << "conflicting variable visibilities: " << s << ", " + << *vis; + + vis = r; + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else if (n == "overridable") + { + try + { + // Treat absent value (represented as NULL) as true. + // + bool r (v.null || convert<bool> (move (v))); + + if (ovr && r != *ovr) + fail (l) << "conflicting variable overridabilities"; + + ovr = r; + } + catch (const invalid_argument& e) + { + fail (l) << "invalid " << n << " attribute value: " << e; + } + } + else if (const value_type* t = find_value_type (root_, n)) { + if (!v.null) + fail (l) << "unexpected value in attribute " << a; + if (type != nullptr && t != type) - fail (l) << "multiple variable types: " << n << ", " << type->name; + fail (l) << "conflicting variable types: " << n << ", " + << type->name; type = t; - // Fall through. } else fail (l) << "unknown variable attribute " << a; - - if (!v.null) - fail (l) << "unexpected value in attribute " << a; } if (type != nullptr && var.type != nullptr) @@ -4704,15 +6249,33 @@ namespace build2 << var.type->name << " to " << type->name; } - //@@ TODO: the same checks for vis and ovr (when we have the corresponding - // attributes). + if (vis) + { + // Note that this logic naturally makes sure that a project-private + // variable doesn't have global visibility (since it would have been + // entered with the project visibility). + // + if (var.visibility == *vis) + vis = nullopt; + else if (var.visibility > *vis) // See variable_pool::update(). + fail (l) << "changing variable " << var << " visibility from " + << var.visibility << " to " << *vis; + } + + if (ovr) + { + // Note that the overridability incompatibilities are diagnosed by + // update(). So we just need to diagnose the project-private case. + // + if (*ovr && var.owner != &ctx->var_pool) + fail (l) << "private variable " << var << " cannot be overridable"; + } if (type || vis || ovr) - ctx.var_pool.update (const_cast<variable&> (var), - type, - vis ? &*vis : nullptr, - ovr ? &*ovr : nullptr); - + var.owner->update (const_cast<variable&> (var), + type, + vis ? &*vis : nullptr, + ovr ? &*ovr : nullptr); } void parser:: @@ -4722,7 +6285,7 @@ namespace build2 type kind) { attributes as (attributes_pop ()); - const location& l (as.loc); + const location& l (as.loc); // This points to value if no attributes. // Essentially this is an attribute-augmented assign/append/prepend. // @@ -4736,16 +6299,18 @@ namespace build2 if (n == "null") { + // @@ Looks like here we assume representationally empty? + // if (rhs && !rhs.empty ()) // Note: null means we had an expansion. fail (l) << "value with null attribute"; null = true; // Fall through. } - else if (const value_type* t = map_type (n)) + else if (const value_type* t = find_value_type (root_, n)) { if (type != nullptr && t != type) - fail (l) << "multiple value types: " << n << ", " << type->name; + fail (l) << "conflicting value types: " << n << ", " << type->name; type = t; // Fall through. @@ -4793,6 +6358,13 @@ namespace build2 bool rhs_type (false); if (rhs.type != nullptr) { + // Our heuristics is to not reduce typed RHS empty simple values for + // prepend/append and additionally for assign provided LHS is a + // container. + // + bool reduce (kind == type::assign && + (type == nullptr || !type->container)); + // Only consider RHS type if there is no explicit or variable type. // if (type == nullptr) @@ -4803,7 +6375,7 @@ namespace build2 // Reduce this to the untyped value case for simplicity. // - untypify (rhs); + untypify (rhs, reduce); } if (kind == type::assign) @@ -4832,6 +6404,17 @@ namespace build2 } else { + auto df = make_diag_frame ( + [this, var, &l](const diag_record& dr) + { + if (!l.empty ()) + { + dr << info (l); + if (var != nullptr) dr << "variable " << var->name << ' '; + dr << "value is assigned here"; + } + }); + if (kind == type::assign) { if (rhs) @@ -5186,17 +6769,38 @@ namespace build2 if (pre_parse_) return v; // Empty. - if (v.type != nullptr || !v || v.as<names> ().size () != 1) - fail (l) << "expected target before ':'"; - + // We used to return this as a <target>:<name> pair but that meant we + // could not handle an out-qualified target (which is represented as + // <target>@<out> pair). As a somewhat of a hack, we deal with this by + // changing the order of the name and target to be <name>:<target> with + // the qualified case becoming a "tripple pair" <name>:<target>@<out>. + // + // @@ This is actually not great since it's possible to observe such a + // tripple pair, for example with `print (file{x}@./:y)`. + // if (n.type != nullptr || !n || n.as<names> ().size () != 1 || n.as<names> ()[0].pattern) fail (nl) << "expected variable name after ':'"; - names& ns (v.as<names> ()); + names& ns (n.as<names> ()); ns.back ().pair = ':'; - ns.push_back (move (n.as<names> ().back ())); - return v; + + if (v.type == nullptr && v) + { + names& ts (v.as<names> ()); + + size_t s (ts.size ()); + if (s == 1 || (s == 2 && ts.front ().pair == '@')) + { + ns.push_back (move (ts.front ())); + if (s == 2) + ns.push_back (move (ts.back ())); + + return n; + } + } + + fail (l) << "expected target before ':'" << endf; } else { @@ -5265,8 +6869,13 @@ namespace build2 } pair<bool, location> parser:: - attributes_push (token& t, type& tt, bool standalone) + attributes_push (token& t, type& tt, bool standalone, bool next_token) { + // To make sure that the attributes are not standalone we need to read the + // token which follows ']'. + // + assert (standalone || next_token); + location l (get_location (t)); bool has (tt == type::lsbrace); @@ -5340,28 +6949,27 @@ namespace build2 if (tt != type::rsbrace) fail (t) << "expected ']' instead of " << t; - next (t, tt); - - if (tt == type::newline || tt == type::eos) + if (next_token) { - if (!standalone) - fail (t) << "standalone attributes"; + next (t, tt); + + if (tt == type::newline || tt == type::eos) + { + if (!standalone) + fail (t) << "standalone attributes"; + } + // + // Verify that the attributes are separated from the following word or + // "word-producing" token. + // + else if (!t.separated && (tt == type::word || + tt == type::dollar || + tt == type::lparen || + tt == type::lcbrace)) + fail (t) << "whitespace required after attributes" << + info (l) << "use the '\\[' escape sequence if this is a wildcard " + << "pattern"; } - // - // We require attributes to be separated from the following word or - // "word-producing" tokens (`$` for variable expansions/function calls, - // `(` for eval contexts, and `{` for name generation) to reduce the - // possibility of confusing them with wildcard patterns. Consider: - // - // ./: [abc]-foo.txt - // - else if (!t.separated && (tt == type::word || - tt == type::dollar || - tt == type::lparen || - tt == type::lcbrace)) - fail (t) << "whitespace required after attributes" << - info (l) << "use the '\\[' escape sequence if this is a wildcard " - << "pattern"; return make_pair (has, l); } @@ -5596,9 +7204,11 @@ namespace build2 // May throw invalid_path. // auto include_pattern = - [&r, &append, &include_match, sp, &l, this] (string&& p, - optional<string>&& e, - bool a) + [this, + &append, &include_match, + &r, sp, &l, &dir] (string&& p, + optional<string>&& e, + bool a) { // If we don't already have any matches and our pattern doesn't contain // multiple recursive wildcards, then the result will be unique and we @@ -5645,14 +7255,62 @@ namespace build2 // multiple entries for each pattern. // if (!interm) - d.appf (move (m).representation (), optional<string> (d.e)); + { + // If the extension is empty (meaning there should be no extension, + // for example hxx{Q*.}), skip entries with extensions. + // + if (!d.e || !d.e->empty () || m.extension_cstring () == nullptr) + d.appf (move (m).representation (), optional<string> (d.e)); + } return true; }; + const function<bool (const dir_entry&)> dangling ( + [&dir] (const dir_entry& de) + { + bool sl (de.ltype () == entry_type::symlink); + + const path& n (de.path ()); + + // One case where this turned out to be not worth it practically + // (too much noise) is the backlinks to executables (and the + // associated DLL assemblies for Windows). So we now have this + // heuristics that if this looks like an executable (or DLL for + // Windows), then we omit the warning. On POSIX, where executables + // don't have extensions, we will consider it an executable only if + // we are not looking for directories (which also normally don't + // have extension). + // + // @@ PEDANTIC: re-enable if --pedantic. + // + if (sl) + { + string e (n.extension ()); + + if ((e.empty () && !dir) || + path_traits::compare (e, "exe") == 0 || + path_traits::compare (e, "dll") == 0 || + path_traits::compare (e, "pdb") == 0 || // .{exe,dll}.pdb + (path_traits::compare (e, "dlls") == 0 && // .exe.dlls assembly + path_traits::compare (n.base ().extension (), "exe") == 0)) + return true; + } + + warn << "skipping " + << (sl ? "dangling symlink" : "inaccessible entry") + << ' ' << de.base () / n; + + return true; + }); + try { - path_search (path (move (p)), process, *sp); + path_search (path (move (p)), + process, + *sp, + path_match_flags::follow_symlinks, + dangling); } catch (const system_error& e) { @@ -5820,6 +7478,7 @@ namespace build2 if ((n.pair & 0x02) != 0) { e = move (n.type); + n.type.clear (); // Remove non-empty extension from the name (it got to be there, see // above). @@ -6097,9 +7756,35 @@ namespace build2 bool concat_quoted_first (false); name concat_data; - auto concat_typed = [this, &vnull, &vtype, - &concat, &concat_data] (value&& rhs, - const location& loc) + auto concat_diag_multiple = [this] (const location& loc, + const char* what_expansion) + { + diag_record dr (fail (loc)); + + dr << "concatenating " << what_expansion << " contains multiple values"; + + // See if this looks like a subscript without an evaluation context and + // help the user out. + // + if (mode () != lexer_mode::eval) + { + const token& t (peeked ()); // Should be peeked at. + + if (t.type == type::word && + t.qtype == quote_type::unquoted && + t.value[0] == '[') + { + dr << info << "wrap it in (...) evaluation context if this " + << "is value subscript"; + } + } + }; + + auto concat_typed = [this, what, &vnull, &vtype, + &concat, &concat_data, + &concat_diag_multiple] (value&& rhs, + const location& loc, + const char* what_expansion) { // If we have no LHS yet, then simply copy value/type. // @@ -6116,6 +7801,10 @@ namespace build2 // RHS. // + // Note that if RHS contains multiple values then we expect the result + // to be a single value somehow or, more likely, there to be no + // suitable $builtin.concat() overload. + // a.push_back (move (rhs)); const char* l ((a[0].type != nullptr ? a[0].type->name : "<untyped>")); @@ -6132,7 +7821,10 @@ namespace build2 dr << info << "use quoting to force untyped concatenation"; }); - p = ctx.functions.try_call ( + if (ctx == nullptr) + fail << "literal " << what << " expected"; + + p = ctx->functions.try_call ( scope_, "builtin.concat", vector_view<value> (a), loc); } @@ -6154,18 +7846,22 @@ namespace build2 if (!vnull) { if (vtype != nullptr) - untypify (rhs); + untypify (rhs, true /* reduce */); names& d (rhs.as<names> ()); - // If the value is empty, then untypify() will (typically; no pun - // intended) represent it as an empty sequence of names rather than - // a sequence of one empty name. This is usually what we need (see - // simple_reverse() for details) but not in this case. + // If the value is empty, then we asked untypify() to reduce it to + // an empty sequence of names rather than a sequence of one empty + // name. // - if (!d.empty ()) + if (size_t n = d.size ()) { - assert (d.size () == 1); // Must be a single value. + if (n != 1) + { + assert (what_expansion != nullptr); + concat_diag_multiple (loc, what_expansion); + } + concat_data = move (d[0]); } } @@ -6272,6 +7968,8 @@ namespace build2 // continue accumulating or inject. We inject if the next token is not a // word, var expansion, or eval context or if it is separated. // + optional<pair<const value_type*, name>> path_concat; // Backup. + if (concat && last_concat ()) { // Concatenation does not affect the tokens we get, only what we do @@ -6311,6 +8009,13 @@ namespace build2 // dir/{$str} // file{$str} // + // And yet another exception: if the type is path or dir_path and the + // pattern mode is not ignore, then we will inject to try our luck in + // interpreting the concatenation result as a path pattern. This makes + // sure patterns like `$src_base/*.txt` work, naturally. Failed that, + // we will handle this concatenation as we do for other types (via the + // path_concat backup). + // // A concatenation cannot produce value/NULL. // @@ -6322,12 +8027,14 @@ namespace build2 bool e1 (tt == type::lcbrace && !peeked ().separated); bool e2 (pp || dp != nullptr || tp != nullptr); + const value_type* pt (&value_traits<path>::value_type); + const value_type* dt (&value_traits<dir_path>::value_type); + if (e1 || e2) { - if (vtype == &value_traits<path>::value_type || - vtype == &value_traits<string>::value_type) + if (vtype == pt || vtype == &value_traits<string>::value_type) ; // Representation is already in concat_data.value. - else if (vtype == &value_traits<dir_path>::value_type) + else if (vtype == dt) concat_data.value = move (concat_data.dir).representation (); else { @@ -6342,6 +8049,20 @@ namespace build2 vtype = nullptr; // Fall through to injection. } + else if (pmode != pattern_mode::ignore && + (vtype == pt || vtype == dt)) + { + path_concat = make_pair (vtype, concat_data); + + // Note: for path the representation is already in + // concat_data.value. + // + if (vtype == dt) + concat_data.value = move (concat_data.dir).representation (); + + vtype = nullptr; + // Fall through to injection. + } else { // This is either a simple name (untyped concatenation; in which @@ -6435,7 +8156,7 @@ namespace build2 // names ns; ns.push_back (name (move (val))); - concat_typed (value (move (ns)), get_location (t)); + concat_typed (value (move (ns)), get_location (t), nullptr); } else { @@ -6537,6 +8258,8 @@ namespace build2 if (ttp == nullptr) ppat = pinc = false; + else if (ttp->factory == nullptr) + fail (loc) << "abstract target type " << ttp->name << "{}"; } } @@ -6602,7 +8325,7 @@ namespace build2 // See if this is a pattern, path or regex. // // A path pattern either contains an unquoted wildcard character or, - // in the curly context, start with unquoted/unescaped `+`. + // in the curly context, starts with unquoted/unescaped `+`. // // A regex pattern starts with unquoted/unescaped `~` followed by a // non-alphanumeric delimiter and has the following form: @@ -6680,7 +8403,7 @@ namespace build2 // Note that we have to check for regex patterns first since // they may also be detected as path patterns. // - if (!quoted_first && regex_pattern ()) + if (!quoted_first && !path_concat && regex_pattern ()) { // Note: we may decide to support regex-based name generation // some day (though a substitution won't make sense here). @@ -6698,6 +8421,9 @@ namespace build2 ? scope_->find_target_type (*tp) : nullptr); + if (ttp != nullptr && ttp->factory == nullptr) + fail (loc) << "abstract target type " << ttp->name << "{}"; + if (tp == nullptr || ttp != nullptr) { if (pmode == pattern_mode::detect) @@ -6748,7 +8474,7 @@ namespace build2 // there isn't any good reason to; see also to_stream(name) for // the corresponding serialization logic). // - if (!quoted_first && regex_pattern ()) + if (!quoted_first && !path_concat && regex_pattern ()) { const char* w; if (val[0] == '~') @@ -6806,6 +8532,24 @@ namespace build2 } } + // If this is a concatenation of the path or dir_path type and it is + // not a pattern, then handle it in the same way as concatenations of + // other types (see above). + // + if (path_concat && !pat) + { + ns.push_back (move (path_concat->second)); + + // Restore the type information if that's the only name. + // + if (start == ns.size () && last_token ()) + vtype = path_concat->first; + + // Restart the loop. + // + continue; + } + // If we are a second half of a pair, add another first half // unless this is the first instance. // @@ -6860,6 +8604,9 @@ namespace build2 // if (tt == type::dollar || tt == type::lparen) { + if (ctx == nullptr) + fail << "literal " << what << " expected"; + // These cases are pretty similar in that in both we quickly end up // with a list of names that we need to splice into the result. // @@ -6881,11 +8628,15 @@ namespace build2 // token is a paren or a word, we turn it on and switch to the eval // mode if what we get next is a paren. // - // Also sniff out the special variables string from mode data for - // the ad hoc $() handling below. - // mode (lexer_mode::variable); + // Sniff out the special variables string from mode data and use + // that to recognize special variables in the ad hoc $() handling + // below. + // + // Note: must be done before calling next() which may expire the + // mode. + // auto special = [s = reinterpret_cast<const char*> (mode_data ())] (const token& t) -> char { @@ -6924,156 +8675,202 @@ namespace build2 next (t, tt); loc = get_location (t); - name qual; - string name; - - if (t.separated) - ; // Leave the name empty to fail below. - else if (tt == type::word) + if (tt == type::escape) { - name = move (t.value); + // For now we only support all the simple C/C++ escape sequences + // plus \0 (which in C/C++ is an octal escape sequence). See the + // lexer part for details. + // + // Note: cannot be subscripted. + // + if (!pre_parse_) + { + string s; + switch (char c = t.value[0]) + { + case '\'': + case '"': + case '?': + case '\\': s = c; break; + case '0': s = '\0'; break; + case 'a': s = '\a'; break; + case 'b': s = '\b'; break; + case 'f': s = '\f'; break; + case 'n': s = '\n'; break; + case 'r': s = '\r'; break; + case 't': s = '\t'; break; + case 'v': s = '\v'; break; + default: + assert (false); + } + + result_data = name (move (s)); + what = "escape sequence expansion"; + } + + tt = peek (); } - else if (tt == type::lparen) + else { - expire_mode (); - mode (lexer_mode::eval, '@'); - next_with_attributes (t, tt); + names qual; + string name; - // Handle the $(x) case ad hoc. We do it this way in order to get - // the variable name even during pre-parse. It should also be - // faster. - // - char c; - if ((tt == type::word - ? path_traits::rfind_separator (t.value) == string::npos - : (c = special (t))) && - peek () == type::rparen) + if (t.separated) + ; // Leave the name empty to fail below. + else if (tt == type::word) { - name = (tt == type::word ? move (t.value) : string (1, c)); - next (t, tt); // Get `)`. + name = move (t.value); } - else + else if (tt == type::lparen) { - using name_type = build2::name; + expire_mode (); + mode (lexer_mode::eval, '@'); + next_with_attributes (t, tt); - //@@ OUT will parse @-pair and do well? + // Handle the $(x) case ad hoc. We do it this way in order to + // get the variable name even during pre-parse. It should also + // be faster. // - values vs (parse_eval (t, tt, pmode)); - - if (!pre_parse_) + char c ('\0'); + if ((tt == type::word + ? path_traits::rfind_separator (t.value) == string::npos + : (c = special (t))) && + peek () == type::rparen) + { + name = (tt == type::word ? move (t.value) : string (1, c)); + next (t, tt); // Get `)`. + } + else { - if (vs.size () != 1) - fail (loc) << "expected single variable/function name"; + using name_type = build2::name; - value& v (vs[0]); + values vs (parse_eval (t, tt, pmode)); - if (!v) - fail (loc) << "null variable/function name"; + if (!pre_parse_) + { + if (vs.size () != 1) + fail (loc) << "expected single variable/function name"; - names storage; - vector_view<name_type> ns (reverse (v, storage)); // Movable. - size_t n (ns.size ()); + value& v (vs[0]); - // We cannot handle scope-qualification in the eval context as - // we do for target-qualification (see eval-qual) since then - // we would be treating all paths as qualified variables. So - // we have to do it here. - // - if (n == 2 && ns[0].pair == ':') // $(foo: x) - { - qual = move (ns[0]); + if (!v) + fail (loc) << "null variable/function name"; - if (qual.empty ()) - fail (loc) << "empty variable/function qualification"; - } - else if (n == 2 && ns[0].directory ()) // $(foo/ x) - { - qual = move (ns[0]); - qual.pair = '/'; - } - else if (n > 1) - fail (loc) << "expected variable/function name instead of '" - << ns << "'"; + names storage; + vector_view<name_type> ns ( + reverse (v, storage, true /* reduce */)); // Movable. + size_t n (ns.size ()); - // Note: checked for empty below. - // - if (!ns[n - 1].simple ()) - fail (loc) << "expected variable/function name instead of '" - << ns[n - 1] << "'"; + // We cannot handle scope-qualification in the eval context + // as we do for target-qualification (see eval-qual) since + // then we would be treating all paths as qualified + // variables. So we have to do it here. + // + if (n >= 2 && ns[0].pair == ':') // $(foo: x) + { + // Note: name is first (see eval for details). + // + qual.push_back (move (ns[1])); - size_t p; - if (n == 1 && // $(foo/x) - (p = path_traits::rfind_separator (ns[0].value)) != - string::npos) - { - // Note that p cannot point to the last character since then - // it would have been a directory, not a simple name. + if (qual.back ().empty ()) + fail (loc) << "empty variable/function qualification"; + + if (n > 2) + qual.push_back (move (ns[2])); + + // Move name to the last position (see below). + // + swap (ns[0], ns[n - 1]); + } + else if (n == 2 && ns[0].directory ()) // $(foo/ x) + { + qual.push_back (move (ns[0])); + qual.back ().pair = '/'; + } + else if (n > 1) + fail (loc) << "expected variable/function name instead of '" + << ns << "'"; + + // Note: checked for empty below. // - string& s (ns[0].value); + if (!ns[n - 1].simple ()) + fail (loc) << "expected variable/function name instead of '" + << ns[n - 1] << "'"; + + size_t p; + if (n == 1 && // $(foo/x) + (p = path_traits::rfind_separator (ns[0].value)) != + string::npos) + { + // Note that p cannot point to the last character since + // then it would have been a directory, not a simple name. + // + string& s (ns[0].value); - name = string (s, p + 1); - s.resize (p + 1); - qual = name_type (dir_path (move (s))); - qual.pair = '/'; + name = string (s, p + 1); + s.resize (p + 1); + qual.push_back (name_type (dir_path (move (s)))); + qual.back ().pair = '/'; + } + else + name = move (ns[n - 1].value); } - else - name = move (ns[n - 1].value); } } - } - else - fail (t) << "expected variable/function name instead of " << t; - - if (!pre_parse_ && name.empty ()) - fail (loc) << "empty variable/function name"; - - // Figure out whether this is a variable expansion with potential - // subscript or a function call. - // - if (sub) enable_subscript (); - tt = peek (); + else + fail (t) << "expected variable/function name instead of " << t; - // Note that we require function call opening paren to be - // unseparated; consider: $x ($x == 'foo' ? 'FOO' : 'BAR'). - // - if (tt == type::lparen && !peeked ().separated) - { - // Function call. - // - next (t, tt); // Get '('. - mode (lexer_mode::eval, '@'); - next_with_attributes (t, tt); + if (!pre_parse_ && name.empty ()) + fail (loc) << "empty variable/function name"; - // @@ Should we use (target/scope) qualification (of name) as the - // context in which to call the function? Hm, interesting... + // Figure out whether this is a variable expansion with potential + // subscript or a function call. // - values args (parse_eval (t, tt, pmode)); - if (sub) enable_subscript (); tt = peek (); - // Note that we "move" args to call(). + // Note that we require function call opening paren to be + // unseparated; consider: $x ($x == 'foo' ? 'FOO' : 'BAR'). // - if (!pre_parse_) + if (tt == type::lparen && !peeked ().separated) { - result_data = ctx.functions.call (scope_, name, args, loc); - what = "function call"; + // Function call. + // + next (t, tt); // Get '('. + mode (lexer_mode::eval, '@'); + next_with_attributes (t, tt); + + // @@ Should we use (target/scope) qualification (of name) as + // the context in which to call the function? Hm, interesting... + // + values args (parse_eval (t, tt, pmode)); + + if (sub) enable_subscript (); + tt = peek (); + + // Note that we "move" args to call(). + // + if (!pre_parse_) + { + result_data = ctx->functions.call (scope_, name, args, loc); + what = "function call"; + } + else + lookup_function (move (name), loc); } else - lookup_function (move (name), loc); - } - else - { - // Variable expansion. - // - lookup l (lookup_variable (move (qual), move (name), loc)); - - if (!pre_parse_) { - if (l.defined ()) - result = l.value; // Otherwise leave as NULL result_data. + // Variable expansion. + // + lookup l (lookup_variable (move (qual), move (name), loc)); + + if (!pre_parse_) + { + if (l.defined ()) + result = l.value; // Otherwise leave as NULL result_data. - what = "variable expansion"; + what = "variable expansion"; + } } } } @@ -7105,85 +8902,132 @@ namespace build2 // Handle value subscript. // - if (tt == type::lsbrace) + if (mode () == lexer_mode::eval) // Note: not if(sub)! { - location bl (get_location (t)); - next (t, tt); // `[` - mode (lexer_mode::subscript, '\0' /* pair */); - next (t, tt); - - location l (get_location (t)); - value v ( - tt != type::rsbrace - ? parse_value (t, tt, pattern_mode::ignore, "value subscript") - : value (names ())); - - if (tt != type::rsbrace) + while (tt == type::lsbrace) { - // Note: wildcard pattern should have `]` as well so no escaping - // suggestion. - // - fail (t) << "expected ']' instead of " << t; - } + location bl (get_location (t)); + next (t, tt); // `[` + mode (lexer_mode::subscript, '\0' /* pair */); + next (t, tt); - if (!pre_parse_) - { - uint64_t j; - try - { - j = convert<uint64_t> (move (v)); - } - catch (const invalid_argument& e) + location l (get_location (t)); + value v ( + tt != type::rsbrace + ? parse_value (t, tt, pattern_mode::ignore, "value subscript") + : value (names ())); + + if (tt != type::rsbrace) { - fail (l) << "invalid value subscript: " << e << - info (bl) << "use the '\\[' escape sequence if this is a " - << "wildcard pattern" << endf; + // Note: wildcard pattern should have `]` as well so no escaping + // suggestion. + // + fail (t) << "expected ']' instead of " << t; } - // Similar to expanding an undefined variable, we return NULL if - // the index is out of bounds. - // - // Note that result may or may not point to result_data. - // - if (result->null) - result_data = value (); - else if (result->type == nullptr) + if (!pre_parse_) { - const names& ns (result->as<names> ()); - - // Pair-aware subscript. + // For type-specific subscript implementations we pass the + // subscript value as is. // - names r; - for (auto i (ns.begin ()); i != ns.end (); ++i, --j) + if (auto f = (result->type != nullptr + ? result->type->subscript + : nullptr)) { - if (j == 0) + result_data = f (*result, &result_data, move (v), l, bl); + } + else + { + uint64_t j; + try { - r.push_back (*i); - if (i->pair) - r.push_back (*++i); - break; + j = convert<uint64_t> (move (v)); } + catch (const invalid_argument& e) + { + fail (l) << "invalid value subscript: " << e << + info (bl) << "use the '\\[' escape sequence if this is a " + << "wildcard pattern" << endf; + } + + // Similar to expanding an undefined variable, we return NULL + // if the index is out of bounds. + // + // Note that result may or may not point to result_data. + // + if (result->null) + result_data = value (); + else if (result->type == nullptr) + { + const names& ns (result->as<names> ()); + + // Pair-aware subscript. + // + names r; + for (auto i (ns.begin ()); i != ns.end (); ++i, --j) + { + if (j == 0) + { + r.push_back (*i); + if (i->pair) + r.push_back (*++i); + break; + } - if (i->pair) - ++i; + if (i->pair) + ++i; + } + + result_data = r.empty () ? value () : value (move (r)); + } + else + { + // Similar logic to parse_for(). + // + const value_type* etype (result->type->element_type); + + value val (result == &result_data + ? value (move (result_data)) + : value (*result)); + + untypify (val, false /* reduce */); + + names& ns (val.as<names> ()); + + // Pair-aware subscript. + // + names r; + for (auto i (ns.begin ()); i != ns.end (); ++i, --j) + { + bool p (i->pair); + + if (j == 0) + { + r.push_back (move (*i)); + if (p) + r.push_back (move (*++i)); + break; + } + + if (p) + ++i; + } + + result_data = r.empty () ? value () : value (move (r)); + + if (etype != nullptr) + typify (result_data, *etype, nullptr /* var */); + } } - result_data = r.empty () ? value () : value (move (r)); - } - else - { - // @@ TODO: we would want to return a value with element type. - // - //result_data = ... - fail (l) << "typed value subscript not yet supported" << - info (bl) << "use the '\\[' escape sequence if this is a " - << "wildcard pattern"; + result = &result_data; } - result = &result_data; + // See if we have chained subscript. + // + enable_subscript (); + tt = peek (); } - - tt = peek (); } if (pre_parse_) @@ -7227,7 +9071,8 @@ namespace build2 // then it should not be overloaded for a type). In a quoted // context we use $string() which returns a "canonical // representation" (e.g., a directory path without a trailing - // slash). + // slash). Note: looks like we use typed $concat() now in the + // unquoted context. // if (result->type != nullptr && quoted) { @@ -7250,7 +9095,10 @@ namespace build2 dr << info (loc) << "while converting " << t << " to string"; }); - p = ctx.functions.try_call ( + if (ctx == nullptr) + fail << "literal " << what << " expected"; + + p = ctx->functions.try_call ( scope_, "string", vector_view<value> (&result_data, 1), loc); } @@ -7258,7 +9106,11 @@ namespace build2 fail (loc) << "no string conversion for " << t; result_data = move (p.first); - untypify (result_data); // Convert to untyped simple name. + + // Convert to untyped simple name reducing empty string to empty + // names as an optimization. + // + untypify (result_data, true /* reduce */); } if ((concat && vtype != nullptr) || // LHS typed. @@ -7267,13 +9119,13 @@ namespace build2 if (result != &result_data) // Same reason as above. result = &(result_data = *result); - concat_typed (move (result_data), loc); + concat_typed (move (result_data), loc, what); } // // Untyped concatenation. Note that if RHS is NULL/empty, we still // set the concat flag. // - else if (!result->null && !result->empty ()) + else if (!result->null) { // This can only be an untyped value. // @@ -7281,34 +9133,36 @@ namespace build2 // const names& lv (cast<names> (*result)); - // This should be a simple value or a simple directory. - // - if (lv.size () > 1) - fail (loc) << "concatenating " << what << " contains multiple " - << "values"; + if (size_t s = lv.size ()) + { + // This should be a simple value or a simple directory. + // + if (s > 1) + concat_diag_multiple (loc, what); - const name& n (lv[0]); + const name& n (lv[0]); - if (n.qualified ()) - fail (loc) << "concatenating " << what << " contains project " - << "name"; + if (n.qualified ()) + fail (loc) << "concatenating " << what << " contains project " + << "name"; - if (n.typed ()) - fail (loc) << "concatenating " << what << " contains type"; + if (n.typed ()) + fail (loc) << "concatenating " << what << " contains target type"; - if (!n.dir.empty ()) - { - if (!n.value.empty ()) - fail (loc) << "concatenating " << what << " contains " - << "directory"; + if (!n.dir.empty ()) + { + if (!n.value.empty ()) + fail (loc) << "concatenating " << what << " contains " + << "directory"; - // Note that here we cannot assume what's in dir is really a - // path (think s/foo/bar/) so we have to reverse it exactly. - // - concat_data.value += n.dir.representation (); + // Note that here we cannot assume what's in dir is really a + // path (think s/foo/bar/) so we have to reverse it exactly. + // + concat_data.value += n.dir.representation (); + } + else + concat_data.value += n.value; } - else - concat_data.value += n.value; } // The same little hack as in the word case ($empty+foo). @@ -7334,16 +9188,27 @@ namespace build2 // Nothing else to do here if the result is NULL or empty. // - if (result->null || result->empty ()) - continue; - - // @@ Could move if nv is result_data; see untypify(). + // Note that we cannot use value::empty() here since we are + // interested in representationally empty. // - names nv_storage; - names_view nv (reverse (*result, nv_storage)); + if (!result->null) + { + // @@ Could move if nv is result_data; see untypify(). + // + // Nuance: we should only be reducing empty simple value to empty + // list if we are not a second half of a pair. + // + bool pair (!ns.empty () && ns.back ().pair); + + names nv_storage; + names_view nv (reverse (*result, nv_storage, !pair /* reduce */)); - count = splice_names ( - loc, nv, move (nv_storage), ns, what, pairn, pp, dp, tp); + if (!nv.empty ()) + { + count = splice_names ( + loc, nv, move (nv_storage), ns, what, pairn, pp, dp, tp); + } + } } continue; @@ -7578,14 +9443,16 @@ namespace build2 buildspec parser:: parse_buildspec (istream& is, const path_name& in) { - // We do "effective escaping" and only for ['"\$(] (basically what's - // necessary inside a double-quoted literal plus the single quote). + // We do "effective escaping" of the special `'"\$(` characters (basically + // what's escapable inside a double-quoted literal plus the single quote; + // note, however, that we exclude line continuations and `)` since they + // would make directory paths on Windows unusable). // path_ = ∈ lexer l (is, *path_, 1 /* line */, "\'\"\\$("); lexer_ = &l; - root_ = &ctx.global_scope.rw (); + root_ = &ctx->global_scope.rw (); scope_ = root_; target_ = nullptr; prerequisite_ = nullptr; @@ -7820,8 +9687,11 @@ namespace build2 } lookup parser:: - lookup_variable (name&& qual, string&& name, const location& loc) + lookup_variable (names&& qual, string&& name, const location& loc) { + // Note that this function can be called during execute (for example, from + // scripts). In particular, this means we cannot use enter_{scope,target}. + if (pre_parse_) return lookup (); @@ -7833,9 +9703,6 @@ namespace build2 // If we are qualified, it can be a scope or a target. // - enter_scope sg; - enter_target tg; - if (qual.empty ()) { s = scope_; @@ -7844,36 +9711,70 @@ namespace build2 } else { - switch (qual.pair) + // What should we do if we cannot find the qualification (scope or + // target)? We can "fall through" to an outer scope (there is always the + // global scope backstop), we can return NULL straight away, or we can + // fail. It feels like in most cases unknown scope or target is a + // mistake and doing anything other than failing is just making things + // harder to debug. + // + switch (qual.front ().pair) { case '/': { - assert (qual.directory ()); - sg = enter_scope (*this, move (qual.dir)); - s = scope_; + assert (qual.front ().directory ()); + + dir_path& d (qual.front ().dir); + enter_scope::complete_normalize (*scope_, d); + + s = &ctx->scopes.find_out (d); + + if (s->out_path () != d) + fail (loc) << "unknown scope " << d << " in scope-qualified " + << "variable " << name << " expansion" << + info << "did you forget to include the corresponding buildfile?"; + break; } - case ':': + default: { - qual.pair = '\0'; + build2::name n (move (qual.front ())), o; + + if (n.pair) + o = move (qual.back ()); - // @@ OUT TODO + t = enter_target::find_target (*this, n, o, loc, trace); + + if (t == nullptr || !operator>= (t->decl, target_decl::implied)) // VC14 + { + diag_record dr (fail (loc)); + + dr << "unknown target " << n; + + if (n.pair && !o.dir.empty ()) + dr << '@' << o.dir; + + dr << " in target-qualified variable " << name << " expansion"; + } + + // Use the target's var_pool for good measure. // - tg = enter_target ( - *this, move (qual), build2::name (), true, loc, trace); - t = target_; + s = &t->base_scope (); + break; } - default: assert (false); } } // Lookup. // - if (const variable* pvar = scope_->var_pool ().find (name)) + if (const variable* pvar = + (s != nullptr ? s : scope_)->var_pool ().find (name)) { auto& var (*pvar); + // Note: the order of the following blocks is important. + if (p != nullptr) { // The lookup depth is a bit of a hack but should be harmless since @@ -7960,62 +9861,213 @@ namespace build2 return r; } + // file.cxx + // + extern const dir_path std_export_dir; + extern const dir_path alt_export_dir; + void parser:: - process_default_target (token& t) + process_default_target (token& t, const buildfile* bf) { tracer trace ("parser::process_default_target", &path_); // The logic is as follows: if we have an explicit current directory - // target, then that's the default target. Otherwise, we take the - // first target and use it as a prerequisite to create an implicit - // current directory target, effectively making it the default - // target via an alias. If there are no targets in this buildfile, - // then we don't do anything. + // target, then that's the default target. Otherwise, we take the first + // target and use it as a prerequisite to create an implicit current + // directory target, effectively making it the default target via an + // alias. If this is a project root buildfile, then also add exported + // buildfiles. And if there are no targets in this buildfile, then we + // don't do anything (reasonably assuming it's not root). // if (default_target_ == nullptr) // No targets in this buildfile. return; - target& dt (*default_target_); - target* ct ( - const_cast<target*> ( // Ok (serial execution). - ctx.targets.find (dir::static_type, // Explicit current dir target. - scope_->out_path (), - dir_path (), // Out tree target. - string (), - nullopt, - trace))); - - if (ct == nullptr) - { - l5 ([&]{trace (t) << "creating current directory alias for " << dt;}); - - // While this target is not explicitly mentioned in the buildfile, we - // say that we behave as if it were. Thus not implied. - // - ct = &ctx.targets.insert (dir::static_type, - scope_->out_path (), - dir_path (), - string (), - nullopt, - target_decl::real, - trace).first; - // Fall through. - } - else if (ct->decl != target_decl::real) + const_cast<target*> ( // Ok (serial execution). + ctx->targets.find (dir::static_type, // Explicit current dir target. + scope_->out_path (), + dir_path (), // Out tree target. + string (), + nullopt, + trace))); + + if (ct != nullptr && ct->decl == target_decl::real) + ; // Existing and not implied. + else { - ct->decl = target_decl::real; - // Fall through. + target& dt (*default_target_); + + if (ct == nullptr) + { + l5 ([&]{trace (t) << "creating current directory alias for " << dt;}); + + // While this target is not explicitly mentioned in the buildfile, we + // say that we behave as if it were. Thus not implied. + // + ct = &ctx->targets.insert (dir::static_type, + scope_->out_path (), + dir_path (), + string (), + nullopt, + target_decl::real, + trace).first; + } + else + ct->decl = target_decl::real; + + ct->prerequisites_state_.store (2, memory_order_relaxed); + ct->prerequisites_.push_back (prerequisite (dt)); } - else - return; // Existing and not implied. - ct->prerequisites_state_.store (2, memory_order_relaxed); - ct->prerequisites_.emplace_back (prerequisite (dt)); + // See if this is a root buildfile and not in a simple project. + // + if (bf != nullptr && + root_ != nullptr && + root_->root_extra != nullptr && + root_->root_extra->loaded && + *root_->root_extra->project != nullptr && + bf->dir == root_->src_path () && + bf->name == root_->root_extra->buildfile_file.string ()) + { + // See if we have any exported buildfiles. + // + const dir_path& export_dir ( + root_->root_extra->altn ? alt_export_dir : std_export_dir); + + dir_path d (root_->src_path () / export_dir); + if (exists (d)) + { + // Make sure prerequisites are set. + // + ct->prerequisites_state_.store (2, memory_order_relaxed); + + const string& build_ext (root_->root_extra->build_ext); + + // Return true if entered any exported buildfiles. + // + // Note: recursive lambda. + // + auto iterate = [this, &trace, + ct, &build_ext] (const dir_path& d, + const auto& iterate) -> bool + { + bool r (false); + + try + { + for (const dir_entry& e: + dir_iterator (d, dir_iterator::detect_dangling)) + { + switch (e.type ()) + { + case entry_type::directory: + { + r = iterate (d / path_cast<dir_path> (e.path ()), iterate) || r; + break; + } + case entry_type::regular: + { + const path& n (e.path ()); + + // Besides the buildfile also export buildscript and C++ files + // that are used to provide recipe implementations (see + // parse_recipe() for details). + // + string e (n.extension ()); + if (const target_type* tt = ( + e == build_ext ? &buildfile::static_type : + e == "buildscript" ? &buildscript::static_type : + e == "cxx" || + e == "cpp" || + e == "cc" ? &file::static_type : nullptr)) + { + // Enter as if found by search_existing_file(). Note that + // entering it as real would cause file_rule not to match + // for clean. + // + // Note that these targets may already be entered (for + // example, if already imported). + // + const target& bf ( + ctx->targets.insert (*tt, + d, + (root_->out_eq_src () + ? dir_path () + : out_src (d, *root_)), + n.base ().string (), + move (e), + target_decl::prereq_file, + trace).first); + + ct->prerequisites_.push_back (prerequisite (bf)); + r = true; + } + + break; + } + case entry_type::unknown: + { + bool sl (e.ltype () == entry_type::symlink); + + fail << (sl ? "dangling symlink" : "inaccessible entry") + << ' ' << d / e.path (); + + break; + } + default: + break; + } + } + } + catch (const system_error& e) + { + fail << "unable to iterate over " << d << ": " << e; + } + + return r; + }; + + if (iterate (d, iterate)) + { + // Arrange for the exported buildfiles to be installed, recreating + // subdirectories inside export/. Essentially, we are arranging for + // this: + // + // build/export/file{*}: + // { + // install = buildfile/ + // install.subdirs = true + // } + // + if (cast_false<bool> (root_->vars["install.loaded"])) + { + enter_scope es (*this, dir_path (export_dir)); + auto& vars (scope_->target_vars[file::static_type]["*"]); + + // @@ TODO: get cached variables from the module once we have one. + // + { + auto r (vars.insert (*root_->var_pool ().find ("install"))); + + if (r.second) // Already set by the user? + r.first = path_cast<path> (dir_path ("buildfile")); + } + + { + auto r (vars.insert ( + *root_->var_pool (true).find ("install.subdirs"))); + if (r.second) + r.first = true; + } + } + } + } + } } - void parser:: - enter_buildfile (const path& p) + template <typename T> + const T& parser:: + enter_buildfile (const path& p, optional<dir_path> out) { tracer trace ("parser::enter_buildfile", &path_); @@ -8023,17 +10075,20 @@ namespace build2 // Figure out if we need out. // - dir_path out; - if (scope_->src_path_ != nullptr && - scope_->src_path () != scope_->out_path () && - d.sub (scope_->src_path ())) + dir_path o; + if (out) + o = move (*out); + else if (root_ != nullptr && + root_->src_path_ != nullptr && + !root_->out_eq_src () && + d.sub (*root_->src_path_)) { - out = out_src (d, *root_); + o = out_src (d, *root_); } - ctx.targets.insert<buildfile> ( + return ctx->targets.insert<T> ( move (d), - move (out), + move (o), p.leaf ().base ().string (), p.extension (), // Always specified. trace); |