From 6362c4e4eda8340eedc73dfdbf6b92b281ccbadd Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Fri, 10 Mar 2017 11:51:24 +0200 Subject: Implement support for wildcard patterns --- build2/b.cxx | 8 +- build2/bin/target.cxx | 8 + build2/cc/target.cxx | 3 + build2/cli/target.cxx | 2 + build2/context.cxx | 189 +++++----- build2/cxx/target.cxx | 4 + build2/parser | 101 ++++-- build2/parser.cxx | 779 +++++++++++++++++++++++++++++++++--------- build2/target | 8 + build2/target-type | 6 + build2/target.cxx | 94 ++++- build2/target.txx | 81 ++++- build2/test/script/parser.cxx | 45 ++- build2/test/target.cxx | 23 ++ build2/utility.cxx | 10 +- doc/manual.cli | 76 +++-- tests/buildfile | 2 +- tests/name/buildfile | 5 + tests/name/pattern.test | 237 +++++++++++++ 19 files changed, 1358 insertions(+), 323 deletions(-) create mode 100644 tests/name/buildfile create mode 100644 tests/name/pattern.test diff --git a/build2/b.cxx b/build2/b.cxx index 2ada09b..15ace65 100644 --- a/build2/b.cxx +++ b/build2/b.cxx @@ -321,6 +321,11 @@ main (int argc, char* argv[]) trace << "jobs: " << jobs; } + // Set the build state before parsing the buildspec since it relies on + // global scope being setup. + // + variable_overrides var_ovs (reset (cmd_vars)); + // Parse the buildspec. // buildspec bspec; @@ -349,8 +354,7 @@ main (int argc, char* argv[]) // opspec* lifted (nullptr); size_t skip (0); - bool dirty (true); - variable_overrides var_ovs; + bool dirty (false); // We already (re)set for the first run. for (auto mit (bspec.begin ()); mit != bspec.end (); ) { diff --git a/build2/bin/target.cxx b/build2/bin/target.cxx index cf187fa..2be5686 100644 --- a/build2/bin/target.cxx +++ b/build2/bin/target.cxx @@ -41,6 +41,7 @@ namespace build2 &file::static_type, &objx_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_target, // Note: not _file(); don't look for an existing file. false @@ -52,6 +53,7 @@ namespace build2 &file::static_type, &objx_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_target, // Note: not _file(); don't look for an existing file. false @@ -63,6 +65,7 @@ namespace build2 &file::static_type, &objx_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_target, // Note: not _file(); don't look for an existing file. false @@ -103,6 +106,7 @@ namespace build2 &obj_factory, nullptr, nullptr, + nullptr, &search_target, false }; @@ -139,6 +143,7 @@ namespace build2 &file::static_type, &libx_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false @@ -150,6 +155,7 @@ namespace build2 &file::static_type, &libx_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false @@ -199,6 +205,7 @@ namespace build2 &lib_factory, nullptr, nullptr, + nullptr, &search_target, false }; @@ -211,6 +218,7 @@ namespace build2 &file::static_type, &target_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false diff --git a/build2/cc/target.cxx b/build2/cc/target.cxx index d8c3b97..f269b35 100644 --- a/build2/cc/target.cxx +++ b/build2/cc/target.cxx @@ -17,6 +17,7 @@ namespace build2 nullptr, nullptr, nullptr, + nullptr, &search_target, false }; @@ -30,6 +31,7 @@ namespace build2 &file::static_type, &target_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false @@ -42,6 +44,7 @@ namespace build2 &cc::static_type, &target_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false diff --git a/build2/cli/target.cxx b/build2/cli/target.cxx index 015c034..e3ce7e2 100644 --- a/build2/cli/target.cxx +++ b/build2/cli/target.cxx @@ -22,6 +22,7 @@ namespace build2 &file::static_type, &target_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false @@ -68,6 +69,7 @@ namespace build2 &cli_cxx_factory, nullptr, nullptr, + nullptr, &search_target, true // "See through" default iteration mode. }; diff --git a/build2/context.cxx b/build2/context.cxx index 62822a3..c055035 100644 --- a/build2/context.cxx +++ b/build2/context.cxx @@ -250,6 +250,100 @@ namespace build2 scope& gs (make_global_scope ()); + // Setup the global scope before parsing any variable overrides since they + // may reference these things. + // + + // Target extension. + // + vp.insert ("extension", variable_visibility::target); + + gs.assign ("build.work") = work; + gs.assign ("build.home") = home; + + // Build system driver process path. + // + gs.assign ("build.path") = + process_path (nullptr, // Will be filled by value assignment. + path (argv0.recall_string ()), + path (argv0.effect)); + + // Build system version. + // + { + gs.assign ("build.version") = uint64_t (BUILD2_VERSION); + gs.assign ("build.version.string") = BUILD2_VERSION_STR; + + // AABBCCDD + // + auto comp = [] (unsigned int d) -> uint64_t + { + return (BUILD2_VERSION / d) % 100; + }; + + gs.assign ("build.version.release") = comp (1); + gs.assign ("build.version.patch") = comp (100); + gs.assign ("build.version.minor") = comp (10000); + gs.assign ("build.version.major") = comp (1000000); + } + + // Enter the host information. Rather than jumping through hoops like + // config.guess, for now we are just going to use the compiler target we + // were built with. While it is not as precise (for example, a binary + // built for i686 might be running on x86_64), it is good enough of an + // approximation/fallback since most of the time we are interested in just + // the target class (e.g., linux, windows, macosx). + // + { + // Did the user ask us to use config.guess? + // + string orig ( + ops.config_guess_specified () + ? run (ops.config_guess (), [] (string& l) {return move (l);}) + : BUILD2_HOST_TRIPLET); + + l5 ([&]{trace << "original host: '" << orig << "'";}); + + try + { + target_triplet t (orig); + + l5 ([&]{trace << "canonical host: '" << t.string () << "'; " + << "class: " << t.class_;}); + + // Also enter as build.host.{cpu,vendor,system,version,class} for + // convenience of access. + // + gs.assign ("build.host.cpu") = t.cpu; + gs.assign ("build.host.vendor") = t.vendor; + gs.assign ("build.host.system") = t.system; + gs.assign ("build.host.version") = t.version; + gs.assign ("build.host.class") = t.class_; + + gs.assign ("build.host") = move (t); + } + catch (const invalid_argument& e) + { + fail << "unable to parse build host '" << orig << "': " << e << + info << "consider using the --config-guess option"; + } + } + + // Register builtin target types. + // + { + target_type_map& t (gs.target_types); + + t.insert (); + t.insert (); + t.insert (); + t.insert (); + t.insert (); + t.insert (); + t.insert (); + t.insert (); + } + // Parse and enter the command line variables. We do it before entering // any other variables so that all the variables that are overriden are // marked as such first. Then, as we enter variables, we can verify that @@ -337,10 +431,11 @@ namespace build2 o = o->override.get (); // Currently we expand project overrides in the global scope to keep - // things simple. Pass original variable for diagnostics. + // things simple. Pass original variable for diagnostics. Use current + // working directory as pattern base. // parser p; - pair r (p.parse_variable_value (l, gs, var)); + pair r (p.parse_variable_value (l, gs, &work, var)); if (r.second.type != token_type::eos) fail << "unexpected " << r.second << " in variable assignment " @@ -410,96 +505,6 @@ namespace build2 var_import_target = &vp.insert ("import.target"); - // Target extension. - // - vp.insert ("extension", variable_visibility::target); - - gs.assign ("build.work") = work; - gs.assign ("build.home") = home; - - // Build system driver process path. - // - gs.assign ("build.path") = - process_path (nullptr, // Will be filled by value assignment. - path (argv0.recall_string ()), - path (argv0.effect)); - - // Build system version. - // - { - gs.assign ("build.version") = uint64_t (BUILD2_VERSION); - gs.assign ("build.version.string") = BUILD2_VERSION_STR; - - // AABBCCDD - // - auto comp = [] (unsigned int d) -> uint64_t - { - return (BUILD2_VERSION / d) % 100; - }; - - gs.assign ("build.version.release") = comp (1); - gs.assign ("build.version.patch") = comp (100); - gs.assign ("build.version.minor") = comp (10000); - gs.assign ("build.version.major") = comp (1000000); - } - - // Enter the host information. Rather than jumping through hoops like - // config.guess, for now we are just going to use the compiler target we - // were built with. While it is not as precise (for example, a binary - // built for i686 might be running on x86_64), it is good enough of an - // approximation/fallback since most of the time we are interested in just - // the target class (e.g., linux, windows, macosx). - // - { - // Did the user ask us to use config.guess? - // - string orig ( - ops.config_guess_specified () - ? run (ops.config_guess (), [] (string& l) {return move (l);}) - : BUILD2_HOST_TRIPLET); - - l5 ([&]{trace << "original host: '" << orig << "'";}); - - try - { - target_triplet t (orig); - - l5 ([&]{trace << "canonical host: '" << t.string () << "'; " - << "class: " << t.class_;}); - - // Also enter as build.host.{cpu,vendor,system,version,class} for - // convenience of access. - // - gs.assign ("build.host.cpu") = t.cpu; - gs.assign ("build.host.vendor") = t.vendor; - gs.assign ("build.host.system") = t.system; - gs.assign ("build.host.version") = t.version; - gs.assign ("build.host.class") = t.class_; - - gs.assign ("build.host") = move (t); - } - catch (const invalid_argument& e) - { - fail << "unable to parse build host '" << orig << "': " << e << - info << "consider using the --config-guess option"; - } - } - - // Register builtin target types. - // - { - target_type_map& t (gs.target_types); - - t.insert (); - t.insert (); - t.insert (); - t.insert (); - t.insert (); - t.insert (); - t.insert (); - t.insert (); - } - // Register builtin rules. // { diff --git a/build2/cxx/target.cxx b/build2/cxx/target.cxx index 37f33cb..7b19312 100644 --- a/build2/cxx/target.cxx +++ b/build2/cxx/target.cxx @@ -19,6 +19,7 @@ namespace build2 &file::static_type, &target_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false @@ -31,6 +32,7 @@ namespace build2 &file::static_type, &target_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false @@ -43,6 +45,7 @@ namespace build2 &file::static_type, &target_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false @@ -55,6 +58,7 @@ namespace build2 &cc::static_type, &target_factory, &target_extension_var, + &target_pattern_var, nullptr, &search_file, false diff --git a/build2/parser b/build2/parser index df2797b..91d574b 100644 --- a/build2/parser +++ b/build2/parser @@ -41,7 +41,7 @@ namespace build2 parse_variable (lexer&, scope&, const variable&, token_type kind); pair - parse_variable_value (lexer&, scope&, const variable&); + parse_variable_value (lexer&, scope&, const dir_path*, const variable&); names parse_export_stub (istream& is, const path& p, scope& r, scope& b) @@ -53,6 +53,16 @@ namespace build2 // Recursive descent parser. // protected: + + // Pattern expansion mode. + // + enum class pattern_mode + { + ignore, // Treat as ordinary names. + detect, // Ignore pair/dir/type if the first name is a pattern. + expand // Expand to ordinary names. + }; + // If one is true then parse a single (logical) line (logical means it // can actually be several lines, e.g., an if-block). Return false if // nothing has been parsed (i.e., we are on the same token). @@ -115,25 +125,25 @@ namespace build2 // result is an empty pack, not a pack of one empty. // values - parse_eval (token&, token_type&); + parse_eval (token&, token_type&, pattern_mode); values - parse_eval_comma (token&, token_type&, bool = false); + parse_eval_comma (token&, token_type&, pattern_mode, bool = false); value - parse_eval_ternary (token&, token_type&, bool = false); + parse_eval_ternary (token&, token_type&, pattern_mode, bool = false); value - parse_eval_or (token&, token_type&, bool = false); + parse_eval_or (token&, token_type&, pattern_mode, bool = false); value - parse_eval_and (token&, token_type&, bool = false); + parse_eval_and (token&, token_type&, pattern_mode, bool = false); value - parse_eval_comp (token&, token_type&, bool = false); + parse_eval_comp (token&, token_type&, pattern_mode, bool = false); value - parse_eval_value (token&, token_type&, bool = false); + parse_eval_value (token&, token_type&, pattern_mode, bool = false); // Attributes stack. We can have nested attributes, for example: // @@ -189,17 +199,25 @@ namespace build2 // project separator. Note that even if it is NULL, the result may still // contain non-simple names due to variable expansions. // + static const string name_separators; names parse_names (token& t, token_type& tt, + pattern_mode pmode, bool chunk = false, const char* what = "name", const string* separators = &name_separators) { names ns; - parse_names ( - t, tt, ns, chunk, what, separators, 0, nullopt, nullptr, nullptr); + parse_names (t, tt, + ns, + pmode, + chunk, + what, + separators, + 0, + nullopt, nullptr, nullptr); return ns; } @@ -207,28 +225,35 @@ namespace build2 // value parse_value (token& t, token_type& tt, + pattern_mode pmode, const char* what = "name", const string* separators = &name_separators, bool chunk = false) { names ns; - pair p ( - parse_names ( - t, tt, ns, chunk, what, separators, 0, nullopt, nullptr, nullptr)); + auto r (parse_names (t, tt, + ns, + pmode, + chunk, + what, + separators, + 0, + nullopt, nullptr, nullptr)); - value r (p.second); // Potentially typed NULL value. + value v (r.type); // Potentially typed NULL value. // This should not fail since we are typing the result of reversal from // the typed value. // - if (p.first) // Not NULL. - r.assign (move (ns), nullptr); + if (r.not_null) + v.assign (move (ns), nullptr); - return r; + return v; } - // Append names and return the indication if the parsed value is NOT NULL - // (first) and whether it is typed (second). + // Append names and return the indication if the parsed value is not NULL + // and whether it is typed (and whether it is a pattern if pattern_mode is + // detect). // // You may have noticed that what we return here is essentially a value // and doing it this way (i.e., reversing it to untyped names and @@ -243,13 +268,21 @@ namespace build2 // it is the result of a sole, unquoted variable expansion, function call, // or context evaluation. // - pair + struct parse_names_result + { + bool not_null; + const value_type* type; + optional pattern; + }; + + parse_names_result parse_names (token&, token_type&, names&, + pattern_mode, bool chunk = false, const char* what = "name", const string* separators = &name_separators, - size_t pair = 0, + size_t pairn = 0, const optional& prj = nullopt, const dir_path* dir = nullptr, const string* type = nullptr); @@ -257,13 +290,35 @@ namespace build2 size_t parse_names_trailer (token&, token_type&, names&, + pattern_mode, const char* what, const string* separators, - size_t pair, + size_t pairn, const optional& prj, const dir_path* dir, const string* type); + size_t + expand_name_pattern (const location&, + names&&, + names&, + const char* what, + size_t pairn, + const dir_path* dir, + const string* type, + const target_type*); + + size_t + splice_names (const location&, + const names_view&, + names&&, + names&, + const char* what, + size_t pairn, + const optional& prj, + const dir_path* dir, + const string* type); + // Skip until newline or eos. // void @@ -529,6 +584,8 @@ namespace build2 scope* scope_; // Current base scope (out_base). scope* root_; // Current root scope (out_root). + const dir_path* pbase_ = nullptr; // Current pattern base directory. + std::stack attributes_; target* default_target_; diff --git a/build2/parser.cxx b/build2/parser.cxx index 20d7bd2..9dcd4f5 100644 --- a/build2/parser.cxx +++ b/build2/parser.cxx @@ -6,6 +6,8 @@ #include // cout +#include // path_search(), path_match() + #include #include @@ -27,9 +29,10 @@ namespace build2 class parser::enter_scope { public: - enter_scope (): p_ (nullptr), r_ (nullptr), s_ (nullptr) {} + enter_scope (): p_ (nullptr), r_ (nullptr), s_ (nullptr), b_ (nullptr) {} - enter_scope (parser& p, dir_path&& d): p_ (&p), r_ (p.root_), s_ (p.scope_) + enter_scope (parser& p, dir_path&& d) + : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_) { // Try hard not to call normalize(). Most of the time we will go just // one level deeper. @@ -61,14 +64,25 @@ namespace build2 { p_->scope_ = s_; p_->root_ = r_; + p_->pbase_ = b_; } } // Note: move-assignable to empty only. // enter_scope (enter_scope&& x) {*this = move (x);} - enter_scope& operator= (enter_scope&& x) { - p_ = x.p_; r_ = x.r_; s_ = x.s_; x.p_ = nullptr; return *this;} + enter_scope& operator= (enter_scope&& x) + { + if (this != &x) + { + p_ = x.p_; + r_ = x.r_; + s_ = x.s_; + b_ = x.b_; + x.p_ = nullptr; + } + return *this; + } enter_scope (const enter_scope&) = delete; enter_scope& operator= (const enter_scope&) = delete; @@ -77,6 +91,7 @@ namespace build2 parser* p_; scope* r_; scope* s_; + const dir_path* b_; // Pattern base. }; class parser::enter_target @@ -167,9 +182,10 @@ namespace build2 lexer l (is, *path_); lexer_ = &l; - target_ = nullptr; - scope_ = &base; root_ = &root; + scope_ = &base; + pbase_ = scope_->src_path_; + target_ = nullptr; default_target_ = nullptr; enter_buildfile (p); // Needs scope_. @@ -193,6 +209,7 @@ namespace build2 lexer_ = &l; target_ = nullptr; scope_ = &s; + pbase_ = scope_->src_path_; // Normally NULL. token t; type tt; @@ -201,12 +218,16 @@ namespace build2 } pair parser:: - parse_variable_value (lexer& l, scope& s, const variable& var) + parse_variable_value (lexer& l, + scope& s, + const dir_path* b, + const variable& var) { path_ = &l.name (); lexer_ = &l; target_ = nullptr; scope_ = &s; + pbase_ = b; token t; type tt; @@ -331,14 +352,8 @@ namespace build2 } } - // ': foo' is equvalent to '{}: foo' and to 'dir{}: foo'. - // - // @@ I think we should make ': foo' invalid. - // const location nloc (get_location (t)); - names ns (tt != type::colon - ? parse_names (t, tt) - : names ({name ("dir", string ())})); + names ns (parse_names (t, tt, pattern_mode::ignore)); if (tt == type::colon) { @@ -444,9 +459,11 @@ namespace build2 tt == type::newline || tt == type::eos) { + // @@ PAT: currently we pattern-expand scope/target-specific vars. + // const location ploc (get_location (t)); names pns (tt != type::newline && tt != type::eos - ? parse_names (t, tt) + ? parse_names (t, tt, pattern_mode::expand) : names ()); // Scope/target-specific variable assignment. @@ -785,7 +802,11 @@ namespace build2 next (t, tt); const location l (get_location (t)); names ns (tt != type::newline && tt != type::eos - ? parse_names (t, tt, false, "path", nullptr) + ? parse_names (t, tt, + pattern_mode::expand, + false, + "path", + nullptr) : names ()); for (name& n: ns) @@ -861,7 +882,11 @@ namespace build2 next (t, tt); const location l (get_location (t)); names ns (tt != type::newline && tt != type::eos - ? parse_names (t, tt, false, "path", nullptr) + ? parse_names (t, tt, + pattern_mode::expand, + false, + "path", + nullptr) : names ()); for (name& n: ns) @@ -918,6 +943,7 @@ namespace build2 // scope* ors (root_); scope* ocs (scope_); + const dir_path* opb (pbase_); switch_scope (out_base); // Use the new scope's src_base to get absolute buildfile path @@ -931,6 +957,7 @@ namespace build2 if (!root_->buildfiles.insert (p).second) // Note: may be "new" root. { l5 ([&]{trace (l) << "skipping already included " << p;}); + pbase_ = opb; scope_ = ocs; root_ = ors; continue; @@ -975,6 +1002,7 @@ namespace build2 fail (l) << "unable to read buildfile " << p << ": " << e; } + pbase_ = opb; scope_ = ocs; root_ = ors; } @@ -1100,12 +1128,13 @@ namespace build2 attributes_pop (); } - // The rest should be a list of projects and/or targets. Parse - // them as names to get variable expansion and directory prefixes. + // The rest should be a list of projects and/or targets. Parse them as + // names to get variable expansion and directory prefixes. Note: doesn't + // make sense to expand patterns (what's the base directory?) // const location l (get_location (t)); names ns (tt != type::newline && tt != type::eos - ? parse_names (t, tt) + ? parse_names (t, tt, pattern_mode::ignore) : names ()); for (name& n: ns) @@ -1197,7 +1226,11 @@ namespace build2 next (t, tt); const location l (get_location (t)); names ns (tt != type::newline && tt != type::eos - ? parse_names (t, tt, false, "module", nullptr) + ? parse_names (t, tt, + pattern_mode::ignore, + false, + "module", + nullptr) : names ()); for (auto i (ns.begin ()); i != ns.end (); ++i) @@ -1332,7 +1365,9 @@ namespace build2 if (tt == type::newline || tt == type::eos) fail (t) << "expected " << k << "-expression instead of " << t; - // Parse as names to get variable expansion, evaluation, etc. + // Parse as names to get variable expansion, evaluation, etc. Note + // that we also expand patterns (could be used in nested contexts, + // etc; e.g., "if pattern expansion is empty" condition). // const location l (get_location (t)); @@ -1342,7 +1377,10 @@ namespace build2 // bool e ( convert ( - parse_value (t, tt, "expression", nullptr))); + parse_value (t, tt, + pattern_mode::expand, + "expression", + nullptr))); take = (k.back () == '!' ? !e : e); } @@ -1437,7 +1475,11 @@ namespace build2 // bool e ( convert ( - parse_value (t, tt, "expression", nullptr, true))); + parse_value (t, tt, + pattern_mode::expand, + "expression", + nullptr, + true))); e = (neg ? !e : e); if (e) @@ -1456,7 +1498,11 @@ namespace build2 // any, with expansion. Then fail. // names ns (tt != type::newline && tt != type::eos - ? parse_names (t, tt, false, "description", nullptr) + ? parse_names (t, tt, + pattern_mode::ignore, + false, + "description", + nullptr) : names ()); diag_record dr (fail (al)); @@ -1534,7 +1580,7 @@ namespace build2 attributes_push (t, tt, true); return tt != type::newline && tt != type::eos - ? parse_value (t, tt) + ? parse_value (t, tt, pattern_mode::expand) : value (names ()); } @@ -1751,7 +1797,7 @@ namespace build2 } values parser:: - parse_eval (token& t, type& tt) + parse_eval (token& t, type& tt, pattern_mode pmode) { // enter: lparen // leave: rparen @@ -1762,7 +1808,7 @@ namespace build2 if (tt == type::rparen) return values (); - values r (parse_eval_comma (t, tt, true)); + values r (parse_eval_comma (t, tt, pmode, true)); if (tt != type::rparen) fail (t) << "unexpected " << t; // E.g., stray ':'. @@ -1771,7 +1817,7 @@ namespace build2 } values parser:: - parse_eval_comma (token& t, type& tt, bool first) + parse_eval_comma (token& t, type& tt, pattern_mode pmode, bool first) { // enter: first token of LHS // leave: next token after last RHS @@ -1779,7 +1825,7 @@ namespace build2 // Left-associative: parse in a loop for as long as we can. // values r; - value lhs (parse_eval_ternary (t, tt, first)); + value lhs (parse_eval_ternary (t, tt, pmode, first)); if (!pre_parse_) r.push_back (move (lhs)); @@ -1787,7 +1833,7 @@ namespace build2 while (tt == type::comma) { next (t, tt); - value rhs (parse_eval_ternary (t, tt)); + value rhs (parse_eval_ternary (t, tt, pmode)); if (!pre_parse_) r.push_back (move (rhs)); @@ -1797,7 +1843,7 @@ namespace build2 } value parser:: - parse_eval_ternary (token& t, type& tt, bool first) + parse_eval_ternary (token& t, type& tt, pattern_mode pmode, bool first) { // enter: first token of LHS // leave: next token after last RHS @@ -1813,7 +1859,7 @@ namespace build2 // a ? (x ? y : z) : (b ? c : d) // location l (get_location (t)); - value lhs (parse_eval_or (t, tt, first)); + value lhs (parse_eval_or (t, tt, pmode, first)); if (tt != type::question) return lhs; @@ -1833,7 +1879,7 @@ namespace build2 pre_parse_ = !q; // Short-circuit middle? next (t, tt); - value mhs (parse_eval_ternary (t, tt)); + value mhs (parse_eval_ternary (t, tt, pmode)); if (tt != type::colon) fail (t) << "expected ':' instead of " << t; @@ -1842,14 +1888,14 @@ namespace build2 pre_parse_ = q; // Short-circuit right? next (t, tt); - value rhs (parse_eval_ternary (t, tt)); + value rhs (parse_eval_ternary (t, tt, pmode)); pre_parse_ = pp; return q ? move (mhs) : move (rhs); } value parser:: - parse_eval_or (token& t, type& tt, bool first) + parse_eval_or (token& t, type& tt, pattern_mode pmode, bool first) { // enter: first token of LHS // leave: next token after last RHS @@ -1857,7 +1903,7 @@ namespace build2 // Left-associative: parse in a loop for as long as we can. // location l (get_location (t)); - value lhs (parse_eval_and (t, tt, first)); + value lhs (parse_eval_and (t, tt, pmode, first)); // Use the pre-parse mechanism to implement short-circuit. // @@ -1872,7 +1918,7 @@ namespace build2 next (t, tt); l = get_location (t); - value rhs (parse_eval_and (t, tt)); + value rhs (parse_eval_and (t, tt, pmode)); if (pre_parse_) continue; @@ -1889,7 +1935,7 @@ namespace build2 } value parser:: - parse_eval_and (token& t, type& tt, bool first) + parse_eval_and (token& t, type& tt, pattern_mode pmode, bool first) { // enter: first token of LHS // leave: next token after last RHS @@ -1897,7 +1943,7 @@ namespace build2 // Left-associative: parse in a loop for as long as we can. // location l (get_location (t)); - value lhs (parse_eval_comp (t, tt, first)); + value lhs (parse_eval_comp (t, tt, pmode, first)); // Use the pre-parse mechanism to implement short-circuit. // @@ -1912,7 +1958,7 @@ namespace build2 next (t, tt); l = get_location (t); - value rhs (parse_eval_comp (t, tt)); + value rhs (parse_eval_comp (t, tt, pmode)); if (pre_parse_) continue; @@ -1929,14 +1975,14 @@ namespace build2 } value parser:: - parse_eval_comp (token& t, type& tt, bool first) + parse_eval_comp (token& t, type& tt, pattern_mode pmode, bool first) { // enter: first token of LHS // leave: next token after last RHS // Left-associative: parse in a loop for as long as we can. // - value lhs (parse_eval_value (t, tt, first)); + value lhs (parse_eval_value (t, tt, pmode, first)); while (tt == type::equal || tt == type::not_equal || @@ -1949,7 +1995,7 @@ namespace build2 location l (get_location (t)); next (t, tt); - value rhs (parse_eval_value (t, tt)); + value rhs (parse_eval_value (t, tt, pmode)); if (pre_parse_) continue; @@ -2005,7 +2051,7 @@ namespace build2 } value parser:: - parse_eval_value (token& t, type& tt, bool first) + parse_eval_value (token& t, type& tt, pattern_mode pmode, bool first) { // enter: first token of value // leave: next token after value @@ -2023,7 +2069,7 @@ namespace build2 case type::log_not: { next (t, tt); - v = parse_eval_value (t, tt); + v = parse_eval_value (t, tt, pmode); if (pre_parse_) break; @@ -2058,7 +2104,7 @@ namespace build2 tt != type::log_or && tt != type::log_and - ? parse_value (t, tt) + ? parse_value (t, tt, pmode) : value (names ())); } } @@ -2075,7 +2121,7 @@ namespace build2 const location nl (get_location (t)); next (t, tt); - value n (parse_value (t, tt)); + value n (parse_value (t, tt, pattern_mode::ignore)); if (tt != type::rparen) fail (t) << "expected ')' after variable name"; @@ -2133,7 +2179,9 @@ namespace build2 if (tt != type::rsbrace) { - names ns (parse_names (t, tt, false, "attribute", nullptr)); + names ns ( + parse_names ( + t, tt, pattern_mode::ignore, false, "attribute", nullptr)); if (!pre_parse_) { @@ -2183,12 +2231,363 @@ namespace build2 return make_pair (true, l); } + // Splice names from the name view into the destination name list while + // doing sensible things with pairs, types, etc. Return the number of + // the names added. + // + // If nv points to nv_storage then the names can be moved. + // + size_t parser:: + splice_names (const location& loc, + const names_view& nv, + names&& nv_storage, + names& ns, + const char* what, + size_t pairn, + const optional& pp, + const dir_path* dp, + const string* tp) + { + // We could be asked to splice 0 elements (see the name pattern + // expansion). In this case may need to pop the first half of the + // pair. + // + if (nv.size () == 0) + { + if (pairn != 0) + ns.pop_back (); + + return 0; + } + + size_t start (ns.size ()); + + // Move if nv points to nv_storage, + // + bool m (nv.data () == nv_storage.data ()); + + for (const name& cn: nv) + { + name* n (m ? const_cast (&cn) : nullptr); + + // Project. + // + optional p; + if (cn.proj) + { + if (pp) + fail (loc) << "nested project name " << *cn.proj << " in " << what; + + p = m ? move (n->proj) : cn.proj; + } + else if (pp) + p = pp; + + // Directory. + // + dir_path d; + if (!cn.dir.empty ()) + { + if (dp != nullptr) + { + if (cn.dir.absolute ()) + fail (loc) << "nested absolute directory " << cn.dir << " in " + << what; + + d = *dp / cn.dir; + } + else + d = m ? move (n->dir) : cn.dir; + } + else if (dp != nullptr) + d = *dp; + + // Type. + // + string t; + if (!cn.type.empty ()) + { + if (tp != nullptr) + fail (loc) << "nested type name " << cn.type << " in " << what; + + t = m ? move (n->type) : cn.type; + } + else if (tp != nullptr) + t = *tp; + + // Value. + // + string v (m ? move (n->value) : cn.value); + + // If we are a second half of a pair. + // + if (pairn != 0) + { + // Check that there are no nested pairs. + // + if (cn.pair) + fail (loc) << "nested pair in " << what; + + // And add another first half unless this is the first instance. + // + if (pairn != ns.size ()) + ns.push_back (ns[pairn - 1]); + } + + ns.emplace_back (move (p), move (d), move (t), move (v)); + ns.back ().pair = cn.pair; + } + + return ns.size () - start; + } + + // Expand a name pattern. Note that the result can empty (as in "no + // elements"). + // + size_t parser:: + expand_name_pattern (const location& l, + names&& pat, + names& ns, + const char* what, + size_t pairn, + const dir_path* dp, + const string* tp, + const target_type* tt) + { + assert (!pat.empty () && (tp == nullptr || tt != nullptr)); + + // We are going to accumulate the result in a vector which can result in + // quite a few linear searches. However, thanks to a few optimizations, + // this shouldn't be an issue for the common cases (e.g., a pattern plus + // a few exclusions). + // + names r; + bool dir (false); + + // Compare string to name as paths and according to dir. + // + auto equal = [&dir] (const string& v, const name& n) -> bool + { + // Use path comparison (which may be slash/case-insensitive). + // + return path::traits::compare ( + v, dir ? n.dir.representation () : n.value) == 0; + }; + + // Compare name to pattern as paths and according to dir. + // + auto match = [&dir] (const string& p, const name& n) -> bool + { + // Currently exclusions only match the last component to a pattern. + // + string c (dir + ? n.dir.leaf ().representation () + : string (n.value, path::traits::find_leaf (n.value))); + + return butl::path_match (p, c); + }; + + // Append string to result according to dir. Store an indication of + // whether it was amended in the pair flag. + // + auto append = [&r, &dir] (string&& v, bool a) + { + r.push_back (dir ? name (dir_path (move (v))) : name (move (v))); + r.back ().pair = a ? 'a' : '\0'; + }; + + auto include_match = [&r, &equal, &append] (string&& m, bool a) + { + auto i (find_if ( + r.begin (), + r.end (), + [&m, &equal] (const name& n) {return equal (m, n);})); + + if (i == r.end ()) + append (move (m), a); + }; + + auto include_pattern = [&r, &append, &include_match, dp, this] + (string&& p, bool a) + { + // If we don't already have any matches and our pattern doesn't contain + // multiple recursive wildcards, then the result will be unique and we + // can skip checking for duplicated. This should help quite a bit in the + // common cases where we have a pattern plus maybe a few exclusions. + // + bool unique (false); + if (r.empty ()) + { + size_t i (p.find ("**")); + unique = (i == string::npos || p.find ("**", i + 2) == string::npos); + } + + //@@ PAT TODO: weed out starting with dot (unless pattern starts + // with dot; last component? intermediate components?). + // + function func; + if (unique) + func = [a, &append] (path&& m) + { + append (move (m).representation (), a); + return true; + }; + else + func = [a, &include_match] (path&& m) + { + include_match (move (m).representation (), a); + return true; + }; + + // Figure out the start directory. + // + const dir_path* sp; + dir_path s; + if (dp != nullptr) + { + if (dp->absolute ()) + sp = dp; + else + { + s = *pbase_ / *dp; + sp = &s; + } + } + else + sp = pbase_; + + butl::path_search (path (move (p)), func, *sp); + }; + + auto exclude_match = [&r, &equal] (const string& m) + { + // We know there can only be one element so we use find_if() instead of + // remove_if() for efficiency. + // + auto i (find_if ( + r.begin (), + r.end (), + [&m, &equal] (const name& n) {return equal (m, n);})); + + if (i != r.end ()) + r.erase (i); + }; + + auto exclude_pattern = [&r, &match] (const string& p) + { + for (auto i (r.begin ()); i != r.end (); ) + { + if (match (p, *i)) + i = r.erase (i); + else + ++i; + } + }; + + // Process the pattern and inclusions/exclusions. + // + for (auto b (pat.begin ()), i (b), e (pat.end ()); i != e; ++i) + { + name& n (*i); + bool first (i == b); + + if (n.empty () || !(n.simple () || n.directory ())) + fail (l) << "invalid '" << n << "' in " << what << " pattern"; + + string v (n.simple () ? move (n.value) : move (n.dir).representation ()); + + // Figure out if this is inclusion or exclusion. + // + char s; // +/- + if (first) + s = '+'; // Treat as inclusion. + else + { + s = v[0]; + + if (s != '-' && s != '+') + fail (l) << "missing leading +/- in '" << v << "' " << what + << " pattern"; + + v.erase (0, 1); + + if (v.empty ()) + fail (l) << "empty " << what << " pattern"; + } + + // Amend the pattern or match in a target type-specific manner. + // + bool a (tt != nullptr && + tt->pattern != nullptr && + tt->pattern (*tt, *scope_, v, false)); + + // Figure out if this is a pattern. + // + bool p (v.find_first_of ("*?") != string::npos); + assert (p || !first); // First must be a pattern. + + // Based on the first pattern figure out if the result is a directory or + // a file. For inclusions/exclusions verify it is consistent. + // + { + bool d (path::traits::is_separator (v.back ())); + + if (first) + dir = d; + else if (d != dir) + fail (l) << "inconsistent file/directory result in " << what + << " pattern"; + } + + if (s == '+') + { + if (p) + include_pattern (move (v), a); + else + include_match (move (v), a); + } + else + { + if (p) + exclude_pattern (v); + else + exclude_match (v); + } + } + + // Reverse target type-specific pattern/match amendments from the result. + // Essentially: cxx{*} -> *.cxx -> foo.cxx -> cxx{foo}. + // + if (tt != nullptr && tt->pattern != nullptr) + { + for (name& n: r) + { + if (n.pair) + { + string v (dir ? move (n.dir).representation () : move (n.value)); + tt->pattern (*tt, *scope_, v, true); + + if (dir) + n.dir = dir_path (move (v)); + else + n.value = move (v); + + n.pair = '\0'; + } + } + } + + return splice_names ( + l, names_view (r), move (r), ns, what, pairn, nullopt, dp, tp); + } + // Parse names inside {} and handle the following "crosses" (i.e., // {a b}{x y}) if any. Return the number of names added to the list. // size_t parser:: parse_names_trailer (token& t, type& tt, names& ns, + pattern_mode pmode, const char* what, const string* separators, size_t pairn, @@ -2198,23 +2597,67 @@ namespace build2 { assert (!pre_parse_); + if (pp) + pmode = pattern_mode::ignore; + next (t, tt); // Get what's after '{'. - size_t count (ns.size ()); - parse_names (t, tt, - ns, - false, - what, - separators, - (pairn != 0 - ? pairn - : (ns.empty () || ns.back ().pair ? ns.size () : 0)), - pp, dp, tp); - count = ns.size () - count; + size_t start (ns.size ()); + + if (pairn == 0 && start != 0 && ns.back ().pair) + pairn = start; + + // This can be an ordinary name group or a pattern (with inclusions and + // exclusions). We want to detect which one it is since for patterns we + // want just the list of simple names without pair/dir/type added (those + // are added after the pattern expansion in parse_names_pattern()). + // + // Detecting which one it is is tricky. We cannot just peek at the token + // and look for some wildcards since the pattern can be the result of an + // expansion (or, worse, concatenation). Thus pattern_mode::detect: we are + // going to ask parse_names() to detect for us if the first name is a + // pattern. And if it is, to refrain from adding pair/dir/type. On our + // side we will also have to move the pattern names out of the result. + // + const location loc (get_location (t)); + + optional pat_tt ( + parse_names ( + t, tt, + ns, + pmode == pattern_mode::expand ? pattern_mode::detect : pmode, + false, + what, + separators, + pairn, + pp, dp, tp).pattern); if (tt != type::rcbrace) fail (t) << "expected } instead of " << t; + size_t count; + + // See if this is a pattern. + // + if (pat_tt) + { + // Move the pattern names our of the result. + // + names ps; + if (start == 0) + ps = move (ns); + else + ps.insert (ps.end (), + make_move_iterator (ns.begin () + start), + make_move_iterator (ns.end ())); + ns.resize (start); + + count = expand_name_pattern ( + loc, move (ps), ns, what, pairn, dp, tp, *pat_tt); + } + else + count = ns.size () - start; + // See if we have a cross. See tests/names. // if (peek () == type::lcbrace && !peeked ().separated) @@ -2224,9 +2667,13 @@ namespace build2 names x; // Parse into a separate list of names. parse_names_trailer ( - t, tt, x, what, separators, 0, nullopt, nullptr, nullptr); + t, tt, x, pmode, what, separators, 0, nullopt, nullptr, nullptr); - if (size_t n = x.size ()) + size_t n (x.size ()); + + // If LHS or RHS are empty, then the result is empty as well. + // + if (count != 0 && n != 0) { // Now cross the last 'count' names in 'ns' with 'x'. First we will // allocate n - 1 additional sets of last 'count' names in 'ns'. @@ -2296,6 +2743,11 @@ namespace build2 count *= n; } + else if (count != 0) + { + ns.resize (start); // Drop LHS. + count = 0; + } } return count; @@ -2307,31 +2759,36 @@ namespace build2 const string parser::name_separators ( string (path::traits::directory_separators) + '%'); - pair parser:: + auto parser:: parse_names (token& t, type& tt, names& ns, + pattern_mode pmode, bool chunk, const char* what, const string* separators, size_t pairn, const optional& pp, const dir_path* dp, - const string* tp) + const string* tp) -> parse_names_result { // Note that support for pre-parsing is partial, it does not handle // groups ({}). + // + // If pairn is not 0, then it is an index + 1 of the first half of the + // pair for which we are parsing the second halves, for example: + // + // a@{b c d{e f} {}} tracer trace ("parser::parse_names", &path_); - // Returned value NULL/type (see below). + if (pp) + pmode = pattern_mode::ignore; + + // Returned value NULL/type and pattern (see below). // bool vnull (false); const value_type* vtype (nullptr); - - // If pair is not 0, then it is an index + 1 of the first half of - // the pair for which we are parsing the second halves, e.g., - // a@{b c d{e f} {}}. - // + optional rpat; // Buffer that is used to collect the complete name in case of an // unseparated variable expansion or eval context, e.g., foo$bar($baz)fox. @@ -2342,6 +2799,7 @@ namespace build2 // simple (i.e., just a string). // bool concat (false); + bool concat_quoted (false); name concat_data; auto concat_typed = [&vnull, &vtype, &concat, &concat_data, this] @@ -2420,6 +2878,12 @@ namespace build2 // Note that here we assume that, except for the first iterartion, // tt contains the type of the peeked token. + // Automatically reset the detect pattern mode to expand after the + // first element. + // + if (pmode == pattern_mode::detect && start != ns.size ()) + pmode = pattern_mode::expand; + // Return true if the next token which should be peeked at won't be part // of the name. // @@ -2461,7 +2925,11 @@ namespace build2 // parsing. // assert (!pre_parse_); + + bool quoted (concat_quoted); + concat = false; + concat_quoted = false; // If this is a result of typed concatenation, then don't inject. For // one we don't want any of the "interpretations" performed in the @@ -2514,13 +2982,17 @@ namespace build2 } } - // Replace the current token with our injection (after handling it - // we will peek at the current token again). + // Replace the current token with our injection (after handling it we + // will peek at the current token again). + // + // We don't know what exactly was quoted so approximating as partially + // mixed quoted. // tt = type::word; t = token (move (concat_data.value), true, - quote_type::unquoted, false, // @@ Not quite true. + quoted ? quote_type::mixed : quote_type::unquoted, + false, t.line, t.column); } else if (!first) @@ -2574,6 +3046,7 @@ namespace build2 } concat = true; + concat_quoted = t.qtype != quote_type::unquoted || concat_quoted; continue; } @@ -2659,11 +3132,56 @@ namespace build2 } count = parse_names_trailer ( - t, tt, ns, what, separators, pairn, *pp1, dp1, tp1); + t, tt, ns, pmode, what, separators, pairn, *pp1, dp1, tp1); tt = peek (); continue; } + // See if this is a wildcard pattern. + // + if (pmode != pattern_mode::ignore && + !*pp1 && // Cannot be project-qualified. + t.qtype == quote_type::unquoted && // Cannot be quoted. + ((dp != nullptr && dp->absolute ()) || pbase_ != nullptr) && + val.find_first_of ("*?") != string::npos) + { + // Resolve the target if there is one. If we fail, then this is not + // a pattern. + // + const target_type* tt (tp != nullptr && scope_ != nullptr + ? scope_->find_target_type (*tp) + : nullptr); + + if (tp == nullptr || tt != nullptr) + { + if (pmode == pattern_mode::expand) + { + count = expand_name_pattern (get_location (t), + names {name (move (val))}, + ns, + what, + pairn, + dp, tp, tt); + continue; + } + + // The goal of the detect mode is to assemble the "raw" list (the + // pattern itself plus inclusions/exclusions) that will be passed + // to parse_names_pattern(). So clear pair, directory, and type + // (they will be added during pattern expansion) and change the + // mode to ignore (to prevent any expansions in + // inclusions/exclusions). + // + pairn = 0; + dp = nullptr; + tp = nullptr; + pmode = pattern_mode::ignore; + rpat = tt; + + // Fall through. + } + } + // If we are a second half of a pair, add another first half // unless this is the first instance. // @@ -2745,7 +3263,7 @@ namespace build2 else if (tt == type::lparen) { expire_mode (); - values vs (parse_eval (t, tt)); //@@ OUT will parse @-pair and do well? + values vs (parse_eval (t, tt, pmode)); //@@ OUT will parse @-pair and do well? if (!pre_parse_) { @@ -2807,7 +3325,7 @@ namespace build2 // @@ Should we use (target/scope) qualification (of name) as the // context in which to call the function? // - values args (parse_eval (t, tt)); + values args (parse_eval (t, tt, pmode)); tt = peek (); if (pre_parse_) @@ -2840,7 +3358,7 @@ namespace build2 // loc = get_location (t); - values vs (parse_eval (t, tt)); + values vs (parse_eval (t, tt, pmode)); tt = peek (); if (pre_parse_) @@ -2860,11 +3378,11 @@ namespace build2 // assert (!pre_parse_); - // Should we accumulate? If the buffer is not empty, then - // we continue accumulating (the case where we are separated - // should have been handled by the injection code above). If - // the next token is a word or var expansion and it is not - // separated, then we need to start accumulating. + // Should we accumulate? If the buffer is not empty, then we continue + // accumulating (the case where we are separated should have been + // handled by the injection code above). If the next token is a word + // or var expansion and it is not separated, then we need to start + // accumulating. // if (concat || // Continue. !last_concat ()) // Start. @@ -2984,12 +3502,13 @@ namespace build2 } concat = true; + concat_quoted = quoted || concat_quoted; } else { // See if we should propagate the value NULL/type. We only do this // if this is the only expansion, that is, it is the first and the - // text token is not part of the name. + // next token is not part of the name. // if (first && last_token ()) { @@ -3002,82 +3521,13 @@ namespace build2 if (result->null || result->empty ()) continue; - // @@ Could move if lv is lv_storage (or even result_data; see - // untypify()). + // @@ Could move if nv is result_data; see untypify(). // - names lv_storage; - names_view lv (reverse (*result, lv_storage)); - - // Copy the names from the variable into the resulting name list - // while doing sensible things with the types and directories. - // - for (const name& n: lv) - { - const optional* pp1 (&pp); - const dir_path* dp1 (dp); - const string* tp1 (tp); + names nv_storage; + names_view nv (reverse (*result, nv_storage)); - optional p1; - if (n.proj) - { - if (!pp) - { - p1 = n.proj; - pp1 = &p1; - } - else - fail (loc) << "nested project name " << *n.proj << " in " - << what; - } - - dir_path d1; - if (!n.dir.empty ()) - { - if (dp != nullptr) - { - if (n.dir.absolute ()) - fail (loc) << "nested absolute directory " << n.dir - << " in " << what; - - d1 = *dp / n.dir; - dp1 = &d1; - } - else - dp1 = &n.dir; - } - - if (!n.type.empty ()) - { - if (tp == nullptr) - tp1 = &n.type; - else - fail (loc) << "nested type name " << n.type << " in " << what; - } - - // If we are a second half of a pair. - // - if (pairn != 0) - { - // Check that there are no nested pairs. - // - if (n.pair) - fail (loc) << "nested pair in " << what; - - // And add another first half unless this is the first instance. - // - if (pairn != ns.size ()) - ns.push_back (ns[pairn - 1]); - } - - ns.emplace_back (*pp1, - (dp1 != nullptr ? *dp1 : dir_path ()), - (tp1 != nullptr ? *tp1 : string ()), - n.value); - - ns.back ().pair = n.pair; - } - - count = lv.size (); + count = splice_names ( + loc, nv, move (nv_storage), ns, what, pairn, pp, dp, tp); } continue; @@ -3088,7 +3538,7 @@ namespace build2 if (tt == type::lcbrace) { count = parse_names_trailer ( - t, tt, ns, what, separators, pairn, pp, dp, tp); + t, tt, ns, pmode, what, separators, pairn, pp, dp, tp); tt = peek (); continue; } @@ -3180,7 +3630,7 @@ namespace build2 string ()); } - return make_pair (!vnull, vtype); + return parse_names_result {!vnull, vtype, rpat}; } void parser:: @@ -3294,6 +3744,7 @@ namespace build2 lexer_ = &l; target_ = nullptr; scope_ = root_ = scope::global_; + pbase_ = &work; // Use current working directory. // Turn on the value mode/pairs recognition with '@' as the pair separator // (e.g., src_root/@out_root/exe{foo bar}). @@ -3346,10 +3797,13 @@ namespace build2 const location l (get_location (t)); // Start of names. - // This call will parse the next chunk of output and produce - // zero or more names. + // This call will parse the next chunk of output and produce zero or + // more names. // - names ns (parse_names (t, tt, true)); + names ns (parse_names (t, tt, pattern_mode::expand, true)); + + if (ns.empty ()) // Can happen if pattern expansion. + fail (l) << "operation or target expected"; // What these names mean depends on what's next. If it is an // opening paren, then they are operation/meta-operation names. @@ -3536,6 +3990,7 @@ namespace build2 auto p (build2::switch_scope (*root_, d)); scope_ = &p.first; + pbase_ = scope_->src_path_; if (p.second != nullptr && p.second != root_) { diff --git a/build2/target b/build2/target index b5a8f78..ef1e34d 100644 --- a/build2/target +++ b/build2/target @@ -1602,6 +1602,10 @@ namespace build2 optional target_extension_fix (const target_key&, const scope&, bool); + template + bool + target_pattern_fix (const target_type&, const scope&, string&, bool); + // Get the extension from the variable or use the default if none set. If // the default is NULL, then return NULL. // @@ -1609,6 +1613,10 @@ namespace build2 optional target_extension_var (const target_key&, const scope&, bool); + template + bool + target_pattern_var (const target_type&, const scope&, string&, bool); + // Always return NULL extension. // optional diff --git a/build2/target-type b/build2/target-type index aa4bbf7..191c5d8 100644 --- a/build2/target-type +++ b/build2/target-type @@ -36,6 +36,10 @@ namespace build2 // (called for a target with the last argument false); see their // implementations for details. // + // If the pattern function is not NULL, then it is used to amend a pattern + // or match (reverse is false) and then, if the amendment call returned + // true, to reverse it in the resulting matches. + // struct target_type { const char* name; @@ -53,6 +57,8 @@ namespace build2 const scope&, bool search); + bool (*pattern) (const target_type&, const scope&, string&, bool reverse); + void (*print) (ostream&, const target_key&); const target* (*search) (const prerequisite_key&); diff --git a/build2/target.cxx b/build2/target.cxx index fa8378c..676dc6b 100644 --- a/build2/target.cxx +++ b/build2/target.cxx @@ -659,6 +659,7 @@ namespace build2 nullptr, nullptr, nullptr, + nullptr, &search_target, false }; @@ -670,6 +671,7 @@ namespace build2 nullptr, nullptr, nullptr, + nullptr, &search_target, false }; @@ -681,6 +683,7 @@ namespace build2 nullptr, nullptr, nullptr, + nullptr, &search_target, false }; @@ -714,6 +717,7 @@ namespace build2 &path_target::static_type, &file_factory, &target_extension_var, + &target_pattern_var, &target_print_1_ext_verb, // Print extension even at verbosity level 0. &search_file, false @@ -740,14 +744,15 @@ namespace build2 &target_factory, nullptr, // Extension not used. nullptr, + nullptr, &search_alias, false }; static const target* - search_dir (const prerequisite_key& pk) + dir_target_search (const prerequisite_key& pk) { - tracer trace ("search_dir"); + tracer trace ("dir_target_search"); // The first step is like in search_alias(): looks for an existing target. // @@ -831,14 +836,36 @@ namespace build2 info << "did you forget to include the corresponding buildfile?" << endf; } + static bool + dir_target_pattern (const target_type&, const scope&, string& v, bool r) + { + // Add/strip trailing directory separator unless already there. + // + bool d (path::traits::is_separator (v.back ())); + + if (r) + { + assert (d); + v.resize (v.size () - 1); + } + else if (!d) + { + v += path::traits::directory_separator; + return true; + } + + return false; + } + const target_type dir::static_type { "dir", &alias::static_type, &target_factory, - nullptr, // Extension not used. + nullptr, // Extension not used. + &dir_target_pattern, nullptr, - &search_dir, + &dir_target_search, false }; @@ -847,14 +874,15 @@ namespace build2 "fsdir", &target::static_type, &target_factory, - nullptr, // Extension not used. + nullptr, // Extension not used. + &dir_target_pattern, nullptr, &search_target, false }; static optional - exe_extension (const target_key&, const scope&, bool search) + exe_target_extension (const target_key&, const scope&, bool search) { // If we are searching for an executable that is not a target, then // use the build machine executable extension. Otherwise, if this is @@ -871,12 +899,38 @@ namespace build2 : nullopt; } +#ifdef _WIN32 + static bool + exe_target_pattern (const target_type&, const scope&, string& v, bool r) + { + size_t p (path::traits::find_extension (v)); + + if (r) + { + assert (p != string::npos); + v.resize (p); + } + else if (p == string::npos) + { + v += ".exe"; + return true; + } + + return false; + } +#endif + const target_type exe::static_type { "exe", &file::static_type, &target_factory, - &exe_extension, + &exe_target_extension, +#ifdef _WIN32 + &exe_target_pattern, +#else + nullptr, +#endif nullptr, &search_file, false @@ -904,12 +958,35 @@ namespace build2 return *tk.name == "buildfile" ? string () : "build"; } + static bool + buildfile_target_pattern (const target_type&, + const scope&, + string& v, + bool r) + { + size_t p (path::traits::find_extension (v)); + + if (r) + { + assert (p != string::npos); + v.resize (p); + } + else if (p == string::npos && v != "buildfile") + { + v += ".build"; + return true; + } + + return false; + } + const target_type buildfile::static_type { "build", &file::static_type, &buildfile_factory, &buildfile_target_extension, + &buildfile_target_pattern, nullptr, &search_file, false @@ -921,6 +998,7 @@ namespace build2 &file::static_type, &file_factory, // No extension by default. &target_extension_var, // Same as file. + &target_pattern_var, // Same as file. &target_print_1_ext_verb, // Same as file. &search_file, false @@ -945,6 +1023,7 @@ namespace build2 &doc::static_type, &man_factory, &target_extension_null, // Should be specified explicitly (see factory). + nullptr, &target_print_1_ext_verb, // Print extension even at verbosity level 0. &search_file, false @@ -958,6 +1037,7 @@ namespace build2 &man::static_type, &file_factory, &target_extension_fix, + &target_pattern_fix, &target_print_0_ext_verb, // Fixed extension, no use printing. &search_file, false diff --git a/build2/target.txx b/build2/target.txx index f885e80..dd087c0 100644 --- a/build2/target.txx +++ b/build2/target.txx @@ -46,13 +46,43 @@ namespace build2 return string (ext); } - template - optional - target_extension_var (const target_key& tk, const scope& s, bool) + template + bool + target_pattern_fix (const target_type&, const scope&, string& v, bool r) + { + size_t p (path::traits::find_extension (v)); + + if (r) + { + // If we get called to reverse then it means we've added the extension + // in the first place. So simply strip it. + // + assert (p != string::npos); + v.resize (p); + } + // + // We only add our extension if there isn't one already. + // + else if (p == string::npos) + { + v += '.'; + v += ext; + return true; + } + + return false; + } + + inline optional + target_extension_var (const target_type& tt, + const string& tn, + const scope& s, + const char* var, + const char* def) { // Include target type/pattern-specific variables. // - if (auto l = s.find (var_pool[var], tk)) + if (auto l = s.find (var_pool[var], tt, tn)) { // Help the user here and strip leading '.' from the extension. // @@ -62,4 +92,47 @@ namespace build2 return def != nullptr ? optional (def) : nullopt; } + + template + optional + target_extension_var (const target_key& tk, const scope& s, bool) + { + return target_extension_var (*tk.type, *tk.name, s, var, def); + } + + template + bool + target_pattern_var (const target_type& tt, const scope& s, string& v, bool r) + { + size_t p (path::traits::find_extension (v)); + + if (r) + { + // If we get called to reverse then it means we've added the extension + // in the first place. So simply strip it. + // + assert (p != string::npos); + v.resize (p); + } + // + // We only add our extension if there isn't one already. + // + else if (p == string::npos) + { + // Use empty name as a target since we only want target type/pattern- + // specific variables that match any target (e.g., '*' but not '*.txt'). + // + if (auto e = target_extension_var (tt, string (), s, var, def)) + { + if (!e->empty ()) // Don't add empty extension (means no extension). + { + v += '.'; + v += *e; + return true; + } + } + } + + return false; + } } diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx index 4e6759f..fd21a58 100644 --- a/build2/test/script/parser.cxx +++ b/build2/test/script/parser.cxx @@ -73,6 +73,8 @@ namespace build2 include_set_ = &ins; scope_ = nullptr; + //@@ PAT TODO: set pbase_? + // Start location of the implied script group is the beginning of // the file. End location -- end of the file. // @@ -968,7 +970,11 @@ namespace build2 if (tt != type::newline) { pre_parse_ = false; - args = parse_names (t, tt, false, "directive argument", nullptr); + args = parse_names (t, tt, + pattern_mode::expand, + false, + "directive argument", + nullptr); pre_parse_ = true; } @@ -1275,8 +1281,15 @@ namespace build2 // attributes_push (t, tt, true); + // @@ PAT: Should we expand patterns? Note that it will only be + // simple ones since we have disabled {}. Also, what would be the + // pattern base directory? + // return tt != type::newline && tt != type::semi - ? parse_value (t, tt, "variable value", nullptr) + ? parse_value (t, tt, + pattern_mode::ignore, + "variable value", + nullptr) : value (names ()); } @@ -2165,8 +2178,17 @@ namespace build2 // Note that we do it in the chunking mode to detect whether // anything in each chunk is quoted. // + // @@ PAT: should we support pattern expansion? This is even + // fuzzier than the variable case above. Though this is the + // shell semantics. Think what happens when we do rm *.txt? + // reset_quoted (t); - parse_names (t, tt, ns, true, "command line", nullptr); + parse_names (t, tt, + ns, + pattern_mode::ignore, + true, + "command line", + nullptr); if (pre_parse_) // Nothing else to do if we are pre-parsing. break; @@ -2404,7 +2426,11 @@ namespace build2 // next (t, tt); location l (get_location (t)); - names ns (parse_names (t, tt, true, "exit status", nullptr)); + names ns (parse_names (t, tt, + pattern_mode::ignore, + true, + "exit status", + nullptr)); unsigned long es (256); if (!pre_parse_) @@ -2570,8 +2596,15 @@ namespace build2 // Expand the line (can be blank). // + // @@ PAT: one could argue that if we do it in variables, then we + // should do it here as well. Though feels bizarre. + // names ns (tt != type::newline - ? parse_names (t, tt, false, "here-document line", nullptr) + ? parse_names (t, tt, + pattern_mode::ignore, + false, + "here-document line", + nullptr) : names ()); if (!pre_parse_) @@ -2819,6 +2852,8 @@ namespace build2 include_set_ = nullptr; scope_ = ≻ + //@@ PAT TODO: set pbase_? + exec_scope_body (); } diff --git a/build2/test/target.cxx b/build2/test/target.cxx index 3bf00c1..b6f9854 100644 --- a/build2/test/target.cxx +++ b/build2/test/target.cxx @@ -34,12 +34,35 @@ namespace build2 return *tk.name == "testscript" ? string () : "test"; } + static bool + testscript_target_pattern (const target_type&, + const scope&, + string& v, + bool r) + { + size_t p (path::traits::find_extension (v)); + + if (r) + { + assert (p != string::npos); + v.resize (p); + } + else if (p == string::npos && v != "testscript") + { + v += ".test"; + return true; + } + + return false; + } + const target_type testscript::static_type { "test", &file::static_type, &testscript_factory, &testscript_target_extension, + &testscript_target_pattern, nullptr, &search_file, false diff --git a/build2/utility.cxx b/build2/utility.cxx index 4bd4583..d26265d 100644 --- a/build2/utility.cxx +++ b/build2/utility.cxx @@ -533,7 +533,15 @@ namespace build2 // Figure out work and home directories. // - work = dir_path::current_directory (); + try + { + work = dir_path::current_directory (); + } + catch (const system_error& e) + { + fail << "invalid current working directory: " << e; + } + home = dir_path::home_directory (); } } diff --git a/doc/manual.cli b/doc/manual.cli index 8f6bebd..232b0fa 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -28,15 +28,23 @@ exe{hello}: {hxx cxx}{**} # All C++ header/source files. pattern = '*.txt' # Literal '*.txt'. \ -Pattern-based name generation is only performed in certain contexts. -Specifically, it is performed in prerequisite names and variable assignments -but not in target names (where it may be interpreted as a pattern at a high -level), for example (@@ TODO: clarify after implemented): +Pattern-based name generation is not performed in certain contexts. +Specifically, it is not performed in target names where it is interpreted +as a pattern for target type/pattern-specific variable assignments. For +example. \ -s = *.txt # Variable assignment. -./: cxx{*} # Prerequisite names. -cxx{*}: dist = false # Target pattern. +s = *.txt # Variable assignment (performed). +./: cxx{*} # Prerequisite names (performed). +cxx{*}: dist = false # Target pattern (not performed). +\ + +In contexts where it is performed, it can be inhibited with quoting, for +example: + +\ +pat = 'foo*bar' +./: cxx{'foo*bar'} \ The following characters are wildcards: @@ -58,20 +66,33 @@ it is matched just like \c{*} but in all the subdirectories, recursively. The itself. For example: \ -exe{hello}: cxx{**} # All C++ source files, including in subdirectories. +exe{hello}: cxx{**} # All C++ source files recursively. +\ + +A group-enclosed (\c{{\}}) pattern value may be followed by +inclusion/exclusion patterns/matches. A subsequent value is treated as an +inclusion if it starts with a plus sign (\c{+}) and as an exclusion if it +starts with a minus (\c{-}). A subsequent value that does not start with +either of these signs is illegal. For example: + +\ +exe{hello}: cxx{f* -foo} # Exclude foo if present. +exe{hello}: cxx{f* +foo} # Include foo if not present. +exe{hello}: cxx{f* -fo?} # Exclude foo and fox if present. +exe{hello}: cxx{f* +b* -foo -bar} # Exclude foo and bar if present. \ -A \c{{\}}-enclosed pattern value may be followed by inclusion/exclusion -patterns/matches. A subsequent value is treated as an inclusion if it starts -with a plus sign (\c{+}) and as an exclusion if it starts with a minus -(\c{-}). A subsequent value that does not start with either of these signs is -illegal. For example: +Inclusion and exclusion are applied in the order specified and only to the +result produced up to that point. The order of names in the result is +unspecified, however, it is guaranteed not to contain duplicates. The +pattern and the following inclusions/exclusions must be consistent with +regards to the type of filesystem entry they match. That is, they should +all match either files or directories. For example: \ -exe{hello}: cxx{f* -foo} # Exclude foo if present. -exe{hello}: cxx{f* +foo} # Include foo if not present. -exe{hello}: cxx{f* -fo?} # Exclude foo and fox if present. -exe{hello}: cxx{f* +b* -foo -bar} # Exclude foo and bar if present. +exe{hello}: cxx{f* -foo +*oo} # Exclusion has no effect. +exe{hello}: cxx{f* +*oo} # Ok, no duplicates. +./: {*/ -build} # Error: exclusion must match a directory. \ One common situation that calls for exclusions is auto-generated source @@ -91,14 +112,13 @@ If the name pattern includes an absolute directory, then the pattern match is performed in that directory and the generated names include absolute directories as well. Otherwise, the pattern match is performed in the \i{pattern base} directory. In buildfiles this is \c{src_base} while on the -command line \- the current working directory (@@ TODO: spec for testscript, -other contexts). In this case the generated names are relative to the base -directory. For example, assuming we have the \c{foo.cxx} and \c{b/bar.cxx} -source files: +command line \- the current working directory. In this case the generated +names are relative to the base directory. For example, assuming we have the +\c{foo.cxx} and \c{b/bar.cxx} source files: \ -exe{hello}: $src_base/cxx{**} # $src_base/cxx{foo} $src_base/b/cxx{bar} -exe{hello}: cxx{**} # cxx{foo} b/cxx{bar} +exe{hello}: $src_base/cxx{**} # $src_base/cxx{foo} $src_base/b/cxx{bar} +exe{hello}: cxx{**} # cxx{foo} b/cxx{bar} \ Pattern matching as well as inclusion/exclusion logic is target @@ -112,7 +132,7 @@ already end with one. Then the filesystem search is performed for matching directories. For example: \ -./: dir{* -build} # Search for */, exclude build/. +./: dir{* -build} # Search for */, exclude build/. \ For the \c{file{\}} and \c{file{\}}-based target types the default extension @@ -135,13 +155,15 @@ exe{hello}: {cxx}{* -foo -bar.cxx} \ The pattern match will first search for all the files matching the \c{*.cxx} -pattern in \c{src_base} and then exclude \c{foo.cxx} and \c{bar.cpp} from the -result. +pattern in \c{src_base} and then exclude \c{foo.cxx} and \c{bar.cxx} from the +result. Note also that target type-specific decorations are removed from the +result. So in the above example if the pattern match produces \c{baz.cxx}, +then the prerequisite name is \c{cxx{baz\}}, not \c{cxx{baz.cxx\}}. If the name generation cannot be performed because the base directory is unknown, target type is unknown, or the target type is not directory or file-based, then the name pattern is returned as is (that is, as an ordinary -name). +name). Project-qualified names are never considered to be patterns. \h1#grammar|Grammar| diff --git a/tests/buildfile b/tests/buildfile index 4b5f149..1938139 100644 --- a/tests/buildfile +++ b/tests/buildfile @@ -2,6 +2,6 @@ # copyright : Copyright (c) 2014-2017 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -./: directive/ eval/ expansion/ function/ search/ test/ value/ \ +./: directive/ eval/ expansion/ function/ name/ search/ test/ value/ \ variable/ file{common.test} diff --git a/tests/name/buildfile b/tests/name/buildfile new file mode 100644 index 0000000..8eaf31e --- /dev/null +++ b/tests/name/buildfile @@ -0,0 +1,5 @@ +# file : tests/name/buildfile +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +./: test{pattern} $b diff --git a/tests/name/pattern.test b/tests/name/pattern.test new file mode 100644 index 0000000..e9e1c45 --- /dev/null +++ b/tests/name/pattern.test @@ -0,0 +1,237 @@ +# file : tests/name/pattern.test +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +.include ../common.test + ++cat <=build/root.build +define txt: file +txt{*}: extension = txt +EOI + +$* <'print p%*.txt' >'p%*.txt' : project-simple +$* <'print p%{*.txt -x*}' >'p%*.txt p%-x*' : project-group + +$* <"print '*.txt'" >'*.txt' : quoted-single +$* <'print "*.txt"' >'*.txt' : quoted-double +$* <'*.txt' : quoted-expansion +pat = '*' +print "$(pat).txt" +EOI + +: detect +: +: Test pattern_mode parsing logic. +{ + : second-pattern + : + touch foo.txt; + $* <'print {foo *.txt}' >'foo foo.txt' + + : second-inclusion + : + touch foo.txt bar.txt; + $* <'print {f*.txt +b*.txt}' >'foo.txt bar.txt' +} + +: diagnostics +: +{ + : simple + : + $* <'print {*.txt file{foo}}' 2>>EOE != 0 + :1:8: error: invalid 'file{foo}' in name pattern + EOE + + : inclusion-exclusion-sign + : + $* <'print {*.txt foo}' 2>>EOE != 0 + :1:8: error: missing leading +/- in 'foo' name pattern + EOE + + : empty-inclusion-exclusion + : + $* <'print {*.txt -}' 2>>EOE != 0 + :1:8: error: empty name pattern + EOE + + : inconsistent-result + : + $* <'print {*.txt +foo/}' 2>>EOE != 0 + :1:8: error: inconsistent file/directory result in name pattern + EOE +} + +: basics +: +{ + touch foo.txt; + $* <'print *.txt' >'foo.txt' : simple-file + + mkdir foo; + $* <'print */' >/'foo/' : simple-dir + + touch foo.txt; + $* <'print {*.txt}' >'foo.txt' : group + + mkdir dir && touch dir/foo.txt; + $* <'print dir/{*.txt}' >'dir/foo.txt' : dir + + touch foo.txt; + $* <'print file{*.txt}' >'file{foo.txt}' : type + + touch foo.txt; + $* <'print x@{*.txt}' >'x@foo.txt' : pair + + touch bar.txt; + $* <'print x@dir/file{f*.txt}' >'' : empty + + mkdir dir && touch dir/foo.txt; + $* <'print **.txt' >/'dir/foo.txt' : recursive + + mkdir dir && touch dir/foo.txt; + $* <'print d*/*.txt' >/'dir/foo.txt' : multi-pattern + + touch foo.txt bar.txt; + $* <'print {*.txt -bar.txt}' >'foo.txt' : exclude-match + + touch foo.txt bar.txt; + $* <'print {*.txt -b*.txt}' >'foo.txt' : exclude-pattern + + touch bar.txt; + $* <'print {f*.txt +bar.txt}' >'bar.txt' : include-match + + touch bar.txt; + $* <'print {f*.txt +b*.txt}' >'bar.txt' : include-pattern + + touch foo.txt fox.txt; + $* <'print {*.txt -f*.txt +*x.txt}' >'fox.txt' : include-exclude-order +} + +: target-type +: +: Test target type-specific pattern amendment logic. +{ + : append-extension + : + touch foo.txt bar.txt; + $* <'print txt{* -bar}' >'txt{foo}' + + : existing-extension + : + touch foo.txt bar.txt; + $* <'print txt{*.txt -bar.txt}' >'txt{foo.txt}' + + : append-slash + : + mkdir foo bar; + $* <'print dir{* -bar}' >/'dir{foo/}' + + : existing-slash + : + mkdir foo bar; + $* <'print dir{*/ -bar/}' >/'dir{foo/}' +} + +: expansion +: +: Test interaction with expansion/concatenation/re-parse. +{ + # Do we want to recognize patterns in non-concatenating expansion? + # + # pat = '*.txt' + # print $pat + # + # While this case is probably better rewritten as (i.e., move pattern search + # to variable assignment): + # + # pat = *.txt + # print $pat + # + # One may also want to do something like this: + # + # pat = '*.txt' + # print dir1/{$pat} + # print dir2/{$pat} + # + # Note that if we make it work, escaping this case will be pretty hairy: + # + # filters = --include '*.txt' --exclude '*.obj' + # options += $filters + + #: pattern-via-expansion + #: + #$* <'<*.txt>' + #pat = '*.txt' + #print $pat + #EOI + + #: pattern-via-expansion-list + #: + #$* <'<*.hxx+*.txt>' + #pats = '*.hxx' '+*.txt' + #print {$pats} + #EOI + + #: pattern-via-expansion-type + #: + #$* <'<*.txt>' + #pat = '*.txt' + #print txt{$pat} + #EOI + + #: pattern-via-expansion-dir + #: + #$* <'<*.txt>' + #pat = '*.txt' + #print dir/{$pat} + #EOI + + : pattern-via-concat + : + touch foo.txt; + $* <'foo.txt' + ext = txt + print *.$ext + EOI + + : pattern-via-concat-expansion + : + touch foo.txt; + $* <'foo.txt' + pat = 'f*' + ext = txt + print $pat.$ext + EOI +} + +: command-line +: +: Test pattern expansion on the command line. +{ + : variable + : + { + mkdir dir; + $* <'print $d' 'd=*/' >/'dir/' : dir + + mkdir dir; + $* <'print $d' 'd=dir{*}' >/'dir{dir/}' : dir-type + + touch foo.txt; + $* <'print $f' 'f=*.txt' >'foo.txt' : feil + } + + : buildspec + : + { + test.arguments = + test.options += --buildfile buildfile + + mkdir dir && cat <'./:' >=dir/buildfile; + $* '*/' : dir + + mkdir dir dir1 && cat <'./:' >=dir/buildfile; + $* 'update(dir{* -dir1})' : dir-type + } +} -- cgit v1.1