From 20e49b4e63779abc0e25bec4c74399a83ec8a83c Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Mon, 12 Feb 2024 09:34:50 +0200 Subject: Add ability to specify recipes in separate files This can now be achieved with the new `recipe` directive: recipe Note that similar to the use of if-else and switch directives with recipes, this directive requires explicit % recipe header. For example, instead of: file{foo.output}: {{ echo 'hello' >$path($>) }} We can now write: file{foo.output}: % recipe buildscript hello.buildscript With hello.buildscript containing: echo 'hello' >$path($>) Similarly, for C++ recipes (this time for a pattern), instead of: [rule_name=hello] file{~'/(.+)\.output/'}: % update clean {{ c++ 1 -- -- ... }} We can now write: [rule_name=hello] file{~'/(.+)\.output/'}: % update clean recipe c++ hello.cxx With hello.cxx containing: // c++ 1 -- -- ... Relative paths are resolved using the buildfile directory that contains the `recipe` directive as a base. Note also that this mechanism can be used in exported buildfiles with recipe files placed into build/export/ together with buildfiles. --- libbuild2/parser.cxx | 563 +++++++++++++++++++++++++++---------- libbuild2/parser.hxx | 15 +- libbuild2/target.cxx | 49 ++++ libbuild2/target.hxx | 16 ++ tests/dependency/recipe/testscript | 2 +- 5 files changed, 495 insertions(+), 150 deletions(-) diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index 7cc01e3..6ccae8a 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -311,7 +311,7 @@ namespace build2 : auto_project_env ()); const buildfile* bf (enter && path_->path != nullptr - ? &enter_buildfile (*path_->path) + ? &enter_buildfile (*path_->path) : nullptr); token t; type tt; @@ -345,7 +345,7 @@ namespace build2 ? out_src (name.path->directory (), rs) : dir_path ()); - enter_buildfile (*name.path, move (out)); + enter_buildfile (*name.path, move (out)); } parse_buildfile (is, name, &gs, ts, nullptr, nullptr, false /* enter */); @@ -617,6 +617,12 @@ namespace build2 { f = &parser::parse_config_environment; } + else if (n == "recipe") + { + // Valid only after recipe header (%). + // + fail (t) << n << " directive without % recipe header"; + } if (f != nullptr) { @@ -1917,7 +1923,7 @@ namespace build2 // Parse a recipe chain. // // % [] [] - // [if|if!|switch ...] + // [if|if!|switch|recipe ...] // {{ [ ...] // ... // }} @@ -1994,7 +2000,131 @@ namespace build2 attributes& as; buildspec& bs; const location& bsloc; - } d {ttype, name, recipes, first, clean, i, as, bs, bsloc}; + function parse_trailer; + } d {ttype, name, recipes, first, clean, i, as, bs, bsloc, {}}; + + d.parse_trailer = [this, &d] (string&& text) + { + if (d.first) + { + adhoc_rule& ar (*d.recipes.back ()); + + // Translate each buildspec entry into action and add it to the + // recipe entry. + // + const location& l (d.bsloc); + + for (metaopspec& m: d.bs) + { + meta_operation_id mi (ctx->meta_operation_table.find (m.name)); + + if (mi == 0) + fail (l) << "unknown meta-operation " << m.name; + + const meta_operation_info* mf ( + root_->root_extra->meta_operations[mi]); + + if (mf == nullptr) + fail (l) << "project " << *root_ << " does not support meta-" + << "operation " << ctx->meta_operation_table[mi].name; + + for (opspec& o: m) + { + operation_id oi; + if (o.name.empty ()) + { + if (mf->operation_pre == nullptr) + oi = update_id; + else + // Calling operation_pre() to translate doesn't feel + // appropriate here. + // + fail (l) << "default operation in recipe action" << endf; + } + else + oi = ctx->operation_table.find (o.name); + + if (oi == 0) + fail (l) << "unknown operation " << o.name; + + const operation_info* of (root_->root_extra->operations[oi]); + + if (of == nullptr) + fail (l) << "project " << *root_ << " does not support " + << "operation " << ctx->operation_table[oi]; + + // Note: for now always inner (see match_rule_impl() for + // details). + // + action a (mi, oi); + + // Check for duplicates (local). + // + if (find_if ( + d.recipes.begin (), d.recipes.end (), + [a] (const shared_ptr& r) + { + auto& as (r->actions); + return find (as.begin (), as.end (), a) != as.end (); + }) != d.recipes.end ()) + { + fail (l) << "duplicate " << mf->name << '(' << of->name + << ") recipe"; + } + + ar.actions.push_back (a); + } + } + + // Set the recipe text. + // + if (ar.recipe_text ( + *scope_, + d.ttype != nullptr ? *d.ttype : target_->type (), + move (text), + d.as)) + d.clean = true; + + // Verify we have no unhandled attributes. + // + for (attribute& a: d.as) + fail (d.as.loc) << "unknown recipe attribute " << a << endf; + } + + // Copy the recipe over to the target verifying there are no + // duplicates (global). + // + if (target_ != nullptr) + { + const shared_ptr& r (d.recipes[d.i]); + + for (const shared_ptr& er: target_->adhoc_recipes) + { + auto& as (er->actions); + + for (action a: r->actions) + { + if (find (as.begin (), as.end (), a) != as.end ()) + { + const meta_operation_info* mf ( + root_->root_extra->meta_operations[a.meta_operation ()]); + + const operation_info* of ( + root_->root_extra->operations[a.operation ()]); + + fail (d.bsloc) + << "duplicate " << mf->name << '(' << of->name + << ") recipe for target " << *target_; + } + } + } + + target_->adhoc_recipes.push_back (r); + + // Note that "registration" of configure_* and dist_* actions + // (similar to ad hoc rules) is provided by match_adhoc_recipe(). + } + }; // Note that this function must be called at most once per iteration. // @@ -2037,7 +2167,7 @@ namespace build2 // to rule_name. shared_ptr ar; - if (!lang) + if (!lang || icasecmp (*lang, "buildscript") == 0) { // Buildscript // @@ -2133,133 +2263,198 @@ namespace build2 } if (!skip) - { - if (d.first) - { - adhoc_rule& ar (*d.recipes.back ()); - - // Translate each buildspec entry into action and add it to the - // recipe entry. - // - const location& l (d.bsloc); - - for (metaopspec& m: d.bs) - { - meta_operation_id mi (ctx->meta_operation_table.find (m.name)); + d.parse_trailer (move (t.value)); - if (mi == 0) - fail (l) << "unknown meta-operation " << m.name; + next (t, tt); + assert (tt == type::multi_rcbrace); - const meta_operation_info* mf ( - root_->root_extra->meta_operations[mi]); + next (t, tt); // Newline. + next_after_newline (t, tt, token (t)); // Should be on its own line. + }; - if (mf == nullptr) - fail (l) << "project " << *root_ << " does not support meta-" - << "operation " << ctx->meta_operation_table[mi].name; + auto parse_recipe_directive = [this, &d] (token& t, type& tt, + const string&) + { + // Parse recipe directive: + // + // recipe + // + // Note that here is not optional. + // + // @@ We could guess from the extension. - for (opspec& o: m) - { - operation_id oi; - if (o.name.empty ()) - { - if (mf->operation_pre == nullptr) - oi = update_id; - else - // Calling operation_pre() to translate doesn't feel - // appropriate here. - // - fail (l) << "default operation in recipe action" << endf; - } - else - oi = ctx->operation_table.find (o.name); + // Use value mode to minimize the number of special characters. + // + mode (lexer_mode::value, '@'); - if (oi == 0) - fail (l) << "unknown operation " << o.name; + // Parse . + // + if (next (t, tt) != type::word) + fail (t) << "expected recipe language instead of " << t; - const operation_info* of (root_->root_extra->operations[oi]); + location lloc (get_location (t)); + string lang (t.value); + next (t, tt); - if (of == nullptr) - fail (l) << "project " << *root_ << " does not support " - << "operation " << ctx->operation_table[oi]; + // Parse as names to get variable expansion, etc. + // + location nloc (get_location (t)); + names ns (parse_names (t, tt, pattern_mode::ignore, "file name")); - // Note: for now always inner (see match_rule_impl() for - // details). - // - action a (mi, oi); + path file; + try + { + file = convert (move (ns)); + } + catch (const invalid_argument& e) + { + fail (nloc) << "invalid recipe file path: " << e; + } - // Check for duplicates (local). - // - if (find_if ( - d.recipes.begin (), d.recipes.end (), - [a] (const shared_ptr& r) - { - auto& as (r->actions); - return find (as.begin (), as.end (), a) != as.end (); - }) != d.recipes.end ()) - { - fail (l) << "duplicate " << mf->name << '(' << of->name - << ") recipe"; - } + string text; + if (d.first) + { + // Source relative to the buildfile rather than src scope. In + // particular, this make sourcing from exported buildfiles work. + // + if (file.relative () && path_->path != nullptr) + { + // Note: all sourced/included/imported paths are absolute and + // normalized. + // + file = path_->path->directory () / file; + } - ar.actions.push_back (a); - } - } + file.normalize (); - // Set the recipe text. - // - if (ar.recipe_text ( - *scope_, - d.ttype != nullptr ? *d.ttype : target_->type (), - move (t.value), - d.as)) - d.clean = true; - - // Verify we have no unhandled attributes. - // - for (attribute& a: d.as) - fail (d.as.loc) << "unknown recipe attribute " << a << endf; + try + { + ifdstream ifs (file); + text = ifs.read_text (); + } + catch (const io_error& e) + { + fail (nloc) << "unable to read recipe file " << file << ": " << e; } - // Copy the recipe over to the target verifying there are no - // duplicates (global). - // - if (target_ != nullptr) + shared_ptr ar; { - const shared_ptr& r (d.recipes[d.i]); + // This is expected to be the location of the opening multi-curly + // with the recipe body starting from the following line. So we + // need to fudge the line number a bit. + // + location loc (file, 0, 1); - for (const shared_ptr& er: target_->adhoc_recipes) + if (icasecmp (lang, "buildscript") == 0) { - auto& as (er->actions); + // Buildscript + // + ar.reset ( + new adhoc_buildscript_rule ( + d.name.empty () ? "" : d.name, + loc, + 2)); // Use `{{` and `}}` for dump. - for (action a: r->actions) + // Enter as buildfile-like so that it gets automatically + // distributed. Note: must be consistent with build/export/ + // handling in process_default_target(). + // + enter_buildfile (file); + } + else if (icasecmp (lang, "c++") == 0) + { + // C++ + // + // We expect to find a C++ comment line with version and + // optional fragment separator before the first non-comment, + // non-blank line: + // + // // c++ [] + // + string s; + location sloc (file, 1, 1); { - if (find (as.begin (), as.end (), a) != as.end ()) + // @@ Line is inaccurate since we skip consecutive newlines! + // + size_t b (0), e (0); + for (; next_word (text, b, e, '\n', '\r'); sloc.line++) { - const meta_operation_info* mf ( - root_->root_extra->meta_operations[a.meta_operation ()]); + s.assign (text, b, e - b); - const operation_info* of ( - root_->root_extra->operations[a.operation ()]); + if (!trim (s).empty ()) + { + if (icasecmp (s, "// c++ ", 7) == 0) + break; + + if (s[0] != '/' || s[1] != '/') + { + b = e; + break; + } + } + } + + if (b == e) + fail (sloc) << "no '// c++ []' line"; + } + + uint64_t ver; + optional sep; + { + size_t b (7), e (7); + if (next_word (s, b, e, ' ', '\t') == 0) + fail (sloc) << "missing c++ recipe version" << endf; + + try + { + ver = convert (build2::name (string (s, b, e - b))); + } + catch (const invalid_argument& e) + { + fail (sloc) << "invalid c++ recipe version: " << e << endf; + } - fail (d.bsloc) - << "duplicate " << mf->name << '(' << of->name - << ") recipe for target " << *target_; + if (next_word (s, b, e, ' ', '\t') != 0) + { + sep = string (s, b, e - b); + + if (next_word (s, b, e, ' ', '\t') != 0) + fail (sloc) << "junk after fragment separator"; } } - } - target_->adhoc_recipes.push_back (r); + ar.reset ( + new adhoc_cxx_rule ( + d.name.empty () ? "" : d.name, + loc, + 2, // Use `{{` and `}}` for dump. + ver, + move (sep))); - // Note that "registration" of configure_* and dist_* actions - // (similar to ad hoc rules) is provided by match_adhoc_recipe(). + // Enter as buildfile-like so that it gets automatically + // distributed. Note: must be consistent with build/export/ + // handling in process_default_target(). + // + // While ideally we would want to use the cxx{} target type, + // it's defined in a seperate build system module (which may not + // even be loaded by this project, so even runtime lookup won't + // work). So we use file{} instead. + // + enter_buildfile (file); + } + else + fail (lloc) << "unknown recipe language '" << lang << "'"; } + + assert (d.recipes[d.i] == nullptr); + d.recipes[d.i] = move (ar); } + else + assert (d.recipes[d.i] != nullptr); - next (t, tt); - assert (tt == type::multi_rcbrace); + d.parse_trailer (move (text)); - next (t, tt); // Newline. - next_after_newline (t, tt, token (t)); // Should be on its own line. + next_after_newline (t, tt); }; bsloc = get_location (t); // Fallback location. @@ -2319,7 +2514,7 @@ namespace build2 expire_mode (); next_after_newline (t, tt, "recipe action"); - // See if this is if-else or switch. + // See if this is if-else/switch or `recipe`. // // We want the keyword test similar to parse_clause() but we cannot do // it if replaying. So we skip it with understanding that if it's not @@ -2337,12 +2532,19 @@ namespace build2 if (n == "if" || n == "if!") { - parse_if_else (t, tt, true /* multi */, parse_block); + parse_if_else (t, tt, true /* multi */, + parse_block, parse_recipe_directive); continue; } else if (n == "switch") { - parse_switch (t, tt, true /* multi */, parse_block); + parse_switch (t, tt, true /* multi */, + parse_block, parse_recipe_directive); + continue; + } + else if (n == "recipe") + { + parse_recipe_directive (t, tt, "" /* kind */); continue; } @@ -2350,7 +2552,7 @@ namespace build2 } if (tt != type::multi_lcbrace) - fail (t) << "expected recipe block instead of " << t; + fail (t) << "expected recipe block or 'recipe' instead of " << t; // Fall through. } @@ -3188,7 +3390,7 @@ namespace build2 l5 ([&]{trace (loc) << "entering " << in;}); const buildfile* bf (in.path != nullptr - ? &enter_buildfile (*in.path) + ? &enter_buildfile (*in.path) : nullptr); const path_name* op (path_); @@ -4680,14 +4882,17 @@ namespace build2 [this] (token& t, type& tt, bool s, const string& k) { return parse_clause_block (t, tt, s, k); - }); + }, + {}); } void parser:: parse_if_else (token& t, type& tt, bool multi, const function& parse_block) + token&, type&, bool, const string&)>& parse_block, + const function& parse_recipe_directive) { // Handle the whole if-else chain. See tests/if-else. // @@ -4772,35 +4977,65 @@ namespace build2 parse_block (t, tt, !take, k); taken = taken || take; } - else if (!multi) // No lines in multi-curly if-else. + else { - if (tt == type::multi_lcbrace) - fail (t) << "expected " << k << "-line instead of " << t << - info << "did you forget to specify % recipe header?"; - - if (take) + // The only valid line in multi-curly if-else is `recipe`. + // + if (multi) { - if (!parse_clause (t, tt, true)) - fail (t) << "expected " << k << "-line instead of " << t; + // Note that we cannot do the keyword test if we are replaying. So + // we skip it with the understanding that if it's not a keywords, + // then we wouldn't have gotten here on the replay. + // + if (tt == type::word && + (replay_ == replay::play || keyword (t)) && + t.value == "recipe") + { + if (take) + { + parse_recipe_directive (t, tt, k); + taken = true; + } + else + { + skip_line (t, tt); - taken = true; + if (tt == type::newline) + next (t, tt); + } + } + else + fail (t) << "expected " << k << "-block or 'recipe' instead of " + << t; } else { - skip_line (t, tt); + if (tt == type::multi_lcbrace) + fail (t) << "expected " << k << "-line instead of " << t << + info << "did you forget to specify % recipe header?"; - if (tt == type::newline) - next (t, tt); + if (take) + { + if (!parse_clause (t, tt, true)) + fail (t) << "expected " << k << "-line instead of " << t; + + taken = true; + } + else + { + skip_line (t, tt); + + if (tt == type::newline) + next (t, tt); + } } } - else - fail (t) << "expected " << k << "-block instead of " << t; // See if we have another el* keyword. // // Note that we cannot do the keyword test if we are replaying. So we // skip it with the understanding that if it's not a keywords, then we - // wouldn't have gotten here on the reply (see parse_recipe() for + // wouldn't have gotten here on the replay (see parse_recipe() for // details). // if (k != "else" && @@ -4831,14 +5066,17 @@ namespace build2 [this] (token& t, type& tt, bool s, const string& k) { return parse_clause_block (t, tt, s, k); - }); + }, + {}); } void parser:: parse_switch (token& t, type& tt, bool multi, const function& parse_block) + token&, type&, bool, const string&)>& parse_block, + const function& parse_recipe_directive) { // switch [: []] [, ...] // { @@ -4933,7 +5171,7 @@ namespace build2 { // Note that we cannot do the keyword test if we are replaying. So we // skip it with the understanding that if it's not a keywords, then we - // wouldn't have gotten here on the reply (see parse_recipe() for + // wouldn't have gotten here on the replay (see parse_recipe() for // details). Note that this appears to mean that replay cannot be used // if we allow lines, only blocks. Consider: // @@ -5139,25 +5377,49 @@ namespace build2 parse_block (t, tt, !take, k); taken = taken || take; } - else if (!multi) // No lines in multi-curly if-else. + else { - if (take) + if (multi) { - if (!parse_clause (t, tt, true)) - fail (t) << "expected " << k << "-line instead of " << t; + if (tt == type::word && + (replay_ == replay::play || keyword (t)) && + t.value == "recipe") + { + if (take) + { + parse_recipe_directive (t, tt, k); + taken = true; + } + else + { + skip_line (t, tt); - taken = true; + if (tt == type::newline) + next (t, tt); + } + } + else + fail (t) << "expected " << k << "-block or 'recipe' instead of " + << t; } else { - skip_line (t, tt); + if (take) + { + if (!parse_clause (t, tt, true)) + fail (t) << "expected " << k << "-line instead of " << t; - if (tt == type::newline) - next (t, tt); + taken = true; + } + else + { + skip_line (t, tt); + + if (tt == type::newline) + next (t, tt); + } } } - else - fail (t) << "expected " << k << "-block instead of " << t; } if (tt != type::rcbrace) @@ -9642,7 +9904,17 @@ namespace build2 { const path& n (e.path ()); - if (n.extension () == build_ext) + // Besides the buildfile also export buildscript and C++ files + // that are used to provide recipe implementations (see + // parse_recipe() for details). + // + string e (n.extension ()); + if (const target_type* tt = ( + e == build_ext ? &buildfile::static_type : + e == "buildscript" ? &buildscript::static_type : + e == "cxx" || + e == "cpp" || + e == "cc" ? &file::static_type : nullptr)) { // Enter as if found by search_existing_file(). Note that // entering it as real would cause file_rule not to match @@ -9652,13 +9924,13 @@ namespace build2 // example, if already imported). // const target& bf ( - ctx->targets.insert (buildfile::static_type, + ctx->targets.insert (*tt, d, (root_->out_eq_src () ? dir_path () : out_src (d, *root_)), n.base ().string (), - build_ext, + move (e), target_decl::prereq_file, trace).first); @@ -9696,7 +9968,7 @@ namespace build2 // subdirectories inside export/. Essentially, we are arranging for // this: // - // build/export/buildfile{*}: + // build/export/file{*}: // { // install = buildfile/ // install.subdirs = true @@ -9705,7 +9977,7 @@ namespace build2 if (cast_false (root_->vars["install.loaded"])) { enter_scope es (*this, dir_path (export_dir)); - auto& vars (scope_->target_vars[buildfile::static_type]["*"]); + auto& vars (scope_->target_vars[file::static_type]["*"]); // @@ TODO: get cached variables from the module once we have one. // @@ -9728,7 +10000,8 @@ namespace build2 } } - const buildfile& parser:: + template + const T& parser:: enter_buildfile (const path& p, optional out) { tracer trace ("parser::enter_buildfile", &path_); @@ -9748,7 +10021,7 @@ namespace build2 o = out_src (d, *root_); } - return ctx->targets.insert ( + return ctx->targets.insert ( move (d), move (o), p.leaf ().base ().string (), diff --git a/libbuild2/parser.hxx b/libbuild2/parser.hxx index b756984..0645b5a 100644 --- a/libbuild2/parser.hxx +++ b/libbuild2/parser.hxx @@ -265,7 +265,9 @@ namespace build2 parse_if_else (token&, token_type&, bool, const function&); + token&, token_type&, bool, const string&)>&, + const function&); void parse_switch (token&, token_type&); @@ -274,7 +276,9 @@ namespace build2 parse_switch (token&, token_type&, bool, const function&); + token&, token_type&, bool, const string&)>&, + const function&); void parse_for (token&, token_type&); @@ -617,9 +621,12 @@ namespace build2 void process_default_target (token&, const buildfile*); - // Enter buildfile as a target. + private: + // Enter buildfile or buildfile-file like file (e.g., a recipe file) as a + // target. // - const buildfile& + template + const T& enter_buildfile (const path&, optional out = nullopt); // Lexer. diff --git a/libbuild2/target.cxx b/libbuild2/target.cxx index a70830e..2a134a4 100644 --- a/libbuild2/target.cxx +++ b/libbuild2/target.cxx @@ -1775,6 +1775,55 @@ namespace build2 target_type::flag::none }; + static const char* + buildscript_target_extension (const target_key& tk, const scope*) + { + // If the name is special 'buildscript', then there is no extension, + // otherwise it is .buildscript. + // + return *tk.name == "buildscript" ? "" : "buildscript"; + } + + static bool + buildscript_target_pattern (const target_type&, + const scope&, + string& v, + optional& e, + const location& l, + bool r) + { + if (r) + { + assert (e); + e = nullopt; + } + else + { + e = target::split_name (v, l); + + if (!e && v != "buildscript") + { + e = "buildscript"; + return true; + } + } + + return false; + } + + const target_type buildscript::static_type + { + "buildscript", + &file::static_type, + &target_factory, + &buildscript_target_extension, + nullptr, /* default_extension */ + &buildscript_target_pattern, + nullptr, + &file_search, + target_type::flag::none + }; + const target_type doc::static_type { "doc", diff --git a/libbuild2/target.hxx b/libbuild2/target.hxx index 41bf095..b97d562 100644 --- a/libbuild2/target.hxx +++ b/libbuild2/target.hxx @@ -2501,6 +2501,22 @@ namespace build2 static const target_type static_type; }; + // This target type is primarily used for files mentioned in the `recipe` + // directive. + // + class LIBBUILD2_SYMEXPORT buildscript: public file + { + public: + buildscript (context& c, dir_path d, dir_path o, string n) + : file (c, move (d), move (o), move (n)) + { + dynamic_type = &static_type; + } + + public: + static const target_type static_type; + }; + // Common documentation file target. // class LIBBUILD2_SYMEXPORT doc: public file diff --git a/tests/dependency/recipe/testscript b/tests/dependency/recipe/testscript index f43111e..a581724 100644 --- a/tests/dependency/recipe/testscript +++ b/tests/dependency/recipe/testscript @@ -406,7 +406,7 @@ alias{x}: echo } EOI -:3:1: error: expected recipe block instead of '{' +:3:1: error: expected recipe block or 'recipe' instead of '{' EOE : duplicate-action-single -- cgit v1.1