From 8c54405d78b87b8756106eceec0a53ef0225d05e Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Fri, 24 Mar 2023 20:32:25 +0300 Subject: Add support for package build config worker steps and steps enabling/disabling --- bbot/worker/worker.cxx | 2611 ++++++++++++++++++++++++------------------ doc/manual.cli | 133 ++- tests/integration/testscript | 14 +- 3 files changed, 1622 insertions(+), 1136 deletions(-) diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index 6c96b5b..f1e5e8c 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -151,6 +151,9 @@ catch (const system_error& e) // Step IDs. // +// NOTE: keep ids ordered according to the sequence of steps and remember to +// update unreachable breakpoint checks if changing anything here. +// enum class step_id { // Common fallbacks for bpkg_*_create/b_test_installed_create and @@ -302,6 +305,12 @@ static const strings step_id_str { "end"}; +static inline const string& +to_string (step_id s) +{ + return step_id_str[static_cast (s)]; +} + using std::regex; namespace regex_constants = std::regex_constants; using regexes = vector; @@ -390,7 +399,7 @@ run_cmd (step_id step, auto prompt_step = [step, &t, &log, &bkp_step, &prompt] () { - const string& sid (step_id_str[static_cast (step)]); + const string& sid (to_string (step)); // Prompt the user if the breakpoint is reached. // @@ -789,7 +798,7 @@ upload_manifest (tracer& trace, } } -static const string worker_checksum ("4"); // Logic version. +static const string worker_checksum ("5"); // Logic version. static int bbot:: build (size_t argc, const char* argv[]) @@ -915,53 +924,105 @@ build (size_t argc, const char* argv[]) } // Split the argument into prefix (empty if not present) and unquoted - // value. Return nullopt if the prefix is invalid. + // value (absent if not present) and determine the step status. If the + // prefix is present and is prepended with the '+'/'-' character, then the + // respective step needs to be enabled/disabled. Return nullopt if the + // prefix is invalid. // - auto parse_arg = [] (const string& a) -> optional> + // Note that arguments with absent values are normally used to + // enable/disable steps and are omitted from the command lines. + // + struct argument + { + string prefix; + + // Absent if the argument value is an empty unquoted string. + // + optional value; + + // True - enable, false - disable, nullopt - neutral. + // + optional step_status; + }; + + auto parse_arg = [] (const string& a) -> optional { size_t p (a.find_first_of (":=\"'")); + auto value = [] (const string& v) + { + return !v.empty () ? unquote (v) : optional (); + }; + if (p == string::npos || a[p] != ':') // No prefix. - return make_pair (string (), unquote (a)); + return argument {string (), value (a), nullopt}; + + string prefix (a, 0, p); + + optional enable; + if (prefix[0] == '+' || prefix[0] == '-') + { + enable = (prefix[0] == '+'); + + prefix.erase (0, 1); + + if (prefix != "bpkg.update" && + prefix != "bpkg.test" && + prefix != "bpkg.test-separate.update" && + prefix != "bpkg.test-separate.test" && + prefix != "bpkg.install" && + prefix != "b.test-installed.test" && + prefix != "bpkg.test-separate-installed.update" && + prefix != "bpkg.test-separate-installed.test") + { + return nullopt; // Prefix is invalid. + } + } for (const string& id: step_id_str) { - if (a.compare (0, p, id, 0, p) == 0 && - (id.size () == p || (id.size () > p && id[p] == '.'))) - return make_pair (a.substr (0, p), unquote (a.substr (p + 1))); + size_t n (prefix.size ()); + if (id.compare (0, n, prefix) == 0 && + (id.size () == n || (id.size () > n && id[n] == '.'))) + return argument {move (prefix), value (a.substr (p + 1)), enable}; } return nullopt; // Prefix is invalid. }; - // Parse configuration arguments. Report failures to the bbot controller. + // Keep track of explicitly enabled/disabled steps. // - std::multimap tgt_args; + std::map step_statuses; - for (const string& c: tm.target_config) + // Return true if the step is explicitly enabled via a +:[] + // environment/configuration argument. + // + // @@ TMP Use it for bpkg.bindist step. + // +#if 0 + auto step_enabled = [&step_statuses] (step_id step) -> bool { - optional> v (parse_arg (c)); - - if (!v) - { - rm.status |= result_status::abort; - l3 ([&]{trace << "invalid configuration argument prefix in " - << "'" << c << "'";}); - break; - } - - if (v->second[0] != '-' && v->second.find ('=') == string::npos) - { - rm.status |= result_status::abort; - l3 ([&]{trace << "invalid configuration argument '" << c << "'";}); - break; - } + auto i (step_statuses.find (to_string (step))); + return i != step_statuses.end () && i->second; + }; +#endif - tgt_args.emplace (move (*v)); - } + // Return true if the step is explicitly disabled via a -:[] + // environment/configuration argument. + // + auto step_disabled = [&step_statuses] (step_id step) -> bool + { + auto i (step_statuses.find (to_string (step))); + return i != step_statuses.end () && !i->second; + }; - if (!rm.status) - break; + // Parse the environment, target configuration, and build package + // configuration arguments. + // + // NOTE: keep this parsing order intact so that, for example, a build + // package configuration argument can override step status specified + // by a target configuration argument. + // // Parse environment arguments. // @@ -971,147 +1032,73 @@ build (size_t argc, const char* argv[]) for (size_t i (1); i != argc; ++i) { const char* a (argv[i]); - optional> v (parse_arg (a)); + optional v (parse_arg (a)); if (!v) fail << "invalid environment argument prefix in '" << a << "'"; - bool mod (v->second[0] != '-' && v->second.find ('=') == string::npos); - - if (mod && !v->first.empty () && - v->first != "b.create" && - v->first != "bpkg.create" && - v->first != "bpkg.target.create" && - v->first != "bpkg.host.create" && - v->first != "bpkg.module.create" && - v->first != "b.test-installed.create" && - v->first != "bpkg.test-separate-installed.create" && - v->first != "bpkg.test-separate-installed.create_for_target" && - v->first != "bpkg.test-separate-installed.create_for_host" && - v->first != "bpkg.test-separate-installed.create_for_module") + bool mod (v->value && + (*v->value)[0] != '-' && + v->value->find ('=') == string::npos); + + if (mod && + !v->prefix.empty () && + v->prefix != "b.create" && + v->prefix != "bpkg.create" && + v->prefix != "bpkg.target.create" && + v->prefix != "bpkg.host.create" && + v->prefix != "bpkg.module.create" && + v->prefix != "b.test-installed.create" && + v->prefix != "bpkg.test-separate-installed.create" && + v->prefix != "bpkg.test-separate-installed.create_for_target" && + v->prefix != "bpkg.test-separate-installed.create_for_host" && + v->prefix != "bpkg.test-separate-installed.create_for_module") fail << "invalid module prefix in '" << a << "'"; - (mod ? modules : env_args).emplace (move (*v)); + if (v->step_status) + step_statuses[v->prefix] = *v->step_status; + + if (v->value) + (mod ? modules : env_args).emplace (make_pair (move (v->prefix), + move (*v->value))); } - // Return command arguments for the specified step id, complementing - // *.create[_for_*] steps with un-prefixed arguments. If no arguments are - // specified for the step then use the specified fallbacks, potentially - // both. Arguments with more specific prefixes come last. + // Parse target configuration arguments. Report failures to the bbot + // controller. // - auto step_args = [] (const std::multimap& args, - step_id step, - optional fallback1 = nullopt, - optional fallback2 = nullopt) -> cstrings - { - cstrings r; - - // Add arguments for a specified, potentially empty, prefix. - // - auto add_args = [&args, &r] (const string& prefix) - { - auto range (args.equal_range (prefix)); + std::multimap tgt_args; - for (auto i (range.first); i != range.second; ++i) - r.emplace_back (i->second.c_str ()); - }; + for (const string& c: tm.target_config) + { + optional v (parse_arg (c)); - // Add un-prefixed arguments if this is one of the *.create[_for_*] - // steps. - // - switch (step) + if (!v) { - case step_id::b_create: - case step_id::bpkg_create: - case step_id::bpkg_target_create: - case step_id::bpkg_host_create: - case step_id::bpkg_module_create: - case step_id::b_test_installed_create: - case step_id::bpkg_test_separate_installed_create: - case step_id::bpkg_test_separate_installed_create_for_target: - case step_id::bpkg_test_separate_installed_create_for_host: - case step_id::bpkg_test_separate_installed_create_for_module: - { - add_args (""); - break; - } - default: break; + rm.status |= result_status::abort; + l3 ([&]{trace << "invalid target configuration argument prefix in " + << "'" << c << "'";}); + break; } - auto add_step_args = [&add_args] (step_id step) - { - const string& s (step_id_str[static_cast (step)]); - - for (size_t n (0);; ++n) - { - n = s.find ('.', n); - - add_args (n == string::npos ? s : string (s, 0, n)); - - if (n == string::npos) - break; - } - }; - - // If no arguments found for the step id, then use the fallback step - // ids, if specified. - // - if (args.find (step_id_str[static_cast (step)]) != args.end ()) - { - add_step_args (step); - } - else + if (v->value && + (*v->value)[0] != '-' && + v->value->find ('=') == string::npos) { - // Note that if we ever need to specify fallback pairs with common - // ancestors, we may want to suppress duplicate ancestor step ids. - // - if (fallback1) - add_step_args (*fallback1); - - if (fallback2) - add_step_args (*fallback2); + rm.status |= result_status::abort; + l3 ([&]{trace << "invalid target configuration argument '" << c + << "'";}); + break; } - return r; - }; - - // bpkg-rep-fetch trust options. - // - cstrings trust_ops; - { - const char* t ("--trust-no"); - - for (const string& fp: tm.trust) - { - if (fp == "yes") - t = "--trust-yes"; - else - { - trust_ops.push_back ("--trust"); - trust_ops.push_back (fp.c_str ()); - } - } + if (v->step_status) + step_statuses[v->prefix] = *v->step_status; - trust_ops.push_back (t); + if (v->value) + tgt_args.emplace (make_pair (move (v->prefix), move (*v->value))); } - const string& pkg (tm.name.string ()); - const version& ver (tm.version); - const string repo (tm.repository.string ()); - const dir_path pkg_dir (pkg + '-' + ver.string ()); - const string pkg_var (tm.name.variable ()); - - // Specify the revision explicitly for the bpkg-build command not to end - // up with a race condition building the latest revision rather than the - // zero revision. - // - const string pkg_rev (pkg + - '/' + - version (ver.epoch, - ver.upstream, - ver.release, - ver.effective_revision (), - ver.iteration).string ()); + if (!rm.status) + break; // Parse the build package configuration represented as a whitespace // separated list of the following potentially quoted bpkg-pkg-build @@ -1120,12 +1107,21 @@ build (size_t argc, const char* argv[]) //