From f37129f4c6cc9e60f547fdfe0a28a984ddd8de73 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Wed, 16 Nov 2022 22:16:21 +0300 Subject: Add support for package-config task manifest value --- bbot/buildfile | 4 +- bbot/worker/worker.cxx | 490 +++++++++++++++++++++++++++++++------------ doc/manual.cli | 300 +++++++++++++++----------- tests/integration/testscript | 61 +++++- 4 files changed, 594 insertions(+), 261 deletions(-) diff --git a/bbot/buildfile b/bbot/buildfile index 1d9a409..cb7b576 100644 --- a/bbot/buildfile +++ b/bbot/buildfile @@ -91,8 +91,10 @@ if $cli.configured --cxx-prologue "#include " \ --cli-namespace bbot::cli --generate-specifier --generate-parse + # No usage. + # cli.cxx{common-options}: cli.options += --include-prefix bbot \ ---guard-prefix BBOT # No usage. +--guard-prefix BBOT --generate-vector-scanner --generate-group-scanner # Usage options. # diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index 4b1a16d..0bf480f 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -17,7 +17,7 @@ #include #include #include -#include // to_utf8(), eof() +#include // to_utf8(), eof(), alpha() #include #include #include @@ -789,7 +789,7 @@ upload_manifest (tracer& trace, } } -static const string worker_checksum ("3"); // Logic version. +static const string worker_checksum ("4"); // Logic version. static int bbot:: build (size_t argc, const char* argv[]) @@ -936,9 +936,9 @@ build (size_t argc, const char* argv[]) // Parse configuration arguments. Report failures to the bbot controller. // - std::multimap config_args; + std::multimap tgt_args; - for (const string& c: tm.config) + for (const string& c: tm.target_config) { optional> v (parse_arg (c)); @@ -957,7 +957,7 @@ build (size_t argc, const char* argv[]) break; } - config_args.emplace (move (*v)); + tgt_args.emplace (move (*v)); } if (!rm.status) @@ -1113,6 +1113,105 @@ build (size_t argc, const char* argv[]) ver.effective_revision (), ver.iteration).string ()); + // Parse the build package configuration represented as a whitespace + // separated list of the following potentially quoted bpkg-pkg-build + // command arguments: + // + // ... ({ ... }+ ?[sys:][])... + // + // If the package configuration is specified, then parse it into the main + // package-specific configuration variables list and the dependency + // packages list, potentially with their own configuration variables. + // + strings pkg_config_vars; + vector> pkg_config_deps; + + if (!tm.package_config.empty ()) + { + struct abort {}; + + auto fail = [&tm, &add_result, &fail_operation] (const string& d, + bool throw_abort = true) + { + fail_operation (add_result ("configure"), + "invalid package configuration: " + d + + "\n info: package configuration: '" + + tm.package_config + '\'', + result_status::abort); + + if (throw_abort) + throw abort (); + }; + + try + { + strings argsv (string_parser::parse_quoted (tm.package_config, + true /* unquote */)); + + cli::vector_scanner scanv (argsv); + cli::group_scanner args (scanv); + + while (args.more ()) + { + // Return true if the argument is a configuration variable. + // + auto var = [] (const string& a) + { + // Note: we need to be careful not to misinterpret + // '?libfoo == 1.0.0' as a variable. + // + return a.compare (0, 7, "config.") == 0 && + a.find ('=') != string::npos; + }; + + string a (args.next ()); + bool v (var (a)); + + // Fail if this is not a configuration variable nor a dependency. + // + if (v || a[0] == '?') + { + cli::scanner& ag (args.group ()); + + if (v) // Configuration variable. + { + if (ag.more ()) + fail ("unexpected options group for configuration variable '" + + a + '\''); + + pkg_config_vars.push_back (move (a)); + } + else // Dependency. + { + strings vars; + while (ag.more ()) + { + string da (ag.next ()); + if (!var (da)) + fail ("argument is not a configuration variable for " + "dependency " + a + ": '" + da + '\''); + + vars.push_back (move (da)); + } + + pkg_config_deps.push_back (make_pair (move (a), move (vars))); + } + } + else + fail ("not a configuration variable nor dependency: '" + a + '\''); + } + } + catch (const string_parser::invalid_string& e) + { + fail (e.what (), false /* throw_abort */); + break; + } + catch (const abort&) + { + break; + } + } + // Query the project's build system information with `b info`. // auto prj_info = [&trace] (const dir_path& d, @@ -1290,8 +1389,7 @@ build (size_t argc, const char* argv[]) size_t n (19); auto space = [] (char c) {return c == ' ' || c == '\t';}; - for (const char* a: - reverse_iterate (step_args (config_args, s, f1, f2))) + for (const char* a: reverse_iterate (step_args (tgt_args, s, f1, f2))) { if (strncmp (a, "config.install.root", n) == 0 && (a[n] == '=' || space (a[n]))) @@ -1461,7 +1559,7 @@ build (size_t argc, const char* argv[]) // // Create the target configuration. // - // bpkg create + // bpkg create // if (create_target) { @@ -1478,9 +1576,9 @@ build (size_t argc, const char* argv[]) "create", "-d", target_conf, !target_pkg ? cstrings ({"--uuid", target_uuid}) : cstrings (), - step_args (modules, s, f1, f2), - step_args (env_args, s, f1, f2), - step_args (config_args, s, f1, f2)); + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2)); if (!r.status) break; @@ -1501,7 +1599,7 @@ build (size_t argc, const char* argv[]) step_id f1 (step_id::b_create); step_id f2 (step_id::bpkg_create); - // bpkg create --type host + // bpkg create --type host // r.status |= run_bpkg ( b, @@ -1512,9 +1610,9 @@ build (size_t argc, const char* argv[]) "-d", host_conf, "--type", "host", "--uuid", host_uuid, - step_args (modules, s, f1, f2), - step_args (env_args, s, f1, f2), - step_args (config_args, s, f1, f2)); + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2)); if (!r.status) break; @@ -1522,7 +1620,7 @@ build (size_t argc, const char* argv[]) // Create the install configuration. // - // bpkg create + // bpkg create // if (create_install) { @@ -1538,9 +1636,9 @@ build (size_t argc, const char* argv[]) "create", "-d", install_conf, "--uuid", install_uuid, - step_args (modules, s, f1, f2), - step_args (env_args, s, f1, f2), - step_args (config_args, s, f1, f2)); + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2)); if (!r.status) break; @@ -1591,15 +1689,15 @@ build (size_t argc, const char* argv[]) // Create the module configuration. // { - // b create() config.config.load=~build2 [ ] + // b create() config.config.load=~build2 [ ] // // Note also that we suppress warnings about unused config.* values. // - // What if a module wants to use CLI? The current thinking is that we - // will be "whitelisting" base (i.e., those that can plausibly be used - // by multiple modules) libraries and tools for use by build system - // modules. So if and when we whitelist CLI, we will add it here, next - // to cc. + // What if a module wants to use CLI? The current thinking is that + // we will be "whitelisting" base (i.e., those that can plausibly be + // used by multiple modules) libraries and tools for use by build + // system modules. So if and when we whitelist CLI, we will add it + // here, next to cc. // string mods; cstrings eas; @@ -1617,8 +1715,8 @@ build (size_t argc, const char* argv[]) mods += m; } - eas = step_args (env_args, s); - cas = step_args (config_args, s); + eas = step_args (env_args, s); + cas = step_args (tgt_args, s); } else mods = "cc"; @@ -1669,7 +1767,7 @@ build (size_t argc, const char* argv[]) mods += m; } - // b create() config.config.load=~build2 [ ] + // b create() config.config.load=~build2 [ ] // r.status |= run_b ( b, @@ -1679,8 +1777,8 @@ build (size_t argc, const char* argv[]) "create(" + install_conf.representation () + ',' + mods + ')', "config.config.load=~build2", "config.config.persist+='config.*'@unused=drop", - step_args (env_args, s), - step_args (config_args, s)); + step_args (env_args, s), + step_args (tgt_args, s)); if (!r.status) break; @@ -1799,7 +1897,7 @@ build (size_t argc, const char* argv[]) // configuration for external build-time tests, if any, and the install // configuration, if present. // - // bpkg add + // bpkg add // { step_id b (step_id::bpkg_configure_add); @@ -1812,15 +1910,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", main_pkg_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), repo); if (!r.status) break; } - // bpkg fetch + // bpkg fetch // { step_id b (step_id::bpkg_configure_fetch); @@ -1833,8 +1931,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", main_pkg_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), trust_ops); if (!r.status) @@ -1843,7 +1941,7 @@ build (size_t argc, const char* argv[]) if (create_install) { - // bpkg add + // bpkg add // { step_id b (step_id::bpkg_configure_add); @@ -1856,15 +1954,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", install_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), repo); if (!r.status) break; } - // bpkg fetch + // bpkg fetch // { step_id b (step_id::bpkg_configure_fetch); @@ -1877,8 +1975,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", install_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), trust_ops); if (!r.status) @@ -1888,7 +1986,7 @@ build (size_t argc, const char* argv[]) if (has_buildtime_tests) { - // bpkg add + // bpkg add // { step_id b (step_id::bpkg_configure_add); @@ -1901,15 +1999,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", target_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), repo); if (!r.status) break; } - // bpkg fetch + // bpkg fetch // { step_id b (step_id::bpkg_configure_fetch); @@ -1922,8 +2020,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", target_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), trust_ops); if (!r.status) @@ -1935,28 +2033,48 @@ build (size_t argc, const char* argv[]) // // First, prepare the common and package arguments. // - // Add the config..develop=false variables for the main and - // external test packages to trigger their package skeleton creation and - // loading. This way we make sure that these packages can be used as - // dependencies of dependents with configuration clauses. + // If no variables are specified in the package configuration, then add + // the config..develop=false variable for the main package instead + // to trigger its package skeleton creation and loading. Also add this + // variable for the external test packages for the same purpose. This + // way we make sure that these packages can be used as dependencies of + // dependents with configuration clauses. + // + // Also add the dependency packages specified in the package + // configuration, if any, to configurations where the main package is + // being configured. + // + // Should we also add the dependency packages to configurations where + // the test packages are being configured? It feels like we shouldn't. + // Moreover, in the future we may decide to support specifying tests + // package configuration in the tests manifest value or some such. In + // this case a test package may have its own dependencies to be + // configured. What we could probably do now, is to never share a bpkg + // configuration between the main package and the tests packages if we + // configure any dependencies in it. Note that such dependencies may + // potentially be unsatisfactory for the test packages (unsatisfactory + // version, etc). This, however, seems rather far fetched so let's keep + // it simple for now. // strings common_args; strings pkg_args; if (target_pkg) // The simple common case (see above)? { - // The overall command looks like this: + // The overall command looks like this (but some parts may be omitted): // - // bpkg build --configure-only -- - // { config..develop=false }+ - // { config..develop=false }+ ... + // bpkg build --configure-only -- + // { |config..develop=false }+ + // { config..develop=false }+ ... + // { }+ ... + // ... // step_id s (step_id::bpkg_target_configure_build); step_id f1 (step_id::b_configure); step_id f2 (step_id::bpkg_configure_build); - cstrings eas (step_args (env_args, s, f1, f2)); - cstrings cas (step_args (config_args, s, f1, f2)); + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); common_args.push_back ("--checkout-root"); common_args.push_back (dist_root.string ()); @@ -1964,16 +2082,29 @@ build (size_t argc, const char* argv[]) common_args.insert (common_args.end (), eas.begin (), eas.end ()); common_args.insert (common_args.end (), cas.begin (), cas.end ()); + // Add the main package. + // + pkg_args.push_back ("{"); + // @@ config..develop=false // + // Only add the config..develop variable if there are no package + // configuration variables specified. + // + if (!pkg_config_vars.empty ()) + pkg_args.insert (pkg_args.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); #if 1 - pkg_args.push_back ("{"); - pkg_args.push_back ("config." + pkg_var + ".develop=false"); - pkg_args.push_back ("}+"); + else + pkg_args.push_back ("config." + pkg_var + ".develop=false"); #endif + pkg_args.push_back ("}+"); + pkg_args.push_back (pkg_rev); + // Add the runtime test packages. + // for (const auto& t: runtime_tests) { // @@ config..develop=false @@ -1991,17 +2122,37 @@ build (size_t argc, const char* argv[]) // pkg_args.push_back (t.string ()); } + + // Add the main package dependencies. + // + for (const pair& d: pkg_config_deps) + { + if (!d.second.empty ()) + { + pkg_args.push_back ("{"); + pkg_args.insert (pkg_args.end (), d.second.begin (), d.second.end ()); + pkg_args.push_back ("}+"); + } + + pkg_args.push_back (d.first); + } } else { // The overall command looks like this (but some parts may be omitted): // - // bpkg build --configure-only -- - // { config..develop=false }+ - // { config..develop=false }+ ... - // { }+ - // { config..develop=false }+ ... + // bpkg build --configure-only -- + // { |config..develop=false }+ + // { config..develop=false }+ ... + // { }+ + // { config..develop=false }+ ... + // { }+ ... + // { }+ { ... } + // + + // Main package configuration name. // + const char* conf_uuid (host_pkg ? host_uuid : module_uuid); // Add the main package args. // @@ -2016,14 +2167,8 @@ build (size_t argc, const char* argv[]) step_id f1 (step_id::b_configure); step_id f2 (step_id::bpkg_configure_build); - cstrings eas (step_args (env_args, s, f1, f2)); - cstrings cas (step_args (config_args, s, f1, f2)); - - // Main package configuration name. - // - const char* conf_uuid (target_pkg ? target_uuid : - host_pkg ? host_uuid : - module_uuid); + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); // Add the main package. // @@ -2041,8 +2186,15 @@ build (size_t argc, const char* argv[]) // @@ config..develop=false // + // Only add the config..develop variable if there are no + // package configuration variables specified. + // + if (!pkg_config_vars.empty ()) + pkg_args.insert (pkg_args.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); #if 1 - pkg_args.push_back ("config." + pkg_var + ".develop=false"); + else + pkg_args.push_back ("config." + pkg_var + ".develop=false"); #endif pkg_args.push_back ("}+"); @@ -2114,15 +2266,15 @@ build (size_t argc, const char* argv[]) } // Add the main package configured in the install configuration and - // the external build-time test packages + // the external build-time test packages. // { step_id s (step_id::bpkg_target_configure_build); step_id f1 (step_id::b_configure); step_id f2 (step_id::bpkg_configure_build); - cstrings eas (step_args (env_args, s, f1, f2)); - cstrings cas (step_args (config_args, s, f1, f2)); + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); // Add the main package. // @@ -2146,6 +2298,9 @@ build (size_t argc, const char* argv[]) pkg_args.insert (pkg_args.end (), eas.begin (), eas.end ()); pkg_args.insert (pkg_args.end (), cas.begin (), cas.end ()); + pkg_args.insert (pkg_args.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); + pkg_args.push_back ("}+"); pkg_args.push_back (pkg_rev); @@ -2217,6 +2372,69 @@ build (size_t argc, const char* argv[]) } #endif } + + // Add the main package dependencies to those configurations where + // the main package is configured. + // + { + // Add dependencies which have some configuration variables + // specified and count the number of others. + // + size_t no_vars (0); + for (const pair& d: pkg_config_deps) + { + if (!d.second.empty ()) + { + pkg_args.push_back ("{"); + + pkg_args.push_back ("--config-uuid"); + pkg_args.push_back (conf_uuid); + + if (create_install) + { + pkg_args.push_back ("--config-uuid"); + pkg_args.push_back (install_uuid); + } + + pkg_args.insert (pkg_args.end (), d.second.begin (), d.second.end ()); + pkg_args.push_back ("}+"); + + pkg_args.push_back (d.first); + } + else + ++no_vars; + } + + // Add dependencies which have no configuration variables specified. + // + if (no_vars != 0) + { + pkg_args.push_back ("{"); + + pkg_args.push_back ("--config-uuid"); + pkg_args.push_back (conf_uuid); + + if (create_install) + { + pkg_args.push_back ("--config-uuid"); + pkg_args.push_back (install_uuid); + } + + pkg_args.push_back ("}+"); + + if (no_vars != 1) + pkg_args.push_back ("{"); + + for (const pair& d: pkg_config_deps) + { + if (d.second.empty ()) + pkg_args.push_back (d.first); + } + + if (no_vars != 1) + pkg_args.push_back ("}"); + } + } } // Finally, configure all the packages. @@ -2238,8 +2456,8 @@ build (size_t argc, const char* argv[]) tm.dependency_checksum ? *tm.dependency_checksum : "", "--yes", "-d", root_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), common_args, "--", pkg_args); @@ -2344,7 +2562,7 @@ build (size_t argc, const char* argv[]) change_wd (trace, &r.log, rwd / main_pkg_conf); - // bpkg update + // bpkg update // step_id b (step_id::bpkg_update); step_id s (step_id::bpkg_update); @@ -2355,8 +2573,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", "update", - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), pkg); if (!r.status) @@ -2379,7 +2597,7 @@ build (size_t argc, const char* argv[]) // auto test = [&trace, &wre, &bkp_step, &bkp_status, &last_cmd, - &step_args, &config_args, &env_args, + &step_args, &tgt_args, &env_args, &bootstrap_import, &redist] (operation_result& r, @@ -2440,7 +2658,7 @@ build (size_t argc, const char* argv[]) // Update. // - // bpkg update + // bpkg update // { step_id b (installed @@ -2458,8 +2676,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", "update", - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), import, pkg); @@ -2472,7 +2690,7 @@ build (size_t argc, const char* argv[]) // Note that we assume that the package supports the test operation // since this is its main purpose. // - // bpkg test + // bpkg test // { step_id b (installed @@ -2491,8 +2709,8 @@ build (size_t argc, const char* argv[]) "-v", "test", "--package-cwd", // See above for details. - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), import, pkg); @@ -2532,7 +2750,7 @@ build (size_t argc, const char* argv[]) // much sense, thus we don't pass the config.import.* variable on // the command line for modules that require bootstrap. // - // bpkg test + // bpkg test // step_id b (step_id::bpkg_test); step_id s (step_id::bpkg_test); @@ -2544,8 +2762,8 @@ build (size_t argc, const char* argv[]) "-v", "test", "--package-cwd", - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), pkg); if (!r.status) @@ -2614,7 +2832,7 @@ build (size_t argc, const char* argv[]) if (!target_pkg && create_target) rm_r (trace, &r.log, rwd / target_conf); - // bpkg install + // bpkg install // step_id b (step_id::bpkg_install); step_id s (step_id::bpkg_install); @@ -2625,8 +2843,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", "install", - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), pkg); if (!r.status) @@ -2699,7 +2917,7 @@ build (size_t argc, const char* argv[]) { // Create the configuration. // - // b create(, ) + // b create(, ) // // Amalgamation directory that will contain configuration subdirectory // for package tests out of source tree build. @@ -2725,8 +2943,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-V", "create('" + out_dir.representation () + '\'' + mods + ')', - step_args (env_args, s, f), - step_args (config_args, s, f)); + step_args (env_args, s, f), + step_args (tgt_args, s, f)); if (!r.status) break; @@ -2735,11 +2953,16 @@ build (size_t argc, const char* argv[]) // Configure testable subprojects sequentially and test/build in // parallel afterwards. // + // It feels right to configure internal tests also passing the main + // package configuration variables, since they may need to align with + // the main package setup (enable some testscripts, etc). + // strings test_specs; for (const dir_path& d: subprj_dirs) { // b configure(@) - // + // + // // step_id b (step_id::b_test_installed_configure); step_id s (step_id::b_test_installed_configure); @@ -2760,8 +2983,9 @@ build (size_t argc, const char* argv[]) "configure('" + subprj_src_dir.representation () + "'@'" + subprj_out_dir.representation () + "')", - step_args (env_args, s, f), - step_args (config_args, s, f)); + step_args (env_args, s, f), + step_args (tgt_args, s, f), + pkg_config_vars); if (!r.status) break; @@ -2775,7 +2999,7 @@ build (size_t argc, const char* argv[]) // Build/test subprojects. // - // b test()... + // b test()... // { step_id b (step_id::b_test_installed_test); @@ -2788,8 +3012,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", test_specs, - step_args (env_args, s), - step_args (config_args, s)); + step_args (env_args, s), + step_args (tgt_args, s)); if (!r.status) break; @@ -2842,7 +3066,7 @@ build (size_t argc, const char* argv[]) // Create the target configuration. // - // bpkg create + // bpkg create // if (create_target) { @@ -2874,9 +3098,9 @@ build (size_t argc, const char* argv[]) "-V", "create", "-d", target_conf, - step_args (modules, s, f), - step_args (env_args, s, f), - step_args (config_args, s, f)); + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f)); if (!r.status) break; @@ -2886,7 +3110,7 @@ build (size_t argc, const char* argv[]) // if (create_host) { - // bpkg create --type host + // bpkg create --type host // step_id b (step_id::bpkg_test_separate_installed_create); @@ -2910,9 +3134,9 @@ build (size_t argc, const char* argv[]) "-d", host_conf, "--type", "host", "--name", "host", - step_args (modules, s, f), - step_args (env_args, s, f), - step_args (config_args, s, f)); + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f)); if (!r.status) break; @@ -3022,7 +3246,7 @@ build (size_t argc, const char* argv[]) // if (has_runtime_tests) { - // bpkg add + // bpkg add // { step_id b (step_id::bpkg_test_separate_installed_configure_add); @@ -3036,15 +3260,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", runtime_tests_conf, - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), repo); if (!r.status) break; } - // bpkg fetch + // bpkg fetch // { step_id b (step_id::bpkg_test_separate_installed_configure_fetch); @@ -3058,8 +3282,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", runtime_tests_conf, - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), trust_ops); if (!r.status) @@ -3069,7 +3293,7 @@ build (size_t argc, const char* argv[]) if (has_buildtime_tests) { - // bpkg add + // bpkg add // { step_id b (step_id::bpkg_test_separate_installed_configure_add); @@ -3083,15 +3307,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", target_conf, - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), repo); if (!r.status) break; } - // bpkg fetch + // bpkg fetch // { step_id b (step_id::bpkg_test_separate_installed_configure_fetch); @@ -3105,8 +3329,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", target_conf, - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), trust_ops); if (!r.status) @@ -3116,7 +3340,7 @@ build (size_t argc, const char* argv[]) // Configure all the packages using a single bpkg-pkg-build command. // - // bpkg build --configure-only + // bpkg build --configure-only // { }+ { ... } // ... // ?sys: @@ -3194,10 +3418,10 @@ build (size_t argc, const char* argv[]) "--checkout-root", dist_installed_root, "--yes", "-d", root_conf, - step_args (env_args, g), - step_args (env_args, s, f), - step_args (config_args, g), - step_args (config_args, s, f), + step_args (env_args, g), + step_args (env_args, s, f), + step_args (tgt_args, g), + step_args (tgt_args, s, f), "--", pkg_args); @@ -3252,7 +3476,7 @@ build (size_t argc, const char* argv[]) change_wd (trace, &r.log, install_conf); - // bpkg uninstall + // bpkg uninstall // step_id b (step_id::bpkg_uninstall); step_id s (step_id::bpkg_uninstall); @@ -3263,8 +3487,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", "uninstall", - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), pkg); if (!r.status) diff --git a/doc/manual.cli b/doc/manual.cli index 3b3bd4b..59b20c7 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -90,12 +90,12 @@ build logs hosted by the controller. \h#arch-machine-config|Configurations| -The \c{bbot} architecture distinguishes between a \i{machine configuration} -and a \i{build configuration}. The machine configuration captures the -operating system, installed compiler toolchain, and so on. The same build -machine may be used to \"generate\" multiple \i{build configurations}. For -example, the same machine can normally be used to produce 32/64-bit and -debug/optimized builds. +The \c{bbot} architecture distinguishes between a \i{machine configuration}, +\i{build target configuration}, and a \i{build package configuration}. The +machine configuration captures the operating system, installed compiler +toolchain, and so on. The same build machine may be used to \"generate\" +multiple \i{build target configurations}. For example, the same machine can +normally be used to produce 32/64-bit and debug/optimized builds. The machine configuration is \i{approximately} encoded in its \i{machine name}. The machine name is a list of components separated with \c{-}. @@ -156,22 +156,31 @@ linux_ubuntu_16.04-gcc_6.3 aarch64_linux_debian_11-gcc_12.2 \ -Similarly, the build configuration is encoded in a \i{configuration name} -using the same overall format. As described in \l{#arch-controller Controller -Logic}, build configurations are generated from machine configurations. As a -result, it usually makes sense to have the first component identify the -operating systems and the second component \- the toolchain with the rest -identifying a particular build configuration variant, for example, optimized, -sanitized, etc. For example: +Similarly, the build target configuration is encoded in a \i{configuration +name} using the same overall format. As described in \l{#arch-controller +Controller Logic}, target configurations are generated from machine +configurations. As a result, it usually makes sense to have the first +component identify the operating systems and the second component \- the +toolchain with the rest identifying a particular target configuration variant, +for example, optimized, sanitized, etc. For example: \ windows-vc_14-O2 linux-gcc_6-O3_asan \ -While we can also specify the \c{} component in a build configuration, -this information is best conveyed as part of \c{} as described in -\l{#arch-controller Controller Logic}. +While we can also specify the \c{} component in a build target +configuration, this information is best conveyed as part of \c{} as +described in \l{#arch-controller Controller Logic}. + +A package can be built in multiple package configurations per target +configuration. A build package configuration normally specifies the package +configuration variables that need to be used for the build. It may also +include the information regarding the dependency packages which need to +additionally be configured. The build package configurations originate from +the package manifest \c{*-build-config}, \c{*-builds}, \c{*-build-include}, +and \c{*-build-exclude} values. See \l{bpkg#manifest-package Package Manifest} +for more information on these values. \h#arch-machine-header|Machine Header Manifest| @@ -348,7 +357,8 @@ repository-url: machine: target: [environment]: -[config]: +[target-config]: +[package-config]: [host]: true|false [warning-regex]: [interactive]: @@ -469,24 +479,50 @@ The name of the build environment to use. See \l{#arch-worker Worker Logic} for details. -\h2#arch-task-config|\c{config}| +\h2#arch-task-target-config|\c{target-config}| \ -[config]: +[target-config]: \ -The additional configuration options and variables. A single level of quotes +The additional target configuration options and variables. A single level of +quotes (either single or double) is removed in each value before being passed +to \c{bpkg}. For example, the following value: + +\ +target-config: config.cc.coptions=\"-O3 -stdlib='libc++'\" +\ + +Will be passed to \c{bpkg} as the following (single) argument: + +\ +config.cc.coptions=-O3 -stdlib='libc++' +\ + +Values can be separated with spaces or newlines. See \l{#arch-controller +Controller Logic} for details. + + +\h2#arch-task-package-config|\c{package-config}| + +\ +[package-config]: +\ + +The primary package manifest \c{*-build-config} value for the build +configuration the build task is issued for. See \l{bpkg#manifest-package +Package Manifest} for more information on this value. A single level of quotes (either single or double) is removed in each value before being passed to \c{bpkg}. For example, the following value: \ -config: config.cc.coptions=\"-O3 -stdlib='libc++'\" +package-config: \"?libcurl ~7.76.0\" \ Will be passed to \c{bpkg} as the following (single) argument: \ -config.cc.coptions=-O3 -stdlib='libc++' +?libcurl ~7.76.0 \ Values can be separated with spaces or newlines. See \l{#arch-controller @@ -499,8 +535,9 @@ Controller Logic} for details. [host]: true|false \ -If \c{true}, then the build configuration is self-hosted. If not specified, -\c{false} is assumed. See \l{#arch-controller Controller Logic} for details. +If \c{true}, then the build target configuration is self-hosted. If not +specified, \c{false} is assumed. See \l{#arch-controller Controller Logic} for +details. \h2#arch-task-warning-regex|\c{warning-regex}| @@ -924,20 +961,24 @@ modules (\c{}) and the list of configuration options and variables The re-executed \c{bbot} worker then proceeds to test the package from the repository by executing the following commands, collectively called a \i{worker script}. Each command has a unique \i{step id} that can be used as a -breakpoint and normally as a prefix in the \c{}, +breakpoint and normally as a prefix in the \c{}, \c{}, and \c{} values as discussed in \l{#arch-controller Controller Logic}. The \c{<>}-values are from the task manifest and the environment though some are assigned by the worker during the -script execution (configuration directories, UUIDs, etc). +script execution (configuration directories, UUIDs, etc). In particular, the +\c{}, \c{}, +\c{}, and \c{} values result +from parsing the \l{#arch-task-package-config \c{package-config}} task +manifest value. Some prefix step ids have fallback step ids which are used in the absence of the primary step id values. If the prefix step id differs from the breakpoint step id and/or has the fallback step ids, then they are listed in parenthesis: the prefix id before the colon and the fallback ids after it. -Some commands have no configuration or environment options or variables. Such -commands have only breakpoint step ids associated, which are listed in square -brackets. +Some commands have no target configuration or environment options or +variables. Such commands have only breakpoint step ids associated, which are +listed in square brackets. Note that the worker script varies for different primary package types. The \c{bbot} worker classifies the primary package based on the configuration type @@ -952,19 +993,21 @@ by always using the \c{bpkg.global.configure.build} prefix step id for global (as opposed to package-specific) \l{bpkg-pkg-build(1)} options. The \c{bpkg.global.configure.build} prefix id has no fallback ids. -Note finally that the worker adds the \c{config..develop=false} -configuration variables for the main and external test packages at the -\c{bpkg.configure.build} step to trigger their package skeleton creation and -loading. This makes sure that these packages can be used as dependencies of -dependents with configuration clauses. To keep the below listings concise, -these variables are not shown. +Note finally that if no configuration variables are specified in the main +package configuration, then the worker adds the +\c{config..develop=false} configuration variable for the main package at +the \c{bpkg.configure.build} step to trigger its package skeleton creation and +loading. It also adds this variable for external test packages at this step +and for the same purpose. This makes sure that these packages can be used as +dependencies of dependents with configuration clauses. To keep the below +listings concise, these variables are not shown. Worker script for \c{target} packages: \ # bpkg.create (bpkg.target.create : b.create, bpkg.create) # -bpkg -V create +bpkg -V create # bpkg.configure.add # @@ -978,9 +1021,11 @@ bpkg -v fetch --trust # bpkg.global.configure.build, # (bpkg.target.configure.build : b.configure, bpkg.configure.build)) # -bpkg -v build --configure-only \\ - / \\ - [[ ]...] +bpkg -v build --configure-only \\ + [{ }+] / \\ + [[ ]...] \\ + [([{ }+] \\ + ?[sys:][ ])...] # bpkg.update # @@ -1020,14 +1065,14 @@ bpkg -v update { # b.test-installed.create ( : b.create) # - b -V create + b -V create # For each test subproject: # { # b.test-installed.configure ( : b.configure) # - b -v configure + b -v configure [] } # b.test-installed.test @@ -1043,7 +1088,7 @@ bpkg -v update # bpkg.test-separate-installed.create_for_target : # bpkg.test-separate-installed.create) # - bpkg -V create + bpkg -V create # bpkg.test-separate-installed.configure.add ( # : bpkg.configure.add) @@ -1060,7 +1105,7 @@ bpkg -v update # (bpkg.test-separate-installed.configure.build_for_target : # bpkg.test-separate-installed.configure.build)) # - bpkg -v build --configure-only \\ + bpkg -v build --configure-only \\ [ ]... \\ ?sys: @@ -1097,7 +1142,7 @@ Worker script for \c{host} packages: # bpkg.create (bpkg.host.create : b.create, bpkg.create) # bpkg -V create --type host -d \\ - + } # # Otherwise: @@ -1123,7 +1168,7 @@ bpkg -v fetch -d --trust # bpkg.create (bpkg.target.create : b.create, bpkg.create) # bpkg -V create -d \\ - + # [bpkg.link] # @@ -1145,7 +1190,7 @@ bpkg -v fetch -d --trust # bpkg.create (bpkg.target.create : b.create, bpkg.create) # bpkg -V create -d \\ - + # [bpkg.create] # @@ -1192,19 +1237,25 @@ bpkg -v fetch -d --trust # - All parts have the same fallback step ids: b.configure and # bpkg.configure.build. # -bpkg -v build --configure-only \\ +bpkg -v build --configure-only \\ \\ -{ --config-uuid }+ \\ +{ --config-uuid \\ + [] }+ \\ / \\ \\ -{ --config-uuid }+ \\ +{ --config-uuid \\ + [] }+ \\ / \\ \\ -{ --config-uuid }+ \\ +{ --config-uuid }+ \\ { [ test-version-constraint>]... } \\ \\ -{ --config-uuid }+ \\ -{ [ test-version-constraint>]... } +{ --config-uuid }+ \\ +{ [ test-version-constraint>]... } \\ +\\ +({ --config-uuid [--config-uuid ] \\ + [] }+ \\ + ?[sys:][ ])... # bpkg.update # @@ -1257,14 +1308,14 @@ bpkg -v update -d { # b.test-installed.create ( : b.create) # - b -V create + b -V create # For each test subproject: # { # b.test-installed.configure ( : b.configure) # - b -v configure + b -v configure [] } # b.test-installed.test @@ -1281,7 +1332,7 @@ bpkg -v update -d # bpkg.test-separate-installed.create) # bpkg -V create --type host -d \\ - + # If task manifest refers to any runtime tests, examples, or # benchmarks packages: @@ -1307,7 +1358,7 @@ bpkg -v update -d # bpkg.test-separate-installed.create) # bpkg -V create -d \\ - + # [bpkg.test-separate-installed.create] # @@ -1339,7 +1390,7 @@ bpkg -v update -d # Note that any of the runtime or build-time tests related parts # (but not both) may be omitted. # - bpkg -v build --configure-only \\ + bpkg -v build --configure-only \\ \\ { --config-name }+ \\ { [ ]... } \\ @@ -1382,7 +1433,7 @@ Worker script for \c{module} packages: # bpkg.create (bpkg.module.create) # b -V create(, ) config.config.load=~build2 \\ - + bpkg -v create --existing --type build2 -d } # @@ -1409,7 +1460,7 @@ bpkg -v fetch -d --trust # bpkg.create (bpkg.module.create) # b -V create(, ) \\ - config.config.load=~build2 + config.config.load=~build2 bpkg -v create --existing --type build2 -d # bpkg.configure.add @@ -1428,7 +1479,7 @@ bpkg -v fetch -d --trust # bpkg.create (bpkg.target.create : b.create, bpkg.create) # bpkg -V create -d \\ - + # [bpkg.create] # @@ -1466,16 +1517,22 @@ bpkg -v fetch -d --trust # - All parts have the same fallback step ids: b.configure and # bpkg.configure.build. # -bpkg -v build --configure-only \\ +bpkg -v build --configure-only \\ \\ -{ --config-uuid }+ \\ +{ --config-uuid \\ + [] }+ \\ / \\ \\ -{ --config-uuid }+ \\ +{ --config-uuid \\ + [] }+ \\ / \\ \\ -{ --config-uuid }+ \\ -{ [ test-version-constraint>]... } +{ --config-uuid }+ \\ +{ [ test-version-constraint>]... } \\ +\\ +({ --config-uuid [--config-uuid ] \\ + [] }+ \\ + ?[sys:][ ])... # bpkg.update # @@ -1523,14 +1580,14 @@ bpkg -v update -d # bpkg.test-separate-installed.create) # bpkg -V create -d \\ - + # bpkg.test-separate-installed.create ( # bpkg.test-separate-installed.create_for_module : # bpkg.test-separate-installed.create) # bpkg -V create --type host -d \\ - + # [bpkg.test-separate-installed.link] # @@ -1553,7 +1610,7 @@ bpkg -v update -d # (bpkg.test-separate-installed.configure.build_for_module : # bpkg.test-separate-installed.configure.build)) # - bpkg -v build --configure-only \\ + bpkg -v build --configure-only \\ \\ { --config-name }+ \\ [ ]... \\ @@ -1635,49 +1692,50 @@ exec \"$@\" cc config.c=\"gcc-9 $mode\" config.cxx=\"g++-9 $mode\" \h#arch-controller|Controller Logic| A \c{bbot} controller that issues own build tasks maps available build -machines (as reported by agents) to \i{build configurations} according to the -\c{buildtab} configuration file. Blank lines and lines that start with \c{#} -are ignored. All other lines in this file have the following format: +machines (as reported by agents) to \i{build target configurations} according +to the \c{buildtab} configuration file. Blank lines and lines that start with +\c{#} are ignored. All other lines in this file have the following format: \ - [/] []* []* + [/] []* []* - = [:](|