diff options
| -rw-r--r-- | bbot/buildfile | 4 | ||||
| -rw-r--r-- | bbot/worker/worker.cxx | 490 | ||||
| -rw-r--r-- | doc/manual.cli | 300 | ||||
| -rw-r--r-- | tests/integration/testscript | 61 |
4 files changed, 594 insertions, 261 deletions
diff --git a/bbot/buildfile b/bbot/buildfile index 1d9a409..cb7b576 100644 --- a/bbot/buildfile +++ b/bbot/buildfile @@ -91,8 +91,10 @@ if $cli.configured --cxx-prologue "#include <bbot/types-parsers.hxx>" \ --cli-namespace bbot::cli --generate-specifier --generate-parse + # No usage. + # cli.cxx{common-options}: cli.options += --include-prefix bbot \ ---guard-prefix BBOT # No usage. +--guard-prefix BBOT --generate-vector-scanner --generate-group-scanner # Usage options. # diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index 4b1a16d..0bf480f 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -17,7 +17,7 @@ #include <libbutl/b.hxx> #include <libbutl/pager.hxx> #include <libbutl/prompt.hxx> -#include <libbutl/utility.hxx> // to_utf8(), eof() +#include <libbutl/utility.hxx> // to_utf8(), eof(), alpha() #include <libbutl/timestamp.hxx> #include <libbutl/filesystem.hxx> #include <libbutl/string-parser.hxx> @@ -789,7 +789,7 @@ upload_manifest (tracer& trace, } } -static const string worker_checksum ("3"); // Logic version. +static const string worker_checksum ("4"); // Logic version. static int bbot:: build (size_t argc, const char* argv[]) @@ -936,9 +936,9 @@ build (size_t argc, const char* argv[]) // Parse configuration arguments. Report failures to the bbot controller. // - std::multimap<string, string> config_args; + std::multimap<string, string> tgt_args; - for (const string& c: tm.config) + for (const string& c: tm.target_config) { optional<pair<string, string>> v (parse_arg (c)); @@ -957,7 +957,7 @@ build (size_t argc, const char* argv[]) break; } - config_args.emplace (move (*v)); + tgt_args.emplace (move (*v)); } if (!rm.status) @@ -1113,6 +1113,105 @@ build (size_t argc, const char* argv[]) ver.effective_revision (), ver.iteration).string ()); + // Parse the build package configuration represented as a whitespace + // separated list of the following potentially quoted bpkg-pkg-build + // command arguments: + // + // <config-var>... ({ <config-var>... }+ ?[sys:]<pkg-name>[<version-spec>])... + // + // If the package configuration is specified, then parse it into the main + // package-specific configuration variables list and the dependency + // packages list, potentially with their own configuration variables. + // + strings pkg_config_vars; + vector<pair<string, strings>> pkg_config_deps; + + if (!tm.package_config.empty ()) + { + struct abort {}; + + auto fail = [&tm, &add_result, &fail_operation] (const string& d, + bool throw_abort = true) + { + fail_operation (add_result ("configure"), + "invalid package configuration: " + d + + "\n info: package configuration: '" + + tm.package_config + '\'', + result_status::abort); + + if (throw_abort) + throw abort (); + }; + + try + { + strings argsv (string_parser::parse_quoted (tm.package_config, + true /* unquote */)); + + cli::vector_scanner scanv (argsv); + cli::group_scanner args (scanv); + + while (args.more ()) + { + // Return true if the argument is a configuration variable. + // + auto var = [] (const string& a) + { + // Note: we need to be careful not to misinterpret + // '?libfoo == 1.0.0' as a variable. + // + return a.compare (0, 7, "config.") == 0 && + a.find ('=') != string::npos; + }; + + string a (args.next ()); + bool v (var (a)); + + // Fail if this is not a configuration variable nor a dependency. + // + if (v || a[0] == '?') + { + cli::scanner& ag (args.group ()); + + if (v) // Configuration variable. + { + if (ag.more ()) + fail ("unexpected options group for configuration variable '" + + a + '\''); + + pkg_config_vars.push_back (move (a)); + } + else // Dependency. + { + strings vars; + while (ag.more ()) + { + string da (ag.next ()); + if (!var (da)) + fail ("argument is not a configuration variable for " + "dependency " + a + ": '" + da + '\''); + + vars.push_back (move (da)); + } + + pkg_config_deps.push_back (make_pair (move (a), move (vars))); + } + } + else + fail ("not a configuration variable nor dependency: '" + a + '\''); + } + } + catch (const string_parser::invalid_string& e) + { + fail (e.what (), false /* throw_abort */); + break; + } + catch (const abort&) + { + break; + } + } + // Query the project's build system information with `b info`. // auto prj_info = [&trace] (const dir_path& d, @@ -1290,8 +1389,7 @@ build (size_t argc, const char* argv[]) size_t n (19); auto space = [] (char c) {return c == ' ' || c == '\t';}; - for (const char* a: - reverse_iterate (step_args (config_args, s, f1, f2))) + for (const char* a: reverse_iterate (step_args (tgt_args, s, f1, f2))) { if (strncmp (a, "config.install.root", n) == 0 && (a[n] == '=' || space (a[n]))) @@ -1461,7 +1559,7 @@ build (size_t argc, const char* argv[]) // // Create the target configuration. // - // bpkg create <env-modules> <env-config-args> <config-args> + // bpkg create <env-modules> <env-config-args> <tgt-config-args> // if (create_target) { @@ -1478,9 +1576,9 @@ build (size_t argc, const char* argv[]) "create", "-d", target_conf, !target_pkg ? cstrings ({"--uuid", target_uuid}) : cstrings (), - step_args (modules, s, f1, f2), - step_args (env_args, s, f1, f2), - step_args (config_args, s, f1, f2)); + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2)); if (!r.status) break; @@ -1501,7 +1599,7 @@ build (size_t argc, const char* argv[]) step_id f1 (step_id::b_create); step_id f2 (step_id::bpkg_create); - // bpkg create --type host <env-modules> <env-config-args> <config-args> + // bpkg create --type host <env-modules> <env-config-args> <tgt-config-args> // r.status |= run_bpkg ( b, @@ -1512,9 +1610,9 @@ build (size_t argc, const char* argv[]) "-d", host_conf, "--type", "host", "--uuid", host_uuid, - step_args (modules, s, f1, f2), - step_args (env_args, s, f1, f2), - step_args (config_args, s, f1, f2)); + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2)); if (!r.status) break; @@ -1522,7 +1620,7 @@ build (size_t argc, const char* argv[]) // Create the install configuration. // - // bpkg create <env-modules> <env-config-args> <config-args> + // bpkg create <env-modules> <env-config-args> <tgt-config-args> // if (create_install) { @@ -1538,9 +1636,9 @@ build (size_t argc, const char* argv[]) "create", "-d", install_conf, "--uuid", install_uuid, - step_args (modules, s, f1, f2), - step_args (env_args, s, f1, f2), - step_args (config_args, s, f1, f2)); + step_args (modules, s, f1, f2), + step_args (env_args, s, f1, f2), + step_args (tgt_args, s, f1, f2)); if (!r.status) break; @@ -1591,15 +1689,15 @@ build (size_t argc, const char* argv[]) // Create the module configuration. // { - // b create(<dir>) config.config.load=~build2 [<env-config-args> <config-args>] + // b create(<dir>) config.config.load=~build2 [<env-config-args> <tgt-config-args>] // // Note also that we suppress warnings about unused config.* values. // - // What if a module wants to use CLI? The current thinking is that we - // will be "whitelisting" base (i.e., those that can plausibly be used - // by multiple modules) libraries and tools for use by build system - // modules. So if and when we whitelist CLI, we will add it here, next - // to cc. + // What if a module wants to use CLI? The current thinking is that + // we will be "whitelisting" base (i.e., those that can plausibly be + // used by multiple modules) libraries and tools for use by build + // system modules. So if and when we whitelist CLI, we will add it + // here, next to cc. // string mods; cstrings eas; @@ -1617,8 +1715,8 @@ build (size_t argc, const char* argv[]) mods += m; } - eas = step_args (env_args, s); - cas = step_args (config_args, s); + eas = step_args (env_args, s); + cas = step_args (tgt_args, s); } else mods = "cc"; @@ -1669,7 +1767,7 @@ build (size_t argc, const char* argv[]) mods += m; } - // b create(<dir>) config.config.load=~build2 [<env-config-args> <config-args>] + // b create(<dir>) config.config.load=~build2 [<env-config-args> <tgt-config-args>] // r.status |= run_b ( b, @@ -1679,8 +1777,8 @@ build (size_t argc, const char* argv[]) "create(" + install_conf.representation () + ',' + mods + ')', "config.config.load=~build2", "config.config.persist+='config.*'@unused=drop", - step_args (env_args, s), - step_args (config_args, s)); + step_args (env_args, s), + step_args (tgt_args, s)); if (!r.status) break; @@ -1799,7 +1897,7 @@ build (size_t argc, const char* argv[]) // configuration for external build-time tests, if any, and the install // configuration, if present. // - // bpkg add <env-config-args> <config-args> <repository-url> + // bpkg add <env-config-args> <tgt-config-args> <repository-url> // { step_id b (step_id::bpkg_configure_add); @@ -1812,15 +1910,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", main_pkg_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), repo); if (!r.status) break; } - // bpkg fetch <env-config-args> <config-args> <trust-options> + // bpkg fetch <env-config-args> <tgt-config-args> <trust-options> // { step_id b (step_id::bpkg_configure_fetch); @@ -1833,8 +1931,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", main_pkg_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), trust_ops); if (!r.status) @@ -1843,7 +1941,7 @@ build (size_t argc, const char* argv[]) if (create_install) { - // bpkg add <env-config-args> <config-args> <repository-url> + // bpkg add <env-config-args> <tgt-config-args> <repository-url> // { step_id b (step_id::bpkg_configure_add); @@ -1856,15 +1954,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", install_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), repo); if (!r.status) break; } - // bpkg fetch <env-config-args> <config-args> <trust-options> + // bpkg fetch <env-config-args> <tgt-config-args> <trust-options> // { step_id b (step_id::bpkg_configure_fetch); @@ -1877,8 +1975,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", install_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), trust_ops); if (!r.status) @@ -1888,7 +1986,7 @@ build (size_t argc, const char* argv[]) if (has_buildtime_tests) { - // bpkg add <env-config-args> <config-args> <repository-url> + // bpkg add <env-config-args> <tgt-config-args> <repository-url> // { step_id b (step_id::bpkg_configure_add); @@ -1901,15 +1999,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", target_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), repo); if (!r.status) break; } - // bpkg fetch <env-config-args> <config-args> <trust-options> + // bpkg fetch <env-config-args> <tgt-config-args> <trust-options> // { step_id b (step_id::bpkg_configure_fetch); @@ -1922,8 +2020,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", target_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), trust_ops); if (!r.status) @@ -1935,28 +2033,48 @@ build (size_t argc, const char* argv[]) // // First, prepare the common and package arguments. // - // Add the config.<pkg>.develop=false variables for the main and - // external test packages to trigger their package skeleton creation and - // loading. This way we make sure that these packages can be used as - // dependencies of dependents with configuration clauses. + // If no variables are specified in the package configuration, then add + // the config.<pkg>.develop=false variable for the main package instead + // to trigger its package skeleton creation and loading. Also add this + // variable for the external test packages for the same purpose. This + // way we make sure that these packages can be used as dependencies of + // dependents with configuration clauses. + // + // Also add the dependency packages specified in the package + // configuration, if any, to configurations where the main package is + // being configured. + // + // Should we also add the dependency packages to configurations where + // the test packages are being configured? It feels like we shouldn't. + // Moreover, in the future we may decide to support specifying tests + // package configuration in the tests manifest value or some such. In + // this case a test package may have its own dependencies to be + // configured. What we could probably do now, is to never share a bpkg + // configuration between the main package and the tests packages if we + // configure any dependencies in it. Note that such dependencies may + // potentially be unsatisfactory for the test packages (unsatisfactory + // version, etc). This, however, seems rather far fetched so let's keep + // it simple for now. // strings common_args; strings pkg_args; if (target_pkg) // The simple common case (see above)? { - // The overall command looks like this: + // The overall command looks like this (but some parts may be omitted): // - // bpkg build --configure-only <env-config-args> <config-args> -- - // { config.<pkg-name>.develop=false }+ <pkg> - // { config.<runtime-test-name>.develop=false }+ <runtime-test>... + // bpkg build --configure-only <env-config-args> <tgt-config-args> -- + // { <pkg-config-vars>|config.<pkg-name>.develop=false }+ <pkg> + // { config.<runtime-test-name>.develop=false }+ <runtime-test>... + // { <dep-config-vars> }+ <dep>... + // <dep>... // step_id s (step_id::bpkg_target_configure_build); step_id f1 (step_id::b_configure); step_id f2 (step_id::bpkg_configure_build); - cstrings eas (step_args (env_args, s, f1, f2)); - cstrings cas (step_args (config_args, s, f1, f2)); + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); common_args.push_back ("--checkout-root"); common_args.push_back (dist_root.string ()); @@ -1964,16 +2082,29 @@ build (size_t argc, const char* argv[]) common_args.insert (common_args.end (), eas.begin (), eas.end ()); common_args.insert (common_args.end (), cas.begin (), cas.end ()); + // Add the main package. + // + pkg_args.push_back ("{"); + // @@ config.<pkg>.develop=false // + // Only add the config.<pkg>.develop variable if there are no package + // configuration variables specified. + // + if (!pkg_config_vars.empty ()) + pkg_args.insert (pkg_args.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); #if 1 - pkg_args.push_back ("{"); - pkg_args.push_back ("config." + pkg_var + ".develop=false"); - pkg_args.push_back ("}+"); + else + pkg_args.push_back ("config." + pkg_var + ".develop=false"); #endif + pkg_args.push_back ("}+"); + pkg_args.push_back (pkg_rev); + // Add the runtime test packages. + // for (const auto& t: runtime_tests) { // @@ config.<pkg>.develop=false @@ -1991,17 +2122,37 @@ build (size_t argc, const char* argv[]) // pkg_args.push_back (t.string ()); } + + // Add the main package dependencies. + // + for (const pair<string, strings>& d: pkg_config_deps) + { + if (!d.second.empty ()) + { + pkg_args.push_back ("{"); + pkg_args.insert (pkg_args.end (), d.second.begin (), d.second.end ()); + pkg_args.push_back ("}+"); + } + + pkg_args.push_back (d.first); + } } else { // The overall command looks like this (but some parts may be omitted): // - // bpkg build --configure-only <env-config-args> <config-args> -- - // { <build-config> <env-config-args> <config-args> config.<pkg-name>.develop=false }+ <pkg> - // { <build-config> <env-config-args> <config-args> config.<runtime-test-name>.develop=false }+ <runtime-test>... - // { <install-config> <env-config-args> <config-args> }+ <pkg> - // { <target-config> <env-config-args> <config-args> config.<buildtime-test-name>.develop=false }+ <buildtime-test>... + // bpkg build --configure-only <env-config-args> <tgt-config-args> -- + // { <build-config> <env-config-args> <tgt-config-args> <pkg-config-vars>|config.<pkg-name>.develop=false }+ <pkg> + // { <build-config> <env-config-args> <tgt-config-args> config.<runtime-test-name>.develop=false }+ <runtime-test>... + // { <install-config> <env-config-args> <tgt-config-args> <pkg-config-vars> }+ <pkg> + // { <target-config> <env-config-args> <tgt-config-args> config.<buildtime-test-name>.develop=false }+ <buildtime-test>... + // { <build-config> <install-config> <dep-config-vars> }+ <dep>... + // { <build-config> <install-config> }+ { <dep>... } + // + + // Main package configuration name. // + const char* conf_uuid (host_pkg ? host_uuid : module_uuid); // Add the main package args. // @@ -2016,14 +2167,8 @@ build (size_t argc, const char* argv[]) step_id f1 (step_id::b_configure); step_id f2 (step_id::bpkg_configure_build); - cstrings eas (step_args (env_args, s, f1, f2)); - cstrings cas (step_args (config_args, s, f1, f2)); - - // Main package configuration name. - // - const char* conf_uuid (target_pkg ? target_uuid : - host_pkg ? host_uuid : - module_uuid); + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); // Add the main package. // @@ -2041,8 +2186,15 @@ build (size_t argc, const char* argv[]) // @@ config.<pkg>.develop=false // + // Only add the config.<pkg>.develop variable if there are no + // package configuration variables specified. + // + if (!pkg_config_vars.empty ()) + pkg_args.insert (pkg_args.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); #if 1 - pkg_args.push_back ("config." + pkg_var + ".develop=false"); + else + pkg_args.push_back ("config." + pkg_var + ".develop=false"); #endif pkg_args.push_back ("}+"); @@ -2114,15 +2266,15 @@ build (size_t argc, const char* argv[]) } // Add the main package configured in the install configuration and - // the external build-time test packages + // the external build-time test packages. // { step_id s (step_id::bpkg_target_configure_build); step_id f1 (step_id::b_configure); step_id f2 (step_id::bpkg_configure_build); - cstrings eas (step_args (env_args, s, f1, f2)); - cstrings cas (step_args (config_args, s, f1, f2)); + cstrings eas (step_args (env_args, s, f1, f2)); + cstrings cas (step_args (tgt_args, s, f1, f2)); // Add the main package. // @@ -2146,6 +2298,9 @@ build (size_t argc, const char* argv[]) pkg_args.insert (pkg_args.end (), eas.begin (), eas.end ()); pkg_args.insert (pkg_args.end (), cas.begin (), cas.end ()); + pkg_args.insert (pkg_args.end (), + pkg_config_vars.begin (), pkg_config_vars.end ()); + pkg_args.push_back ("}+"); pkg_args.push_back (pkg_rev); @@ -2217,6 +2372,69 @@ build (size_t argc, const char* argv[]) } #endif } + + // Add the main package dependencies to those configurations where + // the main package is configured. + // + { + // Add dependencies which have some configuration variables + // specified and count the number of others. + // + size_t no_vars (0); + for (const pair<string, strings>& d: pkg_config_deps) + { + if (!d.second.empty ()) + { + pkg_args.push_back ("{"); + + pkg_args.push_back ("--config-uuid"); + pkg_args.push_back (conf_uuid); + + if (create_install) + { + pkg_args.push_back ("--config-uuid"); + pkg_args.push_back (install_uuid); + } + + pkg_args.insert (pkg_args.end (), d.second.begin (), d.second.end ()); + pkg_args.push_back ("}+"); + + pkg_args.push_back (d.first); + } + else + ++no_vars; + } + + // Add dependencies which have no configuration variables specified. + // + if (no_vars != 0) + { + pkg_args.push_back ("{"); + + pkg_args.push_back ("--config-uuid"); + pkg_args.push_back (conf_uuid); + + if (create_install) + { + pkg_args.push_back ("--config-uuid"); + pkg_args.push_back (install_uuid); + } + + pkg_args.push_back ("}+"); + + if (no_vars != 1) + pkg_args.push_back ("{"); + + for (const pair<string, strings>& d: pkg_config_deps) + { + if (d.second.empty ()) + pkg_args.push_back (d.first); + } + + if (no_vars != 1) + pkg_args.push_back ("}"); + } + } } // Finally, configure all the packages. @@ -2238,8 +2456,8 @@ build (size_t argc, const char* argv[]) tm.dependency_checksum ? *tm.dependency_checksum : "", "--yes", "-d", root_conf, - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), common_args, "--", pkg_args); @@ -2344,7 +2562,7 @@ build (size_t argc, const char* argv[]) change_wd (trace, &r.log, rwd / main_pkg_conf); - // bpkg update <env-config-args> <config-args> <package-name> + // bpkg update <env-config-args> <tgt-config-args> <package-name> // step_id b (step_id::bpkg_update); step_id s (step_id::bpkg_update); @@ -2355,8 +2573,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", "update", - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), pkg); if (!r.status) @@ -2379,7 +2597,7 @@ build (size_t argc, const char* argv[]) // auto test = [&trace, &wre, &bkp_step, &bkp_status, &last_cmd, - &step_args, &config_args, &env_args, + &step_args, &tgt_args, &env_args, &bootstrap_import, &redist] (operation_result& r, @@ -2440,7 +2658,7 @@ build (size_t argc, const char* argv[]) // Update. // - // bpkg update <env-config-args> <config-args> <package-name> + // bpkg update <env-config-args> <tgt-config-args> <package-name> // { step_id b (installed @@ -2458,8 +2676,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", "update", - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), import, pkg); @@ -2472,7 +2690,7 @@ build (size_t argc, const char* argv[]) // Note that we assume that the package supports the test operation // since this is its main purpose. // - // bpkg test <env-config-args> <config-args> <package-name> + // bpkg test <env-config-args> <tgt-config-args> <package-name> // { step_id b (installed @@ -2491,8 +2709,8 @@ build (size_t argc, const char* argv[]) "-v", "test", "--package-cwd", // See above for details. - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), import, pkg); @@ -2532,7 +2750,7 @@ build (size_t argc, const char* argv[]) // much sense, thus we don't pass the config.import.* variable on // the command line for modules that require bootstrap. // - // bpkg test <env-config-args> <config-args> <package-name> + // bpkg test <env-config-args> <tgt-config-args> <package-name> // step_id b (step_id::bpkg_test); step_id s (step_id::bpkg_test); @@ -2544,8 +2762,8 @@ build (size_t argc, const char* argv[]) "-v", "test", "--package-cwd", - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), pkg); if (!r.status) @@ -2614,7 +2832,7 @@ build (size_t argc, const char* argv[]) if (!target_pkg && create_target) rm_r (trace, &r.log, rwd / target_conf); - // bpkg install <env-config-args> <config-args> <package-name> + // bpkg install <env-config-args> <tgt-config-args> <package-name> // step_id b (step_id::bpkg_install); step_id s (step_id::bpkg_install); @@ -2625,8 +2843,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", "install", - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), pkg); if (!r.status) @@ -2699,7 +2917,7 @@ build (size_t argc, const char* argv[]) { // Create the configuration. // - // b create(<dir>, <env-modules>) <env-config-args> <config-args> + // b create(<dir>, <env-modules>) <env-config-args> <tgt-config-args> // // Amalgamation directory that will contain configuration subdirectory // for package tests out of source tree build. @@ -2725,8 +2943,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-V", "create('" + out_dir.representation () + '\'' + mods + ')', - step_args (env_args, s, f), - step_args (config_args, s, f)); + step_args (env_args, s, f), + step_args (tgt_args, s, f)); if (!r.status) break; @@ -2735,11 +2953,16 @@ build (size_t argc, const char* argv[]) // Configure testable subprojects sequentially and test/build in // parallel afterwards. // + // It feels right to configure internal tests also passing the main + // package configuration variables, since they may need to align with + // the main package setup (enable some testscripts, etc). + // strings test_specs; for (const dir_path& d: subprj_dirs) { // b configure(<subprj-src-dir>@<subprj-out-dir>) <env-config-args> - // <config-args> + // <tgt-config-args> + // <pkg-vars> // step_id b (step_id::b_test_installed_configure); step_id s (step_id::b_test_installed_configure); @@ -2760,8 +2983,9 @@ build (size_t argc, const char* argv[]) "configure('" + subprj_src_dir.representation () + "'@'" + subprj_out_dir.representation () + "')", - step_args (env_args, s, f), - step_args (config_args, s, f)); + step_args (env_args, s, f), + step_args (tgt_args, s, f), + pkg_config_vars); if (!r.status) break; @@ -2775,7 +2999,7 @@ build (size_t argc, const char* argv[]) // Build/test subprojects. // - // b test(<subprj-out-dir>)... <env-config-args> <config-args> + // b test(<subprj-out-dir>)... <env-config-args> <tgt-config-args> // { step_id b (step_id::b_test_installed_test); @@ -2788,8 +3012,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", test_specs, - step_args (env_args, s), - step_args (config_args, s)); + step_args (env_args, s), + step_args (tgt_args, s)); if (!r.status) break; @@ -2842,7 +3066,7 @@ build (size_t argc, const char* argv[]) // Create the target configuration. // - // bpkg create <env-modules> <env-config-args> <config-args> + // bpkg create <env-modules> <env-config-args> <tgt-config-args> // if (create_target) { @@ -2874,9 +3098,9 @@ build (size_t argc, const char* argv[]) "-V", "create", "-d", target_conf, - step_args (modules, s, f), - step_args (env_args, s, f), - step_args (config_args, s, f)); + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f)); if (!r.status) break; @@ -2886,7 +3110,7 @@ build (size_t argc, const char* argv[]) // if (create_host) { - // bpkg create --type host <env-modules> <env-config-args> <config-args> + // bpkg create --type host <env-modules> <env-config-args> <tgt-config-args> // step_id b (step_id::bpkg_test_separate_installed_create); @@ -2910,9 +3134,9 @@ build (size_t argc, const char* argv[]) "-d", host_conf, "--type", "host", "--name", "host", - step_args (modules, s, f), - step_args (env_args, s, f), - step_args (config_args, s, f)); + step_args (modules, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f)); if (!r.status) break; @@ -3022,7 +3246,7 @@ build (size_t argc, const char* argv[]) // if (has_runtime_tests) { - // bpkg add <env-config-args> <config-args> <repository-url> + // bpkg add <env-config-args> <tgt-config-args> <repository-url> // { step_id b (step_id::bpkg_test_separate_installed_configure_add); @@ -3036,15 +3260,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", runtime_tests_conf, - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), repo); if (!r.status) break; } - // bpkg fetch <env-config-args> <config-args> <trust-options> + // bpkg fetch <env-config-args> <tgt-config-args> <trust-options> // { step_id b (step_id::bpkg_test_separate_installed_configure_fetch); @@ -3058,8 +3282,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", runtime_tests_conf, - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), trust_ops); if (!r.status) @@ -3069,7 +3293,7 @@ build (size_t argc, const char* argv[]) if (has_buildtime_tests) { - // bpkg add <env-config-args> <config-args> <repository-url> + // bpkg add <env-config-args> <tgt-config-args> <repository-url> // { step_id b (step_id::bpkg_test_separate_installed_configure_add); @@ -3083,15 +3307,15 @@ build (size_t argc, const char* argv[]) "-v", "add", "-d", target_conf, - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), repo); if (!r.status) break; } - // bpkg fetch <env-config-args> <config-args> <trust-options> + // bpkg fetch <env-config-args> <tgt-config-args> <trust-options> // { step_id b (step_id::bpkg_test_separate_installed_configure_fetch); @@ -3105,8 +3329,8 @@ build (size_t argc, const char* argv[]) "-v", "fetch", "-d", target_conf, - step_args (env_args, s, f), - step_args (config_args, s, f), + step_args (env_args, s, f), + step_args (tgt_args, s, f), trust_ops); if (!r.status) @@ -3116,7 +3340,7 @@ build (size_t argc, const char* argv[]) // Configure all the packages using a single bpkg-pkg-build command. // - // bpkg build --configure-only <env-config-args> <config-args> + // bpkg build --configure-only <env-config-args> <tgt-config-args> // { <config> }+ { <runtime-test>... } // <buildtime-test>... // ?sys:<pkg> @@ -3194,10 +3418,10 @@ build (size_t argc, const char* argv[]) "--checkout-root", dist_installed_root, "--yes", "-d", root_conf, - step_args (env_args, g), - step_args (env_args, s, f), - step_args (config_args, g), - step_args (config_args, s, f), + step_args (env_args, g), + step_args (env_args, s, f), + step_args (tgt_args, g), + step_args (tgt_args, s, f), "--", pkg_args); @@ -3252,7 +3476,7 @@ build (size_t argc, const char* argv[]) change_wd (trace, &r.log, install_conf); - // bpkg uninstall <env-config-args> <config-args> <package-name> + // bpkg uninstall <env-config-args> <tgt-config-args> <package-name> // step_id b (step_id::bpkg_uninstall); step_id s (step_id::bpkg_uninstall); @@ -3263,8 +3487,8 @@ build (size_t argc, const char* argv[]) bkp_step, bkp_status, last_cmd, "-v", "uninstall", - step_args (env_args, s), - step_args (config_args, s), + step_args (env_args, s), + step_args (tgt_args, s), pkg); if (!r.status) diff --git a/doc/manual.cli b/doc/manual.cli index 3b3bd4b..59b20c7 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -90,12 +90,12 @@ build logs hosted by the controller. \h#arch-machine-config|Configurations| -The \c{bbot} architecture distinguishes between a \i{machine configuration} -and a \i{build configuration}. The machine configuration captures the -operating system, installed compiler toolchain, and so on. The same build -machine may be used to \"generate\" multiple \i{build configurations}. For -example, the same machine can normally be used to produce 32/64-bit and -debug/optimized builds. +The \c{bbot} architecture distinguishes between a \i{machine configuration}, +\i{build target configuration}, and a \i{build package configuration}. The +machine configuration captures the operating system, installed compiler +toolchain, and so on. The same build machine may be used to \"generate\" +multiple \i{build target configurations}. For example, the same machine can +normally be used to produce 32/64-bit and debug/optimized builds. The machine configuration is \i{approximately} encoded in its \i{machine name}. The machine name is a list of components separated with \c{-}. @@ -156,22 +156,31 @@ linux_ubuntu_16.04-gcc_6.3 aarch64_linux_debian_11-gcc_12.2 \ -Similarly, the build configuration is encoded in a \i{configuration name} -using the same overall format. As described in \l{#arch-controller Controller -Logic}, build configurations are generated from machine configurations. As a -result, it usually makes sense to have the first component identify the -operating systems and the second component \- the toolchain with the rest -identifying a particular build configuration variant, for example, optimized, -sanitized, etc. For example: +Similarly, the build target configuration is encoded in a \i{configuration +name} using the same overall format. As described in \l{#arch-controller +Controller Logic}, target configurations are generated from machine +configurations. As a result, it usually makes sense to have the first +component identify the operating systems and the second component \- the +toolchain with the rest identifying a particular target configuration variant, +for example, optimized, sanitized, etc. For example: \ windows-vc_14-O2 linux-gcc_6-O3_asan \ -While we can also specify the \c{<arch>} component in a build configuration, -this information is best conveyed as part of \c{<target>} as described in -\l{#arch-controller Controller Logic}. +While we can also specify the \c{<arch>} component in a build target +configuration, this information is best conveyed as part of \c{<target>} as +described in \l{#arch-controller Controller Logic}. + +A package can be built in multiple package configurations per target +configuration. A build package configuration normally specifies the package +configuration variables that need to be used for the build. It may also +include the information regarding the dependency packages which need to +additionally be configured. The build package configurations originate from +the package manifest \c{*-build-config}, \c{*-builds}, \c{*-build-include}, +and \c{*-build-exclude} values. See \l{bpkg#manifest-package Package Manifest} +for more information on these values. \h#arch-machine-header|Machine Header Manifest| @@ -348,7 +357,8 @@ repository-url: <repository-url> machine: <machine-name> target: <target-triplet> [environment]: <environment-name> -[config]: <config-args> +[target-config]: <tgt-config-args> +[package-config]: <pkg-config-args> [host]: true|false [warning-regex]: <warning-regex> [interactive]: <breakpoint> @@ -469,24 +479,50 @@ The name of the build environment to use. See \l{#arch-worker Worker Logic} for details. -\h2#arch-task-config|\c{config}| +\h2#arch-task-target-config|\c{target-config}| \ -[config]: <config-args> +[target-config]: <tgt-config-args> \ -The additional configuration options and variables. A single level of quotes +The additional target configuration options and variables. A single level of +quotes (either single or double) is removed in each value before being passed +to \c{bpkg}. For example, the following value: + +\ +target-config: config.cc.coptions=\"-O3 -stdlib='libc++'\" +\ + +Will be passed to \c{bpkg} as the following (single) argument: + +\ +config.cc.coptions=-O3 -stdlib='libc++' +\ + +Values can be separated with spaces or newlines. See \l{#arch-controller +Controller Logic} for details. + + +\h2#arch-task-package-config|\c{package-config}| + +\ +[package-config]: <pkg-config-args> +\ + +The primary package manifest \c{*-build-config} value for the build +configuration the build task is issued for. See \l{bpkg#manifest-package +Package Manifest} for more information on this value. A single level of quotes (either single or double) is removed in each value before being passed to \c{bpkg}. For example, the following value: \ -config: config.cc.coptions=\"-O3 -stdlib='libc++'\" +package-config: \"?libcurl ~7.76.0\" \ Will be passed to \c{bpkg} as the following (single) argument: \ -config.cc.coptions=-O3 -stdlib='libc++' +?libcurl ~7.76.0 \ Values can be separated with spaces or newlines. See \l{#arch-controller @@ -499,8 +535,9 @@ Controller Logic} for details. [host]: true|false \ -If \c{true}, then the build configuration is self-hosted. If not specified, -\c{false} is assumed. See \l{#arch-controller Controller Logic} for details. +If \c{true}, then the build target configuration is self-hosted. If not +specified, \c{false} is assumed. See \l{#arch-controller Controller Logic} for +details. \h2#arch-task-warning-regex|\c{warning-regex}| @@ -924,20 +961,24 @@ modules (\c{<env-modules>}) and the list of configuration options and variables The re-executed \c{bbot} worker then proceeds to test the package from the repository by executing the following commands, collectively called a \i{worker script}. Each command has a unique \i{step id} that can be used as a -breakpoint and normally as a prefix in the \c{<config-args>}, +breakpoint and normally as a prefix in the \c{<tgt-config-args>}, \c{<env-config-args>}, and \c{<env-modules>} values as discussed in \l{#arch-controller Controller Logic}. The \c{<>}-values are from the task manifest and the environment though some are assigned by the worker during the -script execution (configuration directories, UUIDs, etc). +script execution (configuration directories, UUIDs, etc). In particular, the +\c{<pkg-config-vars>}, \c{<dependency-name>}, +\c{<dependency-version-constraint>}, and \c{<dep-config-vars>} values result +from parsing the \l{#arch-task-package-config \c{package-config}} task +manifest value. Some prefix step ids have fallback step ids which are used in the absence of the primary step id values. If the prefix step id differs from the breakpoint step id and/or has the fallback step ids, then they are listed in parenthesis: the prefix id before the colon and the fallback ids after it. -Some commands have no configuration or environment options or variables. Such -commands have only breakpoint step ids associated, which are listed in square -brackets. +Some commands have no target configuration or environment options or +variables. Such commands have only breakpoint step ids associated, which are +listed in square brackets. Note that the worker script varies for different primary package types. The \c{bbot} worker classifies the primary package based on the configuration type @@ -952,19 +993,21 @@ by always using the \c{bpkg.global.configure.build} prefix step id for global (as opposed to package-specific) \l{bpkg-pkg-build(1)} options. The \c{bpkg.global.configure.build} prefix id has no fallback ids. -Note finally that the worker adds the \c{config.<name>.develop=false} -configuration variables for the main and external test packages at the -\c{bpkg.configure.build} step to trigger their package skeleton creation and -loading. This makes sure that these packages can be used as dependencies of -dependents with configuration clauses. To keep the below listings concise, -these variables are not shown. +Note finally that if no configuration variables are specified in the main +package configuration, then the worker adds the +\c{config.<name>.develop=false} configuration variable for the main package at +the \c{bpkg.configure.build} step to trigger its package skeleton creation and +loading. It also adds this variable for external test packages at this step +and for the same purpose. This makes sure that these packages can be used as +dependencies of dependents with configuration clauses. To keep the below +listings concise, these variables are not shown. Worker script for \c{target} packages: \ # bpkg.create (bpkg.target.create : b.create, bpkg.create) # -bpkg -V create <env-modules> <env-config-args> <config-args> +bpkg -V create <env-modules> <env-config-args> <tgt-config-args> # bpkg.configure.add # @@ -978,9 +1021,11 @@ bpkg -v fetch --trust <repository-fp> # bpkg.global.configure.build, # (bpkg.target.configure.build : b.configure, bpkg.configure.build)) # -bpkg -v build --configure-only <env-config-args> <config-args> \\ - <package-name>/<package-version> \\ - [<test-package-name>[ <test-version-constraint>]...] +bpkg -v build --configure-only <env-config-args> <tgt-config-args> \\ + [{ <pkg-config-vars> }+] <package-name>/<package-version> \\ + [<test-package-name>[ <test-version-constraint>]...] \\ + [([{ <dep-config-vars> }+] \\ + ?[sys:]<dependency-name>[ <dependency-version-constraint>])...] # bpkg.update # @@ -1020,14 +1065,14 @@ bpkg -v update <package-name> { # b.test-installed.create ( : b.create) # - b -V create <env-modules> <env-config-args> <config-args> + b -V create <env-modules> <env-config-args> <tgt-config-args> # For each test subproject: # { # b.test-installed.configure ( : b.configure) # - b -v configure + b -v configure [<pkg-config-vars>] } # b.test-installed.test @@ -1043,7 +1088,7 @@ bpkg -v update <package-name> # bpkg.test-separate-installed.create_for_target : # bpkg.test-separate-installed.create) # - bpkg -V create <env-modules> <env-config-args> <config-args> + bpkg -V create <env-modules> <env-config-args> <tgt-config-args> # bpkg.test-separate-installed.configure.add ( # : bpkg.configure.add) @@ -1060,7 +1105,7 @@ bpkg -v update <package-name> # (bpkg.test-separate-installed.configure.build_for_target : # bpkg.test-separate-installed.configure.build)) # - bpkg -v build --configure-only <env-config-args> <config-args> \\ + bpkg -v build --configure-only <env-config-args> <tgt-config-args> \\ <test-package-name>[ <test-version-constraint>]... \\ ?sys:<package-name> @@ -1097,7 +1142,7 @@ Worker script for \c{host} packages: # bpkg.create (bpkg.host.create : b.create, bpkg.create) # bpkg -V create --type host -d <host-conf> \\ - <env-modules> <env-config-args> <config-args> + <env-modules> <env-config-args> <tgt-config-args> } # # Otherwise: @@ -1123,7 +1168,7 @@ bpkg -v fetch -d <host-conf> --trust <repository-fp> # bpkg.create (bpkg.target.create : b.create, bpkg.create) # bpkg -V create -d <install-conf> \\ - <env-modules> <env-config-args> <config-args> + <env-modules> <env-config-args> <tgt-config-args> # [bpkg.link] # @@ -1145,7 +1190,7 @@ bpkg -v fetch -d <host-conf> --trust <repository-fp> # bpkg.create (bpkg.target.create : b.create, bpkg.create) # bpkg -V create -d <target-conf> \\ - <env-modules> <env-config-args> <config-args> + <env-modules> <env-config-args> <tgt-config-args> # [bpkg.create] # @@ -1192,19 +1237,25 @@ bpkg -v fetch -d <host-conf> --trust <repository-fp> # - All parts have the same fallback step ids: b.configure and # bpkg.configure.build. # -bpkg -v build --configure-only <env-config-args> <config-args> \\ +bpkg -v build --configure-only <env-config-args> <tgt-config-args> \\ \\ -{ --config-uuid <host-uuid> <env-config-args> <config-args> }+ \\ +{ --config-uuid <host-uuid> \\ + <env-config-args> <tgt-config-args> [<pkg-config-vars>] }+ \\ <package-name>/<package-version> \\ \\ -{ --config-uuid <install-uuid> <env-config-args> <config-args> }+ \\ +{ --config-uuid <install-uuid> \\ + <env-config-args> <tgt-config-args> [<pkg-config-vars>] }+ \\ <package-name>/<package-version> \\ \\ -{ --config-uuid <host-uuid> <env-config-args> <config-args> }+ \\ +{ --config-uuid <host-uuid> <env-config-args> <tgt-config-args> }+ \\ { <runtime-test-package-name>[ test-version-constraint>]... } \\ \\ -{ --config-uuid <target-uuid> <env-config-args> <config-args> }+ \\ -{ <buildtime-test-package-name>[ test-version-constraint>]... } +{ --config-uuid <target-uuid> <env-config-args> <tgt-config-args> }+ \\ +{ <buildtime-test-package-name>[ test-version-constraint>]... } \\ +\\ +({ --config-uuid <host-uuid> [--config-uuid <install-uuid>] \\ + [<dep-config-vars>] }+ \\ + ?[sys:]<dependency-name>[ <dependency-version-constraint>])... # bpkg.update # @@ -1257,14 +1308,14 @@ bpkg -v update -d <host-conf> <package-name> { # b.test-installed.create ( : b.create) # - b -V create <env-modules> <env-config-args> <config-args> + b -V create <env-modules> <env-config-args> <tgt-config-args> # For each test subproject: # { # b.test-installed.configure ( : b.configure) # - b -v configure + b -v configure [<pkg-config-vars>] } # b.test-installed.test @@ -1281,7 +1332,7 @@ bpkg -v update -d <host-conf> <package-name> # bpkg.test-separate-installed.create) # bpkg -V create --type host -d <host-conf> \\ - <env-modules> <env-config-args> <config-args> + <env-modules> <env-config-args> <tgt-config-args> # If task manifest refers to any runtime tests, examples, or # benchmarks packages: @@ -1307,7 +1358,7 @@ bpkg -v update -d <host-conf> <package-name> # bpkg.test-separate-installed.create) # bpkg -V create -d <target-conf> \\ - <env-modules> <env-config-args> <config-args> + <env-modules> <env-config-args> <tgt-config-args> # [bpkg.test-separate-installed.create] # @@ -1339,7 +1390,7 @@ bpkg -v update -d <host-conf> <package-name> # Note that any of the runtime or build-time tests related parts # (but not both) may be omitted. # - bpkg -v build --configure-only <env-config-args> <config-args> \\ + bpkg -v build --configure-only <env-config-args> <tgt-config-args> \\ \\ { --config-name <host-conf> }+ \\ { <runtime-test-package-name>[ <test-version-constraint>]... } \\ @@ -1382,7 +1433,7 @@ Worker script for \c{module} packages: # bpkg.create (bpkg.module.create) # b -V create(<module-conf>, <env-modules>) config.config.load=~build2 \\ - <env-config-args> <config-args> + <env-config-args> <tgt-config-args> bpkg -v create --existing --type build2 -d <module-conf> } # @@ -1409,7 +1460,7 @@ bpkg -v fetch -d <module-conf> --trust <repository-fp> # bpkg.create (bpkg.module.create) # b -V create(<install-conf>, <env-modules>) \\ - config.config.load=~build2 <env-config-args> <config-args> + config.config.load=~build2 <env-config-args> <tgt-config-args> bpkg -v create --existing --type build2 -d <install-conf> # bpkg.configure.add @@ -1428,7 +1479,7 @@ bpkg -v fetch -d <module-conf> --trust <repository-fp> # bpkg.create (bpkg.target.create : b.create, bpkg.create) # bpkg -V create -d <target-conf> \\ - <env-modules> <env-config-args> <config-args> + <env-modules> <env-config-args> <tgt-config-args> # [bpkg.create] # @@ -1466,16 +1517,22 @@ bpkg -v fetch -d <module-conf> --trust <repository-fp> # - All parts have the same fallback step ids: b.configure and # bpkg.configure.build. # -bpkg -v build --configure-only <env-config-args> <config-args> \\ +bpkg -v build --configure-only <env-config-args> <tgt-config-args> \\ \\ -{ --config-uuid <module-uuid> <env-config-args> <config-args> }+ \\ +{ --config-uuid <module-uuid> \\ + <env-config-args> <tgt-config-args> [<pkg-config-vars>] }+ \\ <package-name>/<package-version> \\ \\ -{ --config-uuid <install-uuid> <env-config-args> <config-args> }+ \\ +{ --config-uuid <install-uuid> \\ + <env-config-args> <tgt-config-args> [<pkg-config-vars>] }+ \\ <package-name>/<package-version> \\ \\ -{ --config-uuid <target-uuid> <env-config-args> <config-args> }+ \\ -{ <buildtime-test-package-name>[ test-version-constraint>]... } +{ --config-uuid <target-uuid> <env-config-args> <tgt-config-args> }+ \\ +{ <buildtime-test-package-name>[ test-version-constraint>]... } \\ +\\ +({ --config-uuid <host-uuid> [--config-uuid <install-uuid>] \\ + [<dep-config-vars>] }+ \\ + ?[sys:]<dependency-name>[ <dependency-version-constraint>])... # bpkg.update # @@ -1523,14 +1580,14 @@ bpkg -v update -d <module-conf> <package-name> # bpkg.test-separate-installed.create) # bpkg -V create -d <target-conf> \\ - <env-modules> <env-config-args> <config-args> + <env-modules> <env-config-args> <tgt-config-args> # bpkg.test-separate-installed.create ( # bpkg.test-separate-installed.create_for_module : # bpkg.test-separate-installed.create) # bpkg -V create --type host -d <host-conf> \\ - <env-modules> <env-config-args> <config-args> + <env-modules> <env-config-args> <tgt-config-args> # [bpkg.test-separate-installed.link] # @@ -1553,7 +1610,7 @@ bpkg -v update -d <module-conf> <package-name> # (bpkg.test-separate-installed.configure.build_for_module : # bpkg.test-separate-installed.configure.build)) # - bpkg -v build --configure-only <env-config-args> <config-args> \\ + bpkg -v build --configure-only <env-config-args> <tgt-config-args> \\ \\ { --config-name <target-conf> }+ \\ <buildtime-test-package-name>[ <test-version-constraint>]... \\ @@ -1635,49 +1692,50 @@ exec \"$@\" cc config.c=\"gcc-9 $mode\" config.cxx=\"g++-9 $mode\" \h#arch-controller|Controller Logic| A \c{bbot} controller that issues own build tasks maps available build -machines (as reported by agents) to \i{build configurations} according to the -\c{buildtab} configuration file. Blank lines and lines that start with \c{#} -are ignored. All other lines in this file have the following format: +machines (as reported by agents) to \i{build target configurations} according +to the \c{buildtab} configuration file. Blank lines and lines that start with +\c{#} are ignored. All other lines in this file have the following format: \ -<machine-pattern> <config> <target>[/<environment>] <classes> [<config-arg>]* [<warning-regex>]* +<machine-pattern> <target-config> <target>[/<environment>] <classes> [<tgt-config-arg>]* [<warning-regex>]* -<config-arg> = [<prefix>:](<variable>|<option>) +<tgt-config-arg> = [<prefix>:](<variable>|<option>) <prefix> = <tool>[.<phase>][.<operation>[.<command>]] \ Where \c{<machine-pattern>} is filesystem wildcard pattern that is matched -against available machine names, \c{<config>} is the configuration name, -\c{<target>} is the build target, optional \c{<environment>} is the build -environment name, \c{<classes>} is a space-separated list of configuration -classes that is matched against the package \c{builds} values, optional -\c{<config-arg>} list is additional configuration options and variables, and -optional \c{<warning-regex>} list is additional regular expressions that -should be used to detect warnings in the logs. - -The build configurations can belong to multiple classes with their names -reflecting some common configuration aspects, such as the operating system, -compiler, build options, etc. Predefined class names are \c{default}, \c{all}, -\c{none}, \c{host}, and \c{build2}. The default configurations are built by -default. A configuration must also belong to the \c{all} unless it is hidden. -A configuration that is self-hosted must also belong to the \c{host} class -and, if it is also self-hosted for build system modules, to the \c{build2} -class. Valid custom class names must contain only alpha-numeric characters, -\c{_}, \c{+}, \c{-}, and \c{.}, except as the first character for the last -three. Class names that start with \c{_} are reserved for the future +against available machine names, \c{<target-config>} is the target +configuration name, \c{<target>} is the build target, optional +\c{<environment>} is the build environment name, \c{<classes>} is a +space-separated list of configuration classes that is matched against the +package configuration \c{*-builds} values, optional \c{<tgt-config-arg>} list +is additional configuration options and variables, and optional +\c{<warning-regex>} list is additional regular expressions that should be used +to detect warnings in the logs. + +The build target configurations can belong to multiple classes with their +names reflecting some common configuration aspects, such as the operating +system, compiler, build options, etc. Predefined class names are \c{default}, +\c{all}, \c{none}, \c{host}, and \c{build2}. The default target configurations +are built by default. A configuration must also belong to the \c{all} unless +it is hidden. A configuration that is self-hosted must also belong to the +\c{host} class and, if it is also self-hosted for build system modules, to the +\c{build2} class. Valid custom class names must contain only alpha-numeric +characters, \c{_}, \c{+}, \c{-}, and \c{.}, except as the first character for +the last three. Class names that start with \c{_} are reserved for the future hidden/special class functionality. -Regular expressions must start with \c{~}, to be distinguished from -configuration options and variables. Note that the \c{<config-arg>} and -\c{<warning-regex>} lists have the same quoting semantics as in the \c{config} -and the \c{warning-regex} value in the build task manifest. The matched -machine name, the target, the environment name, configuration -options/variables, and regular expressions are included into the build task -manifest. - -Values in the \c{<config-arg>} list can be opionally prefixed with the \i{step -id} or a leading portion thereof to restrict it to a specific step, operation, -phase, or tool in the \i{worker script} (see \l{#arch-worker Worker +Regular expressions must start with \c{~}, to be distinguished from target +configuration options and variables. Note that the \c{<tgt-config-arg>} and +\c{<warning-regex>} lists have the same quoting semantics as in the +\c{target-config} and the \c{warning-regex} value in the build task +manifest. The matched machine name, the target, the environment name, +configuration options/variables, and regular expressions are included into the +build task manifest. + +Values in the \c{<tgt-config-arg>} list can be opionally prefixed with the +\i{step id} or a leading portion thereof to restrict it to a specific step, +operation, phase, or tool in the \i{worker script} (see \l{#arch-worker Worker Logic}). Unprefixed values only apply to the \c{*.create[_for_*]} steps. Note that options with values can only be specified using the single argument notation. For example: @@ -1687,15 +1745,16 @@ bpkg:--fetch-timeout=600 bpkg.configure.fetch:--fetch-timeout=60 b:-j1 \ Note that each machine name is matched against every pattern and all the -patterns that match produce a configuration. If a machine does not match any -pattern, then it is ignored (meaning that this controller is not interested in -testing its packages with this machine). If multiple machines match the same -pattern, then only a single configuration using any of the machines is -produced (meaning that this controller considers these machines equivalent). +patterns that match produce a target configuration. If a machine does not +match any pattern, then it is ignored (meaning that this controller is not +interested in testing its packages with this machine). If multiple machines +match the same pattern, then only a single target configuration using any of +the machines is produced (meaning that this controller considers these +machines equivalent). As an example, let's say we have a machine named \c{windows_10-vc_14.3}. If we wanted to test both 32 and 64-bit as well as debug and optimized builds, then -we could have generated the following configurations: +we could have generated the following target configurations: \ windows*-msvc_14* windows-msvc_14-Z7 i686-microsoft-win32-msvc14.0 \"all default msvc i686 debug\" config.cc.coptions=/Z7 config.cc.loptions=/DEBUG ~\"warning C4\d{3}: \" @@ -1722,7 +1781,7 @@ aarch64_linux_debian_11*-gcc_12.2 linux_debian_11-gcc_12.2 aarch64-linux-gnu ... As another example, let's say we have \c{linux_fedora_25-gcc_6} and \c{linux_ubuntu_16.04-gcc_6}. If all we cared about is testing GCC 6 64-bit -builds on Linux, then our configurations could look like this: +builds on Linux, then our target configurations could look like this: \ linux*-gcc_6 linux-gcc_6-g x86_64-linux-gnu \"all default gcc debug\" config.cc.coptions=-g @@ -1730,11 +1789,12 @@ linux*-gcc_6 linux-gcc_6-g x86_64-linux-gnu \"all default gcc debug\" config.cc. linux*-gcc_6 linux-gcc_6-O3 x86_64-linux-gnu \"all default gcc optimized\" config.cc.coptions=-O3 \ -A build configuration class can derive from another class in which case -configurations that belong to the derived class are treated as also belonging -to the base class (or classes, recursively). The derived and base class names -are separated with \c{:} (no leading or trailing spaces allowed) and the base -must be present in the first mentioning of the derived class. For example: +A build target configuration class can derive from another class in which case +target configurations that belong to the derived class are treated as also +belonging to the base class (or classes, recursively). The derived and base +class names are separated with \c{:} (no leading or trailing spaces allowed) +and the base must be present in the first mentioning of the derived class. For +example: \ linux*-gcc_6 linux-gcc_6-g x86_64-linux-gnu \"all gcc-6+ debug\" config.cc.coptions=-g @@ -1758,7 +1818,7 @@ linux*-gcc_6 linux-gcc_6 x86_64-linux-gnu \"all gcc-6+ \" linux*-gcc_8 linux-gcc_8 x86_64-linux-gnu \"all gcc-8+:gcc-7+\" \ -If the \c{<config-arg>} list contains the \c{config.install.root} variable +If the \c{<tgt-config-arg>} list contains the \c{config.install.root} variable that applies to the \c{bpkg.create} step, then in addition to building and possibly running tests, the \c{bbot} worker will also test installing and uninstalling each package. Furthermore, if the package contains subprojects @@ -1777,8 +1837,8 @@ preferable, it may not be always usable because of the potential conflicts with the already installed software, for example, by the system package manager. -As an example, the following two configurations could be used to test system -and private installations: +As an example, the following two target configurations could be used to test +system and private installations: \ linux*-gcc* linux-gcc-sysinstall x86_64-linux-gnu \"all default gcc\" config.install.root=/usr config.install.sudo=sudo @@ -1790,4 +1850,10 @@ Note also that while building and running tests against the installation the worker makes the \c{bin} subdirectory of \c{config.install.root} the first entry in the \c{PATH} environment variable, except for build system modules which supposedly don't install any executables. + +The \c{bbot} controller normally issues the build task by picking an unbuilt +package configuration and one of the produced (via the machine names match) +target configurations, which is not excluded from building due to this package +configuration \c{*-builds}, \c{*-build-include}, and \c{*-build-exclude} +manifest values. " diff --git a/tests/integration/testscript b/tests/integration/testscript index e60e09c..16d7002 100644 --- a/tests/integration/testscript +++ b/tests/integration/testscript @@ -1,6 +1,9 @@ # file : tests/integration/testscript # license : MIT; see accompanying LICENSE file +# NOTE: to see the worker execution progress run the test with the +# --no-diag-buffer option. + #\ # Requirement: # @@ -93,17 +96,18 @@ host='host: true' # #\ pkg = libbuild2-kconfig -ver = 0.2.0-a.0.20220629083402.1af9a330bb2b +ver = 0.3.0-a.0.20221118053819.f702eb65da87 rep_url = "https://github.com/build2/libbuild2-kconfig.git#master" rep_type = git #ver = 0.1.0-a.0.20200910053253.a71aa3f3938b #rep_url = https://stage.build2.org/1 #rep_type = pkg rfp = yes -requires='requires: bootstrap' -tests="tests: * libbuild2-kconfig-tests == $ver +requires = 'requires: bootstrap' +tests = "tests: * libbuild2-kconfig-tests == $ver examples: * kconfig-hello == $ver" -host='host: true' +host = 'host: true' +package_config = 'package-config: config.libbuild2-kconfig.develop=true' #dependency_checksum = 'dependency-checksum: 72ae02bed9a05aaf022147297a99b84d63b712e15d05cc073551da39003e87e8' #\ @@ -115,21 +119,36 @@ ver = 0.1.0-a.0.20220629084006.3c4928213194 rep_url = "https://github.com/build2/libbuild2-autoconf.git#master" rep_type = git rfp = yes -tests="tests: * libbuild2-autoconf-tests == $ver" -host='host: true' +tests = "tests: * libbuild2-autoconf-tests == $ver" +host = 'host: true' #\ #\ pkg = curl -ver = 7.76.0 -rep_url = https://pkg.cppget.org/1/testing +ver = 7.84.0 +rep_url = https://pkg.cppget.org/1/stable rep_type = pkg rfp = yes #\ #\ +pkg = bpkg +ver = 0.16.0-a.0.20221118054047.a6c6065d5c2a +rep_url = https://stage.build2.org/1 +rep_type = pkg +rfp = yes +package_config = 'package-config: +\ +config.bpkg.tests.remote=true +?libodb-sqlite +{ config.libodb_sqlite.develop=true } +?cli +{ config.cli.develop=true } +?sys:libsqlite3 +\' +#\ + +#\ pkg = cli -ver = 1.2.0-b.8.20220218142801.83de480a8bab +ver = 1.2.0-b.9.20221122090425.15876dcb479e rep_url = "https://git.codesynthesis.com/cli/cli.git#master" rep_type = git #rep_url = https://stage.build2.org/1 @@ -139,6 +158,7 @@ requires='requires: host' tests="tests: * cli-tests == $ver examples: * cli-examples == $ver" host='host: true' +package_config = 'package-config: ?libcutl +{ config.libcutl.develop=true }' #\ #\ @@ -154,7 +174,7 @@ tests="tests: libxsd-tests == $ver" #\ pkg = xsd -ver = 4.2.0-b.3.20220224113525.516981000564 +ver = 4.2.0-b.4.20221031121010.4fdbd8298c4d rep_url = "https://git.codesynthesis.com/xsd/xsd.git#master" rep_type = git #rep_url = https://queue.stage.build2.org/1 @@ -164,10 +184,30 @@ requires='requires: host' tests="tests: * xsd-tests == $ver examples: * xsd-examples == $ver" host='host: true' +package_config = 'package-config: +\ +?libxerces-c +{ config.libxerces_c.network=true } +?libcurl/7.76.0 +?sys:libz +\' #dependency_checksum = 'dependency-checksum: 40a0ad4546d836a3afc83a9e7da22f2b5d224af4e62996d88f7103eaee23e9e1' #\ #\ +pkg = libxerces-c +ver = 3.2.4 +rep_url = https://pkg.cppget.org/1/stable +rep_type = pkg +rfp = yes +package_config = 'package-config: +\ +config.libxerces_c.network=true +"?libcurl ~7.76.0" +?sys:libz +\' +#\ + +#\ pkg = odb ver = 2.5.0-b.22.20220629083600.4a9af07ee566 rep_url = "https://git.codesynthesis.com/odb/odb.git#master" @@ -230,6 +270,7 @@ bpkg.test-separate-installed.create:\"config.cc.loptions=-L'$~/install/lib'\"" machine: $machine target: $target config: $config + $package_config $interactive $host worker-checksum: 1 |