// if (create_target) { step_id b (step_id::bpkg_create); // Breakpoint. step_id s (step_id::bpkg_target_create); // Step. step_id f1 (step_id::b_create); // First fallback. step_id f2 (step_id::bpkg_create); // Second fallback. r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create", "-d", target_conf, !target_pkg ? cstrings ({"--uuid", target_uuid}) : cstrings (), step_args (modules, s, f1, f2), step_args (env_args, s, f1, f2), step_args (config_args, s, f1, f2)); if (!r.status) break; } // Create the host configurations. // if (create_host) { step_id b (step_id::bpkg_create); if (host_pkg && selfhost) { // Create the host configuration. // { step_id s (step_id::bpkg_host_create); step_id f1 (step_id::b_create); step_id f2 (step_id::bpkg_create); // bpkg create --type host

// r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create", "-d", host_conf, "--type", "host", "--uuid", host_uuid, step_args (modules, s, f1, f2), step_args (env_args, s, f1, f2), step_args (config_args, s, f1, f2)); if (!r.status) break; } // Create the install configuration. // // bpkg create

// if (create_install) { step_id s (step_id::bpkg_target_create); step_id f1 (step_id::b_create); step_id f2 (step_id::bpkg_create); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create", "-d", install_conf, "--uuid", install_uuid, step_args (modules, s, f1, f2), step_args (env_args, s, f1, f2), step_args (config_args, s, f1, f2)); if (!r.status) break; } } else { // b create() config.config.load=~host // // Note also that we suppress warnings about unused config.* values. // r.status |= run_b ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create(" + host_conf.representation () + ",cc)", "config.config.load=~host", "config.config.persist+='config.*'@unused=drop"); if (!r.status) break; // bpkg create --existing --type host // r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "create", "--existing", "-d", host_conf, "--type", "host", "--uuid", host_uuid); if (!r.status) break; } } // Create the module configurations. // if (create_module) { step_id b (step_id::bpkg_create); // Create the module configuration. // { // b create() config.config.load=~build2 [

] // // Note also that we suppress warnings about unused config.* values. // // What if a module wants to use CLI? The current thinking is that we // will be "whitelisting" base (i.e., those that can plausibly be used // by multiple modules) libraries and tools for use by build system // modules. So if and when we whitelist CLI, we will add it here, next // to cc. // string mods; cstrings eas; cstrings cas; if (module_pkg && selfhost) { step_id s (step_id::bpkg_module_create); for (const char* m: step_args (modules, s)) { if (!mods.empty ()) mods += ' '; mods += m; } eas = step_args (env_args, s); cas = step_args (config_args, s); } else mods = "cc"; r.status |= run_b ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create(" + module_conf.representation () + "," + mods + ")", "config.config.load=~build2", "config.config.persist+='config.*'@unused=drop", eas, cas); if (!r.status) break; // bpkg create --existing --type build2 // r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "create", "--existing", "-d", module_conf, "--type", "build2", "--uuid", module_uuid); if (!r.status) break; } // Create the install configuration. // if (create_install && module_pkg) { step_id s (step_id::bpkg_module_create); string mods; for (const char* m: step_args (modules, s)) { if (!mods.empty ()) mods += ' '; mods += m; } // b create() config.config.load=~build2 [

] // r.status |= run_b ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create(" + install_conf.representation () + "," + mods + ")", "config.config.load=~build2", "config.config.persist+='config.*'@unused=drop", step_args (env_args, s), step_args (config_args, s)); if (!r.status) break; // bpkg create --existing // r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "create", "--existing", "-d", install_conf, "--uuid", install_uuid); if (!r.status) break; } } // Link the configurations. // // bpkg link -d // { step_id b (step_id::bpkg_link); if (create_target) { if (create_host) { r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "link", "-d", target_conf, host_conf); if (!r.status) break; } if (create_module) { r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "link", "-d", target_conf, module_conf); if (!r.status) break; } } if (create_host) { if (create_module) { r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "link", "-d", host_conf, module_conf); if (!r.status) break; } } // Link the install configuration only for the host package. Note that // the module package may not have build-time dependencies and so // doesn't need configurations for them. // if (create_install && host_pkg) { r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "link", "-d", install_conf, host_conf); if (!r.status) break; if (create_module) { r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "link", "-d", install_conf, module_conf); if (!r.status) break; } } } // Fetch repositories into the main package configuration, the target // configuration for external build-time tests, if any, and the install // configuration, if present. // // bpkg add

// { step_id b (step_id::bpkg_configure_add); step_id s (step_id::bpkg_configure_add); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "add", "-d", main_pkg_conf, step_args (env_args, s), step_args (config_args, s), repo); if (!r.status) break; } // bpkg fetch

// { step_id b (step_id::bpkg_configure_add); step_id s (step_id::bpkg_configure_add); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "add", "-d", install_conf, step_args (env_args, s), step_args (config_args, s), repo); if (!r.status) break; } // bpkg fetch

// { step_id b (step_id::bpkg_configure_add); step_id s (step_id::bpkg_configure_add); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "add", "-d", target_conf, step_args (env_args, s), step_args (config_args, s), repo); if (!r.status) break; } // bpkg fetch

// { step_id b (step_id::bpkg_configure_fetch); step_id s (step_id::bpkg_configure_fetch); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "fetch", "-d", target_conf, step_args (env_args, s), step_args (config_args, s), trust_ops); if (!r.status) break; } } // Configure all the packages using a single bpkg-pkg-build command. // // First, prepare the common and package arguments. // strings common_args; strings pkg_args; if (target_pkg) // The simple common case (see above)? { // The overall command looks like this: // // bpkg build --configure-only

-- //

... // step_id s (step_id::bpkg_target_configure_build); step_id f1 (step_id::b_configure); step_id f2 (step_id::bpkg_configure_build); cstrings eas (step_args (env_args, s, f1, f2)); cstrings cas (step_args (config_args, s, f1, f2)); common_args.push_back ("--checkout-root"); common_args.push_back (dist_root.string ()); common_args.insert (common_args.end (), eas.begin (), eas.end ()); common_args.insert (common_args.end (), cas.begin (), cas.end ()); pkg_args.push_back (pkg_rev); // Add test dependency package constraints (for example 'bar > 1.0.0'). // for (auto t: runtime_tests) pkg_args.push_back (t.string ()); } else { // The overall command looks like this (but some parts may be omitted): // // bpkg build --configure-only

// step_id b (step_id::bpkg_update); step_id s (step_id::bpkg_update); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "update", step_args (env_args, s), step_args (config_args, s), pkg); if (!r.status) break; rm.status |= r.status; } // Re-distribute if comes from a version control-based repository, update, // and test external test packages in the bpkg configuration in the // current working directory. Optionally pass the config.import.* variable // override and/or set the environment variables for the bpkg processes. // Return true if all operations for all packages succeeded. // // Pass true as the installed argument to use the test separate installed // phase step ids (bpkg.test-separate-installed.*) and the test separate // phase step ids (bpkg.test-separate.*) otherwise. In both cases fall // back to the main phase step ids (bpkg.*) when no environment/ // configuration arguments are specified for them. // auto test = [&trace, &wre, &bkp_step, &bkp_status, &last_cmd, &step_args, &config_args, &env_args, &bootstrap_import, &redist] (operation_result& r, const small_vector& tests, const dir_path& dist_root, bool installed, const small_vector& envvars = {}) { const optional& import (!installed ? bootstrap_import : nullopt); for (const test_dependency& td: tests) { const string& pkg (td.name.string ()); // Re-distribute. // if (exists (dist_root)) { // Note that re-distributing the test package is a bit tricky since // we don't know its version and so cannot deduce its source // directory name easily. We could potentially run the bpkg-status // command after the package is configured and parse the output to // obtain the version. Let's, however, keep it simple and find the // source directory using the package directory name pattern. // try { dir_path pkg_dir; path_search (dir_path (pkg + "-*/"), [&pkg_dir] (path&& pe, const string&, bool interm) { if (!interm) pkg_dir = path_cast (move (pe)); return interm; }, dist_root); if (!pkg_dir.empty ()) { step_id b ( installed ? step_id::bpkg_test_separate_installed_configure_build : step_id::bpkg_configure_build); if (!redist (b, r, dist_root, pkg_dir, import, envvars)) return false; } } catch (const system_error& e) { fail << "unable to scan directory " << dist_root << ": " << e; } } // Update. // // bpkg update

// { step_id b (installed ? step_id::bpkg_test_separate_installed_update : step_id::bpkg_test_separate_update); step_id s (b); step_id f (step_id::bpkg_update); r.status |= run_bpkg ( b, envvars, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "update", step_args (env_args, s, f), step_args (config_args, s, f), import, pkg); if (!r.status) return false; } // Test. // // Note that we assume that the package supports the test operation // since this is its main purpose. // // bpkg test

// { step_id b (installed ? step_id::bpkg_test_separate_installed_test : step_id::bpkg_test_separate_test); step_id s (b); step_id f (step_id::bpkg_test); r.status |= run_bpkg ( b, envvars, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "test", "--package-cwd", // See above for details. step_args (env_args, s, f), step_args (config_args, s, f), import, pkg); if (!r.status) return false; } } return true; }; // Test the main package. // b_project_info prj (prj_info (pkg_dir, true /* ext_mods */, "project")); // Run the internal tests if the test operation is supported by the // project but only for the target package or if the configuration is // self-hosted. // bool has_internal_tests ((target_pkg || selfhost) && find (prj.operations.begin (), prj.operations.end (), "test") != prj.operations.end ()); if (has_internal_tests || has_runtime_tests || has_buildtime_tests) { operation_result& r (add_result ("test")); // Run internal tests. // if (has_internal_tests) { // Use --package-cwd to help ported to build2 third-party packages a // bit (see bpkg-pkg-test(1) for details). // // Note that internal tests that load the module itself don't make // much sense, thus we don't pass the config.import.* variable on // the command line for modules that require bootstrap. // // bpkg test

// step_id b (step_id::bpkg_test); step_id s (step_id::bpkg_test); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "test", "--package-cwd", step_args (env_args, s), step_args (config_args, s), pkg); if (!r.status) break; } // External runtime tests. // // Note that we assume that these packages belong to the dependent // package's repository or its complement repositories, recursively. // Thus, we test them in the configuration used to build the dependent // package. // if (has_runtime_tests) { if (!test (r, runtime_tests, dist_root, false /* installed */)) break; } // External build-time tests. // if (has_buildtime_tests) { change_wd (trace, &r.log, rwd / target_conf); if (!test (r, buildtime_tests, dist_root, false /* installed */)) break; } rm.status |= r.status; } // Install the package, optionally test the installation and uninstall // afterwards. // // These operations are triggered by presence of config.install.root // configuration variable having a non-empty value for // bpkg.configure.create step. // if (install_root.empty ()) break; // If this is not a self-hosted configuration, then skip installing host // and module packages. // if (!target_pkg && !selfhost) break; // Now the overall plan is as follows: // // 1. Install the package. // // 2. If the package has subprojects that support the test operation, then // configure, build, and test them out of the source tree against the // installed package using the build system directly. // // 3. If any of the test packages are specified, then configure, build, // and test them in a separate bpkg configuration(s) against the // installed package. // // 4. Uninstall the package. install_conf = rwd / (create_install ? install_conf : main_pkg_conf); // Install. // { operation_result& r (add_result ("install")); change_wd (trace, &r.log, install_conf); // Note that for a host or module package we don't need the target // configuration anymore, if present. So let's free up the space a // little bit. // if (!target_pkg && create_target) rm_r (trace, &r.log, rwd / target_conf); // bpkg install

// step_id b (step_id::bpkg_install); step_id s (step_id::bpkg_install); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "install", step_args (env_args, s), step_args (config_args, s), pkg); if (!r.status) break; rm.status |= r.status; } // Run the internal tests if the project contains "testable" subprojects, // but not for a module. // has_internal_tests = false; dir_paths subprj_dirs; // "Testable" package subprojects. if (!module_pkg) { // Collect the "testable" subprojects. // for (const b_project_info::subproject& sp: prj.subprojects) { // Retrieve the subproject information similar to how we've done it // for the package. // b_project_info si (prj_info (pkg_dir / sp.path, true /* ext_mods */, "subproject")); const strings& ops (si.operations); if (find (ops.begin (), ops.end (), "test") != ops.end ()) subprj_dirs.push_back (sp.path); } has_internal_tests = !subprj_dirs.empty (); } if (has_internal_tests || has_runtime_tests || has_buildtime_tests) { operation_result& r (add_result ("test-installed")); change_wd (trace, &r.log, rwd); // Make sure that the installed package executables are properly // imported when configuring and running tests, unless we are testing // the build system module (that supposedly doesn't install any // executables). // small_vector envvars; if (!module_pkg) { // Note that we add the $config.install.root/bin directory at the // beginning of the PATH environment variable value, so the installed // executables are found first. // string paths ("PATH=" + (install_root / "bin").string ()); if (optional s = getenv ("PATH")) { paths += path::traits_type::path_separator; paths += *s; } envvars.push_back (move (paths)); } // Run internal tests. // if (has_internal_tests) { // Create the configuration. // // b create(, )

// // Amalgamation directory that will contain configuration subdirectory // for package tests out of source tree build. // dir_path out_dir ("build-installed"); { step_id b (step_id::b_test_installed_create); step_id s (step_id::b_test_installed_create); step_id f (step_id::b_create); string mods; // build2 create meta-operation parameters. for (const char* m: step_args (modules, s, f)) { mods += mods.empty () ? ", " : " "; mods += m; } r.status |= run_b ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create('" + out_dir.representation () + "'" + mods + ")", step_args (env_args, s, f), step_args (config_args, s, f)); if (!r.status) break; } // Configure testable subprojects sequentially and test/build in // parallel afterwards. // strings test_specs; for (const dir_path& d: subprj_dirs) { // b configure(@) // // step_id b (step_id::b_test_installed_configure); step_id s (step_id::b_test_installed_configure); step_id f (step_id::b_configure); dir_path subprj_src_dir (exists (dist_src) ? dist_src / d : main_pkg_conf / pkg_dir / d); dir_path subprj_out_dir (out_dir / d); r.status |= run_b ( b, envvars, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "configure('" + subprj_src_dir.representation () + "'@'" + subprj_out_dir.representation () + "')", step_args (env_args, s, f), step_args (config_args, s, f)); if (!r.status) break; test_specs.push_back ( "test('" + subprj_out_dir.representation () + "')"); } if (!r.status) break; // Build/test subprojects. // // b test()...

// { step_id b (step_id::b_test_installed_test); step_id s (step_id::b_test_installed_test); r.status |= run_b ( b, envvars, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", test_specs, step_args (env_args, s), step_args (config_args, s)); if (!r.status) break; } } // Run runtime and build-time tests. // // Note that we only build runtime tests for target packages and for // host packages in self-hosted configurations. // if (has_runtime_tests || has_buildtime_tests) { // Create the required build configurations. // dir_path target_conf ("build-installed-bpkg"); dir_path host_conf ("build-installed-bpkg-host"); dir_path module_conf ("build-installed-bpkg-module"); // Create the target configuration if this is a target package having // external runtime tests or a host/module package having external // build-time tests. // bool create_target (target_pkg || has_buildtime_tests); // Note that even if there are no runtime tests for a host/module // package, we still need to create the host/build2 configuration to // configure the system package in. // bool create_host (host_pkg || module_pkg); bool create_module (module_pkg || (host_pkg && has_buildtime_tests)); // Note: a module package cannot have runtime tests and so the module // configuration is only created to serve build-time tests. Thus, the // host or target configuration is always created as well and the // module configuration is never a root configuration. // assert (create_target || create_host); // Root configuration through which we will be configuring the // cluster. // const dir_path& root_conf (create_target ? target_conf : host_conf); // Runtime tests configuration. Should only be used if there are any. // const dir_path& runtime_tests_conf (target_pkg ? target_conf : host_conf); // Create the target configuration. // // bpkg create

// if (create_target) { step_id b (step_id::bpkg_test_separate_installed_create); // Note that here and below the _for_* step ids are determined by // the main package type (and, yes, that means we will use the same // step ids for target and host configuration -- that, however, // should be ok since host configuration will only be created in // the self-hosted case). // step_id s ( target_pkg ? step_id::bpkg_test_separate_installed_create_for_target : host_pkg ? step_id::bpkg_test_separate_installed_create_for_host : step_id::bpkg_test_separate_installed_create_for_module); // Note: no fallback for modules. // optional f (!module_pkg ? step_id::bpkg_test_separate_installed_create : optional ()); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create", "-d", target_conf, step_args (modules, s, f), step_args (env_args, s, f), step_args (config_args, s, f)); if (!r.status) break; } // Create the host configuration. // if (create_host) { // bpkg create --type host

// step_id b (step_id::bpkg_test_separate_installed_create); step_id s ( host_pkg ? step_id::bpkg_test_separate_installed_create_for_host : step_id::bpkg_test_separate_installed_create_for_module); // Note: no fallback for modules. // optional f (!module_pkg ? step_id::bpkg_test_separate_installed_create : optional ()); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create", "-d", host_conf, "--type", "host", "--name", "host", step_args (modules, s, f), step_args (env_args, s, f), step_args (config_args, s, f)); if (!r.status) break; } // Create the module configuration. // // Note that we never build any tests in it but only configure the // system package. Note, however, that the host/module package // build-time tests can potentially build some other modules here. // if (create_module) { // b create() config.config.load=~build2 // step_id b (step_id::bpkg_test_separate_installed_create); r.status |= run_b ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-V", "create(" + module_conf.representation () + ",cc)", "config.config.load=~build2", "config.config.persist+='config.*'@unused=drop"); if (!r.status) break; // bpkg create --existing --type build2 // r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "create", "--existing", "-d", module_conf, "--type", "build2", "--name", "module"); if (!r.status) break; } // Link the configurations. // // bpkg link -d // { step_id b (step_id::bpkg_test_separate_installed_link); if (create_target) { if (create_host) { r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "link", "-d", target_conf, host_conf); if (!r.status) break; } if (create_module) { r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "link", "-d", target_conf, module_conf); if (!r.status) break; } } if (create_host) { if (create_module) { r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "link", "-d", host_conf, module_conf); if (!r.status) break; } } } // Add and fetch the repositories. // if (has_runtime_tests) { // bpkg add

// { step_id b (step_id::bpkg_test_separate_installed_configure_add); step_id s (step_id::bpkg_test_separate_installed_configure_add); step_id f (step_id::bpkg_configure_add); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "add", "-d", runtime_tests_conf, step_args (env_args, s, f), step_args (config_args, s, f), repo); if (!r.status) break; } // bpkg fetch

// { step_id b (step_id::bpkg_test_separate_installed_configure_fetch); step_id s (step_id::bpkg_test_separate_installed_configure_fetch); step_id f (step_id::bpkg_configure_fetch); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "fetch", "-d", runtime_tests_conf, step_args (env_args, s, f), step_args (config_args, s, f), trust_ops); if (!r.status) break; } } if (has_buildtime_tests) { // bpkg add

// { step_id b (step_id::bpkg_test_separate_installed_configure_add); step_id s (step_id::bpkg_test_separate_installed_configure_add); step_id f (step_id::bpkg_configure_add); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "add", "-d", target_conf, step_args (env_args, s, f), step_args (config_args, s, f), repo); if (!r.status) break; } // bpkg fetch

// { step_id b (step_id::bpkg_test_separate_installed_configure_fetch); step_id s (step_id::bpkg_test_separate_installed_configure_fetch); step_id f (step_id::bpkg_configure_fetch); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "fetch", "-d", target_conf, step_args (env_args, s, f), step_args (config_args, s, f), trust_ops); if (!r.status) break; } } // Configure all the packages using a single bpkg-pkg-build command. // // bpkg build --configure-only

// { }+ { ... } // ... // ?sys: // strings pkg_args; if (has_runtime_tests) { // Note that only host package runtime tests can (but not // necessarily) be configured in a linked configuration and require // --config-name to be specified for them. // assert (!module_pkg); string conf_name (runtime_tests_conf == root_conf ? "" : "host"); bool og (!conf_name.empty ()); if (og) { pkg_args.push_back ("{"); pkg_args.push_back ("--config-name"); pkg_args.push_back (conf_name); pkg_args.push_back ("}+"); } if (og && runtime_tests.size () != 1) pkg_args.push_back ("{"); for (auto t: runtime_tests) pkg_args.push_back (t.string ()); if (og && runtime_tests.size () != 1) pkg_args.push_back ("}"); } if (has_buildtime_tests) { // Strip the build-time mark. // for (auto t: buildtime_tests) pkg_args.push_back (t.dependency::string ()); } pkg_args.push_back ("?sys:" + pkg); // Finally, configure all the test packages. // { step_id b (step_id::bpkg_test_separate_installed_configure_build); step_id g (step_id::bpkg_global_configure_build); // Global. step_id s ( target_pkg ? step_id::bpkg_test_separate_installed_create_for_target : host_pkg ? step_id::bpkg_test_separate_installed_create_for_host : step_id::bpkg_test_separate_installed_create_for_module); step_id f (step_id::bpkg_test_separate_installed_configure_build); r.status |= run_bpkg ( b, envvars, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "build", "--configure-only", "--checkout-root", dist_installed_root, "--yes", "-d", root_conf, step_args (env_args, g), step_args (env_args, s, f), step_args (config_args, g), step_args (config_args, s, f), "--", pkg_args); if (!r.status) break; } #ifdef _WIN32 Sleep (5000); // See above. #endif // Run external runtime tests. // if (has_runtime_tests) { const dir_path& runtime_tests_conf (target_pkg ? target_conf : host_conf); change_wd (trace, &r.log, runtime_tests_conf); if (!test (r, runtime_tests, dist_installed_root, true /* installed */, envvars)) break; } // Run external build-time tests. // if (has_buildtime_tests) { change_wd (trace, &r.log, rwd / target_conf); if (!test (r, buildtime_tests, dist_installed_root, true /* installed */, envvars)) break; } rm.status |= r.status; } } // Uninstall. // { operation_result& r (add_result ("uninstall")); change_wd (trace, &r.log, install_conf); // bpkg uninstall

// step_id b (step_id::bpkg_uninstall); step_id s (step_id::bpkg_uninstall); r.status |= run_bpkg ( b, trace, r.log, wre, bkp_step, bkp_status, last_cmd, "-v", "uninstall", step_args (env_args, s), step_args (config_args, s), pkg); if (!r.status) break; rm.status |= r.status; } break; } if (!rm.results.empty ()) { operation_result& r (rm.results.back ()); rm.status |= r.status; // Merge last in case of a break. // Unless there is an error (or worse) encountered, log the special 'end' // step and, if this step is specified in the interactive manifest value, // ask the user if to continue the task execution. // bool error (!rm.status); if (!error) { r.status |= run_cmd (step_id::end, trace, r.log, nullptr /* out */, regexes (), "" /* name */, bkp_step, bkp_status, last_cmd, process_env ()); rm.status |= r.status; } // Truncate logs if they would exceed the upload limit. // // @@ TMP: currently this limit is hard-coded. In the future it should be // sent along with the task manifest. // const size_t upload_limit (10 * 1024 * 1024); { // Reserve 10K for other manifest values (alternatively, we could do it // exactly in upload_manifest()). // const size_t manifest_size (10 * 1024); size_t n (manifest_size); for (const operation_result& r: rm.results) n += r.log.size (); if (n > upload_limit) { // Divide the size equally among all the operations and truncate any // that exceed their allowance. This way we will get some information // for each operation. // n = (upload_limit - manifest_size) / rm.results.size (); for (operation_result& r: rm.results) { if (r.log.size () <= n) continue; // We need to be careful not to truncate it in the middle of UTF-8 // sequence. So we look for the last newline that still fits. // size_t p (n - 80 /* for the "log truncated" note */); for (; p != 0 && r.log[p] != '\n'; --p) ; r.log.resize (p != 0 ? p + 1 : 0); // Keep the newline. r.log += "-------------------------------LOG TRUNCATED-------------------------------\n"; } } } } else assert (rm.status == result_status::abort || rm.status == result_status::skip); if (!rwd.empty ()) change_wd (trace, nullptr /* log */, rwd); // Upload the result. // const string url ("tftp://" + ops.tftp_host () + "/result.manifest.lz4"); try { upload_manifest (trace, url, rm, "result"); // We use exit code 2 to signal abnormal termination but where we managed // to upload the result manifest. See startup() for details. // return rm.status != result_status::abnormal ? 0 : 2; } catch (const io_error& e) { error << "unable to upload result manifest to " << url << ": " << e; } // We use exit code 3 to signal an unsuccessful attempt to upload the result // manifest. See startup() for details. // return 3; } static int startup () { tracer trace ("startup"); // Our overall plan is as follows: // // 1. Download the task manifest into the build directory (CWD). // // 2. Parse it and get the target. // // 3. Find the environment setup executable for this target. // // 4. Execute the environment setup executable. // // 5. If the environment setup executable fails, then upload the (failed) // result ourselves. // const string url ("tftp://" + ops.tftp_host () + "/task.manifest"); const path mf ("task.manifest"); // If we fail, try to upload the result manifest (abnormal termination). The // idea is that the machine gets suspended and we can investigate what's // going on by logging in and examining the diagnostics (e.g., via // journalctl, etc). // task_manifest tm; try { // Download the task. // // We are downloading from our host so there shouldn't normally be any // connectivity issues. Unless, of course, we are on Windows where all // kinds of flakiness is business as usual. Note that having a long enough // timeout is not enough: if we try to connect before the network is up, // we will keep waiting forever, even after it is up. So we have to // timeout and try again. This is also pretty bad (unlike, say during // bootstrap which doesn't happen very often) since we are wasting the // machine time. So we are going to log it as a warning and not merely a // trace since if this is a common occurrence, then something has to be // done about it. // for (size_t retry (1);; ++retry) { try { tftp_curl c (trace, nullfd, mf, curl::get, url, "--tftp-blksize", tftp_blksize, "--max-time", tftp_get_timeout); if (!c.wait ()) throw_generic_error (EIO); break; } catch (const system_error& e) { bool bail (retry > tftp_get_retries); diag_record dr (bail ? error : warn); dr << "unable to download task manifest from " << url << " on " << retry << " try: " << e; if (bail) throw failed (); } } // Parse it. // tm = parse_manifest (mf, "task"); // Find the environment setup executable. // // While the executable path contains a directory (so the PATH search does // not apply) we still use process::path_search() to automatically handle // appending platform-specific executable extensions (.exe/.bat, etc). // process_path pp; if (tm.environment) { try { pp = process::try_path_search (env_dir / *tm.environment, false /* init */); } catch (const invalid_path& e) { fail << "invalid environment name '" << e.path << "': " << e; } if (pp.empty ()) fail << "no environment setup executable in " << env_dir << " " << "for environment name '" << *tm.environment << "'"; } else { pp = process::try_path_search (env_dir / "default", false /* init */); if (pp.empty ()) fail << "no default environment setup executable in " << env_dir; } // Run it. // strings os; string tg (tm.target.string ()); // Use the name=value notation for options to minimize the number of // arguments passed to the environment setup executable. Note that the // etc/environments/default-*.bat scripts can only handle the limited // number of arguments. // if (ops.systemd_daemon ()) os.push_back ("--systemd-daemon"); if (ops.verbose_specified ()) os.push_back ("--verbose=" + to_string (ops.verbose ())); if (ops.tftp_host_specified ()) os.push_back ("--tftp-host=" + ops.tftp_host ()); os.push_back (string ("--env-script=") + pp.effect_string ()); os.push_back ("--env-target=" + tg); // Note that we use the effective (absolute) path instead of recall since // we may have changed the CWD. // // Also note that the worker can ask the user if to continue the task // execution when the interactive build breakpoint is reached. Thus, we // don't redirect stdin to /dev/null. // // Exit code 2 signals abnormal termination but where the worker uploaded // the result itself. // // Exit code 3 signals an unsuccessful attempt by the worker to upload the // result manifest. There is no reason to retry (most likely there is // nobody listening on the other end anymore). // switch (run_io_exit (trace, 0, 2, 2, pp, tg, argv0.effect_string (), os)) { case 3: case 2: return 1; case 0: return 0; default: fail << "process " << pp << " exited with non-zero code" << endf; } } catch (const failed&) { const string url ("tftp://" + ops.tftp_host () + "/result.manifest.lz4"); // If we failed before being able to parse the task manifest, use the // "unknown" values for the package name and version. // result_manifest rm { tm.name.empty () ? bpkg::package_name ("unknown") : tm.name, tm.version.empty () ? bpkg::version ("0") : tm.version, result_status::abnormal, operation_results {}, worker_checksum, nullopt /* dependency_checksum */ }; try { upload_manifest (trace, url, rm, "result"); } catch (const io_error& e) { fail << "unable to upload result manifest to " << url << ": " << e; } return 1; } } static int bootstrap () { bootstrap_manifest bm { bootstrap_manifest::versions_type { {"bbot", standard_version (BBOT_VERSION_STR)}, {"libbbot", standard_version (LIBBBOT_VERSION_STR)}, {"libbpkg", standard_version (LIBBPKG_VERSION_STR)}, {"libbutl", standard_version (LIBBUTL_VERSION_STR)} } }; serialize_manifest (bm, cout, "stdout", "bootstrap"); return 0; } int bbot:: main (int argc, char* argv[]) try { tracer trace ("main"); // This is a little hack to make our baseutils for Windows work when called // with absolute path. In a nutshell, MSYS2's exec*p() doesn't search in the // parent's executable directory, only in PATH. And since we are running // without a shell (that would read /etc/profile which sets PATH to some // sensible values), we are only getting Win32 PATH values. And MSYS2 /bin // is not one of them. So what we are going to do is add /bin at the end of // PATH (which will be passed as is by the MSYS2 machinery). This will make // MSYS2 search in /bin (where our baseutils live). And for everyone else // this should be harmless since it is not a valid Win32 path. // #ifdef _WIN32 { string mp; if (optional p = getenv ("PATH")) { mp = move (*p); mp += ';'; } mp += "/bin"; setenv ("PATH", mp); } #endif // On POSIX ignore SIGPIPE which is signaled to a pipe-writing process if // the pipe reading end is closed. Note that by default this signal // terminates a process. Also note that there is no way to disable this // behavior on a file descriptor basis or for the write() function call. // // On Windows disable displaying error reporting dialog box. Note that the // error mode is inherited by child processes. // #ifndef _WIN32 if (signal (SIGPIPE, SIG_IGN) == SIG_ERR) fail << "unable to ignore broken pipe (SIGPIPE) signal: " << system_error (errno, std::generic_category ()); // Sanitize. #else SetErrorMode (SetErrorMode (0) | // Returns the current mode. SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); #endif cli::argv_scanner scan (argc, argv, true); ops.parse (scan); verb = ops.verbose (); // @@ systemd 231 added JOURNAL_STREAM environment variable which allows // detecting if stderr is connected to the journal. // if (ops.systemd_daemon ()) systemd_diagnostics (false); // Version. // if (ops.version ()) { cout << "bbot-worker " << BBOT_VERSION_ID << endl << "libbbot " << LIBBBOT_VERSION_ID << endl << "libbpkg " << LIBBPKG_VERSION_ID << endl << "libbutl " << LIBBUTL_VERSION_ID << endl << "Copyright (c) " << BBOT_COPYRIGHT << "." << endl << "This is free software released under the MIT license." << endl; return 0; } // Help. // if (ops.help ()) { pager p ("bbot-worker help", false); print_bbot_worker_usage (p.stream ()); // If the pager failed, assume it has issued some diagnostics. // return p.wait () ? 0 : 1; } // Figure out our mode. // if (ops.bootstrap () && ops.startup ()) fail << "--bootstrap and --startup are mutually exclusive"; enum class mode {boot, start, build} m (mode::build); if (ops.bootstrap ()) m = mode::boot; if (ops.startup ()) m = mode::start; if (ops.systemd_daemon ()) { info << "bbot worker " << BBOT_VERSION_ID; } // Figure out our path (used for re-exec). // argv0 = process::path_search (argv[0], true); // Sort out the build directory. // if (ops.build_specified ()) change_wd (trace, nullptr /* log */, ops.build (), true /* create */); // Sort out the environment directory. // try { env_dir = ops.environments_specified () ? ops.environments () : dir_path::home_directory (); if (!dir_exists (env_dir)) throw_generic_error (ENOENT); } catch (const system_error& e) { fail << "invalid environment directory: " << e; } int r (1); switch (m) { case mode::boot: r = bootstrap (); break; case mode::start: r = startup (); break; case mode::build: r = build (static_cast (argc), const_cast (argv)); break; } return r; } catch (const failed&) { return 1; // Diagnostics has already been issued. } catch (const cli::exception& e) { error << e; return 1; } int main (int argc, char* argv[]) { return bbot::main (argc, argv); }