diff options
| -rw-r--r-- | bbot/worker/worker.cxx | 379 | ||||
| -rw-r--r-- | doc/manual.cli | 8 | ||||
| -rw-r--r-- | tests/integration/testscript | 7 |
3 files changed, 299 insertions, 95 deletions
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index 93e3ca2..803e755 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -10,6 +10,7 @@ # include <stdlib.h> // getenv(), _putenv() #endif +#include <map> #include <regex> #include <cstring> // strchr() #include <sstream> @@ -17,6 +18,7 @@ #include <libbutl/pager.mxx> #include <libbutl/filesystem.mxx> +#include <libbutl/string-parser.mxx> #include <libbbot/manifest.hxx> @@ -137,7 +139,7 @@ run_cmd (tracer& t, // if (r != result_status::warning) { - for (const auto& re: warn_detect) + for (const regex& re: warn_detect) { // Only examine the first 512 bytes. Long lines (e.g., linker // command lines) could trigger implementation-specific limitations @@ -204,8 +206,8 @@ run_b (tracer& t, return run_cmd (t, log, warn_detect, "b " + buildspec, - process_env ("b", envvars), - "-v", forward<A> (a)..., buildspec); + process_env ("b", envvars), + "-v", buildspec, forward<A> (a)...); } template <typename... A> @@ -228,8 +230,8 @@ build (size_t argc, const char* argv[]) // 1. Parse the task manifest (it should be in CWD). // // 2. Run bpkg to create the configuration, add the repository, and - // configure, build, test, install and uninstall the package all while - // saving the logs in the result manifest. + // configure, build, test, optionally install, test installed and + // uninstall the package all while saving the logs in the result manifest. // // 3. Upload the result manifest. // @@ -269,49 +271,241 @@ build (size_t argc, const char* argv[]) // regex::flag_type f (regex_constants::optimize); // ECMAScript is implied. - regexes wre ({ + regexes wre { regex ("^warning: ", f), - regex ("^.+: warning: ", f)}); + regex ("^.+: warning: ", f)}; - for (const auto& re: tm.unquoted_warning_regex ()) + for (const string& re: tm.unquoted_warning_regex ()) wre.emplace_back (re, f); - strings config (tm.unquoted_config ()); - const vector_view<const char*> env (argv + 1, argc - 1); + // Step IDs. + // + enum class step_id + { + bpkg_configure_create, + bpkg_configure_add, + bpkg_configure_fetch, + bpkg_configure_build, + bpkg_update_update, + bpkg_test_test, + bpkg_install_install, + b_test_installed_create, + b_test_installed_configure, + b_test_installed_test, + bpkg_uninstall_uninstall + }; + + const strings step_id_str { + "bpkg.configure.create", + "bpkg.configure.add", + "bpkg.configure.fetch", + "bpkg.configure.build", + "bpkg.update.update", + "bpkg.test.test", + "bpkg.install.install", + "b.test_installed.create", + "b.test_installed.configure", + "b.test_installed.test", + "bpkg.uninstall.uninstall"}; + + // Split the argument into prefix (empty if not present) and unquoted + // value. Return nullopt if the prefix is invalid. + // + auto parse_arg = + [&step_id_str] (const string& a) -> optional<pair<string, string>> + { + using string_parser::unquote; + + size_t p (a.find_first_of (":=\"'")); + + if (p == string::npos || a[p] != ':') // No prefix. + return make_pair (string (), unquote (a)); + + for (const string& id: step_id_str) + { + if (a.compare (0, p, id, 0, p) == 0 && + (id.size () == p || (id.size () > p && id[p] == '.'))) + return make_pair (a.substr (0, p), unquote (a.substr (p + 1))); + } + + return nullopt; // Prefix is invalid. + }; + + // Enter split arguments into a map. Those without a prefix are + // entered for the *.create steps. + // + auto add_arg = [] (std::multimap<string, string>& args, + pair<string, string>&& a) + { + if (!a.first.empty ()) + args.emplace (move (a)); + else + { + args.emplace ("bpkg.configure.create", a.second); + args.emplace ("b.test_installed.create", move (a.second)); + } + }; + + // Parse configuration arguments. Report failures to the bbot controller. + // + std::multimap<string, string> config_args; + + for (const string& c: tm.config) + { + optional<pair<string, string>> v (parse_arg (c)); + + if (!v) + { + rm.status |= result_status::abort; + l3 ([&]{trace << "invalid configuration argument prefix in " + << "'" << c << "'";}); + break; + } + + if (v->second[0] != '-' && v->second.find ('=') == string::npos) + { + rm.status |= result_status::abort; + l3 ([&]{trace << "invalid configuration argument '" << c << "'";}); + break; + } + + add_arg (config_args, move (*v)); + } + + if (!rm.status) + break; + + // Parse environment arguments. + // + std::multimap<string, string> modules; + std::multimap<string, string> env_args; + + for (size_t i (1); i != argc; ++i) + { + const char* a (argv[i]); + optional<pair<string, string>> v (parse_arg (a)); + + if (!v) + fail << "invalid environment argument prefix in '" << a << "'"; + + bool mod (v->second[0] != '-' && v->second.find ('=') == string::npos); + + if (mod && !v->first.empty () && + v->first != "bpkg.configure.create" && + v->first != "b.test_installed.create") + fail << "invalid module prefix in '" << a << "'"; + + add_arg (mod ? modules : env_args, move (*v)); + } + + // Return command arguments for the specified step id. Arguments with more + // specific prefixes come last. + // + // @@ When CLI will support <name>=<value> option notation the splitting + // become redundant. + // + auto step_args = [&step_id_str] (const std::multimap<string, string>& args, + step_id step) -> strings + { + strings r; + const string& s (step_id_str[static_cast<size_t> (step)]); + + for (size_t n (0);; ++n) + { + n = s.find ('.', n); + + auto range ( + args.equal_range (n == string::npos ? s : string (s, 0, n))); + + for (auto i (range.first); i != range.second; ++i) + { + const string& a (i->second); + + size_t p; + if (a[0] == '-' && (p = a.find ('=')) != string::npos) + { + r.emplace_back (a, 0, p); + r.emplace_back (a, p + 1); + } + else + r.emplace_back (a); + } + + if (n == string::npos) + break; + } + + return r; + }; // Configure. // + // Search for config.install.root variable. If it is present and has a + // non-empty value, then test the package installation and uninstall. Note + // that passing [null] value would be meaningless, so we don't recognize it + // as a special one. + // + dir_path install_root; + // Configuration directory name. // dir_path build_dir ("build"); { + strings cargs (step_args (config_args, step_id::bpkg_configure_create)); + { + size_t n (19); + auto space = [] (char c) {return c == ' ' || c == '\t';}; + + for (const string& s: reverse_iterate (cargs)) + { + if (s.compare (0, n, "config.install.root") == 0 && + (s[n] == '=' || space (s[n]))) + { + while (space (s[n])) ++n; // Skip spaces. + if (s[n] == '=') ++n; // Skip the equal sign. + while (space (s[n])) ++n; // Skip spaces. + + install_root = dir_path (s, n, s.size () - n); + break; + } + } + } + operation_result& r (add_result ("configure")); // bpkg create <env-modules> <config-args> <env-config-args> // // bpkg.configure.create // - r.status |= run_bpkg (trace, r.log, wre, - "create", - "-d", build_dir.string (), - "--wipe", - config, - env); + r.status |= run_bpkg ( + trace, r.log, wre, + "create", + "-d", build_dir.string (), + "--wipe", + step_args (modules, step_id::bpkg_configure_create), + cargs, + step_args (env_args, step_id::bpkg_configure_create)); + if (!r.status) break; rwd = change_wd (build_dir); - // bpkg add <repository-url> + // bpkg add <config-args> <env-config-args> <repository-url> // // bpkg.configure.add // - r.status |= run_bpkg (trace, r.log, wre, "add", tm.repository.string ()); + r.status |= run_bpkg ( + trace, r.log, wre, + "add", + step_args (config_args, step_id::bpkg_configure_add), + step_args (env_args, step_id::bpkg_configure_add), + tm.repository.string ()); if (!r.status) break; - // bpkg fetch + // bpkg fetch <config-args> <env-config-args> <trust-options> // // bpkg.configure.fetch // @@ -329,20 +523,31 @@ build (size_t argc, const char* argv[]) } } - r.status |= run_bpkg (trace, r.log, wre, "fetch", ts, t); + r.status |= run_bpkg ( + trace, r.log, wre, + "fetch", + step_args (config_args, step_id::bpkg_configure_fetch), + step_args (env_args, step_id::bpkg_configure_fetch), + ts, + t); if (!r.status) break; - // bpkg build --configure-only <package-name>/<package-version> + // bpkg build --configure-only <config-args> <env-config-args> + // <package-name>/<package-version> // // bpkg.configure.build // - r.status |= run_bpkg (trace, r.log, wre, - "build", - "--configure-only", - "--yes", - tm.name + '/' + tm.version.string ()); + r.status |= run_bpkg ( + trace, r.log, wre, + "build", + "--configure-only", + "--yes", + step_args (config_args, step_id::bpkg_configure_build), + step_args (env_args, step_id::bpkg_configure_build), + tm.name + '/' + tm.version.string ()); + if (!r.status) break; @@ -354,11 +559,16 @@ build (size_t argc, const char* argv[]) { operation_result& r (add_result ("update")); - // bpkg update <package-name> + // bpkg update <config-args> <env-config-args> <package-name> // // bpkg.update.update // - r.status |= run_bpkg (trace, r.log, wre, "update", tm.name); + r.status |= run_bpkg ( + trace, r.log, wre, + "update", + step_args (config_args, step_id::bpkg_update_update), + step_args (env_args, step_id::bpkg_update_update), + tm.name); if (!r.status) break; @@ -371,11 +581,16 @@ build (size_t argc, const char* argv[]) { operation_result& r (add_result ("test")); - // bpkg test <package-name> + // bpkg test <config-args> <env-config-args> <package-name> // // bpkg.test.test // - r.status |= run_bpkg (trace, r.log, wre, "test", tm.name); + r.status |= run_bpkg ( + trace, r.log, wre, + "test", + step_args (config_args, step_id::bpkg_test_test), + step_args (env_args, step_id::bpkg_test_test), + tm.name); if (!r.status) break; @@ -387,31 +602,11 @@ build (size_t argc, const char* argv[]) // afterwards. // // These operations are triggered by presence of config.install.root - // configuration variable having a non-empty value. Passing [null] value - // would be meaningless, so we don't recognize it as a special one. + // configuration variable having a non-empty value for + // bpkg.configure.create step. // - dir_path install_root; - { - size_t n (19); - auto space = [] (char c) {return c == ' ' || c == '\t';}; - - for (const auto& s: reverse_iterate (config)) - { - if (s.compare (0, n, "config.install.root") == 0 && - (s[n] == '=' || space (s[n]))) - { - while (space (s[n])) ++n; // Skip spaces. - if (s[n] == '=') ++n; // Skip the equal sign. - while (space (s[n])) ++n; // Skip spaces. - - install_root = dir_path (s, n, s.size () - n); - break; - } - } - - if (install_root.empty ()) - break; - } + if (install_root.empty ()) + break; // Now the overall plan is as follows: // @@ -428,11 +623,16 @@ build (size_t argc, const char* argv[]) { operation_result& r (add_result ("install")); - // bpkg install <package-name> + // bpkg install <config-args> <env-config-args> <package-name> // // bpkg.install.install // - r.status |= run_bpkg (trace, r.log, wre, "install", tm.name); + r.status |= run_bpkg ( + trace, r.log, wre, + "install", + step_args (config_args, step_id::bpkg_install_install), + step_args (env_args, step_id::bpkg_install_install), + tm.name); if (!r.status) break; @@ -456,25 +656,13 @@ build (size_t argc, const char* argv[]) change_wd (rwd); - // Sort environment arguments into modules and configuration variables. - // string mods; // build2 create meta-operation parameters. - cstrings vars; - for (const auto& a: env) + for (const string& m: + step_args (modules, step_id::b_test_installed_create)) { - // Note that we don't check for the argument emptiness, as this is - // already done by 'bpkg create' (see above). - // - if (strchr (a, '=') != nullptr) - { - vars.push_back (a); - } - else - { - mods += mods.empty () ? ", " : " "; - mods += a; - } + mods += mods.empty () ? ", " : " "; + mods += m; } // b create(<dir>, <env-modules>) <config-args> <env-config-args> @@ -489,8 +677,8 @@ build (size_t argc, const char* argv[]) r.status |= run_b ( trace, r.log, wre, "create(" + out_dir.representation () + mods + ")", - config, - vars); + step_args (config_args, step_id::b_test_installed_create), + step_args (env_args, step_id::b_test_installed_create)); if (!r.status) break; @@ -512,32 +700,39 @@ build (size_t argc, const char* argv[]) paths += s; } - small_vector<string, 1> envvars ({move (paths)}); + small_vector<string, 1> envvars {move (paths)}; - // b configure(<tests-dir>@<tests-out-dir>) + // b configure(<tests-dir>@<tests-out-dir>) <config-args> + // <env-config-args> // // b.test-installed.configure // dir_path tests_out_dir (out_dir / dir_path ("tests")); - r.status |= run_b (trace, r.log, wre, - envvars, - "configure(" + - (build_dir / tests_dir).representation () + '@' + - tests_out_dir.representation () + ")"); + r.status |= run_b ( + trace, r.log, wre, + envvars, + "configure(" + + (build_dir / tests_dir).representation () + '@' + + tests_out_dir.representation () + ")", + step_args (config_args, step_id::b_test_installed_configure), + step_args (env_args, step_id::b_test_installed_configure)); if (!r.status) break; rm.status |= r.status; - // b test(<tests-out-dir>) + // b test(<tests-out-dir>) <config-args> <env-config-args> // // b.test-installed.test // - r.status |= run_b (trace, r.log, wre, - envvars, - "test(" + tests_out_dir.representation () + ')'); + r.status |= run_b ( + trace, r.log, wre, + envvars, + "test(" + tests_out_dir.representation () + ')', + step_args (config_args, step_id::b_test_installed_test), + step_args (env_args, step_id::b_test_installed_test)); if (!r.status) break; @@ -552,11 +747,16 @@ build (size_t argc, const char* argv[]) { operation_result& r (add_result ("uninstall")); - // bpkg uninstall <package-name> + // bpkg uninstall <config-args> <env-config-args> <package-name> // // bpkg.uninstall.uninstall // - r.status |= run_bpkg (trace, r.log, wre, "uninstall", tm.name); + r.status |= run_bpkg ( + trace, r.log, wre, + "uninstall", + step_args (config_args, step_id::bpkg_uninstall_uninstall), + step_args (env_args, step_id::bpkg_uninstall_uninstall), + tm.name); if (!r.status) break; @@ -567,7 +767,10 @@ build (size_t argc, const char* argv[]) break; } - rm.status |= rm.results.back ().status; // Merge last in case of a break. + if (!rm.results.empty ()) + rm.status |= rm.results.back ().status; // Merge last in case of a break. + else + assert (rm.status == result_status::abort); if (!rwd.empty ()) change_wd (rwd); diff --git a/doc/manual.cli b/doc/manual.cli index 5183343..a49b255 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -368,7 +368,7 @@ triplet} format as autotools for \c{target}, it is not flexible enough for \ The additional configuration options and variables. A single level of quotes -(either single or double) is removed in each variable before being passed to +(either single or double) is removed in each value before being passed to \c{bpkg}. For example, the following value: \ @@ -705,9 +705,9 @@ modules (\c{<env-modules>}) and the list of configuration options and variables The re-executed \c{bbot} worker then proceeds to test the package from the repository by executing the following commands, collectively called a \i{worker script}. Each command has a unique \i{step id} that can be used as a -prefix in the \c{<config-args>} and \c{<env-config-args>} values as discussed -in \l{#arch-controller Controller Logic}. The \c{<>}-values are from the task -manifest and the environment: +prefix in the \c{<config-args>}, \c{<env-config-args>}, and \c{<env-modules>} +values as discussed in \l{#arch-controller Controller Logic}. The +\c{<>}-values are from the task manifest and the environment: \ # bpkg.configure.create diff --git a/tests/integration/testscript b/tests/integration/testscript index 6084d32..d30b1cc 100644 --- a/tests/integration/testscript +++ b/tests/integration/testscript @@ -72,9 +72,10 @@ rfp = 37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2 # executables are runnable. # config = "config.install.root='$~/install' \ -config.cc.poptions=-I'$~/install/include' \ -config.cc.loptions=-L'$~/install/lib' \ -config.bin.rpath='$~/install/lib'" +bpkg:--fetch-timeout=60 \ +b.test_installed.configure:config.cc.loptions=-L'$~/install/lib' \ +b.test_installed.configure:config.bin.rpath='$~/install/lib' \ +b.test_installed:--progress" +cat <<"EOI" >=task : 1 |