diff options
| -rw-r--r-- | bbot/agent/agent.cxx | 39 | ||||
| -rw-r--r-- | bbot/worker/worker.cxx | 260 | ||||
| -rw-r--r-- | doc/manual.cli | 24 | ||||
| -rw-r--r-- | tests/integration/testscript | 29 |
4 files changed, 302 insertions, 50 deletions
diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx index eacd05a..80b4fb7 100644 --- a/bbot/agent/agent.cxx +++ b/bbot/agent/agent.cxx @@ -64,6 +64,17 @@ file_sync (const path& f) throw_system_error (errno); } +static bool +file_not_empty (const path& f) +{ + if (file_exists (f)) + { + file_sync (f); + return !file_empty (f); + } + return false; +} + // The btrfs tool likes to print informational messages, like "Created // snapshot such and such". Luckily, it writes them to stdout while proper // diagnostics to stderr. @@ -226,12 +237,8 @@ bootstrap_machine (const dir_path& md, // for (to = bootstrap_to; to != 0; tftpd.serve (to, 2)) { - if (file_exists (mf)) - { - file_sync (mf); - if (!file_empty (mf)) - break; - } + if (file_not_empty (mf)) + break; } if (to == 0) @@ -615,9 +622,19 @@ try { // Simply wait for the file to appear. // - for (size_t i (0); !file_exists (rf); sleep (1)) + for (size_t i (0);; sleep (1)) + { + if (file_not_empty (rf)) + { + // Wait a bit to make sure we see complete manifest. + // + sleep (2); + break; + } + if (i++ % 10 == 0) l3 ([&]{trace << "waiting for result manifest";}); + } r = parse_manifest<result_manifest> (rf, "result"); } @@ -711,12 +728,8 @@ try // for (to = build_to; to != 0; tftpd.serve (to, 2)) { - if (file_exists (rf)) - { - file_sync (rf); - if (!file_empty (rf)) - break; - } + if (file_not_empty (rf)) + break; } if (to == 0) diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index 67bbebe..2107dc3 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -9,6 +9,7 @@ #endif #include <regex> +#include <cstring> // strchr() #include <iostream> #include <libbutl/pager.hxx> @@ -59,15 +60,17 @@ catch (const system_error& e) using regexes = vector<regex>; -// Match lines read from the command's stderr against the regular expressions -// and return the warning result status (instead of success) in case of a -// match. +// Run a named command. Name is used for logging and diagnostics only. Match +// lines read from the command's stderr against the regular expressions and +// return the warning result status (instead of success) in case of a match. // template <typename... A> static result_status -run_bpkg (tracer& t, - string& log, const regexes& warn_detect, - const string& cmd, A&&... a) +run_cmd (tracer& t, + string& log, const regexes& warn_detect, + const string& name, + const process_env& pe, + A&&... a) { try { @@ -90,9 +93,7 @@ run_bpkg (tracer& t, fdnull (), // Never reads from stdout. 2, // 1>&2 pipe.out, - "bpkg", - "-v", - cmd, + pe, forward<A> (a)...)); pipe.out.close (); @@ -136,7 +137,7 @@ run_bpkg (tracer& t, if (pr.wait ()) return r; - log += "bpkg " + cmd; + log += name; const process_exit& e (*pr.exit); if (e.normal ()) @@ -153,14 +154,50 @@ run_bpkg (tracer& t, } catch (const process_error& e) { - fail << "unable to execute bpkg " << cmd << ": " << e << endf; + fail << "unable to execute " << name << ": " << e << endf; } catch (const io_error& e) { - fail << "unable to read bpkg " << cmd << " diagnostics: " << e << endf; + fail << "unable to read " << name << " diagnostics: " << e << endf; } } +template <typename... A> +static result_status +run_bpkg (tracer& t, + string& log, const regexes& warn_detect, + const string& cmd, A&&... a) +{ + return run_cmd (t, + log, warn_detect, + "bpkg " + cmd, + "bpkg", "-v", cmd, forward<A> (a)...); +} + +template <typename V, typename... A> +static result_status +run_b (tracer& t, + string& log, const regexes& warn_detect, + const V& envvars, + const string& buildspec, A&&... a) +{ + return run_cmd (t, + log, warn_detect, + "b " + buildspec, + process_env ("b", envvars), + "-v", forward<A> (a)..., buildspec); +} + +template <typename... A> +static result_status +run_b (tracer& t, + string& log, const regexes& warn_detect, + const string& buildspec, A&&... a) +{ + const char* const* envvars (nullptr); + return run_b (t, log, warn_detect, envvars, buildspec, forward<A> (a)...); +} + static void build (size_t argc, const char* argv[]) { @@ -171,8 +208,8 @@ build (size_t argc, const char* argv[]) // 1. Parse the task manifest (it should be in CWD). // // 2. Run bpkg to create the configuration, add the repository, and - // configure, build, and test the package all while saving the logs in - // the result manifest. + // configure, build, test, install and uninstall the package all while + // saving the logs in the result manifest. // // 3. Upload the result manifest. // @@ -198,7 +235,7 @@ build (size_t argc, const char* argv[]) return rm.results.back (); }; - dir_path owd; + dir_path rwd; // Root working directory. for (;;) // The "breakout" loop. { @@ -219,29 +256,29 @@ build (size_t argc, const char* argv[]) for (const auto& re: tm.unquoted_warning_regex ()) wre.emplace_back (re, f); + strings config (tm.unquoted_config ()); + const vector_view<const char*> env (argv + 1, argc - 1); + // Configure. // + // Configuration directory name. + // + dir_path build_dir ("build"); { operation_result& r (add_result ("configure")); // bpkg create <config-vars> <env-module> <env-config-vars> // - const vector_view<const char*> env (argv + 1, argc - 1); - - // Configuration directory name. - // - dir_path dir ("build"); - r.status |= run_bpkg (trace, r.log, wre, "create", - "-d", dir.string (), + "-d", build_dir.string (), "--wipe", - tm.unquoted_config (), + config, env); if (!r.status) break; - owd = change_wd (dir); + rwd = change_wd (build_dir); // bpkg add <repository-url> // @@ -315,13 +352,184 @@ build (size_t argc, const char* argv[]) rm.status |= r.status; } + // Install the package, optionally test the installation and uninstall + // afterwards. + // + // These operations are triggered by presence of config.install.root + // configuration variable having a non-empty value. Passing [null] value + // would be meaningless, so we don't recognize it as a special one. + // + dir_path install_root; + { + size_t n (19); + auto space = [] (char c) {return c == ' ' || c == '\t';}; + + for (const auto& s: reverse_iterate (config)) + { + if (s.compare (0, n, "config.install.root") == 0 && + (s[n] == '=' || space (s[n]))) + { + while (space (s[n])) ++n; // Skip spaces. + if (s[n] == '=') ++n; // Skip the equal sign. + while (space (s[n])) ++n; // Skip spaces. + + install_root = dir_path (s, n, s.size () - n); + break; + } + } + + if (install_root.empty ()) + break; + } + + // Now the overall plan is as follows: + // + // 1. Install the package. + // + // 2. If the package project has the 'tests' subdirectory that is a + // subproject, then configure, build and test it out of the source tree + // against the installed package. + // + // 3. Uninstall the package. + // + // Install. + // + { + operation_result& r (add_result ("install")); + + // bpkg install <package-name> + // + r.status |= run_bpkg (trace, r.log, wre, "install", tm.name); + + if (!r.status) + break; + + rm.status |= r.status; + } + + // Test installed. + // + // The package tests subdirectory path (may not exist). + // + dir_path tests_dir (tm.name + "-" + tm.version.string ()); + tests_dir /= "tests"; + + // We will consider the tests subdirectory to be a subproject if it + // contains a build2 bootstrap file. + // + if (file_exists (tests_dir / path ("build/bootstrap.build"))) + { + operation_result& r (add_result ("test-installed")); + + change_wd (rwd); + + // Sort environment arguments into modules and configuration variables. + // + string mods; // build2 create meta-operation parameters. + cstrings vars; + + for (const auto& a: env) + { + // Note that we don't check for the argument emptiness, as this is + // already done by 'bpkg create' (see above). + // + if (strchr (a, '=') != nullptr) + { + vars.push_back (a); + } + else + { + mods += mods.empty () ? ", " : " "; + mods += a; + } + } + + // b <config-vars> <env-config-vars> create(<dir>, <modules>) + // + // Amalgamation directory that will contain configuration subdirectory + // for package tests out of source tree build. + // + dir_path out_dir ("build-installed"); + + r.status |= run_b ( + trace, r.log, wre, + "create(" + out_dir.representation () + mods + ")", + config, + vars); + + if (!r.status) + break; + + rm.status |= r.status; + + // Make sure that the installed package executables are properly imported + // when configuring and running tests. + // + // Note that we add the $config.install.root/bin directory at the + // beginning of the PATH environment variable value, so the installed + // executables are found first. + // + string paths ("PATH=" + (install_root / "bin").string ()); + + if (char const* s = getenv ("PATH")) + { + paths += path::traits::path_separator; + paths += s; + } + + small_vector<string, 1> envvars ({move (paths)}); + + // b configure(<tests-dir>@<tests-out-dir>) + // + dir_path tests_out_dir (out_dir / dir_path ("tests")); + + r.status |= run_b (trace, r.log, wre, + envvars, + "configure(" + + (build_dir / tests_dir).representation () + '@' + + tests_out_dir.representation () + ")"); + + if (!r.status) + break; + + rm.status |= r.status; + + // b test(<tests-out-dir>) + // + r.status |= run_b (trace, r.log, wre, + envvars, + "test(" + tests_out_dir.representation () + ')'); + + if (!r.status) + break; + + rm.status |= r.status; + + change_wd (build_dir); + } + + // Uninstall. + // + { + operation_result& r (add_result ("uninstall")); + + // bpkg uninstall <package-name> + // + r.status |= run_bpkg (trace, r.log, wre, "uninstall", tm.name); + + if (!r.status) + break; + + rm.status |= r.status; + } + break; } rm.status |= rm.results.back ().status; // Merge last in case of a break. - if (!owd.empty ()) - change_wd (owd); + if (!rwd.empty ()) + change_wd (rwd); // Upload the result. // diff --git a/doc/manual.cli b/doc/manual.cli index 46f947f..9d67377 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -370,14 +370,20 @@ SYNOPSIS name: <package-name> version: <package-version> -status: <status> -configure-status: <status> -update-status: <status> -test-status: <status> - -configure-log: <text> -update-log: <text> -test-log: <text> +status: <status> +configure-status: <status> +update-status: <status> +test-status: <status> +install-status: <status> +test-installed-status: <status> +uninstall-status: <status> + +configure-log: <text> +update-log: <text> +test-log: <text> +install-log: <text> +test-installed-log: <text> +uninstall-log: <text> \ The result manifest describes a build result. @@ -677,7 +683,7 @@ and private installations: \ linux*-gcc* linux-gcc-sys x86_64-linux-gnu config.install.root=/usr config.install.sudo=sudo -linux*-gcc* linux-gcc-prv x86_64-linux-gnu config.install.root=/tmp/install config.cc.poptions=-I/tmp/install/include config.cc.loptions=-L/tmp/install/lib +linux*-gcc* linux-gcc-prv x86_64-linux-gnu config.install.root=/tmp/install config.cc.poptions=-I/tmp/install/include config.cc.loptions=-L/tmp/install/lib config.bin.rpath=/tmp/install/bin \ Note also that while building and run tests against the installation the diff --git a/tests/integration/testscript b/tests/integration/testscript index 2ae3b3c..6084d32 100644 --- a/tests/integration/testscript +++ b/tests/integration/testscript @@ -48,10 +48,34 @@ controller = https://stage.build2.org/?build-task wait=1s controller = --fake-request ../task --dump-result -pkg = hello +pkg = libhello ver = 1.0.0 rep = https://build2.org/pkg/1/hello/stable rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94 + +#\ +pkg = bpkg +ver = 0.6.0-a.0.1499870184.53fff46c5e5fd4df +rep = https://stage.build2.org/1 +rfp = 37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65 +#\ + +#\ +pkg = build2 +ver = 0.6.0-a.0.1500478402.30578be48463b93f +rep = https://stage.build2.org/1 +rfp = 37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65 +#\ + +# Note that we also need to make sure that the installed package libraries are +# properly imported when configuring and running tests, and that the installed +# executables are runnable. +# +config = "config.install.root='$~/install' \ +config.cc.poptions=-I'$~/install/include' \ +config.cc.loptions=-L'$~/install/lib' \ +config.bin.rpath='$~/install/lib'" + +cat <<"EOI" >=task : 1 name: $pkg @@ -60,6 +84,7 @@ rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30 trust: $rfp machine: $machine target: $target + config: $config EOI # @@ -96,5 +121,5 @@ a = $0 chmod ugo+x $target; sleep $wait; $w --verbose 3 --startup --tftp-host $tftp --environments $~ \ - &build/*** &manifest 2>| + &build/*** &?build-installed/*** &manifest 2>| } |