From 29f9eb0c59740519c1b356e7e925f5743ae33bfb Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Thu, 10 Oct 2024 21:00:05 +0300 Subject: Clean tests subprojects and external test packages to free up disk space in worker --- bbot/worker/worker.cxx | 143 +++++++++++++++++++++++++++++++++---------- tests/integration/testscript | 14 ++--- 2 files changed, 116 insertions(+), 41 deletions(-) diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx index a5dd92d..285bf5e 100644 --- a/bbot/worker/worker.cxx +++ b/bbot/worker/worker.cxx @@ -1037,6 +1037,27 @@ run_b (step_id step, template static result_status +run_b (step_id step, + tracer& t, + string& log, const regexes& warn_detect, + const optional& bkp_step, + const optional& bkp_status, + const strings& aux_env, + string& last_cmd, + const char* verbosity, + const strings& buildspecs, A&&... a) +{ + const char* const* envvars (nullptr); + return run_b (step, + envvars, + t, + log, warn_detect, + bkp_step, bkp_status, aux_env, last_cmd, + verbosity, buildspecs, forward (a)...); +} + +template +static result_status run_ldconfig (step_id step, tracer& t, string& log, const regexes& warn_detect, @@ -3771,6 +3792,27 @@ build (size_t argc, const char* argv[]) if (!r.status) return false; } + + // Now, as the external tests package is tested, clean it to free + // up a bit of space. + // + { + step_id b (installed + ? step_id::bpkg_test_separate_installed_test + : step_id::bpkg_test_separate_test); + + r.status |= run_bpkg ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-q", + "clean", + pkg); + + if (!r.status) + return false; + } } return true; @@ -3787,6 +3829,35 @@ build (size_t argc, const char* argv[]) prj.operations.end (), "test") != prj.operations.end ()); + // Collect the "testable" subprojects. + // + // Note that while we run the package tests using the bpkg-test command, + // we still collect the "testable" subprojects to clean them at the end + // to free up a bit of space. We will also reuse this list for testing + // against the installed package. + // + dir_paths subprj_dirs; // "Testable" package subprojects. + strings clean_specs; // While at it, collect the clean specs. + + for (const b_project_info::subproject& sp: prj.subprojects) + { + dir_path d (pkg_dir / sp.path); + + // Retrieve the subproject information similar to how we've done it + // for the package. + // + b_project_info si (prj_info (d, + b_info_flags::ext_mods, + "subproject")); + + const strings& ops (si.operations); + if (find (ops.begin (), ops.end (), "test") != ops.end ()) + { + subprj_dirs.push_back (sp.path); + clean_specs.push_back ("clean('" + d.representation () + "')"); + } + } + if (has_internal_tests || has_runtime_tests || has_buildtime_tests) { operation_result& r (add_result ("test")); @@ -3833,6 +3904,24 @@ build (size_t argc, const char* argv[]) break; } + // Now, as the subprojects are tested, clean them to free up a bit of + // space. + // + if (!clean_specs.empty ()) + { + step_id b (step_id::bpkg_test); + + r.status |= run_b ( + b, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-q", + clean_specs); + + if (!r.status) + break; + } + // External tests. // // Note that if the bpkg.test-separate.update step is disabled, we @@ -4651,38 +4740,7 @@ build (size_t argc, const char* argv[]) // Run the internal tests if the project contains "testable" // subprojects, but not for a module. // - has_internal_tests = false; - - dir_paths subprj_dirs; // "Testable" package subprojects. - - // Collect the "testable" subprojects. - // - if (!module_pkg) - { - assert (!rm.results.empty ()); - - // Result of the install or sys-install operation. - // - operation_result& r (rm.results.back ()); - - change_wd (trace, &r.log, effective_install_conf); - - for (const b_project_info::subproject& sp: prj.subprojects) - { - // Retrieve the subproject information similar to how we've done it - // for the package. - // - b_project_info si (prj_info (pkg_dir / sp.path, - b_info_flags::ext_mods, - "subproject")); - - const strings& ops (si.operations); - if (find (ops.begin (), ops.end (), "test") != ops.end ()) - subprj_dirs.push_back (sp.path); - } - - has_internal_tests = !subprj_dirs.empty (); - } + has_internal_tests = !module_pkg && !subprj_dirs.empty (); if (has_internal_tests || has_runtime_tests || has_buildtime_tests) { @@ -4770,6 +4828,7 @@ build (size_t argc, const char* argv[]) // etc). // strings test_specs; + strings clean_specs; for (const dir_path& d: subprj_dirs) { // b configure(@) @@ -4804,8 +4863,10 @@ build (size_t argc, const char* argv[]) if (!r.status) break; - test_specs.push_back ( - "test('" + subprj_out_dir.representation () + "')"); + string rp (subprj_out_dir.representation ()); + + test_specs.push_back ("test('" + rp + "')"); + clean_specs.push_back ("clean('" + rp + "')"); } if (!r.status) @@ -4835,6 +4896,20 @@ build (size_t argc, const char* argv[]) if (!r.status) break; + + // Now, as the subprojects are tested, clean them to free up a + // bit of space. + // + r.status |= run_b ( + b, + envvars, + trace, r.log, wre, + bkp_step, bkp_status, aux_env, last_cmd, + "-q", + clean_specs); + + if (!r.status) + break; } // // Fail if the breakpoint refers to the b.test-installed.test step diff --git a/tests/integration/testscript b/tests/integration/testscript index 942d8ec..8d6ad4e 100644 --- a/tests/integration/testscript +++ b/tests/integration/testscript @@ -372,7 +372,7 @@ sys:libz/* #\ pkg = odb -ver = 2.5.0-b.22.20220629083600.4a9af07ee566 +ver = 2.5.0-b.28.20241009172401.fa4b44129d84 rep_url = "https://git.codesynthesis.com/odb/odb.git#master" rep_type = git rfp = yes @@ -408,8 +408,8 @@ rfp = yes #\ pkg = libodb-sqlite -ver = 2.5.0-b.26.20240131175206.1c7f67f47770 -rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git" +ver = 2.5.0-b.28.20241009172401.fa4b44129d84 +rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master" rep_type = git rfp = yes tests="tests: odb-tests == $ver"' ? (!$defined(config.odb_tests.database)) config.odb_tests.database=sqlite' @@ -424,7 +424,7 @@ package_config = 'package-config: #\ pkg = libodb-pgsql ver = 2.5.0-b.26.20240131175206.1c7f67f47770 -rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git" +rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master" rep_type = git rfp = yes tests="tests: odb-tests == $ver"' ? (!$defined(config.odb_tests.database)) config.odb_tests.database=pgsql' @@ -439,7 +439,7 @@ package_config = 'package-config: #\ pkg = odb-tests ver = 2.5.0-b.26.20240131175206.1c7f67f47770 -rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git" +rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master" rep_type = git rfp = yes #\ @@ -453,7 +453,7 @@ config.odb_tests.database="sqlite pgsql" #\ pkg = libodb-oracle ver = 2.5.0-b.26.20240201133448.3fa01c83a095 -rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git" +rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master" rep_type = git rfp = yes package_config = 'package-config: @@ -465,7 +465,7 @@ config.cc.poptions+=-I/usr/include/oracle/12.2/client64 config.cc.loptions+=-L/u #\ pkg = libodb-qt ver = 2.5.0-b.26.20240201180613.633ad7ccad39 -rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git" +rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master" rep_type = git rfp = yes #\ -- cgit v1.1