Clean tests subprojects and external test packages to free up disk space in worker

2 files changed, 116 insertions, 41 deletions

diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index a5dd92d..285bf5e 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -1037,6 +1037,27 @@ run_b (step_id step,
 
 template <typename... A>
 static result_status
+run_b (step_id step,
+       tracer& t,
+       string& log, const regexes& warn_detect,
+       const optional<step_id>& bkp_step,
+       const optional<result_status>& bkp_status,
+       const strings& aux_env,
+       string& last_cmd,
+       const char* verbosity,
+       const strings& buildspecs, A&&... a)
+{
+  const char* const* envvars (nullptr);
+  return run_b (step,
+                envvars,
+                t,
+                log, warn_detect,
+                bkp_step, bkp_status, aux_env, last_cmd,
+                verbosity, buildspecs, forward<A> (a)...);
+}
+
+template <typename... A>
+static result_status
 run_ldconfig (step_id step,
               tracer& t,
               string& log, const regexes& warn_detect,
@@ -3771,6 +3792,27 @@ build (size_t argc, const char* argv[])
               if (!r.status)
                 return false;
             }
+
+            // Now, as the external tests package is tested, clean it to free
+            // up a bit of space.
+            //
+            {
+              step_id b (installed
+                         ? step_id::bpkg_test_separate_installed_test
+                         : step_id::bpkg_test_separate_test);
+
+              r.status |= run_bpkg (
+                b,
+                envvars,
+                trace, r.log, wre,
+                bkp_step, bkp_status, aux_env, last_cmd,
+                "-q",
+                "clean",
+                pkg);
+
+              if (!r.status)
+                return false;
+            }
           }
 
           return true;
@@ -3787,6 +3829,35 @@ build (size_t argc, const char* argv[])
                                      prj.operations.end (),
                                      "test") != prj.operations.end ());
 
+      // Collect the "testable" subprojects.
+      //
+      // Note that while we run the package tests using the bpkg-test command,
+      // we still collect the "testable" subprojects to clean them at the end
+      // to free up a bit of space. We will also reuse this list for testing
+      // against the installed package.
+      //
+      dir_paths subprj_dirs; // "Testable" package subprojects.
+      strings clean_specs;   // While at it, collect the clean specs.
+
+      for (const b_project_info::subproject& sp: prj.subprojects)
+      {
+        dir_path d (pkg_dir / sp.path);
+
+        // Retrieve the subproject information similar to how we've done it
+        // for the package.
+        //
+        b_project_info si (prj_info (d,
+                                     b_info_flags::ext_mods,
+                                     "subproject"));
+
+        const strings& ops (si.operations);
+        if (find (ops.begin (), ops.end (), "test") != ops.end ())
+        {
+          subprj_dirs.push_back (sp.path);
+          clean_specs.push_back ("clean('" + d.representation () + "')");
+        }
+      }
+
       if (has_internal_tests || has_runtime_tests || has_buildtime_tests)
       {
         operation_result& r (add_result ("test"));
@@ -3833,6 +3904,24 @@ build (size_t argc, const char* argv[])
           break;
         }
 
+        // Now, as the subprojects are tested, clean them to free up a bit of
+        // space.
+        //
+        if (!clean_specs.empty ())
+        {
+          step_id b (step_id::bpkg_test);
+
+          r.status |= run_b (
+            b,
+            trace, r.log, wre,
+            bkp_step, bkp_status, aux_env, last_cmd,
+            "-q",
+            clean_specs);
+
+          if (!r.status)
+            break;
+        }
+
         // External tests.
         //
         // Note that if the bpkg.test-separate.update step is disabled, we
@@ -4651,38 +4740,7 @@ build (size_t argc, const char* argv[])
         // Run the internal tests if the project contains "testable"
         // subprojects, but not for a module.
         //
-        has_internal_tests = false;
-
-        dir_paths subprj_dirs; // "Testable" package subprojects.
-
-        // Collect the "testable" subprojects.
-        //
-        if (!module_pkg)
-        {
-          assert (!rm.results.empty ());
-
-          // Result of the install or sys-install operation.
-          //
-          operation_result& r (rm.results.back ());
-
-          change_wd (trace, &r.log, effective_install_conf);
-
-          for (const b_project_info::subproject& sp: prj.subprojects)
-          {
-            // Retrieve the subproject information similar to how we've done it
-            // for the package.
-            //
-            b_project_info si (prj_info (pkg_dir / sp.path,
-                                         b_info_flags::ext_mods,
-                                         "subproject"));
-
-            const strings& ops (si.operations);
-            if (find (ops.begin (), ops.end (), "test") != ops.end ())
-              subprj_dirs.push_back (sp.path);
-          }
-
-          has_internal_tests = !subprj_dirs.empty ();
-        }
+        has_internal_tests = !module_pkg && !subprj_dirs.empty ();
 
         if (has_internal_tests || has_runtime_tests || has_buildtime_tests)
         {
@@ -4770,6 +4828,7 @@ build (size_t argc, const char* argv[])
             // etc).
             //
             strings test_specs;
+            strings clean_specs;
             for (const dir_path& d: subprj_dirs)
             {
               // b configure(<subprj-src-dir>@<subprj-out-dir>) <env-config-args>
@@ -4804,8 +4863,10 @@ build (size_t argc, const char* argv[])
               if (!r.status)
                 break;
 
-              test_specs.push_back (
-                "test('" + subprj_out_dir.representation () + "')");
+              string rp (subprj_out_dir.representation ());
+
+              test_specs.push_back ("test('" + rp + "')");
+              clean_specs.push_back ("clean('" + rp + "')");
             }
 
             if (!r.status)
@@ -4835,6 +4896,20 @@ build (size_t argc, const char* argv[])
 
               if (!r.status)
                 break;
+
+              // Now, as the subprojects are tested, clean them to free up a
+              // bit of space.
+              //
+              r.status |= run_b (
+                b,
+                envvars,
+                trace, r.log, wre,
+                bkp_step, bkp_status, aux_env, last_cmd,
+                "-q",
+                clean_specs);
+
+              if (!r.status)
+                break;
             }
             //
             // Fail if the breakpoint refers to the b.test-installed.test step
diff --git a/tests/integration/testscript b/tests/integration/testscript
index 942d8ec..8d6ad4e 100644
--- a/tests/integration/testscript
+++ b/tests/integration/testscript
@@ -372,7 +372,7 @@ sys:libz/*
 
 #\
 pkg = odb
-ver = 2.5.0-b.22.20220629083600.4a9af07ee566
+ver = 2.5.0-b.28.20241009172401.fa4b44129d84
 rep_url = "https://git.codesynthesis.com/odb/odb.git#master"
 rep_type = git
 rfp = yes
@@ -408,8 +408,8 @@ rfp = yes
 
 #\
 pkg = libodb-sqlite
-ver = 2.5.0-b.26.20240131175206.1c7f67f47770
-rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git"
+ver = 2.5.0-b.28.20241009172401.fa4b44129d84
+rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master"
 rep_type = git
 rfp = yes
 tests="tests: odb-tests == $ver"' ? (!$defined(config.odb_tests.database)) config.odb_tests.database=sqlite'
@@ -424,7 +424,7 @@ package_config = 'package-config:
 #\
 pkg = libodb-pgsql
 ver = 2.5.0-b.26.20240131175206.1c7f67f47770
-rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git"
+rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master"
 rep_type = git
 rfp = yes
 tests="tests: odb-tests == $ver"' ? (!$defined(config.odb_tests.database)) config.odb_tests.database=pgsql'
@@ -439,7 +439,7 @@ package_config = 'package-config:
 #\
 pkg = odb-tests
 ver = 2.5.0-b.26.20240131175206.1c7f67f47770
-rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git"
+rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master"
 rep_type = git
 rfp = yes
 #\
@@ -453,7 +453,7 @@ config.odb_tests.database="sqlite pgsql"
 #\
 pkg = libodb-oracle
 ver = 2.5.0-b.26.20240201133448.3fa01c83a095
-rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git"
+rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master"
 rep_type = git
 rfp = yes
 package_config = 'package-config:
@@ -465,7 +465,7 @@ config.cc.poptions+=-I/usr/include/oracle/12.2/client64 config.cc.loptions+=-L/u
 #\
 pkg = libodb-qt
 ver = 2.5.0-b.26.20240201180613.633ad7ccad39
-rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git"
+rep_url = "https://git.codesynthesis.com/var/scm/odb/odb.git#master"
 rep_type = git
 rfp = yes
 #\