Add support for install, test installed and uninstall operations

4 files changed, 302 insertions, 50 deletions

diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx
index eacd05a..80b4fb7 100644
--- a/bbot/agent/agent.cxx
+++ b/bbot/agent/agent.cxx
@@ -64,6 +64,17 @@ file_sync (const path& f)
     throw_system_error (errno);
 }
 
+static bool
+file_not_empty (const path& f)
+{
+  if (file_exists (f))
+  {
+    file_sync (f);
+    return !file_empty (f);
+  }
+  return false;
+}
+
 // The btrfs tool likes to print informational messages, like "Created
 // snapshot such and such". Luckily, it writes them to stdout while proper
 // diagnostics to stderr.
@@ -226,12 +237,8 @@ bootstrap_machine (const dir_path& md,
         //
         for (to = bootstrap_to; to != 0; tftpd.serve (to, 2))
         {
-          if (file_exists (mf))
-          {
-            file_sync (mf);
-            if (!file_empty (mf))
-              break;
-          }
+          if (file_not_empty (mf))
+            break;
         }
 
         if (to == 0)
@@ -615,9 +622,19 @@ try
   {
     // Simply wait for the file to appear.
     //
-    for (size_t i (0); !file_exists (rf); sleep (1))
+    for (size_t i (0);; sleep (1))
+    {
+      if (file_not_empty (rf))
+      {
+        // Wait a bit to make sure we see complete manifest.
+        //
+        sleep (2);
+        break;
+      }
+
       if (i++ % 10 == 0)
         l3 ([&]{trace << "waiting for result manifest";});
+    }
 
     r = parse_manifest<result_manifest> (rf, "result");
   }
@@ -711,12 +728,8 @@ try
         //
         for (to = build_to; to != 0; tftpd.serve (to, 2))
         {
-          if (file_exists (rf))
-          {
-            file_sync (rf);
-            if (!file_empty (rf))
-              break;
-          }
+          if (file_not_empty (rf))
+            break;
         }
 
         if (to == 0)
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index 67bbebe..2107dc3 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -9,6 +9,7 @@
 #endif
 
 #include <regex>
+#include <cstring>  // strchr()
 #include <iostream>
 
 #include <libbutl/pager.hxx>
@@ -59,15 +60,17 @@ catch (const system_error& e)
 
 using regexes = vector<regex>;
 
-// Match lines read from the command's stderr against the regular expressions
-// and return the warning result status (instead of success) in case of a
-// match.
+// Run a named command. Name is used for logging and diagnostics only. Match
+// lines read from the command's stderr against the regular expressions and
+// return the warning result status (instead of success) in case of a match.
 //
 template <typename... A>
 static result_status
-run_bpkg (tracer& t,
-          string& log, const regexes& warn_detect,
-          const string& cmd, A&&... a)
+run_cmd (tracer& t,
+         string& log, const regexes& warn_detect,
+         const string& name,
+         const process_env& pe,
+         A&&... a)
 {
   try
   {
@@ -90,9 +93,7 @@ run_bpkg (tracer& t,
                      fdnull (), // Never reads from stdout.
                      2,         // 1>&2
                      pipe.out,
-                     "bpkg",
-                     "-v",
-                     cmd,
+                     pe,
                      forward<A> (a)...));
 
     pipe.out.close ();
@@ -136,7 +137,7 @@ run_bpkg (tracer& t,
     if (pr.wait ())
       return r;
 
-    log += "bpkg " + cmd;
+    log += name;
     const process_exit& e (*pr.exit);
 
     if (e.normal ())
@@ -153,14 +154,50 @@ run_bpkg (tracer& t,
   }
   catch (const process_error& e)
   {
-    fail << "unable to execute bpkg " << cmd << ": " << e << endf;
+    fail << "unable to execute " << name << ": " << e << endf;
   }
   catch (const io_error& e)
   {
-    fail << "unable to read bpkg " << cmd << " diagnostics: " << e << endf;
+    fail << "unable to read " << name << " diagnostics: " << e << endf;
   }
 }
 
+template <typename... A>
+static result_status
+run_bpkg (tracer& t,
+          string& log, const regexes& warn_detect,
+          const string& cmd, A&&... a)
+{
+  return run_cmd (t,
+                  log, warn_detect,
+                  "bpkg " + cmd,
+                  "bpkg", "-v", cmd, forward<A> (a)...);
+}
+
+template <typename V, typename... A>
+static result_status
+run_b (tracer& t,
+       string& log, const regexes& warn_detect,
+       const V& envvars,
+       const string& buildspec, A&&... a)
+{
+  return run_cmd (t,
+                  log, warn_detect,
+                  "b " + buildspec,
+                  process_env ("b",  envvars),
+                  "-v", forward<A> (a)..., buildspec);
+}
+
+template <typename... A>
+static result_status
+run_b (tracer& t,
+       string& log, const regexes& warn_detect,
+       const string& buildspec, A&&... a)
+{
+  const char* const* envvars (nullptr);
+  return run_b (t, log, warn_detect, envvars, buildspec, forward<A> (a)...);
+}
+
 static void
 build (size_t argc, const char* argv[])
 {
@@ -171,8 +208,8 @@ build (size_t argc, const char* argv[])
   // 1. Parse the task manifest (it should be in CWD).
   //
   // 2. Run bpkg to create the configuration, add the repository, and
-  //    configure, build, and test the package all while saving the logs in
-  //    the result manifest.
+  //    configure, build, test, install and uninstall the package all while
+  //    saving the logs in the result manifest.
   //
   // 3. Upload the result manifest.
   //
@@ -198,7 +235,7 @@ build (size_t argc, const char* argv[])
     return rm.results.back ();
   };
 
-  dir_path owd;
+  dir_path rwd; // Root working directory.
 
   for (;;) // The "breakout" loop.
   {
@@ -219,29 +256,29 @@ build (size_t argc, const char* argv[])
     for (const auto& re: tm.unquoted_warning_regex ())
       wre.emplace_back (re, f);
 
+    strings config (tm.unquoted_config ());
+    const vector_view<const char*> env (argv + 1, argc - 1);
+
     // Configure.
     //
+    // Configuration directory name.
+    //
+    dir_path build_dir ("build");
     {
       operation_result& r (add_result ("configure"));
 
       // bpkg create <config-vars> <env-module> <env-config-vars>
       //
-      const vector_view<const char*> env (argv + 1, argc - 1);
-
-      // Configuration directory name.
-      //
-      dir_path dir ("build");
-
       r.status |= run_bpkg (trace, r.log, wre,
                             "create",
-                            "-d", dir.string (),
+                            "-d", build_dir.string (),
                             "--wipe",
-                            tm.unquoted_config (),
+                            config,
                             env);
       if (!r.status)
         break;
 
-      owd = change_wd (dir);
+      rwd = change_wd (build_dir);
 
       // bpkg add <repository-url>
       //
@@ -315,13 +352,184 @@ build (size_t argc, const char* argv[])
       rm.status |= r.status;
     }
 
+    // Install the package, optionally test the installation and uninstall
+    // afterwards.
+    //
+    // These operations are triggered by presence of config.install.root
+    // configuration variable having a non-empty value. Passing [null] value
+    // would be meaningless, so we don't recognize it as a special one.
+    //
+    dir_path install_root;
+    {
+      size_t n (19);
+      auto space = [] (char c) {return c == ' ' || c == '\t';};
+
+      for (const auto& s: reverse_iterate (config))
+      {
+        if (s.compare (0, n, "config.install.root") == 0 &&
+            (s[n] == '=' || space (s[n])))
+        {
+          while (space (s[n])) ++n; // Skip spaces.
+          if (s[n] == '=')     ++n; // Skip the equal sign.
+          while (space (s[n])) ++n; // Skip spaces.
+
+          install_root = dir_path (s, n, s.size () - n);
+          break;
+        }
+      }
+
+      if (install_root.empty ())
+        break;
+    }
+
+    // Now the overall plan is as follows:
+    //
+    // 1. Install the package.
+    //
+    // 2. If the package project has the 'tests' subdirectory that is a
+    //    subproject, then configure, build and test it out of the source tree
+    //    against the installed package.
+    //
+    // 3. Uninstall the package.
+    //
+    // Install.
+    //
+    {
+      operation_result& r (add_result ("install"));
+
+      // bpkg install <package-name>
+      //
+      r.status |= run_bpkg (trace, r.log, wre, "install", tm.name);
+
+      if (!r.status)
+        break;
+
+      rm.status |= r.status;
+    }
+
+    // Test installed.
+    //
+    // The package tests subdirectory path (may not exist).
+    //
+    dir_path tests_dir (tm.name + "-" + tm.version.string ());
+    tests_dir /= "tests";
+
+    // We will consider the tests subdirectory to be a subproject if it
+    // contains a build2 bootstrap file.
+    //
+    if (file_exists (tests_dir / path ("build/bootstrap.build")))
+    {
+      operation_result& r (add_result ("test-installed"));
+
+      change_wd (rwd);
+
+      // Sort environment arguments into modules and configuration variables.
+      //
+      string mods;  // build2 create meta-operation parameters.
+      cstrings vars;
+
+      for (const auto& a: env)
+      {
+        // Note that we don't check for the argument emptiness, as this is
+        // already done by 'bpkg create' (see above).
+        //
+        if (strchr (a, '=') != nullptr)
+        {
+          vars.push_back (a);
+        }
+        else
+        {
+          mods += mods.empty () ? ", " : " ";
+          mods += a;
+        }
+      }
+
+      // b <config-vars> <env-config-vars> create(<dir>, <modules>)
+      //
+      // Amalgamation directory that will contain configuration subdirectory
+      // for package tests out of source tree build.
+      //
+      dir_path out_dir ("build-installed");
+
+      r.status |= run_b (
+        trace, r.log, wre,
+        "create(" + out_dir.representation () + mods + ")",
+        config,
+        vars);
+
+      if (!r.status)
+        break;
+
+      rm.status |= r.status;
+
+      // Make sure that the installed package executables are properly imported
+      // when configuring and running tests.
+      //
+      // Note that we add the $config.install.root/bin directory at the
+      // beginning of the PATH environment variable value, so the installed
+      // executables are found first.
+      //
+      string paths ("PATH=" + (install_root / "bin").string ());
+
+      if (char const* s = getenv ("PATH"))
+      {
+        paths += path::traits::path_separator;
+        paths += s;
+      }
+
+      small_vector<string, 1> envvars ({move (paths)});
+
+      // b configure(<tests-dir>@<tests-out-dir>)
+      //
+      dir_path tests_out_dir (out_dir / dir_path ("tests"));
+
+      r.status |= run_b (trace, r.log, wre,
+                         envvars,
+                         "configure(" +
+                         (build_dir / tests_dir).representation ()  + '@' +
+                         tests_out_dir.representation () + ")");
+
+      if (!r.status)
+        break;
+
+      rm.status |= r.status;
+
+      // b test(<tests-out-dir>)
+      //
+      r.status |= run_b (trace, r.log, wre,
+                         envvars,
+                         "test(" + tests_out_dir.representation () + ')');
+
+      if (!r.status)
+        break;
+
+      rm.status |= r.status;
+
+      change_wd (build_dir);
+    }
+
+    // Uninstall.
+    //
+    {
+      operation_result& r (add_result ("uninstall"));
+
+      // bpkg uninstall <package-name>
+      //
+      r.status |= run_bpkg (trace, r.log, wre, "uninstall", tm.name);
+
+      if (!r.status)
+        break;
+
+      rm.status |= r.status;
+    }
+
     break;
   }
 
   rm.status |= rm.results.back ().status; // Merge last in case of a break.
 
-  if (!owd.empty ())
-    change_wd (owd);
+  if (!rwd.empty ())
+    change_wd (rwd);
 
   // Upload the result.
   //
diff --git a/doc/manual.cli b/doc/manual.cli
index 46f947f..9d67377 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -370,14 +370,20 @@ SYNOPSIS
 name:     <package-name>
 version:  <package-version>
 
-status:           <status>
-configure-status: <status>
-update-status:    <status>
-test-status:      <status>
-
-configure-log: <text>
-update-log:    <text>
-test-log:      <text>
+status:                <status>
+configure-status:      <status>
+update-status:         <status>
+test-status:           <status>
+install-status:        <status>
+test-installed-status: <status>
+uninstall-status:      <status>
+
+configure-log:      <text>
+update-log:         <text>
+test-log:           <text>
+install-log:        <text>
+test-installed-log: <text>
+uninstall-log:      <text>
 \
 
 The result manifest describes a build result.
@@ -677,7 +683,7 @@ and private installations:
 
 \
 linux*-gcc* linux-gcc-sys x86_64-linux-gnu config.install.root=/usr config.install.sudo=sudo
-linux*-gcc* linux-gcc-prv x86_64-linux-gnu config.install.root=/tmp/install config.cc.poptions=-I/tmp/install/include config.cc.loptions=-L/tmp/install/lib
+linux*-gcc* linux-gcc-prv x86_64-linux-gnu config.install.root=/tmp/install config.cc.poptions=-I/tmp/install/include config.cc.loptions=-L/tmp/install/lib config.bin.rpath=/tmp/install/bin
 \
 
 Note also that while building and run tests against the installation the
diff --git a/tests/integration/testscript b/tests/integration/testscript
index 2ae3b3c..6084d32 100644
--- a/tests/integration/testscript
+++ b/tests/integration/testscript
@@ -48,10 +48,34 @@ controller = https://stage.build2.org/?build-task
 wait=1s
 controller = --fake-request ../task --dump-result
 
-pkg = hello
+pkg = libhello
 ver = 1.0.0
 rep = https://build2.org/pkg/1/hello/stable
 rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94
+
+#\
+pkg = bpkg
+ver = 0.6.0-a.0.1499870184.53fff46c5e5fd4df
+rep = https://stage.build2.org/1
+rfp = 37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65
+#\
+
+#\
+pkg = build2
+ver = 0.6.0-a.0.1500478402.30578be48463b93f
+rep = https://stage.build2.org/1
+rfp = 37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65
+#\
+
+# Note that we also need to make sure that the installed package libraries are
+# properly imported when configuring and running tests, and that the installed
+# executables are runnable.
+#
+config = "config.install.root='$~/install' \
+config.cc.poptions=-I'$~/install/include' \
+config.cc.loptions=-L'$~/install/lib' \
+config.bin.rpath='$~/install/lib'"
+
 +cat <<"EOI" >=task
   : 1
   name: $pkg
@@ -60,6 +84,7 @@ rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30
   trust: $rfp
   machine: $machine
   target: $target
+  config: $config
   EOI
 
 #
@@ -96,5 +121,5 @@ a = $0
   chmod ugo+x $target;
   sleep $wait;
   $w --verbose 3 --startup --tftp-host $tftp --environments $~ \
-    &build/*** &manifest 2>|
+    &build/*** &?build-installed/*** &manifest 2>|
 }