Implement machine enumeration

13 files changed, 813 insertions, 236 deletions

diff --git a/bbot/agent.cli b/bbot/agent.cli
index c4ee356..7a6c201 100644
--- a/bbot/agent.cli
+++ b/bbot/agent.cli
@@ -37,6 +37,9 @@ namespace bbot
   {
     "\h|OPTIONS|"
 
+    bool --help {"Print usage information and exit."}
+    bool --version {"Print version and exit."}
+
     bool --systemd-daemon
     {
       "Start as a simple systemd daemon."
@@ -46,16 +49,41 @@ namespace bbot
     {
       "<num>",
       "Number of CPUs (threads) to use, 1 by default."
-    };
+    }
 
     size_t --ram (1024 * 1024) // 1G
     {
       "<num>",
       "Amount of RAM (in kB) to use, 1G by default."
-    };
+    }
 
-    bool --help {"Print usage information and exit."}
-    bool --version {"Print version and exit."}
+    dir_path --machines = "/build/machines/"
+    {
+      "<dir>",
+      "The location of the build machines with \cb{/build/machines/} being
+       the default."
+    }
+
+    uint16_t --verbose = 1
+    {
+      "<level>",
+      "Set the diagnostics verbosity to <level> between 0 and 6 with level 1
+       being the default."
+    }
+
+    // Testing options.
+    //
+    bool --dump-machines
+    {
+      "Dump available machines to \c{stdout}, (re)-bootstrapping any if
+       necessary, and exit."
+    }
+
+    bool --fake-bootstrap
+    {
+      "Fake the machine bootstrap process by creating the expected bootstrapped
+       machine manifest."
+    }
   };
 
   "
diff --git a/bbot/agent.cxx b/bbot/agent.cxx
index 3e4f8dc..76c3a86 100644
--- a/bbot/agent.cxx
+++ b/bbot/agent.cxx
@@ -9,12 +9,8 @@
 #include <iostream>
 
 #include <butl/pager>
-#include <butl/fdstream>
 #include <butl/filesystem> // dir_iterator
 
-#include <butl/manifest-parser>
-#include <butl/manifest-serializer>
-
 #include <bbot/manifest>
 
 #include <bbot/types>
@@ -29,50 +25,64 @@ using namespace std;
 using namespace butl;
 using namespace bbot;
 
+// The btrfs tool likes to print informational messages, like "Created
+// snapshot such and such". Luckily, it writes them to stdout while proper
+// diagnostics to stderr.
+//
+template <typename... A>
+inline void
+btrfs (tracer& t, A&&... a)
+{
+  if (verb >= 3)
+    run (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...);
+  else
+    run (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
+}
+
+template <typename... A>
+inline butl::process_exit::code_type
+btrfs_exit (tracer& t, A&&... a)
+{
+  return verb >= 3
+    ? run_exit (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...)
+    : run_exit (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
+}
+
+agent_options ops;
+
 const string bs_prot ("1"); // Bootstrap protocol version.
 
 string tc_name; // Toolchain name.
 string tc_num;  // Toolchain number.
 string tc_id;   // Toolchain id.
 
-template <typename T>
-static T
-parse_manifest (const path& f, const char* what, bool ignore_unknown = true)
+static bootstrapped_machine_manifest
+bootstrap_machine (const dir_path& md, const machine_manifest& mm)
 {
-  try
-  {
-    if (!file_exists (f))
-      fail << what << " manifest file " << f << " does not exist";
+  bootstrapped_machine_manifest r {
+    mm,
+    toolchain_manifest {tc_id},
+    bootstrap_manifest {
+      bootstrap_manifest::versions_type {
+        {"bbot",    BBOT_VERSION},
+        {"libbbot", LIBBBOT_VERSION},
+        {"libbpkg", LIBBPKG_VERSION},
+        {"libbutl", LIBBUTL_VERSION}
+      }
+    }
+  };
 
-    ifdstream ifs (f);
-    manifest_parser mp (ifs, f.string ());
-    return T (mp, ignore_unknown);
-  }
-  catch (const manifest_parsing& e)
-  {
-    fail << "invalid " << what << " manifest: "
-         << f << ':' << e.line << ':' << e.column << ": " << e.description
-         << endf;
-  }
-  catch (const io_error& e)
-  {
-    fail << "unable to read " << what << " manifest " << f << ": " << e
-         << endf;
-  }
-  catch (const system_error& e) // EACCES, etc.
+  if (!ops.fake_bootstrap ())
   {
-    fail << "unable to access " << what << " manifest " << f << ": " << e
-         << endf;
   }
-}
-
-/*
 
-static bootstrapped_machine_manifest
-bootstrap_machine (const dir_path&);
+  serialize_manifest (r, md / "manifest", "bootstrapped machine");
+  return r;
+}
 
 static machine_manifests
 enumerate_machines (const dir_path& rd)
+try
 {
   tracer trace ("enumerate_machines");
 
@@ -93,231 +103,242 @@ enumerate_machines (const dir_path& rd)
 
     // Inside we have machines.
     //
-    for (const dir_entry& me: dir_iterator (vd))
+    try
     {
-      const string mn (me.path ().string ());
-
-      if (me.type () != entry_type::directory || mn[0] == '.')
-        continue;
-
-      const dir_path md (dir_path (vd) /= mn);
-
-      // Our endgoal here is to obtain a bootstrapped snapshot of this machine
-      // while watching out for potential race conditions (machines being
-      // added/upgraded/removed; see the manual for details).
-      //
-      // So here is our overall plan:
-      //
-      // 1. Resolve current subvolume link for our bootstrap protocol.
-      //
-      // 2. If there is no link, cleanup and ignore this machine.
-      //
-      // 3. Try to create a snapshot of current subvolume (this operation is
-      //    atomic). If failed (e.g., someone changed the link and removed the
-      //    subvolume in the meantime), retry from #1.
-      //
-      // 4. Compare the snapshot to the already bootstrapped version (if any)
-      //    and see if we need to re-bootstrap. If so, use the snapshot as a
-      //    starting point. Rename to bootstrapped at the end (atomic).
-      //
-      const dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P>
-      const dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<toolc...>
-      bool te (dir_exists (tp));
-
-      auto delete_t = [&tp] ()
+      for (const dir_entry& me: dir_iterator (vd))
       {
-        // btrfs property set -ts $tp ro false
-        // btrfs subvolume delete $tp
-      };
+        const string mn (me.path ().string ());
 
-      for (size_t retry (0);; ++retry)
-      {
-        if (retry != 0)
-          sleep (1);
+        if (me.type () != entry_type::directory || mn[0] == '.')
+          continue;
+
+        const dir_path md (dir_path (vd) /= mn);
 
-        // Resolve the link to subvolume path.
+        // Our endgoal here is to obtain a bootstrapped snapshot of this
+        // machine while watching out for potential race conditions (machines
+        // being added/upgraded/removed; see the manual for details).
         //
-        dir_path sp; // <name>-<P>.<R>
-        try
+        // So here is our overall plan:
+        //
+        // 1. Resolve current subvolume link for our bootstrap protocol.
+        //
+        // 2. If there is no link, cleanup and ignore this machine.
+        //
+        // 3. Try to create a snapshot of current subvolume (this operation is
+        //    atomic). If failed (e.g., someone changed the link and removed
+        //    the subvolume in the meantime), retry from #1.
+        //
+        // 4. Compare the snapshot to the already bootstrapped version (if
+        //    any) and see if we need to re-bootstrap. If so, use the snapshot
+        //    as a starting point. Rename to bootstrapped at the end (atomic).
+        //
+        const dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P>
+        const dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<too...>
+        bool te (dir_exists (tp));
+
+        auto delete_t = [&tp, &trace] ()
+        {
+          btrfs (trace, "property", "set", "-ts", tp, "ro", "false");
+          btrfs (trace, "subvolume", "delete", tp);
+        };
+
+        for (size_t retry (0);; ++retry)
         {
-          char b [PATH_MAX + 1];
-          ssize_t r (readlink (lp.string ().c_str (), b, sizeof (b)));
+          if (retry != 0)
+            sleep (1);
 
-          if (r == -1)
+          // Resolve the link to subvolume path.
+          //
+          dir_path sp; // <name>-<P>.<R>
+          try
           {
-            if (errno != ENOENT)
-              throw_generic_error (errno);
+            char b [PATH_MAX + 1];
+            ssize_t r (readlink (lp.string ().c_str (), b, sizeof (b)));
+
+            if (r == -1)
+            {
+              if (errno != ENOENT)
+                throw_generic_error (errno);
+            }
+            else if (static_cast<size_t> (r) >= sizeof (b))
+              throw_generic_error (EINVAL);
+            else
+            {
+              b[r] = '\0';
+              sp = dir_path (b);
+              if (sp.relative ())
+                sp = md / sp;
+            }
           }
-          else if (static_cast<size_t> (r) >= sizeof (b))
-            throw_generic_error (EINVAL);
-          else
+          catch (const system_error& e)
           {
-            b[r] = '\0';
-            sp = dir_path (b);
-            if (sp.relative ())
-              sp = md / sp;
+            fail << "unable to read subvolume link " << lp << ": " << e;
           }
-        }
-        catch (const system_error& e)
-        {
-          fail << "unable to read subvolume link " << lp << ": " << e;
-        }
 
-        // If the resolution fails, then this means there is no current
-        // machine subvolume (for this bootstrap protocol). In this case we
-        // clean up our toolchain subvolume (<name>-<toolchain>) and ignore
-        // this machine.
-        //
-        if (sp.empty ())
-        {
-          if (te)
-            delete_t ();
+          // If the resolution fails, then this means there is no current
+          // machine subvolume (for this bootstrap protocol). In this case we
+          // clean up our toolchain subvolume (<name>-<toolchain>) and ignore
+          // this machine.
+          //
+          if (sp.empty ())
+          {
+            if (te)
+              delete_t ();
 
-          break;
-        }
+            l2 ([&]{trace << "skipping " << md << ": no subvolume link";});
+            break;
+          }
 
-        // <name>-<toolchain>-<xxx>
-        //
-        const dir_path xp (dir_path (md) /=
-                           path::traits::temp_name (mn + '-' + tc_name));
+          // <name>-<toolchain>-<xxx>
+          //
+          const dir_path xp (dir_path (md) /=
+                             path::traits::temp_name (mn + '-' + tc_name));
 
-        // btrfs subvolume snapshot $sp $xp
-        if (false)
-        {
-          if (retry >= 10)
-            fail << "unable to snapshot subvolume " << sp;
+          if (btrfs_exit (trace, "subvolume", "snapshot", sp, xp) != 0)
+          {
+            if (retry >= 10)
+              fail << "unable to snapshot subvolume " << sp;
 
-          continue;
-        }
+            continue;
+          }
 
-        // Load the (original) machine manifest.
-        //
-        auto mm (
-          parse_manifest<machine_manifest> (sp / "manifest", "machine"));
+          // Load the (original) machine manifest.
+          //
+          auto mm (
+            parse_manifest<machine_manifest> (sp / "manifest", "machine"));
 
-        // If we already have <name>-<toolchain>, see if it needs to be re-
-        // bootstrapped. Things that render it obsolete:
-        //
-        // 1. New machine revision  (compare machine ids).
-        // 2. New toolchain         (compare toolchain ids).
-        // 3. New bbot/libbbot      (compare versions).
-        //
-        // The last case has a complication: what should we do if we have
-        // bootstrapped a newer version of bbot? This would mean that we are
-        // about to be stopped and upgraded (and the upgraded version will
-        // probably be able to use the result). So we simply ignore this
-        // machine for this run.
+          // If we already have <name>-<toolchain>, see if it needs to be re-
+          // bootstrapped. Things that render it obsolete:
+          //
+          // 1. New machine revision  (compare machine ids).
+          // 2. New toolchain         (compare toolchain ids).
+          // 3. New bbot/libbbot      (compare versions).
+          //
+          // The last case has a complication: what should we do if we have
+          // bootstrapped a newer version of bbot? This would mean that we are
+          // about to be stopped and upgraded (and the upgraded version will
+          // probably be able to use the result). So we simply ignore this
+          // machine for this run.
 
-        // Return -1 if older, 0 if the same, and +1 if newer.
-        //
-        auto compare_bbot = [] (const bootstrap_manifest& m) -> int
-        {
-          auto cmp = [&m] (const string& n, uint64_t v) -> int
+          // Return -1 if older, 0 if the same, and +1 if newer.
+          //
+          auto compare_bbot = [] (const bootstrap_manifest& m) -> int
           {
-            auto i = m.versions.find (n);
+            auto cmp = [&m] (const string& n, uint64_t v) -> int
+            {
+              auto i = m.versions.find (n);
+              return
+                i == m.versions.end () || i->second < v
+                ? -1
+                : i->second > v ? 1 : 0;
+            };
+
+            // Start from the top assuming a new dependency cannot be added
+            // without changing the dependent's version.
+            //
+            int r;
             return
-              i == m.versions.end () || i->second < v
-              ? -1
-              : i->second > v ? 1 : 0;
+              (r = cmp ("bbot",       BBOT_VERSION)) != 0 ? r :
+              (r = cmp ("libbbot", LIBBBOT_VERSION)) != 0 ? r :
+              (r = cmp ("libbpkg", LIBBPKG_VERSION)) != 0 ? r :
+              (r = cmp ("libbutl", LIBBUTL_VERSION)) != 0 ? r : 0;
           };
 
-          // Start from the top assuming a new dependency cannot be added
-          // without changing the dependent's version.
-          //
-          int r;
-          return
-            (r = cmp ("bbot",       BBOT_VERSION)) != 0 ? r :
-            (r = cmp ("libbbot", LIBBBOT_VERSION)) != 0 ? r :
-            (r = cmp ("libbpkg", LIBBPKG_VERSION)) != 0 ? r :
-            (r = cmp ("libbutl", LIBBUTL_VERSION)) != 0 ? r : 0;
-        };
-
-        if (te)
-        {
-          auto bmm (
-            parse_manifest<bootstrapped_machine_manifest> (
-              tp / "manifest",
-              "bootstrapped machine"));
-
-          if (bmm.machine.id != mm.id)
+          if (te)
           {
-            trace << "re-bootstrapping " << tp << ": new machine";
-            te = false;
-          }
+            auto bmm (
+              parse_manifest<bootstrapped_machine_manifest> (
+                tp / "manifest",
+                "bootstrapped machine"));
 
-          if (bmm.toolchain.id != tc_id)
-          {
-            trace << "re-bootstrapping " << tp << ": new toolchain";
-            te = false;
-          }
+            if (bmm.machine.id != mm.id)
+            {
+              l2 ([&]{trace << "re-bootstrapping " << tp << ": new machine";});
+              te = false;
+            }
 
-          if (int i = compare_bbot (bmm.bootstrap))
-          {
-            if (i < 0)
+            if (bmm.toolchain.id != tc_id)
             {
-              trace << "re-bootstrapping " << tp << ": new bbot";
+              l2 ([&]{trace << "re-bootstrapping " << tp << ": new toolchain";});
               te = false;
             }
-            else
+
+            if (int i = compare_bbot (bmm.bootstrap))
             {
-              trace << "ignoring " << tp << ": newer bbot";
-              // btrfs subvolume snapshot $xp
-              break;
+              if (i < 0)
+              {
+                l2 ([&]{trace << "re-bootstrapping " << tp << ": new bbot";});
+                te = false;
+              }
+              else
+              {
+                l2 ([&]{trace << "ignoring " << tp << ": old bbot";});
+                btrfs (trace, "subvolume", "delete", xp);
+                break;
+              }
             }
+
+            if (!te)
+              delete_t ();
           }
+          else
+            l2 ([&]{trace << "bootstrapping " << tp;});
 
           if (!te)
-            delete_t ();
-        }
-
-        if (!te)
-        {
-          // Use the <name>-<toolchain>-<xxx> snapshot that we have made to
-          // bootstrap the new machine. Then atomically rename it to
-          // <name>-<toolchain>.
-          //
-          bootstrapped_machine_manifest bmm (bootstrap_machine (xp));
-
-          try
-          {
-            mvdir (xp, tp);
-          }
-          catch (const system_error& e)
           {
-            fail << "unable to rename " << xp << " to " << tp;
-          }
+            // Use the <name>-<toolchain>-<xxx> snapshot that we have made to
+            // bootstrap the new machine. Then atomically rename it to
+            // <name>-<toolchain>.
+            //
+            bootstrapped_machine_manifest bmm (bootstrap_machine (xp, mm));
 
-          te = true;
+            try
+            {
+              mvdir (xp, tp);
+            }
+            catch (const system_error& e)
+            {
+              fail << "unable to rename " << xp << " to " << tp;
+            }
 
-          // Check the boostrapped bbot version as above and ignore this
-          // machine if it's newer than us.
-          //
-          if (int i = compare_bbot (bmm.bootstrap))
-          {
-            assert (i > 0);
-            trace << "ignoring " << tp << ": newer bbot";
-            break;
+            te = true;
+
+            // Check the boostrapped bbot version as above and ignore this
+            // machine if it's newer than us.
+            //
+            if (int i = compare_bbot (bmm.bootstrap))
+            {
+              assert (i > 0);
+              l2 ([&]{trace << "ignoring " << tp << ": old bbot";});
+              break;
+            }
           }
-        }
-        else
-          ;// btrfs subvolume snapshot $xp
+          else
+            btrfs (trace, "subvolume", "delete", xp);
 
-        // Add the machine to the list.
-        //
-        // In order not to forget to clear new fields, we are instead going to
-        // create a new instance with just the required fields.
-        //
-        r.push_back (machine_manifest (mm.id, mm.name, mm.summary));
+          // Add the machine to the list.
+          //
+          // In order not to forget to clear new fields, we are instead going
+          // to create a new instance with just the required fields.
+          //
+          r.push_back (machine_manifest (mm.id, mm.name, mm.summary));
 
-        break;
+          break;
+        }
       }
     }
+    catch (const system_error& e)
+    {
+      fail << "unable to iterate over " << vd << ": " << e << endf;
+    }
   }
 
   return r;
 }
-
-*/
+catch (const system_error& e)
+{
+  fail << "unable to iterate over " << rd << ": " << e << endf;
+}
 
 extern "C" void
 handle_signal (int sig)
@@ -339,7 +360,9 @@ main (int argc, char* argv[])
 try
 {
   cli::argv_scanner scan (argc, argv, true);
-  agent_options ops (scan);
+  ops.parse (scan);
+
+  verb = ops.verbose ();
 
   if (ops.systemd_daemon ())
   {
@@ -359,6 +382,11 @@ try
     warn.type_  = "<4>";
     info.type_  = "<6>";
     trace_type  = "<7>";
+
+    info << "bbot agent for " << tc_name << '/' << tc_num <<
+      info << "toolchain id " << tc_id <<
+      info << "CPU(s)       " << ops.cpu () <<
+      info << "RAM(kB)      " << ops.ram ();
   }
 
   tracer trace ("main");
@@ -412,16 +440,28 @@ try
     fail << "unable to set signal handler: "
          << system_error (errno, generic_category ()); // Sanitize.
 
-  info << "bbot agent for " << tc_name << '/' << tc_num <<
-    info << "toolchain id " << tc_id <<
-    info << "CPU(s)       " << ops.cpu () <<
-    info << "RAM(kB)      " << ops.ram ();
-
-  for (;;)
+  // The work loop. The steps we go through are:
+  //
+  // 1. Enumerate the available machines, (re-)bootstrapping any of necessary.
+  //
+  // 2. Poll controller(s) for build tasks.
+  //
+  // 3. If no build tasks are available, go to #1 after sleeping a bit.
+  //
+  // 4. If a build task is returned, do it, upload the result, and go to #1
+  //    immediately.
+  //
+  for (unsigned int s; (s = 60); sleep (s))
   {
-    error << "sleeping" <<
-      warn << "lightly";
-    sleep (10);
+    machine_manifests mms (enumerate_machines (ops.machines ()));
+
+    if (ops.dump_machines ())
+    {
+      for (const machine_manifest& mm: mms)
+        serialize_manifest (mm, cout, "stdout", "machine manifest");
+
+      return 0;
+    }
   }
 }
 catch (const failed&)
diff --git a/bbot/bbot-agent@.service b/bbot/bbot-agent@.service
index 496692f..f8349ff 100644
--- a/bbot/bbot-agent@.service
+++ b/bbot/bbot-agent@.service
@@ -7,12 +7,14 @@ Type=simple
 
 Environment=CPU=1
 Environment=RAM=1048576
+Environment=VERBOSE=1
 Environment=TOOLCHAIN_ID=123abc
 Environment=TOOLCHAIN_NUM=1
 
 ExecStart=/build/bbot/%i/bin/bbot-agent --systemd-daemon \
   --cpu ${CPU} \
   --ram ${RAM} \
+  --verbose ${VERBOSE} \
   %i \
   ${TOOLCHAIN_NUM} \
   ${TOOLCHAIN_ID}
diff --git a/bbot/bootstrap-manifest b/bbot/bootstrap-manifest
index c29406a..6007c6e 100644
--- a/bbot/bootstrap-manifest
+++ b/bbot/bootstrap-manifest
@@ -26,6 +26,9 @@ namespace bbot
     //
     string id;
 
+    explicit
+    toolchain_manifest (string i): id (i) {}
+
   public:
     toolchain_manifest () = default; // VC export.
     toolchain_manifest (butl::manifest_parser&, bool ignore_unknown = false);
@@ -51,7 +54,12 @@ namespace bbot
     // libbbot-version: 1010100  # 1.1.1
     // bbot-version:    1010200  # 1.1.2
     //
-    std::map<string, uint64_t> versions;
+    using versions_type = std::map<string, uint64_t>;
+    versions_type versions;
+
+    explicit
+    bootstrap_manifest (versions_type v)
+        : versions (move (v)) {}
 
   public:
     bootstrap_manifest () = default; // VC export.
@@ -74,6 +82,11 @@ namespace bbot
     toolchain_manifest toolchain;
     bootstrap_manifest bootstrap;
 
+    bootstrapped_machine_manifest (machine_manifest m,
+                                   toolchain_manifest t,
+                                   bootstrap_manifest b)
+        : machine (move (m)), toolchain (move (t)), bootstrap (move (b)) {}
+
   public:
     bootstrapped_machine_manifest () = default; // VC export.
     bootstrapped_machine_manifest (butl::manifest_parser&,
diff --git a/bbot/buildfile b/bbot/buildfile
index 4045888..8c1e5b4 100644
--- a/bbot/buildfile
+++ b/bbot/buildfile
@@ -30,7 +30,7 @@ if ($cxx.target.class == "linux")
   {hxx         cxx}{ diagnostics        }                                 \
   {hxx            }{ types              }                                 \
   {hxx         cxx}{ types-parsers      }                                 \
-  {hxx         cxx}{ utility            }                                 \
+  {hxx     txx cxx}{ utility            }                                 \
   {hxx            }{ version-impl       }                                 \
                      $libs
 }
@@ -43,7 +43,7 @@ exe{bbot-worker}:                                                        \
 {hxx         cxx}{ diagnostics         }                                 \
 {hxx            }{ types               }                                 \
 {hxx         cxx}{ types-parsers       }                                 \
-{hxx         cxx}{ utility             }                                 \
+{hxx     txx cxx}{ utility             }                                 \
 {hxx            }{ version-impl        }                                 \
                    $libs
 
@@ -57,7 +57,7 @@ if $cli.configured
 
   cli.options += -I $src_root --include-with-brackets --include-prefix bbot \
 --guard-prefix BBOT --cxx-prologue "#include <bbot/types-parsers>" \
---cli-namespace bbot::cli --generate-specifier
+--cli-namespace bbot::cli --generate-specifier --generate-parse
 
   cli.cxx{common-options}: cli.options = $cli.options # No usage.
 
diff --git a/bbot/diagnostics b/bbot/diagnostics
index 6699c45..ebf658d 100644
--- a/bbot/diagnostics
+++ b/bbot/diagnostics
@@ -7,8 +7,7 @@
 
 #include <butl/diagnostics>
 
-#include <bbot/types>
-#include <bbot/utility>
+#include <bbot/types> // Note: not <bbot/utility>.
 
 namespace bbot
 {
@@ -106,7 +105,12 @@ namespace bbot
     trace_mark_base (const char* name, const void* data = nullptr);
   };
   using trace_mark = butl::diag_mark<trace_mark_base>;
-  using tracer = trace_mark;
+
+  // using tracer = trace_mark;
+  class tracer: public trace_mark
+  {
+  public: using trace_mark::trace_mark;
+  };
 
   // fail
   //
@@ -144,7 +148,7 @@ namespace bbot
   using fail_end = butl::diag_noreturn_end<fail_end_base>;
 
   extern fail_mark fail;
-  extern const fail_end  endf;
+  extern const fail_end endf;
 }
 
 #endif // BBOT_DIAGNOSTICS
diff --git a/bbot/diagnostics.cxx b/bbot/diagnostics.cxx
index 5c356f5..2779770 100644
--- a/bbot/diagnostics.cxx
+++ b/bbot/diagnostics.cxx
@@ -4,6 +4,8 @@
 
 #include <bbot/diagnostics>
 
+#include <bbot/utility>
+
 using namespace std;
 using namespace butl;
 
diff --git a/bbot/utility b/bbot/utility
index 81c6c87..ebab971 100644
--- a/bbot/utility
+++ b/bbot/utility
@@ -13,8 +13,8 @@
 
 #include <butl/ft/lang>
 
+#include <butl/process>
 #include <butl/utility> // casecmp(), reverse_iterate(), etc
-
 #include <butl/filesystem>
 
 #include <bbot/types>
@@ -39,6 +39,51 @@ namespace bbot
 
   using butl::exception_guard;
   using butl::make_exception_guard;
+
+  // Process execution.
+  //
+  class tracer;
+
+  template <typename I, typename O, typename E, typename P, typename... A>
+  void
+  run (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+
+  template <typename I, typename O, typename E, typename P, typename... A>
+  butl::process_exit::code_type
+  run_exit (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+
+  template <typename P, typename... A>
+  inline void
+  run (tracer& t, P&& p, A&&... a)
+  {
+    run (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
+  }
+
+  template <typename P, typename... A>
+  inline butl::process_exit::code_type
+  run_exit (tracer& t, P&& p, A&&... a)
+  {
+    return run (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
+  }
+
+  // Manifest parsing and serialization.
+  //
+  template <typename T>
+  T
+  parse_manifest (const path&, const char* what, bool ignore_unknown = true);
+
+  template <typename T>
+  void
+  serialize_manifest (const T&, const path&, const char* what);
+
+  template <typename T>
+  void
+  serialize_manifest (const T&,
+                      ostream&,
+                      const string& name,
+                      const char* what);
 }
 
+#include <bbot/utility.txx>
+
 #endif // BBOT_UTILITY
diff --git a/bbot/utility.txx b/bbot/utility.txx
new file mode 100644
index 0000000..d641612
--- /dev/null
+++ b/bbot/utility.txx
@@ -0,0 +1,144 @@
+// file      : bbot/utility.txx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <butl/fdstream>
+
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+#include <bbot/diagnostics>
+
+namespace bbot
+{
+  template <typename I, typename O, typename E, typename P, typename... A>
+  butl::process_exit::code_type
+  run_exit (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args)
+  {
+    using namespace butl;
+
+    try
+    {
+      auto cmdc = [&t] (const char* cmd[], size_t n)
+      {
+        if (verb >= 2)
+        {
+          diag_record dr (t);
+          process::print (dr.os, cmd, n);
+        }
+      };
+
+      process_exit r (process_run (cmdc,
+                                   forward<I> (in),
+                                   forward<O> (out),
+                                   forward<E> (err),
+                                   dir_path (),
+                                   p,
+                                   forward<A> (args)...));
+
+      if (!r.normal ())
+        fail << "process " << p << " terminated abnormally: "
+             << r.description ();
+
+      return r.code ();
+    }
+    catch (const process_error& e)
+    {
+      fail << "unable to execute " << p << ": " << e << endf;
+    }
+  }
+
+  template <typename I, typename O, typename E, typename P, typename... A>
+  void
+  run (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args)
+  {
+    if (run_exit (t,
+                  forward<I> (in),
+                  forward<O> (out),
+                  forward<E> (err),
+                  p,
+                  forward<A> (args)...) != 0)
+      fail << "process " << p << " terminated with non-zero exit code";
+  }
+
+  template <typename T>
+  T
+  parse_manifest (const path& f, const char* what, bool ignore_unknown)
+  {
+    using namespace butl;
+
+    try
+    {
+      if (!file_exists (f))
+        fail << what << " manifest file " << f << " does not exist";
+
+      ifdstream ifs (f);
+      manifest_parser p (ifs, f.string ());
+      return T (p, ignore_unknown);
+    }
+    catch (const manifest_parsing& e)
+    {
+      fail << "invalid " << what << " manifest: "
+           << f << ':' << e.line << ':' << e.column << ": " << e.description
+           << endf;
+    }
+    catch (const io_error& e)
+    {
+      fail << "unable to read " << what << " manifest " << f << ": " << e
+           << endf;
+    }
+    catch (const system_error& e) // EACCES, etc.
+    {
+      fail << "unable to access " << what << " manifest " << f << ": " << e
+           << endf;
+    }
+  }
+
+  template <typename T>
+  void
+  serialize_manifest (const T& m, const path& f, const char* what)
+  {
+    using namespace std;
+    using namespace butl;
+
+    try
+    {
+      ofdstream ofs (f, ios::binary);
+      auto_rmfile arm (f); // Try to remove on failure ignoring errors.
+
+      serialize_manifest (m, ofs, f.string (), what);
+
+      ofs.close ();
+      arm.cancel ();
+    }
+    catch (const system_error& e) // EACCES, etc.
+    {
+      fail << "unable to access " << what << " manifest " << f << ": " << e;
+    }
+  }
+
+  template <typename T>
+  void
+  serialize_manifest (const T& m,
+                      ostream& os,
+                      const string& name,
+                      const char* what)
+  {
+    using namespace std;
+    using namespace butl;
+
+    try
+    {
+      manifest_serializer s (os, name);
+      m.serialize (s);
+    }
+    catch (const manifest_serialization& e)
+    {
+      fail << "invalid " << what << " manifest: " << e.description;
+    }
+    catch (const io_error& e)
+    {
+      fail << "unable to write " << what << " manifest " << name << ": " << e;
+    }
+  }
+}
diff --git a/tests/agent/btrfs-cpdir b/tests/agent/btrfs-cpdir
new file mode 100755
index 0000000..d2dbd43
--- /dev/null
+++ b/tests/agent/btrfs-cpdir
@@ -0,0 +1,113 @@
+#! /usr/bin/env bash
+
+# Copy a directory on (the same) btrfs filesystem. Inside, subvolumes
+# are copied with btrfs subvolume snapshot and everything else with
+# cp --reflink=always.
+#
+# If the -f option is specified and <dst> exists, it is first removed by
+# calling btrfs-rmdir (which is expected to be found next to btrfs-cpdir).
+#
+# Notes:
+#
+#  1. <src> should not be a subvolume (use snapshot directly in this case).
+#
+#  2. Copying of symlinks is not supported (symlinks in submodules are ok).
+#
+#  3. Read-only subvolumes are snapshotted as read-only.
+#
+# Note also that <src> is clones as <dst>, not into <dst> (i.e., like cp -T).
+#
+usage="usage: $0 [-f] <src>/ <dst>/"
+
+owd="$(pwd)"
+trap "{ cd '$owd'; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+force=
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -f)
+      shift
+      force="true"
+      ;;
+    --)
+      break
+      ;;
+    -*)
+      error "unknown option: $1"
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+src="${1%/}"
+dst="${2%/}"
+
+if [ -z "$src" -o -z "$dst" ]; then
+  error "$usage"
+fi
+
+shopt -s nullglob dotglob
+
+function cp_dir () # <src> <dst>
+{
+  local src="$1"
+  local dst="$2"
+
+  mkdir "$dst"
+
+  local s d
+  for s in "$src"/*; do
+
+    d="$dst/${s#$src/}"
+
+    if [ -f "$s" ]; then
+      cp -p --reflink=always "$s" "$d"
+      continue
+    fi
+
+    if [ -d "$s" ]; then
+
+      # See if this is a subvolume: btrfs subvolume list requires root
+      # priviliges so we use the inode number which for subvolumes is always
+      # 256.
+      #
+      if [ "$(stat --format=%i "$s")" -eq 256 ]; then
+	cp_subvol "$s" "$d"
+      else
+	cp_dir "$s" "$d"
+      fi
+      continue
+    fi
+
+    error "$s is not a file/directory, not supported"
+  done
+
+  chmod --reference="$src" "$dst"
+  chown --reference="$src" "$dst"
+}
+
+function cp_subvol () # <src> <dst>
+{
+  local src="$1"
+  local dst="$2"
+
+  local o=()
+  if [ "$(btrfs property get -ts "$src" ro)" = "ro=true" ]; then
+    o+=(-r)
+  fi
+
+  btrfs subvolume snapshot "${o[@]}" "$src" "$dst" >/dev/null
+}
+
+if [ -d "$dst" -a -n "$force" ]; then
+  "$(dirname "$(realpath ${BASH_SOURCE[0]})")/btrfs-rmdir" "$dst"
+fi
+
+cp_dir "$src" "$dst"
diff --git a/tests/agent/btrfs-rmdir b/tests/agent/btrfs-rmdir
new file mode 100755
index 0000000..cc51dc2
--- /dev/null
+++ b/tests/agent/btrfs-rmdir
@@ -0,0 +1,66 @@
+#! /usr/bin/env bash
+
+# Remove a directory on a btrfs filesystem. Inside, subvolumes are removed
+# with btrfs subvolume delete and everything else with rm.
+#
+# Notes:
+#
+#  1. <dir> should not be a subvolume (use delete directly in this case).
+#
+#  2. Read-only subvolumes are changed to read-write before deleting.
+#
+usage="usage: $0 <dir>/"
+
+owd="$(pwd)"
+trap "{ cd '$owd'; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+dir="${1%/}"
+
+if [ -z "$dir" ]; then
+  error "$usage"
+fi
+
+shopt -s nullglob dotglob
+
+function rm_dir () # <dir>
+{
+  local dir="$1"
+
+  local p
+  for p in "$dir"/*; do
+
+    if [ -d "$p" -a ! -L "$p" ]; then
+
+      # See if this is a subvolume: btrfs subvolume list requires root
+      # priviliges so we use the inode number which for subvolumes is always
+      # 256.
+      #
+      if [ "$(stat --format=%i "$p")" -eq 256 ]; then
+	rm_subvol "$p"
+      else
+	rm_dir "$p"
+      fi
+    else
+      rm "$p"
+    fi
+  done
+
+  rmdir "$dir"
+}
+
+function rm_subvol () # <dir>
+{
+  local dir="$1"
+
+  if [ "$(btrfs property get -ts "$dir" ro)" = "ro=true" ]; then
+    btrfs property set -ts "$dir" ro false >/dev/null
+  fi
+
+  btrfs subvolume delete "$dir" >/dev/null
+}
+
+rm_dir "$dir"
diff --git a/tests/agent/buildfile b/tests/agent/buildfile
new file mode 100644
index 0000000..1c6d59b
--- /dev/null
+++ b/tests/agent/buildfile
@@ -0,0 +1,8 @@
+# file      : tests/agent/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: ../../bbot/exe{bbot-agent} test{testscript} file{btrfs-cpdir btrfs-rmdir}
+dir{./}: test = ../../bbot/exe{bbot-agent}
+
+include ../../bbot/
diff --git a/tests/agent/testscript b/tests/agent/testscript
new file mode 100644
index 0000000..fa465a4
--- /dev/null
+++ b/tests/agent/testscript
@@ -0,0 +1,112 @@
+# file      : tests/agent/testscript
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# The /build/machines directory should be on a btrfs filesystem and have the
+# following layout and contents:
+#
+# /build/machines/
+# └── default/
+#     └── linux-gcc/
+#         ├── linux-gcc-1.0/
+#         │    └── manifest
+#         └── linux-gcc-1.1/
+#               └── manifest
+#
+# Notes:
+#
+#  - The test must be run serially (@@ TODO: serial directive)
+
+test.options   = --verbose 2
+test.arguments = stage 1
+
+cp = $src_base/btrfs-cpdir -f /build/machines.orig /build/machines
+rm = $src_base/btrfs-rmdir /build/machines
+
+: dump-machines
+:
+{
+  m = /build/machines/default/linux-gcc
+
+  test.options += --dump-machines --fake-bootstrap
+
+  +$cp
+
+  : no-current-machine-symlink
+  :
+  $* 123 2>>"EOE"
+    trace: enumerate_machines: skipping $m/: no subvolume link
+    EOE
+
+  ln -T -s linux-gcc-1.0 $m/linux-gcc-1
+
+  : bootstrap
+  :
+  $* 123 >>EOO 2>>~"%EOE%d"
+    : 1
+    id: linux-gcc-1.0
+    name: linux-gcc
+    summary: Linux with GCC
+    EOO
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-stage-\\.+%
+    trace: enumerate_machines: bootstrapping $m/linux-gcc-stage/
+    EOE
+
+  ln -T -f -s linux-gcc-1.1 $m/linux-gcc-1
+
+  : re-bootstrap-machine
+  :
+  $* 123 >>EOO 2>>~"%EOE%d"
+    : 1
+    id: linux-gcc-1.1
+    name: linux-gcc
+    summary: Linux with GCC
+    EOO
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
+    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new machine
+    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
+    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+    EOE
+
+  : re-bootstrap-toolchain
+  :
+  $* 124 >>EOO 2>>~"%EOE%d"
+    : 1
+    id: linux-gcc-1.1
+    name: linux-gcc
+    summary: Linux with GCC
+    EOO
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
+    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new toolchain
+    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
+    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+    EOE
+
+  sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-stage/manifest
+
+  : re-bootstrap-bbot
+  :
+  $* 124 >>EOO 2>>~"%EOE%d"
+    : 1
+    id: linux-gcc-1.1
+    name: linux-gcc
+    summary: Linux with GCC
+    EOO
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
+    trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new bbot
+    trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false
+    trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage
+    EOE
+
+  sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-stage/manifest
+
+  : re-bootstrap-bbot-newer
+  :
+  $* 124 2>>~"%EOE%d"
+    %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+%
+    trace: enumerate_machines: ignoring $m/linux-gcc-stage/: old bbot
+    %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage-\\.+%
+    EOE
+
+  -$rm
+}