Implement vm startup and shutdown

16 files changed, 891 insertions, 87 deletions

diff --git a/bbot/agent b/bbot/agent
new file mode 100644
index 0000000..c333fe5
--- /dev/null
+++ b/bbot/agent
@@ -0,0 +1,24 @@
+// file      : bbot/agent -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BBOT_AGENT
+#define BBOT_AGENT
+
+#include <bbot/types>
+#include <bbot/utility>
+
+#include <bbot/agent-options>
+
+namespace bbot
+{
+  extern agent_options ops;
+
+  extern const string bs_prot; // Bootstrap protocol version.
+
+  extern string tc_name; // Toolchain name.
+  extern string tc_num;  // Toolchain number.
+  extern string tc_id;   // Toolchain id.
+}
+
+#endif // BBOT_AGENT
diff --git a/bbot/agent.cxx b/bbot/agent.cxx
index 76c3a86..e608af2 100644
--- a/bbot/agent.cxx
+++ b/bbot/agent.cxx
@@ -2,6 +2,8 @@
 // copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 // license   : MIT; see accompanying LICENSE file
 
+#include <bbot/agent>
+
 #include <limits.h> // PATH_MAX
 #include <signal.h> // signal()
 #include <unistd.h> // sleep(), realink()
@@ -15,12 +17,22 @@
 
 #include <bbot/types>
 #include <bbot/utility>
-
 #include <bbot/diagnostics>
-#include <bbot/agent-options>
 
+#include <bbot/machine>
 #include <bbot/bootstrap-manifest>
 
+namespace bbot
+{
+  agent_options ops;
+
+  const string bs_prot ("1"); // Bootstrap protocol version.
+
+  string tc_name; // Toolchain name.
+  string tc_num;  // Toolchain number.
+  string tc_id;   // Toolchain id.
+}
+
 using namespace std;
 using namespace butl;
 using namespace bbot;
@@ -34,9 +46,9 @@ inline void
 btrfs (tracer& t, A&&... a)
 {
   if (verb >= 3)
-    run (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...);
+    run_io (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...);
   else
-    run (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
+    run_io (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
 }
 
 template <typename... A>
@@ -44,20 +56,14 @@ inline butl::process_exit::code_type
 btrfs_exit (tracer& t, A&&... a)
 {
   return verb >= 3
-    ? run_exit (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...)
-    : run_exit (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
+    ? run_io_exit (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...)
+    : run_io_exit (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...);
 }
 
-agent_options ops;
-
-const string bs_prot ("1"); // Bootstrap protocol version.
-
-string tc_name; // Toolchain name.
-string tc_num;  // Toolchain number.
-string tc_id;   // Toolchain id.
-
 static bootstrapped_machine_manifest
-bootstrap_machine (const dir_path& md, const machine_manifest& mm)
+bootstrap_machine (const dir_path& md,
+                   const machine_manifest& mm,
+                   optional<bootstrapped_machine_manifest> obmm)
 {
   bootstrapped_machine_manifest r {
     mm,
@@ -72,21 +78,40 @@ bootstrap_machine (const dir_path& md, const machine_manifest& mm)
     }
   };
 
-  if (!ops.fake_bootstrap ())
+  if (ops.fake_bootstrap ())
   {
+    r.machine.mac = "de:ad:be:ef:de:ad";
+  }
+  else
+  {
+    unique_ptr<machine> m (
+      start_machine (md,
+                     mm,
+                     obmm ? obmm->machine.mac : nullopt));
+
+    r.machine.mac = m->mac;
+
+    sleep (10);
+
+    if (!m->shutdown ())
+    {
+      error << "forcing machine " << md << " down";
+      m->forcedown ();
+      throw failed ();
+    }
   }
 
   serialize_manifest (r, md / "manifest", "bootstrapped machine");
   return r;
 }
 
-static machine_manifests
+static machine_header_manifests
 enumerate_machines (const dir_path& rd)
 try
 {
   tracer trace ("enumerate_machines");
 
-  machine_manifests r;
+  machine_header_manifests r;
 
   // The first level are machine volumes.
   //
@@ -244,26 +269,25 @@ try
               (r = cmp ("libbutl", LIBBUTL_VERSION)) != 0 ? r : 0;
           };
 
+          optional<bootstrapped_machine_manifest> obmm;
           if (te)
           {
-            auto bmm (
-              parse_manifest<bootstrapped_machine_manifest> (
-                tp / "manifest",
-                "bootstrapped machine"));
+            obmm = parse_manifest<bootstrapped_machine_manifest> (
+              tp / "manifest", "bootstrapped machine");
 
-            if (bmm.machine.id != mm.id)
+            if (obmm->machine.id != mm.id)
             {
               l2 ([&]{trace << "re-bootstrapping " << tp << ": new machine";});
               te = false;
             }
 
-            if (bmm.toolchain.id != tc_id)
+            if (obmm->toolchain.id != tc_id)
             {
               l2 ([&]{trace << "re-bootstrapping " << tp << ": new toolchain";});
               te = false;
             }
 
-            if (int i = compare_bbot (bmm.bootstrap))
+            if (int i = compare_bbot (obmm->bootstrap))
             {
               if (i < 0)
               {
@@ -290,7 +314,8 @@ try
             // bootstrap the new machine. Then atomically rename it to
             // <name>-<toolchain>.
             //
-            bootstrapped_machine_manifest bmm (bootstrap_machine (xp, mm));
+            bootstrapped_machine_manifest bmm (
+              bootstrap_machine (xp, mm, move (obmm)));
 
             try
             {
@@ -318,10 +343,10 @@ try
 
           // Add the machine to the list.
           //
-          // In order not to forget to clear new fields, we are instead going
-          // to create a new instance with just the required fields.
-          //
-          r.push_back (machine_manifest (mm.id, mm.name, mm.summary));
+          r.push_back (
+            machine_header_manifest (move (mm.id),
+                                     move (mm.name),
+                                     move (mm.summary)));
 
           break;
         }
@@ -453,11 +478,11 @@ try
   //
   for (unsigned int s; (s = 60); sleep (s))
   {
-    machine_manifests mms (enumerate_machines (ops.machines ()));
+    machine_header_manifests mms (enumerate_machines (ops.machines ()));
 
     if (ops.dump_machines ())
     {
-      for (const machine_manifest& mm: mms)
+      for (const machine_header_manifest& mm: mms)
         serialize_manifest (mm, cout, "stdout", "machine manifest");
 
       return 0;
diff --git a/bbot/bootstrap-manifest b/bbot/bootstrap-manifest
index 6007c6e..48139ad 100644
--- a/bbot/bootstrap-manifest
+++ b/bbot/bootstrap-manifest
@@ -12,7 +12,7 @@
 #include <bbot/types>
 #include <bbot/utility>
 
-#include <bbot/manifest> // machine_manifest
+#include <bbot/machine-manifest>
 
 namespace bbot
 {
diff --git a/bbot/bootstrap-manifest.cxx b/bbot/bootstrap-manifest.cxx
index 7d635ca..231e56a 100644
--- a/bbot/bootstrap-manifest.cxx
+++ b/bbot/bootstrap-manifest.cxx
@@ -221,7 +221,10 @@ namespace bbot
     if (nv.empty ())
       bad_value ("machine manifest expected");
 
-    machine = machine_manifest (p, nv, false, iu);
+    machine = machine_manifest (p, nv, iu);
+
+    if (!machine.mac)
+      bad_name ("mac address must be present in machine manifest");
 
     nv = p.next ();
     if (nv.empty ())
@@ -251,6 +254,10 @@ namespace bbot
     s.next ("", "1"); // Start of manifest.
     s.next ("", "");  // End of manifest.
 
+    if (!machine.mac)
+      throw serialization (s.name (),
+                           "mac address must be present in machine manifest");
+
     machine.serialize (s);
     toolchain.serialize (s);
     bootstrap.serialize (s);
diff --git a/bbot/buildfile b/bbot/buildfile
index 8c1e5b4..b43bc63 100644
--- a/bbot/buildfile
+++ b/bbot/buildfile
@@ -25,9 +25,11 @@ if ($cxx.target.class == "linux")
   ./: exe{bbot-agent} service{'bbot-agent@'}
 
   exe{bbot-agent}:                                                        \
-  {            cxx}{ agent              } {hxx ixx cxx}{ agent-options  } \
+  {hxx         cxx}{ agent              } {hxx ixx cxx}{ agent-options  } \
   {hxx         cxx}{ bootstrap-manifest } {hxx ixx cxx}{ common-options } \
   {hxx         cxx}{ diagnostics        }                                 \
+  {hxx         cxx}{ machine-manifest   }                                 \
+  {hxx         cxx}{ machine            }                                 \
   {hxx            }{ types              }                                 \
   {hxx         cxx}{ types-parsers      }                                 \
   {hxx     txx cxx}{ utility            }                                 \
diff --git a/bbot/machine b/bbot/machine
new file mode 100644
index 0000000..5ab8c15
--- /dev/null
+++ b/bbot/machine
@@ -0,0 +1,50 @@
+// file      : bbot/machine -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BBOT_MACHINE
+#define BBOT_MACHINE
+
+#include <bbot/types>
+#include <bbot/utility>
+
+namespace bbot
+{
+  // A running build machine (container, vm, etc).
+  //
+  class machine
+  {
+  public:
+    // Shut the machine down cleanly. Return false if machine is still
+    // running, true if machine exited successfully, and throw failed
+    // otherwise.
+    //
+    virtual bool
+    shutdown () = 0;
+
+    // Force the machine down.
+    //
+    virtual void
+    forcedown () = 0;
+
+  public:
+    const string mac; // MAC address (inside the machine).
+
+  public:
+    virtual
+    ~machine () = default;
+
+  protected:
+    machine (string m)
+        : mac (move (m)) {}
+  };
+
+  class machine_manifest;
+
+  unique_ptr<machine>
+  start_machine (const dir_path&,
+                 const machine_manifest&,
+                 const optional<string>& mac);
+}
+
+#endif // BBOT_MACHINE
diff --git a/bbot/machine-manifest b/bbot/machine-manifest
new file mode 100644
index 0000000..0d71bd7
--- /dev/null
+++ b/bbot/machine-manifest
@@ -0,0 +1,58 @@
+// file      : bbot/machine-manifest -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BBOT_MACHINE_MANIFEST
+#define BBOT_MACHINE_MANIFEST
+
+#include <map>
+
+#include <butl/manifest-forward>
+
+#include <bbot/types>
+#include <bbot/utility>
+
+#include <bbot/manifest> // machine_header
+
+namespace bbot
+{
+  // Machine type.
+  //
+  enum class machine_type {kvm, nspawn};
+
+  string
+  to_string (machine_type);
+
+  machine_type
+  to_machine_type (const string&); // Throws invalid_argument.
+
+  // Machine.
+  //
+  class machine_manifest: public machine_header_manifest
+  {
+  public:
+    machine_type type;
+    optional<string> mac; // Required in bootstrapped machine manifest.
+
+    machine_manifest (std::string i,
+                      std::string n,
+                      std::string s,
+                      machine_type t)
+        : machine_header_manifest (std::move (i),
+                                   std::move (n),
+                                   std::move (s)),
+          type (t) {}
+
+  public:
+    machine_manifest () = default; // VC export.
+    machine_manifest (butl::manifest_parser&, bool ignore_unknown = false);
+    machine_manifest (butl::manifest_parser&,
+                      butl::manifest_name_value start,
+                      bool ignore_unknown = false);
+
+    void
+    serialize (butl::manifest_serializer&) const;
+  };
+}
+
+#endif // BBOT_MACHINE_MANIFEST
diff --git a/bbot/machine-manifest.cxx b/bbot/machine-manifest.cxx
new file mode 100644
index 0000000..9145ed4
--- /dev/null
+++ b/bbot/machine-manifest.cxx
@@ -0,0 +1,175 @@
+// file      : bbot/machine-manifest.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <bbot/machine-manifest>
+
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+using namespace butl;
+
+namespace bbot
+{
+  using parser = manifest_parser;
+  using parsing = manifest_parsing;
+  using serializer = manifest_serializer;
+  using serialization = manifest_serialization;
+  using name_value = manifest_name_value;
+
+  // machine_type
+  //
+  string
+  to_string (machine_type t)
+  {
+    switch (t)
+    {
+    case machine_type::kvm:    return "kvm";
+    case machine_type::nspawn: return "nspawn";
+    }
+
+    assert (false);
+    return string ();
+  }
+
+  machine_type
+  to_machine_type (const string& t)
+  {
+         if (t == "kvm")    return machine_type::kvm;
+    else if (t == "nspawn") return machine_type::nspawn;
+    else throw invalid_argument ("invalid machine type '" + t + "'");
+  }
+
+  // machine_manifest
+  //
+  machine_manifest::
+  machine_manifest (parser& p, bool iu)
+      : machine_manifest (p, p.next (), iu)
+  {
+    // Make sure this is the end.
+    //
+    name_value nv (p.next ());
+    if (!nv.empty ())
+      throw parsing (p.name (), nv.name_line, nv.name_column,
+                     "single machine manifest expected");
+  }
+
+  machine_manifest::
+  machine_manifest (parser& p, name_value nv, bool iu)
+  {
+    auto bad_name = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.name_line, nv.name_column, d);
+    };
+
+    auto bad_value = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.value_line, nv.value_column, d);
+    };
+
+    // Make sure this is the start and we support the version.
+    //
+    if (!nv.name.empty ())
+      bad_name ("start of machine manifest expected");
+
+    if (nv.value != "1")
+      bad_value ("unsupported format version");
+
+    optional<machine_type> type;
+
+    for (nv = p.next (); !nv.empty (); nv = p.next ())
+    {
+      string& n (nv.name);
+      string& v (nv.value);
+
+      if (n == "id")
+      {
+        if (!id.empty ())
+          bad_name ("machine id redefinition");
+
+        if (v.empty ())
+          bad_value ("empty machine id");
+
+        id = move (v);
+      }
+      else if (n == "name")
+      {
+        if (!name.empty ())
+          bad_name ("machine name redefinition");
+
+        if (v.empty ())
+          bad_value ("empty machine name");
+
+        name = move (v);
+      }
+      else if (n == "summary")
+      {
+        if (!summary.empty ())
+          bad_name ("machine summary redefinition");
+
+        if (v.empty ())
+          bad_value ("empty machine summary");
+
+        summary = move (v);
+      }
+      else if (n == "type")
+      {
+        if (type)
+          bad_name ("machine type redefinition");
+
+        try
+        {
+          type = to_machine_type (v);
+        }
+        catch (const invalid_argument&)
+        {
+          bad_value ("invalid machine type");
+        }
+      }
+      else if (n == "mac")
+      {
+        if (mac)
+          bad_name ("machine mac redefinition");
+
+        mac = move (v);
+      }
+      else if (!iu)
+        bad_name ("unknown name '" + n + "' in machine manifest");
+    }
+
+    // Verify all non-optional values were specified.
+    //
+    if (id.empty ())
+      bad_value ("no machine id specified");
+
+    if (name.empty ())
+      bad_value ("no machine name specified");
+
+    if (summary.empty ())
+      bad_value ("no machine summary specified");
+
+    if (!type)
+      bad_value ("no machine type specified");
+
+    this->type = *type;
+  }
+
+  void machine_manifest::
+  serialize (serializer& s) const
+  {
+    // @@ Should we check that all non-optional values are specified and all
+    //    values are valid?
+    //
+    s.next ("", "1"); // Start of manifest.
+    s.next ("id", id);
+    s.next ("name", name);
+    s.next ("summary", summary);
+    s.next ("type", to_string (type));
+
+    if (mac)
+      s.next ("mac", *mac);
+
+    s.next ("", ""); // End of manifest.
+  }
+
+}
diff --git a/bbot/machine.cxx b/bbot/machine.cxx
new file mode 100644
index 0000000..bebcc3f
--- /dev/null
+++ b/bbot/machine.cxx
@@ -0,0 +1,294 @@
+// file      : bbot/machine.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <bbot/machine>
+
+#include <sys/un.h>   // sockaddr_un
+#include <sys/socket.h>
+
+#include <unistd.h>    // getuid()
+#include <sys/types.h> // getuid()
+
+#include <cstdio>  // snprintf()
+#include <cstring> // strcpy()
+
+#include <bbot/agent>
+#include <bbot/machine-manifest>
+
+using namespace std;
+using namespace butl;
+
+namespace bbot
+{
+  static string
+  create_tap ()
+  {
+    tracer trace ("create_tap");
+
+    string b ("br1");          // Use private bridge for now.
+    string t ("tap" + tc_num);
+
+    auto uid (getuid ());
+
+    // First try to delete it in case there is one from a previous run.
+    //
+    run_exit (trace, "sudo", "ip", "tuntap", "delete", t, "mode", "tap");
+
+    run (trace, "sudo", "ip", "tuntap", "add", t, "mode", "tap", "user", uid);
+    run (trace, "sudo", "ip", "link", "set", t, "up");
+    //sleep (1);
+    run (trace, "sudo", "ip", "link", "set", t, "master", b);
+
+    return t;
+  }
+
+  static void
+  destroy_tap (const string& t)
+  {
+    tracer trace ("create_tap");
+    run (trace, "sudo", "ip", "tuntap", "delete", t, "mode", "tap");
+  }
+
+  static string
+  generate_mac ()
+  {
+    // The last two bits of the first byte are special: bit 1 indicates a
+    // multicast address (which we don't want) while bit 1 -- local assignment
+    // (which we do want).
+    //
+    char r[6 * 2 + 5 + 1];
+    snprintf (r, sizeof (r),
+              "%02x:%02x:%02x:%02x:%02x:%02x",
+              (genrand<uint8_t> () & 0xFE) | 0x02,
+              genrand<uint8_t> (),
+              genrand<uint8_t> (),
+              genrand<uint8_t> (),
+              genrand<uint8_t> (),
+              genrand<uint8_t> ());
+    return r;
+  }
+
+  class kvm_machine: public machine
+  {
+  public:
+    kvm_machine (const dir_path&,
+                 const machine_manifest&,
+                 const optional<string>& mac);
+
+    virtual bool
+    shutdown () override;
+
+    virtual void
+    forcedown () override;
+
+  private:
+    bool
+    wait (size_t seconds);
+
+    void
+    monitor_command (const string&);
+
+  private:
+    path kvm;     // Hypervisor binary.
+    string tap;   // Tap network interface.
+    path monitor; // QEMU monitor UNIX socket.
+    process proc;
+  };
+
+  kvm_machine::
+  kvm_machine (const dir_path& md,
+               const machine_manifest& mm,
+               const optional<string>& omac)
+      : machine (mm.mac ? *mm.mac : // Fixed mac from machine manifest.
+                 omac   ? *omac   : // Generated mac from previous bootstrap.
+                 generate_mac ()),
+        kvm ("kvm"),
+        tap (create_tap ()),
+        monitor ("/tmp/" + tc_name + "-monitor")
+  {
+    tracer trace ("kvm_machine");
+
+    if (sizeof (sockaddr_un::sun_path) <= monitor.size ())
+      throw invalid_argument ("monitor unix socket path too long");
+
+    // Start the VM.
+    //
+    // Notes:
+    //
+    // 1. For now we let qemu calculate sockets/cores/threads from the
+    //    total number of CPUs (i.e., threads).
+    //
+    // 2. echo system_powerdown | socat - UNIX-CONNECT:.../monitor
+    //
+    proc = run_io_start (
+      trace,
+      fdnull (),
+      2,
+      2,
+      md, // Run from the machine's directory.
+      kvm,
+      "-boot", "c", // Boot from disk.
+      "-no-reboot", // Exit on VM reboot.
+      //
+      // Machine.
+      //
+      "-m",   to_string (ops.ram () / 1024) + "M",
+      "-cpu", "host",
+      "-smp", ops.cpu (),
+      //
+      // Network.
+      //
+      "-device", "virtio-net-pci,netdev=net0,mac=" + mac,
+      "-netdev", "tap,id=net0,script=no,ifname=" + tap,
+      //
+      // Disk.
+      //
+      "-device", "virtio-scsi-pci,id=scsi",
+      "-device", "scsi-hd,drive=disk0",
+      "-drive",  "if=none,id=disk0,format=raw,file=disk.img",
+      //
+      // VNC & monitor.
+      //
+      "-vnc", "localhost:" + tc_num, // 5900 + tc_num
+      "-monitor", "unix:" + monitor.string () + ",server,nowait");
+  }
+
+  // Connect to the QEMU monitor via the UNIX socket and send system_reset.
+  // You may be wondering why not system_powerdown? The reason is that while
+  // not all OS know how to power-down the machine, pretty much all of them
+  // can reboot. So combined with the -no-reboot option above, we get the
+  // same result in a more robust way.
+  //
+  // Note that this setup has one side effect: if the VM decided to reboot,
+  // say, during bootstrap, then we will interpret it as a shutdown. Current
+  // thinking saying this is good since we don't want our VMs to reboot
+  // uncontrollably for security and predictability reasons (e.g., we don't
+  // want Windows to decide to install updates -- this stuff should all be
+  // disabled during the VM preparation).
+  //
+  // Actually, this turned out not to be entirely accurate: reset appears to
+  // be a "hard reset" while powerdown causes a clean shutdown. So we use
+  // powerdown to implement shutdown() and reset/-no-reboot for implement
+  // forcedown().
+  //
+  bool kvm_machine::
+  shutdown ()
+  {
+    monitor_command ("system_powerdown");
+
+    // Wait for up to 10 seconds for the machine to shutdown.
+    //
+    return wait (10);
+  }
+
+  void kvm_machine::
+  forcedown ()
+  {
+    monitor_command ("system_reset");
+    wait (size_t (~0)); // Wait indefinitely.
+  }
+
+  bool kvm_machine::
+  wait (size_t sec)
+  try
+  {
+    tracer trace ("kvm_machine::wait");
+
+    bool t;
+    for (size_t i (0); !(t = proc.try_wait ()) && i != sec; ++i)
+      sleep (1);
+
+    if (t)
+    {
+      run_io_finish (trace, proc, kvm);
+
+      destroy_tap (tap);
+      try_rmfile (monitor, true); // QEMU doesn't seem to remove it.
+    }
+
+    return t;
+  }
+  catch (const process_error& e)
+  {
+    fail << "unable to execute " << kvm << ": " << e << endf;
+  }
+
+  void kvm_machine::
+  monitor_command (const string& c)
+  try
+  {
+    sockaddr_un addr;
+    addr.sun_family = AF_LOCAL;
+    strcpy (addr.sun_path, monitor.string ().c_str ()); // Size check in ctor
+
+    auto_fd sock (socket (AF_LOCAL, SOCK_STREAM, 0));
+
+    if (sock.get () == -1)
+      throw_system_error (errno);
+
+    if (connect (sock.get (),
+                 reinterpret_cast<sockaddr*> (&addr),
+                 sizeof (addr)) == -1)
+      throw_system_error (errno);
+
+    // Read until we get something.
+    //
+    auto readsome = [&sock] ()
+    {
+      ifdstream ifs (move (sock),
+                     fdstream_mode::non_blocking,
+                     ostream::badbit);
+
+      char buf[256];
+      for (streamsize n (0), m (0);
+           n == 0 || m != 0;
+           m = ifs.readsome (buf, sizeof (buf) - 1))
+      {
+        if (m != 0)
+        {
+          n += m;
+
+          //buf[m] = '\0';
+          //text << buf;
+        }
+      }
+
+      sock = move (ifs.release ());
+    };
+
+    // Read QEMU welcome.
+    //
+    readsome ();
+
+    // Write our command.
+    //
+    {
+      ofdstream ofs (move (sock), fdstream_mode::blocking);
+      ofs << c << endl;
+      sock = move (ofs.release ());
+    }
+
+    // Read QEMU reply (may hit eof).
+    //
+    readsome ();
+  }
+  catch (const system_error& e)
+  {
+    fail << "unable to communicate with qemu monitor: " << e;
+  }
+
+  unique_ptr<machine>
+  start_machine (const dir_path& md,
+                 const machine_manifest& mm,
+                 const optional<string>& mac)
+  {
+    switch (mm.type)
+    {
+    case machine_type::kvm: return make_unique<kvm_machine> (md, mm, mac);
+    case machine_type::nspawn: assert (false); //@@ TODO
+    }
+
+    return nullptr;
+  }
+}
diff --git a/bbot/utility b/bbot/utility
index ebab971..5bd4821 100644
--- a/bbot/utility
+++ b/bbot/utility
@@ -40,32 +40,67 @@ namespace bbot
   using butl::exception_guard;
   using butl::make_exception_guard;
 
+  // Random number generator (currently not MT-safe and limited to RAND_MAX).
+  //
+  size_t
+  genrand ();
+
+  template <typename T>
+  inline T
+  genrand () {return static_cast<T> (genrand ());}
+
   // Process execution.
   //
   class tracer;
 
+  using butl::process;
+  using butl::process_exit;
+  using butl::process_error;
+
   template <typename I, typename O, typename E, typename P, typename... A>
   void
-  run (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+  run_io (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
 
   template <typename I, typename O, typename E, typename P, typename... A>
-  butl::process_exit::code_type
-  run_exit (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+  process_exit::code_type
+  run_io_exit (tracer&, I&& in, O&& out, E&& err, P&&, A&&...);
+
+  template <typename I, typename O, typename E, typename P, typename... A>
+  process
+  run_io_start (tracer&,
+                I&& in,
+                O&& out,
+                E&& err,
+                const dir_path& cwd,
+                P&&,
+                A&&...);
+
+  template <typename P>
+  void
+  run_io_finish (tracer&, process&, P&&);
+
+  template <typename P>
+  process_exit::code_type
+  run_io_finish_exit (tracer&, process&, P&&);
 
   template <typename P, typename... A>
   inline void
   run (tracer& t, P&& p, A&&... a)
   {
-    run (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
+    run_io (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
   }
 
   template <typename P, typename... A>
-  inline butl::process_exit::code_type
+  inline process_exit::code_type
   run_exit (tracer& t, P&& p, A&&... a)
   {
-    return run (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
+    return run_io_exit (
+      t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...);
   }
 
+  void
+  run_trace (tracer&, const char*[], size_t);
+
   // Manifest parsing and serialization.
   //
   template <typename T>
diff --git a/bbot/utility.cxx b/bbot/utility.cxx
index 591ff52..95c31e9 100644
--- a/bbot/utility.cxx
+++ b/bbot/utility.cxx
@@ -4,6 +4,9 @@
 
 #include <bbot/utility>
 
+#include <chrono>
+#include <cstdlib> // rand_r()
+
 #include <bbot/diagnostics>
 
 using namespace std;
@@ -11,4 +14,25 @@ using namespace butl;
 
 namespace bbot
 {
+  static unsigned int rand_seed; // Seed for rand_r();
+
+  size_t
+  genrand ()
+  {
+    if (rand_seed == 0)
+      rand_seed = static_cast<unsigned int> (
+        chrono::system_clock::now ().time_since_epoch ().count ());
+
+    return static_cast<size_t> (rand_r (&rand_seed));
+  }
+
+  void
+  run_trace (tracer& t, const char* cmd[], size_t n)
+  {
+    if (verb >= 2)
+    {
+      diag_record dr (t);
+      process::print (dr.os, cmd, n);
+    }
+  }
 }
diff --git a/bbot/utility.txx b/bbot/utility.txx
index d641612..383db74 100644
--- a/bbot/utility.txx
+++ b/bbot/utility.txx
@@ -12,35 +12,45 @@
 namespace bbot
 {
   template <typename I, typename O, typename E, typename P, typename... A>
-  butl::process_exit::code_type
-  run_exit (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args)
+  process
+  run_io_start (tracer& t,
+                I&& in,
+                O&& out,
+                E&& err,
+                const dir_path& cwd,
+                P&& p,
+                A&&... args)
   {
-    using namespace butl;
+    try
+    {
+      return butl::process_start (
+        [&t] (const char* c[], size_t n) {run_trace (t, c, n);},
+        forward<I> (in),
+        forward<O> (out),
+        forward<E> (err),
+        cwd,
+        p,
+        forward<A> (args)...);
+    }
+    catch (const process_error& e)
+    {
+      fail << "unable to execute " << p << ": " << e << endf;
+    }
+  }
 
+  template <typename P>
+  process_exit::code_type
+  run_io_finish_exit (tracer&, process& pr, P&& p)
+  {
     try
     {
-      auto cmdc = [&t] (const char* cmd[], size_t n)
-      {
-        if (verb >= 2)
-        {
-          diag_record dr (t);
-          process::print (dr.os, cmd, n);
-        }
-      };
-
-      process_exit r (process_run (cmdc,
-                                   forward<I> (in),
-                                   forward<O> (out),
-                                   forward<E> (err),
-                                   dir_path (),
-                                   p,
-                                   forward<A> (args)...));
-
-      if (!r.normal ())
+      pr.wait ();
+
+      if (!pr.exit->normal ())
         fail << "process " << p << " terminated abnormally: "
-             << r.description ();
+             << pr.exit->description ();
 
-      return r.code ();
+      return pr.exit->code ();
     }
     catch (const process_error& e)
     {
@@ -48,19 +58,43 @@ namespace bbot
     }
   }
 
-  template <typename I, typename O, typename E, typename P, typename... A>
-  void
-  run (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args)
+  template <typename P>
+  inline void
+  run_io_finish (tracer& t, process& pr, P&& p)
   {
-    if (run_exit (t,
-                  forward<I> (in),
-                  forward<O> (out),
-                  forward<E> (err),
-                  p,
-                  forward<A> (args)...) != 0)
+    if (run_io_finish_exit (t, pr, p) != 0)
       fail << "process " << p << " terminated with non-zero exit code";
   }
 
+  template <typename I, typename O, typename E, typename P, typename... A>
+  inline process_exit::code_type
+  run_io_exit (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args)
+  {
+    process pr (run_io_start (t,
+                              forward<I> (in),
+                              forward<O> (out),
+                              forward<E> (err),
+                              dir_path (),
+                              p,
+                              forward<A> (args)...));
+
+    return run_io_finish_exit (t, pr, p);
+  }
+
+  template <typename I, typename O, typename E, typename P, typename... A>
+  inline void
+  run_io (tracer& t, I&& in, O&& out, E&& err, const P& p, A&&... args)
+  {
+    process pr (run_io_start (t,
+                              forward<I> (in),
+                              forward<O> (out),
+                              forward<E> (err),
+                              dir_path (),
+                              p,
+                              forward<A> (args)...));
+    run_io_finish (t, pr, p);
+  }
+
   template <typename T>
   T
   parse_manifest (const path& f, const char* what, bool ignore_unknown)
diff --git a/doc/manual.cli b/doc/manual.cli
index d5ab992..3617ca0 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -155,20 +155,19 @@ windows-vc_14-32-debug
 linux-gcc_6-cross-arm-eabi
 \
 
-\h#arch-machine-manifest|Machine Manifest|
+\h#arch-machine-header-manifest|Machine Header Manifest|
 
 \
 SYNOPSIS
 
 id:      <machine-id>
 name:    <machine-name>
-type:    <machine-type>
 summary: <string>
 \
 
-The build machine manifest describes the build machine on the build host (see
-the Build OS documentation for their origin and location). A list of machine
-manifests is also sent by \c{bbot} agents to controllers.
+The build machine header manifest contains basic information about a build
+machine on the build host. A list of machine header manifests is sent by
+\c{bbot} agents to controllers.
 
 \dl|
 
@@ -183,23 +182,59 @@ manifests is also sent by \c{bbot} agents to controllers.
 
   The machine name as described above.|
 
-\li|\n\c{type: <machine-type>}\n
-
-  The machine type. Valid values are \c{kvm} (QEMU/KVM virtual machine) and
-  \c{nspawn} (\c{systemd-nspawn} container). Note that this value is not sent
-  by agents to controllers.|
-
 \li|\n\c{summary: <string>}\n
 
   A one-line description of the machine. For example:
 
   \
+  id: windows_10-msvc_14-1.3
   name: windows_10-msvc_14
   summary: Windows 10 build 1607 with VC 14 update 3
   \
 
 ||
 
+\h#arch-machine-manifest|Machine Manifest|
+
+\
+SYNOPSIS
+
+id:      <machine-id>
+name:    <machine-name>
+summary: <string>
+
+type: <machine-type>
+mac:  <macaddr>
+\
+
+The build machine manifest contains the complete description of a build
+machine on the build host (see the Build OS documentation for their origin and
+location). The machine manifest starts with the machine manifest header. All
+the header values must appear before any non-header values.
+
+\dl|
+
+\li|\n\c{type: <machine-type>}\n
+
+  The machine type. Valid values are \c{kvm} (QEMU/KVM virtual machine) and
+  \c{nspawn} (\c{systemd-nspawn} container).|
+
+\li|\n\c{mac: <macaddr>}\n
+
+  Optional fixed MAC address for the machine in the hexadecimal,
+  comma-separated format. For example:
+
+  \
+  mac: de:ad:be:ef:de:ad
+  \
+
+  If it is not specified, then a random address is generated on the first
+  machine bootstrap which is then reused for each build/re-bootstrap. Note
+  that it you specify a fixed address, then the machine can only be used by a
+  single \c{bbot} agent.
+
+||
+
 
 \h#arch-task-manifest|Task Manifest|
 
diff --git a/tests/agent/testscript b/tests/agent/testscript
index fa465a4..55f9d7f 100644
--- a/tests/agent/testscript
+++ b/tests/agent/testscript
@@ -110,3 +110,29 @@ rm = $src_base/btrfs-rmdir /build/machines
 
   -$rm
 }
+
+#\
+: bootstrap
+:
+{
+  m = /build/machines/default/linux-gcc
+
+  test.options += --dump-machines
+
+  +$cp
+
+  ln -T -s linux-gcc-1.0 $m/linux-gcc-1
+
+  : bootstrap
+  :
+  $* 123 >>EOO 2>>EOE #2>>~"%EOE%d"
+    : 1
+    id: linux-gcc-1.0
+    name: linux-gcc
+    summary: Linux with GCC
+    EOO
+    EOE
+
+  -$rm
+}
+#\
diff --git a/unit-tests/bootstrap-manifest/buildfile b/unit-tests/bootstrap-manifest/buildfile
index aa598ee..ff85794 100644
--- a/unit-tests/bootstrap-manifest/buildfile
+++ b/unit-tests/bootstrap-manifest/buildfile
@@ -5,7 +5,7 @@
 import libs = libbutl%lib{butl}
 import libs += libbbot%lib{bbot}
 
-exe{driver}: cxx{driver} ../../bbot/cxx{bootstrap-manifest} $libs \
+exe{driver}: cxx{driver} ../../bbot/cxx{*-manifest} $libs \
              test{testscript}
 
 include ../../bbot/
diff --git a/unit-tests/bootstrap-manifest/testscript b/unit-tests/bootstrap-manifest/testscript
index 2a104be..74b4992 100644
--- a/unit-tests/bootstrap-manifest/testscript
+++ b/unit-tests/bootstrap-manifest/testscript
@@ -84,6 +84,7 @@
   name: windows_10-msvc_14
   summary: Windows 10 build 1607 with VC 14 update 3
   type: kvm
+  mac: de:ad:be:ef:de:ad
   :
   id: a2b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
   :
@@ -104,26 +105,40 @@
   : 1
   EOI
 
+  : no-machine-mac
+  :
+  $* <<EOI 2>'stdin:2:1: error: mac address must be present in machine manifest' == 1
+  : 1
+  :
+  id: a2b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  name: windows_10-msvc_14
+  summary: Windows 10 build 1607 with VC 14 update 3
+  type: kvm
+
+  EOI
+
   : no-toolchain
   :
-  $* <<EOI 2>'stdin:7:1: error: toolchain manifest expected' == 1
+  $* <<EOI 2>'stdin:8:1: error: toolchain manifest expected' == 1
   : 1
   :
   id: a2b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
   name: windows_10-msvc_14
   summary: Windows 10 build 1607 with VC 14 update 3
   type: kvm
+  mac: de:ad:be:ef:de:ad
   EOI
 
   : no-bootstrap
   :
-  $* <<EOI 2>'stdin:9:1: error: bootstrap manifest expected' == 1
+  $* <<EOI 2>'stdin:10:1: error: bootstrap manifest expected' == 1
   : 1
   :
   id: a2b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
   name: windows_10-msvc_14
   summary: Windows 10 build 1607 with VC 14 update 3
   type: kvm
+  mac: de:ad:be:ef:de:ad
   :
   id: a2b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
   EOI