path: root/bbot/worker/worker.cxx

// file      : bbot/worker.cxx -*- C++ -*-
// license   : MIT; see accompanying LICENSE file

#ifndef _WIN32
#  include <signal.h> // signal()
#else
#  include <libbutl/win32-utility.hxx> // SetErrorMode(), Sleep()
#endif

#include <map>
#include <regex>
#include <cstring>   // strchr(), strncmp()
#include <sstream>
#include <iostream>

#include <libbutl/b.hxx>
#include <libbutl/pager.hxx>
#include <libbutl/prompt.hxx>
#include <libbutl/utility.hxx>              // to_utf8(), eof()
#include <libbutl/timestamp.hxx>
#include <libbutl/filesystem.hxx>
#include <libbutl/string-parser.hxx>
#include <libbutl/manifest-serializer.hxx>

#include <libbutl/json/parser.hxx>

#include <libbbot/manifest.hxx>

#include <bbot/types.hxx>
#include <bbot/utility.hxx>

#include <bbot/diagnostics.hxx>
#include <bbot/bootstrap-manifest.hxx>

#include <bbot/worker/worker-options.hxx>

using namespace butl;
using namespace bbot;

using std::cout;
using std::endl;

namespace bbot
{
  int
  main (int argc, char* argv[]);

  static int
  build (size_t argc, const char* argv[]);

  process_path argv0;
  worker_options ops;

  dir_path env_dir;

  // Note that upload can be quite large and take a while to upload under high
  // load.
  //
  const size_t tftp_blksize     (1468); // Between 512 (default) and 65464.
  const size_t tftp_put_timeout (3600); // 1 hour (also the default).
  const size_t tftp_get_timeout (10);   // 10 seconds.
  const size_t tftp_get_retries (3);    // Task request retries (see startup()).
}

static bool
exists (const dir_path& d)
try
{
  return dir_exists (d);
}
catch (const system_error& e)
{
  fail << "unable to stat path " << d << ": " << e << endf;
}

static bool
exists (const path& f)
try
{
  return file_exists (f, true /* follow_symlinks */);
}
catch (const system_error& e)
{
  fail << "unable to stat path " << f << ": " << e << endf;
}

static dir_path
current_directory ()
try
{
  return dir_path::current_directory ();
}
catch (const system_error& e)
{
  fail << "unable to obtain current directory: " << e << endf;
}

static void
mk_p (tracer& t, string* log, const dir_path& d)
try
{
  if (verb >= 3)
    t << "mkdir -p " << d;

  if (log != nullptr)
    *log += "mkdir -p " + d.representation () + '\n';

  try_mkdir_p (d);
}
catch (const system_error& e)
{
  fail << "unable to create directory " << d << ": " << e << endf;
}

static void
mk (tracer& t, string* log, const dir_path& d)
try
{
  if (verb >= 3)
    t << "mkdir " << d;

  if (log != nullptr)
    *log += "mkdir " + d.representation () + '\n';

  try_mkdir (d);
}
catch (const system_error& e)
{
  fail << "unable to create directory " << d << ": " << e << endf;
}

static bool
empty (const dir_path& d)
try
{
  return dir_empty (d);
}
catch (const system_error& e)
{
  fail << "unable to scan directory " << d << ": " << e << endf;
}

static void
cp_into (tracer& t, string* log, const path& p, const dir_path& d)
try
{
  if (verb >= 3)
    t << "cp " << p << ' ' << d;

  if (log != nullptr)
    *log += "cp " + p.string () + ' ' + d.representation () + '\n';

  cpfile_into (p, d);
}
catch (const system_error& e)
{
  fail << "unable to copy file " << p << " into " << d <<  ": " << e << endf;
}

static void
mv (tracer& t, string* log, const dir_path& from, const dir_path& to)
try
{
  if (verb >= 3)
    t << "mv " << from << ' ' << to;

  if (log != nullptr)
    *log += "mv " + from.representation () + ' ' + to.representation () +
            "\n";

  mvdir (from, to);
}
catch (const system_error& e)
{
  fail << "unable to move directory '" << from << "' to '" << to << "': " << e
       << endf;
}

static void
mv_into (tracer& t, string* log, const path& from, const dir_path& into)
try
{
  if (verb >= 3)
    t << "mv " << from << ' ' << into;

  if (log != nullptr)
    *log += "mv " + from.representation () + ' ' + into.representation () +
            "\n";

  mventry_into (from, into);
}
catch (const system_error& e)
{
  fail << "unable to move entry '" << from << "' into '" << into << "': " << e
       << endf;
}

static void
rm_r (tracer& t, string* log, const dir_path& d)
try
{
  if (verb >= 3)
    t << "rm -r " << d;

  if (log != nullptr)
    *log += "rm -r " + d.representation () + '\n';

  rmdir_r (d);
}
catch (const system_error& e)
{
  fail << "unable to remove directory " << d << ": " << e << endf;
}

static dir_path
change_wd (tracer& t, string* log, const dir_path& d, bool create = false)
try
{
  if (create)
    mk_p (t, log, d);

  dir_path r (current_directory ());

  if (verb >= 3)
    t << "cd " << d;

  if (log != nullptr)
    *log += "cd " + d.representation () + '\n';

  dir_path::current_directory (d);
  return r;
}
catch (const system_error& e)
{
  fail << "unable to change current directory to " << d << ": " << e << endf;
}

// Step IDs.
//
// NOTE: keep ids ordered according to the sequence of steps and remember to
//       update unreachable breakpoint checks if changing anything here.
//
enum class step_id
{
  // Common fallbacks for bpkg_*_create/b_test_installed_create and
  // bpkg_*_configure_build/b_test_installed_configure, respectively. Note:
  // not breakpoints.
  //
  b_create,
  b_configure,

  // Note that bpkg_module_* options are only used if the main package is a
  // build system module (using just ~build2 otherwise). They also have no
  // fallback (build system modules are just too different to try to handle
  // them together with target and host; e.g., install root). However,
  // bpkg_module_create is complemented with arguments from un-prefixed step
  // ids, the same way as other *.create[_for_*] steps (note that un-prefixed
  // steps are not fallbacks, they are always added first).
  //
  bpkg_create,         // Breakpoint and base.
  bpkg_target_create,  //: b_create, bpkg_create
  bpkg_host_create,    //: b_create, bpkg_create
  bpkg_module_create,  //: no fallback

  bpkg_link,

  bpkg_configure_add,
  bpkg_configure_fetch,

  // Global (as opposed to package-specific) bpkg-pkg-build options (applies
  // to all *_configure_build* steps). Note: not a breakpoint.
  //
  bpkg_global_configure_build,

  // Note that bpkg_configure_build serves as a breakpoint for the
  // bpkg-pkg-build call that configures (at once) the main package and all
  // its external tests.
  //
  bpkg_configure_build,         // Breakpoint and base.
  bpkg_target_configure_build,  //: b_configure, bpkg_configure_build
  bpkg_host_configure_build,    //: b_configure, bpkg_configure_build
  bpkg_module_configure_build,  //: b_configure, bpkg_configure_build

  bpkg_update,
  bpkg_test,

  // Note that separate test packages are configured as part of the
  // bpkg_configure_build step above with options taken from
  // bpkg_{target,host}_configure_build, depending on tests package type.
  //
  bpkg_test_separate_update, //: bpkg_update
  bpkg_test_separate_test,   //: bpkg_test

  // Note that we only perform the installation tests if this is a target
  // package or a self-hosted configuration. Also note that this step is
  // considered disabled if any of the bpkg_bindist_* steps is explicitly
  // enabled.
  //
  bpkg_install,

  bbot_install_ldconfig, // Note: disabled by default.

  // Note that the bpkg_bindist_* steps are mutually exclusive and the latest
  // status change for them (via the leading +/- characters in the prefix)
  // overrides all the previous ones. Disabled by default.
  //
  bpkg_bindist_debian,
  bpkg_bindist_fedora,
  bpkg_bindist_archive,

  // Note that this step is considered disabled unless one of the
  // bpkg_bindist_* steps is explicitly enabled. Note: not a breakpoint.
  //
  bbot_sys_install,

  bbot_sys_install_apt_get_update,
  bbot_sys_install_apt_get_install,
  bbot_sys_install_dnf_install,
  bbot_sys_install_tar_extract,

  bbot_sys_install_ldconfig, // Note: disabled by default.

  // Note: skipped for modules.
  //
  b_test_installed_create,    //: b_create
  b_test_installed_configure, //: b_configure
  b_test_installed_test,

  // Note that for a host package this can involve both run-time and build-
  // time tests (which means we may also need a shared configuration for
  // modules).
  //
  // The *_for_{target,host,module} denote main package type, not
  // configuration being created, which will always be target (more precisely,
  // target or host, but host only in a self-hosted case, which means it's
  // the same as target).
  //
  // Note that if this is a non-self-hosted configuration, we can only end up
  // here if building target package and so can just use *_create and *_build
  // values in buildtabs.
  //
  bpkg_test_separate_installed_create,            // Breakpoint and base.
  bpkg_test_separate_installed_create_for_target, //: bpkg_test_separate_installed_create
  bpkg_test_separate_installed_create_for_host,   //: bpkg_test_separate_installed_create
  bpkg_test_separate_installed_create_for_module, //: no fallback

  bpkg_test_separate_installed_link,            //  breakpoint only
  bpkg_test_separate_installed_configure_add,   //: bpkg_configure_add
  bpkg_test_separate_installed_configure_fetch, //: bpkg_configure_fetch

  bpkg_test_separate_installed_configure_build,            // Breakpoint and base.
  bpkg_test_separate_installed_configure_build_for_target, //: bpkg_test_separate_installed_configure_build
  bpkg_test_separate_installed_configure_build_for_host,   //: bpkg_test_separate_installed_configure_build
  bpkg_test_separate_installed_configure_build_for_module, //: bpkg_test_separate_installed_configure_build

  bpkg_test_separate_installed_update,          //: bpkg_update
  bpkg_test_separate_installed_test,            //: bpkg_test

  bbot_sys_uninstall_apt_get_remove,
  bbot_sys_uninstall_dnf_remove,

  bpkg_uninstall,

  bbot_bindist_upload, // Note: disabled by default, not a breakpoint.

  // Note that this step is considered disabled unless the upload/ directory
  // is not empty. Note: not a breakpoint.
  //
  bbot_upload,

  bbot_upload_tar_create,
  bbot_upload_tar_list,

  end
};

static const strings step_id_str {
  "b.create",
  "b.configure",

  "bpkg.create",
  "bpkg.target.create",
  "bpkg.host.create",
  "bpkg.module.create",

  "bpkg.link",

  "bpkg.configure.add",
  "bpkg.configure.fetch",

  "bpkg.global.configure.build",

  "bpkg.configure.build",
  "bpkg.target.configure.build",
  "bpkg.host.configure.build",
  "bpkg.module.configure.build",

  "bpkg.update",
  "bpkg.test",

  "bpkg.test-separate.update",
  "bpkg.test-separate.test",

  "bpkg.install",
  "bbot.install.ldconfig",

  "bpkg.bindist.debian",
  "bpkg.bindist.fedora",
  "bpkg.bindist.archive",

  "bbot.sys-install",
  "bbot.sys-install.apt-get.update",
  "bbot.sys-install.apt-get.install",
  "bbot.sys-install.dnf.install",
  "bbot.sys-install.tar.extract",
  "bbot.sys-install.ldconfig",

  "b.test-installed.create",
  "b.test-installed.configure",
  "b.test-installed.test",

  "bpkg.test-separate-installed.create",
  "bpkg.test-separate-installed.create_for_target",
  "bpkg.test-separate-installed.create_for_host",
  "bpkg.test-separate-installed.create_for_module",

  "bpkg.test-separate-installed.link",
  "bpkg.test-separate-installed.configure.add",
  "bpkg.test-separate-installed.configure.fetch",

  "bpkg.test-separate-installed.configure.build",
  "bpkg.test-separate-installed.configure.build_for_target",
  "bpkg.test-separate-installed.configure.build_for_host",
  "bpkg.test-separate-installed.configure.build_for_module",

  "bpkg.test-separate-installed.update",
  "bpkg.test-separate-installed.test",

  "bbot.sys-uninstall.apt-get.remove",
  "bbot.sys-uninstall.dnf.remove",

  "bpkg.uninstall",

  "bbot.bindist.upload",

  "bbot.upload",
  "bbot.upload.tar.create",
  "bbot.upload.tar.list",

  "end"};

static inline const string&
to_string (step_id s)
{
  return step_id_str[static_cast<size_t> (s)];
}

using std::regex;
namespace regex_constants = std::regex_constants;
using regexes = vector<regex>;

// UTF-8-sanitize and log the line. Also print it to a tracer, if specified,
// or to stderr otherwise at verbosity level 3 or higher.
//
static void
log_line (string&& l, string& log, tracer* trace = nullptr)
{
  if (verb >= 3)
  {
    if (trace != nullptr)
      *trace << l;
    else
      text << l;
  }

  to_utf8 (l, '?', codepoint_types::graphic, U"\n\r\t");

  log += l;
  log += '\n';
}

#ifndef _WIN32
const char* comment_begin ("#");
#else
const char* comment_begin ("rem");
#endif

static void
log_step_id (tracer& t, string& log, step_id id)
{
  string ts (to_string (system_clock::now (),
                        "%Y-%m-%d %H:%M:%S %Z",
                        true /* special */,
                        true /* local */));

  const string& sid (to_string (id));

  l3 ([&]{t << "step id: " << sid << ' ' << ts;});

  log += comment_begin;
  log += " step id: ";
  log += sid;
  log += ' ';
  log += ts;
  log += '\n';
}

// Run the worker script command. Name is used for logging and diagnostics
// only. Match lines read from the command's stderr against the regular
// expressions and return the warning result status (instead of success) in
// case of a match. Save the executed command into last_cmd.
//
// Redirect stdout to stderr if the out_* arguments are not specified (out_str
// is NULL and out_file is empty; must never be specified both). Otherwise,
// save the process output into the variable referenced by out_str, if
// specified, and to the file referenced by out_file otherwise. Note: in the
// former case assumes that the output will always fit into the pipe buffer.
//
// If bkp_step is present and is equal to the command step, then prior to
// running this command ask the user if to continue or abort the task
// execution. If bkp_status is present, then ask for that if the command
// execution results with the specified or more critical status.
//
// For the special end step no command is executed. In this case only the user
// is potentially prompted and the step is traced/logged.
//
// If specified, the pre-run callback is called after the step id is logged
// but before the command is logged/executed.
//
using pre_run_function = void ();

template <typename... A>
static result_status
run_cmd (step_id step,
         tracer& t,
         string& log,
         const function<pre_run_function>& pre_run,
         optional<string>* out_str, const path& out_file,
         const regexes& warn_detect,
         const string& name,
         const optional<step_id>& bkp_step,
         const optional<result_status>& bkp_status,
         string& last_cmd,
         const process_env& pe,
         A&&... a)
{
  assert (out_str == nullptr || out_file.empty ());

  string next_cmd;

  // Prompt the user if to continue the task execution and, if they refuse,
  // log this and throw abort.
  //
  struct abort {};

  auto prompt = [&last_cmd, &next_cmd, &t, &log] (const string& what)
  {
    diag_record dr (text);

    dr << '\n'
       << what << '\n'
       << "  current dir:  " << current_directory () << '\n'
       << "  environment:  " << ops.env_script () << ' ' << ops.env_target ();

    if (!last_cmd.empty ())
      dr << '\n'
         << "  last command: " << last_cmd;

    if (!next_cmd.empty ())
      dr << '\n'
         << "  next command: " << next_cmd;

    dr.flush ();

    if (!yn_prompt (
          "continue execution (or you may shutdown the machine)? [y/n]"))
    {
      log_line ("execution aborted by interactive user", log, &t);
      throw abort ();
    }
  };

  auto prompt_step = [step, &t, &log, &bkp_step, &prompt, &pre_run] ()
  {
    // Prompt the user if the breakpoint is reached.
    //
    if (bkp_step && *bkp_step == step)
      prompt (to_string (step) + " step reached");

    log_step_id (t, log, step);

    if (pre_run)
      pre_run ();
  };

  try
  {
    // Trace, log, and save the command line.
    //
    auto cmdc = [&t, &log, &next_cmd, &prompt_step]
                (const char* c[], size_t n)
    {
      std::ostringstream os;
      process::print (os, c, n);
      next_cmd = os.str ();

      prompt_step ();

      // Log the command to be executed.
      //
      t (c, n);

      log += next_cmd;
      log += '\n';
    };

    result_status r (result_status::success);

    if (step != step_id::end)
    {
      try
      {
        // Redirect stdout to stderr, if the caller is not interested in it.
        //
        // Text mode seems appropriate.
        //
        fdpipe out_pipe (out_str != nullptr ? fdopen_pipe () : fdpipe ());
        fdpipe err_pipe (fdopen_pipe ());

        // If the output file is specified, then open "half-pipe".
        //
        if (!out_file.empty ())
        try
        {
          out_pipe.out = fdopen (out_file,
                                 fdopen_mode::out | fdopen_mode::create);
        }
        catch (const io_error& e)
        {
          fail << "unable to open " << out_file << ": " << e;
        }

        process pr (
          process_start_callback (
            cmdc,
            fdopen_null (), // Never reads from stdin.
            out_pipe.out != nullfd ? out_pipe.out.get () : 2,
            err_pipe,
            pe,
            forward<A> (a)...));

        out_pipe.out.close ();
        err_pipe.out.close ();

        {
          // Skip on exception.
          //
          ifdstream is (move (err_pipe.in),
                        fdstream_mode::skip,
                        ifdstream::badbit);

          for (string l; !eof (getline (is, l)); )
          {
            // Match the log line with the warning-detecting regular
            // expressions until the first match.
            //
            if (r != result_status::warning)
            {
              for (const regex& re: warn_detect)
              {
                // Only examine the first 512 bytes. Long lines (e.g., linker
                // command lines) could trigger implementation-specific
                // limitations (like stack overflow). Plus, it is a
                // performance concern.
                //
                if (regex_search (l.begin (),
                                  (l.size () < 512
                                   ? l.end ()
                                   : l.begin () + 512),
                                  re))
                {
                  r = result_status::warning;
                  break;
                }
              }
            }

            log_line (move (l), log);
          }
        }

        if (!pr.wait ())
        {
          const process_exit& e (*pr.exit);
          log_line (name + ' ' + to_string (e), log, &t);
          r = e.normal () ? result_status::error : result_status::abnormal;
        }

        // Only read the buffered output if the process terminated normally.
        //
        if (out_str != nullptr && pr.exit->normal ())
        {
          // Note: shouldn't throw since the output is buffered.
          //
          ifdstream is (move (out_pipe.in));
          *out_str = is.read_text ();
        }

        last_cmd = move (next_cmd);

        if (bkp_status && r >= *bkp_status)
        {
          next_cmd.clear (); // Note: used by prompt().
          prompt (!r ? "error occured" : "warning is issued");
        }
      }
      catch (const process_error& e)
      {
        fail << "unable to execute " << name << ": " << e;
      }
      catch (const io_error& e)
      {
        fail << "unable to read " << name << " diagnostics: " << e;
      }
    }
    else
    {
      next_cmd.clear (); // Note: used by prompt_step().
      prompt_step ();
    }

    return r;
  }
  catch (const abort&)
  {
    return result_status::abort;
  }
}

template <typename... A>
static result_status
run_cmd (step_id step,
         tracer& t,
         string& log,
         optional<string>* out_str, const path& out_file,
         const regexes& warn_detect,
         const string& name,
         const optional<step_id>& bkp_step,
         const optional<result_status>& bkp_status,
         string& last_cmd,
         const process_env& pe,
         A&&... a)
{
  return run_cmd (step,
                  t,
                  log,
                  nullptr /* pre_run */,
                  out_str, out_file,
                  warn_detect,
                  name,
                  bkp_step,
                  bkp_status,
                  last_cmd,
                  pe,
                  forward<A> (a)...);
}

template <typename V, typename... A>
static result_status
run_bpkg (step_id step,
          const V& envvars,
          tracer& t,
          string& log,
          const function<pre_run_function>& pre_run,
          optional<string>& out,
          const regexes& warn_detect,
          const optional<step_id>& bkp_step,
          const optional<result_status>& bkp_status,
          string& last_cmd,
          const char* verbosity,
          const string& cmd, A&&... a)
{
  return run_cmd (step,
                  t,
                  log,
                  pre_run,
                  &out, path () /* out_file */,
                  warn_detect,
                  "bpkg " + cmd,
                  bkp_step, bkp_status, last_cmd,
                  process_env ("bpkg", envvars),
                  verbosity, cmd, forward<A> (a)...);
}

template <typename V, typename... A>
static result_status
run_bpkg (step_id step,
          const V& envvars,
          tracer& t,
          string& log, const regexes& warn_detect,
          const optional<step_id>& bkp_step,
          const optional<result_status>& bkp_status,
          string& last_cmd,
          const char* verbosity,
          const string& cmd, A&&... a)
{
  return run_cmd (step,
                  t,
                  log,
                  nullptr /* out_str */, path () /* out_file */,
                  warn_detect,
                  "bpkg " + cmd,
                  bkp_step, bkp_status, last_cmd,
                  process_env ("bpkg", envvars),
                  verbosity, cmd, forward<A> (a)...);
}

template <typename... A>
static result_status
run_bpkg (step_id step,
          tracer& t,
          string& log,
          const function<pre_run_function>& pre_run,
          optional<string>& out,
          const regexes& warn_detect,
          const optional<step_id>& bkp_step,
          const optional<result_status>& bkp_status,
          string& last_cmd,
          const char* verbosity,
          const string& cmd, A&&... a)
{
  const char* const* envvars (nullptr);

  return run_bpkg (step,
                   envvars,
                   t,
                   log,
                   pre_run,
                   out,
                   warn_detect,
                   bkp_step, bkp_status, last_cmd,
                   verbosity, cmd, forward<A> (a)...);
}

template <typename V, typename... A>
static result_status
run_bpkg (step_id step,
          const V& envvars,
          tracer& t,
          string& log, const path& out, const regexes& warn_detect,
          const optional<step_id>& bkp_step,
          const optional<result_status>& bkp_status,
          string& last_cmd,
          const char* verbosity,
          const string& cmd, A&&... a)
{
  return run_cmd (step,
                  t,
                  log, nullptr /* out_str */, out, warn_detect,
                  "bpkg " + cmd,
                  bkp_step, bkp_status, last_cmd,
                  process_env ("bpkg", envvars),
                  verbosity, cmd, forward<A> (a)...);
}

template <typename... A>
static result_status
run_bpkg (step_id step,
          tracer& t,
          string& log, const path& out, const regexes& warn_detect,
          const optional<step_id>& bkp_step,
          const optional<result_status>& bkp_status,
          string& last_cmd,
          const char* verbosity,
          const string& cmd, A&&... a)
{
  const char* const* envvars (nullptr);

  return run_bpkg (step,
                   envvars,
                   t,
                   log, out, warn_detect,
                   bkp_step, bkp_status, last_cmd,
                   verbosity, cmd, forward<A> (a)...);
}

template <typename... A>
static result_status
run_bpkg (step_id step,
          tracer& t,
          string& log, const regexes& warn_detect,
          const optional<step_id>& bkp_step,
          const optional<result_status>& bkp_status,
          string& last_cmd,
          const char* verbosity,
          const string& cmd, A&&... a)
{
  const char* const* envvars (nullptr);

  return run_bpkg (step,
                   envvars,
                   t,
                   log, warn_detect,
                   bkp_step, bkp_status, last_cmd,
                   verbosity, cmd, forward<A> (a)...);
}

template <typename V, typename... A>
static result_status
run_b (step_id step,
       const V& envvars,
       tracer& t,
       string& log, const regexes& warn_detect,
       const optional<step_id>& bkp_step,
       const optional<result_status>& bkp_status,
       string& last_cmd,
       const char* verbosity,
       const strings& buildspecs, A&&... a)
{
  string name ("b");
  for (const string& s: buildspecs)
  {
    if (!name.empty ())
      name += ' ';

    name += s;
  }

  return run_cmd (step,
                  t, log,
                  nullptr /* out_str */, path () /* out_file */,
                  warn_detect,
                  name,
                  bkp_step, bkp_status, last_cmd,
                  process_env ("b", envvars),
                  verbosity, buildspecs, forward<A> (a)...);
}

template <typename V, typename... A>
static result_status
run_b (step_id step,
       const V& envvars,
       tracer& t,
       string& log, const regexes& warn_detect,
       const optional<step_id>& bkp_step,
       const optional<result_status>& bkp_status,
       string& last_cmd,
       const char* verbosity,
       const string& buildspec, A&&... a)
{
  return run_cmd (step,
                  t, log,
                  nullptr /* out_str */, path () /* out_file */,
                  warn_detect,
                  "b " + buildspec,
                  bkp_step, bkp_status, last_cmd,
                  process_env ("b", envvars),
                  verbosity, buildspec, forward<A> (a)...);
}

template <typename... A>
static result_status
run_b (step_id step,
       tracer& t,
       string& log, const regexes& warn_detect,
       const optional<step_id>& bkp_step,
       const optional<result_status>& bkp_status,
       string& last_cmd,
       const char* verbosity,
       const string& buildspec, A&&... a)
{
  const char* const* envvars (nullptr);
  return run_b (step,
                envvars,
                t,
                log, warn_detect,
                bkp_step, bkp_status, last_cmd,
                verbosity, buildspec, forward<A> (a)...);
}

template <typename... A>
static result_status
run_ldconfig (step_id step,
              tracer& t,
              string& log, const regexes& warn_detect,
              const optional<step_id>& bkp_step,
              const optional<result_status>& bkp_status,
              string& last_cmd,
              A&&... a)
{
  return run_cmd (step,
                  t,
                  log,
                  nullptr /* out_str */, path () /* out_file*/,
                  warn_detect,
                  "sudo ldconfig",
                  bkp_step, bkp_status, last_cmd,
                  process_env ("sudo"),
                  "ldconfig", forward<A> (a)...);
}

template <typename... A>
static result_status
run_apt_get (step_id step,
             tracer& t,
             string& log, const regexes& warn_detect,
             const optional<step_id>& bkp_step,
             const optional<result_status>& bkp_status,
             string& last_cmd,
             const string& cmd, A&&... a)
{
  // Note: dumps some of its diagnostics to stdout.
  //
  return run_cmd (step,
                  t,
                  log,
                  nullptr /* out_str */, path () /* out_file*/,
                  warn_detect,
                  "sudo apt-get " + cmd,
                  bkp_step, bkp_status, last_cmd,
                  process_env ("sudo"),
                  "apt-get", cmd, forward<A> (a)...);
}

template <typename... A>
static result_status
run_dnf (step_id step,
         tracer& t,
         string& log, const regexes& warn_detect,
         const optional<step_id>& bkp_step,
         const optional<result_status>& bkp_status,
         string& last_cmd,
         const string& cmd, A&&... a)
{
  // Note: dumps some of its diagnostics to stdout.
  //
  return run_cmd (step,
                  t,
                  log,
                  nullptr /* out_str */, path () /* out_file*/,
                  warn_detect,
                  "sudo dnf " + cmd,
                  bkp_step, bkp_status, last_cmd,
                  process_env ("sudo"),
                  "dnf", cmd, forward<A> (a)...);
}

#ifndef _WIN32
template <typename... A>
static result_status
run_tar (step_id step,
         tracer& t,
         string& log, const regexes& warn_detect,
         const optional<step_id>& bkp_step,
         const optional<result_status>& bkp_status,
         string& last_cmd,
         bool sudo,
         A&&... a)
{
  return run_cmd (step,
                  t,
                  log,
                  nullptr /* out_str */, path () /* out_file*/,
                  warn_detect,
                  sudo ? "sudo tar" : "tar",
                  bkp_step, bkp_status, last_cmd,
                  process_env (sudo ? "sudo" : "tar"),
                  sudo ? "tar" : nullptr, forward<A> (a)...);
}
#else
template <typename... A>
static result_status
run_tar (step_id step,
         tracer& t,
         string& log, const regexes& warn_detect,
         const optional<step_id>& bkp_step,
         const optional<result_status>& bkp_status,
         string& last_cmd,
         bool /* sudo */,
         A&&... a)
{
  // Note: using bsdtar which can unpack .zip archives (and also not an MSYS
  //       executable).
  //
  return run_cmd (step,
                  t,
                  log,
                  nullptr /* out_str */, path () /* out_file*/,
                  warn_detect,
                  "bsdtar",
                  bkp_step, bkp_status, last_cmd,
                  process_env ("bsdtar"),
                  forward<A> (a)...);
}
#endif

// Upload compressed manifest to the specified TFTP URL with curl. Issue
// diagnostics and throw failed on invalid manifest or process management
// errors and throw io_error for input/output errors or non-zero curl exit.
//
template <typename T>
static void
upload_manifest (tracer& trace,
                 const string& url,
                 const T& m,
                 const string& what)
{
  try
  {
    // Piping the data directly into curl's stdin sometimes results in the
    // broken pipe error on the client and partial/truncated upload on the
    // server. This happens quite regularly on older Linux distributions
    // (e.g., Debian 8, Ubuntu 16.04) but also sometimes on Windows. On the
    // other hand, uploading from a file appears to work reliably (we still
    // get an odd error on Windows from time to time with larger uploads).
    //
    // Let's not break lines in the manifest values not to further increase
    // the size of the manifest encoded representation. Also here we don't
    // care much about readability of the manifest since it will only be read
    // by the bbot agent anyway.
    //
#if 0
    // Note: need to add compression support if re-enable this.
    tftp_curl c (trace,
                 path ("-"),
                 nullfd,
                 curl::put,
                 url,
                 "--tftp-blksize", tftp_blksize,
                 "--max-time", tftp_put_timeout);

    manifest_serializer s (c.out, url, true /* long_lines */);
    m.serialize (s);
    c.out.close ();
#else
    auto_rmfile tmp;
    try
    {
      tmp = auto_rmfile (path::temp_path (what + "-manifest.lz4"));
      ofdstream ofs (tmp.path, fdopen_mode::binary);
      olz4stream ozs (ofs, 9, 5 /* 256KB */, nullopt /* content_size */);
      manifest_serializer s (ozs, tmp.path.string (), true /* long_lines */);
      m.serialize (s);
      ozs.close ();
      ofs.close ();
    }
    catch (const io_error& e) // In case not derived from system_error.
    {
      fail << "unable to save " << what << " manifest: " << e;
    }
    catch (const system_error& e)
    {
      fail << "unable to save " << what << " manifest: " << e;
    }

    tftp_curl c (trace,
                 tmp.path,
                 nullfd,
                 curl::put,
                 url,
                 "--tftp-blksize", tftp_blksize,
                 "--max-time", tftp_put_timeout);
#endif

    if (!c.wait ())
      throw_generic_ios_failure (EIO, "non-zero curl exit code");
  }
  catch (const manifest_serialization& e)
  {
    fail << "invalid " << what << " manifest: " << e.description;
  }
  catch (const process_error& e)
  {
    fail << "unable to execute curl: " << e;
  }
  catch (const system_error& e)
  {
    const auto& c (e.code ());

    if (c.category () == generic_category ())
      throw_generic_ios_failure (c.value (), e.what ());
    else
      throw_system_ios_failure (c.value (), e.what ());
  }
}

static const string worker_checksum ("5"); // Logic version.

static int bbot::
build (size_t argc, const char* argv[])
{
  using std::map;
  using std::multimap;

  using string_parser::unquote;

  using serializer    = manifest_serializer;
  using serialization = manifest_serialization;

  using namespace bpkg;

  tracer trace ("build");

  // Our overall plan is as follows:
  //
  // 1. Parse the task manifest (it should be in CWD).
  //
  // 2. Run bpkg to create the package/tests configurations, add the
  //    repository to them, and configure, build, test, optionally install or
  //    alternatively bindist and sys-install, test installed, and
  //    (sys-)uninstall the package all while saving the logs in the result
  //    manifest.
  //
  // 3. Upload the result manifest and, optionally, the build artifacts.
  //
  // NOTE: consider updating worker_checksum if making any logic changes.
  //
  // Note also that we are being "watched" by the startup version of us which
  // will upload an appropriate result in case we exit with an error. So here
  // for abnormal situations (like a failure to parse the manifest), we just
  // fail.
  //
  task_manifest tm (
    parse_manifest<task_manifest> (path ("task.manifest"), "task"));

  // Reset the dependency checksum if the task's worker checksum doesn't match
  // the current one.
  //
  if (!tm.worker_checksum || *tm.worker_checksum != worker_checksum)
    tm.dependency_checksum = nullopt;

  result_manifest rm {
    tm.name,
    tm.version,
    result_status::success,
    operation_results {},
    worker_checksum,
    nullopt /* dependency_checksum */
  };

  auto add_result = [&rm] (string o) -> operation_result&
  {
    rm.results.push_back (
      operation_result {move (o), result_status::success, ""});

    return rm.results.back ();
  };

  dir_path rwd; // Root working directory.

  // Archive of the build artifacts for upload.
  //
  path upload_archive ("upload.tar");

  // Resolve the breakpoint specified by the interactive manifest value into
  // the step id or the result status breakpoint. If the breakpoint is
  // invalid, then log the error and abort the build. Note that we reuse the
  // configure operation log here not to complicate things.
  //
  optional<step_id> bkp_step;
  optional<result_status> bkp_status;
  string last_cmd; // Used in the user prompt.

  for (;;) // The "breakout" loop.
  {
    auto fail_operation = [&trace] (operation_result& r,
                                    const string& e,
                                    result_status s,
                                    const string& name = "",
                                    uint64_t line = 0,
                                    uint64_t column = 0)
    {
      string prefix;

      if (!name.empty ())
      {
        prefix += name;
        prefix += ':';

        if (line != 0)
        {
          prefix += to_string (line);
          prefix += ':';

          if (column != 0)
          {
            prefix += to_string (column);
            prefix += ':';
          }
        }

        prefix += ' ';
      }

      l3 ([&]{trace << prefix << e;});

      r.log    += prefix + "error: " + e + '\n';
      r.status  = s;
    };

    // Regular expressions that detect different forms of build2 toolchain
    // warnings. Accidently (or not), they also cover GCC and Clang warnings
    // (for the English locale).
    //
    // The expressions will be matched multiple times, so let's make the
    // matching faster, with the potential cost of making regular expressions
    // creation slower.
    //
    regex::flag_type f (regex_constants::optimize); // ECMAScript is implied.

    regexes wre {
      regex ("^warning: ", f),
      regex ("^.+: warning: ", f)};

    for (const string& re: tm.unquoted_warning_regex ())
      wre.emplace_back (re, f);

    if (tm.interactive && *tm.interactive != "none")
    {
      const string& b (*tm.interactive);

      if (b == "error")
        bkp_status = result_status::error;
      else if (b == "warning")
        bkp_status = result_status::warning;
      else
      {
        for (size_t i (0); i < step_id_str.size (); ++i)
        {
          if (b == step_id_str[i])
          {
            bkp_step = static_cast<step_id> (i);
            break;
          }
        }
      }

      if (!bkp_step && !bkp_status)
      {
        fail_operation (add_result ("configure"),
                        "invalid interactive build breakpoint '" + b + '\'',
                        result_status::abort);

        break;
      }
    }

    // Split the argument into prefix (empty if not present) and unquoted
    // value (absent if not present) and determine the step status. If the
    // prefix is present and is prepended with the '+'/'-' character, then the
    // respective step needs to be enabled/disabled. Return nullopt if the
    // prefix is invalid.
    //
    // Note that arguments with absent values are normally used to
    // enable/disable steps and are omitted from the command lines.
    //
    struct argument
    {
      string prefix;

      // Absent if the argument value is an empty unquoted string.
      //
      optional<string> value;

      // True - enable, false - disable, nullopt - neutral.
      //
      optional<bool> step_status;
    };

    auto parse_arg = [] (const string& a) -> optional<argument>
    {
      size_t p (a.find_first_of (":=\"'"));

      auto value = [] (const string& v)
      {
        return !v.empty () ? unquote (v) : optional<string> ();
      };

      if (p == string::npos || a[p] != ':') // No prefix.
        return argument {string (), value (a), nullopt};

      string prefix (a, 0, p);

      optional<bool> enable;
      if (prefix[0] == '+' || prefix[0] == '-')
      {
        enable = (prefix[0] == '+');

        prefix.erase (0, 1);

        if (prefix != "bpkg.update"                         &&
            prefix != "bpkg.test"                           &&
            prefix != "bpkg.test-separate.update"           &&
            prefix != "bpkg.test-separate.test"             &&
            prefix != "bpkg.install"                        &&
            prefix != "bbot.install.ldconfig"               &&
            prefix != "bpkg.bindist.debian"                 &&
            prefix != "bpkg.bindist.fedora"                 &&
            prefix != "bpkg.bindist.archive"                &&
            prefix != "bbot.sys-install"                    &&
            prefix != "bbot.sys-install.ldconfig"           &&
            prefix != "b.test-installed.test"               &&
            prefix != "bpkg.test-separate-installed.update" &&
            prefix != "bpkg.test-separate-installed.test"   &&
            prefix != "bbot.bindist.upload"                 &&
            prefix != "bbot.upload")
        {
          return nullopt; // Prefix is invalid.
        }
      }

      for (const string& id: step_id_str)
      {
        size_t n (prefix.size ());
        if (id.compare (0, n, prefix) == 0 &&
            (id.size () == n || (id.size () > n && id[n] == '.')))
          return argument {move (prefix), value (a.substr (p + 1)), enable};
      }

      return nullopt; // Prefix is invalid.
    };

    // Keep track of explicitly enabled/disabled steps.
    //
    map<string, bool> step_statuses;

    // Return true if the step is explicitly enabled via a +<prefix>:[<value>]
    // environment/configuration argument.
    //
    auto step_enabled = [&step_statuses] (step_id step) -> bool
    {
      auto i (step_statuses.find (to_string (step)));
      return i != step_statuses.end () && i->second;
    };

    // Return true if the step is explicitly disabled via a -<prefix>:[<value>]
    // environment/configuration argument.
    //
    auto step_disabled = [&step_statuses] (step_id step) -> bool
    {
      auto i (step_statuses.find (to_string (step)));
      return i != step_statuses.end () && !i->second;
    };

    // Save a step status.
    //
    // Note that since the bpkg.bindist.* steps are mutually exclusive we only
    // keep the latest status change (see above for details).
    //
    auto step_status = [&step_statuses] (const string& step, bool status)
    {
      if (step.compare (0, 13, "bpkg.bindist.") == 0)
      {
        step_statuses.erase ("bpkg.bindist.debian");
        step_statuses.erase ("bpkg.bindist.fedora");
        step_statuses.erase ("bpkg.bindist.archive");
      }

      step_statuses[step] = status;
    };

    // Parse the environment, target configuration, and build package
    // configuration arguments.
    //
    // NOTE: keep this parsing order intact so that, for example, a build
    //       package configuration argument can override step status specified
    //       by a target configuration argument.
    //

    // Parse environment arguments.
    //
    multimap<string, string> modules;
    multimap<string, string> env_args;

    for (size_t i (1); i != argc; ++i)
    {
      const char* a (argv[i]);
      optional<argument> v (parse_arg (a));

      if (!v)
        fail << "invalid environment argument prefix in '" << a << "'";

      bool mod (v->value              &&
                (*v->value)[0] != '-' &&
                v->value->find ('=') == string::npos);

      if (mod                                                           &&
          !v->prefix.empty ()                                           &&
          v->prefix != "b.create"                                       &&
          v->prefix != "bpkg.create"                                    &&
          v->prefix != "bpkg.target.create"                             &&
          v->prefix != "bpkg.host.create"                               &&
          v->prefix != "bpkg.module.create"                             &&
          v->prefix != "b.test-installed.create"                        &&
          v->prefix != "bpkg.test-separate-installed.create"            &&
          v->prefix != "bpkg.test-separate-installed.create_for_target" &&
          v->prefix != "bpkg.test-separate-installed.create_for_host"   &&
          v->prefix != "bpkg.test-separate-installed.create_for_module")
        fail << "invalid module prefix in '" << a << "'";

      if (v->step_status)
        step_status (v->prefix, *v->step_status);

      if (v->value)
        (mod ? modules : env_args).emplace (make_pair (move (v->prefix),
                                                       move (*v->value)));
    }

    // Parse target configuration arguments. Report failures to the bbot
    // controller.
    //
    multimap<string, string> tgt_args;

    for (const string& c: tm.target_config)
    {
      optional<argument> v (parse_arg (c));

      if (!v)
      {
        rm.status |= result_status::abort;
        l3 ([&]{trace << "invalid target configuration argument prefix in "
                      << "'" << c << "'";});
        break;
      }

      if (v->value               &&
          (*v->value)[0] != '-'  &&
          v->value->find ('=') == string::npos)
      {
        rm.status |= result_status::abort;
        l3 ([&]{trace << "invalid target configuration argument '" << c
                      << "'";});
        break;
      }

      if (v->step_status)
        step_status (v->prefix, *v->step_status);

      if (v->value)
        tgt_args.emplace (make_pair (move (v->prefix), move (*v->value)));
    }

    if (!rm.status)
      break;

    // Parse the build package configuration represented as a whitespace
    // separated list of the following potentially quoted bpkg-pkg-build
    // command arguments:
    //
    // <option>...
    // <config-var>...
    // ([{ <config-var>... }+] (?[sys:]|sys:)<pkg-name>[<version-spec>])...
    // ( { <config-var>... }+  <pkg-name>)...
    //
    // If the package configuration is specified, then parse it into the
    // following lists/maps:
    //
    // - The prefixed global options and configuration variables map
    //   (pkg_args). Added to the command lines at the corresponding steps
    //   after potential environment and target configuration arguments.
    //
    // - The unprefixed global options list (pkg_config_opts). Specified after
    //   all the prefixed global options on the bpkg-pkg-build command line at
    //   the bpkg.configure.build step.
    //
    // - The main package-specific configuration variables list
    //   (pkg_config_vars). Specified for the main package only on the
    //   bpkg-pkg-build command line at the bpkg.configure.build step,
    //   wherever it is configured. Also specified on the b-configure command
    //   line at the b.test-installed.configure step.
    //
    // - The main package-specific dependency packages list
    //   (pkg_config_main_deps), potentially with their own configuration
    //   variables (but not options). Only configured where the main package
    //   is configured with the bpkg-pkg-build command line at the
    //   bpkg.configure.build step.
    //
    // - The global system dependency packages list (pkg_config_glob_deps).
    //   Configured in all configurations with the bpkg-pkg-build command line
    //   at the bpkg.configure.build step.
    //
    // - The main and external test package-specific configuration variables
    //   map (pkg_config_pkgs). Specified on the bpkg-pkg-build command lines
    //   at the bpkg.configure.build and
    //   bpkg.test-separate-installed.configure.build steps. Package names
    //   other than the main and external test package names are silently
    //   ignored.
    //
    multimap<string, string>      pkg_args;
    strings                       pkg_config_opts;
    strings                       pkg_config_vars;
    vector<pair<string, strings>> pkg_config_main_deps; // ?<pkg>, sys:<pkg>
    vector<pair<string, strings>> pkg_config_glob_deps; // ?sys:<pkg>
    map<string, strings>          pkg_config_pkgs;      // <pkg>

    if (!tm.package_config.empty ())
    {
      struct abort {};

      auto fail = [&tm, &add_result, &fail_operation] (const string& d,
                                                       bool throw_abort = true)
      {
        fail_operation (add_result ("configure"),
                        "invalid package configuration: " + d +
                        "\n  info: package configuration: '"  +
                        tm.package_config + '\'',
                        result_status::abort);

        if (throw_abort)
          throw abort ();
      };

      try
      {
        strings argsv (string_parser::parse_quoted (tm.package_config,
                                                    false /* unquote */));

        cli::vector_scanner scanv (argsv);
        cli::group_scanner args (scanv);

        while (args.more ())
        {
          string a (args.next ());

          // Unless the argument is an unquoted dependency (starts with `?` or
          // `sys:`), first try to interpret it as a prefixed option/variable
          // and/or step id status (enabled/disabled).
          //
          if (!(a[0] == '?' || a.compare (0, 4, "sys:") == 0))
          {
            optional<argument> v (parse_arg (a));

            // Note that we only assume an argument as prefixed if the prefix
            // is a known step id. Otherwise, we interpret the argument as
            // unprefixed global option, variable, or a package spec.
            //
            if (v && !v->prefix.empty ())
            {
              if (v->value              &&
                  (*v->value)[0] != '-' &&
                  v->value->find ('=') == string::npos)
                fail ("invalid prefixed argument '" + a + '\'');

              if (args.group ().more ())
                fail ("unexpected options group for prefixed argument '" + a +
                      '\'');

              if (v->step_status)
                step_status (v->prefix, *v->step_status);

              if (v->value)
                pkg_args.emplace (make_pair (move (v->prefix),
                                             move (*v->value)));

              continue;
            }
          }

          a = unquote (a);

          // Return true if the argument is an option.
          //
          // Note that options with values can only be specified using
          // the single argument notation.
          //
          auto opt = [] (const string& a)
          {
            // Make sure that -- or - is always followed by some characters.
            //
            return a.compare (0, 2, "--") == 0 ? a.size () > 2 :
                   a[0] == '-'                 ? a.size () > 1 :
                                                 false         ;
          };

          // Return true if the argument is a configuration variable.
          //
          auto var = [] (const string& a)
          {
            // Note: we need to be careful not to misinterpret
            // '?libfoo == 1.0.0' as a variable.
            //
            return a.compare (0, 7, "config.") == 0 &&
                   a.find ('=') != string::npos;
          };

          bool o (opt (a));
          bool v (var (a));

          cli::scanner& ag (args.group ());

          if (o)      // Option.
          {
            if (ag.more ())
              fail ("unexpected options group for option '" + a + '\'');

            pkg_config_opts.push_back (move (a));
          }
          else if (v) // Configuration variable.
          {
            if (ag.more ())
              fail ("unexpected options group for configuration variable '" +
                    a + '\'');

            pkg_config_vars.push_back (move (a));
          }
          else        // Dependency or build-to-hold package.
          {
            // Note that we consider a system package as a dependency
            // regardless whether it is prefixed with '?' or not.
            //
            strings vars;
            while (ag.more ())
            {
              string da (unquote (ag.next ()));
              if (!var (da))
                fail ("argument is not a configuration variable for "
                      "dependency " + a + ": '" + da + '\'');

              vars.push_back (move (da));
            }

            // Add the system dependency packages (prefixed with `?sys:`) to
            // a separate list, to specify them globally on the
            // bpkg-pkg-build command line for configuring them in all the
            // (being) created configurations.
            //
            // Note, though, that we will handle the build-to-hold system
            // packages (prefixed with `sys:`) in the same way as non system
            // dependencies, since such an auto-configuration is only
            // supported by bpkg-pkg-build for system dependencies. In the
            // future, we may support that on the bbot worker level by, for
            // example, specifying all the configurations manually for the
            // build-to-hold system packages and also specifying them as a
            // system dependencies globally. We need to be careful to make
            // sure that these dependencies are also auto-configured for the
            // private configurations potentially created by bpkg-pkg-build.
            //
            // Also note that in the future we may allow package-specific
            // --config-uuid options to only configure such packages in the
            // specified configurations. We may also invent the special
            // 00000000-0000-0000-0000-000000000005 configuration id to, for
            // example, only configure them at the
            // bpkg.test-separate-installed.configure.build step.
            //
            if (a.compare (0, 5, "?sys:") == 0) // Global system dependency.
            {
              pkg_config_glob_deps.push_back (make_pair (move (a),
                                                         move (vars)));
            }
            else if (a[0] == '?' ||             // Main package dependency.
                     a.compare (0, 4, "sys:") == 0)
            {
              pkg_config_main_deps.push_back (make_pair (move (a),
                                                         move (vars)));
            }
            else                                // Build-to-hold package.
            {
              if (vars.empty ())
                fail ("no configuration variables specified for package '" +
                      a + '\'');

              auto i (pkg_config_pkgs.find (a));

              if (i == pkg_config_pkgs.end ())
              {
                pkg_config_pkgs.emplace (move (a), move (vars));
              }
              else
              {
                strings& vs (i->second);
                vs.insert (vs.end (),
                           make_move_iterator (vars.begin ()),
                           make_move_iterator (vars.end ()));
              }
            }
          }
        }
      }
      catch (const string_parser::invalid_string& e)
      {
        fail (e.what (), false /* throw_abort */);
        break;
      }
      catch (const abort&)
      {
        break;
      }
    }

    // Return command arguments for the specified step id, complementing
    // *.create[_for_*] steps with un-prefixed arguments. If no arguments are
    // specified for the step then use the specified fallbacks, potentially
    // both. Arguments with more specific prefixes come last.
    //
    auto step_args = [] (const multimap<string, string>& args,
                         step_id step,
                         optional<step_id> fallback1 = nullopt,
                         optional<step_id> fallback2 = nullopt) -> cstrings
    {
      cstrings r;

      // Add arguments for a specified, potentially empty, prefix.
      //
      auto add_args = [&args, &r] (const string& prefix)
      {
        auto range (args.equal_range (prefix));

        for (auto i (range.first); i != range.second; ++i)
          r.emplace_back (i->second.c_str ());
      };

      // Add un-prefixed arguments if this is one of the *.create[_for_*]
      // steps.
      //
      switch (step)
      {
      case step_id::b_create:
      case step_id::bpkg_create:
      case step_id::bpkg_target_create:
      case step_id::bpkg_host_create:
      case step_id::bpkg_module_create:
      case step_id::b_test_installed_create:
      case step_id::bpkg_test_separate_installed_create:
      case step_id::bpkg_test_separate_installed_create_for_target:
      case step_id::bpkg_test_separate_installed_create_for_host:
      case step_id::bpkg_test_separate_installed_create_for_module:
        {
          add_args ("");
          break;
        }
      default: break;
      }

      auto add_step_args = [&add_args] (step_id step)
      {
        const string& s (to_string (step));

        for (size_t n (0);; ++n)
        {
          n = s.find ('.', n);

          add_args (n == string::npos ? s : string (s, 0, n));

          if (n == string::npos)
            break;
        }
      };

      // If no arguments found for the step id, then use the fallback step
      // ids, if specified.
      //
      if (args.find (to_string (step)) != args.end ())
      {
        add_step_args (step);
      }
      else
      {
        // Note that if we ever need to specify fallback pairs with common
        // ancestors, we may want to suppress duplicate ancestor step ids.
        //
        if (fallback1)
          add_step_args (*fallback1);

        if (fallback2)
          add_step_args (*fallback2);
      }

      return r;
    };

    // bpkg-rep-fetch trust options.
    //
    cstrings trust_ops;
    {
      const char* t ("--trust-no");

      for (const string& fp: tm.trust)
      {
        if (fp == "yes")
          t = "--trust-yes";
        else
        {
          trust_ops.push_back ("--trust");
          trust_ops.push_back (fp.c_str ());
        }
      }

      trust_ops.push_back (t);
    }

    const string&  pkg     (tm.name.string ());
    const version& ver     (tm.version);
    const string   repo    (tm.repository.string ());
    const dir_path pkg_dir (pkg + '-' + ver.string ());
    const string   pkg_var (tm.name.variable ());

    // Specify the revision explicitly for the bpkg-build command not to end
    // up with a race condition building the latest revision rather than the
    // zero revision.
    //
    const string pkg_rev (pkg +
                          '/' +
                          version (ver.epoch,
                                   ver.upstream,
                                   ver.release,
                                   ver.effective_revision (),
                                   ver.iteration).string ());

    // Query the project's build system information with `b info`.
    //
    auto prj_info = [&trace] (const dir_path& d,
                              b_info_flags fl,
                              const char* what)
    {
      // Note that the `b info` diagnostics won't be copied into any of the
      // build logs. This is fine as this is likely to be an infrastructure
      // problem, given that the project distribution has been successfully
      // created. It's actually not quite clear which log this diagnostics
      // could go into.
      //
      try
      {
        return b_info (d, fl, verb, trace);
      }
      catch (const b_error& e)
      {
        if (e.normal ())
          throw failed (); // Assume the build2 process issued diagnostics.

        fail << "unable to query " << what << ' ' << d << " info: " << e
             << endf;
      }
    };

    rwd = current_directory ();

    // Create directory for the build artifacts to archive and upload.
    //
    dir_path upload_dir ("upload");

    mk (trace, nullptr /* log */, upload_dir);

    // If the package comes from a version control-based repository, then we
    // will also test its dist meta-operation. Specifically, we will checkout
    // the package outside the configuration directory passing --checkout-root
    // to the configure-only pkg-build command, re-distribute the checked out
    // directory in the load distribution mode, and then use this distribution
    // as a source to build the package.
    //
    dir_path dist_root (rwd / dir_path ("dist"));
    dir_path dist_src  (dist_root / pkg_dir);

    dir_path dist_install_root (rwd / dir_path ("dist-install"));
    dir_path dist_install_src  (dist_install_root / pkg_dir);

    dir_path dist_installed_root (rwd / dir_path ("dist-installed"));

    // Redistribute the package source directory (pkg_dir) checked out into
    // the directory other than the configuration directory (dist_root) and
    // replace it with the newly created distribution. Assume that the current
    // directory is the package configuration directory. Optionally pass the
    // config.import.* variable override and/or set the environment variables
    // for the build2 process. Return true if the dist meta-operation
    // succeeds.
    //
    auto redist = [&trace, &wre, &bkp_step, &bkp_status, &last_cmd]
                  (step_id step,
                   operation_result& r,
                   const dir_path& dist_root,
                   const dir_path& pkg_dir, // <name>-<version>
                   const optional<string>& import = nullopt,
                   const small_vector<string, 1>& envvars = {})
    {
      // Temporarily change the current directory to the distribution root
      // parent directory from the configuration directory to shorten the
      // command line paths and try to avoid the '..' path prefix.
      //
      dir_path dp (dist_root.directory ());
      dir_path dn (dist_root.leaf ());

      // Redistribute the package using the configured output directory.
      //
      dir_path cnf_dir (change_wd (trace, &r.log, dp));
      dir_path out_dir (cnf_dir.relative (dp) / pkg_dir);
      dir_path src_dir (dn / pkg_dir);

      // Create the re-distribution root directory next to the distribution
      // root.
      //
      dir_path redist_root ("re" + dn.string ());

      r.status |= run_b (
        step,
        envvars,
        trace, r.log, wre,
        bkp_step, bkp_status, last_cmd,
        "-v",
        "config.dist.root=" + redist_root.string (),
        import,
        ("dist('" + src_dir.representation () + "'@'" +
         out_dir.representation () + "')"));

      if (!r.status)
        return false;

      // Replace the package source directory with the re-distribution result.
      //
      rm_r (trace, &r.log, src_dir);
      mv   (trace, &r.log, redist_root / pkg_dir, src_dir);

      change_wd (trace, &r.log, cnf_dir); // Return back to the configuration.
      return true;
    };

    // Note that if this is not a self-hosted configuration, then we do not
    // build external runtime tests nor run internal for host or module
    // packages because the assumption is that they have been built/run (and
    // with buildtab settings such as warnings, etc) when testing the
    // self-hosted configuration this non-self-hosted one is based on. Also,
    // by the same reason, we don't install tools or modules nor generate the
    // binary distribution packages for them for non-self-hosted
    // configurations.
    //
    // Actually, it could make sense to build and install tools and module
    // from a target configuration in this case. But that means for a
    // non-self-hosted configuration a tool/module may want to test two
    // things: its output build and its own build, which means we would need a
    // way to control which of the two things (or both) are to be tested
    // (think of two cross-compiler configurations, Emscripten and MinGW: for
    // the former a source code generator would normally only want to test the
    // output while for the latter -- both; maybe we could have a `cross-host`
    // class, meaning that the configuration is not host itself but its target
    // is). In any case, seeing that there is no way to verify such own build
    // works, we ignore this for now.
    //
    // Also note that build system modules can only have external build-time
    // tests (which is verified by bpkg-rep-fetch) and target packages cannot
    // have external build-time tests (which we verify ourselves).
    //
    bool selfhost (tm.host && *tm.host);

    // Detect if the package is of the target, host, or module type.
    //
    auto requirement = [&tm] (const char* id)
    {
      return find_if (tm.requirements.begin (),
                      tm.requirements.end (),
                      [id] (const requirement_alternatives& r)
                      {
                        if (r.size () == 1)
                        {
                          const requirement_alternative& a (r[0]);
                          return find (a.begin (), a.end (), id) != a.end ();
                        }

                        return false;
                      }) != tm.requirements.end ();
    };

    bool module_pkg (pkg.compare (0, 10, "libbuild2-") == 0);
    bool bootstrap  (module_pkg && requirement ("bootstrap"));
    bool host_pkg   (!module_pkg && requirement ("host"));
    bool target_pkg (!module_pkg && !host_pkg);

    // Don't generate binary packages for tools or modules for non-self-hosted
    // configurations (see above for details).
    //
    optional<step_id> bindist;

    if (target_pkg || selfhost)
    {
      if (step_enabled (step_id::bpkg_bindist_debian))
        bindist = step_id::bpkg_bindist_debian;
      else if (step_enabled (step_id::bpkg_bindist_fedora))
        bindist = step_id::bpkg_bindist_fedora;
      else if (step_enabled (step_id::bpkg_bindist_archive))
        bindist = step_id::bpkg_bindist_archive;
    }

    bool sys_install (bindist && !step_disabled (step_id::bbot_sys_install));
    bool bindist_upload (bindist && step_enabled (step_id::bbot_bindist_upload));

    // Unless a bpkg.bindist.* step is enabled or bpkg.install step is
    // disabled, search for config.install.root variable. If it is present and
    // has a non-empty value, then test the package installation and
    // uninstall. Note that passing [null] value would be meaningless, so we
    // don't recognize it as a special one.
    //
    // Note that the host package can only be installed for a self-hosted
    // configuration, using bpkg configuration of the target type.
    //
    // Also note that the module package is always installed for a self-hosted
    // configuration (and never otherwise), using config.install.root
    // specified for ~build2 configuration.
    //
    // If present, indicates that the install, test installed, and uninstall
    // operations need to be tested.
    //
    optional<dir_path> install_root;

    // While building and running tests against the installation created
    // either from source or from the archive distribution package we will
    // make the bin/ subdirectory of config.install.root, if specified, the
    // first entry in the PATH environment variable, except for build system
    // modules which supposedly don't install any executables.
    //
    optional<dir_path> install_bin;

    auto config_install_root = [&step_args, &tgt_args] () -> optional<dir_path>
    {
      step_id s  (step_id::bpkg_target_create);
      step_id f1 (step_id::b_create);
      step_id f2 (step_id::bpkg_create);

      size_t n (19);
      auto space = [] (char c) {return c == ' ' || c == '\t';};

      for (const char* a: reverse_iterate (step_args (tgt_args, s, f1, f2)))
      {
        if (strncmp (a, "config.install.root", n) == 0 &&
            (a[n] == '=' || space (a[n])))
        {
          while (space (a[n])) ++n; // Skip spaces.
          if (a[n] == '=')     ++n; // Skip the equal sign.
          while (space (a[n])) ++n; // Skip spaces.

          // Note that the config.install.root variable value may potentially
          // be quoted.
          //
          return dir_path (unquote (a + n));
        }
      }

      return nullopt;
    };

    if ((target_pkg || selfhost) &&
        !bindist                 &&
        !step_disabled (step_id::bpkg_install))
    {
      if (!module_pkg)
      {
        install_root = config_install_root ();

        if (install_root)
          install_bin = *install_root / dir_path ("bin");
      }
      else
        install_root = dir_path ();
    }

    // Split external test packages into the runtime and build-time lists.
    //
    // Note that runtime and build-time test packages are always configured in
    // different bpkg configurations, since they can depend on different
    // versions of the same package.
    //
    small_vector<test_dependency, 1> runtime_tests;
    small_vector<test_dependency, 1> buildtime_tests;

    for (test_dependency& t: tm.tests)
    {
      if (t.buildtime)
        buildtime_tests.push_back (move (t));
      else if (target_pkg || selfhost)
        runtime_tests.push_back (move (t));
    }

    bool has_buildtime_tests (!buildtime_tests.empty ());
    bool has_runtime_tests   (!runtime_tests.empty ());

    // Abort if a target package has external build-time tests.
    //
    if (target_pkg && has_buildtime_tests)
    {
      fail_operation (
        add_result ("configure"),
        "build-time tests in package not marked with `requires: host`",
        result_status::abort);

      break;
    }

    // Create the required build configurations.
    //
    // Note that if this is a target package, then we intentionally do not
    // create host or module configuration letting the automatic private
    // configuration creation to take its course (since that would probably be
    // the most typical usage scenario).
    //
    // Also note that we may need a separate target configuration to build the
    // host package for installation. This is required to avoid a potential
    // conflict between the main package and a tool it may try to run during
    // the build. We also do the same for module packages which, while cannot
    // have build-time dependencies, could have private code generators. This
    // configuration needs to have the target type (so that it uses any
    // build-time dependencies from build-host/module configurations). Note
    // also that we currently only do this for self-hosted configuration
    // (since we don't install otherwise, see above).
    //
    dir_path target_conf  ("build");
    dir_path host_conf    ("build-host");
    dir_path module_conf  ("build-module");
    dir_path install_conf ("build-install");

    // Main package config.
    //
    const dir_path& main_pkg_conf (target_pkg ? target_conf :
                                   host_pkg   ? host_conf   :
                                                module_conf);

    // Create the target configuration if this is a target package or if the
    // host/module package has external build-time tests.
    //
    bool create_target (target_pkg || has_buildtime_tests);

    // Create the host configuration if this is a host package.
    //
    // Also create it for the module package with external build-time tests.
    // The idea is to be able to test a tool which might only be tested via
    // the module. To be precise, we need to check that the tests package has
    // a build-time dependency (on the tool) but that's not easy to do and so
    // we will create a host configuration if a module has any build-time
    // tests.
    //
    bool create_host (host_pkg || (module_pkg && has_buildtime_tests));

    // Create the module configuration if the package is a build system
    // module.
    //
    // Also create it for the host package with the external build-time tests,
    // so that a single build2 configuration is used for both target and host
    // packages (this is important in case they happen to use the same
    // module).
    //
    bool create_module (module_pkg || (host_pkg && has_buildtime_tests));

    // Create the configuration for installing the main package (potentially
    // as a part of generating binary distribution package) of the host or
    // module type, unless it's not supposed to be installed.
    //
    bool create_install (!target_pkg && (install_root || bindist));

    // Configuration where the package will be installed from.
    //
    dir_path effective_install_conf (
      rwd / (create_install ? install_conf : main_pkg_conf));

    // Root configuration through which we will be configuring the cluster
    // (note: does not necessarily match the main package type).
    //
    // In other words, this is configuration that will be specified for
    // bpkg-pkg-build as the current configuration (via -d). It must be the
    // configuration that links to all the other configurations, except
    // install.
    //
    // Note that the install configuration, if present, is either the
    // cluster's "second root" (for a host package) or is an independent
    // cluster (for a module package). In either case it needs to additionally
    // be specified as a current configuration on the command line.
    //
    const dir_path& root_conf (create_target ? target_conf :
                               create_host   ? host_conf   :
                                               module_conf);

    // Note that bpkg doesn't support configuring bootstrap module
    // dependents well, not distinguishing such modules from regular ones
    // (see pkg_configure() for details). Thus, we need to pass the
    // !config.import.* global override wherever required ourselves.
    //
    // Also note that since this override is global, it may only be specified
    // globally on the bpkg command line (as opposed to package-specific
    // overrides).
    //
    optional<string> bootstrap_import;

    if (bootstrap)
      bootstrap_import = "!config.import." + pkg_var + '=' +
        (rwd / main_pkg_conf).string ();

    // Configure.
    //
    {
      operation_result& r (add_result ("configure"));

      // Noop, just for the log record.
      //
      change_wd (trace, &r.log, rwd);

      // If we end up with multiple current configurations (root and install)
      // then when running the bpkg-pkg-build command we need to specify the
      // configuration for each package explicitly via --config-uuid.
      //
      // While it's tempting to use the --config-name option instead of
      // --config-uuid, that wouldn't work well for multiple current
      // configurations. For --config-name the configuration search is carried
      // out among configurations explicitly linked to the main configuration
      // only. That's in contrast to --config-uuid, when the whole
      // configuration cluster is searched (see bpkg-pkg-build implementation
      // for details).
      //
      // Let's not generate random UUIDs but use some predefined values which
      // we can easily recognize in the build logs.
      //
      const char* target_uuid  ("00000000-0000-0000-0000-000000000001");
      const char* host_uuid    ("00000000-0000-0000-0000-000000000002");
      const char* module_uuid  ("00000000-0000-0000-0000-000000000003");
      const char* install_uuid ("00000000-0000-0000-0000-000000000004");

      // Let's however distinguish the target package as a simple common case
      // and simplify the configuration creation and packages configuration
      // commands making them more readable in the build log. For this simple
      // case only one configuration needs to be created explicitly and so it
      // doesn't need the UUID. Also there is no need in any package-specific
      // options for the bpkg-pkg-build command in this case.
      //
      // Create the target configuration.
      //
      // bpkg create <env-modules> <env-config-args>
      //                           <tgt-config-args>
      //                           <pkg-config-args>
      //
      if (create_target)
      {
        step_id b  (step_id::bpkg_create);        // Breakpoint.
        step_id s  (step_id::bpkg_target_create); // Step.
        step_id f1 (step_id::b_create);           // First fallback.
        step_id f2 (step_id::bpkg_create);        // Second fallback.

        r.status |= run_bpkg (
          b,
          trace, r.log, wre,
          bkp_step, bkp_status, last_cmd,
          "-V",
          "create",
          "-d", target_conf,
          !target_pkg ? cstrings ({"--uuid", target_uuid}) : cstrings (),
          step_args (modules,  s, f1, f2),
          step_args (env_args, s, f1, f2),
          step_args (tgt_args, s, f1, f2),
          step_args (pkg_args, s, f1, f2));

        if (!r.status)
          break;
      }

      // Create the host configurations.
      //
      if (create_host)
      {
        step_id b (step_id::bpkg_create);

        if (host_pkg && selfhost)
        {
          // Create the host configuration.
          //
          {
            step_id s  (step_id::bpkg_host_create);
            step_id f1 (step_id::b_create);
            step_id f2 (step_id::bpkg_create);

            // bpkg create --type host <env-modules> <env-config-args>
            //                                       <tgt-config-args>
            //                                       <pkg-config-args>
            //
            r.status |= run_bpkg (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              "-V",
              "create",
              "-d", host_conf,
              "--type", "host",
              "--uuid", host_uuid,
              step_args (modules,  s, f1, f2),
              step_args (env_args, s, f1, f2),
              step_args (tgt_args, s, f1, f2),
              step_args (pkg_args, s, f1, f2));

            if (!r.status)
              break;
          }

          // Create the install configuration.
          //
          // bpkg create <env-modules> <env-config-args>
          //                           <tgt-config-args>
          //                           <pkg-config-args>
          //
          if (create_install)
          {
            step_id s  (step_id::bpkg_target_create);
            step_id f1 (step_id::b_create);
            step_id f2 (step_id::bpkg_create);

            r.status |= run_bpkg (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              "-V",
              "create",
              "-d", install_conf,
              "--uuid", install_uuid,
              step_args (modules,  s, f1, f2),
              step_args (env_args, s, f1, f2),
              step_args (tgt_args, s, f1, f2),
              step_args (pkg_args, s, f1, f2));

            if (!r.status)
              break;
          }
        }
        else
        {
          // b create(<dir>) config.config.load=~host
          //
          // Note also that we suppress warnings about unused config.* values.
          //
          r.status |= run_b (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-V",
            "create(" + host_conf.representation () + ",cc)",
            "config.config.load=~host",
            "config.config.persist+='config.*'@unused=drop");

          if (!r.status)
            break;

          // bpkg create --existing --type host
          //
          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "create",
            "--existing",
            "-d", host_conf,
            "--type", "host",
            "--uuid", host_uuid);

          if (!r.status)
            break;
        }
      }

      // Create the module configurations.
      //
      if (create_module)
      {
        step_id b (step_id::bpkg_create);

        // Create the module configuration.
        //
        {
          // b create(<dir>) config.config.load=~build2 [<env-config-args>
          //                                             <tgt-config-args>
          //                                             <pkg-config-args>]
          //
          // Note also that we suppress warnings about unused config.* values.
          //
          // What if a module wants to use CLI? The current thinking is that
          // we will be "whitelisting" base (i.e., those that can plausibly be
          // used by multiple modules) libraries and tools for use by build
          // system modules. So if and when we whitelist CLI, we will add it
          // here, next to cc.
          //
          string  mods;
          cstrings eas;
          cstrings cas;
          cstrings pas;

          if (module_pkg && selfhost)
          {
            step_id s (step_id::bpkg_module_create);

            for (const char* m: step_args (modules, s))
            {
              if (!mods.empty ())
                mods += ' ';

              mods += m;
            }

            eas = step_args (env_args, s);
            cas = step_args (tgt_args, s);
            pas = step_args (pkg_args, s);
          }
          else
            mods = "cc";

          r.status |= run_b (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-V",
            "create(" + module_conf.representation () + ',' + mods + ')',
            "config.config.load=~build2",
            "config.config.persist+='config.*'@unused=drop",
            eas,
            cas,
            pas);

          if (!r.status)
            break;

          // bpkg create --existing --type build2
          //
          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "create",
            "--existing",
            "-d", module_conf,
            "--type", "build2",
            "--uuid", module_uuid);

          if (!r.status)
            break;
        }

        // Create the install configuration.
        //
        if (create_install && module_pkg)
        {
          step_id s (step_id::bpkg_module_create);

          string mods;
          for (const char* m: step_args (modules, s))
          {
            if (!mods.empty ())
              mods += ' ';

            mods += m;
          }

          // b create(<dir>) config.config.load=~build2 [<env-config-args>
          //                                             <tgt-config-args>
          //                                             <pkg-config-args>]
          //
          r.status |= run_b (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-V",
            "create(" + install_conf.representation () + ',' + mods + ')',
            "config.config.load=~build2",
            "config.config.persist+='config.*'@unused=drop",
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s));

          if (!r.status)
            break;

          // bpkg create --existing
          //
          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "create",
            "--existing",
            "-d", install_conf,
            "--uuid", install_uuid);

          if (!r.status)
            break;
        }
      }

      // Link the configurations.
      //
      // bpkg link -d <dir> <dir>
      //
      {
        step_id b (step_id::bpkg_link);

        if (create_target)
        {
          if (create_host)
          {
            r.status |= run_bpkg (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              "-v",
              "link",
              "-d", target_conf,
              host_conf);

            if (!r.status)
              break;
          }

          if (create_module)
          {
            r.status |= run_bpkg (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              "-v",
              "link",
              "-d", target_conf,
              module_conf);

            if (!r.status)
              break;
          }
        }

        if (create_host)
        {
          if (create_module)
          {
            r.status |= run_bpkg (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              "-v",
              "link",
              "-d", host_conf,
              module_conf);

            if (!r.status)
              break;
          }
        }

        // Link the install configuration only for the host package. Note that
        // the module package may not have build-time dependencies and so
        // doesn't need configurations for them.
        //
        if (create_install && host_pkg)
        {
          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "link",
            "-d", install_conf,
            host_conf);

          if (!r.status)
            break;

          if (create_module)
          {
            r.status |= run_bpkg (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              "-v",
              "link",
              "-d", install_conf,
              module_conf);

            if (!r.status)
              break;
          }
        }
      }

      // Fetch repositories into the main package configuration, the target
      // configuration for external build-time tests, if any, and the install
      // configuration, if present.
      //
      // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args>
      //          <repository-url>
      //
      {
        step_id b (step_id::bpkg_configure_add);
        step_id s (step_id::bpkg_configure_add);

        r.status |= run_bpkg (
          b,
          trace, r.log, wre,
          bkp_step, bkp_status, last_cmd,
          "-v",
          "add",
          "-d", main_pkg_conf,
          step_args (env_args, s),
          step_args (tgt_args, s),
          step_args (pkg_args, s),
          repo);

        if (!r.status)
          break;
      }

      // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args>
      //            <trust-options>
      //
      {
        step_id b (step_id::bpkg_configure_fetch);
        step_id s (step_id::bpkg_configure_fetch);

        r.status |= run_bpkg (
          b,
          trace, r.log, wre,
          bkp_step, bkp_status, last_cmd,
          "-v",
          "fetch",
          "-d", main_pkg_conf,
          step_args (env_args, s),
          step_args (tgt_args, s),
          step_args (pkg_args, s),
          trust_ops);

        if (!r.status)
          break;
      }

      if (create_install)
      {
        // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args>
        //          <repository-url>
        //
        {
          step_id b (step_id::bpkg_configure_add);
          step_id s (step_id::bpkg_configure_add);

          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "add",
            "-d", install_conf,
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            repo);

          if (!r.status)
            break;
        }

        // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args>
        //            <trust-options>
        //
        {
          step_id b (step_id::bpkg_configure_fetch);
          step_id s (step_id::bpkg_configure_fetch);

          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "fetch",
            "-d", install_conf,
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            trust_ops);

          if (!r.status)
            break;
        }
      }

      if (has_buildtime_tests)
      {
        // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args>
        //          <repository-url>
        //
        {
          step_id b (step_id::bpkg_configure_add);
          step_id s (step_id::bpkg_configure_add);

          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "add",
            "-d", target_conf,
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            repo);

          if (!r.status)
            break;
        }

        // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args>
        //            <trust-options>
        //
        {
          step_id b (step_id::bpkg_configure_fetch);
          step_id s (step_id::bpkg_configure_fetch);

          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "fetch",
            "-d", target_conf,
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            trust_ops);

          if (!r.status)
            break;
        }
      }

      // Configure all the packages using a single bpkg-pkg-build command.
      //
      // First, prepare the common and package arguments.
      //
      // If no variables are specified in the package configuration, then add
      // the config.<pkg>.develop=false variable for the main package instead
      // to trigger its package skeleton creation and loading. Also add this
      // variable for the external test packages for the same purpose. This
      // way we make sure that these packages can be used as dependencies of
      // dependents with configuration clauses.
      //
      // Also add the dependency packages specified in the package
      // configuration, if any, to configurations where the main package is
      // being configured.
      //
      // Should we also add the dependency packages to configurations where
      // the test packages are being configured? It feels like we shouldn't.
      // Moreover, in the future we may decide to support specifying tests
      // package configuration in the tests manifest value or some such. In
      // this case a test package may have its own dependencies to be
      // configured. What we could probably do now, is to never share a bpkg
      // configuration between the main package and the tests packages if we
      // configure any dependencies in it. Note that such dependencies may
      // potentially be unsatisfactory for the test packages (unsatisfactory
      // version, etc). This, however, seems rather far fetched so let's keep
      // it simple for now.
      //
      strings common_args;
      strings pkgs;

      if (target_pkg) // The simple common case (see above)?
      {
        // The overall command looks like this (but some parts may be omitted):
        //
        // bpkg build --configure-only <env-config-args>
        //                             <tgt-config-args>
        //                             <pkg-config-args>
        //                             <pkg-config-opts>
        //      --
        //      { <pkg-config-vars>|config.<pkg-name>.develop=false          }+ <pkg>
        //      { <rtt-config-vars>|config.<runtime-test-name>.develop=false }+ <runtime-test>...
        //      { <dep-config-vars>                                          }+ <main-dep>...
        //                                                                      <main-dep>...
        //                                                                      <glob-dep>...
        //
        step_id s  (step_id::bpkg_target_configure_build);
        step_id f1 (step_id::b_configure);
        step_id f2 (step_id::bpkg_configure_build);

        cstrings eas (step_args (env_args, s, f1, f2));
        cstrings cas (step_args (tgt_args, s, f1, f2));
        cstrings pas (step_args (pkg_args, s, f1, f2));

        common_args.push_back ("--checkout-root");
        common_args.push_back (dist_root.string ());

        common_args.insert (common_args.end (), eas.begin (), eas.end ());
        common_args.insert (common_args.end (), cas.begin (), cas.end ());
        common_args.insert (common_args.end (), pas.begin (), pas.end ());

        // Add the main package.
        //
        pkgs.push_back ("{");

        // @@ config.<pkg>.develop=false
        //
        // Only add the config.<pkg>.develop variable if there are no package
        // configuration variables specified.
        //
        auto i (pkg_config_pkgs.find (tm.name.string ()));

        if (!pkg_config_vars.empty () || i != pkg_config_pkgs.end ())
        {
          if (!pkg_config_vars.empty ())
            pkgs.insert (pkgs.end (),
                         pkg_config_vars.begin (), pkg_config_vars.end ());

          if (i != pkg_config_pkgs.end ())
            pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ());
        }
#if 1
        else
          pkgs.push_back ("config." + pkg_var + ".develop=false");
#endif

        pkgs.push_back ("}+");

        pkgs.push_back (pkg_rev);

        // Add the runtime test packages.
        //
        for (const auto& t: runtime_tests)
        {
          pkgs.push_back ("{");

          // @@ config.<pkg>.develop=false
          //
          // Only add the config.<pkg>.develop variable if there are no
          // package configuration variables specified.
          //
          auto i (pkg_config_pkgs.find (t.name.string ()));

          if (i != pkg_config_pkgs.end ())
            pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ());
#if 1
          else
            pkgs.push_back ("config." + t.name.variable () + ".develop=false");
#endif

          pkgs.push_back ("}+");

          // Add test dependency package constraints (for example
          // 'bar > 1.0.0').
          //
          pkgs.push_back (t.string ());
        }

        // Add the main package dependencies.
        //
        for (const pair<string, strings>& d: pkg_config_main_deps)
        {
          if (!d.second.empty ())
          {
            pkgs.push_back ("{");
            pkgs.insert (pkgs.end (), d.second.begin (), d.second.end ());
            pkgs.push_back ("}+");
          }

          pkgs.push_back (d.first);
        }
      }
      else
      {
        // The overall command looks like this (but some parts may be omitted):
        //
        // bpkg build --configure-only            <env-config-args>
        //                                        <tgt-config-args>
        //                                        <pkg-config-args>
        //                                        <pkg-config-opts>
        //      --
        //      { <build-config>                  <env-config-args>
        //                                        <tgt-config-args>
        //                                        <pkg-config-args>
        //                                        <pkg-config-vars>|config.<pkg-name>.develop=false            }+   <pkg>
        //
        //      { <build-config>                  <env-config-args>
        //                                        <tgt-config-args>
        //                                        <pkg-config-args>
        //                                        <rtt-config-vars>|config.<runtime-test-name>.develop=false   }+   <runtime-test>...
        //
        //      { <install-config>                <env-config-args>
        //                                        <tgt-config-args>
        //                                        <pkg-config-args>
        //                                        <pkg-config-vars>                                            }+   <pkg>
        //
        //      { <target-config>                 <env-config-args>
        //                                        <tgt-config-args>
        //                                        <pkg-config-args>
        //                                        <btt-config-vars>|config.<buildtime-test-name>.develop=false }+   <buildtime-test>...
        //
        //      { <build-config> <install-config> <dep-config-vars>                                            }+   <main-dep>...
        //      { <build-config> <install-config>                                                              }+ { <main-dep>... }
        //                                                                                                          <glob-dep>...
        //

        // Main package configuration name.
        //
        const char* conf_uuid (host_pkg ? host_uuid : module_uuid);

        // Add the main package args.
        //
        // Also add the external runtime test packages here since they share
        // the configuration directory with the main package.
        //
        {
          step_id s (target_pkg ? step_id::bpkg_target_configure_build :
                     host_pkg   ? step_id::bpkg_host_configure_build   :
                                  step_id::bpkg_module_configure_build);

          step_id f1 (step_id::b_configure);
          step_id f2 (step_id::bpkg_configure_build);

          cstrings eas (step_args (env_args, s, f1, f2));
          cstrings cas (step_args (tgt_args, s, f1, f2));
          cstrings pas (step_args (pkg_args, s, f1, f2));

          // Add the main package.
          //
          {
            pkgs.push_back ("{");

            pkgs.push_back ("--config-uuid");
            pkgs.push_back (conf_uuid);

            pkgs.push_back ("--checkout-root");
            pkgs.push_back (dist_root.string ());

            pkgs.insert (pkgs.end (), eas.begin (), eas.end ());
            pkgs.insert (pkgs.end (), cas.begin (), cas.end ());
            pkgs.insert (pkgs.end (), pas.begin (), pas.end ());

            // @@ config.<pkg>.develop=false
            //
            // Only add the config.<pkg>.develop variable if there are no
            // package configuration variables specified.
            //
            auto i (pkg_config_pkgs.find (tm.name.string ()));

            if (!pkg_config_vars.empty () || i != pkg_config_pkgs.end ())
            {
              if (!pkg_config_vars.empty ())
                pkgs.insert (pkgs.end (),
                             pkg_config_vars.begin (), pkg_config_vars.end ());

              if (i != pkg_config_pkgs.end ())
                pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ());
            }
#if 1
            else
              pkgs.push_back ("config." + pkg_var + ".develop=false");
#endif

            pkgs.push_back ("}+");

            pkgs.push_back (pkg_rev);
          }

          // Add the runtime test packages.
          //
          for (const auto& t: runtime_tests)
          {
            pkgs.push_back ("{");

            pkgs.push_back ("--config-uuid");
            pkgs.push_back (conf_uuid);

            pkgs.push_back ("--checkout-root");
            pkgs.push_back (dist_root.string ());

            pkgs.insert (pkgs.end (), eas.begin (), eas.end ());
            pkgs.insert (pkgs.end (), cas.begin (), cas.end ());
            pkgs.insert (pkgs.end (), pas.begin (), pas.end ());

            // @@ config.<pkg>.develop=false
            //
            // Only add the config.<pkg>.develop variable if there are no
            // package configuration variables specified.
            //
            auto i (pkg_config_pkgs.find (t.name.string ()));

            if (i != pkg_config_pkgs.end ())
              pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ());
#if 1
            else
              pkgs.push_back ("config." + t.name.variable () + ".develop=false");
#endif

            pkgs.push_back ("}+");

            pkgs.push_back (t.string ());
          }
        }

        // Add the main package configured in the install configuration and
        // the external build-time test packages.
        //
        {
          step_id s  (step_id::bpkg_target_configure_build);
          step_id f1 (step_id::b_configure);
          step_id f2 (step_id::bpkg_configure_build);

          cstrings eas (step_args (env_args, s, f1, f2));
          cstrings cas (step_args (tgt_args, s, f1, f2));
          cstrings pas (step_args (pkg_args, s, f1, f2));

          // Add the main package.
          //
          if (create_install)
          {
            common_args.push_back ("-d");
            common_args.push_back (install_conf.string ());

            pkgs.push_back ("{");

            pkgs.push_back ("--config-uuid");
            pkgs.push_back (install_uuid);

            // Note that we do another re-distribution (with a separate
            // --checkout-root) in case the package is missing file that
            // are only used during installation.
            //
            pkgs.push_back ("--checkout-root");
            pkgs.push_back (dist_install_root.string ());

            pkgs.insert (pkgs.end (), eas.begin (), eas.end ());
            pkgs.insert (pkgs.end (), cas.begin (), cas.end ());
            pkgs.insert (pkgs.end (), pas.begin (), pas.end ());

            pkgs.insert (pkgs.end (),
                         pkg_config_vars.begin (), pkg_config_vars.end ());

            auto i (pkg_config_pkgs.find (tm.name.string ()));

            if (i != pkg_config_pkgs.end ())
              pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ());

            pkgs.push_back ("}+");

            pkgs.push_back (pkg_rev);
          }

          // Add the build-time test packages.
          //
          // @@ config.<pkg>.develop=false
          //
          for (const auto& t: buildtime_tests)
          {
            pkgs.push_back ("{");

            pkgs.push_back ("--config-uuid");
            pkgs.push_back (target_uuid);

            pkgs.push_back ("--checkout-root");
            pkgs.push_back (dist_root.string ());

            pkgs.insert (pkgs.end (), eas.begin (), eas.end ());
            pkgs.insert (pkgs.end (), cas.begin (), cas.end ());
            pkgs.insert (pkgs.end (), pas.begin (), pas.end ());

            // @@ config.<pkg>.develop=false
            //
            // Only add the config.<pkg>.develop variable if there are no
            // package configuration variables specified.
            //
            auto i (pkg_config_pkgs.find (t.name.string ()));

            if (i != pkg_config_pkgs.end ())
              pkgs.insert (pkgs.end (), i->second.begin (), i->second.end ());
#if 1
            else
              pkgs.push_back ("config." + t.name.variable () + ".develop=false");
#endif

            pkgs.push_back ("}+");

            // Strip the build-time mark.
            //
            pkgs.push_back (t.dependency::string ());
          }
        }

        // Add the main package dependencies to those configurations where
        // the main package is configured.
        //
        {
          // Add dependencies which have some configuration variables
          // specified and count the number of others.
          //
          size_t no_vars (0);
          for (const pair<string, strings>& d: pkg_config_main_deps)
          {
            if (!d.second.empty ())
            {
              pkgs.push_back ("{");

              pkgs.push_back ("--config-uuid");
              pkgs.push_back (conf_uuid);

              if (create_install)
              {
                pkgs.push_back ("--config-uuid");
                pkgs.push_back (install_uuid);
              }

              pkgs.insert (pkgs.end (), d.second.begin (), d.second.end ());

              pkgs.push_back ("}+");

              pkgs.push_back (d.first);
            }
            else
              ++no_vars;
          }

          // Add dependencies which have no configuration variables specified.
          //
          if (no_vars != 0)
          {
            pkgs.push_back ("{");

            pkgs.push_back ("--config-uuid");
            pkgs.push_back (conf_uuid);

            if (create_install)
            {
              pkgs.push_back ("--config-uuid");
              pkgs.push_back (install_uuid);
            }

            pkgs.push_back ("}+");

            if (no_vars != 1)
              pkgs.push_back ("{");

            for (const pair<string, strings>& d: pkg_config_main_deps)
            {
              if (d.second.empty ())
                pkgs.push_back (d.first);
            }

            if (no_vars != 1)
              pkgs.push_back ("}");
          }
        }
      }

      // Add the global system dependencies.
      //
      for (const pair<string, strings>& d: pkg_config_glob_deps)
        pkgs.push_back (d.first);

      // Finally, configure all the packages.
      //
      {
        step_id b (step_id::bpkg_configure_build);
        step_id s (step_id::bpkg_global_configure_build);

        optional<string> dependency_checksum;

        // Only log configuration UUIDs if they are specified on the command
        // line.
        //
        function<pre_run_function> log_uuids (
          [&r, &trace,
           target_uuid, host_uuid, module_uuid, install_uuid,
           target_pkg] ()
          {
            if (!target_pkg)
            {
              auto log = [&r, &trace] (const char* uuid, const char* name)
              {
                string s (uuid);
                s += " - ";
                s += name;

                l3 ([&]{trace << s;});

                r.log += comment_begin;
                r.log += ' ';
                r.log += s;
                r.log += '\n';
              };

              r.log += comment_begin;
              r.log += '\n';

              log (target_uuid,  "target");
              log (host_uuid,    "host");
              log (module_uuid,  "module");
              log (install_uuid, "install");

              r.log += comment_begin;
              r.log += '\n';
            }
          });

        r.status |= run_bpkg (
          b,
          trace, r.log,
          log_uuids,
          dependency_checksum,
          wre,
          bkp_step, bkp_status, last_cmd,
          "-v",
          "build",
          "--configure-only",
          "--rebuild-checksum",
          tm.dependency_checksum ? *tm.dependency_checksum : "",
          "--yes",
          "-d", root_conf,
          step_args (env_args, s),
          step_args (tgt_args, s),
          step_args (pkg_args, s),
          common_args,
          pkg_config_opts,
          (has_runtime_tests || has_buildtime_tests
           ? bootstrap_import
           : nullopt),
          "--",
          pkgs);

        // The dependency checksum is tricky, here are the possibilities:
        //
        // - absent: bpkg terminated abnormally (or was not executed due to
        //           a breakpoint) -- nothing to do here.
        //
        // - empty: bpkg terminated normally with error before calculating the
        //          checksum -- nothing to do here either.
        //
        // - one line: bpkg checksum that we want.
        //
        // - many lines: someone else (e.g., buildfile) printed to stdout,
        //               which we consider an error.
        //
        if (dependency_checksum && !dependency_checksum->empty ())
        {
          string& s (*dependency_checksum);

          // Make sure that the output contains a single line, and bail out
          // with the error status if that's not the case.
          //
          if (s.find ('\n') == s.size () - 1)
          {
            s.pop_back ();

            // If the dependency checksum didn't change, then save it to the
            // result manifest, clean the logs and bail out with the skip
            // result status.
            //
            if (tm.dependency_checksum && *tm.dependency_checksum == s)
            {
              l3 ([&]{trace << "skip";});

              rm.status = result_status::skip;
              rm.dependency_checksum = move (s);
              rm.results.clear ();
              break;
            }

            // Save the (new) dependency checksum to the result manifest.
            //
            // Also note that we save the checksum if bpkg failed after the
            // checksum was printed. As a result, we won't be rebuilding the
            // package until the error is fixed (in a package or worker) and
            // the checksum changes, which feels like a proper behavior.
            //
            rm.dependency_checksum = move (s);
          }
          else
            fail_operation (r,
                            "unexpected bpkg output:\n'" + s + '\'',
                            result_status::error);
        }

        if (!r.status)
          break;
      }

      // Redistribute the main package in both build and install
      // configurations, if required (test packages will be handled later).
      //
      if (exists (dist_src))
      {
        change_wd (trace, &r.log, main_pkg_conf);

        step_id b (step_id::bpkg_configure_build);

        if (!redist (b, r, dist_root, pkg_dir))
          break;
      }

      if (exists (dist_install_src))
      {
        change_wd (trace, &r.log, rwd / install_conf);

        step_id b (step_id::bpkg_configure_build);

        if (!redist (b, r, dist_install_root, pkg_dir))
          break;
      }

      rm.status |= r.status;
    }

#ifdef _WIN32
    // Give Windows a chance to (presumably) scan any files we may have just
    // unpacked. Failed that, if we try to overwrite any such file (e.g., a
    // generated header) we may end up with a permission denied error. Note
    // also that this is in addition to the 2 seconds retry we have in our
    // fdopen() implementation, which is not always enough.
    //
    Sleep (5000);
#endif

    auto fail_unreached_breakpoint = [&bkp_step, &fail_operation]
                                     (operation_result& r)
    {
      assert (bkp_step);

      fail_operation (r,
                      "interactive build breakpoint " +
                      to_string (*bkp_step) + " cannot be reached",
                      result_status::abort);
    };

    // Note that if the bpkg.update step is disabled, we also skip all the
    // test and install related steps.
    //
    if (!step_disabled (step_id::bpkg_update))
    {
      // Update the main package.
      //
      {
        operation_result& r (add_result ("update"));

        change_wd (trace, &r.log, rwd / main_pkg_conf);

        // bpkg update <env-config-args> <tgt-config-args> <pkg-config-args>
        //             <package-name>
        //
        step_id b (step_id::bpkg_update);
        step_id s (step_id::bpkg_update);

        r.status |= run_bpkg (
          b,
          trace, r.log, wre,
          bkp_step, bkp_status, last_cmd,
          "-v",
          "update",
          step_args (env_args, s),
          step_args (tgt_args, s),
          step_args (pkg_args, s),
          pkg);

        if (!r.status)
          break;

        rm.status |= r.status;
      }

      // Re-distribute if comes from a version control-based repository,
      // update, and test external test packages in the bpkg configuration in
      // the current working directory. Optionally pass the config.import.*
      // variable override and/or set the environment variables for the bpkg
      // processes. Return true if all operations for all packages succeeded.
      //
      // Pass true as the installed argument to use the test separate installed
      // phase step ids (bpkg.test-separate-installed.*) and the test separate
      // phase step ids (bpkg.test-separate.*) otherwise. In both cases fall
      // back to the main phase step ids (bpkg.*) when no environment/
      // configuration arguments are specified for them.
      //
      auto test = [&trace, &wre,
                   &bkp_step, &bkp_status, &last_cmd,
                   &step_args, &env_args, &tgt_args, &pkg_args,
                   &bootstrap_import,
                   &redist]
        (operation_result& r,
         const small_vector<test_dependency, 1>& tests,
         const dir_path& dist_root,
         bool installed,
         bool update_only,
         const small_vector<string, 1>& envvars = {})
        {
          const optional<string>& import (!installed
                                          ? bootstrap_import
                                          : nullopt);

          for (const test_dependency& td: tests)
          {
            const string& pkg (td.name.string ());

            // Re-distribute.
            //
            if (exists (dist_root))
            {
              // Note that re-distributing the test package is a bit tricky
              // since we don't know its version and so cannot deduce its
              // source directory name easily. We could potentially run the
              // bpkg-status command after the package is configured and parse
              // the output to obtain the version. Let's, however, keep it
              // simple and find the source directory using the package
              // directory name pattern.
              //
              try
              {
                dir_path pkg_dir;

                // Note: doesn't follow symlinks.
                //
                path_search (dir_path (pkg + "-*/"),
                             [&pkg_dir] (path&& pe, const string&, bool interm)
                             {
                               if (!interm)
                                 pkg_dir = path_cast<dir_path> (move (pe));

                               return interm;
                             },
                             dist_root,
                             path_match_flags::none);

                if (!pkg_dir.empty ())
                {
                  step_id b (
                    installed
                    ? step_id::bpkg_test_separate_installed_configure_build
                    : step_id::bpkg_configure_build);

                  if (!redist (b, r, dist_root, pkg_dir, import, envvars))
                    return false;
                }
              }
              catch (const system_error& e)
              {
                fail << "unable to scan directory " << dist_root << ": " << e;
              }
            }

            // Update.
            //
            // bpkg update <env-config-args> <tgt-config-args> <pkg-config-args>
            //             <package-name>
            //
            {
              step_id b (installed
                         ? step_id::bpkg_test_separate_installed_update
                         : step_id::bpkg_test_separate_update);

              step_id s (b);

              step_id f (step_id::bpkg_update);

              r.status |= run_bpkg (
                b,
                envvars,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-v",
                "update",
                step_args (env_args, s, f),
                step_args (tgt_args, s, f),
                step_args (pkg_args, s, f),
                import,
                pkg);

              if (!r.status)
                return false;
            }

            // Test.
            //
            // Note that we assume that the package supports the test operation
            // since this is its main purpose.
            //
            // bpkg test <env-config-args> <tgt-config-args> <pkg-config-args>
            //           <package-name>
            //
            if (!update_only)
            {
              step_id b (installed
                         ? step_id::bpkg_test_separate_installed_test
                         : step_id::bpkg_test_separate_test);

              step_id s (b);

              step_id f (step_id::bpkg_test);

              r.status |= run_bpkg (
                b,
                envvars,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-v",
                "test",
                "--package-cwd", // See above for details.
                step_args (env_args, s, f),
                step_args (tgt_args, s, f),
                step_args (pkg_args, s, f),
                import,
                pkg);

              if (!r.status)
                return false;
            }
          }

          return true;
        };

      // Test the main package.
      //
      b_project_info prj (
        prj_info (pkg_dir,
                  b_info_flags::ext_mods | b_info_flags::subprojects,
                  "project"));

      // Run the internal tests if the test operation is supported by the
      // project but only for the target package or if the configuration is
      // self-hosted.
      //
      bool has_internal_tests ((target_pkg || selfhost) &&
                               find (prj.operations.begin (),
                                     prj.operations.end (),
                                     "test") != prj.operations.end ());

      if (has_internal_tests || has_runtime_tests || has_buildtime_tests)
      {
        operation_result& r (add_result ("test"));

        // Run internal tests.
        //
        if (has_internal_tests && !step_disabled (step_id::bpkg_test))
        {
          // Use --package-cwd to help ported to build2 third-party packages a
          // bit (see bpkg-pkg-test(1) for details).
          //
          // Note that internal tests that load the module itself don't make
          // much sense, thus we don't pass the config.import.*  variable on
          // the command line for modules that require bootstrap.
          //
          // bpkg test <env-config-args> <tgt-config-args> <pkg-config-args>
          //           <package-name>
          //
          step_id b (step_id::bpkg_test);
          step_id s (step_id::bpkg_test);

          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "test",
            "--package-cwd",
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            pkg);

          if (!r.status)
            break;
        }
        //
        // Fail if the breakpoint refers to the bpkg.test step but the package
        // has no internal tests or this step is disabled.
        //
        else if (bkp_step && *bkp_step == step_id::bpkg_test)
        {
          fail_unreached_breakpoint (r);
          break;
        }

        // External tests.
        //
        // Note that if the bpkg.test-separate.update step is disabled, we
        // also skip bpkg.test-separate.test.
        //
        if ((has_runtime_tests || has_buildtime_tests) &&
            !step_disabled (step_id::bpkg_test_separate_update))
        {
          bool update_only (step_disabled (step_id::bpkg_test_separate_test));

          // Fail if the breakpoint refers to the bpkg.test-separate.test step
          // but this step is disabled.
          //
          if (update_only &&
              bkp_step    &&
              *bkp_step == step_id::bpkg_test_separate_test)
          {
            fail_unreached_breakpoint (r);
            break;
          }

          // External runtime tests.
          //
          // Note that we assume that these packages belong to the dependent
          // package's repository or its complement repositories, recursively.
          // Thus, we test them in the configuration used to build the
          // dependent package.
          //
          if (has_runtime_tests)
          {
            if (!test (r,
                       runtime_tests,
                       dist_root,
                       false /* installed */,
                       update_only))
              break;
          }

          // External build-time tests.
          //
          if (has_buildtime_tests)
          {
            change_wd (trace, &r.log, rwd / target_conf);

            if (!test (r,
                       buildtime_tests,
                       dist_root,
                       false /* installed */,
                       update_only))
              break;
          }
        }
        //
        // Fail if the breakpoint refers to some of the bpkg.test-separate.*
        // steps but the package either has no external tests or the
        // bpkg.test-separate.update step is disabled.
        //
        else if (bkp_step                                        &&
                 *bkp_step >= step_id::bpkg_test_separate_update &&
                 *bkp_step <= step_id::bpkg_test_separate_test)
        {
          fail_unreached_breakpoint (r);
          break;
        }

        rm.status |= r.status;
      }
      //
      // Fail if the breakpoint refers to some of the test steps but the
      // package has no tests.
      //
      else if (bkp_step                        &&
               *bkp_step >= step_id::bpkg_test &&
               *bkp_step <= step_id::bpkg_test_separate_test)
      {
        fail_unreached_breakpoint (add_result ("test"));
        break;
      }

      // Install from source.
      //
      if (install_root)
      {
        operation_result& r (add_result ("install"));

        change_wd (trace, &r.log, effective_install_conf);

        // Note that for a host or module package we don't need the target
        // configuration anymore, if present. So let's free up the space a
        // little bit.
        //
        if (!target_pkg && create_target)
          rm_r (trace, &r.log, rwd / target_conf);

        // Install.
        //
        {
          // bpkg install <env-config-args> <tgt-config-args> <pkg-config-args>
          //              <package-name>
          //
          step_id b (step_id::bpkg_install);
          step_id s (step_id::bpkg_install);

          r.status |= run_bpkg (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "-v",
            "install",
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            pkg);

          if (!r.status)
            break;
        }

        // Run ldconfig.
        //
        if (step_enabled (step_id::bbot_install_ldconfig))
        {
          // sudo ldconfig <env-config-args> <tgt-config-args> <pkg-config-args>
          //
          step_id b (step_id::bbot_install_ldconfig);
          step_id s (step_id::bbot_install_ldconfig);

          r.status |= run_ldconfig (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s));

          if (!r.status)
            break;
        }
        //
        // Fail if the breakpoint refers to the bbot.install.ldconfig step but
        // this step is disabled.
        //
        else if (bkp_step && *bkp_step == step_id::bbot_install_ldconfig)
        {
          fail_unreached_breakpoint (r);
          break;
        }

        rm.status |= r.status;
      }
      //
      // Fail if the breakpoint refers to the bpkg.install related steps but
      // the package is not supposed to be installed from source.
      //
      else if (bkp_step                           &&
               *bkp_step >= step_id::bpkg_install &&
               *bkp_step <= step_id::bbot_install_ldconfig)
      {
        fail_unreached_breakpoint (add_result ("install"));
        break;
      }

      // Generate the binary distribution package.
      //
      // Note that if bbot.bindist.upload step is enabled, it makes sense to
      // only copy the generated binary distribution files to the
      // upload/bindist/<distribution>/ directory after the binary
      // distribution packages are testes, i.e. after the potential
      // bbot.sys-uninstall.* steps.
      //
      // The following bindist_* structures contain a subset of members of the
      // corresponding structures described in the STRUCTURED RESULT section
      // of the bpkg-pkg-bindist(1) man page. Note: needed later for
      // uninstall, upload.
      //
      struct bindist_os_release
      {
        string name_id;
        optional<string> version_id;
      };

      struct bindist_file
      {
        string type;
        bbot::path path;              // Absolute and normalized.
        optional<string> system_name;
      };

      struct bindist_package
      {
        string name;
        string version;
        optional<string> system_version;
        vector<bindist_file> files;
      };

      struct bindist_result_type
      {
        string distribution;
        string architecture;
        bindist_os_release os_release;
        bindist_package package;
        vector<bindist_package> dependencies;
      };

      bindist_result_type bindist_result;

      const dir_path& bindist_conf (
        create_install ? install_conf : main_pkg_conf);

      // Make it absolute for the sake of diagnostics.
      //
      path bindist_result_file (rwd / bindist_conf / "bindist-result.json");

      if (bindist)
      {
        operation_result& r (add_result ("bindist"));

        // Fail if the breakpoint refers to a bpkg.bindist.* step but this
        // step differs from the enabled one.
        //
        if (bkp_step                                     &&
            (*bkp_step == step_id::bpkg_bindist_debian ||
             *bkp_step == step_id::bpkg_bindist_fedora ||
             *bkp_step == step_id::bpkg_bindist_archive) &&
            *bkp_step != *bindist)
        {
          fail_unreached_breakpoint (r);
          break;
        }

        change_wd (trace, &r.log, rwd);

        // Note that for a host or module package we don't need the target
        // configuration anymore, if present. So let's free up the space a
        // little bit.
        //
        if (!target_pkg && create_target)
          rm_r (trace, &r.log, rwd / target_conf);

        string distribution;
        dir_path output_root;

        switch (*bindist)
        {
        case step_id::bpkg_bindist_debian:
          {
            distribution = "debian";
            output_root = dir_path ("bindist");
            break;
          }
        case step_id::bpkg_bindist_fedora:
          {
            distribution = "fedora";
            break;
          }
        case step_id::bpkg_bindist_archive:
          {
            distribution = "archive";
            output_root = dir_path ("bindist");
            break;
          }
        default: assert (false);
        }

        // bpkg bindist --distribution <distribution>
        //              <env-config-args> <tgt-config-args> <pkg-config-args>
        //              <package-name>
        //
        // Note that if we are installing the result, we need to generate
        // packages for all the dependencies unless they are included in the
        // package (with --recursive). The way we are going to arrange for
        // this is by specifying --recursive=separate first and letting any
        // user --recursive option override that.
        //
        step_id b (*bindist);
        step_id s (*bindist);

        r.status |= run_bpkg (
          b,
          trace, r.log, bindist_result_file, wre,
          bkp_step, bkp_status, last_cmd,
          "-v",
          "bindist",
          "--distribution", distribution,
          sys_install ? cstrings ({"--recursive", "separate"}) : cstrings (),
          "--structured-result", "json",
          (!output_root.empty ()
           ? cstrings ({"--output-root", output_root.string ().c_str ()})
           : cstrings ()),
          "-d", bindist_conf,
          step_args (env_args, s),
          step_args (tgt_args, s),
          step_args (pkg_args, s),
          pkg);

        if (!r.status)
          break;

        // Parse the structured result JSON.
        //
        try
        {
          ifdstream is (bindist_result_file);
          json::parser p (is, bindist_result_file.string ());

          using event = json::event;

          auto bad_json = [&p] (string d)
          {
            throw json::invalid_json_input (p.input_name,
                                            p.line (),
                                            p.column (),
                                            p.position (),
                                            move (d));
          };

          // Parse bindist_os_release object.
          //
          auto parse_os_release = [&p] ()
          {
            // enter: after begin_object
            // leave: after end_object

            bindist_os_release r;

            // Skip unknown/uninteresting members.
            //
            while (p.next_expect (event::name, event::end_object))
            {
              const string& n (p.name ());

              if (n == "name_id")
              {
                r.name_id = p.next_expect_string ();
              }
              else if (n == "version_id")
              {
                r.version_id = p.next_expect_string ();
              }
              else
                p.next_expect_value_skip ();
            }

            return r;
          };

          // Parse a bindist_file object.
          //
          auto parse_file = [&p, &bad_json] ()
          {
            // enter: after begin_object
            // leave: after end_object

            bindist_file r;

            // Skip unknown/uninteresting members.
            //
            while (p.next_expect (event::name, event::end_object))
            {
              const string& n (p.name ());

              if (n == "type")
              {
                r.type = p.next_expect_string ();
              }
              else if (n == "path")
              {
                try
                {
                  r.path =
                    path (p.next_expect_string ()).complete ().normalize ();
                }
                catch (const invalid_path& e)
                {
                  bad_json ("invalid package file path '" + e.path + "'");
                }
              }
              else if (n == "system_name")
              {
                r.system_name = p.next_expect_string ();
              }
              else
                p.next_expect_value_skip ();
            }

            return r;
          };

          // Parse a bindist_package object.
          //
          auto parse_package = [&p, &parse_file] ()
          {
            // enter: after begin_object
            // leave: after end_object

            bindist_package r;

            // Skip unknown/uninteresting members.
            //
            while (p.next_expect (event::name, event::end_object))
            {
              const string& n (p.name ());

              if (n == "name")
              {
                r.name = p.next_expect_string ();
              }
              else if (n == "version")
              {
                r.version = p.next_expect_string ();
              }
              else if (n == "system_version")
              {
                r.system_version = p.next_expect_string ();
              }
              else if (n == "files")
              {
                p.next_expect (event::begin_array);

                while (p.next_expect (event::begin_object, event::end_array))
                  r.files.push_back (parse_file ());
              }
              else
                p.next_expect_value_skip ();
            }

            return r;
          };

          // Parse the bindist_result.
          //
          // Note that if the bbot.bindist.upload step is enabled, then we
          // require bindist_result.os_release.version_id to be present. This
          // way the uploaded binary package can be published for a specific
          // version of the distribution.
          //
          p.next_expect (event::begin_object);

          while (p.next_expect (event::name, event::end_object))
          {
            const string& n (p.name ());

            if (n == "distribution")
            {
              bindist_result.distribution = p.next_expect_string ();

              if (bindist_result.distribution != distribution)
                bad_json ("expected distribution '" + distribution +
                          "' instead of '" + bindist_result.distribution + "'");
            }
            else if (n == "architecture")
            {
              bindist_result.architecture = p.next_expect_string ();
            }
            else if (n == "os_release")
            {
              p.next_expect (event::begin_object);
              bindist_result.os_release = parse_os_release ();

              if (!bindist_result.os_release.version_id && bindist_upload)
                bad_json ("version_id must be present if bbot.bindist.upload "
                          "step is enabled");
            }
            else if (n == "package")
            {
              p.next_expect (event::begin_object);
              bindist_result.package = parse_package ();
            }
            else if (n == "dependencies")
            {
              p.next_expect (event::begin_array);

              while (p.next_expect (event::begin_object, event::end_array))
                bindist_result.dependencies.push_back (parse_package ());
            }
            else
              p.next_expect_value_skip ();
          }
        }
        catch (const json::invalid_json_input& e)
        {
          fail_operation (
            r,
            string ("invalid bpkg-pkg-bindist result json input: ") +
            e.what (),
            result_status::abort,
            e.name,
            e.line,
            e.column);

          // Fall through.
        }
        catch (const io_error& e)
        {
          fail << "unable to read " << bindist_result_file << ": " << e;
        }

        if (!r.status)
          break;

        log_line ("generated " + distribution + " package for " + pkg + '/' +
                  ver.string () + ':',
                  r.log);

        for (const bindist_file& f: bindist_result.package.files)
          log_line ("  " + f.path.string (), r.log);

        rm.status |= r.status;
      }
      //
      // Fail if the breakpoint refers to a bpkg.bindist.* step but this step
      // is disabled.
      //
      else if (bkp_step &&
               (*bkp_step == step_id::bpkg_bindist_debian ||
                *bkp_step == step_id::bpkg_bindist_fedora ||
                *bkp_step == step_id::bpkg_bindist_archive))
      {
        fail_unreached_breakpoint (add_result ("bindist"));
        break;
      }

      // Install from the binary distribution package generated on a
      // bpkg.bindist.* step.
      //
      if (sys_install)
      {
        operation_result& r (add_result ("sys-install"));

        // Fail if the breakpoint refers to the bbot.sys-install step since
        // it has no specific command associated.
        //
        if (bkp_step && *bkp_step == step_id::bbot_sys_install)
        {
          fail_unreached_breakpoint (add_result ("sys-install"));
          break;
        }

        // Noop, just for the log record.
        //
        change_wd (trace, &r.log, rwd);

        // Collect the binary package files.
        //
        // Specifically, for now we consider files with the system name
        // specified as package files.
        //
        cstrings pfs;

        auto add_package_files = [&pfs] (const vector<bindist_file>& bfs)
        {
          for (const bindist_file& f: bfs)
          {
            if (f.system_name)
              pfs.push_back (f.path.string ().c_str ());
          }
        };

        add_package_files (bindist_result.package.files);

        for (const bindist_package& d: bindist_result.dependencies)
          add_package_files (d.files);

        // Install for the `debian` distribution.
        //
        if (*bindist == step_id::bpkg_bindist_debian)
        {
          // Update package index.
          //
          {
            // sudo apt-get update <env-config-args>
            //                     <tgt-config-args>
            //                     <pkg-config-args>
            //
            step_id b (step_id::bbot_sys_install_apt_get_update);
            step_id s (step_id::bbot_sys_install_apt_get_update);

            r.status |= run_apt_get (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              "update",
              "--assume-yes",
              step_args (env_args, s),
              step_args (tgt_args, s),
              step_args (pkg_args, s));

            if (!r.status)
              break;
          }

          // Install.
          //
          {
            // sudo apt-get install <env-config-args>
            //                      <tgt-config-args>
            //                      <pkg-config-args>
            //                      <distribution-package-file>...
            //
            // Note that apt-get install requires a directory separator for an
            // argument to be treated as a file rather than name. The paths we
            // pass are absolute.
            //
            step_id b (step_id::bbot_sys_install_apt_get_install);
            step_id s (step_id::bbot_sys_install_apt_get_install);

            r.status |= run_apt_get (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              "install",
              "--assume-yes",
              step_args (env_args, s),
              step_args (tgt_args, s),
              step_args (pkg_args, s),
              pfs);

            if (!r.status)
              break;
          }
        }
        //
        // Fail if the breakpoint refers to a bbot.sys-install.apt_get.* step
        // but the distribution is other than `debian`.
        //
        else if (bkp_step                                              &&
                 *bkp_step >= step_id::bbot_sys_install_apt_get_update &&
                 *bkp_step <= step_id::bbot_sys_install_apt_get_install)
        {
          fail_unreached_breakpoint (r);
          break;
        }
        //
        // Install for the `fedora` distribution.
        //
        else if (*bindist == step_id::bpkg_bindist_fedora)
        {
          // sudo dnf install <env-config-args>
          //                  <tgt-config-args>
          //                  <pkg-config-args>
          //                  <distribution-package-file>...
          //
          step_id b (step_id::bbot_sys_install_dnf_install);
          step_id s (step_id::bbot_sys_install_dnf_install);

          r.status |= run_dnf (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "install",
            "--refresh",
            "--assumeyes",
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            pfs);

          if (!r.status)
            break;
        }
        //
        // Fail if the breakpoint refers to a bbot.sys-install.dnf.* step but
        // the distribution is other than `fedora`.
        //
        else if (bkp_step && *bkp_step == step_id::bbot_sys_install_dnf_install)
        {
          fail_unreached_breakpoint (r);
          break;
        }
        //
        // Install for the `archive` distribution.
        //
        // Since there is no easy way to extract from multiple archives with a
        // single command, we run tar in a loop.
        //
        // Note that it is assumed that the --directory and --strip-components
        // options are passed via <*-config-args>. The extracted executables
        // can be arranged to be found by setting config.install.root for
        // bpkg.target.create, etc (the same way as for installing from
        // source).
        //
        else if (*bindist == step_id::bpkg_bindist_archive)
        {
          for (const char* f: pfs)
          {
            // [sudo] tar -xf <distribution-package-file> <env-config-args>
            //                                            <tgt-config-args>
            //                                            <pkg-config-args>
            //
            step_id b (step_id::bbot_sys_install_tar_extract);
            step_id s (step_id::bbot_sys_install_tar_extract);

            r.status |= run_tar (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              true /* sudo */,
              "-xf",
              f,
              step_args (env_args, s),
              step_args (tgt_args, s),
              step_args (pkg_args, s));

            if (!r.status)
              break;
          }

          if (!r.status)
            break;

          // Run ldconfig.
          //
          if (step_enabled (step_id::bbot_sys_install_ldconfig))
          {
            // sudo ldconfig <env-config-args>
            //               <tgt-config-args>
            //               <pkg-config-args>
            //
            step_id b (step_id::bbot_sys_install_ldconfig);
            step_id s (step_id::bbot_sys_install_ldconfig);

            r.status |= run_ldconfig (
              b,
              trace, r.log, wre,
              bkp_step, bkp_status, last_cmd,
              step_args (env_args, s),
              step_args (tgt_args, s),
              step_args (pkg_args, s));

            if (!r.status)
              break;
          }
          //
          // Fail if the breakpoint refers to the bbot.sys-install.ldconfig
          // step but this step is disabled.
          //
          else if (bkp_step && *bkp_step == step_id::bbot_sys_install_ldconfig)
          {
            fail_unreached_breakpoint (r);
            break;
          }

          if (!module_pkg)
          {
            if (optional<dir_path> ir = config_install_root ())
              install_bin = *ir / dir_path ("bin");
          }
        }
        //
        // Fail if the breakpoint refers to a
        // bbot.sys-install.{tar.extract,ldconfig} step but the distribution
        // is other than `archive`.
        //
        else if (bkp_step &&
                 *bkp_step >= step_id::bbot_sys_install_tar_extract &&
                 *bkp_step <= step_id::bbot_sys_install_ldconfig)
        {
          fail_unreached_breakpoint (r);
          break;
        }
        else
          assert (false);

        rm.status |= r.status;
      }
      //
      // Fail if the breakpoint refers to a bbot.sys-install.* step but this
      // step is disabled.
      //
      else if (bkp_step                               &&
               *bkp_step >= step_id::bbot_sys_install &&
               *bkp_step <= step_id::bbot_sys_install_ldconfig)
      {
        fail_unreached_breakpoint (add_result ("sys-install"));
        break;
      }

      // Now, if the package is installed, either from source or from the
      // binary distribution package, the overall plan is as follows:
      //
      // 1. If the package has subprojects that support the test operation,
      //    then configure, build, and test them out of the source tree
      //    against the installed package using the build system directly.
      //
      // 2. If any of the test packages are specified, then configure, build,
      //    and test them in a separate bpkg configuration(s) against the
      //    installed package.
      //
      if (install_root || sys_install)
      {
        // Run the internal tests if the project contains "testable"
        // subprojects, but not for a module.
        //
        has_internal_tests = false;

        dir_paths subprj_dirs; // "Testable" package subprojects.

        // Collect the "testable" subprojects.
        //
        if (!module_pkg)
        {
          assert (!rm.results.empty ());

          // Result of the install or sys-install operation.
          //
          operation_result& r (rm.results.back ());

          change_wd (trace, &r.log, effective_install_conf);

          for (const b_project_info::subproject& sp: prj.subprojects)
          {
            // Retrieve the subproject information similar to how we've done it
            // for the package.
            //
            b_project_info si (prj_info (pkg_dir / sp.path,
                                         b_info_flags::ext_mods,
                                         "subproject"));

            const strings& ops (si.operations);
            if (find (ops.begin (), ops.end (), "test") != ops.end ())
              subprj_dirs.push_back (sp.path);
          }

          has_internal_tests = !subprj_dirs.empty ();
        }

        if (has_internal_tests || has_runtime_tests || has_buildtime_tests)
        {
          operation_result& r (add_result ("test-installed"));

          change_wd (trace, &r.log, rwd);

          // Make sure that the installed package executables are properly
          // imported when configuring and running tests, unless we are testing
          // the build system module (that supposedly doesn't install any
          // executables).
          //
          small_vector<string, 1> envvars;

          if (install_bin)
          {
            // Note that we add the $config.install.root/bin directory at the
            // beginning of the PATH environment variable value, so the
            // installed executables are found first.
            //
            string paths ("PATH=" + install_bin->string ());

            if (optional<string> s = getenv ("PATH"))
            {
              paths += path::traits_type::path_separator;
              paths += *s;
            }

            envvars.push_back (move (paths));
          }

          // Run internal tests.
          //
          if (has_internal_tests)
          {
            // Create the configuration.
            //
            // b create(<dir>, <env-modules>) <env-config-args>
            //                                <tgt-config-args>
            //                                <pkg-config-args>
            //
            // Amalgamation directory that will contain configuration
            // subdirectory for package tests out of source tree build.
            //
            dir_path out_dir ("build-installed");

            {
              step_id b (step_id::b_test_installed_create);
              step_id s (step_id::b_test_installed_create);
              step_id f (step_id::b_create);

              string mods; // build2 create meta-operation parameters.

              for (const char* m: step_args (modules, s, f))
              {
                mods += mods.empty () ? ", " : " ";
                mods += m;
              }

              r.status |= run_b (
                b,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-V",
                "create('" + out_dir.representation () + '\'' + mods + ')',
                step_args (env_args, s, f),
                step_args (tgt_args, s, f),
                step_args (pkg_args, s, f));

              if (!r.status)
                break;
            }

            // Configure testable subprojects sequentially and test/build in
            // parallel afterwards.
            //
            // It feels right to configure internal tests also passing the
            // main package configuration variables, since they may need to
            // align with the main package setup (enable some testscripts,
            // etc).
            //
            strings test_specs;
            for (const dir_path& d: subprj_dirs)
            {
              // b configure(<subprj-src-dir>@<subprj-out-dir>) <env-config-args>
              //                                                <tgt-config-args>
              //                                                <pkg-config-args>
              //                                                <pkg-vars>
              //
              step_id b (step_id::b_test_installed_configure);
              step_id s (step_id::b_test_installed_configure);
              step_id f (step_id::b_configure);

              dir_path subprj_src_dir (exists (dist_src)
                                       ? dist_src / d
                                       : main_pkg_conf / pkg_dir / d);

              dir_path subprj_out_dir (out_dir / d);

              r.status |= run_b (
                b,
                envvars,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-v",
                "configure('" +
                subprj_src_dir.representation () + "'@'" +
                subprj_out_dir.representation () + "')",
                step_args (env_args, s, f),
                step_args (tgt_args, s, f),
                step_args (pkg_args, s, f),
                pkg_config_vars);

              if (!r.status)
                break;

              test_specs.push_back (
                "test('" + subprj_out_dir.representation () + "')");
            }

            if (!r.status)
              break;

            // Build/test subprojects.
            //
            // b test(<subprj-out-dir>)... <env-config-args>
            //                             <tgt-config-args>
            //                             <pkg-config-args>
            //
            if (!step_disabled (step_id::b_test_installed_test))
            {
              step_id b (step_id::b_test_installed_test);
              step_id s (step_id::b_test_installed_test);

              r.status |= run_b (
                b,
                envvars,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-v",
                test_specs,
                step_args (env_args, s),
                step_args (tgt_args, s),
                step_args (pkg_args, s));

              if (!r.status)
                break;
            }
            //
            // Fail if the breakpoint refers to the b.test-installed.test step
            // but this step is disabled.
            //
            else if (bkp_step && *bkp_step == step_id::b_test_installed_test)
            {
              fail_unreached_breakpoint (r);
              break;
            }
          }
          //
          // Fail if the breakpoint refers to some of the b.test-installed.*
          // steps but the package doesn't have any internal tests.
          //
          else if (bkp_step                                      &&
                   *bkp_step >= step_id::b_test_installed_create &&
                   *bkp_step <= step_id::b_test_installed_test)
          {
            fail_unreached_breakpoint (r);
            break;
          }

          // Run runtime and build-time tests.
          //
          // Note that we only build runtime tests for target packages and for
          // host packages in self-hosted configurations.
          //
          if (has_runtime_tests || has_buildtime_tests)
          {
            // Create the required build configurations.
            //
            dir_path target_conf ("build-installed-bpkg");
            dir_path host_conf   ("build-installed-bpkg-host");
            dir_path module_conf ("build-installed-bpkg-module");

            // Create the target configuration if this is a target package
            // having external runtime tests or a host/module package having
            // external build-time tests.
            //
            bool create_target (target_pkg || has_buildtime_tests);

            // Note that even if there are no runtime tests for a host/module
            // package, we still need to create the host/build2 configuration
            // to configure the system package in.
            //
            bool create_host (host_pkg || module_pkg);

            bool create_module (module_pkg ||
                                (host_pkg && has_buildtime_tests));

            // Note: a module package cannot have runtime tests and so the
            // module configuration is only created to serve build-time tests.
            // Thus, the host or target configuration is always created as
            // well and the module configuration is never a root
            // configuration.
            //
            assert (create_target || create_host);

            // Root configuration through which we will be configuring the
            // cluster.
            //
            const dir_path& root_conf (create_target ? target_conf : host_conf);

            // Runtime tests configuration. Should only be used if there are
            // any.
            //
            const dir_path& runtime_tests_conf (target_pkg
                                                ? target_conf
                                                : host_conf);

            // Create the target configuration.
            //
            // bpkg create <env-modules> <env-config-args>
            //                           <tgt-config-args>
            //                           <pkg-config-args>
            //
            if (create_target)
            {
              step_id b (step_id::bpkg_test_separate_installed_create);

              // Note that here and below the _for_* step ids are determined
              // by the main package type (and, yes, that means we will use
              // the same step ids for target and host configuration -- that,
              // however, should be ok since host configuration will only be
              // created in the self-hosted case).
              //
              step_id s (
                target_pkg
                ? step_id::bpkg_test_separate_installed_create_for_target
                : host_pkg
                  ? step_id::bpkg_test_separate_installed_create_for_host
                  : step_id::bpkg_test_separate_installed_create_for_module);

              // Note: no fallback for modules.
              //
              optional<step_id> f (!module_pkg
                                   ? step_id::bpkg_test_separate_installed_create
                                   : optional<step_id> ());

              r.status |= run_bpkg (
                b,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-V",
                "create",
                "-d", target_conf,
                step_args (modules,  s, f),
                step_args (env_args, s, f),
                step_args (tgt_args, s, f),
                step_args (pkg_args, s, f));

              if (!r.status)
                break;
            }

            // Create the host configuration.
            //
            if (create_host)
            {
              // bpkg create --type host <env-modules> <env-config-args>
              //                                       <tgt-config-args>
              //                                       <pkg-config-args>
              //
              step_id b (step_id::bpkg_test_separate_installed_create);

              step_id s (host_pkg
                         ? step_id::bpkg_test_separate_installed_create_for_host
                         : step_id::bpkg_test_separate_installed_create_for_module);

              // Note: no fallback for modules.
              //
              optional<step_id> f (!module_pkg
                                   ? step_id::bpkg_test_separate_installed_create
                                   : optional<step_id> ());

              r.status |= run_bpkg (
                b,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-V",
                "create",
                "-d", host_conf,
                "--type", "host",
                "--name", "host",
                step_args (modules,  s, f),
                step_args (env_args, s, f),
                step_args (tgt_args, s, f),
                step_args (pkg_args, s, f));

              if (!r.status)
                break;
            }

            // Create the module configuration.
            //
            // Note that we never build any tests in it but only configure the
            // system package. Note, however, that the host/module package
            // build-time tests can potentially build some other modules here.
            //
            if (create_module)
            {
              // b create(<dir>) config.config.load=~build2
              //
              step_id b (step_id::bpkg_test_separate_installed_create);

              r.status |= run_b (
                b,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-V",
                "create(" + module_conf.representation () + ",cc)",
                "config.config.load=~build2",
                "config.config.persist+='config.*'@unused=drop");

              if (!r.status)
                break;

              // bpkg create --existing --type build2
              //
              r.status |= run_bpkg (
                b,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-v",
                "create",
                "--existing",
                "-d", module_conf,
                "--type", "build2",
                "--name", "module");

              if (!r.status)
                break;
            }

            // Link the configurations.
            //
            // bpkg link -d <dir> <dir>
            //
            {
              step_id b (step_id::bpkg_test_separate_installed_link);

              if (create_target)
              {
                if (create_host)
                {
                  r.status |= run_bpkg (
                    b,
                    trace, r.log, wre,
                    bkp_step, bkp_status, last_cmd,
                    "-v",
                    "link",
                    "-d", target_conf,
                    host_conf);

                  if (!r.status)
                    break;
                }

                if (create_module)
                {
                  r.status |= run_bpkg (
                    b,
                    trace, r.log, wre,
                    bkp_step, bkp_status, last_cmd,
                    "-v",
                    "link",
                    "-d", target_conf,
                    module_conf);

                  if (!r.status)
                    break;
                }
              }

              if (create_host)
              {
                if (create_module)
                {
                  r.status |= run_bpkg (
                    b,
                    trace, r.log, wre,
                    bkp_step, bkp_status, last_cmd,
                    "-v",
                    "link",
                    "-d", host_conf,
                    module_conf);

                  if (!r.status)
                    break;
                }
              }
            }

            // Add and fetch the repositories.
            //
            if (has_runtime_tests)
            {
              // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args>
              //          <repository-url>
              //
              {
                step_id b (step_id::bpkg_test_separate_installed_configure_add);
                step_id s (step_id::bpkg_test_separate_installed_configure_add);
                step_id f (step_id::bpkg_configure_add);

                r.status |= run_bpkg (
                  b,
                  trace, r.log, wre,
                  bkp_step, bkp_status, last_cmd,
                  "-v",
                  "add",
                  "-d", runtime_tests_conf,
                  step_args (env_args, s, f),
                  step_args (tgt_args, s, f),
                  step_args (pkg_args, s, f),
                  repo);

                if (!r.status)
                  break;
              }

              // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args>
              //            <trust-options>
              //
              {
                step_id b (step_id::bpkg_test_separate_installed_configure_fetch);
                step_id s (step_id::bpkg_test_separate_installed_configure_fetch);
                step_id f (step_id::bpkg_configure_fetch);

                r.status |= run_bpkg (
                  b,
                  trace, r.log, wre,
                  bkp_step, bkp_status, last_cmd,
                  "-v",
                  "fetch",
                  "-d", runtime_tests_conf,
                  step_args (env_args, s, f),
                  step_args (tgt_args, s, f),
                  step_args (pkg_args, s, f),
                  trust_ops);

                if (!r.status)
                  break;
              }
            }

            if (has_buildtime_tests)
            {
              // bpkg add <env-config-args> <tgt-config-args> <pkg-config-args>
              //          <repository-url>
              //
              {
                step_id b (step_id::bpkg_test_separate_installed_configure_add);
                step_id s (step_id::bpkg_test_separate_installed_configure_add);
                step_id f (step_id::bpkg_configure_add);

                r.status |= run_bpkg (
                  b,
                  trace, r.log, wre,
                  bkp_step, bkp_status, last_cmd,
                  "-v",
                  "add",
                  "-d", target_conf,
                  step_args (env_args, s, f),
                  step_args (tgt_args, s, f),
                  step_args (pkg_args, s, f),
                  repo);

                if (!r.status)
                  break;
              }

              // bpkg fetch <env-config-args> <tgt-config-args> <pkg-config-args>
              //            <trust-options>
              //
              {
                step_id b (step_id::bpkg_test_separate_installed_configure_fetch);
                step_id s (step_id::bpkg_test_separate_installed_configure_fetch);
                step_id f (step_id::bpkg_configure_fetch);

                r.status |= run_bpkg (
                  b,
                  trace, r.log, wre,
                  bkp_step, bkp_status, last_cmd,
                  "-v",
                  "fetch",
                  "-d", target_conf,
                  step_args (env_args, s, f),
                  step_args (tgt_args, s, f),
                  step_args (pkg_args, s, f),
                  trust_ops);

                if (!r.status)
                  break;
              }
            }

            // Configure all the packages using a single bpkg-pkg-build command.
            //
            // bpkg build --configure-only <env-config-args>
            //                             <tgt-config-args>
            //                             <pkg-config-args>
            //      { <config> <rtt-config-vars> }+   <runtime-test>...
            //      { <config>                   }+ { <runtime-test>... }
            //      {          <btt-config-vars> }+   <buildtime-test>...
            //                                        ?sys:<pkg>
            //                                        <glob-dep>...
            //
            strings pkgs;

            if (has_runtime_tests)
            {
              // Note that only host package runtime tests can (but not
              // necessarily) be configured in a linked configuration and
              // require --config-name to be specified for them.
              //
              assert (!module_pkg);

              string conf_name (runtime_tests_conf == root_conf
                                ? ""
                                : "host");

              // If there are any runtime tests with configuration variables,
              // then add them to the command line as following:
              //
              // { --config-name <name> <config-var>... }+ <runtime-test>...
              //
              // Also count the number of runtime tests without configuration
              // variables.
              //
              size_t no_vars (0);
              for (const auto& t: runtime_tests)
              {
                auto i (pkg_config_pkgs.find (t.name.string ()));

                if (i != pkg_config_pkgs.end ())
                {
                  pkgs.push_back ("{");

                  if (!conf_name.empty ())
                  {
                    pkgs.push_back ("--config-name");
                    pkgs.push_back (conf_name);
                  }

                  pkgs.insert (pkgs.end (),
                               i->second.begin (), i->second.end ());

                  pkgs.push_back ("}+");

                  pkgs.push_back (t.string ());
                }
                else
                  ++no_vars;
              }

              // If there are any runtime tests without configuration
              // variables, then add them to the command line as following:
              //
              // { --config-name <name> }+ { <runtime-test>... }
              //
              if (no_vars != 0)
              {
                bool og (!conf_name.empty ());

                if (og)
                {
                  pkgs.push_back ("{");

                  pkgs.push_back ("--config-name");
                  pkgs.push_back (conf_name);

                  pkgs.push_back ("}+");
                }

                if (og && no_vars != 1)
                  pkgs.push_back ("{");

                for (const auto& t: runtime_tests)
                {
                  if (pkg_config_pkgs.find (t.name.string ()) ==
                      pkg_config_pkgs.end ())
                  {
                    pkgs.push_back (t.string ());
                  }
                }

                if (og && no_vars != 1)
                  pkgs.push_back ("}");
              }
            }

            if (has_buildtime_tests)
            {
              for (const auto& t: buildtime_tests)
              {
                auto i (pkg_config_pkgs.find (t.name.string ()));

                if (i != pkg_config_pkgs.end ())
                {
                  pkgs.push_back ("{");

                  pkgs.insert (pkgs.end (),
                               i->second.begin (), i->second.end ());

                  pkgs.push_back ("}+");
                }

                // Strip the build-time mark.
                //
                pkgs.push_back (t.dependency::string ());
              }
            }

            pkgs.push_back ("?sys:" + pkg_rev);

            // Add the global system dependencies.
            //
            for (const pair<string, strings>& d: pkg_config_glob_deps)
              pkgs.push_back (d.first);

            // Finally, configure all the test packages.
            //
            {
              step_id b (step_id::bpkg_test_separate_installed_configure_build);

              step_id g (step_id::bpkg_global_configure_build); // Global.

              step_id s (
                target_pkg
                ? step_id::bpkg_test_separate_installed_configure_build_for_target
                  : host_pkg
                  ? step_id::bpkg_test_separate_installed_configure_build_for_host
                  : step_id::bpkg_test_separate_installed_configure_build_for_module);

              step_id f (step_id::bpkg_test_separate_installed_configure_build);

              r.status |= run_bpkg (
                b,
                envvars,
                trace, r.log, wre,
                bkp_step, bkp_status, last_cmd,
                "-v",
                "build",
                "--configure-only",
                "--checkout-root", dist_installed_root,
                "--yes",
                "-d", root_conf,
                step_args (env_args, g),
                step_args (env_args, s, f),
                step_args (tgt_args, g),
                step_args (tgt_args, s, f),
                step_args (pkg_args, g),
                step_args (pkg_args, s, f),
                "--",
                pkgs);

              if (!r.status)
                break;
            }

#ifdef _WIN32
            Sleep (5000); // See above.
#endif

            // Note that if bpkg.test-separate-installed.update step is
            // disabled, we also skip bpkg.test-separate-installed.test.
            //
            if (!step_disabled (step_id::bpkg_test_separate_installed_update))
            {
              bool update_only (
                step_disabled (step_id::bpkg_test_separate_installed_test));

              // Fail if the breakpoint refers to the
              // bpkg.test-separate-installed.test step but this step is
              // disabled.
              //
              if (update_only &&
                  bkp_step    &&
                  *bkp_step == step_id::bpkg_test_separate_installed_test)
              {
                fail_unreached_breakpoint (r);
                break;
              }

              // External runtime tests.
              //
              if (has_runtime_tests)
              {
                const dir_path& runtime_tests_conf (target_pkg
                                                    ? target_conf
                                                    : host_conf);

                change_wd (trace, &r.log, runtime_tests_conf);

                if (!test (r,
                           runtime_tests,
                           dist_installed_root,
                           true /* installed */,
                           update_only,
                           envvars))
                  break;
              }

              // External build-time tests.
              //
              if (has_buildtime_tests)
              {
                change_wd (trace, &r.log, rwd / target_conf);

                if (!test (r,
                           buildtime_tests,
                           dist_installed_root,
                           true /* installed */,
                           update_only,
                           envvars))
                  break;
              }
            }
            //
            // Fail if the breakpoint refers to some of the
            // bpkg.test-separate-installed.{update,test} steps but the
            // bpkg.test-separate-installed.update step is disabled.
            //
            else if (bkp_step                                                  &&
                     *bkp_step >= step_id::bpkg_test_separate_installed_update &&
                     *bkp_step <= step_id::bpkg_test_separate_installed_test)
            {
              fail_unreached_breakpoint (r);
              break;
            }
          }
          //
          // Fail if the breakpoint refers to some of the
          // bpkg.test-separate-installed.* steps but the package has no
          // external tests.
          //
          else if (bkp_step                                                  &&
                   *bkp_step >= step_id::bpkg_test_separate_installed_create &&
                   *bkp_step <= step_id::bpkg_test_separate_installed_test)
          {
            fail_unreached_breakpoint (r);
            break;
          }

          rm.status |= r.status;
        }
        //
        // Fail if the breakpoint refers to some of the test installed steps
        // but the package has no tests.
        //
        else if (bkp_step                                      &&
                 *bkp_step >= step_id::b_test_installed_create &&
                 *bkp_step <= step_id::bpkg_test_separate_installed_test)
        {
          fail_unreached_breakpoint (add_result ("test-installed"));
          break;
        }
      }
      //
      // Fail if the breakpoint refers to some of the test installed steps but
      // the package is not supposed to be installed neither from source nor
      // from the binary distribution package.
      //
      else if (bkp_step                                      &&
               *bkp_step >= step_id::b_test_installed_create &&
               *bkp_step <= step_id::bpkg_test_separate_installed_test)
      {
        fail_unreached_breakpoint (add_result ("test-installed"));
        break;
      }

      // Uninstall, if installed from the binary distribution package.
      //
      // Note: noop for the archive distribution.
      //
      if (sys_install &&
          (*bindist == step_id::bpkg_bindist_debian ||
           *bindist == step_id::bpkg_bindist_fedora))
      {
        operation_result& r (add_result ("sys-uninstall"));

        // Noop, just for the log record.
        //
        change_wd (trace, &r.log, rwd);

        // Collect the binary package system names.
        //
        cstrings pns;

        auto add_package_names = [&pns] (const vector<bindist_file>& bfs)
        {
          for (const bindist_file& f: bfs)
          {
            if (f.system_name)
              pns.push_back (f.system_name->c_str ());
          }
        };

        add_package_names (bindist_result.package.files);

        for (const bindist_package& d: bindist_result.dependencies)
          add_package_names (d.files);

        // Uninstall for the `debian` distribution.
        //
        if (*bindist == step_id::bpkg_bindist_debian)
        {
          // sudo apt-get remove <env-config-args>
          //                     <tgt-config-args>
          //                     <pkg-config-args>
          //                     <distribution-package-name>...
          //
          step_id b (step_id::bbot_sys_uninstall_apt_get_remove);
          step_id s (step_id::bbot_sys_uninstall_apt_get_remove);

          r.status |= run_apt_get (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "remove",
            "--assume-yes",
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            pns);

          if (!r.status)
            break;
        }
        //
        // Fail if the breakpoint refers to the
        // bbot.sys-uninstall.apt-get.remove step but the distribution is
        // other than `debian`.
        //
        else if (bkp_step &&
                 *bkp_step == step_id::bbot_sys_uninstall_apt_get_remove)
        {
          fail_unreached_breakpoint (r);
          break;
        }
        //
        // Uninstall for the `fedora` distribution.
        //
        else if (*bindist == step_id::bpkg_bindist_fedora)
        {
          // sudo dnf remove <env-config-args>
          //                 <tgt-config-args>
          //                 <pkg-config-args>
          //                 <distribution-package-name>...
          //
          step_id b (step_id::bbot_sys_uninstall_dnf_remove);
          step_id s (step_id::bbot_sys_uninstall_dnf_remove);

          r.status |= run_dnf (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            "remove",
            "--assumeyes",
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            pns);

          if (!r.status)
            break;
        }
        //
        // Fail if the breakpoint refers to the bbot.sys-uninstall.dnf.remove
        // step but the distribution is other than `fedora`.
        //
        else if (bkp_step &&
                 *bkp_step == step_id::bbot_sys_uninstall_dnf_remove)
        {
          fail_unreached_breakpoint (r);
          break;
        }
        else
          assert (false);

        rm.status |= r.status;
      }
      //
      // Fail if the breakpoint refers to a bbot.sys-uninstall.* step but
      // this step is disabled.
      //
      else if (bkp_step                                                &&
               *bkp_step >= step_id::bbot_sys_uninstall_apt_get_remove &&
               *bkp_step <= step_id::bbot_sys_uninstall_dnf_remove)
      {
        fail_unreached_breakpoint (add_result ("sys-uninstall"));
        break;
      }

      // Uninstall, if installed from source.
      //
      if (install_root)
      {
        operation_result& r (add_result ("uninstall"));

        change_wd (trace, &r.log, effective_install_conf);

        // bpkg uninstall <env-config-args> <tgt-config-args> <pkg-config-args>
        //                <package-name>
        //
        step_id b (step_id::bpkg_uninstall);
        step_id s (step_id::bpkg_uninstall);

        r.status |= run_bpkg (
          b,
          trace, r.log, wre,
          bkp_step, bkp_status, last_cmd,
          "-v",
          "uninstall",
          step_args (env_args, s),
          step_args (tgt_args, s),
          step_args (pkg_args, s),
          pkg);

        if (!r.status)
          break;

        rm.status |= r.status;
      }
      //
      // Fail if the breakpoint refers to the bpkg.uninstall step but the
      // package was not installed from source.
      //
      else if (bkp_step && *bkp_step == step_id::bpkg_uninstall)
      {
        fail_unreached_breakpoint (add_result ("uninstall"));
        break;
      }

      // Now, after the package is fully tested, let's prepare the build
      // artifacts for the upload, using the upload operation log.
      //

      // Prepare the bindist artifacts.
      //
      // Move the binary distribution files generated for the main package and
      // bindist-result.json to the upload/bindist/<distribution>/ directory.
      // Also serialize the subset of the bindist result as
      // bindist-result.manifest.
      //
      // Fail if the breakpoint refers to the bbot.bindist.upload step since
      // it has no specific command associated.
      //
      if (bkp_step && *bkp_step == step_id::bbot_bindist_upload)
      {
        fail_unreached_breakpoint (add_result ("upload"));
        break;
      }

      if (bindist_upload)
      {
        operation_result& r (add_result ("upload"));

        change_wd (trace, &r.log, rwd);

        dir_path d (upload_dir           /
                    dir_path ("bindist") /
                    dir_path (bindist_result.distribution));

        log_step_id (trace, r.log, step_id::bbot_bindist_upload);

        mk_p (trace, &r.log, d);

        // Move a file to the upload/bindist/<distribution>/ directory.
        //
        // On Windows copy the file instead of moving not to fail if it is
        // being scanned by the Windows Defender or some such.
        //
        auto mv = [&d, &r, &rwd, &trace] (const path& p)
        {
#ifndef _WIN32
          bool mv (true);
#else
          bool mv (false);
#endif
          // Use relative path, if possible, to keep the operation log tidy
          // (won't be the case on Fedora).
          //
          const path& rp (p.sub (rwd) ? p.leaf (rwd) : p);

          if (mv)
            mv_into (trace, &r.log, rp, d);
          else
            cp_into (trace, &r.log, rp, d);
        };

        // Main package files.
        //
        for (const bindist_file& f: bindist_result.package.files)
          mv (f.path);

        // Bindist result JSON.
        //
        mv (bindist_result_file);

        // Bindist result manifest.
        //
        path mf (d / "bindist-result.manifest");

        try
        {
          ofdstream os (mf);
          serializer s (os, mf.string ());

          // Serialize package manifest.
          //
          s.next ("", "1"); // Start of manifest.

          s.next ("distribution", bindist_result.distribution);
          s.next ("architecture", bindist_result.architecture);

          s.next ("os-release-name-id", bindist_result.os_release.name_id);

          // Should have failed earlier.
          //
          assert (bindist_result.os_release.version_id);

          s.next ("os-release-version-id",
                  *bindist_result.os_release.version_id);

          s.next ("package-name", bindist_result.package.name);
          s.next ("package-version", bindist_result.package.version);

          if (bindist_result.package.system_version)
            s.next ("package-system-version",
                    *bindist_result.package.system_version);

          s.next ("", "");  // End of manifest.

          // Serialize package file manifests.
          //
          for (const bindist_file& f: bindist_result.package.files)
          {
            s.next ("", "1"); // Start of manifest.

            s.next ("package-file-type", f.type);

            // Note: the simple path representation is POSIX.
            //
            s.next ("package-file-path", f.path.leaf ().string ());

            if (f.system_name)
              s.next ("package-file-system-name", *f.system_name);

            s.next ("", "");  // End of manifest.
          }

          s.next ("", ""); // End of stream.

          os.close ();
        }
        catch (const io_error& e)
        {
          fail << "unable to write to '" << mf << "': " << e;
        }
        catch (const serialization& e)
        {
          fail << "unable to serialize bindist result: " << e;
        }
      }

      // Create the archive of the build artifacts for subsequent upload, if
      // the upload/ directory is not empty.
      //
      // Note that the archive will be uploaded later, right before uploading
      // the result manifest, unless the task has been aborted by the user or
      // the result status is error or worse.
      //
      if (!empty (rwd / upload_dir) && !step_disabled (step_id::bbot_upload))
      {
        // The upload operation log must have been added as part of the
        // build artifacts preparation for upload.
        //
        operation_result& r (rm.results.back ());

        // Fail if the breakpoint refers to the bbot.upload step since it has
        // no specific command associated.
        //
        if (bkp_step && *bkp_step == step_id::bbot_upload)
        {
          fail_unreached_breakpoint (r);
          break;
        }

        change_wd (trace, &r.log, rwd);

        // Archive the build artifacts.
        //
        {
          // tar -cf upload.tar <env-config-args>
          //                    <tgt-config-args>
          //                    <pkg-config-args>
          //         upload/
          //
          step_id b (step_id::bbot_upload_tar_create);
          step_id s (step_id::bbot_upload_tar_create);

          // Make sure the archive is portable.
          //
          // Note that OpenBSD tar does not support --format but it appear
          // ustar is the default (see bpkg/system-package-manager-archive.cxx
          // for details).
          //
          r.status |= run_tar (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            false /* sudo */,
#ifndef __OpenBSD__
            "--format", "ustar",
#endif
            "-cf",
            upload_archive,
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s),
            upload_dir);

          if (!r.status)
            break;
        }

        // It feels useful to also print the archive content to the log.
        //
        {
          // tar -tf upload.tar <env-config-args>
          //                    <tgt-config-args>
          //                    <pkg-config-args>
          //
          step_id b (step_id::bbot_upload_tar_list);
          step_id s (step_id::bbot_upload_tar_list);

          r.status |= run_tar (
            b,
            trace, r.log, wre,
            bkp_step, bkp_status, last_cmd,
            false /* sudo */,
            "-tf",
            upload_archive,
            step_args (env_args, s),
            step_args (tgt_args, s),
            step_args (pkg_args, s));

          if (!r.status)
            break;
        }

        rm.status |= r.status;
      }
      //
      // Fail if the breakpoint refers to some of the bbot.upload.* steps but
      // there is either nothing to upload or the bbot.upload step is
      // disabled.
      //
      else if (bkp_step                          &&
               *bkp_step >= step_id::bbot_upload &&
               *bkp_step <= step_id::bbot_upload_tar_list)
      {
        fail_unreached_breakpoint (add_result ("upload"));
        break;
      }
    }
    //
    // Fail if the breakpoint refers to bpkg.update or any dependent step but
    // the bpkg.update step is disabled.
    //
    else if (bkp_step                          &&
             *bkp_step >= step_id::bpkg_update &&
             *bkp_step <= step_id::bbot_upload)
    {
      fail_unreached_breakpoint (add_result ("update"));
      break;
    }

    break;
  }

  if (!rm.results.empty ())
  {
    operation_result& r (rm.results.back ());

    rm.status |= r.status; // Merge last in case of a break.

    // Unless there is an error (or worse) encountered, log the special 'end'
    // step and, if this step is specified in the interactive manifest value,
    // ask the user if to continue the task execution.
    //
    bool error (!rm.status);

    if (!error)
    {
      r.status |= run_cmd (step_id::end,
                           trace, r.log,
                           nullptr /* out_str */, path () /* out_file */,
                           regexes (),
                           "" /* name */,
                           bkp_step, bkp_status, last_cmd,
                           process_env ());

      rm.status |= r.status;
    }

    // Truncate logs if they would exceed the upload limit.
    //
    // @@ TMP: currently this limit is hard-coded. In the future it should be
    //    sent along with the task manifest.
    //
    const size_t upload_limit (10 * 1024 * 1024);
    {
      // Reserve 10K for other manifest values (alternatively, we could do it
      // exactly in upload_manifest()).
      //
      const size_t manifest_size (10 * 1024);

      size_t n (manifest_size);
      for (const operation_result& r: rm.results)
        n += r.log.size ();

      if (n > upload_limit)
      {
        // Divide the size equally among all the operations and truncate any
        // that exceed their allowance. This way we will get some information
        // for each operation.
        //
        n = (upload_limit - manifest_size) / rm.results.size ();

        for (operation_result& r: rm.results)
        {
          if (r.log.size () <= n)
            continue;

          // We need to be careful not to truncate it in the middle of UTF-8
          // sequence. So we look for the last newline that still fits.
          //
          size_t p (n - 80 /* for the "log truncated" note */);
          for (; p != 0 && r.log[p] != '\n'; --p) ;
          r.log.resize (p != 0 ? p + 1 : 0); // Keep the newline.
          r.log += "-------------------------------LOG TRUNCATED-------------------------------\n";
        }
      }
    }
  }
  else
    assert (rm.status == result_status::abort ||
            rm.status == result_status::skip);

  if (!rwd.empty ())
  {
    change_wd (trace, nullptr /* log */, rwd);

    // Upload the build artifacts archive, if exists.
    //
    bool error (!rm.status);
    if (exists (upload_archive) && !error)
    {
      const string url (
        "tftp://" + ops.tftp_host () + '/' + upload_archive.string ());

      try
      {
        tftp_curl c (trace,
                     upload_archive,
                     nullfd,
                     curl::put,
                     url,
                     "--tftp-blksize", tftp_blksize,
                     "--max-time", tftp_put_timeout);

        if (!c.wait ())
          fail << "curl " << *c.exit;
      }
      catch (const process_error& e)
      {
        fail << "unable to execute curl: " << e;
      }
      catch (const system_error& e)
      {
        fail << "unable to upload build artifacts to " << url << ": " << e;
      }
    }
  }

  // Upload the result.
  //
  const string url ("tftp://" + ops.tftp_host () + "/result.manifest.lz4");

  try
  {
    upload_manifest (trace, url, rm, "result");

    // We use exit code 2 to signal abnormal termination but where we managed
    // to upload the result manifest. See startup() for details.
    //
    return rm.status != result_status::abnormal ? 0 : 2;
  }
  catch (const io_error& e)
  {
    error << "unable to upload result manifest to " << url << ": " << e;
  }

  // We use exit code 3 to signal an unsuccessful attempt to upload the result
  // manifest. See startup() for details.
  //
  return 3;
}

static int
startup ()
{
  tracer trace ("startup");

  // Our overall plan is as follows:
  //
  // 1. Download the task manifest into the build directory (CWD).
  //
  // 2. Parse it and get the target.
  //
  // 3. Find the environment setup executable for this target.
  //
  // 4. Execute the environment setup executable.
  //
  // 5. If the environment setup executable fails, then upload the (failed)
  //    result ourselves.
  //
  const string url ("tftp://" + ops.tftp_host () + "/task.manifest");
  const path mf ("task.manifest");

  // If we fail, try to upload the result manifest (abnormal termination). The
  // idea is that the machine gets suspended and we can investigate what's
  // going on by logging in and examining the diagnostics (e.g., via
  // journalctl, etc).
  //
  task_manifest tm;

  try
  {
    // Download the task.
    //
    // We are downloading from our host so there shouldn't normally be any
    // connectivity issues. Unless, of course, we are on Windows where all
    // kinds of flakiness is business as usual. Note that having a long enough
    // timeout is not enough: if we try to connect before the network is up,
    // we will keep waiting forever, even after it is up. So we have to
    // timeout and try again. This is also pretty bad (unlike, say during
    // bootstrap which doesn't happen very often) since we are wasting the
    // machine time. So we are going to log it as a warning and not merely a
    // trace since if this is a common occurrence, then something has to be
    // done about it.
    //
    for (size_t retry (1);; ++retry)
    {
      try
      {
        tftp_curl c (trace,
                     nullfd,
                     mf,
                     curl::get,
                     url,
                     "--tftp-blksize", tftp_blksize,
                     "--max-time", tftp_get_timeout);

        if (!c.wait ())
          throw_generic_error (EIO);

        break;
      }
      catch (const system_error& e)
      {
        bool bail (retry > tftp_get_retries);
        diag_record dr (bail ? error : warn);

        dr << "unable to download task manifest from " << url << " on "
           << retry << " try: " << e;

        if (bail)
          throw failed ();
      }
    }

    // Parse it.
    //
    tm = parse_manifest<task_manifest> (mf, "task");

    // Find the environment setup executable.
    //
    // While the executable path contains a directory (so the PATH search does
    // not apply) we still use process::path_search() to automatically handle
    // appending platform-specific executable extensions (.exe/.bat, etc).
    //
    process_path pp;

    if (tm.environment)
    {
      try
      {
        pp = process::try_path_search (env_dir / *tm.environment,
                                       false /* init */);
      }
      catch (const invalid_path& e)
      {
        fail << "invalid environment name '" << e.path << "': " << e;
      }

      if (pp.empty ())
        fail << "no environment setup executable in " << env_dir << " "
             << "for environment name '" << *tm.environment << "'";
    }
    else
    {
      pp = process::try_path_search (env_dir / "default", false /* init */);

      if (pp.empty ())
        fail << "no default environment setup executable in " << env_dir;
    }

    // Run it.
    //
    strings os;

    string tg (tm.target.string ());

    // Use the name=value notation for options to minimize the number of
    // arguments passed to the environment setup executable. Note that the
    // etc/environments/default-*.bat scripts can only handle the limited
    // number of arguments.
    //
    if (ops.systemd_daemon ())
      os.push_back ("--systemd-daemon");

    if (ops.verbose_specified ())
      os.push_back ("--verbose=" + to_string (ops.verbose ()));

    if (ops.tftp_host_specified ())
      os.push_back ("--tftp-host=" + ops.tftp_host ());

    os.push_back (string ("--env-script=") + pp.effect_string ());
    os.push_back ("--env-target=" + tg);

    // Note that we use the effective (absolute) path instead of recall since
    // we may have changed the CWD.
    //
    // Also note that the worker can ask the user if to continue the task
    // execution when the interactive build breakpoint is reached. Thus, we
    // don't redirect stdin to /dev/null.
    //
    // Exit code 2 signals abnormal termination but where the worker uploaded
    // the result itself.
    //
    // Exit code 3 signals an unsuccessful attempt by the worker to upload the
    // result manifest. There is no reason to retry (most likely there is
    // nobody listening on the other end anymore).
    //
    switch (run_io_exit (trace, 0, 2, 2, pp, tg, argv0.effect_string (), os))
    {
    case 3:
    case 2:  return 1;
    case 0:  return 0;
    default: fail << "process " << pp << " exited with non-zero code" << endf;
    }
  }
  catch (const failed&)
  {
    const string url ("tftp://" + ops.tftp_host () + "/result.manifest.lz4");

    // If we failed before being able to parse the task manifest, use the
    // "unknown" values for the package name and version.
    //
    result_manifest rm {
      tm.name.empty ()    ? bpkg::package_name ("unknown") : tm.name,
      tm.version.empty () ? bpkg::version ("0")            : tm.version,
      result_status::abnormal,
      operation_results {},
      worker_checksum,
      nullopt /* dependency_checksum */
    };

    try
    {
      upload_manifest (trace, url, rm, "result");
    }
    catch (const io_error& e)
    {
      fail << "unable to upload result manifest to " << url << ": " << e;
    }

    return 1;
  }
}

static int
bootstrap ()
{
  bootstrap_manifest bm {
    bootstrap_manifest::versions_type {
      {"bbot",    standard_version (BBOT_VERSION_STR)},
      {"libbbot", standard_version (LIBBBOT_VERSION_STR)},
      {"libbpkg", standard_version (LIBBPKG_VERSION_STR)},
      {"libbutl", standard_version (LIBBUTL_VERSION_STR)}
    }
  };

  serialize_manifest (bm, cout, "stdout", "bootstrap");

  return 0;
}

int bbot::
main (int argc, char* argv[])
try
{
  tracer trace ("main");

  // This is a little hack to make our baseutils for Windows work when called
  // with absolute path. In a nutshell, MSYS2's exec*p() doesn't search in the
  // parent's executable directory, only in PATH. And since we are running
  // without a shell (that would read /etc/profile which sets PATH to some
  // sensible values), we are only getting Win32 PATH values. And MSYS2 /bin
  // is not one of them. So what we are going to do is add /bin at the end of
  // PATH (which will be passed as is by the MSYS2 machinery). This will make
  // MSYS2 search in /bin (where our baseutils live). And for everyone else
  // this should be harmless since it is not a valid Win32 path.
  //
#ifdef _WIN32
  {
    string mp;
    if (optional<string> p = getenv ("PATH"))
    {
      mp = move (*p);
      mp += ';';
    }
    mp += "/bin";

    setenv ("PATH", mp);
  }
#endif

  // On POSIX ignore SIGPIPE which is signaled to a pipe-writing process if
  // the pipe reading end is closed. Note that by default this signal
  // terminates a process. Also note that there is no way to disable this
  // behavior on a file descriptor basis or for the write() function call.
  //
  // On Windows disable displaying error reporting dialog box. Note that the
  // error mode is inherited by child processes.
  //
#ifndef _WIN32
  if (signal (SIGPIPE, SIG_IGN) == SIG_ERR)
    fail << "unable to ignore broken pipe (SIGPIPE) signal: "
         << system_error (errno, std::generic_category ()); // Sanitize.
#else
  SetErrorMode (SetErrorMode (0) | // Returns the current mode.
                SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
#endif

  cli::argv_scanner scan (argc, argv, true);
  ops.parse (scan);

  verb = ops.verbose ();

  // @@ systemd 231 added JOURNAL_STREAM environment variable which allows
  //    detecting if stderr is connected to the journal.
  //
  if (ops.systemd_daemon ())
    systemd_diagnostics (false);

  // Version.
  //
  if (ops.version ())
  {
    cout << "bbot-worker " << BBOT_VERSION_ID << endl
         << "libbbot " << LIBBBOT_VERSION_ID << endl
         << "libbpkg " << LIBBPKG_VERSION_ID << endl
         << "libbutl " << LIBBUTL_VERSION_ID << endl
         << "Copyright (c) " << BBOT_COPYRIGHT << "." << endl
         << "This is free software released under the MIT license." << endl;

    return 0;
  }

  // Help.
  //
  if (ops.help ())
  {
    pager p ("bbot-worker help", false);
    print_bbot_worker_usage (p.stream ());

    // If the pager failed, assume it has issued some diagnostics.
    //
    return p.wait () ? 0 : 1;
  }

  // Figure out our mode.
  //
  if (ops.bootstrap () && ops.startup ())
    fail << "--bootstrap and --startup are mutually exclusive";

  enum class mode {boot, start, build} m (mode::build);

  if (ops.bootstrap ()) m = mode::boot;
  if (ops.startup ())   m = mode::start;

  if (ops.systemd_daemon ())
  {
    info << "bbot worker " << BBOT_VERSION_ID;
  }

  // Figure out our path (used for re-exec).
  //
  argv0 = process::path_search (argv[0], true);

  // Sort out the build directory.
  //
  if (ops.build_specified ())
    change_wd (trace,
               nullptr /* log */,
               ops.build (),
               true /* create */);

  // Sort out the environment directory.
  //
  try
  {
    env_dir = ops.environments_specified ()
      ? ops.environments ()
      : dir_path::home_directory ();

    if (!dir_exists (env_dir))
      throw_generic_error (ENOENT);
  }
  catch (const system_error& e)
  {
    fail << "invalid environment directory: " << e;
  }

  int r (1);
  switch (m)
  {
  case mode::boot:  r = bootstrap ();                            break;
  case mode::start: r = startup ();                              break;
  case mode::build: r = build (static_cast<size_t> (argc),
                               const_cast<const char**> (argv)); break;
  }

  return r;
}
catch (const failed&)
{
  return 1; // Diagnostics has already been issued.
}
catch (const cli::exception& e)
{
  error << e;
  return 1;
}

int
main (int argc, char* argv[])
{
  return bbot::main (argc, argv);
}