Move config, dist, test, and install modules into library

8 files changed, 1330 insertions, 0 deletions

diff --git a/libbuild2/dist/init.cxx b/libbuild2/dist/init.cxx
new file mode 100644
index 0000000..959b2dd
--- /dev/null
+++ b/libbuild2/dist/init.cxx
@@ -0,0 +1,192 @@
+// file      : libbuild2/dist/init.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <libbuild2/dist/init.hxx>
+
+#include <libbuild2/scope.hxx>
+#include <libbuild2/file.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+#include <libbuild2/config/utility.hxx>
+
+#include <libbuild2/dist/rule.hxx>
+#include <libbuild2/dist/module.hxx>
+#include <libbuild2/dist/operation.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace build2
+{
+  namespace dist
+  {
+    static const rule rule_;
+
+    bool
+    boot (scope& rs, const location&, unique_ptr<module_base>& mod)
+    {
+      tracer trace ("dist::boot");
+
+      l5 ([&]{trace << "for " << rs;});
+
+      // Register the meta-operation.
+      //
+      rs.insert_meta_operation (dist_id, mo_dist);
+
+      // Enter module variables. Do it during boot in case they get assigned
+      // in bootstrap.build (which is customary for, e.g., dist.package).
+      //
+      auto& vp (var_pool.rw (rs));
+
+      // Note: some overridable, some not.
+      //
+      // config.dist.archives is a list of archive extensions (e.g., zip,
+      // tar.gz) that can be optionally prefixed with a directory. If it is
+      // relative, then it is prefixed with config.dist.root. Otherwise, the
+      // archive is written to the absolute location.
+      //
+      // config.dist.checksums is a list of archive checksum extensions (e.g.,
+      // sha1, sha256) that can also be optionally prefixed with a directory
+      // with the same semantics as config.dist.archives. If the directory is
+      // absent, then the checksum file is written into the same directory as
+      // the corresponding archive.
+      //
+      vp.insert<abs_dir_path> ("config.dist.root",      true);
+      vp.insert<paths>        ("config.dist.archives",  true);
+      vp.insert<paths>        ("config.dist.checksums", true);
+      vp.insert<path>         ("config.dist.cmd",       true);
+
+      // Allow distribution of uncommitted projects. This is enforced by the
+      // version module.
+      //
+      vp.insert<bool> ("config.dist.uncommitted", true);
+
+      vp.insert<dir_path>     ("dist.root");
+      vp.insert<process_path> ("dist.cmd");
+      vp.insert<paths>        ("dist.archives");
+      vp.insert<paths>        ("dist.checksums");
+      vp.insert<paths>        ("dist.uncommitted");
+
+      vp.insert<bool> ("dist", variable_visibility::target); // Flag.
+
+      // Project's package name.
+      //
+      auto& v_d_p (
+        vp.insert<string> ("dist.package", variable_visibility::project));
+
+      // Create the module.
+      //
+      mod.reset (new module (v_d_p));
+
+      return false;
+    }
+
+    bool
+    init (scope& rs,
+          scope&,
+          const location& l,
+          unique_ptr<module_base>&,
+          bool first,
+          bool,
+          const variable_map& config_hints)
+    {
+      tracer trace ("dist::init");
+
+      if (!first)
+      {
+        warn (l) << "multiple dist module initializations";
+        return true;
+      }
+
+      const dir_path& out_root (rs.out_path ());
+      l5 ([&]{trace << "for " << out_root;});
+
+      assert (config_hints.empty ()); // We don't known any hints.
+
+      // Register our wildcard rule. Do it explicitly for the alias to prevent
+      // something like insert<target>(dist_id, test_id) taking precedence.
+      //
+      rs.rules.insert<target> (dist_id, 0, "dist", rule_);
+      rs.rules.insert<alias> (dist_id, 0, "dist.alias", rule_); //@@ outer?
+
+      // Configuration.
+      //
+      // Note that we don't use any defaults for root -- the location
+      // must be explicitly specified or we will complain if and when
+      // we try to dist.
+      //
+      bool s (config::specified (rs, "dist"));
+
+      // Adjust module priority so that the config.dist.* values are saved at
+      // the end of config.build.
+      //
+      if (s)
+        config::save_module (rs, "dist", INT32_MAX);
+
+      // dist.root
+      //
+      {
+        value& v (rs.assign ("dist.root"));
+
+        if (s)
+        {
+          if (lookup l = config::optional (rs, "config.dist.root"))
+            v = cast<dir_path> (l); // Strip abs_dir_path.
+        }
+      }
+
+      // dist.cmd
+      //
+      {
+        value& v (rs.assign<process_path> ("dist.cmd"));
+
+        if (s)
+        {
+          if (lookup l = config::required (rs,
+                                           "config.dist.cmd",
+                                           path ("install")).first)
+            v = run_search (cast<path> (l), true);
+        }
+      }
+
+      // dist.archives
+      // dist.checksums
+      //
+      {
+        value& a (rs.assign ("dist.archives"));
+        value& c (rs.assign ("dist.checksums"));
+
+        if (s)
+        {
+          if (lookup l = config::optional (rs, "config.dist.archives"))
+            a = *l;
+
+          if (lookup l = config::optional (rs, "config.dist.checksums"))
+          {
+            c = *l;
+
+            if (!c.empty () && (!a || a.empty ()))
+              fail << "config.dist.checksums specified without "
+                   << "config.dist.archives";
+
+          }
+        }
+      }
+
+      // dist.uncommitted
+      //
+      // Omit it from the configuration unless specified.
+      //
+      config::omitted (rs, "config.dist.uncommitted");
+
+      return true;
+    }
+
+    module_functions
+    build2_dist_load ()
+    {
+      return module_functions {&boot, &init};
+    }
+  }
+}
diff --git a/libbuild2/dist/init.hxx b/libbuild2/dist/init.hxx
new file mode 100644
index 0000000..41c82a7
--- /dev/null
+++ b/libbuild2/dist/init.hxx
@@ -0,0 +1,36 @@
+// file      : libbuild2/dist/init.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_DIST_INIT_HXX
+#define LIBBUILD2_DIST_INIT_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/module.hxx>
+
+#include <libbuild2/export.hxx>
+
+namespace build2
+{
+  namespace dist
+  {
+    bool
+    boot (scope&, const location&, unique_ptr<module_base>&);
+
+    bool
+    init (scope&,
+          scope&,
+          const location&,
+          unique_ptr<module_base>&,
+          bool,
+          bool,
+          const variable_map&);
+
+    extern "C" LIBBUILD2_SYMEXPORT module_functions
+    build2_dist_load ();
+  }
+}
+
+#endif // LIBBUILD2_DIST_INIT_HXX
diff --git a/libbuild2/dist/module.cxx b/libbuild2/dist/module.cxx
new file mode 100644
index 0000000..e9b9955
--- /dev/null
+++ b/libbuild2/dist/module.cxx
@@ -0,0 +1,15 @@
+// file      : libbuild2/dist/module.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <libbuild2/dist/module.hxx>
+
+using namespace std;
+
+namespace build2
+{
+  namespace dist
+  {
+    const string module::name ("dist");
+  }
+}
diff --git a/libbuild2/dist/module.hxx b/libbuild2/dist/module.hxx
new file mode 100644
index 0000000..abc1400
--- /dev/null
+++ b/libbuild2/dist/module.hxx
@@ -0,0 +1,71 @@
+// file      : libbuild2/dist/module.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_DIST_MODULE_HXX
+#define LIBBUILD2_DIST_MODULE_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/module.hxx>
+#include <libbuild2/variable.hxx>
+
+#include <libbuild2/export.hxx>
+
+namespace build2
+{
+  namespace dist
+  {
+    struct LIBBUILD2_SYMEXPORT module: module_base
+    {
+      static const string name;
+
+      const variable& var_dist_package;
+
+      // Distribution post-processing callbacks.
+      //
+      // The last component in the pattern may contain shell wildcards. If the
+      // path contains a directory, then it is matched from the distribution
+      // root only. Otherwise, it is matched against all the files being
+      // distributed. For example:
+      //
+      // buildfile        - every buildfile
+      // ./buildfile      - root buildfile only
+      // tests/buildfile  - tests/buildfile only
+      //
+      // The callback is called with the absolute path of the matching file
+      // after it has been copied to the distribution directory. The project's
+      // root scope and callback-specific data are passed along.
+      //
+      // Note that if registered, the callbacks are also called (recursively)
+      // in subprojects.
+      //
+      using callback_func = void (const path&, const scope&, void*);
+
+      void
+      register_callback (path pattern, callback_func* f, void* data)
+      {
+        callbacks_.push_back (callback {move (pattern), f, data});
+      }
+
+      // Implementation details.
+      //
+      module (const variable& v_d_p)
+          : var_dist_package (v_d_p) {}
+
+    public:
+      struct callback
+      {
+        const path     pattern;
+        callback_func* function;
+        void*          data;
+      };
+      using callbacks = vector<callback>;
+
+      callbacks callbacks_;
+    };
+  }
+}
+
+#endif // LIBBUILD2_DIST_MODULE_HXX
diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx
new file mode 100644
index 0000000..ac3912e
--- /dev/null
+++ b/libbuild2/dist/operation.cxx
@@ -0,0 +1,868 @@
+// file      : libbuild2/dist/operation.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <libbuild2/dist/operation.hxx>
+
+#include <libbutl/sha1.mxx>
+#include <libbutl/sha256.mxx>
+
+#include <libbutl/filesystem.mxx> // path_match()
+
+#include <libbuild2/file.hxx>
+#include <libbuild2/dump.hxx>
+#include <libbuild2/scope.hxx>
+#include <libbuild2/target.hxx>
+#include <libbuild2/context.hxx>
+#include <libbuild2/algorithm.hxx>
+#include <libbuild2/filesystem.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+#include <libbuild2/dist/module.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace build2
+{
+  namespace dist
+  {
+    // install -d <dir>
+    //
+    static void
+    install (const process_path& cmd, const dir_path&);
+
+    // install <file> <dir>
+    //
+    // Return the destination file path.
+    //
+    static path
+    install (const process_path& cmd, const file&, const dir_path&);
+
+    // tar|zip ... <dir>/<pkg>.<ext> <pkg>
+    //
+    // Return the archive file path.
+    //
+    static path
+    archive (const dir_path& root,
+             const string& pkg,
+             const dir_path& dir,
+             const string& ext);
+
+    // <ext>sum <arc> > <dir>/<arc>.<ext>
+    //
+    // Return the checksum file path.
+    //
+    static path
+    checksum (const path& arc, const dir_path& dir, const string& ext);
+
+    static operation_id
+    dist_operation_pre (const values&, operation_id o)
+    {
+      if (o != default_id)
+        fail << "explicit operation specified for meta-operation dist";
+
+      return o;
+    }
+
+    static void
+    dist_execute (const values&, action, action_targets& ts,
+                  uint16_t, bool prog)
+    {
+      tracer trace ("dist_execute");
+
+      // For now we assume all the targets are from the same project.
+      //
+      const target& t (ts[0].as_target ());
+      const scope* rs (t.base_scope ().root_scope ());
+
+      if (rs == nullptr)
+        fail << "out of project target " << t;
+
+      const dir_path& out_root (rs->out_path ());
+      const dir_path& src_root (rs->src_path ());
+
+      if (out_root == src_root)
+        fail << "in-tree distribution of target " << t <<
+          info << "distribution requires out-of-tree build";
+
+      // Make sure we have the necessary configuration before we get down to
+      // business.
+      //
+      auto l (rs->vars["dist.root"]);
+
+      if (!l || l->empty ())
+        fail << "unknown root distribution directory" <<
+          info << "did you forget to specify config.dist.root?";
+
+      // We used to complain if dist.root does not exist but then, similar
+      // to install, got tired of user's complaints. So now we just let
+      // install -d for the package directory create it if necessary.
+      //
+      const dir_path& dist_root (cast<dir_path> (l));
+
+      l = rs->vars["dist.package"];
+
+      if (!l || l->empty ())
+        fail << "unknown distribution package name" <<
+          info << "did you forget to set dist.package?";
+
+      const string& dist_package (cast<string> (l));
+      const process_path& dist_cmd (cast<process_path> (rs->vars["dist.cmd"]));
+
+      // Verify all the targets are from the same project.
+      //
+      for (const action_target& at: ts)
+      {
+        const target& t (at.as_target ());
+
+        if (rs != t.base_scope ().root_scope ())
+          fail << "target " << t << " is from a different project" <<
+            info << "one dist meta-operation can handle one project" <<
+            info << "consider using several dist meta-operations";
+      }
+
+      // We used to print 'dist <target>' at verbosity level 1 but that has
+      // proven to be just noise. Though we still want to print something
+      // since otherwise, once the progress line is cleared, we may end up
+      // with nothing printed at all.
+      //
+      // Note that because of this we can also suppress diagnostics noise
+      // (e.g., output directory creation) in all the operations below.
+      //
+      if (verb == 1)
+        text << "dist " << dist_package;
+
+      // Match a rule for every operation supported by this project. Skip
+      // default_id.
+      //
+      // Note that we are not calling operation_pre/post() callbacks here
+      // since the meta operation is dist and we know what we are doing.
+      //
+      values params;
+      const path locf ("<dist>");
+      const location loc (&locf); // Dummy location.
+
+      const operations& ops (rs->root_extra->operations);
+
+      for (operations::size_type id (default_id + 1); // Skip default_id.
+           id < ops.size ();
+           ++id)
+      {
+        if (const operation_info* oif = ops[id])
+        {
+          // Skip aliases (e.g., update-for-install). In fact, one can argue
+          // the default update should be sufficient since it is assumed to
+          // update all prerequisites and we no longer support ad hoc stuff
+          // like test.input. Though here we are using the dist meta-operation,
+          // not perform.
+          //
+          if (oif->id != id)
+            continue;
+
+          // Use standard (perform) match.
+          //
+          if (oif->pre != nullptr)
+          {
+            if (operation_id pid = oif->pre (params, dist_id, loc))
+            {
+              const operation_info* poif (ops[pid]);
+              set_current_oif (*poif, oif, false /* diag_noise */);
+              action a (dist_id, poif->id, oif->id);
+              match (params, a, ts,
+                     1     /* diag (failures only) */,
+                     false /* progress */);
+            }
+          }
+
+          set_current_oif (*oif, nullptr, false /* diag_noise */);
+          action a (dist_id, oif->id);
+          match (params, a, ts,
+                 1     /* diag (failures only) */,
+                 false /* progress */);
+
+          if (oif->post != nullptr)
+          {
+            if (operation_id pid = oif->post (params, dist_id))
+            {
+              const operation_info* poif (ops[pid]);
+              set_current_oif (*poif, oif, false /* diag_noise */);
+              action a (dist_id, poif->id, oif->id);
+              match (params, a, ts,
+                     1     /* diag (failures only) */,
+                     false /* progress */);
+            }
+          }
+        }
+      }
+
+      // Add buildfiles that are not normally loaded as part of the project,
+      // for example, the export stub. They will still be ignored on the next
+      // step if the user explicitly marked them dist=false.
+      //
+      auto add_adhoc = [&trace] (const scope& rs, const path& f)
+      {
+        path p (rs.src_path () / f);
+        if (exists (p))
+        {
+          dir_path d (p.directory ());
+
+          // Figure out if we need out.
+          //
+          dir_path out (rs.src_path () != rs.out_path ()
+                        ? out_src (d, rs)
+                        : dir_path ());
+
+          targets.insert<buildfile> (
+            move (d),
+            move (out),
+            p.leaf ().base ().string (),
+            p.extension (),              // Specified.
+            trace);
+        }
+      };
+
+      add_adhoc (*rs, rs->root_extra->export_file);
+
+      // The same for subprojects that have been loaded.
+      //
+      if (auto l = rs->vars[var_subprojects])
+      {
+        for (auto p: cast<subprojects> (l))
+        {
+          const dir_path& pd (p.second);
+          dir_path out_nroot (out_root / pd);
+          const scope& nrs (scopes.find (out_nroot));
+
+          if (nrs.out_path () != out_nroot) // This subproject not loaded.
+            continue;
+
+          if (!nrs.src_path ().sub (src_root)) // Not a strong amalgamation.
+            continue;
+
+          add_adhoc (nrs, nrs.root_extra->export_file);
+        }
+      }
+
+      // Collect the files. We want to take the snapshot of targets since
+      // updating some of them may result in more targets being entered.
+      //
+      // Note that we are not showing progress here (e.g., "N targets to
+      // distribute") since it will be useless (too fast).
+      //
+      action_targets files;
+      const variable& dist_var (var_pool["dist"]);
+
+      for (const auto& pt: targets)
+      {
+        file* ft (pt->is_a<file> ());
+
+        if (ft == nullptr) // Not a file.
+          continue;
+
+        if (ft->dir.sub (src_root))
+        {
+          // Include unless explicitly excluded.
+          //
+          auto l ((*ft)[dist_var]);
+
+          if (l && !cast<bool> (l))
+            l5 ([&]{trace << "excluding " << *ft;});
+          else
+            files.push_back (ft);
+
+          continue;
+        }
+
+        if (ft->dir.sub (out_root))
+        {
+          // Exclude unless explicitly included.
+          //
+          auto l ((*ft)[dist_var]);
+
+          if (l && cast<bool> (l))
+          {
+            l5 ([&]{trace << "including " << *ft;});
+            files.push_back (ft);
+          }
+
+          continue;
+        }
+      }
+
+      // Make sure what we need to distribute is up to date.
+      //
+      {
+        if (mo_perform.meta_operation_pre != nullptr)
+          mo_perform.meta_operation_pre (params, loc);
+
+        // This is a hack since according to the rules we need to completely
+        // reset the state. We could have done that (i.e., saved target names
+        // and then re-searched them in the new tree) but that would just slow
+        // things down while this little cheat seems harmless (i.e., assume
+        // the dist mete-opreation is "compatible" with perform).
+        //
+        // Note also that we don't do any structured result printing.
+        //
+        size_t on (current_on);
+        set_current_mif (mo_perform);
+        current_on = on + 1;
+
+        if (mo_perform.operation_pre != nullptr)
+          mo_perform.operation_pre (params, update_id);
+
+        set_current_oif (op_update, nullptr, false /* diag_noise */);
+
+        action a (perform_id, update_id);
+
+        mo_perform.match   (params, a, files,
+                            1    /* diag (failures only) */,
+                            prog /* progress */);
+
+        mo_perform.execute (params, a, files,
+                            1    /* diag (failures only) */,
+                            prog /* progress */);
+
+        if (mo_perform.operation_post != nullptr)
+          mo_perform.operation_post (params, update_id);
+
+        if (mo_perform.meta_operation_post != nullptr)
+          mo_perform.meta_operation_post (params);
+      }
+
+      dir_path td (dist_root / dir_path (dist_package));
+
+      // Clean up the target directory.
+      //
+      if (build2::rmdir_r (td, true, 2) == rmdir_status::not_empty)
+        fail << "unable to clean target directory " << td;
+
+      auto_rmdir rm_td (td); // Clean it up if things go bad.
+      install (dist_cmd, td);
+
+      // Copy over all the files. Apply post-processing callbacks.
+      //
+      module& mod (*rs->lookup_module<module> (module::name));
+
+      prog = prog && show_progress (1 /* max_verb */);
+      size_t prog_percent (0);
+
+      for (size_t i (0), n (files.size ()); i != n; ++i)
+      {
+        const file& t (*files[i].as_target ().is_a<file> ());
+
+        // Figure out where this file is inside the target directory.
+        //
+        bool src (t.dir.sub (src_root));
+        dir_path dl (src ? t.dir.leaf (src_root) : t.dir.leaf (out_root));
+
+        dir_path d (td / dl);
+        if (!exists (d))
+          install (dist_cmd, d);
+
+        path r (install (dist_cmd, t, d));
+
+        // See if this file is in a subproject.
+        //
+        const scope* srs (rs);
+        const module::callbacks* cbs (&mod.callbacks_);
+
+        if (auto l = rs->vars[var_subprojects])
+        {
+          for (auto p: cast<subprojects> (l))
+          {
+            const dir_path& pd (p.second);
+            if (dl.sub (pd))
+            {
+              srs = &scopes.find (out_root / pd);
+
+              if (auto* m = srs->lookup_module<module> (module::name))
+                cbs = &m->callbacks_;
+              else
+                fail << "dist module not loaded in subproject " << pd;
+
+              break;
+            }
+          }
+        }
+
+        for (module::callback cb: *cbs)
+        {
+          const path& pat (cb.pattern);
+
+          // If we have a directory, then it should be relative to the project
+          // root.
+          //
+          if (!pat.simple ())
+          {
+            assert (pat.relative ());
+
+            dir_path d ((src ? srs->src_path () : srs->out_path ()) /
+                        pat.directory ());
+            d.normalize ();
+
+            if (d != t.dir)
+              continue;
+          }
+
+          if (path_match (pat.leaf ().string (), t.path ().leaf ().string ()))
+            cb.function (r, *srs, cb.data);
+        }
+
+        if (prog)
+        {
+          // Note that this is not merely an optimization since if stderr is
+          // not a terminal, we print real lines for progress.
+          //
+          size_t p ((i * 100) / n);
+
+          if (prog_percent != p)
+          {
+            prog_percent = p;
+
+            diag_progress_lock pl;
+            diag_progress  = ' ';
+            diag_progress += to_string (prog_percent);
+            diag_progress += "% of targets distributed";
+          }
+        }
+      }
+
+      // Clear the progress if shown.
+      //
+      if (prog)
+      {
+        diag_progress_lock pl;
+        diag_progress.clear ();
+      }
+
+      rm_td.cancel ();
+
+      // Archive and checksum if requested.
+      //
+      if (lookup as = rs->vars["dist.archives"])
+      {
+        lookup cs (rs->vars["dist.checksums"]);
+
+        // Split the dist.{archives,checksums} value into a directory and
+        // extension.
+        //
+        auto split = [] (const path& p, const dir_path& r, const char* what)
+        {
+          dir_path d (p.relative () ? r : dir_path ());
+          d /= p.directory ();
+
+          const string& s (p.string ());
+          size_t i (path::traits_type::find_leaf (s));
+
+          if (i == string::npos)
+            fail << "invalid extension '" << s << "' in " << what;
+
+          if (s[i] == '.') // Skip the dot if specified.
+            ++i;
+
+          return pair<dir_path, string> (move (d), string (s, i));
+        };
+
+        for (const path& p: cast<paths> (as))
+        {
+          auto ap (split (p, dist_root, "dist.archives"));
+          path a (archive (dist_root, dist_package, ap.first, ap.second));
+
+          if (cs)
+          {
+            for (const path& p: cast<paths> (cs))
+            {
+              auto cp (split (p, ap.first, "dist.checksums"));
+              checksum (a, cp.first, cp.second);
+            }
+          }
+        }
+      }
+    }
+
+    // install -d <dir>
+    //
+    static void
+    install (const process_path& cmd, const dir_path& d)
+    {
+      path reld (relative (d));
+
+      cstrings args {cmd.recall_string (), "-d"};
+
+      args.push_back ("-m");
+      args.push_back ("755");
+      args.push_back (reld.string ().c_str ());
+      args.push_back (nullptr);
+
+      if (verb >= 2)
+        print_process (args);
+
+      run (cmd, args);
+    }
+
+    // install <file> <dir>
+    //
+    static path
+    install (const process_path& cmd, const file& t, const dir_path& d)
+    {
+      dir_path reld (relative (d));
+      path relf (relative (t.path ()));
+
+      cstrings args {cmd.recall_string ()};
+
+      // Preserve timestamps. This could becomes important if, for
+      // example, we have pre-generated sources. Note that the
+      // install-sh script doesn't support this option, while both
+      // Linux and BSD install's do.
+      //
+      args.push_back ("-p");
+
+      // Assume the file is executable if the owner has execute
+      // permission, in which case we make it executable for
+      // everyone.
+      //
+      args.push_back ("-m");
+      args.push_back (
+        (path_permissions (t.path ()) & permissions::xu) == permissions::xu
+        ? "755"
+        : "644");
+
+      args.push_back (relf.string ().c_str ());
+      args.push_back (reld.string ().c_str ());
+      args.push_back (nullptr);
+
+      if (verb >= 2)
+        print_process (args);
+
+      run (cmd, args);
+
+      return d / relf.leaf ();
+    }
+
+    static path
+    archive (const dir_path& root,
+             const string& pkg,
+             const dir_path& dir,
+             const string& e)
+    {
+      path an (pkg + '.' + e);
+
+      // Delete old archive for good measure.
+      //
+      path ap (dir / an);
+      if (exists (ap, false))
+        rmfile (ap);
+
+      // Use zip for .zip archives. Also recognize and handle a few well-known
+      // tar.xx cases (in case tar doesn't support -a or has other issues like
+      // MSYS). Everything else goes to tar in the auto-compress mode (-a).
+      //
+      cstrings args;
+
+      // Separate compressor (gzip, xz, etc) state.
+      //
+      size_t i (0);        // Command line start or 0 if not used.
+      auto_rmfile out_rm;  // Output file cleanup (must come first).
+      auto_fd out_fd;      // Output file.
+
+      if (e == "zip")
+      {
+        // On Windows we use libarchive's bsdtar (zip is an MSYS executabales).
+        //
+        // While not explicitly stated, the compression-level option works
+        // for zip archives.
+        //
+#ifdef _WIN32
+        args = {"bsdtar",
+                "-a", // -a with the .zip extension seems to be the only way.
+                "--options=compression-level=9",
+                "-cf", ap.string ().c_str (),
+                pkg.c_str (),
+                nullptr};
+#else
+        args = {"zip",
+                "-9",
+                "-rq", ap.string ().c_str (),
+                pkg.c_str (),
+                nullptr};
+#endif
+      }
+      else
+      {
+        // On Windows we use libarchive's bsdtar with auto-compression (tar
+        // itself and quite a few compressors are MSYS executables).
+        //
+        const char* l (nullptr); // Compression level (option).
+
+#ifdef _WIN32
+        const char* tar = "bsdtar";
+
+        if (e == "tar.gz")
+          l = "--options=compression-level=9";
+#else
+        const char* tar = "tar";
+
+        // For gzip it's a good idea to use -9 by default. For bzip2, -9 is
+        // the default. And for xz, -9 is not recommended as the default due
+        // memory requirements.
+        //
+        // Note also that the compression level can be altered via the GZIP
+        // (GZIP_OPT also seems to work), BZIP2, and XZ_OPT environment
+        // variables, respectively.
+        //
+        const char* c (nullptr);
+
+        if      (e == "tar.gz")  { c = "gzip";  l = "-9"; }
+        else if (e == "tar.xz")  { c = "xz";              }
+        else if (e == "tar.bz2") { c = "bzip2";           }
+
+        if (c != nullptr)
+        {
+          args = {tar,
+                  "--format", "ustar",
+                  "-cf", "-",
+                  pkg.c_str (),
+                  nullptr};
+
+          i = args.size ();
+          args.push_back (c);
+          if (l != nullptr)
+            args.push_back (l);
+          args.push_back (nullptr);
+          args.push_back (nullptr); // Pipe end.
+
+          try
+          {
+            out_fd = fdopen (ap,
+                             fdopen_mode::out      | fdopen_mode::binary |
+                             fdopen_mode::truncate | fdopen_mode::create);
+            out_rm = auto_rmfile (ap);
+          }
+          catch (const io_error& e)
+          {
+            fail << "unable to open " << ap << ": " << e;
+          }
+        }
+        else
+#endif
+        if (e == "tar")
+          args = {tar,
+                  "--format", "ustar",
+                  "-cf", ap.string ().c_str (),
+                  pkg.c_str (),
+                  nullptr};
+        else
+        {
+          args = {tar,
+                  "--format", "ustar",
+                  "-a"};
+
+          if (l != nullptr)
+            args.push_back (l);
+
+          args.push_back ("-cf");
+          args.push_back (ap.string ().c_str ());
+          args.push_back (pkg.c_str ());
+          args.push_back (nullptr);
+        }
+      }
+
+      process_path app; // Archiver path.
+      process_path cpp; // Compressor path.
+
+      app = run_search (args[0]);
+
+      if (i != 0)
+        cpp = run_search (args[i]);
+
+      if (verb >= 2)
+        print_process (args);
+      else if (verb)
+        text << args[0] << ' ' << ap;
+
+      process apr;
+      process cpr;
+
+      // Change the archiver's working directory to dist_root.
+      //
+      apr = run_start (app,
+                       args.data (),
+                       0                 /* stdin  */,
+                       (i != 0 ? -1 : 1) /* stdout */,
+                       true              /* error */,
+                       root);
+
+      // Start the compressor if required.
+      //
+      if (i != 0)
+      {
+        cpr = run_start (cpp,
+                         args.data () + i,
+                         apr.in_ofd.get () /* stdin  */,
+                         out_fd.get ()     /* stdout */);
+
+        cpr.in_ofd.reset (); // Close the archiver's stdout on our side.
+        run_finish (args.data () + i, cpr);
+      }
+
+      run_finish (args.data (), apr);
+
+      out_rm.cancel ();
+      return ap;
+    }
+
+    static path
+    checksum (const path& ap, const dir_path& dir, const string& e)
+    {
+      path     an (ap.leaf ());
+      dir_path ad (ap.directory ());
+
+      path cn (an + '.' + e);
+
+      // Delete old checksum for good measure.
+      //
+      path cp (dir / cn);
+      if (exists (cp, false))
+        rmfile (cp);
+
+      auto_rmfile c_rm; // Note: must come first.
+      auto_fd c_fd;
+      try
+      {
+        c_fd = fdopen (cp,
+                       fdopen_mode::out     |
+                       fdopen_mode::create  |
+                       fdopen_mode::truncate);
+        c_rm = auto_rmfile (cp);
+      }
+      catch (const io_error& e)
+      {
+        fail << "unable to open " << cp << ": " << e;
+      }
+
+      // The plan is as follows: look for the <ext>sum program (e.g., sha1sum,
+      // md5sum, etc). If found, then use that, otherwise, fall back to our
+      // built-in checksum calculation support.
+      //
+      // There are two benefits to first trying the external program: it may
+      // supports more checksum algorithms and could be faster than our
+      // built-in code.
+      //
+      string pn (e + "sum");
+      process_path pp (process::try_path_search (pn, true /* init */));
+
+      if (!pp.empty ())
+      {
+        const char* args[] {
+          pp.recall_string (),
+          "-b" /* binary */,
+          an.string ().c_str (),
+          nullptr};
+
+        if (verb >= 2)
+          print_process (args);
+        else if (verb)
+          text << args[0] << ' ' << cp;
+
+        // Note that to only get the archive name (without the directory) in
+        // the output we have to run from the archive's directory.
+        //
+        process pr (run_start (pp,
+                               args,
+                               0             /* stdin */,
+                               c_fd.get ()   /* stdout */,
+                               true          /* error */,
+                               ad            /* cwd */));
+        run_finish (args, pr);
+      }
+      else
+      {
+        string (*f) (ifdstream&);
+
+        // Note: remember to update info: below if adding another algorithm.
+        //
+        if (e == "sha1")
+          f = [] (ifdstream& i) -> string {return sha1 (i).string ();};
+        else if (e == "sha256")
+          f = [] (ifdstream& i) -> string {return sha256 (i).string ();};
+        else
+          fail << "no built-in support for checksum algorithm " << e
+               << " nor " << e << "sum program found" <<
+            info << "built-in support is available for sha1, sha256" << endf;
+
+        if (verb >= 2)
+          text << "cat >" << cp;
+        else if (verb)
+          text << e << "sum " << cp;
+
+        string c;
+        try
+        {
+          ifdstream is (ap, fdopen_mode::in | fdopen_mode::binary);
+          c = f (is);
+          is.close ();
+        }
+        catch (const io_error& e)
+        {
+          fail << "unable to read " << ap << ": " << e;
+        }
+
+        try
+        {
+          ofdstream os (move (c_fd));
+          os << c << " *" << an << endl;
+          os.close ();
+        }
+        catch (const io_error& e)
+        {
+          fail << "unable to write " << cp << ": " << e;
+        }
+      }
+
+      c_rm.cancel ();
+      return cp;
+    }
+
+    static include_type
+    dist_include (action,
+                  const target&,
+                  const prerequisite_member& p,
+                  include_type i)
+    {
+      tracer trace ("dist_include");
+
+      // Override excluded to adhoc so that every source is included into the
+      // distribution. Note that this should be harmless to a custom rule
+      // given the prescribed semantics of adhoc (match/execute but otherwise
+      // ignore) is followed.
+      //
+      if (i == include_type::excluded)
+      {
+        l5 ([&]{trace << "overriding exclusion of " << p;});
+        i = include_type::adhoc;
+      }
+
+      return i;
+    }
+
+    const meta_operation_info mo_dist {
+      dist_id,
+      "dist",
+      "distribute",
+      "distributing",
+      "distributed",
+      "has nothing to distribute", // We cannot "be distributed".
+      true,    // bootstrap_outer
+      nullptr, // meta-operation pre
+      &dist_operation_pre,
+      &load,   // normal load
+      &search, // normal search
+      nullptr, // no match (see dist_execute()).
+      &dist_execute,
+      nullptr, // operation post
+      nullptr, // meta-operation post
+      &dist_include
+    };
+  }
+}
diff --git a/libbuild2/dist/operation.hxx b/libbuild2/dist/operation.hxx
new file mode 100644
index 0000000..aa59c36
--- /dev/null
+++ b/libbuild2/dist/operation.hxx
@@ -0,0 +1,21 @@
+// file      : libbuild2/dist/operation.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_DIST_OPERATION_HXX
+#define LIBBUILD2_DIST_OPERATION_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/operation.hxx>
+
+namespace build2
+{
+  namespace dist
+  {
+    extern const meta_operation_info mo_dist;
+  }
+}
+
+#endif // LIBBUILD2_DIST_OPERATION_HXX
diff --git a/libbuild2/dist/rule.cxx b/libbuild2/dist/rule.cxx
new file mode 100644
index 0000000..357d70e
--- /dev/null
+++ b/libbuild2/dist/rule.cxx
@@ -0,0 +1,88 @@
+// file      : libbuild2/dist/rule.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <libbuild2/dist/rule.hxx>
+
+#include <libbuild2/scope.hxx>
+#include <libbuild2/target.hxx>
+#include <libbuild2/algorithm.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+using namespace std;
+
+namespace build2
+{
+  namespace dist
+  {
+    bool rule::
+    match (action, target&, const string&) const
+    {
+      return true; // We always match.
+    }
+
+    recipe rule::
+    apply (action a, target& t) const
+    {
+      const dir_path& out_root (t.root_scope ().out_path ());
+
+      // If we can, go inside see-through groups.
+      //
+      for (prerequisite_member p:
+             group_prerequisite_members (a, t, members_mode::maybe))
+      {
+        // Note: no exclusion tests, we want all of them (and see also the
+        // dist_include() override).
+
+        // Skip prerequisites imported from other projects.
+        //
+        if (p.proj ())
+          continue;
+
+        // We used to always search and match but that resulted in the
+        // undesirable behavior in case one of the "source" files is
+        // missing. In this case we would enter a target as "output", this
+        // rule would match it, and then dist_execute() would ignore it by
+        // default.
+        //
+        // So now if this is a file target (we still want to always "see
+        // through" other targets like aliases), we will only match it if (1)
+        // it exists in src or (2) it exists as a target. It feels like we
+        // don't need the stronger "... and not implied" condition since if it
+        // is mentioned as a target, then it is in out (we don't do the same
+        // target in both src/out).
+        //
+        // @@ Note that this is still an issue in a custom dist rule.
+        //
+        const target* pt (nullptr);
+        if (p.is_a<file> ())
+        {
+          pt = p.load ();
+
+          if (pt == nullptr)
+          {
+            // Search for an existing target or existing file in src.
+            //
+            const prerequisite_key& k (p.key ());
+            pt = k.tk.type->search (t, k);
+
+            if (pt == nullptr)
+              fail << "prerequisite " << k << " is not existing source file "
+                   << "nor known output target" << endf;
+
+            search_custom (p.prerequisite, *pt); // Cache.
+          }
+        }
+        else
+          pt = &p.search (t);
+
+        // Don't match targets that are outside of our project.
+        //
+        if (pt->dir.sub (out_root))
+          build2::match (a, *pt);
+      }
+
+      return noop_recipe; // We will never be executed.
+    }
+  }
+}
diff --git a/libbuild2/dist/rule.hxx b/libbuild2/dist/rule.hxx
new file mode 100644
index 0000000..df32de5
--- /dev/null
+++ b/libbuild2/dist/rule.hxx
@@ -0,0 +1,39 @@
+// file      : libbuild2/dist/rule.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_DIST_RULE_HXX
+#define LIBBUILD2_DIST_RULE_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/rule.hxx>
+#include <libbuild2/action.hxx>
+#include <libbuild2/target.hxx>
+
+namespace build2
+{
+  namespace dist
+  {
+    // This is the default rule that simply matches all the prerequisites.
+    //
+    // A custom rule (usually the same as perform_update) may be necessary to
+    // establish group links (so that we see the dist variable set on a
+    // group).
+    //
+    class rule: public build2::rule
+    {
+    public:
+      rule () {}
+
+      virtual bool
+      match (action, target&, const string&) const override;
+
+      virtual recipe
+      apply (action, target&) const override;
+    };
+  }
+}
+
+#endif // LIBBUILD2_DIST_RULE_HXX