Add support for shallow fetch

8 files changed, 200 insertions, 84 deletions

diff --git a/bpkg/package.hxx b/bpkg/package.hxx
index 3567b10..5905dd2 100644
--- a/bpkg/package.hxx
+++ b/bpkg/package.hxx
@@ -597,11 +597,12 @@ namespace bpkg
     bool purge_src;
 
     // Path to the output directory of this package, if any. It is
-    // always relative to the configuration directory and currently
-    // is always <name>-<version>. It is only set once the package
-    // is configured and its main purse is to keep track of what
-    // needs to be cleaned by the user before a broken package can
-    // be purged. Note that it could be the same as out_root.
+    // always relative to the configuration directory, and is <name>
+    // for external packages and <name>-<version> for others. It is
+    // only set once the package is configured and its main purse is
+    // to keep track of what needs to be cleaned by the user before
+    // a broken package can be purged. Note that it could be the
+    // same as src_root.
     //
     optional<dir_path> out_root;
 
@@ -618,6 +619,19 @@ namespace bpkg
       return substate == package_substate::system;
     }
 
+    bool
+    external () const
+    {
+      return
+        // pkg-unpack <name>/<version>
+        //
+        (!repository.empty () && repository.directory_based ()) ||
+
+        // pkg-unpack --existing <dir>
+        //
+        (repository.empty () && !archive);
+    }
+
     // Represent the wildcard version with the "*" string. Represent naturally
     // all other versions.
     //
diff --git a/bpkg/pkg-build.cli b/bpkg/pkg-build.cli
index 3c5568f..a4d49ef 100644
--- a/bpkg/pkg-build.cli
+++ b/bpkg/pkg-build.cli
@@ -141,5 +141,11 @@ namespace bpkg
       "Print to \cb{STDOUT} what would be done without actually doing
        anything."
     }
+
+    bool --fetch-shallow
+    {
+      "Do not re-fetch complement and prerequisite repositories. Refer to
+       the \cb{--shallow} option in \l{bpkg-rep-fetch(1)} for details."
+    }
   };
 }
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index cc14f1e..8f80ff5 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -1178,7 +1178,7 @@ namespace bpkg
     }
 
     if (!locations.empty ())
-      rep_fetch (o, c, db, locations);
+      rep_fetch (o, c, db, locations, o.fetch_shallow ());
 
     // Expand <packages>@<location> arguments.
     //
diff --git a/bpkg/pkg-configure.cxx b/bpkg/pkg-configure.cxx
index c839d42..aaa426f 100644
--- a/bpkg/pkg-configure.cxx
+++ b/bpkg/pkg-configure.cxx
@@ -127,16 +127,9 @@ namespace bpkg
                        ? *p->src_root
                        : c / *p->src_root);
 
-    const repository_location& rl (p->repository);
-
-    // For external packages call the output directory <pkg>, rather than
-    // <pkg>-<ver>.
-    //
-    dir_path out_root (
-      (!rl.empty () && rl.directory_based ()) || // pkg-unpack <name>/<version>
-      ( rl.empty () && !p->archive)              // pkg-unpack --existing <dir>
-      ? c / dir_path (p->name)
-      : c / dir_path (p->name + "-" + p->version.string ()));
+    dir_path out_root (p->external ()
+                       ? c / dir_path (p->name)
+                       : c / dir_path (p->name + "-" + p->version.string ()));
 
     l4 ([&]{trace << "src_root: " << src_root << ", "
                   << "out_root: " << out_root;});
diff --git a/bpkg/rep-fetch.cli b/bpkg/rep-fetch.cli
index 719232d..a39aafb 100644
--- a/bpkg/rep-fetch.cli
+++ b/bpkg/rep-fetch.cli
@@ -39,5 +39,12 @@ namespace bpkg
   class rep_fetch_options: configuration_options
   {
     "\h|REP-FETCH OPTIONS|"
+
+    bool --shallow
+    {
+      "Do not re-fetch complement and prerequisite repositories of the
+       specified repositories unless the set of complements and/or
+       prerequisites has changed."
+    }
   };
 }
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 8f1528c..1f0f893 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -5,6 +5,7 @@
 #include <bpkg/rep-fetch.hxx>
 
 #include <set>
+#include <algorithm> // equal()
 
 #include <libbutl/manifest-parser.mxx>
 
@@ -431,6 +432,7 @@ namespace bpkg
              const shared_ptr<repository>& r,
              repositories& fetched,
              repositories& removed,
+             bool shallow,
              const string& reason = string ())
   {
     tracer trace ("rep_fetch(rep)");
@@ -471,42 +473,19 @@ namespace bpkg
         dr << " (" << reason << ")";
     }
 
-    // Register complements and prerequisites for potential removal unless
-    // they are fetched. Clear repository dependency sets afterwards.
-    //
-    auto remove = [&fetched, &removed] (const lazy_shared_ptr<repository>& rp)
-    {
-      shared_ptr<repository> r (rp.load ());
-      if (fetched.find (r) == fetched.end ())
-        removed.insert (move (r));
-    };
-
-    for (const lazy_shared_ptr<repository>& cr: r->complements)
-    {
-      // Remove the complement unless it is the root repository (see
-      // rep_fetch() for details).
-      //
-      if (cr.object_id () != "")
-        remove (cr);
-    }
-
-    for (const lazy_weak_ptr<repository>& pr: r->prerequisites)
-      remove (lazy_shared_ptr<repository> (pr));
-
-    r->complements.clear ();
-    r->prerequisites.clear ();
-
-    // Remove this repository from locations of the available packages it
-    // contains.
-    //
-    rep_remove_package_locations (t, r->name);
-
     // Load the repository and package manifests and use them to populate the
     // prerequisite and complement repository sets as well as available
     // packages.
     //
     rep_fetch_data rfd (rep_fetch (co, &conf, rl, true /* ignore_unknow */));
 
+    // Create the new prerequisite and complement repository sets. While doing
+    // this we may also reset the shallow flag if discover that any of these
+    // sets have changed.
+    //
+    repository::complements_type complements;
+    repository::prerequisites_type prerequisites;
+
     for (repository_manifest& rm: rfd.repositories)
     {
       repository_role rr (rm.effective_role ());
@@ -542,25 +521,16 @@ namespace bpkg
       {
         pr = make_shared<repository> (move (l));
         db.persist (pr); // Enter into session, important if recursive.
+
+        shallow = false;
       }
       else if (pr->location.url () != l.url ())
       {
         pr->location = move (l);
         db.update (r);
-      }
 
-      // Load the prerequisite repository.
-      //
-      string reason;
-      switch (rr)
-      {
-      case repository_role::complement:   reason = "complements ";     break;
-      case repository_role::prerequisite: reason = "prerequisite of "; break;
-      case repository_role::base: assert (false);
+        shallow = false;
       }
-      reason += r->name;
-
-      rep_fetch (co, conf, t, pr, fetched, removed, reason);
 
       // @@ What if we have duplicated? Ideally, we would like to check
       //    this once and as early as possible. The original idea was to
@@ -582,13 +552,13 @@ namespace bpkg
       case repository_role::complement:
         {
           l4 ([&]{trace << pr->name << " complement of " << r->name;});
-          r->complements.insert (lazy_shared_ptr<repository> (db, pr));
+          complements.insert (lazy_shared_ptr<repository> (db, pr));
           break;
         }
       case repository_role::prerequisite:
         {
           l4 ([&]{trace << pr->name << " prerequisite of " << r->name;});
-          r->prerequisites.insert (lazy_weak_ptr<repository> (db, pr));
+          prerequisites.insert (lazy_weak_ptr<repository> (db, pr));
           break;
         }
       case repository_role::base:
@@ -610,8 +580,8 @@ namespace bpkg
     case repository_type::git:
     case repository_type::dir:
       {
-        if (r->complements.empty () && r->prerequisites.empty ())
-          r->complements.insert (lazy_shared_ptr<repository> (db, string ()));
+        if (complements.empty () && prerequisites.empty ())
+          complements.insert (lazy_shared_ptr<repository> (db, string ()));
 
         break;
       }
@@ -624,9 +594,83 @@ namespace bpkg
       }
     }
 
-    // Save the changes to the repository object.
+    // Reset the shallow flag if the set of complements and/or prerequisites
+    // has changed.
     //
-    db.update (r);
+    // Note that weak pointers are generally incomparable (as can point to
+    // expired objects), and thus we can't compare the prerequisite sets
+    // directly.
+    //
+    if (shallow)
+      shallow = r->complements == complements &&
+                equal (r->prerequisites.begin (), r->prerequisites.end (),
+                       prerequisites.begin (), prerequisites.end (),
+                       [] (const lazy_weak_ptr<repository>& x,
+                           const lazy_weak_ptr<repository>& y)
+                       {
+                         return x.object_id () == y.object_id ();
+                       });
+
+    // Fetch prerequisites and complements, unless this is a shallow fetch.
+    //
+    if (!shallow)
+    {
+      // Register complements and prerequisites for potential removal unless
+      // they are fetched. Clear repository dependency sets afterwards.
+      //
+      auto rm = [&fetched, &removed] (const lazy_shared_ptr<repository>& rp)
+      {
+        shared_ptr<repository> r (rp.load ());
+         if (fetched.find (r) == fetched.end ())
+          removed.insert (move (r));
+      };
+
+      for (const lazy_shared_ptr<repository>& cr: r->complements)
+      {
+        // Remove the complement unless it is the root repository (see
+        // rep_fetch() for details).
+        //
+        if (cr.object_id () != "")
+          rm (cr);
+      }
+
+      for (const lazy_weak_ptr<repository>& pr: r->prerequisites)
+        rm (lazy_shared_ptr<repository> (pr));
+
+      r->complements   = move (complements);
+      r->prerequisites = move (prerequisites);
+
+      // Fetch complements.
+      //
+      for (const auto& cr: r->complements)
+      {
+        if (cr.object_id () != "")
+          rep_fetch (co,
+                     conf,
+                     t,
+                     cr.load (),
+                     fetched,
+                     removed,
+                     false /* shallow */,
+                     "complements " + r->name);
+      }
+
+      // Fetch prerequisites.
+      //
+      for (const auto& pr: r->prerequisites)
+        rep_fetch (co,
+                   conf,
+                   t,
+                   pr.load (),
+                   fetched,
+                   removed,
+                   false /* shallow */,
+                   "prerequisite of " + r->name);
+
+      // Save the changes to the repository object.
+      //
+      db.update (r);
+    }
 
     // "Suspend" session while persisting packages to reduce memory
     // consumption.
@@ -634,6 +678,11 @@ namespace bpkg
     session& s (session::current ());
     session::reset_current ();
 
+    // Remove this repository from locations of the available packages it
+    // contains.
+    //
+    rep_remove_package_locations (t, r->name);
+
     for (rep_fetch_data::package& fp: rfd.packages)
     {
       package_manifest& pm (fp.manifest);
@@ -702,7 +751,8 @@ namespace bpkg
   rep_fetch (const common_options& o,
              const dir_path& conf,
              transaction& t,
-             const vector<lazy_shared_ptr<repository>>& repos)
+             const vector<lazy_shared_ptr<repository>>& repos,
+             bool shallow)
   {
     // As a fist step we fetch repositories recursively building the list of
     // the former prerequisites and complements to be considered for removal.
@@ -722,7 +772,7 @@ namespace bpkg
       repositories removed;
 
       for (const lazy_shared_ptr<repository>& r: repos)
-        rep_fetch (o, conf, t, r.load (), fetched, removed);
+        rep_fetch (o, conf, t, r.load (), fetched, removed, shallow);
 
       // Finally, remove dangling repositories.
       //
@@ -752,8 +802,11 @@ namespace bpkg
   rep_fetch (const common_options& o,
              const dir_path& conf,
              database& db,
-             const vector<repository_location>& rls)
+             const vector<repository_location>& rls,
+             bool shallow)
   {
+    assert (session::has_current ());
+
     vector<lazy_shared_ptr<repository>> repos;
     repos.reserve (rls.size ());
 
@@ -775,7 +828,7 @@ namespace bpkg
       repos.emplace_back (r);
     }
 
-    rep_fetch (o, conf, t, repos);
+    rep_fetch (o, conf, t, repos, shallow);
 
     t.commit ();
   }
@@ -846,7 +899,7 @@ namespace bpkg
       }
     }
 
-    rep_fetch (o, c, t, repos);
+    rep_fetch (o, c, t, repos, o.shallow ());
 
     size_t rcount (0), pcount (0);
     if (verb)
diff --git a/bpkg/rep-fetch.hxx b/bpkg/rep-fetch.hxx
index 030b192..18a50e9 100644
--- a/bpkg/rep-fetch.hxx
+++ b/bpkg/rep-fetch.hxx
@@ -51,14 +51,17 @@ namespace bpkg
              bool ignore_unknown);
 
   // Add (or update) repository locations to the configuration and fetch
-  // them. On failure clean up the configuration (see rep_remove_clean() for
-  // details). Note that it should be called in session.
+  // them. If shallow is true, then don't fetch their prerequisite and/or
+  // complements unless the respective sets have changed. On failure clean up
+  // the configuration (see rep_remove_clean() for details). Note that it
+  // should be called in session.
   //
   void
   rep_fetch (const common_options&,
              const dir_path& conf,
              database&,
-             const vector<repository_location>&);
+             const vector<repository_location>&,
+             bool shallow);
 }
 
 #endif // BPKG_REP_FETCH_HXX
diff --git a/tests/rep-fetch.test b/tests/rep-fetch.test
index f0c0e2d..2339b91 100644
--- a/tests/rep-fetch.test
+++ b/tests/rep-fetch.test
@@ -142,19 +142,19 @@ $* 2>>/EOE != 0
 
     $* --trust-yes 2>>EOE;
       fetching pkg:build2.org/rep-fetch/bar/unstable
-      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
-      fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
       fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
       fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+      fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
       5 package(s) in 5 repository(s)
       EOE
 
     $* 2>>EOE
       fetching pkg:build2.org/rep-fetch/bar/unstable
-      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
-      fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
       fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
       fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+      fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
       5 package(s) in 5 repository(s)
       EOE
   }
@@ -166,20 +166,20 @@ $* 2>>/EOE != 0
 
     $* --trust-yes 2>>EOE &cfg/.bpkg/certs/**;
       fetching pkg:build2.org/rep-fetch/bar/unstable
-      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
-      fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
       fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
       fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+      fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
       fetching pkg:build2.org/rep-fetch/hello
       6 package(s) in 6 repository(s)
       EOE
 
     $* 2>>EOE
       fetching pkg:build2.org/rep-fetch/bar/unstable
-      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
-      fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
       fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
       fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+      fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
       fetching pkg:build2.org/rep-fetch/hello
       6 package(s) in 6 repository(s)
       EOE
@@ -193,19 +193,19 @@ $* 2>>/EOE != 0
     $* --trust-yes $rep/bar/unstable 2>>EOE;
       added pkg:build2.org/rep-fetch/bar/unstable
       fetching pkg:build2.org/rep-fetch/bar/unstable
-      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
-      fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
       fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
       fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+      fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
       5 package(s) in 5 repository(s)
       EOE
 
     $* 'pkg:build2.org/rep-fetch/bar/unstable' 2>>EOE;
       fetching pkg:build2.org/rep-fetch/bar/unstable
-      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/unstable)
-      fetching pkg:build2.org/rep-fetch/foo/stable (complements pkg:build2.org/rep-fetch/foo/testing)
       fetching pkg:build2.org/rep-fetch/bar/testing (complements pkg:build2.org/rep-fetch/bar/unstable)
       fetching pkg:build2.org/rep-fetch/bar/stable (complements pkg:build2.org/rep-fetch/bar/testing)
+      fetching pkg:build2.org/rep-fetch/foo/stable (prerequisite of pkg:build2.org/rep-fetch/bar/stable)
+      fetching pkg:build2.org/rep-fetch/foo/testing (prerequisite of pkg:build2.org/rep-fetch/bar/testing)
       5 package(s) in 5 repository(s)
       EOE
 
@@ -369,6 +369,46 @@ if ($remote != true)
         prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git)
       EOO
   }
+
+  : shallow
+  :
+  {
+    +$clone_cfg
+
+    : respected
+    :
+    {
+      $clone_cfg;
+      $* 2>!;
+
+      $* --shallow "dir:($rep/libbar.git)" 2>>"EOE";
+        fetching dir:($rep/libbar.git)
+        3 package\(s\) in 2 repository\(s\)
+        EOE
+
+      $rep_list >>"EOO"
+        dir:($rep/libbar.git) ($rep/libbar.git)
+          prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git)
+        EOO
+    }
+
+    : ignored
+    :
+    {
+      $clone_cfg;
+
+      $* --shallow 2>>"EOE";
+        fetching dir:($rep/libbar.git)
+        fetching dir:($rep/style-basic.git) \(prerequisite of dir:($rep/libbar.git)\)
+        3 package\(s\) in 2 repository\(s\)
+        EOE
+
+      $rep_list >>"EOO"
+        dir:($rep/libbar.git) ($rep/libbar.git)
+          prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git)
+        EOO
+    }
+  }
 }
 
 : git-rep