Fix deinit command to try bpkg-pkg-build --deorphan if bpkg-pkg drop fails

5 files changed, 404 insertions, 44 deletions

diff --git a/bdep/deinit.cli b/bdep/deinit.cli
index e78cdbe..3ec0e72 100644
--- a/bdep/deinit.cli
+++ b/bdep/deinit.cli
@@ -47,6 +47,12 @@ namespace bdep
        configuration. This mode is primarily useful when the configuration
        directory has been removed or is otherwise unusable."
     }
+
+    bool --no-fetch
+    {
+      "Do not re-fetch the repository information before attempting to replace
+       packages being deinitialized with versions from repositories."
+    }
   };
 
   "
diff --git a/bdep/deinit.cxx b/bdep/deinit.cxx
index 45fab57..30152a4 100644
--- a/bdep/deinit.cxx
+++ b/bdep/deinit.cxx
@@ -8,7 +8,8 @@
 #include <bdep/database.hxx>
 #include <bdep/diagnostics.hxx>
 
-#include <bdep/sync.hxx> // configuration_projects(), hook_file
+#include <bdep/sync.hxx>
+#include <bdep/fetch.hxx>
 
 using namespace std;
 
@@ -77,19 +78,44 @@ namespace bdep
       }
     }
 
-    // Note that --keep-dependent is important: if we drop dependent packages
-    // that are managed by bdep, then its view of what has been initialized
-    // in the configuration will become invalid.
+    // Try to drop the packages watching out for their potential dependents.
+    // If there are any dependents, then sync in the deinit mode. This way we
+    // handle the plausible scenario when a dependency is pushed/submitted to
+    // a git/pkg repository after being developed and now we switch (back) to
+    // tracking it from that repository.
     //
     if (!force)
-      run_bpkg (2,
-                o,
-                "drop",
-                "-d", cfg,
-                "--keep-dependent",
-                "--plan", "synchronizing:",
-                "--yes",
-                pkgs);
+    {
+      process pr (start_bpkg (2,
+                              o,
+                              1 /* stdout */,
+                              2 /* stderr */,
+                              "drop",
+                              "-d", cfg,
+                              "--dependent-exit", 125,
+                              "--plan", "synchronizing:",
+                              "--yes",
+                              pkgs));
+
+      if (pr.wait ())
+        return;
+
+      const process_exit& e (*pr.exit);
+      if (e.normal ())
+      {
+        if (e.code () != 125)
+          throw failed (); // Assume the child issued diagnostics.
+
+        // Fall through.
+      }
+      else
+        fail << "process " << name_bpkg (o) << " " << e;
+
+      if (!o.no_fetch ())
+        cmd_fetch (o, prj, c, true /* fetch_full */);
+
+      cmd_sync_deinit (o, prj, c, pkgs);
+    }
   }
 
   int
diff --git a/bdep/sync.cxx b/bdep/sync.cxx
index 0cda5af..602c9aa 100644
--- a/bdep/sync.cxx
+++ b/bdep/sync.cxx
@@ -820,6 +820,8 @@ namespace bdep
   // If upgrade is not nullopt, then: If there are dep_pkgs, then we are
   // upgrading specific dependency packages. Otherwise -- project packages.
   //
+  // If deinit_pkgs is not empty, then sync in the deinit mode.
+  //
   // Note that if origin_prj is not empty, then origin_cfgs are specified as
   // configurations (as opposed to paths). Also upgrade can only be specified
   // with origin_prj.
@@ -847,6 +849,7 @@ namespace bdep
             bool disfigure,
             const package_locations& prj_pkgs,
             const strings&           dep_pkgs,
+            const strings&           deinit_pkgs,
             bool create_host_config,
             bool create_build2_config,
             transaction* origin_tr = nullptr,
@@ -858,7 +861,11 @@ namespace bdep
     //
     bool origin (!origin_prj.empty ());
 
-    assert (prj_pkgs.empty () || dep_pkgs.empty ()); // Can't have both.
+    // Cannot have more than one package list specified.
+    //
+    assert ((prj_pkgs.empty ()    ? 0 : 1) +
+            (dep_pkgs.empty ()    ? 0 : 1) +
+            (deinit_pkgs.empty () ? 0 : 1) <= 1);
 
     // If a transaction is specified, then it must be started on the origin
     // project's database (which therefore must be specified) and it must be
@@ -1076,6 +1083,14 @@ namespace bdep
 
         for (const package_state& pkg: cfg->packages)
         {
+          // In the deinit mode skip the being deinitialized packages to add
+          // them to the command line differently (see below).
+          //
+          if (find (deinit_pkgs.begin (), deinit_pkgs.end (), pkg.name) !=
+              deinit_pkgs.end () &&
+              prj.path == origin_prj)
+            continue;
+
           // Return true if this package is part of the list specified
           // explicitly or, if none are specified, init'ed in the origin
           // project.
@@ -1525,6 +1540,42 @@ namespace bdep
       args.push_back (move (a));
     }
 
+    // Add the being deinitialized packages.
+    //
+    if (!deinit_pkgs.empty ())
+    {
+      assert (!origin_prj.empty ()); // Project the packages belong to.
+
+      // Must contain the configuration where the packages are being
+      // deinitialized.
+      //
+      assert (origin_cfgs.size () == 1);
+
+      string config_uuid (
+        linked_cfgs.find (origin_cfgs.front ().path ())->uuid.string ());
+
+      args.push_back ("--mask-repository-uuid");
+      args.push_back (config_uuid + '=' + repository_name (origin_prj));
+
+      args.push_back ("{");
+
+      if (multi_cfg)
+        args.push_back ("--config-uuid=" + config_uuid);
+
+      args.push_back ("--deorphan");
+      args.push_back ("--dependency");
+      args.push_back ("}+");
+
+      if (deinit_pkgs.size () > 1)
+        args.push_back ("{");
+
+      for (const string& p: deinit_pkgs)
+        args.push_back (p);
+
+      if (deinit_pkgs.size () > 1)
+        args.push_back ("}");
+    }
+
     // We do a separate fetch instead of letting pkg-build do it. This way we
     // get better control of the diagnostics (no "fetching ..." for the
     // project itself). We also make sure that if the user specifies a
@@ -2105,11 +2156,12 @@ namespace bdep
               fetch,
               yes,
               name_cfg,
-              nullopt              /* upgrade   */,
-              nullopt              /* recursive */,
-              false                /* disfigure */,
-              package_locations () /* prj_pkgs  */,
-              strings ()           /* dep_pkgs  */,
+              nullopt              /* upgrade     */,
+              nullopt              /* recursive   */,
+              false                /* disfigure   */,
+              package_locations () /* prj_pkgs    */,
+              strings ()           /* dep_pkgs    */,
+              strings ()           /* deinit_pkgs */,
               create_host_config,
               create_build2_config,
               t,
@@ -2180,11 +2232,12 @@ namespace bdep
                 fetch,
                 yes,
                 name_cfg,
-                nullopt              /* upgrade   */,
-                nullopt              /* recursive */,
-                false                /* disfigure */,
-                package_locations () /* prj_pkgs  */,
-                strings ()           /* dep_pkgs  */,
+                nullopt              /* upgrade     */,
+                nullopt              /* recursive   */,
+                false                /* disfigure   */,
+                package_locations () /* prj_pkgs    */,
+                strings ()           /* dep_pkgs    */,
+                strings ()           /* deinit_pkgs */,
                 create_host_config,
                 create_build2_config,
                 nullptr,
@@ -2220,16 +2273,17 @@ namespace bdep
               dir_path () /* prj */,
               {sync_config (cfg)},
               move (lcfgs),
-              strings ()            /* pkg_args */,
-              true                  /* implicit */,
+              strings ()            /* pkg_args   */,
+              true                  /* implicit   */,
               fetch,
               yes,
               name_cfg,
-              nullopt               /* upgrade   */,
-              nullopt               /* recursive */,
-              false                 /* disfigure */,
-              package_locations ()  /* prj_pkgs  */,
-              strings ()            /* dep_pkgs  */,
+              nullopt               /* upgrade     */,
+              nullopt               /* recursive   */,
+              false                 /* disfigure   */,
+              package_locations ()  /* prj_pkgs    */,
+              strings ()            /* dep_pkgs    */,
+              strings ()            /* deinit_pkgs */,
               create_host_config,
               create_build2_config);
 
@@ -2286,21 +2340,50 @@ namespace bdep
                 dir_path () /* prj */,
                 move (ocfgs),
                 move (lcfgs),
-                strings ()            /* pkg_args */,
-                true                  /* implicit */,
+                strings ()            /* pkg_args   */,
+                true                  /* implicit   */,
                 fetch,
                 yes,
                 name_cfg,
-                nullopt               /* upgrade   */,
-                nullopt               /* recursive */,
-                false                 /* disfigure */,
-                package_locations ()  /* prj_pkgs  */,
-                strings ()            /* dep_pkgs  */,
+                nullopt               /* upgrade     */,
+                nullopt               /* recursive   */,
+                false                 /* disfigure   */,
+                package_locations ()  /* prj_pkgs    */,
+                strings ()            /* dep_pkgs    */,
+                strings ()            /* deinit_pkgs */,
                 create_host_config,
                 create_build2_config);
     }
   }
 
+  void
+  cmd_sync_deinit (const common_options& co,
+                   const dir_path& prj,
+                   const shared_ptr<configuration>& cfg,
+                   const strings& pkgs)
+  {
+    sync_configs ocfgs {cfg};
+    linked_configs lcfgs (find_config_cluster (co, cfg->path));
+
+    cmd_sync (co,
+              prj,
+              move (ocfgs),
+              move (lcfgs),
+              strings ()            /* pkg_args             */,
+              true                  /* implicit             */,
+              true                  /* fetch                */,
+              true                  /* yes                  */,
+              false                 /* name_cfg             */,
+              nullopt               /* upgrade              */,
+              nullopt               /* recursive            */,
+              false                 /* disfigure            */,
+              package_locations ()  /* prj_pkgs             */,
+              strings ()            /* dep_pkgs             */,
+              pkgs,
+              false                 /* create_host_config   */,
+              false                 /* create_build2_config */);
+  }
+
   int
   cmd_sync (cmd_sync_options&& o, cli::group_scanner& args)
   {
@@ -2643,6 +2726,7 @@ namespace bdep
                   o.disfigure (),
                   package_locations ()     /* prj_pkgs  */,
                   dep_pkgs,
+                  strings ()               /* deinit_pkgs */,
                   o.create_host_config (),
                   o.create_build2_config ());
       }
@@ -2665,6 +2749,7 @@ namespace bdep
                   o.disfigure (),
                   prj_pkgs,
                   strings ()               /* dep_pkgs  */,
+                  strings ()               /* deinit_pkgs */,
                   o.create_host_config (),
                   o.create_build2_config ());
       }
@@ -2682,13 +2767,14 @@ namespace bdep
                   pkg_args,
                   o.implicit (),
                   !fetch,
-                  true                     /* yes       */,
-                  o.implicit ()            /* name_cfg  */,
-                  nullopt                  /* upgrade   */,
-                  nullopt                  /* recursive */,
+                  true                     /* yes         */,
+                  o.implicit ()            /* name_cfg    */,
+                  nullopt                  /* upgrade     */,
+                  nullopt                  /* recursive   */,
                   o.disfigure (),
-                  package_locations ()     /* prj_pkgs  */,
-                  strings ()               /* dep_pkgs  */,
+                  package_locations ()     /* prj_pkgs    */,
+                  strings ()               /* dep_pkgs    */,
+                  strings ()               /* deinit_pkgs */,
                   o.create_host_config (),
                   o.create_build2_config ());
       }
diff --git a/bdep/sync.hxx b/bdep/sync.hxx
index b025e40..fdd6d0b 100644
--- a/bdep/sync.hxx
+++ b/bdep/sync.hxx
@@ -112,6 +112,19 @@ namespace bdep
                      bool create_host_config = false,
                      bool create_build2_config = false);
 
+  // As above but deinitialize the specified project packages in the specified
+  // configuration instead of upgrading them.
+  //
+  // Specifically, in bpkg terms, unhold and deorphan these packages in the
+  // specified configuration with their project directory repository being
+  // masked.
+  //
+  void
+  cmd_sync_deinit (const common_options&,
+                   const dir_path& prj,
+                   const shared_ptr<configuration>&,
+                   const strings& pkgs);
+
   int
   cmd_sync (cmd_sync_options&&, cli::group_scanner& args);
 
diff --git a/tests/init.testscript b/tests/init.testscript
index 6d7c5c7..2a4f686 100644
--- a/tests/init.testscript
+++ b/tests/init.testscript
@@ -9,13 +9,14 @@
 config_cxx = [cmdline] cc config.cxx=$quote($recall($cxx.path) $cxx.config.mode, true)
 
 status += -d prj
-deinit += -d prj
 
 : create-cfg
 :
 {
   +$clone_prj
 
+  deinit += -d prj
+
   : with-dependency
   :
   {
@@ -152,6 +153,8 @@ deinit += -d prj
 {
   $clone_prj;
 
+  deinit += -d prj;
+
   cat <<EOI >+prj/manifest;
     depends: libprj
     EOI
@@ -215,6 +218,8 @@ deinit += -d prj
 : project.
 :
 {
+  deinit += -d prj;
+
   # Create (and build) the executable single-package project.
   #
   cp --no-cleanup -pr ../prj ./ &prj/***;
@@ -309,3 +314,227 @@ deinit += -d prj
       drop libprj
     EOE
 }
+
+: deinit-deorphan
+:
+{
+  +$clone_prj
+
+  +cat <<EOI >+prj/manifest
+    depends: libprj
+    EOI
+
+  +$new -t lib -o libprj.git libprj &libprj.git/*** 2>!
+
+  g = [cmdline] git -C libprj.git 2>! >&2
+
+  +$g config user.name 'Test Script'
+  +$g config user.email 'testscript@example.com'
+  +$g add '*'
+  +$g commit -m 'Create' --no-verify
+
+  : dependency-only
+  :
+  {
+    $clone_prj;
+    cp -rp ../libprj.git ./;
+
+    cat <<EOI >+prj/repositories.manifest;
+      :
+      role: prerequisite
+      location: ../libprj.git#master
+      EOI
+
+    $* $config_cxx -C @cfg &prj-cfg/*** 2>!;
+
+    git clone libprj.git &libprj/*** 2>! >&2;
+
+    $init -d libprj -A prj-cfg 2>>/~"%EOE%";
+      initializing in project $~/libprj/
+      added configuration $~/prj-cfg/ 1 target default,forwarded,auto-synchronized
+      synchronizing:
+      %  upgrade libprj.0.1.0-a.0.+#1%
+      %  reconfigure prj.0.1.0-a.0.19700101000000%
+      EOE
+
+    $build prj/ 2>>/~%EOE%;
+      %(mkdir|c\+\+|ld|ln) .+%{4}
+      EOE
+
+    $deinit -d libprj 2>>/~"%EOE%";
+      deinitializing in project $~/libprj/
+      %fetching dir:.+/libprj%
+      %fetching dir:.+/prj%
+      %fetching git:.+/libprj#.+ \\\(prerequisite of dir:.+/prj\\\)%
+      querying $~/libprj.git
+      synchronizing:
+      %  deorphan.downgrade.unhold libprj.0.1.0-a.0.+%
+      %  reconfigure prj.0.1.0-a.0.+%
+      verifying symlinks...
+      %fixing up symlinks...%?
+      %distributing libprj.0.1.0-a.0.+%
+      EOE
+
+    $build prj/ 2>>/~%EOE%;
+      %ln prj-cfg/prj/prj/exe\{prj\} -> prj/prj/%?
+      info: prj-cfg/dir{prj/} is up to date
+      EOE
+
+    $build 'clean:' prj/ 2>>~%EOE%;
+      %(rm|rmdir) .+%{3}
+      EOE
+
+    $deinit -d prj 2>>/"EOE"
+      deinitializing in project $~/prj/
+      synchronizing:
+        drop prj
+        drop libprj
+      EOE
+  }
+
+  : not-only-dependency
+  :
+  : Similar to the above but with an additional libprj/libprj-extras library
+  : which has no initialized dependents. Here we test that it is dropped while
+  : being deorphaned.
+  :
+  {
+    $clone_prj;
+
+    $new -t empty -o libprj.git libprj &libprj.git/*** 2>!;
+    $new --package -t lib libprj -d libprj.git 2>!;
+    $new --package -t lib libprj-extras -d libprj.git 2>!;
+
+    $g config user.name 'Test Script';
+    $g config user.email 'testscript@example.com';
+    $g add '*';
+    $g commit -m 'Create' --no-verify;
+
+    cat <<EOI >+prj/repositories.manifest;
+      :
+      role: prerequisite
+      location: ../libprj.git#master
+      EOI
+
+    $* $config_cxx -C @cfg &prj-cfg/*** 2>!;
+
+    git clone libprj.git &libprj/*** 2>! >&2;
+
+    $init -d libprj -A prj-cfg 2>>/~"%EOE%";
+      initializing in project $~/libprj/
+      added configuration $~/prj-cfg/ 1 target default,forwarded,auto-synchronized
+      initializing package libprj
+      initializing package libprj-extras
+      synchronizing:
+      %  upgrade libprj.0.1.0-a.0.+#1%
+      %  new libprj-extras.0.1.0-a.0.+%
+      %  reconfigure prj.0.1.0-a.0.19700101000000%
+      EOE
+
+    $build prj/ 2>>/~%EOE%;
+      %(mkdir|c\+\+|ld|ln) .+%{4}
+      EOE
+
+    $deinit -d libprj 2>>/~"%EOE%";
+      deinitializing in project $~/libprj/
+      deinitializing package libprj
+      deinitializing package libprj-extras
+      %fetching dir:.+/libprj%
+      %fetching dir:.+/prj%
+      %fetching git:.+/libprj#.+ \\\(prerequisite of dir:.+/prj\\\)%
+      querying $~/libprj.git
+      synchronizing:
+      %  drop libprj-extras.0.1.0-a.0.+ \\\(unused\\\)%
+      %  deorphan.downgrade.unhold libprj.0.1.0-a.0.+%
+      %  reconfigure prj.0.1.0-a.0.+%
+      verifying symlinks...
+      %fixing up symlinks...%?
+      %distributing libprj.0.1.0-a.0.+%
+      EOE
+
+    $build prj/ 2>>/~%EOE%;
+      %ln prj-cfg/prj/prj/exe\{prj\} -> prj/prj/%?
+      info: prj-cfg/dir{prj/} is up to date
+      EOE
+
+    $build 'clean:' prj/ 2>>~%EOE%;
+      %(rm|rmdir) .+%{3}
+      EOE
+
+    $deinit -d prj 2>>/"EOE"
+      deinitializing in project $~/prj/
+      synchronizing:
+        drop prj
+        drop libprj
+      EOE
+  }
+
+  : not-found
+  :
+  {
+    $clone_prj;
+    cp -rp ../libprj.git ./;
+
+    git clone libprj.git &libprj/*** 2>! >&2;
+
+    $init $config_cxx -d libprj -C @cfg &libprj-cfg/*** 2>!;
+
+    $* -A libprj-cfg 2>>/~"%EOE%";
+      initializing in project $~/prj/
+      added configuration $~/libprj-cfg/ 1 target default,forwarded,auto-synchronized
+      synchronizing:
+      %  new prj.0.1.0-a.0.19700101000000%
+      EOE
+
+    $build prj/ 2>>/~%EOE%;
+      %(mkdir|c\+\+|ld|ln) .+%{4}
+      EOE
+
+    $deinit -d libprj 2>>/~"%EOE%" != 0;
+      deinitializing in project $~/libprj/
+      %fetching dir:.+/libprj%
+      %fetching dir:.+/prj%
+      error: unknown package libprj
+      EOE
+
+    # While at it, test that adding the repository to the manifest fixes the
+    # issue.
+    #
+    cat <<EOI >+prj/repositories.manifest;
+      :
+      role: prerequisite
+      location: ../libprj.git#master
+      EOI
+
+    $deinit -d libprj 2>>/~"%EOE%";
+      deinitializing in project $~/libprj/
+      %fetching dir:.+/libprj%
+      %fetching dir:.+/prj%
+      %fetching git:.+/libprj#.+ \\\(prerequisite of dir:.+/prj\\\)%
+      querying $~/libprj.git
+      fetching from $~/libprj.git
+      synchronizing:
+      %  deorphan.unhold libprj.0.1.0-a.0.+%
+      %  reconfigure prj.0.1.0-a.0.+%
+      verifying symlinks...
+      %fixing up symlinks...%?
+      %distributing libprj.0.1.0-a.0.+%
+      EOE
+
+    $build prj/ 2>>/~%EOE%;
+      %ln libprj-cfg/prj/prj/exe\{prj\} -> prj/prj/%?
+      info: libprj-cfg/dir{prj/} is up to date
+      EOE
+
+    $build 'clean:' prj/ 2>>~%EOE%;
+      %(rm|rmdir) .+%{3}
+      EOE
+
+    $deinit -d prj 2>>/"EOE"
+      deinitializing in project $~/prj/
+      synchronizing:
+        drop prj
+        drop libprj
+      EOE
+  }
+}