Next chunk of work on system package manager (still multi-version)

6 files changed, 363 insertions, 64 deletions

diff --git a/bpkg/diagnostics.hxx b/bpkg/diagnostics.hxx
index e8d9f0a..9dcd9cb 100644
--- a/bpkg/diagnostics.hxx
+++ b/bpkg/diagnostics.hxx
@@ -118,9 +118,34 @@ namespace bpkg
   //
   using butl::diag_stream;
   using butl::diag_epilogue;
+  using butl::diag_frame;
 
   // Diagnostic facility, project specifics.
   //
+  template <typename F>
+  struct diag_frame_impl: diag_frame
+  {
+    explicit
+    diag_frame_impl (F f): diag_frame (&thunk), func_ (move (f)) {}
+
+  private:
+    static void
+    thunk (const diag_frame& f, const butl::diag_record& r)
+    {
+      static_cast<const diag_frame_impl&> (f).func_ (
+        static_cast<const diag_record&> (r));
+    }
+
+    const F func_;
+  };
+
+  template <typename F>
+  inline diag_frame_impl<F>
+  make_diag_frame (F f)
+  {
+    return diag_frame_impl<F> (move (f));
+  }
+
   struct simple_prologue_base
   {
     explicit
@@ -184,7 +209,7 @@ namespace bpkg
     basic_mark_base (const char* type,
                      const char* name = nullptr,
                      const void* data = nullptr,
-                     diag_epilogue* epilogue = nullptr)
+                     diag_epilogue* epilogue = &diag_frame::apply)
         : type_ (type), name_ (name), data_ (data), epilogue_ (epilogue) {}
 
     simple_prologue
@@ -288,6 +313,7 @@ namespace bpkg
                            data,
                            [](const diag_record& r, butl::diag_writer* w)
                            {
+                             diag_frame::apply (r);
                              r.flush (w);
                              throw failed ();
                            }) {}
diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx
index 21b7e04..f30cfd6 100644
--- a/bpkg/system-package-manager-debian.cxx
+++ b/bpkg/system-package-manager-debian.cxx
@@ -26,11 +26,204 @@ namespace bpkg
   //    individual system packages. What if we "installed any version"
   //    first and then need to install specific?
 
-  auto system_package_manager_debian::
+
+  // For background, a library in Debian is normally split up into several
+  // packages: the shared library package (e.g., libfoo1 where 1 is the ABI
+  // version), the development files package (e.g., libfoo-dev), the
+  // documentation files package (e.g., libfoo-doc), the debug symbols
+  // package (e.g., libfoo1-dbg), and the architecture-independent files
+  // (e.g., libfoo1-common). All the packages except -dev are optional
+  // and there is quite a bit of variability here. Here are a few examples:
+  //
+  // libz3-4 libz3-dev
+  //
+  // libssl1.1 libssl-dev libssl-doc
+  // libssl3 libssl-dev libssl-doc
+  //
+  // libcurl4 libcurl4-doc libcurl4-openssl-dev
+  // libcurl3-gnutls libcurl4-gnutls-dev
+  //
+  // Based on that, it seems our best bet when trying to automatically map our
+  // library package name to Debian package names is to go for the -dev
+  // package first and figure out the shared library package from that based
+  // on the fact that the -dev package should have the == dependency on the
+  // shared library package with the same version and its name should normally
+  // start with the -dev package's stem.
+  //
+  // For a manual mapping we will require the user to always specify the
+  // shared library package and the -dev package names explicitly.
+  //
+  // For executable packages there is normally no -dev packages but -dbg,
+  // -doc, and -common are plausible.
+  //
+  class system_package_status_debian: public system_package_status
+  {
+  public:
+    string main;
+    string dev;
+    string doc;
+    string dbg;
+    string common;
+    strings extras;
+
+    explicit
+    system_package_status_debian (string m, string d = {})
+        : main (move (m)), dev (move (d))
+    {
+      assert (!main.empty () || !dev.empty ());
+    }
+  };
+
+  using package_status = system_package_status;
+  using package_status_debian = system_package_status_debian;
+
+  const package_status_debian&
+  as_debian (const unique_ptr<package_status>& s)
+  {
+    return static_cast<const package_status_debian&> (*s);
+  }
+
+  package_status_debian&
+  as_debian (unique_ptr<package_status>& s)
+  {
+    return static_cast<package_status_debian&> (*s);
+  }
+
+  // Parse the debian-name (or alike) value.
+  //
+  // The format of this value is a comma-separated list of one or more package
+  // groups:
+  //
+  // <package-group> [, <package-group>...]
+  //
+  // Where each <package-group> is the space-separate list of one or more
+  // package names:
+  //
+  // <package-name> [  <package-name>...]
+  //
+  // All the packages in the group should be "package components" (for the
+  // lack of a better term) of the same "logical package", such as -dev, -doc,
+  // -common packages. They usually have the same version.
+  //
+  // The first group is called the main group and the first package in the
+  // group is called the main package.
+  //
+  // We allow/recommend specifying the -dev package as the main package for
+  // libraries (the name starts with lib), seeing that we will be capable of
+  // detecting the main package automatically. If the library name happens to
+  // end with -dev (which poses an ambiguity), then the -dev package should be
+  // specified explicitly as the second package to disambiguate this situation
+  // (if a non-library name happened to start with lib and end with -dev,
+  // well, you are out of luck, I guess).
+  //
+  // Note also that for now we treat all the packages from the non-main groups
+  // as extras. But in the future we may decide to sort them out like the main
+  // group.
+  //
+  static unique_ptr<package_status_debian>
+  parse_debian_name (const string& nv)
+  {
+    auto split = [] (const string& s, char d) -> strings
+    {
+      strings r;
+      for (size_t b (0), e (0); next_word (s, b, e, d); )
+        r.push_back (string (s, b, e - b));
+      return r;
+    };
+
+    auto suffix = [] (const string& n, const string& s) -> bool
+    {
+      size_t nn (n.size ());
+      size_t sn (s.size ());
+      return nn > sn && n.compare (nn - sn, sn, s) == 0;
+    };
+
+    auto parse_group = [&split, &suffix] (const string& g)
+    {
+      strings ns (split (g, ' '));
+
+      if (ns.empty ())
+        fail << "empty package group";
+
+      unique_ptr<package_status_debian> r;
+
+      // Handle the dev instead of main special case for libraries.
+      //
+      // Check that the following name does not end with -dev. This will be
+      // the only way to disambiguate the case where the library name happens
+      // to end with -dev (e.g., libops-dev libops-dev-dev).
+      //
+      {
+        string& m (ns[0]);
+
+        if (m.compare (0, 3, "lib") == 0 &&
+            suffix (m, "-dev")           &&
+            !(ns.size () > 1 && suffix (ns[1], "-dev")))
+        {
+          r.reset (new package_status_debian ("", move (m)));
+        }
+        else
+          r.reset (new package_status_debian (move (m)));
+      }
+
+      // Handle the rest.
+      //
+      for (size_t i (1); i != ns.size (); ++i)
+      {
+        string& n (ns[i]);
+
+        const char* w;
+        if (string* v = (suffix (n, (w = "-dev"))    ? &r->dev :
+                         suffix (n, (w = "-doc"))    ? &r->doc :
+                         suffix (n, (w = "-dbg"))    ? &r->dbg :
+                         suffix (n, (w = "-common")) ? &r->common : nullptr))
+        {
+          if (!v->empty ())
+            fail << "multiple " << w << " package names in '" << g << "'" <<
+              info << "did you forget to separate package groups with comma?";
+
+          *v = move (n);
+        }
+        else
+          r->extras.push_back (move (n));
+      }
+
+      return r;
+    };
+
+    strings gs (split (nv, ','));
+    assert (!gs.empty ()); // *-name value cannot be empty.
+
+    unique_ptr<package_status_debian> r;
+    for (size_t i (0); i != gs.size (); ++i)
+    {
+      if (i == 0) // Main group.
+        r = parse_group (gs[i]);
+      else
+      {
+        unique_ptr<package_status_debian> g (parse_group (gs[i]));
+
+        if (!g->main.empty ())   r->extras.push_back (move (g->main));
+        if (!g->dev.empty ())    r->extras.push_back (move (g->dev));
+        if (!g->doc.empty ())    r->extras.push_back (move (g->doc));
+        if (!g->dbg.empty ())    r->extras.push_back (move (g->dbg));
+        if (!g->common.empty ()) r->extras.push_back (move (g->common));
+        if (!g->extras.empty ()) r->extras.insert (
+          r->extras.end (),
+          make_move_iterator (g->extras.begin ()),
+          make_move_iterator (g->extras.end ()));
+      }
+    }
+
+    return r;
+  }
+
+  const vector<unique_ptr<package_status>>*
+  system_package_manager_debian::
   pkg_status (const package_name& pn,
               const available_packages* aps,
               bool install,
-              bool fetch) -> const vector<package_status>*
+              bool fetch)
   {
     // First check the cache.
     //
@@ -44,17 +237,75 @@ namespace bpkg
         return nullptr;
     }
 
-    // Translate our package name to the system package names.
+    vector<unique_ptr<package_status>> r;
+
+    // Translate our package name to the Debian package names.
     //
-    strings spns (system_package_names (*aps,
+    {
+      auto df = make_diag_frame (
+        [&pn] (const diag_record& dr)
+        {
+          dr << info << "while mapping " << pn << " to Debian package name";
+        });
+
+      strings ns (system_package_names (*aps,
                                         os_release_.name_id,
                                         os_release_.version_id,
                                         os_release_.like_ids));
+      if (ns.empty ())
+      {
+        // Attempt to automatically translate our package name (see above for
+        // details).
+        //
+        const string& n (pn.string ());
+
+        // The best we can do in trying to detect whether this is a library is
+        // to check for the lib prefix. Libraries without the lib prefix and
+        // non-libraries with the lib prefix (both of which we do not
+        // recomment) will have to provide a manual mapping.
+        //
+        unique_ptr<package_status_debian> s;
+
+        if (n.compare (0, 3, "lib") == 0)
+        {
+          // Keep the main package name empty as an indication that it is to
+          // be discovered.
+          //
+          s.reset (new package_status_debian ("", n + "-dev"));
+        }
+        else
+          s.reset (new package_status_debian (n));
 
-    // @@ TODO: fallback to our package name if empty (plus -dev if lib).
-    // @@ TODO: split into packages/components
+        r.push_back (move (s));
+      }
+      else
+      {
+        // Parse each manual mapping.
+        //
+        for (const string& n: ns)
+        {
+          unique_ptr<package_status_debian> s (parse_debian_name (n));
 
-    vector<package_status> r;
+          // Suppress duplicates for good measure based on the main package
+          // name (and falling back to -dev if empty).
+          //
+          auto i (find_if (r.begin (), r.end (),
+                           [&s] (const unique_ptr<package_status>& x)
+                           {
+                             const package_status_debian& d (as_debian (x));
+                             return s->main.empty ()
+                               ? s->dev == d.dev
+                               : s->main == d.main;
+                           }));
+          if (i == r.end ())
+            r.push_back (move (s));
+          else
+          {
+            // @@ Should we verify the rest matches for good measure?
+          }
+        }
+      }
+    }
 
     // First look for an already installed package.
     //
diff --git a/bpkg/system-package-manager-debian.hxx b/bpkg/system-package-manager-debian.hxx
index ed74db0..a2f56a2 100644
--- a/bpkg/system-package-manager-debian.hxx
+++ b/bpkg/system-package-manager-debian.hxx
@@ -17,7 +17,7 @@ namespace bpkg
   class system_package_manager_debian: public system_package_manager
   {
   public:
-    virtual const vector<package_status>*
+    virtual const vector<unique_ptr<system_package_status>>*
     pkg_status (const package_name&,
                 const available_packages*,
                 bool install,
diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx
index d535140..78c5ce3 100644
--- a/bpkg/system-package-manager.cxx
+++ b/bpkg/system-package-manager.cxx
@@ -17,6 +17,12 @@ using namespace butl;
 
 namespace bpkg
 {
+  system_package_status::
+  ~system_package_status ()
+  {
+    // vtable
+  }
+
   system_package_manager::
   ~system_package_manager ()
   {
diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx
index 78b157a..7d960f4 100644
--- a/bpkg/system-package-manager.hxx
+++ b/bpkg/system-package-manager.hxx
@@ -20,60 +20,44 @@ namespace bpkg
   // The system package manager interface. Used by both pkg-build (to query
   // and install system packages) and by pkg-bindist (to build them).
   //
-  class system_package_manager
+  class system_package_status
   {
   public:
-    struct package_status
-    {
-      // Downstream (as in, bpkg package) version.
-      //
-      bpkg::version version;
-
-      // The system package can be either "available already installed",
-      // "available partially installed" (for example, libfoo but not
-      // libfoo-dev is installed) or "available not yet installed".
-      //
-      // Whether not_installed versions can be returned along with installed
-      // or partially_installed depends on whether the packager manager can
-      // install multiple versions side-by-side.
-      //
-      enum {installed, partially_installed, not_installed} status;
-
-      // System (as in, distribution package) name and version.
-      //
-      // @@ But these could be multiple. Do we really need this?
-      /*
-      string system_name;
-      string system_version;
-      */
-
-      // Package manager implementation-specific data.
-      //
-    public:
-      using data_ptr = unique_ptr<void, void (*) (void*)>;
-
-      template <typename T>
-      T&
-      data () { return *static_cast<T*> (data_.get ()); }
-
-      template <typename T>
-      const T&
-      data () const { return *static_cast<const T*> (data_.get ()); }
-
-      template <typename T>
-      T&
-      data (T* d)
-      {
-        data_ = data_ptr (d, [] (void* p) { delete static_cast<T*> (p); });
-        return *d;
-      }
-
-      static void
-      null_data_deleter (void* p) { assert (p == nullptr); }
-
-      data_ptr data_ = {nullptr, null_data_deleter};
-    };
+    // Downstream (as in, bpkg package) version.
+    //
+    bpkg::version version;
+
+    // The system package can be either "available already installed",
+    // "available partially installed" (for example, libfoo but not
+    // libfoo-dev is installed) or "available not yet installed".
+    //
+    // Whether not_installed versions can be returned along with installed
+    // or partially_installed depends on whether the packager manager can
+    // install multiple versions side-by-side.
+    //
+    enum status_type {installed, partially_installed, not_installed};
+
+    status_type status = not_installed;
+
+    // System (as in, distribution package) name and version.
+    //
+    // @@ But these could be multiple. Do we really need this?
+    // @@ Can now probably provide as virtual functions.
+    /*
+    string system_name;
+    string system_version;
+    */
+
+  public:
+    virtual
+    ~system_package_status ();
+
+    system_package_status () = default;
+  };
 
+  class system_package_manager
+  {
+  public:
     // Query the system package status.
     //
     // This function has two modes: cache-only (available_packages is NULL)
@@ -83,7 +67,13 @@ namespace bpkg
     // the available packages (for the name/version mapping information) if
     // really necessary.
     //
-    // The returned value can be empty, which indicates that no such package
+    // The returned list should be arranged in the preference order with the
+    // first entry having the highest preference. Normally this will be in the
+    // descending version order but can also be something more elaborate, such
+    // as the already installed or partially installed version coming first
+    // with the descending version order after that.
+    //
+    // The returned list can be empty, which indicates that no such package
     // is available from the system package manager. Note that empty is also
     // returned if no fully installed package is available from the system and
     // the install argument is false.
@@ -93,7 +83,7 @@ namespace bpkg
     // the available version of the not yet installed or partially installed
     // packages.
     //
-    virtual const vector<package_status>*
+    virtual const vector<unique_ptr<system_package_status>>*
     pkg_status (const package_name&,
                 const available_packages*,
                 bool install,
@@ -119,7 +109,8 @@ namespace bpkg
 
   protected:
     // Given the available packages (as returned by find_available_all())
-    // return the list of system package names.
+    // return the list of system package names as mapped by the
+    // <distribution>-name values.
     //
     // The name_id, version_id, and like_ids are the values from os_release
     // (refer there for background). If version_id is empty, then it's treated
@@ -140,15 +131,39 @@ namespace bpkg
     // something more elaborate, like translate version_id to the like_id's
     // version and try that).
     //
+    // @@ TODO: allow multiple -name values per same distribution and handle
+    //    here? E.g., libcurl4-openssl-dev libcurl4-gnutls-dev. But they will
+    //    have the same available version, how will we deal with that? How
+    //    will we pick one? Perhaps this should all be handled by the system
+    //    package manager (conceptually, this is configuration negotiation).
+    //
     static strings
     system_package_names (const available_packages&,
                           const string& name_id,
                           const string& version_id,
                           const vector<string>& like_ids);
 
+    // Given the system package version and available packages (as returned by
+    // find_available_all()) return the downstream package version as mapped
+    // by one of the <distribution>-to-downstream-version values.
+    //
+    // The rest of the arguments as well as the overalls semantics is the same
+    // as in system_package_names() above. That is, first consider
+    // <distribution>-to-downstream-version values corresponding to
+    // name_id. If none match, then repeat the above process for every
+    // like_ids entry with version_id equal 0. If still no match, then return
+    // nullopt.
+    //
+    static optional<version>
+    downstream_package_version (const string& system_version,
+                                const available_packages&,
+                                const string& name_id,
+                                const string& version_id,
+                                const vector<string>& like_ids);
   protected:
     os_release os_release_;
-    std::map<package_name, vector<package_status>> status_cache_;
+    std::map<package_name,
+             vector<unique_ptr<system_package_status>>> status_cache_;
   };
 
   // Create a package manager instance corresponding to the specified host
diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx
index 16ad094..896a67b 100644
--- a/bpkg/utility.hxx
+++ b/bpkg/utility.hxx
@@ -52,6 +52,7 @@ namespace bpkg
   using butl::trim;
   using butl::trim_left;
   using butl::trim_right;
+  using butl::next_word;
 
   using butl::make_guard;
   using butl::make_exception_guard;