WIP (use installation manifest)

4 files changed, 414 insertions, 77 deletions

diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx
index ba0db91..07548f8 100644
--- a/bpkg/system-package-manager-debian.cxx
+++ b/bpkg/system-package-manager-debian.cxx
@@ -1848,7 +1848,7 @@ namespace bpkg
   void system_package_manager_debian::
   generate (const packages& pkgs,
             const packages& deps,
-            const strings&,
+            const strings& vars,
             const package_manifest& pm,
             const string& pt,
             const small_vector<language, 1>& langs,
@@ -1956,6 +1956,83 @@ namespace bpkg
       }
     }
 
+    // We override every config.install.* variable in order not to pick
+    // anything configured. Note that we add some more in the rules file
+    // below.
+    //
+    // We make use of the <project> substitution since in the recursive mode
+    // we may be installing multiple projects. Note that the <private>
+    // directory component is automatically removed if this functionality is
+    // not enabled. One side-effect of using <project> is that we will be
+    // using the bpkg package name instead of the main Debian package name.
+    // But perhaps that's correct: on Debian it's usually the source package
+    // name, which is the same. To keep things consistent we use the bpkg
+    // package name for <private> as well.
+    //
+    // @@ Some libraries install what looks like architecture-specific
+    //    configuration files to /usr/include/$(DEB_HOST_MULTIARCH). Maybe we
+    //    should invent something like config.install.include_arch to support
+    //    this distinction?
+    //
+    // NOTE: make sure to update .install files below if changing anyting
+    //       here.
+    //
+    // Note: we need to quote values that contain `$` so that they don't get
+    // expanded as build2 variables in the installed_entries() call.
+    //
+    strings config {
+      "config.install.root=/usr/",
+      "config.install.data_root=root/",
+      "config.install.exec_root=root/",
+
+      "config.install.bin=exec_root/bin/",
+      "config.install.sbin=exec_root/sbin/",
+
+      // On Debian shared libraries should not be executable. Also,
+      // libexec/ is the same as lib/ (note that executables that get
+      // installed there will still have the executable bit set).
+      //
+      "config.install.lib='exec_root/lib/$(DEB_HOST_MULTIARCH)/<private>/'",
+      "config.install.lib.mode=644",
+      "config.install.libexec=lib/<project>/",
+      "config.install.pkgconfig=lib/pkgconfig/",
+
+      "config.install.etc=data_root/etc/",
+      "config.install.include=data_root/include/<private>/",
+      "config.install.share=data_root/share/",
+      "config.install.data=share/<private>/<project>/",
+
+      "config.install.doc=share/doc/<private>/<project>/",
+      "config.install.legal=doc/",
+      "config.install.man=share/man/",
+      "config.install.man1=man/man1/",
+      "config.install.man2=man/man2/",
+      "config.install.man3=man/man3/",
+      "config.install.man4=man/man4/",
+      "config.install.man5=man/man5/",
+      "config.install.man6=man/man6/",
+      "config.install.man7=man/man7/",
+      "config.install.man8=man/man8/"};
+
+    config.push_back ("config.install.private=" +
+                      (priv ? pn.string () : "[null]"));
+
+    // Add user-specified configuration variables last to allow them to
+    // override anything.
+    //
+    for (const string& v: vars)
+      config.push_back (v);
+
+    // Get the map of files that will end up in the binary packages.
+    //
+    // Note that we are passing quoted values with $(DEB_HOST_MULTIARCH) which
+    // will be treated literally.
+    //
+    installed_entry_map ies (installed_entries (*ops_, pkgs, config));
+
+    if (ies.empty ())
+      fail << "specified package(s) do not install any files";
+
     // Start assembling the package "source" directory.
     //
     // It's hard to predict all the files that will be generated (and
@@ -2504,7 +2581,12 @@ namespace bpkg
       //    (probably because stderr redirected to pipe). @@ No, there is
       //    progress. Maybe just keep, doesn't seem harmful. Or pass ours.
       //
-      os << "b := " << search_b (*ops_).effect_string ()            << '\n'
+      // Note: should be consistent with the invocation in installed_entries()
+      //       above.
+      //
+      os << "b := " << search_b (*ops_).effect_string ();
+      for (const string& o: ops_->build_option ()) os << ' ' << o;
+      os << '\n'
          << '\n'
          << "parallel := $(filter parallel=%,$(DEB_BUILD_OPTIONS))" << '\n'
          << "ifneq ($(parallel),)"                                  << '\n'
@@ -2517,70 +2599,47 @@ namespace bpkg
          << "endif"                                                 << '\n'
          << '\n';
 
-      // Note that we override every config.install.* variable in order not to
-      // pick anything configured.
-      //
-      // We make use of the <project> substitution since in the recursive mode
-      // we may be installing multiple projects. Note that the <private>
-      // directory component is automatically removed if this functionality is
-      // not enabled. One side-effect of using <project> is that we will be
-      // using the bpkg package name instead of the main Debian package name.
-      // But perhaps that's correct: on Debian it's usually the source package
-      // name, which is the same. To keep things consistent we use the bpkg
-      // package name for <private> as well.
-      //
-      // @@ Some libraries install what looks like architecture-specific
-      //    configuration files to /usr/include/$(DEB_HOST_MULTIARCH). Maybe
-      //    we should invent something like config.install.include_arch to
-      //    support this distinction?
-      //
-      // NOTE: make sure to update .install files below if changing anyting
-      //       here.
-      //
-      os << "config :=  config.install.chroot=$(DESTDIR)/"    << '\n'
-         << "config += 'config.install.sudo=[null]'"          << '\n'
-
-         << "config +=  config.install.root=/usr/"            << '\n'
-         << "config +=  config.install.data_root=root/"       << '\n'
-         << "config +=  config.install.exec_root=root/"       << '\n'
-
-         << "config +=  config.install.bin=exec_root/bin/"    << '\n'
-         << "config +=  config.install.sbin=exec_root/sbin/"  << '\n'
-
-         // On Debian shared libraries should not be executable. Also,
-         // libexec/ is the same as lib/ (note that executables that get
-         // installed there will still have the executable bit set).
-         //
-         << "config += 'config.install.lib=exec_root/lib/$(DEB_HOST_MULTIARCH)/<private>/'" << '\n'
-         << "config +=  config.install.lib.mode=644"               << '\n'
-         << "config += 'config.install.libexec=lib/<project>/'"    << '\n'
-         << "config +=  config.install.pkgconfig=lib/pkgconfig/"   << '\n'
-
-         << "config +=  config.install.etc=data_root/etc/"               << '\n'
-         << "config += 'config.install.include=data_root/include/<private>/'" << '\n'
-         << "config +=  config.install.share=data_root/share/"           << '\n'
-         << "config += 'config.install.data=share/<private>/<project>/'" << '\n'
-
-         << "config += 'config.install.doc=share/doc/<private>/<project>/'" << '\n'
-         << "config +=  config.install.legal=doc/"                  << '\n'
-         << "config +=  config.install.man=share/man/"              << '\n'
-         << "config +=  config.install.man1=man/man1/"              << '\n'
-         << "config +=  config.install.man2=man/man2/"              << '\n'
-         << "config +=  config.install.man3=man/man3/"              << '\n'
-         << "config +=  config.install.man4=man/man4/"              << '\n'
-         << "config +=  config.install.man5=man/man5/"              << '\n'
-         << "config +=  config.install.man6=man/man6/"              << '\n'
-         << "config +=  config.install.man7=man/man7/"              << '\n'
-         << "config +=  config.install.man8=man/man8/"              << '\n'
-
-         << "config += 'config.install.private="
-         <<            (priv ? pn.string () : "[null]")  << "'" << '\n';
+      // Configuration variables.
+      //
+      // Note: we need to quote values that contain `<>`, `[]`, since they
+      // will be passed through shell. For simplicity, let's just quote
+      // everything.
+      //
+      os << "config := config.install.chroot='$(DESTDIR)/'"   << '\n'
+         << "config += config.install.sudo='[null]'"          << '\n';
 
       // If this is a C-based language, add rpath for private installation.
       //
       if (priv && (lang ("c") || lang ("c++") || lang ("cc")))
-        os << "config += config.bin.rpath=/usr/lib/$(DEB_HOST_MULTIARCH)/"
-           <<            pn << "/" << '\n';
+        os << "config += config.bin.rpath='/usr/lib/$(DEB_HOST_MULTIARCH)/"
+           <<            pn << "/'" << '\n';
+
+      // Keep last to allow user-specified configuration variables to override
+      // anything.
+      //
+      for (const string& c: config)
+      {
+        // Quote the value unless already quoted (see above). Presense of
+        // potentially-quoted user variables complicates things a bit (can
+        // be partially quoted, double-quoted, etc).
+        //
+        size_t p (c.find_first_of ("=+ \t")); // End of name.
+        if (p != string::npos)
+        {
+          p = c.find_first_not_of ("=+ \t", p); // Beginning of value.
+          if (p != string::npos)
+          {
+            if (c.find_first_of ("'\"", p) == string::npos) // Not quoted.
+            {
+              os << "config += " << string (c, 0, p) << '\''
+                 << string (c, p) << "'\n";
+              continue;
+            }
+          }
+        }
+
+        os << "config += " << c << '\n';
+      }
 
       os << '\n';
 
@@ -2687,11 +2746,6 @@ namespace bpkg
     // variable, we can end up with multiple different directories (bundled
     // package).
     //
-    string privdir (priv ? pn.string () + '/' : "");
-    string libdir ("usr/lib/${DEB_HOST_MULTIARCH}/" + privdir);
-    string incdir ("usr/include/" + privdir);
-    string docdir ("usr/share/doc/" + privdir);
-
     path main_install (deb / (st.main + ".install"));
     try
     {
@@ -2700,17 +2754,16 @@ namespace bpkg
       // The main package contains everything that doesn't go to another
       // package.
       //
-      os//<< "usr/bin/*"   << '\n'
-        //<< "usr/sbin/*"  << '\n'
+      if (ies.contains ("/usr/bin/"))  os << "usr/bin/*"  << '\n';
+      if (ies.contains ("/usr/sbin/")) os << "usr/sbin/*" << '\n';
 
-         << libdir << '\n'
+      if (ies.contains ("/usr/lib/$(DEB_HOST_MULTIARCH)/"))
+        os << "usr/lib/${DEB_HOST_MULTIARCH}/*" << '\n';
 
-         << incdir << '\n'
+      if (ies.contains ("/usr/include/")) os << "usr/include/*" << '\n';
 
-         << docdir << "*" << '\n'
-        //<< "/usr/share/man/man1"
-
-        ;
+      if (ies.contains ("/usr/share/doc/")) os << "usr/share/doc/*" << '\n';
+      if (ies.contains ("/usr/share/man/")) os << "usr/share/man/*" << '\n';
 
       os.close ();
     }
@@ -2719,8 +2772,6 @@ namespace bpkg
       fail << "unable to write to " << main_install << ": " << e;
     }
 
-    return;
-
     // Run dpkg-buildpackage.
     //
     // Note that there doesn't seem to be any way to control its verbosity or
diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx
index a04c213..aba5be1 100644
--- a/bpkg/system-package-manager.cxx
+++ b/bpkg/system-package-manager.cxx
@@ -7,6 +7,7 @@
 
 #include <libbutl/regex.hxx>
 #include <libbutl/semantic-version.hxx>
+#include <libbutl/json/parser.hxx>
 
 #include <bpkg/package.hxx>
 #include <bpkg/package-odb.hxx>
@@ -643,4 +644,238 @@ namespace bpkg
 
     return r;
   }
+
+  auto system_package_manager::
+  installed_entries (const common_options& co,
+                     const packages& pkgs,
+                     const strings& vars) -> installed_entry_map
+  {
+    process_path pp (search_b (co));
+
+    // Note that we don't use start_b() here since we want to be consistent
+    // with how things will be run when building the package.
+    //
+    cstrings args {
+      pp.recall_string (),
+      "--quiet", // Note: implies --no-progress.
+      "--dry-run"};
+
+    // Pass our --jobs value, if any.
+    //
+    string jobs;
+    if (size_t n = co.jobs_specified () ? co.jobs () : 0)
+    {
+      jobs = to_string (n);
+      args.push_back ("--jobs");
+      args.push_back (jobs.c_str ());
+    }
+
+    // Pass any --build-option.
+    //
+    for (const string& o: co.build_option ()) args.push_back (o.c_str ());
+
+    // Configuration variables.
+    //
+    for (const string& v: vars) args.push_back (v.c_str ());
+    args.push_back ("!config.install.manifest=-");
+
+    // Package directories to install.
+    //
+    strings dirs;
+    for (const package& p: pkgs) dirs.push_back (p.out_root.representation ());
+    args.push_back ("install:");
+    for (const string& d: dirs) args.push_back (d.c_str ());
+
+    args.push_back (nullptr);
+
+    installed_entry_map r;
+    try
+    {
+      if (verb >= 2)
+        print_process (args);
+      else if (verb == 1)
+        text << "determining filesystem entries that would be installed...";
+
+      // Redirect stdout to a pipe.
+      //
+      process pr (pp,
+                  args,
+                  0  /* stdin */,
+                  -1 /* stdout */,
+                  2  /* stderr */);
+      try
+      {
+        ifdstream is (move (pr.in_ofd), fdstream_mode::skip);
+
+        json::parser p (is,
+                        args[0] /* input_name */,
+                        true    /* multi_value */,
+                        "\n"    /* value_separators */);
+
+        using event = json::event;
+
+        // Note: recursive lambda.
+        //
+        auto parse_entry = [&r, &p] (const auto& parse_entry) -> void
+        {
+          optional<event> e (p.next ());
+
+          // @@ This is really ugly, need to add next_expect() helpers to JSON
+          //    parser (similar to libstudxml).
+
+          if (*e != event::begin_object)
+            fail << "entry object expected";
+
+          // type
+          //
+          if (!(e = p.next ()) || *e != event::name || p.name () != "type")
+            fail << "type member expected";
+
+          if (!(e = p.next ()) || *e != event::string)
+            fail << "type member string value expected";
+
+          string t (p.value ()); // Note: value invalidated after p.next().
+
+          if (t == "target")
+          {
+            // name
+            //
+            if (!(e = p.next ()) || *e != event::name || p.name () != "name")
+              fail << "name member expected";
+
+            if (!(e = p.next ()) || *e != event::string)
+              fail << "name member string value expected";
+
+            // entries
+            //
+            if (!(e = p.next ()) || *e != event::name || p.name () != "entries")
+              fail << "entries member expected";
+
+            if (!(e = p.next ()) || *e != event::begin_array)
+              fail << "entries member array value expected";
+
+            while ((e = p.peek ()) && *e != event::end_array)
+              parse_entry (parse_entry);
+
+            if (!(e = p.next ()) || *e != event::end_array)
+              fail << "entries member array value end expected";
+          }
+          else if (t == "file" || t == "symlink" || t == "directory")
+          {
+            // path
+            //
+            if (!(e = p.next ()) || *e != event::name || p.name () != "path")
+              fail << "path member expected";
+
+            if (!(e = p.next ()) || *e != event::string)
+              fail << "path member string value expected";
+
+            path ep (p.value ()); // Assuming normalized.
+
+            if (t == "file" || t == "directory")
+            {
+              // mode
+              //
+              if (!(e = p.next ()) || *e != event::name || p.name () != "mode")
+                fail << "mode member expected";
+
+              if (!(e = p.next ()) || *e != event::string)
+                fail << "mode member string value expected";
+
+              string em (p.value ());
+
+              if (t == "file")
+              {
+                auto p (
+                  r.emplace (
+                    move (ep), installed_entry {move (em), nullptr}));
+
+                if (!p.second)
+                  fail << p.first->first << " is installed multiple times";
+              }
+            }
+            else
+            {
+              // target
+              //
+              if (!(e = p.next ()) || *e != event::name || p.name () != "target")
+                fail << "target member expected";
+
+              if (!(e = p.next ()) || *e != event::string)
+                fail << "target member string value expected";
+
+              path et (p.value ());
+              if (et.relative ())
+              {
+                et = ep.directory () / et;
+                et.normalize ();
+              }
+
+              auto i (r.find (et));
+              if (i == r.end ())
+                fail << "symlink " << ep << " target " << et << " does not "
+                     << "refer to previously installed entry";
+
+              auto p (r.emplace (move (ep), installed_entry {"", &*i}));
+
+              if (!p.second)
+                fail << p.first->first << " is installed multiple times";
+            }
+          }
+          else
+            fail << "unknown entry type '" << t << "'";
+
+          if (!(e = p.next ()) || *e != event::end_object)
+            fail << "entry object end expected";
+        };
+
+        while (p.peek ()) // More values.
+        {
+          parse_entry (parse_entry);
+
+          if (p.next ()) // Consume value-terminating nullopt.
+            fail << "unexpected data after entry object";
+        }
+
+        is.close ();
+      }
+      catch (const json::invalid_json_input& e)
+      {
+        if (pr.wait ())
+          fail << "invalid " << args[0] << " json input: " << e;
+
+        // Fall through.
+      }
+      catch (const io_error& e)
+      {
+        if (pr.wait ())
+          fail << "unable to read " << args[0] << " output: " << e;
+
+        // Fall through.
+      }
+
+      if (!pr.wait ())
+      {
+        diag_record dr (fail);
+        dr << args[0] << " exited with non-zero code";
+
+        if (verb < 2)
+        {
+          dr << info << "command line: ";
+          print_process (dr, args);
+        }
+      }
+    }
+    catch (const process_error& e)
+    {
+      error << "unable to execute " << args[0] << ": " << e;
+
+      if (e.child)
+        exit (1);
+
+      throw failed ();
+    }
+
+    return r;
+  }
 }
diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx
index 28b0c17..0fd219c 100644
--- a/bpkg/system-package-manager.hxx
+++ b/bpkg/system-package-manager.hxx
@@ -7,6 +7,8 @@
 #include <bpkg/types.hxx>
 #include <bpkg/utility.hxx>
 
+#include <libbutl/path-map.hxx>
+
 #include <bpkg/package.hxx>
 #include <bpkg/common-options.hxx>
 #include <bpkg/host-os-release.hxx>
@@ -329,6 +331,52 @@ namespace bpkg
                                 const string& name_id,
                                 const string& version_id,
                                 const vector<string>& like_ids);
+
+    // Return the map of filesystem entries (files and symlinks) that would be
+    // installed for the specified packages with the specified configuration
+    // variables.
+    //
+    // In essence, this function runs:
+    //
+    // b --dry-run --quiet <vars> !config.install.manifest=- install: <pkgs>
+    //
+    // And converts the printed installation manifest into the path map.
+    //
+    // Note that this function prints an appropriate progress indicator since
+    // even in the dry-run mode it may take some time (see the --dry-run
+    // option documentation for details).
+    //
+    struct installed_entry
+    {
+      string mode;                                     // Empty if symlink.
+      const pair<const path, installed_entry>* target; // Target if symlink.
+    };
+
+    class installed_entry_map: public butl::path_map<installed_entry>
+    {
+    public:
+      // Return true if there are filesystem entries in the specified
+      // directory or its subdirectories.
+      //
+      bool
+      contains (const dir_path& d)
+      {
+        auto p (find_sub (d));
+        return p.first != p.second;
+      }
+
+      bool
+      contains (string d)
+      {
+        return contains (dir_path (move (d)));
+      }
+    };
+
+    installed_entry_map
+    installed_entries (const common_options&,
+                       const packages& pkgs,
+                       const strings& vars);
+
   protected:
     optional<bool>   progress_;      // --[no]-progress (see also stderr_term)
     optional<size_t> fetch_timeout_; // --fetch-timeout
diff --git a/bpkg/utility.txx b/bpkg/utility.txx
index 0f88d53..3fe1d72 100644
--- a/bpkg/utility.txx
+++ b/bpkg/utility.txx
@@ -21,6 +21,9 @@ namespace bpkg
     {
       small_vector<const char*, 1> ops;
 
+      // NOTE: see custom versions in system_package_manager* if adding
+      //       anything new here (search for search_b()).
+
       // Map verbosity level. If we are running quiet or at level 1,
       // then run build2 quiet. Otherwise, run it at the same level
       // as us.