Add support for build-file package manifest value

8 files changed, 247 insertions, 108 deletions

diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx
index 56da715..d6a0fd9 100644
--- a/bpkg/manifest-utility.cxx
+++ b/bpkg/manifest-utility.cxx
@@ -362,13 +362,14 @@ namespace bpkg
   // Return the sorted list of *.build files (first) which are present in the
   // package's build/config/ subdirectory (or their alternatives) together
   // with the *-build manifest value names they correspond to (second). Skip
-  // files which are already present in the specified list. Note: throws
-  // system_error on filesystem errors.
+  // files which are already present in the specified buildfile/path
+  // lists. Note: throws system_error on filesystem errors.
   //
   static vector<pair<path, path>>
   find_buildfiles (const dir_path& config,
                    const string& ext,
-                   const vector<buildfile>& bs)
+                   const vector<buildfile>& bs,
+                   const vector<path>& bps)
   {
     vector<pair<path, path>> r;
 
@@ -386,7 +387,8 @@ namespace bpkg
 
           if (find_if (bs.begin (), bs.end (),
                        [&f] (const auto& v) {return v.path == f;}) ==
-              bs.end ())
+              bs.end () &&
+              find (bps.begin (), bps.end (), f) == bps.end ())
           {
             r.emplace_back (config / p, move (f));
           }
@@ -405,6 +407,7 @@ namespace bpkg
                                const optional<string>& rb,
                                const vector<buildfile>& bs,
                                const dir_path& d,
+                               const vector<path>& bps,
                                optional<bool> an)
   {
     if (d.empty ())
@@ -422,10 +425,10 @@ namespace bpkg
       return cs.string ();
     }
 
-    auto checksum = [&bb, &rb, &bs] (const path& b,
-                                     const path& r,
-                                     const dir_path& c,
-                                     const string& e)
+    auto checksum = [&bb, &rb, &bs, &bps] (const path& b,
+                                           const path& r,
+                                           const dir_path& c,
+                                           const string& e)
     {
       sha256 cs;
 
@@ -464,10 +467,23 @@ namespace bpkg
       for (const buildfile& b: bs)
         cs.append (b.content);
 
+      if (!bps.empty ())
+      {
+        dir_path bd (b.directory ());
+
+        for (const path& p: bps)
+        {
+          path f (bd / p);
+          f += "." + e;
+
+          append_file (f);
+        }
+      }
+
       if (root && exists (c))
       try
       {
-        for (auto& f: find_buildfiles (c, e, bs))
+        for (auto& f: find_buildfiles (c, e, bs, bps))
           append_file (f.first);
       }
       catch (const system_error& e)
@@ -521,6 +537,8 @@ namespace bpkg
   void
   load_package_buildfiles (package_manifest& m, const dir_path& d, bool erp)
   {
+    assert (m.buildfile_paths.empty ()); // build-file values must be expanded.
+
     auto load_buildfiles = [&m, &d, erp] (const path& b,
                                           const path& r,
                                           const dir_path& c,
@@ -559,8 +577,13 @@ namespace bpkg
       if (m.root_build && exists (c))
       try
       {
-        for (auto& f: find_buildfiles (c, ext, m.buildfiles))
+        for (auto& f: find_buildfiles (c,
+                                       ext,
+                                       m.buildfiles,
+                                       m.buildfile_paths))
+        {
           m.buildfiles.emplace_back (move (f.second), load (f.first));
+        }
       }
       catch (const system_error& e)
       {
diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx
index abb85ab..85fbaee 100644
--- a/bpkg/manifest-utility.hxx
+++ b/bpkg/manifest-utility.hxx
@@ -155,16 +155,18 @@ namespace bpkg
                     const package_info*);
 
   // Calculate the checksum of the buildfiles using the *-build manifest
-  // values. If the package source directory is specified (not empty), then
-  // use the files it contains for unspecified values. If the alt_naming flag
-  // is also specified for the latter case, then verify the package's
-  // buildfile naming scheme against its value and fail on mismatch.
+  // values and, if the package source directory is specified (not empty),
+  // build-file values. If the package source directory is specified, then
+  // also use the files it contains for unspecified values. If additionally
+  // the alt_naming flag is specified, then verify the package's buildfile
+  // naming scheme against its value and fail on mismatch.
   //
   string
   package_buildfiles_checksum (const optional<string>& bootstrap_build,
                                const optional<string>& root_build,
                                const vector<buildfile>& buildfiles,
                                const dir_path& src_dir = {},
+                               const vector<path>& buildfile_paths = {},
                                optional<bool> alt_naming = nullopt);
 
   // Load the package's buildfiles for unspecified manifest values. Throw
@@ -173,6 +175,10 @@ namespace bpkg
   // potential error description to be relative to the package source
   // directory.
   //
+  // Note that before calling this function you need to expand the build-file
+  // manifest values into the respective *-build values, for example, by
+  // calling manifest::load_files().
+  //
   void
   load_package_buildfiles (package_manifest&,
                            const dir_path& src_dir,
diff --git a/bpkg/pkg-unpack.cxx b/bpkg/pkg-unpack.cxx
index 2e21c70..99bd161 100644
--- a/bpkg/pkg-unpack.cxx
+++ b/bpkg/pkg-unpack.cxx
@@ -441,6 +441,7 @@ namespace bpkg
                                             m.root_build,
                                             m.buildfiles,
                                             d,
+                                            m.buildfile_paths,
                                             m.alt_naming);
       }
     }
diff --git a/bpkg/pkg-verify.cxx b/bpkg/pkg-verify.cxx
index 229b73d..26393d6 100644
--- a/bpkg/pkg-verify.cxx
+++ b/bpkg/pkg-verify.cxx
@@ -202,31 +202,41 @@ namespace bpkg
           throw failed ();
         }
 
-        // Expand the *-file manifest values, if requested.
+        // If requested, expand file-referencing package manifest values.
         //
-        if (ev)
+        if (ev || lb)
         {
           m.load_files (
-            [&pd, &co, &af, diag_level] (const string& n, const path& p)
+            [ev, &pd, &co, &af, diag_level]
+            (const string& n, const path& p) -> optional<string>
             {
-              path f (pd / p);
-              string s (extract (co, af, f, diag_level != 0));
+              bool bf (n == "build-file");
 
-              if (s.empty ())
+              // Always expand the build-file values.
+              //
+              if (ev || bf)
               {
-                if (diag_level != 0)
-                  error << n << " manifest value in package archive "
-                        << af << " references empty file " << f;
+                path f (pd / p);
+                string s (extract (co, af, f, diag_level != 0));
 
-                throw failed ();
-              }
+                if (s.empty () && !bf)
+                {
+                  if (diag_level != 0)
+                    error << n << " manifest value in package archive "
+                          << af << " references empty file " << f;
+
+                  throw failed ();
+                }
 
-              return s;
+                return s;
+              }
+              else
+                return nullopt;
             },
             iu);
         }
 
-        // Extract the bootstrap, root, and config/*.build buildfiles into the
+        // Load the bootstrap, root, and config/*.build buildfiles into the
         // respective *-build values, if requested and are not already
         // specified in the manifest.
         //
@@ -433,21 +443,53 @@ namespace bpkg
 
       // Load the bootstrap, root, and config/*.build buildfiles into the
       // respective *-build values, if requested and if they are not already
-      // specified in the manifest.
+      // specified in the manifest. But first expand the build-file manifest
+      // values into the respective *-build values.
       //
       // Note that we don't verify that the files are not empty.
       //
       if (lb)
-      try
-      {
-        load_package_buildfiles (m, d);
-      }
-      catch (const runtime_error& e)
       {
-        if (diag_level != 0)
-          error << e;
+        m.load_files (
+          [&d, &mf, diag_level]
+          (const string& n, const path& p) -> optional<string>
+          {
+            // Only expand the build-file values.
+            //
+            if (n == "build-file")
+            {
+              path f (d / p);
 
-        throw failed ();
+              try
+              {
+                ifdstream is (f);
+                return is.read_text ();
+              }
+              catch (const io_error& e)
+              {
+                if (diag_level != 0)
+                  error << "unable to read from " << f << " referenced by "
+                        << n << " manifest value in " << mf << ": " << e;
+
+                throw failed ();
+              }
+            }
+            else
+              return nullopt;
+          },
+          iu);
+
+        try
+        {
+          load_package_buildfiles (m, d);
+        }
+        catch (const runtime_error& e)
+        {
+          if (diag_level != 0)
+            error << e;
+
+          throw failed ();
+        }
       }
 
       // We used to verify package directory is <name>-<version> but it is
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 038d54e..4fceff7 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -192,6 +192,23 @@ namespace bpkg
     return r;
   }
 
+  static void
+  print_package_info (diag_record& dr,
+                      const dir_path& pl,
+                      const repository_location& rl,
+                      const optional<string>& fragment)
+  {
+    dr << "package ";
+
+    if (!pl.current ())
+      dr << "'" << pl.string () << "' "; // Strip trailing '/'.
+
+    dr << "in repository " << rl;
+
+    if (fragment)
+      dr << ' ' << *fragment;
+  }
+
   // Parse package manifests referenced by the package directory manifests.
   //
   static pair<vector<package_manifest>, vector<package_info>>
@@ -199,22 +216,16 @@ namespace bpkg
                            const dir_path& repo_dir,
                            vector<package_manifest>&& pms,
                            bool iu,
-                           bool lb,
                            const repository_location& rl,
                            const optional<string>& fragment) // For diagnostics.
   {
-    auto add_package_info = [&rl, &fragment] (const package_manifest& pm,
-                                              diag_record& dr)
+    auto prn_package_info = [&rl, &fragment] (diag_record& dr,
+                                              const package_manifest& pm)
     {
-      dr << "package ";
-
-      if (!pm.location->current ())
-        dr << "'" << pm.location->string () << "' "; // Strip trailing '/'.
-
-      dr << "in repository " << rl;
-
-      if (fragment)
-        dr << ' ' << *fragment;
+      print_package_info (dr,
+                          path_cast<dir_path> (*pm.location),
+                          rl,
+                          fragment);
     };
 
     // Verify that all the package directories contain the package manifest
@@ -251,19 +262,19 @@ namespace bpkg
         {
           diag_record dr (fail);
           dr << "no manifest file for ";
-          add_package_info (pm, dr);
+          prn_package_info (dr, pm);
         }
 
         // Provide the context if the package compatibility verification fails.
         //
         auto g (
           make_exception_guard (
-            [&pm, &add_package_info] ()
+            [&pm, &prn_package_info] ()
             {
               diag_record dr (info);
 
               dr << "while retrieving information for ";
-              add_package_info (pm, dr);
+              prn_package_info (dr, pm);
             }));
 
         try
@@ -323,30 +334,13 @@ namespace bpkg
         //
         m.location = move (*pm.location);
 
-        // Load the bootstrap, root, and config/*.build buildfiles into the
-        // respective *-build values, if requested and if they are not already
-        // specified in the manifest.
-        //
-        if (lb)
-        try
-        {
-          load_package_buildfiles (m, pds[i], true /* err_path_relative */);
-        }
-        catch (const runtime_error& e)
-        {
-          diag_record dr (fail);
-          dr << e << info;
-          add_package_info (m, dr);
-          dr << endf;
-        }
-
         pm = move (m);
       }
       catch (const manifest_parsing& e)
       {
         diag_record dr (fail (e.name, e.line, e.column));
         dr << e.description << info;
-        add_package_info (pm, dr);
+        prn_package_info (dr, pm);
       }
 
       r.first.push_back  (move (pm));
@@ -374,7 +368,7 @@ namespace bpkg
       ifdstream is (fp);
       string s (is.read_text ());
 
-      if (s.empty ())
+      if (s.empty () && name != "build-file")
         fail << name << " manifest value in " << pkg / manifest_file
              << " references empty file " << rp <<
           info << "repository " << rl
@@ -423,29 +417,60 @@ namespace bpkg
                                rd,
                                move (pms),
                                iu,
-                               lb,
                                rl,
                                empty_string /* fragment */));
 
     fr.packages      = move (pmi.first);
     fr.package_infos = move (pmi.second);
 
-    // Expand file-referencing package manifest values.
+    // If requested, expand file-referencing package manifest values and load
+    // the buildfiles into the respective *-build values.
     //
-    if (ev)
+    if (ev || lb)
     {
       for (package_manifest& m: fr.packages)
+      {
+        dir_path pl (path_cast<dir_path> (*m.location));
+
+        // Load *-file values.
+        //
         m.load_files (
-          [&m, &rd, &rl] (const string& n, const path& p)
+          [ev, &rd, &rl, &pl]
+          (const string& n, const path& p) -> optional<string>
           {
-            return read_package_file (p,
-                                      n,
-                                      path_cast<dir_path> (*m.location),
-                                      rd,
-                                      rl,
-                                      empty_string /* fragment */);
+            // Always expand the build-file values.
+            //
+            if (ev || n == "build-file")
+            {
+              return read_package_file (p,
+                                        n,
+                                        pl,
+                                        rd,
+                                        rl,
+                                        empty_string /* fragment */);
+            }
+            else
+              return nullopt;
           },
           iu);
+
+        // Load the bootstrap, root, and config/*.build buildfiles into the
+        // respective *-build values, if requested and if they are not already
+        // specified in the manifest.
+        //
+        if (lb)
+        try
+        {
+          load_package_buildfiles (m, rd / pl, true /* err_path_relative */);
+        }
+        catch (const runtime_error& e)
+        {
+          diag_record dr (fail);
+          dr << e << info;
+          print_package_info (dr, pl, rl, nullopt /* fragment */);
+          dr << endf;
+        }
+      }
     }
 
     return rep_fetch_data {{move (fr)},
@@ -587,42 +612,74 @@ namespace bpkg
                                  td,
                                  move (pms),
                                  iu,
-                                 lb,
                                  rl,
                                  fr.friendly_name));
 
       fr.packages      = move (pmi.first);
       fr.package_infos = move (pmi.second);
 
-      // Expand file-referencing package manifest values checking out
-      // submodules, if required.
+      // If requested, expand file-referencing package manifest values
+      // checking out submodules, if required, and load the buildfiles into
+      // the respective *-build values.
       //
-      if (ev)
+      if (ev || lb)
       {
         for (package_manifest& m: fr.packages)
+        {
+          dir_path pl (path_cast<dir_path> (*m.location));
+
+          // Load *-file values.
+          //
           m.load_files (
-            [&m, &td, &rl, &fr, &checkout_submodules] (const string& n,
-                                                       const path& p)
+            [ev, &td, &rl, &pl, &fr, &checkout_submodules]
+            (const string& n, const path& p) -> optional<string>
             {
-              // Note that this doesn't work for symlinks on Windows where git
-              // normally creates filesystem-agnostic symlinks that are
-              // indistinguishable from regular files (see fixup_worktree()
-              // for details). It seems like the only way to deal with that is
-              // to unconditionally checkout submodules on Windows. Let's not
-              // pessimize things for now (if someone really wants this to
-              // work, they can always enable real symlinks in git).
+              // Always expand the build-file values.
               //
-              if (!exists (td / *m.location / p))
-                checkout_submodules ();
-
-              return read_package_file (p,
-                                        n,
-                                        path_cast<dir_path> (*m.location),
-                                        td,
-                                        rl,
-                                        fr.friendly_name);
+              if (ev || n == "build-file")
+              {
+                // Check out submodules if the referenced file doesn't exist.
+                //
+                // Note that this doesn't work for symlinks on Windows where
+                // git normally creates filesystem-agnostic symlinks that are
+                // indistinguishable from regular files (see fixup_worktree()
+                // for details). It seems like the only way to deal with that
+                // is to unconditionally checkout submodules on Windows. Let's
+                // not pessimize things for now (if someone really wants this
+                // to work, they can always enable real symlinks in git).
+                //
+                if (!exists (td / pl / p))
+                  checkout_submodules ();
+
+                return read_package_file (p,
+                                          n,
+                                          pl,
+                                          td,
+                                          rl,
+                                          fr.friendly_name);
+              }
+              else
+                return nullopt;
             },
             iu);
+
+          // Load the bootstrap, root, and config/*.build buildfiles into the
+          // respective *-build values, if requested and if they are not
+          // already specified in the manifest.
+          //
+          if (lb)
+          try
+          {
+            load_package_buildfiles (m, td / pl, true /* err_path_relative */);
+          }
+          catch (const runtime_error& e)
+          {
+            diag_record dr (fail);
+            dr << e << info;
+            print_package_info (dr, pl, rl, fr.friendly_name);
+            dr << endf;
+          }
+        }
       }
 
       np += fr.packages.size ();
diff --git a/doc/manual.cli b/doc/manual.cli
index c85b483..0ef08cf 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -733,6 +733,8 @@ license: <licenses> [; <comment>]
 [build-include]: <config>[/<target>] [; <comment>]
 [build-exclude]: <config>[/<target>] [; <comment>]
 
+[build-file]: <path>
+
 [bootstrap-build]: <text>
 [root-build]: <text>
 [*-build]: <text>
@@ -1480,9 +1482,11 @@ Note that the comment of the matching exclusion is used by the web interface
 (\c{brep}) to display the reason for the build configuration exclusion.
 
 
-\h2#manifest-package-x-build|\c{{bootstrap,root,*\}-build[2]}|
+\h2#manifest-package-build-file|\c{build-file}|
 
 \
+[build-file]: <path>
+
 [bootstrap-build]: <text>
 [root-build]: <text>
 [*-build]: <text>
@@ -1499,24 +1503,30 @@ the alternative naming scheme should use the \c{*-build2} values instead of
 \c{*-build}.
 
 These files must reside in the package's \c{build/} subdirectory and have the
-\c{.build} extension (or their alternative names). The respective manifest
-value name prefix must be the file path relative to this subdirectory with the
-extension stripped.
+\c{.build} extension (or their alternative names). They can be provided either
+inline as text fragments or, for additional files, by referring to them with a
+path relative to this subdirectory, but not both. The \c{*-build}/\c{*-build2}
+manifest value name prefixes must be the file paths relative to this
+subdirectory with the extension stripped.
 
-As an example, the following value corresponds to the
+As an example, the following values correspond to the
 \c{build/config/common.build} file:
 
 \
+build-file: config/common.build
+
 config/common-build:
 \\
 config [bool] config.libhello.fancy ?= false
 \\
 \
 
-And the following value corresponds to the \c{build2/config/common.build2}
+And the following values correspond to the \c{build2/config/common.build2}
 file in a package with the alternative naming scheme:
 
 \
+build-file: config/common.build2
+
 config/common-build2:
 \\
 config [bool] config.libhello.fancy ?= false
diff --git a/tests/common/dependency-alternatives/t11a/libbaz-1.0.0.tar.gz b/tests/common/dependency-alternatives/t11a/libbaz-1.0.0.tar.gz
index 9fde627..a41505d 100644
--- a/tests/common/dependency-alternatives/t11a/libbaz-1.0.0.tar.gz
+++ b/tests/common/dependency-alternatives/t11a/libbaz-1.0.0.tar.gz
diff --git a/tests/common/dependency-alternatives/t11a/libbox-1.0.0.tar.gz b/tests/common/dependency-alternatives/t11a/libbox-1.0.0.tar.gz
index 8a417e4..ce7f51f 100644
--- a/tests/common/dependency-alternatives/t11a/libbox-1.0.0.tar.gz
+++ b/tests/common/dependency-alternatives/t11a/libbox-1.0.0.tar.gz