Fix inability to fetch repository containing toolchain-unsatisfied packages (GH issue #305)

Note: the breaking commit is d6d72bc6e454.

12 files changed, 171 insertions, 9 deletions

diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 6f31478..d7f4cc6 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -314,7 +314,10 @@ namespace bpkg
           if (bs                  &&
               r.build2_dependency &&
               !satisfy_build2 (co, *r.build2_dependency))
+          {
             bs = false;
+            pds.clear (); // Won't now be used.
+          }
 
           nvs.push_back (move (r));
         }
@@ -328,7 +331,9 @@ namespace bpkg
         }
 
         mfs.push_back (move (f));
-        pds.push_back (move (d));
+
+        if (bs)
+          pds.push_back (move (d));
       }
 
       // Note that for the directory-based repositories we also query
@@ -1110,12 +1115,6 @@ namespace bpkg
       //
       if (rl.directory_based ())
       {
-        // Wouldn't be here otherwise. Note that rep_fetch_data is retrieved
-        // by rep_fetch() with false passed as the ignore_toolchain argument
-        // (see rep_fetch() for more details).
-        //
-        assert (!pis.empty ());
-
         // Note that we can't check if the external package of this upstream
         // version and revision is already available in the configuration
         // until we fetch all the repositories, as some of the available
@@ -1129,7 +1128,7 @@ namespace bpkg
             path_cast<dir_path> (rl.path () / *pm.location),
             pm.name,
             pm.version,
-            &pis[i],
+            !pis.empty () ? &pis[i] : nullptr,
             false   /* check_external */));
 
         if (v)
@@ -1314,6 +1313,12 @@ namespace bpkg
     // repository fragments list, as well as its prerequisite and complement
     // repository sets.
     //
+    // Note that we do this in the forward compatible manner ignoring
+    // unrecognized manifest values and unsatisfied build2 toolchain
+    // constraints in the package manifests. This approach allows older
+    // toolchains to work with newer repositories, successfully building the
+    // toolchain-satisfied packages and only failing for unsatisfied ones.
+    //
     rep_fetch_data rfd (
       rep_fetch (co,
                  &db.config_orig,
@@ -1321,7 +1326,7 @@ namespace bpkg
                  rl,
                  dependent_trust,
                  true /* ignore_unknow */,
-                 false /* ignore_toolchain */,
+                 true /* ignore_toolchain */,
                  false /* expand_values */,
                  true /* load_buildfiles */));
 
diff --git a/tests/common/compatibility/t15/libbar-1.0.0.tar.gz b/tests/common/compatibility/t15/libbar-1.0.0.tar.gz
new file mode 100644
index 0000000..2c741b2
--- /dev/null
+++ b/tests/common/compatibility/t15/libbar-1.0.0.tar.gz
diff --git a/tests/common/compatibility/t15/libbaz-1.0.0.tar.gz b/tests/common/compatibility/t15/libbaz-1.0.0.tar.gz
new file mode 100644
index 0000000..07e6d04
--- /dev/null
+++ b/tests/common/compatibility/t15/libbaz-1.0.0.tar.gz
diff --git a/tests/common/compatibility/t15/libbiz-1.0.0.tar.gz b/tests/common/compatibility/t15/libbiz-1.0.0.tar.gz
new file mode 100644
index 0000000..52ee52a
--- /dev/null
+++ b/tests/common/compatibility/t15/libbiz-1.0.0.tar.gz
diff --git a/tests/common/compatibility/t15/libfoo-1.0.0.tar.gz b/tests/common/compatibility/t15/libfoo-1.0.0.tar.gz
new file mode 100644
index 0000000..be052c2
--- /dev/null
+++ b/tests/common/compatibility/t15/libfoo-1.0.0.tar.gz
diff --git a/tests/common/compatibility/t15/repositories.manifest b/tests/common/compatibility/t15/repositories.manifest
new file mode 100644
index 0000000..5b70556
--- /dev/null
+++ b/tests/common/compatibility/t15/repositories.manifest
@@ -0,0 +1 @@
+: 1
diff --git a/tests/pkg-build.testscript b/tests/pkg-build.testscript
index 80e83a2..a3149ae 100644
--- a/tests/pkg-build.testscript
+++ b/tests/pkg-build.testscript
@@ -479,6 +479,7 @@ posix = ($cxx.target.class != 'windows')
   cp -r $src/t14e $out/t14e && $rep_create $out/t14e &$out/t14e/packages.manifest
   cp -r $src/t14f $out/t14f && $rep_create $out/t14f &$out/t14f/packages.manifest
   cp -r $src/t14i $out/t14i && $rep_create $out/t14i &$out/t14i/packages.manifest
+  cp -r $src/t15  $out/t15  && $rep_create $out/t15  &$out/t15/packages.manifest --ignore-unknown
 
   # Create git repositories.
   #
@@ -20295,3 +20296,64 @@ else
     }
   }
 }
+
+: compatibility
+:
+{
+  +$clone_cfg
+  +$rep_add $rep/t15 && $rep_fetch
+
+  test.arguments += --yes
+
+  : toolchain
+  :
+  {
+    +$clone_cfg
+
+    : build-satisfied
+    :
+    {
+      $clone_cfg;
+
+      $* libfoo libbar 2>>~%EOE%;
+        fetched libfoo/1.0.0
+        unpacked libfoo/1.0.0
+        fetched libbar/1.0.0
+        unpacked libbar/1.0.0
+        configured libfoo/1.0.0
+        configured libbar/1.0.0
+        %info: .+libfoo-1.0.0.+ is up to date%
+        %info: .+libbar-1.0.0.+ is up to date%
+        updated libfoo/1.0.0
+        updated libbar/1.0.0
+        EOE
+
+      $pkg_drop libfoo libbar
+    }
+
+    : build-fail-unsatisfied
+    :
+    {
+      $clone_cfg;
+
+      $* libbaz 2>>~%EOE% != 0
+        error: unable to satisfy constraint (build2 >= 65536.0.0) for package libbaz
+        %  info: available build2 version is .+%
+        info: while satisfying libbaz/1.0.0
+        EOE
+    }
+
+    : build-fail-unsatisfied-dependency
+    :
+    {
+      $clone_cfg;
+
+      $* libbiz 2>>~%EOE% != 0
+        error: unable to satisfy constraint (build2 >= 65536.0.0) for package libbaz
+        %  info: available build2 version is .+%
+        info: while satisfying libbaz/1.0.0
+        info: while satisfying libbiz/1.0.0
+        EOE
+    }
+  }
+}
diff --git a/tests/pkg-build/t15 b/tests/pkg-build/t15
new file mode 120000
index 0000000..c7ad857
--- /dev/null
+++ b/tests/pkg-build/t15
@@ -0,0 +1 @@
+../common/compatibility/t15
\ No newline at end of file
diff --git a/tests/pkg-verify.testscript b/tests/pkg-verify.testscript
index 7e7b5f3..8057aab 100644
--- a/tests/pkg-verify.testscript
+++ b/tests/pkg-verify.testscript
@@ -8,6 +8,7 @@
 # pkg-verify
 # |-- foo-1.tar.gz
 # |-- foo-2.tar.gz         (manifest with unknown name)
+# |-- libbaz-1.0.0.tar.gz  (manifest with unsatisfiable toolchain constraint)
 # `-- not-a-package.tar.gz
 
 : valid-package
@@ -137,3 +138,29 @@ $* --ignore-unknown --manifest $src/foo-2.tar.gz >>EOO
     %error: unable to find bootstrap.build file in package archive .+/foo-3.tar.gz%
     EOE
 }
+
+: compatibility
+:
+{
+  : fail
+  :
+  $* --manifest $src/libbaz-1.0.0.tar.gz 2>>/~%EOE% != 0
+    %error: unable to satisfy constraint \(build2 >= 65536.0.0\) for package .+/libbaz-1.0.0.tar.gz%
+    %  info: available build2 version is .+%
+    EOE
+
+  : success
+  :
+  $* --manifest --ignore-unknown $src/libbaz-1.0.0.tar.gz >>EOO
+    : 1
+    name: libbaz
+    version: 1.0.0
+    summary: libbaz
+    license: MIT
+    description: libbaz library
+    url: http://example.org
+    email: pkg@example.org
+    depends: * build2 >= 65536.0.0
+    depends: * bpkg >= 65536.0.0
+    EOO
+}
diff --git a/tests/pkg-verify/libbaz-1.0.0.tar.gz b/tests/pkg-verify/libbaz-1.0.0.tar.gz
new file mode 120000
index 0000000..a95a5dc
--- /dev/null
+++ b/tests/pkg-verify/libbaz-1.0.0.tar.gz
@@ -0,0 +1 @@
+../common/compatibility/t15/libbaz-1.0.0.tar.gz
\ No newline at end of file
diff --git a/tests/rep-info.testscript b/tests/rep-info.testscript
index 8db405d..c5f2d3a 100644
--- a/tests/rep-info.testscript
+++ b/tests/rep-info.testscript
@@ -13,6 +13,8 @@
 # |   |-- foo-1.tar.gz
 # |   `-- repositories.manifest
 # |
+# |-- t15 (see pkg-build for details)
+# |
 # `-- git
 #     |-- libbar.git      -> style-basic.git (prerequisite)
 #     `-- style-basic.git
@@ -35,6 +37,11 @@
   $rc --key $key $out/signed &$out/signed/packages.manifest \
                              &$out/signed/signature.manifest
 
+  # Create the compatibility repository.
+  #
+  cp -r $src/t15 $out/compatibility
+  $rc $out/compatibility &$out/compatibility/packages.manifest --ignore-unknown
+
   # Create git repositories.
   #
   $git_extract $src/git/libbar.tar
@@ -432,3 +439,60 @@ else
       EOO
   }
 }
+
+: compatibility
+:
+{
+  : packages
+  :
+  {
+    $* --packages $rep/compatibility >>EOO
+
+      libbar/1.0.0
+      libbaz/1.0.0
+      libbiz/1.0.0
+      libfoo/1.0.0
+      EOO
+  }
+
+  : package-manifests-ignore-toolchain
+  :
+  {
+    $* --packages --manifest --ignore-unknown $rep/compatibility >>~%EOO%
+      : 1
+      name: libbar
+      version: 1.0.0
+      %.+
+      depends: * build2 >= 0.16.0
+      depends: * bpkg >= 0.16.0
+      %.+
+      :
+      name: libbaz
+      version: 1.0.0
+      summary: libbaz
+      %.+
+      depends: * build2 >= 65536.0.0
+      depends: * bpkg >= 65536.0.0
+      %.+
+      :
+      name: libbiz
+      version: 1.0.0
+      summary: libbiz
+      %.+
+      :
+      name: libfoo
+      version: 1.0.0
+      summary: libfoo
+      %.+
+      EOO
+  }
+
+  : package-manifests-fail
+  :
+  {
+    $* --packages --manifest $rep/compatibility 2>>~%EOE% != 0
+      error: unable to satisfy constraint (build2 >= 65536.0.0) for package libbaz
+      %  info: available build2 version is .+%
+      EOE
+  }
+}
diff --git a/tests/rep-info/t15 b/tests/rep-info/t15
new file mode 120000
index 0000000..c7ad857
--- /dev/null
+++ b/tests/rep-info/t15
@@ -0,0 +1 @@
+../common/compatibility/t15
\ No newline at end of file