Reflect separating repository manifests into repositories file

9 files changed, 92 insertions, 59 deletions

diff --git a/brep/package b/brep/package
index c0c9fc9..45660a6 100644
--- a/brep/package
+++ b/brep/package
@@ -251,7 +251,12 @@ namespace brep
 
     // Initialized with timestamp_nonexistent by default.
     //
-    timestamp_type timestamp;
+    timestamp_type packages_timestamp;
+
+    // Initialized with timestamp_nonexistent by default.
+    // For external repositories stays timestamp_nonexistent.
+    //
+    timestamp_type repositories_timestamp;
 
     bool internal;
     package_versions_type package_versions;
diff --git a/loader/loader.cxx b/loader/loader.cxx
index f13349d..503f577 100644
--- a/loader/loader.cxx
+++ b/loader/loader.cxx
@@ -63,6 +63,9 @@ struct internal_repository
 
   path
   packages_path () const {return local_path / path ("packages");}
+
+  path
+  repositories_path () const {return local_path / path ("repositories");}
 };
 
 using internal_repositories = vector<internal_repository>;
@@ -173,6 +176,9 @@ load_repositories (path p)
       if (!file_exists (r.packages_path ()))
         bad_line ("'packages' file does not exist");
 
+      if (!file_exists (r.repositories_path ()))
+        bad_line ("'repositories' file does not exist");
+
       repos.emplace_back (move (r));
 
       // Check that there is no non-whitespace junk at the end.
@@ -206,7 +212,9 @@ changed (const internal_repositories& repos, database& db)
 
     if (pr == nullptr || r.location.string () != pr->location.string () ||
         r.display_name != pr->display_name || r.local_path != pr->local_path ||
-        file_mtime (r.packages_path ()) != pr->timestamp || !pr->internal)
+        file_mtime (r.packages_path ()) != pr->packages_timestamp ||
+        file_mtime (r.repositories_path ()) != pr->repositories_timestamp ||
+        !pr->internal)
       return true;
 
     names.emplace_back (r.location.canonical_name ());
@@ -231,7 +239,7 @@ changed (const internal_repositories& repos, database& db)
 static void
 load_repository (const shared_ptr<repository>& rp, database& db)
 {
-  if (rp->timestamp != timestamp_nonexistent)
+  if (rp->packages_timestamp != timestamp_nonexistent)
     return; // The repository is already loaded.
 
   // Only locally accessible repositories allowed until package manager API is
@@ -239,31 +247,31 @@ load_repository (const shared_ptr<repository>& rp, database& db)
   //
   assert (!rp->local_path.empty ());
 
-  path p (rp->local_path / path ("packages"));
-
-  ifstream ifs (p.string ());
-  if (!ifs.is_open ())
-    throw ifstream::failure (p.string () + ": unable to open");
-  ifs.exceptions (ifstream::badbit | ifstream::failbit);
-
-  // Mark as loaded. This is important in case we try to load this
-  // repository again recursively.
-  //
-  rp->timestamp = file_mtime (p);
-
-  manifest_parser mp (ifs, p.string ());
-  manifests ms (mp);
-
-  // Close to avoid unpredictable number of files being simultaneously
-  // opened due to load_repository() recursive calls.
-  //
-  ifs.close ();
+  auto mstream ([](const path& p, ifstream& f) -> timestamp
+    {
+      f.open (p.string ());
+      if (!f.is_open ())
+        throw ifstream::failure (p.string () + ": unable to open");
+      f.exceptions (ifstream::badbit | ifstream::failbit);
+      return file_mtime (p);
+    });
 
   // Don't add prerequisite repositories for external repositories.
   //
   if (rp->internal)
   {
-    for (auto& rm: ms.repositories)
+    repository_manifests rpm;
+
+    {
+      ifstream ifs;
+      path p (rp->local_path / path ("repositories"));
+      rp->repositories_timestamp = mstream (p, ifs);
+
+      manifest_parser mp (ifs, p.string ());
+      rpm = repository_manifests (mp);
+    }
+
+    for (auto& rm: rpm)
     {
       if (rm.location.empty ())
         continue; // Ignore entry for this repository.
@@ -342,7 +350,22 @@ load_repository (const shared_ptr<repository>& rp, database& db)
   session& s (session::current ());
   session::reset_current ();
 
-  for (auto& pm: ms.packages)
+  package_manifests pkm;
+
+  {
+    ifstream ifs;
+    path p (rp->local_path / path ("packages"));
+
+    // Mark as loaded. This is important in case we try to load this
+    // repository again recursively.
+    //
+    rp->packages_timestamp = mstream (p, ifs);
+
+    manifest_parser mp (ifs, p.string ());
+    pkm = package_manifests (mp);
+  }
+
+  for (auto& pm: pkm)
   {
     max_package_version mv;
 
@@ -521,10 +544,10 @@ main (int argc, char* argv[])
       db.erase_query<package> ();
       db.erase_query<package_version> ();
 
-      // We use repository object timestamp as a flag to signal that
+      // We use repository object packages_timestamp as a flag to signal that
       // we have already loaded this repo. The easiest way to make
       // it work in case of cycles is to use a session. This way,
-      // the repository object on which we updated the timestamp
+      // the repository object on which we updated the packages_timestamp
       // will be the same as the one we may check down the call
       // stack.
       //
@@ -532,7 +555,7 @@ main (int argc, char* argv[])
 
       // On the first pass over the internal repositories list we
       // persist empty repository objects, setting the interal flag
-      // to true and timestamp to non-existent. The idea is to
+      // to true and packages_timestamp to non-existent. The idea is to
       // establish the "final" list of internal repositories.
       //
       for (auto& ir: irs)
diff --git a/tests/loader/driver.cxx b/tests/loader/driver.cxx
index 1dd011f..0c6c995 100644
--- a/tests/loader/driver.cxx
+++ b/tests/loader/driver.cxx
@@ -97,7 +97,9 @@ main (int argc, char* argv[])
       dir_path srp (cp.directory () / dir_path ("internal/1/stable"));
       assert (sr->local_path == srp.normalize ());
 
-      assert (sr->timestamp == srt);
+      assert (sr->packages_timestamp == srt);
+      assert (sr->repositories_timestamp ==
+              file_mtime (dir_path (sr->local_path) / path ("repositories")));
       assert (sr->internal);
       assert (sr->prerequisite_repositories.size () == 2);
 
@@ -335,8 +337,10 @@ main (int argc, char* argv[])
       dir_path mrp (cp.directory () / dir_path ("internal/1/math"));
       assert (mr->local_path == mrp.normalize ());
 
-      assert (mr->timestamp ==
+      assert (mr->packages_timestamp ==
               file_mtime (dir_path (mr->local_path) / path ("packages")));
+      assert (mr->repositories_timestamp ==
+              file_mtime (dir_path (mr->local_path) / path ("repositories")));
       assert (mr->internal);
       assert (mr->prerequisite_repositories.size () == 1);
       assert (mr->prerequisite_repositories[0].load () == cr);
@@ -411,8 +415,9 @@ main (int argc, char* argv[])
       dir_path crp (cp.directory () / dir_path ("external/1/misc"));
       assert (cr->local_path == crp.normalize ());
 
-      assert (cr->timestamp ==
+      assert (cr->packages_timestamp ==
               file_mtime (dir_path (cr->local_path) / path ("packages")));
+      assert (cr->repositories_timestamp == timestamp_nonexistent);
       assert (!cr->internal);
       assert (cr->prerequisite_repositories.empty ());
 
diff --git a/tests/loader/external/1/misc/packages b/tests/loader/external/1/misc/packages
index d60d7fe..de96771 100644
--- a/tests/loader/external/1/misc/packages
+++ b/tests/loader/external/1/misc/packages
@@ -1,15 +1,4 @@
 : 1
-# Foreign repository manifest.
-#
-location: http://pkg.example.org/1/misc
-:
-# Foreign repository manifest.
-#
-location: http://pkg.example.org/1/math
-:
-# Local repository manifest (this repository).
-#
-:
 name: libbar
 version: 2.3.5
 priority: security; Very important to install.
diff --git a/tests/loader/external/1/misc/repositories b/tests/loader/external/1/misc/repositories
new file mode 100644
index 0000000..5128606
--- /dev/null
+++ b/tests/loader/external/1/misc/repositories
@@ -0,0 +1,11 @@
+: 1
+# Foreign repository manifest.
+#
+location: http://pkg.example.org/1/misc
+:
+# Foreign repository manifest.
+#
+location: http://pkg.example.org/1/math
+:
+# Local repository manifest (this repository).
+#
diff --git a/tests/loader/internal/1/math/packages b/tests/loader/internal/1/math/packages
index 2722f6e..e55168f 100644
--- a/tests/loader/internal/1/math/packages
+++ b/tests/loader/internal/1/math/packages
@@ -1,11 +1,4 @@
 : 1
-# Foreign repository manifest.
-#
-location: ../../../external/1/misc
-:
-# Local repository manifest (this repository).
-#
-:
 name: libexp
 version: 1+1.2
 summary: The exponent
diff --git a/tests/loader/internal/1/math/repositories b/tests/loader/internal/1/math/repositories
new file mode 100644
index 0000000..814d1db
--- /dev/null
+++ b/tests/loader/internal/1/math/repositories
@@ -0,0 +1,7 @@
+: 1
+# Foreign repository manifest.
+#
+location: ../../../external/1/misc
+:
+# Local repository manifest (this repository).
+#
diff --git a/tests/loader/internal/1/stable/packages b/tests/loader/internal/1/stable/packages
index a1aa839..59898c1 100644
--- a/tests/loader/internal/1/stable/packages
+++ b/tests/loader/internal/1/stable/packages
@@ -1,15 +1,4 @@
 : 1
-# Foreign repository manifest.
-#
-location: http://pkg.cppget.org/1/math
-:
-# Foreign repository manifest.
-#
-location: ../../../external/1/misc
-:
-# Local repository manifest (this repository).
-#
-:
 name: libfoo
 version: 1.2.3-4
 summary: The Foo library
diff --git a/tests/loader/internal/1/stable/repositories b/tests/loader/internal/1/stable/repositories
new file mode 100644
index 0000000..4c12e72
--- /dev/null
+++ b/tests/loader/internal/1/stable/repositories
@@ -0,0 +1,11 @@
+: 1
+# Foreign repository manifest.
+#
+location: http://pkg.cppget.org/1/math
+:
+# Foreign repository manifest.
+#
+location: ../../../external/1/misc
+:
+# Local repository manifest (this repository).
+#