Add support for test-exclude task manifest valuemigration-example

Note that the last two commits demonstrate an approach to use for migrating
data on the ODB object member type change.

9 files changed, 207 insertions, 39 deletions

diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql
index 7b9fec5..e8d0775 100644
--- a/libbrep/build-extra.sql
+++ b/libbrep/build-extra.sql
@@ -10,6 +10,12 @@ DROP FOREIGN TABLE IF EXISTS build_package_constraints;
 
 DROP FOREIGN TABLE IF EXISTS build_package_builds;
 
+DROP FOREIGN TABLE IF EXISTS build_package_benchmarks;
+
+DROP FOREIGN TABLE IF EXISTS build_package_examples;
+
+DROP FOREIGN TABLE IF EXISTS build_package_tests;
+
 DROP FOREIGN TABLE IF EXISTS build_package;
 
 DROP FOREIGN TABLE IF EXISTS build_repository;
@@ -18,7 +24,6 @@ DROP FOREIGN TABLE IF EXISTS build_tenant;
 
 -- The foreign table for build_tenant object.
 --
---
 CREATE FOREIGN TABLE build_tenant (
   id TEXT NOT NULL,
   archived BOOLEAN NOT NULL)
@@ -26,7 +31,6 @@ SERVER package_server OPTIONS (table_name 'tenant');
 
 -- The foreign table for build_repository object.
 --
---
 CREATE FOREIGN TABLE build_repository (
   tenant TEXT NOT NULL,
   canonical_name TEXT NOT NULL,
@@ -37,7 +41,6 @@ SERVER package_server OPTIONS (table_name 'repository');
 
 -- The foreign table for build_package object.
 --
---
 CREATE FOREIGN TABLE build_package (
   tenant TEXT NOT NULL,
   name CITEXT NOT NULL,
@@ -49,12 +52,71 @@ CREATE FOREIGN TABLE build_package (
   version_release TEXT NULL,
   internal_repository_tenant TEXT NULL,
   internal_repository_canonical_name TEXT NULL,
-  buildable_ BOOLEAN NOT NULL)
+  buildable TEXT NOT NULL)
 SERVER package_server OPTIONS (table_name 'package');
 
--- The foreign table for the build_package object builds member (that is of a
+-- The foreign table for the build_package object tests member (that is of a
 -- container type).
 --
+CREATE FOREIGN TABLE build_package_tests (
+  tenant TEXT NOT NULL,
+  name CITEXT NOT NULL,
+  version_epoch INTEGER NOT NULL,
+  version_canonical_upstream TEXT NOT NULL,
+  version_canonical_release TEXT NOT NULL COLLATE "C",
+  version_revision INTEGER NOT NULL,
+  index BIGINT NOT NULL,
+  dep_name CITEXT NOT NULL,
+  dep_package_tenant TEXT NULL,
+  dep_package_name CITEXT NULL,
+  dep_package_version_epoch INTEGER NULL,
+  dep_package_version_canonical_upstream TEXT NULL,
+  dep_package_version_canonical_release TEXT NULL COLLATE "C",
+  dep_package_version_revision INTEGER NULL)
+SERVER package_server OPTIONS (table_name 'package_tests');
+
+-- The foreign table for the build_package object examples member (that is of a
+-- container type).
+--
+CREATE FOREIGN TABLE build_package_examples (
+  tenant TEXT NOT NULL,
+  name CITEXT NOT NULL,
+  version_epoch INTEGER NOT NULL,
+  version_canonical_upstream TEXT NOT NULL,
+  version_canonical_release TEXT NOT NULL COLLATE "C",
+  version_revision INTEGER NOT NULL,
+  index BIGINT NOT NULL,
+  dep_name CITEXT NOT NULL,
+  dep_package_tenant TEXT NULL,
+  dep_package_name CITEXT NULL,
+  dep_package_version_epoch INTEGER NULL,
+  dep_package_version_canonical_upstream TEXT NULL,
+  dep_package_version_canonical_release TEXT NULL COLLATE "C",
+  dep_package_version_revision INTEGER NULL)
+SERVER package_server OPTIONS (table_name 'package_examples');
+
+-- The foreign table for the build_package object benchmarks member (that is
+-- of a container type).
+--
+CREATE FOREIGN TABLE build_package_benchmarks (
+  tenant TEXT NOT NULL,
+  name CITEXT NOT NULL,
+  version_epoch INTEGER NOT NULL,
+  version_canonical_upstream TEXT NOT NULL,
+  version_canonical_release TEXT NOT NULL COLLATE "C",
+  version_revision INTEGER NOT NULL,
+  index BIGINT NOT NULL,
+  dep_name CITEXT NOT NULL,
+  dep_package_tenant TEXT NULL,
+  dep_package_name CITEXT NULL,
+  dep_package_version_epoch INTEGER NULL,
+  dep_package_version_canonical_upstream TEXT NULL,
+  dep_package_version_canonical_release TEXT NULL COLLATE "C",
+  dep_package_version_revision INTEGER NULL)
+SERVER package_server OPTIONS (table_name 'package_benchmarks');
+
+-- The foreign table for the build_package object builds member (that is of a
+-- container type).
 --
 CREATE FOREIGN TABLE build_package_builds (
   tenant TEXT NOT NULL,
@@ -71,7 +133,6 @@ SERVER package_server OPTIONS (table_name 'package_builds');
 -- The foreign table for the build_package object constraints member (that is
 -- of a container type).
 --
---
 CREATE FOREIGN TABLE build_package_constraints (
   tenant TEXT NOT NULL,
   name CITEXT NOT NULL,
diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx
index ffa9d0b..85d94e6 100644
--- a/libbrep/build-package.hxx
+++ b/libbrep/build-package.hxx
@@ -68,6 +68,19 @@ namespace brep
     build_repository (): canonical_name (id.canonical_name) {}
   };
 
+  // Forward declarations.
+  //
+  class build_package;
+
+  // Build package test dependency.
+  //
+  #pragma db value
+  struct build_dependency
+  {
+    package_name name;
+    lazy_shared_ptr<build_package> package;
+  };
+
   // Foreign object that is mapped to a subset of the package object.
   //
   #pragma db object table("build_package") pointer(shared_ptr) readonly session
@@ -76,8 +89,16 @@ namespace brep
   public:
     package_id id;
     upstream_version version;
+
+    // Mapped to the package object tests, examples, and benchmarks members
+    // using the PostgreSQL foreign table mechanism.
+    //
+    small_vector<build_dependency, 1> tests;
+    small_vector<build_dependency, 1> examples;
+    small_vector<build_dependency, 1> benchmarks;
+
     lazy_shared_ptr<build_repository> internal_repository;
-    bool buildable;
+    buildable_status buildable;
 
     // Mapped to the package object builds member using the PostgreSQL foreign
     // table mechanism.
@@ -89,15 +110,19 @@ namespace brep
     //
     build_constraints constraints;
 
+    bool
+    internal () const noexcept {return internal_repository != nullptr;}
+
     // Database mapping.
     //
     #pragma db member(id) id column("")
     #pragma db member(version) set(this.version.init (this.id.version, (?)))
+    #pragma db member(tests) id_column("") value_column("dep_")
+    #pragma db member(examples) id_column("") value_column("dep_")
+    #pragma db member(benchmarks) id_column("") value_column("dep_")
     #pragma db member(builds) id_column("") value_column("")
     #pragma db member(constraints) id_column("") value_column("")
 
-    #pragma db member(buildable) column("buildable_")
-
   private:
     friend class odb::access;
     build_package () = default;
@@ -111,7 +136,7 @@ namespace brep
   #pragma db view                                                      \
     object(build_package)                                              \
     object(build_repository inner:                                     \
-           build_package::buildable &&                                 \
+           build_package::buildable == "buildable" &&                  \
            brep::operator== (build_package::internal_repository,       \
                              build_repository::id))                    \
     object(build_tenant: build_package::id.tenant == build_tenant::id)
@@ -130,7 +155,7 @@ namespace brep
   #pragma db view                                                      \
     object(build_package)                                              \
     object(build_repository inner:                                     \
-           build_package::buildable &&                                 \
+           build_package::buildable == "buildable" &&                  \
            brep::operator== (build_package::internal_repository,       \
                              build_repository::id))                    \
     object(build_tenant: build_package::id.tenant == build_tenant::id)
diff --git a/libbrep/build.hxx b/libbrep/build.hxx
index a883fa0..8990880 100644
--- a/libbrep/build.hxx
+++ b/libbrep/build.hxx
@@ -25,7 +25,7 @@
 //
 #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 9
 
-#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 11, closed)
+#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 12, closed)
 
 // We have to keep these mappings at the global scope instead of inside
 // the brep namespace because they need to be also effective in the
@@ -285,7 +285,7 @@ namespace brep
   #pragma db view object(build)                                       \
     object(build_package inner:                                       \
            brep::operator== (build::id.package, build_package::id) && \
-           build_package::buildable)                                  \
+           build_package::buildable == "buildable")                   \
     query(distinct)
   struct toolchain
   {
@@ -329,7 +329,7 @@ namespace brep
     object(build)                                                      \
     object(build_package inner:                                        \
            brep::operator== (build::id.package, build_package::id) &&  \
-           build_package::buildable)                                   \
+           build_package::buildable == "buildable")                    \
     object(build_tenant: build_package::id.tenant == build_tenant::id)
   struct package_build
   {
@@ -340,7 +340,7 @@ namespace brep
     object(build)                                                      \
     object(build_package inner:                                        \
            brep::operator== (build::id.package, build_package::id) &&  \
-           build_package::buildable)                                   \
+           build_package::buildable == "buildable")                    \
     object(build_tenant: build_package::id.tenant == build_tenant::id)
   struct package_build_count
   {
diff --git a/libbrep/build.xml b/libbrep/build.xml
index f4ba6cb..03d60e7 100644
--- a/libbrep/build.xml
+++ b/libbrep/build.xml
@@ -1,4 +1,6 @@
 <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1">
+  <changeset version="12"/>
+
   <changeset version="11"/>
 
   <changeset version="10">
diff --git a/libbrep/common.cxx b/libbrep/common.cxx
index 2e9ff34..cddf44e 100644
--- a/libbrep/common.cxx
+++ b/libbrep/common.cxx
@@ -6,4 +6,32 @@
 namespace brep
 {
   const version wildcard_version (0, "0", nullopt, nullopt, 0);
+
+  // buildable_status
+  //
+  string
+  to_string (buildable_status s)
+  {
+    switch (s)
+    {
+    case buildable_status::buildable:   return "buildable";
+    case buildable_status::unbuildable: return "unbuildable";
+    case buildable_status::external:    return "external";
+    case buildable_status::test:        return "test";
+    case buildable_status::stub:        return "stub";
+    }
+
+    return string (); // Should never reach.
+  }
+
+  buildable_status
+  to_buildable_status (const string& s)
+  {
+         if (s == "buildable")   return buildable_status::buildable;
+    else if (s == "unbuildable") return buildable_status::unbuildable;
+    else if (s == "external")    return buildable_status::external;
+    else if (s == "test")        return buildable_status::test;
+    else if (s == "stub")        return buildable_status::stub;
+    else throw invalid_argument ("invalid buildable status '" + s + "'");
+  }
 }
diff --git a/libbrep/common.hxx b/libbrep/common.hxx
index b7fc2da..4aea110 100644
--- a/libbrep/common.hxx
+++ b/libbrep/common.hxx
@@ -323,6 +323,42 @@ namespace brep
 
   #pragma db value(build_constraint) definition
 
+  // Whether a package is buildable by the build bot controller service and
+  // what is the "main" reason if it is not.
+  //
+  // While we could potentially calculate this status on the fly, that would
+  // complicate the database queries significantly.
+  //
+  enum class buildable_status: std::uint8_t
+  {
+    buildable,   // Not a test nor stub, from internal buildable repository.
+    unbuildable, // Not a test nor stub, from internal unbuildable repository.
+    external,    // Not a test nor stub, from external repository.
+    test,        // Test, not a stub.
+    stub         // Stub.
+  };
+
+  string
+  to_string (buildable_status);
+
+  buildable_status
+  to_buildable_status (const string&); // May throw invalid_argument.
+
+  inline ostream&
+  operator<< (ostream& os, buildable_status s) {return os << to_string (s);}
+
+  // Return true if the status is not buildable.
+  //
+  inline bool
+  operator! (buildable_status s)
+  {
+    return s != buildable_status::buildable;
+  }
+
+  #pragma db map type(buildable_status) as(string) \
+    to(to_string (?))                              \
+    from(brep::to_buildable_status (?))
+
   // Version comparison operators.
   //
   // They allow comparing objects that have epoch, canonical_upstream,
diff --git a/libbrep/package.cxx b/libbrep/package.cxx
index ec8e74a..2178392 100644
--- a/libbrep/package.cxx
+++ b/libbrep/package.cxx
@@ -110,12 +110,15 @@ namespace brep
         examples (move (es)),
         benchmarks (move (bms)),
         builds (move (bs)),
-        build_constraints (!stub () ? move (bc) : build_constraints_type ()),
+        build_constraints (move (bc)),
         internal_repository (move (rp)),
         location (move (lc)),
         fragment (move (fr)),
         sha256sum (move (sh)),
-        buildable (!stub () && internal_repository->buildable)
+        buildable (
+          stub ()                         ? buildable_status::stub        :
+          !internal_repository->buildable ? buildable_status::unbuildable :
+                                            buildable_status::buildable)
   {
     assert (internal_repository->internal);
   }
@@ -123,12 +126,18 @@ namespace brep
   package::
   package (package_name nm,
            version_type vr,
+           build_class_exprs bs,
+           build_constraints_type bc,
            shared_ptr<repository_type> rp)
       : id (rp->tenant, move (nm), vr),
         tenant (id.tenant),
         name (id.name),
         version (move (vr)),
-        buildable (false)
+        builds (move (bs)),
+        build_constraints (move (bc)),
+        buildable (stub ()
+                   ? buildable_status::stub
+                   : buildable_status::external)
   {
     assert (!rp->internal);
     other_repositories.emplace_back (move (rp));
diff --git a/libbrep/package.hxx b/libbrep/package.hxx
index f679c62..09bde77 100644
--- a/libbrep/package.hxx
+++ b/libbrep/package.hxx
@@ -20,7 +20,7 @@
 //
 #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 17
 
-#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 18, closed)
+#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 19, closed)
 
 namespace brep
 {
@@ -362,8 +362,7 @@ namespace brep
     using requirements_type = brep::requirements;
     using build_constraints_type = brep::build_constraints;
 
-    // Create internal package object. Note that for stubs the build
-    // constraints are meaningless, and so not saved.
+    // Create internal package object.
     //
     package (package_name,
              version_type,
@@ -400,12 +399,22 @@ namespace brep
 
     // Create external package object.
     //
-    // External repository packages can appear on the WEB interface only in
-    // dependency list in the form of a link to the corresponding WEB page.
-    // The only package information required to compose such a link is the
-    // package name, version, and repository location.
+    // External package can appear on the WEB interface only in dependency
+    // list in the form of a link to the corresponding WEB page. The only
+    // package information required to compose such a link is the package name,
+    // version, and repository location.
     //
-    package (package_name name, version_type, shared_ptr<repository_type>);
+    // External package can also be a separate test for some primary package
+    // (and belong to a complement but yet external repository), and so we may
+    // need its build class expressions and constraints to decide if to build
+    // it together with the primary package or not (see test-exclude task
+    // manifest value for details).
+    //
+    package (package_name name,
+             version_type,
+             build_class_exprs,
+             build_constraints_type,
+             shared_ptr<repository_type>);
 
     bool
     internal () const noexcept {return internal_repository != nullptr;}
@@ -451,9 +460,9 @@ namespace brep
     optional<email_type> build_error_email;
     dependencies_type dependencies;
     requirements_type requirements;
-    small_vector<dependency, 1> tests;
-    small_vector<dependency, 1> examples;
-    small_vector<dependency, 1> benchmarks;
+    small_vector<dependency, 1> tests;        // Note: foreign-mapped in build.
+    small_vector<dependency, 1> examples;     // Note: foreign-mapped in build.
+    small_vector<dependency, 1> benchmarks;   // Note: foreign-mapped in build.
 
     build_class_exprs builds;                 // Note: foreign-mapped in build.
     build_constraints_type build_constraints; // Note: foreign-mapped in build.
@@ -478,16 +487,9 @@ namespace brep
 
     vector<lazy_shared_ptr<repository_type>> other_repositories;
 
-    // Whether the package is buildable by the build bot controller service.
-    // Can only be true for non-stubs that belong to at least one buildable
-    // (internal) repository.
-    //
-    // While we could potentially calculate this flag on the fly, that would
-    // complicate the database queries significantly.
-    //
     // Note: foreign-mapped in build.
     //
-    bool buildable;
+    buildable_status buildable;
 
     // Database mapping.
     //
@@ -583,8 +585,6 @@ namespace brep
     #pragma db member(other_repositories)                     \
       id_column("") value_column("repository_") value_not_null
 
-    #pragma db member(buildable) column("buildable_")
-
     // search_index
     //
     #pragma db member(search_index) virtual(weighted_text) null \
diff --git a/libbrep/package.xml b/libbrep/package.xml
index 292cd05..0dcc472 100644
--- a/libbrep/package.xml
+++ b/libbrep/package.xml
@@ -1,4 +1,11 @@
 <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1">
+  <changeset version="19">
+    <alter-table name="package">
+      <add-column name="buildable" type="TEXT" null="false"/>
+      <drop-column name="buildable_"/>
+    </alter-table>
+  </changeset>
+
   <changeset version="18">
     <alter-table name="package">
       <add-column name="buildable_" type="BOOLEAN" null="false"/>