From f11d8c32c01ab1ac13268484b9e85732176a47d9 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Wed, 1 Apr 2020 21:50:16 +0300 Subject: Add support for test-exclude task manifest value --- libbrep/build-extra.sql | 71 +++++++++++++++++++++-- libbrep/build-package.hxx | 27 +++++++++ libbrep/build.hxx | 2 +- libbrep/build.xml | 2 + libbrep/common.cxx | 26 +++++++++ libbrep/common.hxx | 32 ++++++++++ libbrep/package.cxx | 21 +++++-- libbrep/package.hxx | 41 +++++++------ libbrep/package.xml | 6 ++ load/load.cxx | 52 +++++++++++++---- migrate/migrate.cxx | 11 +++- mod/mod-build-task.cxx | 42 ++++++++++++-- tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz | Bin 262 -> 264 bytes tests/load/1/math/libfoo-examples-1.2.4.tar.gz | Bin 268 -> 263 bytes tests/load/1/math/libfoo-tests-1.2.4.tar.gz | Bin 259 -> 260 bytes tests/load/1/math/packages.manifest | 12 ++-- 16 files changed, 292 insertions(+), 53 deletions(-) diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index 9ecbcb1..b466382 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -10,6 +10,12 @@ DROP FOREIGN TABLE IF EXISTS build_package_constraints; DROP FOREIGN TABLE IF EXISTS build_package_builds; +DROP FOREIGN TABLE IF EXISTS build_package_benchmarks; + +DROP FOREIGN TABLE IF EXISTS build_package_examples; + +DROP FOREIGN TABLE IF EXISTS build_package_tests; + DROP FOREIGN TABLE IF EXISTS build_package; DROP FOREIGN TABLE IF EXISTS build_repository; @@ -18,7 +24,6 @@ DROP FOREIGN TABLE IF EXISTS build_tenant; -- The foreign table for build_tenant object. -- --- CREATE FOREIGN TABLE build_tenant ( id TEXT NOT NULL, archived BOOLEAN NOT NULL) @@ -26,7 +31,6 @@ SERVER package_server OPTIONS (table_name 'tenant'); -- The foreign table for build_repository object. -- --- CREATE FOREIGN TABLE build_repository ( tenant TEXT NOT NULL, canonical_name TEXT NOT NULL, @@ -37,7 +41,6 @@ SERVER package_server OPTIONS (table_name 'repository'); -- The foreign table for build_package object. -- --- CREATE FOREIGN TABLE build_package ( tenant TEXT NOT NULL, name CITEXT NOT NULL, @@ -52,9 +55,68 @@ CREATE FOREIGN TABLE build_package ( buildable BOOLEAN NOT NULL) SERVER package_server OPTIONS (table_name 'package'); --- The foreign table for the build_package object builds member (that is of a +-- The foreign table for the build_package object tests member (that is of a -- container type). -- +CREATE FOREIGN TABLE build_package_tests ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + index BIGINT NOT NULL, + dep_name CITEXT NOT NULL, + dep_package_tenant TEXT NULL, + dep_package_name CITEXT NULL, + dep_package_version_epoch INTEGER NULL, + dep_package_version_canonical_upstream TEXT NULL, + dep_package_version_canonical_release TEXT NULL COLLATE "C", + dep_package_version_revision INTEGER NULL) +SERVER package_server OPTIONS (table_name 'package_tests'); + +-- The foreign table for the build_package object examples member (that is of a +-- container type). +-- +CREATE FOREIGN TABLE build_package_examples ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + index BIGINT NOT NULL, + dep_name CITEXT NOT NULL, + dep_package_tenant TEXT NULL, + dep_package_name CITEXT NULL, + dep_package_version_epoch INTEGER NULL, + dep_package_version_canonical_upstream TEXT NULL, + dep_package_version_canonical_release TEXT NULL COLLATE "C", + dep_package_version_revision INTEGER NULL) +SERVER package_server OPTIONS (table_name 'package_examples'); + +-- The foreign table for the build_package object benchmarks member (that is +-- of a container type). +-- +CREATE FOREIGN TABLE build_package_benchmarks ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + index BIGINT NOT NULL, + dep_name CITEXT NOT NULL, + dep_package_tenant TEXT NULL, + dep_package_name CITEXT NULL, + dep_package_version_epoch INTEGER NULL, + dep_package_version_canonical_upstream TEXT NULL, + dep_package_version_canonical_release TEXT NULL COLLATE "C", + dep_package_version_revision INTEGER NULL) +SERVER package_server OPTIONS (table_name 'package_benchmarks'); + +-- The foreign table for the build_package object builds member (that is of a +-- container type). -- CREATE FOREIGN TABLE build_package_builds ( tenant TEXT NOT NULL, @@ -71,7 +133,6 @@ SERVER package_server OPTIONS (table_name 'package_builds'); -- The foreign table for the build_package object constraints member (that is -- of a container type). -- --- CREATE FOREIGN TABLE build_package_constraints ( tenant TEXT NOT NULL, name CITEXT NOT NULL, diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 702f937..228c5c0 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -68,6 +68,19 @@ namespace brep build_repository (): canonical_name (id.canonical_name) {} }; + // Forward declarations. + // + class build_package; + + // Build package test dependency. + // + #pragma db value + struct build_dependency + { + package_name name; + lazy_shared_ptr package; + }; + // Foreign object that is mapped to a subset of the package object. // #pragma db object table("build_package") pointer(shared_ptr) readonly session @@ -76,6 +89,14 @@ namespace brep public: package_id id; upstream_version version; + + // Mapped to the package object tests, examples, and benchmarks members + // using the PostgreSQL foreign table mechanism. + // + small_vector tests; + small_vector examples; + small_vector benchmarks; + lazy_shared_ptr internal_repository; bool buildable; @@ -89,10 +110,16 @@ namespace brep // build_constraints constraints; + bool + internal () const noexcept {return internal_repository != nullptr;} + // Database mapping. // #pragma db member(id) id column("") #pragma db member(version) set(this.version.init (this.id.version, (?))) + #pragma db member(tests) id_column("") value_column("dep_") + #pragma db member(examples) id_column("") value_column("dep_") + #pragma db member(benchmarks) id_column("") value_column("dep_") #pragma db member(builds) id_column("") value_column("") #pragma db member(constraints) id_column("") value_column("") diff --git a/libbrep/build.hxx b/libbrep/build.hxx index 83b30a8..a883fa0 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -25,7 +25,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 9 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 10, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 11, closed) // We have to keep these mappings at the global scope instead of inside // the brep namespace because they need to be also effective in the diff --git a/libbrep/build.xml b/libbrep/build.xml index bf8920b..f4ba6cb 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,6 @@ + + diff --git a/libbrep/common.cxx b/libbrep/common.cxx index 2e9ff34..8964e0a 100644 --- a/libbrep/common.cxx +++ b/libbrep/common.cxx @@ -6,4 +6,30 @@ namespace brep { const version wildcard_version (0, "0", nullopt, nullopt, 0); + + // unbuildable_reason + // + string + to_string (unbuildable_reason r) + { + switch (r) + { + case unbuildable_reason::stub: return "stub"; + case unbuildable_reason::test: return "test"; + case unbuildable_reason::external: return "external"; + case unbuildable_reason::unbuildable: return "unbuildable"; + } + + return string (); // Should never reach. + } + + unbuildable_reason + to_unbuildable_reason (const string& r) + { + if (r == "stub") return unbuildable_reason::stub; + else if (r == "test") return unbuildable_reason::test; + else if (r == "external") return unbuildable_reason::external; + else if (r == "unbuildable") return unbuildable_reason::unbuildable; + else throw invalid_argument ("invalid unbuildable reason '" + r + "'"); + } } diff --git a/libbrep/common.hxx b/libbrep/common.hxx index b7fc2da..73353c7 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -323,6 +323,38 @@ namespace brep #pragma db value(build_constraint) definition + // The primary reason why a package is unbuildable by the build bot + // controller service. + // + enum class unbuildable_reason: std::uint8_t + { + stub, // A stub, otherwise + test, // A separate test (built as part of primary), otherwise + external, // From an external repository, otherwise + unbuildable // From an internal unbuildable repository. + }; + + string + to_string (unbuildable_reason); + + unbuildable_reason + to_unbuildable_reason (const string&); // May throw invalid_argument. + + inline ostream& + operator<< (ostream& os, unbuildable_reason r) {return os << to_string (r);} + + using optional_unbuildable_reason = optional; + + #pragma db map type(unbuildable_reason) as(string) \ + to(to_string (?)) \ + from(brep::to_unbuildable_reason (?)) + + #pragma db map type(optional_unbuildable_reason) as(brep::optional_string) \ + to((?) ? to_string (*(?)) : brep::optional_string ()) \ + from((?) \ + ? brep::to_unbuildable_reason (*(?)) \ + : brep::optional_unbuildable_reason ()) \ + // Version comparison operators. // // They allow comparing objects that have epoch, canonical_upstream, diff --git a/libbrep/package.cxx b/libbrep/package.cxx index ec8e74a..1b7c4f6 100644 --- a/libbrep/package.cxx +++ b/libbrep/package.cxx @@ -110,25 +110,38 @@ namespace brep examples (move (es)), benchmarks (move (bms)), builds (move (bs)), - build_constraints (!stub () ? move (bc) : build_constraints_type ()), + build_constraints (move (bc)), internal_repository (move (rp)), location (move (lc)), fragment (move (fr)), - sha256sum (move (sh)), - buildable (!stub () && internal_repository->buildable) + sha256sum (move (sh)) { + if (stub ()) + unbuildable_reason = unbuildable_reason::stub; + else if (!internal_repository->buildable) + unbuildable_reason = unbuildable_reason::unbuildable; + + buildable = !unbuildable_reason; + assert (internal_repository->internal); } package:: package (package_name nm, version_type vr, + build_class_exprs bs, + build_constraints_type bc, shared_ptr rp) : id (rp->tenant, move (nm), vr), tenant (id.tenant), name (id.name), version (move (vr)), - buildable (false) + builds (move (bs)), + build_constraints (move (bc)), + buildable (false), + unbuildable_reason (stub () + ? unbuildable_reason::stub + : unbuildable_reason::external) { assert (!rp->internal); other_repositories.emplace_back (move (rp)); diff --git a/libbrep/package.hxx b/libbrep/package.hxx index 07bd2a0..59ee589 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -20,7 +20,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 17 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 17, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 18, closed) namespace brep { @@ -362,8 +362,7 @@ namespace brep using requirements_type = brep::requirements; using build_constraints_type = brep::build_constraints; - // Create internal package object. Note that for stubs the build - // constraints are meaningless, and so not saved. + // Create internal package object. // package (package_name, version_type, @@ -400,12 +399,22 @@ namespace brep // Create external package object. // - // External repository packages can appear on the WEB interface only in - // dependency list in the form of a link to the corresponding WEB page. - // The only package information required to compose such a link is the - // package name, version, and repository location. + // External package can appear on the WEB interface only in dependency + // list in the form of a link to the corresponding WEB page. The only + // package information required to compose such a link is the package name, + // version, and repository location. // - package (package_name name, version_type, shared_ptr); + // External package can also be a separate test for some primary package + // (and belong to a complement but yet external repository), and so we may + // need its build class expressions and constraints to decide if to build + // it together with the primary package or not (see test-exclude task + // manifest value for details). + // + package (package_name name, + version_type, + build_class_exprs, + build_constraints_type, + shared_ptr); bool internal () const noexcept {return internal_repository != nullptr;} @@ -451,9 +460,9 @@ namespace brep optional build_error_email; dependencies_type dependencies; requirements_type requirements; - small_vector tests; - small_vector examples; - small_vector benchmarks; + small_vector tests; // Note: foreign-mapped in build. + small_vector examples; // Note: foreign-mapped in build. + small_vector benchmarks; // Note: foreign-mapped in build. build_class_exprs builds; // Note: foreign-mapped in build. build_constraints_type build_constraints; // Note: foreign-mapped in build. @@ -478,16 +487,14 @@ namespace brep vector> other_repositories; - // Whether the package is buildable by the build bot controller service. - // Can only be true for non-stubs that belong to at least one buildable - // (internal) repository. + // Whether the package is buildable by the build bot controller service + // and the reason if it's not. // // While we could potentially calculate this flag on the fly, that would // complicate the database queries significantly. // - // Note: foreign-mapped in build. - // - bool buildable; + bool buildable; // Note: foreign-mapped in build. + optional unbuildable_reason; // Database mapping. // diff --git a/libbrep/package.xml b/libbrep/package.xml index 785ae0f..12ae36e 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,10 @@ + + + + + + diff --git a/load/load.cxx b/load/load.cxx index 260f557..b289829 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -566,7 +566,11 @@ load_packages (const shared_ptr& rp, else // Create external package object. // - p = make_shared (move (pm.name), move (pm.version), rp); + p = make_shared (move (pm.name), + move (pm.version), + move (pm.builds), + move (pm.build_constraints), + rp); db.persist (p); } @@ -600,9 +604,14 @@ load_packages (const shared_ptr& rp, // A non-stub package is buildable if belongs to at least one // buildable repository (see libbrep/package.hxx for details). + // Note that if this is an external test package it will be marked as + // unbuildable later (see resolve_dependencies() for details). // - if (!p->stub () && !p->buildable) - p->buildable = rp->buildable; + if (rp->buildable && !p->buildable && !p->stub ()) + { + p->buildable = true; + p->unbuildable_reason = nullopt; + } } p->other_repositories.push_back (rp); @@ -852,9 +861,11 @@ find (const lazy_shared_ptr& r, // Resolve package run-time dependencies, tests, examples, and benchmarks. // Make sure that the best matching dependency belongs to the package // repositories, their complements, recursively, or their immediate -// prerequisite repositories (only for run-time dependencies). Fail if unable -// to resolve a dependency, unless ignore_unresolved is true in which case -// leave this dependency NULL. Should be called once per internal package. +// prerequisite repositories (only for run-time dependencies). Set the +// buildable flag to false for the resolved tests, examples, and benchmarks +// packages. Fail if unable to resolve a dependency, unless ignore_unresolved +// is true in which case leave this dependency NULL. Should be called once per +// internal package. // static void resolve_dependencies (package& p, database& db, bool ignore_unresolved) @@ -872,7 +883,7 @@ resolve_dependencies (package& p, database& db, bool ignore_unresolved) p.benchmarks.empty ()) return; - auto resolve = [&p, &db] (dependency& d, bool prereq) + auto resolve = [&p, &db] (dependency& d, bool test) { // Dependency should not be resolved yet. // @@ -934,9 +945,26 @@ resolve_dependencies (package& p, database& db, bool ignore_unresolved) for (const auto& pp: db.query (q + order_by_version_desc (vm))) { - if (find (p.internal_repository, pp, prereq)) + if (find (p.internal_repository, pp, !test)) { d.package.reset (db, pp.id); + + // If the resolved dependency is an external test, then mark it as + // such, unless it is a stub. + // + if (test) + { + shared_ptr dp (d.package.load ()); + + if (!dp->stub ()) + { + dp->buildable = false; + dp->unbuildable_reason = unbuildable_reason::test; + + db.update (dp); + } + } + return true; } } @@ -963,26 +991,26 @@ resolve_dependencies (package& p, database& db, bool ignore_unresolved) // specifying in the manifest file an alternative which can't be // resolved, unless unresolved dependencies are allowed. // - if (!resolve (d, true /* prereq */) && !ignore_unresolved) + if (!resolve (d, false /* test */) && !ignore_unresolved) bail (d, "dependency"); } } for (dependency& d: p.tests) { - if (!resolve (d, false /* prereq */) && !ignore_unresolved) + if (!resolve (d, true /* test */) && !ignore_unresolved) bail (d, "tests"); } for (dependency& d: p.examples) { - if (!resolve (d, false /* prereq */) && !ignore_unresolved) + if (!resolve (d, true /* test */) && !ignore_unresolved) bail (d, "examples"); } for (dependency& d: p.benchmarks) { - if (!resolve (d, false /* prereq */) && !ignore_unresolved) + if (!resolve (d, true /* test */) && !ignore_unresolved) bail (d, "benchmarks"); } diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx index 81c4543..e892a39 100644 --- a/migrate/migrate.cxx +++ b/migrate/migrate.cxx @@ -206,7 +206,6 @@ create (database& db, bool extra_only) const // Register the data migration functions for the package database schema. // -#if 0 template using package_migration_entry_base = data_migration_entry; @@ -218,11 +217,19 @@ struct package_migration_entry: package_migration_entry_base : package_migration_entry_base (f, "package") {} }; +// Set the unbuildable reason for unbuildable packages. +// +// Note that we are unable to restore the exact reason and so always set it +// to 'unbuildable'. +// +//#if 0 static const package_migration_entry<18> package_migrate_v18 ([] (database& db) { + db.execute ("UPDATE package SET unbuildable_reason = 'unbuildable' " + "WHERE NOT buildable"); }); -#endif +//#endif // main() function // diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index edddb89..3eaa0d3 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -220,14 +220,44 @@ handle (request& rq, response& rs) lazy_shared_ptr r (p->internal_repository); - strings fp; + strings fps; if (r->certificate_fingerprint) - fp.emplace_back (move (*r->certificate_fingerprint)); + fps.emplace_back (move (*r->certificate_fingerprint)); + + // Exclude external test packages which exclude the task build + // configuration. + // + small_vector tes; + auto add_exclusions = [&tes, &cm, this] + (const small_vector& tests) + { + for (const build_dependency& t: tests) + { + // Don't exclude unresolved external tests. + // + // Note that this may result in the build task failure. However, + // silently excluding such tests could end up with missed software + // bugs which feels much worse. + // + if (t.package != nullptr) + { + shared_ptr p (t.package.load ()); + + if (exclude (p->builds, p->constraints, *cm.config)) + tes.push_back (package {move (p->id.name), move (p->version)}); + } + } + }; + + add_exclusions (p->tests); + add_exclusions (p->examples); + add_exclusions (p->benchmarks); task_manifest task (move (b->package_name), move (b->package_version), move (r->location), - move (fp), + move (fps), + move (tes), cm.machine->name, cm.config->target, cm.config->environment, @@ -595,7 +625,7 @@ handle (request& rq, response& rs) // // We iterate over buildable packages. // - assert (p->internal_repository != nullptr); + assert (p->internal ()); p->internal_repository.load (); @@ -677,8 +707,8 @@ handle (request& rq, response& rs) shared_ptr p ( build_db_->find (b->id.package)); - if (p != nullptr && - p->internal_repository != nullptr && + if (p != nullptr && + p->internal () && !exclude (p->builds, p->constraints, *cm.config)) { assert (b->status); diff --git a/tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz b/tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz index 606893a..391eb6f 100644 Binary files a/tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz and b/tests/load/1/math/libfoo-benchmarks-1.2.4.tar.gz differ diff --git a/tests/load/1/math/libfoo-examples-1.2.4.tar.gz b/tests/load/1/math/libfoo-examples-1.2.4.tar.gz index c1d5fbd..eac5190 100644 Binary files a/tests/load/1/math/libfoo-examples-1.2.4.tar.gz and b/tests/load/1/math/libfoo-examples-1.2.4.tar.gz differ diff --git a/tests/load/1/math/libfoo-tests-1.2.4.tar.gz b/tests/load/1/math/libfoo-tests-1.2.4.tar.gz index 8e5105f..223e24d 100644 Binary files a/tests/load/1/math/libfoo-tests-1.2.4.tar.gz and b/tests/load/1/math/libfoo-tests-1.2.4.tar.gz differ diff --git a/tests/load/1/math/packages.manifest b/tests/load/1/math/packages.manifest index de3b1c5..ea37e2a 100644 --- a/tests/load/1/math/packages.manifest +++ b/tests/load/1/math/packages.manifest @@ -89,25 +89,25 @@ name: libfoo-benchmarks version: 1.2.4 summary: The Foo Math Library benchmarks license: MIT -builds: none; Is only build to benchmark libfoo. +builds: 64; Fails building for 32 bits. location: libfoo-benchmarks-1.2.4.tar.gz -sha256sum: 2ec3985a540ca5bf74786d0792820cfa8a2790964a5aeaba443dfa91f2a54c04 +sha256sum: ba664343db5b9bd574450175834b0dd39d038dcff7387477b6eff0d5783a8ac4 : name: libfoo-examples version: 1.2.4 summary: The Foo Math Library examples license: MIT -builds: none; Is only built to demo libfoo usage. +builds: 64; Fails building for 32 bits. location: libfoo-examples-1.2.4.tar.gz -sha256sum: 99658b9a5a5b834047b692b93ded9f9af3d255eb5ea3b27594f600b902039995 +sha256sum: 1343d1826c3ae5446ad965bc9aa7b1586e4238c7736c344e63a4a6bae3d57a88 : name: libfoo-tests version: 1.2.4 summary: The Foo Math Library tests license: MIT -builds: none; Is only built to test libfoo. +builds: 64; Fails building for 32 bits. location: libfoo-tests-1.2.4.tar.gz -sha256sum: 16712c90df5ba2ffb920d29c9c25a29564f8ae01f167359c4651572789e6cd6c +sha256sum: c5c0520b4e612fa2f8948c42824f3e199926c2395bf2c2f898e83f9eb19261a4 : name: libpq version: 0 -- cgit v1.1