From c28758a9327efc0f3bfc2f3c14d12b533750d94b Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Mon, 8 Jul 2019 21:27:53 +0300 Subject: Add support for marking internal repository as non-buildable --- libbrep/build-extra.sql | 3 +- libbrep/build-package.hxx | 15 ++++------ libbrep/build.hxx | 41 ++++++++++++++------------- libbrep/build.xml | 2 ++ libbrep/common.hxx | 2 +- libbrep/package.cxx | 17 +++++++----- libbrep/package.hxx | 31 +++++++++++++++++++-- libbrep/package.xml | 9 ++++++ load/load.cxx | 55 +++++++++++++++++++++++++------------ migrate/migrate.cxx | 15 ++++++++-- mod/mod-package-version-details.cxx | 5 +--- tests/load/driver.cxx | 46 +++++++++++++++++++++++++++++-- tests/load/loadtab | 2 +- 13 files changed, 176 insertions(+), 67 deletions(-) diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index 1ada713..9ecbcb1 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -48,7 +48,8 @@ CREATE FOREIGN TABLE build_package ( version_upstream TEXT NOT NULL, version_release TEXT NULL, internal_repository_tenant TEXT NULL, - internal_repository_canonical_name TEXT NULL) + internal_repository_canonical_name TEXT NULL, + buildable BOOLEAN NOT NULL) SERVER package_server OPTIONS (table_name 'package'); -- The foreign table for the build_package object builds member (that is of a diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 3f15012..c491b9e 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -78,6 +78,7 @@ namespace brep package_id id; upstream_version version; lazy_shared_ptr internal_repository; + bool buildable; // Mapped to the package object builds member using the PostgreSQL foreign // table mechanism. @@ -101,7 +102,7 @@ namespace brep build_package () = default; }; - // Packages that can potentially be built (internal non-stub). + // Packages that can potentially be built. // // Note that ADL can't find the equal operator, so we use the function call // notation. @@ -109,11 +110,9 @@ namespace brep #pragma db view \ object(build_package) \ object(build_repository inner: \ + build_package::buildable && \ brep::operator== (build_package::internal_repository, \ - build_repository::id) && \ - brep::compare_version_ne (build_package::id.version, \ - brep::wildcard_version, \ - false)) \ + build_repository::id)) \ object(build_tenant: build_package::id.tenant == build_tenant::id) struct buildable_package { @@ -128,11 +127,9 @@ namespace brep #pragma db view \ object(build_package) \ object(build_repository inner: \ + build_package::buildable && \ brep::operator== (build_package::internal_repository, \ - build_repository::id) && \ - brep::compare_version_ne (build_package::id.version, \ - brep::wildcard_version, \ - false)) \ + build_repository::id)) \ object(build_tenant: build_package::id.tenant == build_tenant::id) struct buildable_package_count { diff --git a/libbrep/build.hxx b/libbrep/build.hxx index 9fbf983..ea51c00 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -26,7 +26,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 7 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 7, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 8, closed) // We have to keep these mappings at the global scope instead of inside // the brep namespace because they need to be also effective in the @@ -262,10 +262,16 @@ namespace brep } }; - #pragma db view object(build) \ - object(build_package inner: \ - brep::operator== (build::id.package, build_package::id) && \ - build_package::internal_repository.canonical_name.is_not_null ()) \ + // Note that ADL can't find the equal operator in join conditions, so we use + // the function call notation for them. + // + + // Toolchains of existing buildable package builds. + // + #pragma db view object(build) \ + object(build_package inner: \ + brep::operator== (build::id.package, build_package::id) && \ + build_package::buildable) \ query(distinct) struct toolchain { @@ -303,27 +309,24 @@ namespace brep canonical_version version_; }; - // Build of an existing internal package. - // - // Note that ADL can't find the equal operator, so we use the function call - // notation. + // Build of an existing buildable package. // - #pragma db view \ - object(build) \ - object(build_package inner: \ - brep::operator== (build::id.package, build_package::id) && \ - build_package::internal_repository.canonical_name.is_not_null ()) \ + #pragma db view \ + object(build) \ + object(build_package inner: \ + brep::operator== (build::id.package, build_package::id) && \ + build_package::buildable) \ object(build_tenant: build_package::id.tenant == build_tenant::id) struct package_build { shared_ptr build; }; - #pragma db view \ - object(build) \ - object(build_package inner: \ - brep::operator== (build::id.package, build_package::id) && \ - build_package::internal_repository.canonical_name.is_not_null ()) \ + #pragma db view \ + object(build) \ + object(build_package inner: \ + brep::operator== (build::id.package, build_package::id) && \ + build_package::buildable) \ object(build_tenant: build_package::id.tenant == build_tenant::id) struct package_build_count { diff --git a/libbrep/build.xml b/libbrep/build.xml index 2f8a8a9..032fd38 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,6 @@ + + diff --git a/libbrep/common.hxx b/libbrep/common.hxx index db7e045..052aae1 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -151,7 +151,7 @@ namespace brep // // The default collation for UTF8-encoded TEXT columns in PostgreSQL is // UCA-compliant. This makes the statement 'a' < '~' to be false, which - // in turn makes the statement 2.1.alpha < 2.1 to be false as well. + // in turn makes the statement 2.1-alpha < 2.1 to be false as well. // // Unicode Collation Algorithm (UCA): http://unicode.org/reports/tr10/ // diff --git a/libbrep/package.cxx b/libbrep/package.cxx index 6c6a565..b17dcc1 100644 --- a/libbrep/package.cxx +++ b/libbrep/package.cxx @@ -105,13 +105,12 @@ namespace brep dependencies (move (dp)), requirements (move (rq)), builds (move (bs)), - build_constraints (version.compare (wildcard_version, true) != 0 - ? move (bc) - : build_constraints_type ()), + build_constraints (!stub () ? move (bc) : build_constraints_type ()), internal_repository (move (rp)), location (move (lc)), fragment (move (fr)), - sha256sum (move (sh)) + sha256sum (move (sh)), + buildable (!stub () && internal_repository->buildable) { assert (internal_repository->internal); } @@ -123,7 +122,8 @@ namespace brep : id (rp->tenant, move (nm), vr), tenant (id.tenant), name (id.name), - version (move (vr)) + version (move (vr)), + buildable (false) { assert (!rp->internal); other_repositories.emplace_back (move (rp)); @@ -192,6 +192,7 @@ namespace brep string d, repository_location h, optional c, + bool b, uint16_t r) : id (move (t), l.canonical_name ()), tenant (id.tenant), @@ -201,7 +202,8 @@ namespace brep priority (r), cache_location (move (h)), certificate (move (c)), - internal (true) + internal (true), + buildable (b) { } @@ -212,7 +214,8 @@ namespace brep canonical_name (id.canonical_name), location (move (l)), priority (0), - internal (false) + internal (false), + buildable (false) { } } diff --git a/libbrep/package.hxx b/libbrep/package.hxx index 86a1922..5e1bebd 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -21,7 +21,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 14 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 14, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 15, closed) namespace brep { @@ -245,6 +245,7 @@ namespace brep string display_name, repository_location cache_location, optional, + bool buildable, uint16_t priority); // Create external repository. @@ -259,8 +260,8 @@ namespace brep repository_location location; // Note: foreign-mapped in build. string display_name; - // The order in the internal repositories configuration file, starting from - // 1. 0 for external repositories. + // The order in the internal repositories configuration file, starting + // from 1. 0 for external repositories. // uint16_t priority; @@ -291,6 +292,12 @@ namespace brep timestamp repositories_timestamp; bool internal; + + // Whether repository packages are buildable by the build bot controller + // service. Can only be true for internal repositories. + // + bool buildable; + vector> complements; vector> prerequisites; @@ -401,6 +408,13 @@ namespace brep bool internal () const noexcept {return internal_repository != nullptr;} + bool + stub () const noexcept + { + return version.compare (wildcard_version, + true /* ignore_revision */) == 0; + } + // Manifest data. // package_id id; @@ -459,6 +473,17 @@ namespace brep vector> other_repositories; + // Whether the package is buildable by the build bot controller service. + // Can only be true for non-stubs that belong to at least one buildable + // (internal) repository. + // + // While we could potentially calculate this flag on the fly, that would + // complicate the database queries significantly. + // + // Note: foreign-mapped in build. + // + bool buildable; + // Database mapping. // #pragma db member(id) id column("") diff --git a/libbrep/package.xml b/libbrep/package.xml index ad0a888..4af583e 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,13 @@ + + + + + + + + +
diff --git a/load/load.cxx b/load/load.cxx index 9c1c31f..57acb3a 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -60,6 +60,7 @@ struct internal_repository string display_name; repository_location cache_location; optional fingerprint; + bool buildable = true; path packages_path () const {return cache_location.path () / packages;} @@ -74,7 +75,7 @@ using internal_repositories = vector; // // loadtab consists of lines in the following format: // -// cache: [fingerprint:] +// cache: [fingerprint:] [buildable:(yes|no)] // // Note that if the remote repository location is a pkg repository, then the // repository cache should be its local copy. Otherwise, the cache directory @@ -162,7 +163,7 @@ load_repositories (path p) r.display_name = move (tl[i++].value); // Parse options, that have : form. Currently defined - // options are cache (mandatory for now) and fingerprint. + // options are cache (mandatory for now), fingerprint, and buildable. // for (; i < n; ++i) { @@ -238,6 +239,15 @@ load_repositories (path p) } } } + else if (strncmp (nv.c_str (), "buildable:", vp = 10) == 0) + { + string v (nv, vp); + + r.buildable = (v == "yes"); + + if (!r.buildable && v != "no") + bad_line ("invalid buildable option value"); + } else bad_line ("invalid option '" + nv + "'"); } @@ -286,6 +296,7 @@ changed (const string& tenant, r.location.string () != pr->location.string () || r.display_name != pr->display_name || r.cache_location.path () != pr->cache_location.path () || + r.buildable != pr->buildable || file_mtime (r.packages_path ()) != pr->packages_timestamp || file_mtime (r.repositories_path ()) != pr->repositories_timestamp || !pr->internal) @@ -544,23 +555,32 @@ load_packages (const shared_ptr& rp, // assert (!rp->internal || p->internal ()); - // Note that the sha256sum manifest value can only be present if the - // package comes from the pkg repository. - // - if (rp->internal && pm.sha256sum) + if (rp->internal) { - // Save the package sha256sum if it is not present yet, match - // otherwise. + // Note that the sha256sum manifest value can only be present if the + // package comes from the pkg repository. + // + if (pm.sha256sum) + { + // Save the package sha256sum if it is not present yet, match + // otherwise. + // + if (!p->sha256sum) + p->sha256sum = move (pm.sha256sum); + else if (*pm.sha256sum != *p->sha256sum) + cerr << "warning: sha256sum mismatch for package " << p->name + << " " << p->version << endl + << " info: " << p->internal_repository.load ()->location + << " has " << *p->sha256sum << endl + << " info: " << rp->location << " has " << *pm.sha256sum + << endl; + } + + // A non-stub package is buildable if belongs to at least one + // buildable repository (see libbrep/package.hxx for details). // - if (!p->sha256sum) - p->sha256sum = move (pm.sha256sum); - else if (*pm.sha256sum != *p->sha256sum) - cerr << "warning: sha256sum mismatch for package " << p->name - << " " << p->version << endl - << " info: " << p->internal_repository.load ()->location - << " has " << *p->sha256sum << endl - << " info: " << rp->location << " has " << *pm.sha256sum - << endl; + if (!p->stub () && !p->buildable) + p->buildable = rp->buildable; } p->other_repositories.push_back (rp); @@ -1267,6 +1287,7 @@ try move (ir.display_name), move (ir.cache_location), move (cert), + ir.buildable, priority++)); load_packages (r, db, ops.ignore_unknown (), overrides); diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx index 35beb20..f012da1 100644 --- a/migrate/migrate.cxx +++ b/migrate/migrate.cxx @@ -207,7 +207,6 @@ create (database& db, bool extra_only) const // Register the data migration functions for the package database schema. // -#if 0 template using package_migration_entry_base = data_migration_entry; @@ -219,11 +218,21 @@ struct package_migration_entry: package_migration_entry_base : package_migration_entry_base (f, "package") {} }; -static const package_migration_entry<12> +static const package_migration_entry<15> package_migrate_v15 ([] (database& db) { + // Set the buildable flag for the internal repositories. + // + db.execute ("UPDATE repository SET buildable = internal"); + + // Set the buildable flag for the internal non-stub packages. + // + db.execute ("UPDATE package SET buildable = " + "internal_repository_tenant IS NOT NULL AND " + "NOT (version_epoch = 0 AND " + "version_canonical_upstream = '' AND " + "version_canonical_release = '~')"); }); -#endif // main() function // diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 1c7fac4..0271e03 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -350,10 +350,7 @@ handle (request& rq, response& rs) << ~TABLE; } - // Don't display the page builds section for stub packages. - // - bool builds (build_db_ != nullptr && - ver.compare (wildcard_version, true) != 0); + bool builds (build_db_ != nullptr && pkg->buildable); if (builds) package_db_->load (*pkg, pkg->build_section); diff --git a/tests/load/driver.cxx b/tests/load/driver.cxx index db4ed8c..2e4dc54 100644 --- a/tests/load/driver.cxx +++ b/tests/load/driver.cxx @@ -243,6 +243,7 @@ test_git_repos (const cstrings& loader_args, assert (r->location.string () == "https://git.example.com/foo.git#master"); assert (r->summary && *r->summary == "foo project repository"); + assert (r->buildable); // Verify libfoo package version. // @@ -263,6 +264,8 @@ test_git_repos (const cstrings& loader_args, dependency_constraint ( version ("1.0"), false, version ("1.0"), false))); + assert (p->buildable); + t.commit (); } } @@ -339,6 +342,8 @@ test_pkg_repos (const cstrings& loader_args, dir_path srp (loadtab.directory () / dir_path ("1/stable")); assert (sr->cache_location.path () == srp.normalize ()); + assert (!sr->buildable); + assert (sr->packages_timestamp == srt); assert (sr->repositories_timestamp == file_mtime (sr->cache_location.path () / repositories)); @@ -384,6 +389,8 @@ test_pkg_repos (const cstrings& loader_args, assert (fpvxy->sha256sum && *fpvxy->sha256sum == "c994fd49f051ab7fb25f3a4e68ca878e484c5d3c2cb132b37d41224b0621b618"); + assert (fpvxy->buildable); + // libfoo-1.0 // shared_ptr fpv1 ( @@ -418,6 +425,8 @@ test_pkg_repos (const cstrings& loader_args, assert (fpv1->sha256sum && *fpv1->sha256sum == "e89c6d746f8b1ea3ec58d294946d2f683d133438d2ac8c88549ba24c19627e76"); + assert (fpv1->buildable); + // libfoo-1.2.2 // shared_ptr fpv2 ( @@ -460,6 +469,8 @@ test_pkg_repos (const cstrings& loader_args, assert (fpv2->sha256sum && *fpv2->sha256sum == "088068ea3d69542a153f829cf836013374763148fba0a43d8047974f58b5efd7"); + assert (!fpv2->buildable); + // libfoo-1.2.2-alpha.1 // shared_ptr fpv2a ( @@ -522,6 +533,8 @@ test_pkg_repos (const cstrings& loader_args, assert (fpv2a->sha256sum && *fpv2a->sha256sum == "f5d3e9e6e8f9621a638b1375d31f0eb50e6279d8066170b25da21e84198cfd82"); + assert (!fpv2a->buildable); + // libfoo-1.2.3-4 // shared_ptr fpv3 ( @@ -558,6 +571,8 @@ test_pkg_repos (const cstrings& loader_args, assert (fpv3->sha256sum && *fpv3->sha256sum == "f2ebecac6cac8addd7c623bc1becf055e76b13a0d2dd385832b92c38c58956d8"); + assert (!fpv3->buildable); + // libfoo-1.2.4 // shared_ptr fpv4 ( @@ -595,6 +610,8 @@ test_pkg_repos (const cstrings& loader_args, assert (fpv4->sha256sum && *fpv4->sha256sum == "aa1606323bfc59b70de642629dc5d8318cc5348e3646f90ed89406d975db1e1d"); + assert (!fpv4->buildable); + // Verify 'math' repository. // assert (mr->location.canonical_name () == "pkg:dev.cppget.org/math"); @@ -612,6 +629,8 @@ test_pkg_repos (const cstrings& loader_args, dir_path mrp (loadtab.directory () / dir_path ("1/math")); assert (mr->cache_location.path () == mrp.normalize ()); + assert (mr->buildable); + assert (mr->packages_timestamp == file_mtime (mr->cache_location.path () / packages)); @@ -680,9 +699,11 @@ test_pkg_repos (const cstrings& loader_args, assert (xpv->sha256sum && *xpv->sha256sum == "1833906dd93ccc0cda832d6a1b3ef9ed7877bb9958b46d9b2666033d4a7919c9"); + assert (xpv->buildable); + // Verify libfoo package versions. // - // libfoo-1.2.4-1 + // libfoo-1.2.4+1 // shared_ptr fpv5 ( db.load ( @@ -797,6 +818,8 @@ test_pkg_repos (const cstrings& loader_args, assert (fpv5->sha256sum && *fpv5->sha256sum == "0a206d2b5e575549914ed43b87470b33512e975fffa4fc8f3eb92b3dea66979e"); + assert (fpv5->buildable); + // Verify libexp package version. // // libexp-+2-1.2 @@ -838,6 +861,8 @@ test_pkg_repos (const cstrings& loader_args, assert (epv->requirements.empty ()); + assert (epv->buildable); + db.load (*epv, epv->build_section); assert ( @@ -862,6 +887,7 @@ test_pkg_repos (const cstrings& loader_args, version ("0")))); assert (qpv->summary == "PostgreSQL C API client library"); + assert (!qpv->buildable); // Verify 'misc' repository. // @@ -879,6 +905,8 @@ test_pkg_repos (const cstrings& loader_args, dir_path crp (loadtab.directory () / dir_path ("1/misc")); assert (cr->cache_location.path () == crp.normalize ()); + assert (!cr->buildable); + assert (cr->packages_timestamp == file_mtime (cr->cache_location.path () / packages)); @@ -903,6 +931,8 @@ test_pkg_repos (const cstrings& loader_args, assert (bpv->other_repositories[0].load () == cr); assert (check_location (bpv)); + assert (!bpv->buildable); + // Verify libfoo package versions. // // libfoo-0.1 @@ -916,7 +946,9 @@ test_pkg_repos (const cstrings& loader_args, assert (fpv0->other_repositories[0].load () == cr); assert (check_location (fpv0)); - // libfoo-1.2.4-2 + assert (!fpv0->buildable); + + // libfoo-1.2.4+2 // shared_ptr fpv6 ( db.load ( @@ -927,6 +959,8 @@ test_pkg_repos (const cstrings& loader_args, assert (fpv6->other_repositories[0].load () == cr); assert (check_location (fpv6)); + assert (!fpv6->buildable); + // Verify 'testing' repository. // assert (tr->location.canonical_name () == "pkg:dev.cppget.org/testing"); @@ -943,6 +977,8 @@ test_pkg_repos (const cstrings& loader_args, dir_path trp (loadtab.directory () / dir_path ("1/testing")); assert (tr->cache_location.path () == trp.normalize ()); + assert (!tr->buildable); + assert (tr->packages_timestamp == file_mtime (tr->cache_location.path () / packages)); @@ -966,6 +1002,7 @@ test_pkg_repos (const cstrings& loader_args, assert (mpv0->other_repositories.size () == 1); assert (mpv0->other_repositories[0].load () == tr); assert (check_location (mpv0)); + assert (!mpv0->buildable); // libmisc-2.3.0+1 // @@ -977,6 +1014,7 @@ test_pkg_repos (const cstrings& loader_args, assert (mpv1->other_repositories.size () == 1); assert (mpv1->other_repositories[0].load () == tr); assert (check_location (mpv1)); + assert (!mpv1->buildable); // Verify 'staging' repository. // @@ -993,6 +1031,7 @@ test_pkg_repos (const cstrings& loader_args, dir_path grp (loadtab.directory () / dir_path ("1/staging")); assert (gr->cache_location.path () == grp.normalize ()); + assert (!gr->buildable); assert (gr->packages_timestamp == file_mtime (gr->cache_location.path () / packages)); @@ -1016,6 +1055,7 @@ test_pkg_repos (const cstrings& loader_args, assert (tpv->other_repositories.size () == 1); assert (tpv->other_repositories[0].load () == gr); assert (check_location (tpv)); + assert (!tpv->buildable); // Verify libgenx package version. // @@ -1029,6 +1069,7 @@ test_pkg_repos (const cstrings& loader_args, assert (gpv->other_repositories.size () == 1); assert (gpv->other_repositories[0].load () == gr); assert (check_location (gpv)); + assert (!gpv->buildable); // Verify libmisc package version. // @@ -1042,6 +1083,7 @@ test_pkg_repos (const cstrings& loader_args, assert (mpv2->other_repositories.size () == 1); assert (mpv2->other_repositories[0].load () == gr); assert (check_location (mpv2)); + assert (!mpv2->buildable); // Change package summary, update the object persistent state, rerun // the loader and make sure the model were not rebuilt. diff --git a/tests/load/loadtab b/tests/load/loadtab index 87b6f67..96e1f00 100644 --- a/tests/load/loadtab +++ b/tests/load/loadtab @@ -1,4 +1,4 @@ -http://dev.cppget.org/1/stable stable cache:1/stable +http://dev.cppget.org/1/stable stable cache:1/stable buildable:no http://dev.cppget.org/1/math math cache:1/math http://dev.cppget.org/1/signed signed cache:pkg/1/dev.cppget.org/signed fingerprint:C3:EC:12:53:AD:64:41:0E:35:3A:9A:A6:EE:57:BF:E6:05:40:42:2B:FF:AF:2C:B0:99:AD:E9:4A:9C:48:40:22 http://dev.cppget.org/1/unsigned unsigned cache:pkg/1/dev.cppget.org/unsigned fingerprint: -- cgit v1.1