diff options
| -rw-r--r-- | clean/clean.cxx | 7 | ||||
| -rw-r--r-- | libbrep/build-extra.sql | 45 | ||||
| -rw-r--r-- | libbrep/build-package.hxx | 111 | ||||
| -rw-r--r-- | libbrep/build.hxx | 2 | ||||
| -rw-r--r-- | libbrep/build.xml | 2 | ||||
| -rw-r--r-- | libbrep/common.cxx | 10 | ||||
| -rw-r--r-- | libbrep/common.hxx | 191 | ||||
| -rwxr-xr-x | libbrep/odb.sh | 4 | ||||
| -rw-r--r-- | libbrep/package.cxx | 31 | ||||
| -rw-r--r-- | libbrep/package.hxx | 127 | ||||
| -rw-r--r-- | libbrep/package.xml | 135 | ||||
| -rw-r--r-- | load/load.cli | 16 | ||||
| -rw-r--r-- | load/load.cxx | 174 | ||||
| -rw-r--r-- | mod/build-config-module.hxx | 3 | ||||
| -rw-r--r-- | mod/build-result-module.cxx | 145 | ||||
| -rw-r--r-- | mod/build-target-config.cxx | 8 | ||||
| -rw-r--r-- | mod/build-target-config.hxx | 23 | ||||
| -rw-r--r-- | mod/database-module.cxx | 10 | ||||
| -rw-r--r-- | mod/database-module.hxx | 5 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 336 | ||||
| -rw-r--r-- | mod/mod-package-details.cxx | 2 | ||||
| -rw-r--r-- | mod/mod-package-version-details.cxx | 8 | ||||
| -rw-r--r-- | mod/mod-packages.cxx | 2 | ||||
| -rw-r--r-- | monitor/monitor.cxx | 93 |
24 files changed, 1190 insertions, 300 deletions
diff --git a/clean/clean.cxx b/clean/clean.cxx index 55fb59b..828ae4b 100644 --- a/clean/clean.cxx +++ b/clean/clean.cxx @@ -480,8 +480,8 @@ namespace brep auto tenant_ids (pq.execute ()); if ((ne = !tenant_ids.empty ())) { - // Cache tenant ids and erase packages, repositories, and tenants at - // once. + // Cache tenant ids and erase packages, repositories, public keys, and + // tenants at once. // strings tids; tids.reserve (tenant_ids.size ()); @@ -497,6 +497,9 @@ namespace brep db.erase_query<repository> ( query<repository>::id.tenant.in_range (tids.begin (), tids.end ())); + db.erase_query<public_key> ( + query<public_key>::id.tenant.in_range (tids.begin (), tids.end ())); + db.erase_query<tenant> ( query<tenant>::id.in_range (tids.begin (), tids.end ())); } diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index a931f31..9e51a51 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -6,6 +6,8 @@ -- package-extra.sql file for details. -- +DROP FOREIGN TABLE IF EXISTS build_package_config_bot_keys; + DROP FOREIGN TABLE IF EXISTS build_package_config_auxiliaries; DROP FOREIGN TABLE IF EXISTS build_package_config_constraints; @@ -14,6 +16,8 @@ DROP FOREIGN TABLE IF EXISTS build_package_config_builds; DROP FOREIGN TABLE IF EXISTS build_package_configs; +DROP FOREIGN TABLE IF EXISTS build_package_bot_keys; + DROP FOREIGN TABLE IF EXISTS build_package_auxiliaries; DROP FOREIGN TABLE IF EXISTS build_package_constraints; @@ -30,6 +34,8 @@ DROP FOREIGN TABLE IF EXISTS build_package_requirements; DROP FOREIGN TABLE IF EXISTS build_package; +DROP FOREIGN TABLE IF EXISTS build_public_key; + DROP FOREIGN TABLE IF EXISTS build_repository; DROP FOREIGN TABLE IF EXISTS build_tenant; @@ -64,6 +70,14 @@ CREATE FOREIGN TABLE build_repository ( certificate_fingerprint TEXT NULL) SERVER package_server OPTIONS (table_name 'repository'); +-- The foreign table for build_public_key object. +-- +CREATE FOREIGN TABLE build_public_key ( + tenant TEXT NOT NULL, + fingerprint TEXT NOT NULL, + "data" TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'public_key'); + -- The foreign table for build_package object. -- CREATE FOREIGN TABLE build_package ( @@ -84,7 +98,8 @@ CREATE FOREIGN TABLE build_package ( build_error_email_comment TEXT NULL, internal_repository_tenant TEXT NULL, internal_repository_canonical_name TEXT NULL, - buildable BOOLEAN NOT NULL) + buildable BOOLEAN NOT NULL, + custom_bot BOOLEAN NULL) SERVER package_server OPTIONS (table_name 'package'); -- The foreign tables for the build_package object requirements member (that @@ -214,6 +229,21 @@ CREATE FOREIGN TABLE build_package_auxiliaries ( comment TEXT NOT NULL) SERVER package_server OPTIONS (table_name 'package_build_auxiliaries'); +-- The foreign table for the build_package object bot_keys member (that is +-- of a container type). +-- +CREATE FOREIGN TABLE build_package_bot_keys ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + index BIGINT NOT NULL, + key_tenant TEXT NOT NULL, + key_fingerprint TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_bot_keys'); + -- The foreign tables for the build_package object configs member (that is a -- container of values containing containers. -- @@ -277,3 +307,16 @@ CREATE FOREIGN TABLE build_package_config_auxiliaries ( config TEXT NOT NULL, comment TEXT NOT NULL) SERVER package_server OPTIONS (table_name 'package_build_config_auxiliaries'); + +CREATE FOREIGN TABLE build_package_config_bot_keys ( + tenant TEXT NOT NULL, + name CITEXT NOT NULL, + version_epoch INTEGER NOT NULL, + version_canonical_upstream TEXT NOT NULL, + version_canonical_release TEXT NOT NULL COLLATE "C", + version_revision INTEGER NOT NULL, + config_index BIGINT NOT NULL, + index BIGINT NOT NULL, + key_tenant TEXT NOT NULL, + key_fingerprint TEXT NOT NULL) +SERVER package_server OPTIONS (table_name 'package_build_config_bot_keys'); diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 08fb781..9a9c277 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -23,9 +23,11 @@ namespace brep // // The mapping is established in build-extra.sql. We also explicitly mark // non-primary key foreign-mapped members in the source object. - // + // Foreign object that is mapped to a subset of the tenant object. // + // Note: table created manually thus assign table name explicitly. + // #pragma db object table("build_tenant") pointer(shared_ptr) class build_tenant { @@ -50,6 +52,8 @@ namespace brep // Foreign object that is mapped to a subset of the repository object. // + // Note: table created manually thus assign table name explicitly. + // #pragma db object table("build_repository") pointer(shared_ptr) readonly class build_repository { @@ -75,6 +79,54 @@ namespace brep build_repository (): canonical_name (id.canonical_name) {} }; + // Foreign object that is mapped to a subset of the public key object. + // + // Note: table created manually thus assign table name explicitly. + // + #pragma db object table("build_public_key") pointer(shared_ptr) readonly + class build_public_key: public string + { + public: + public_key_id id; + + // Database mapping. + // + #pragma db member(id) id column("") + + #pragma db member(data) virtual(string) access(this) + + private: + friend class odb::access; + build_public_key () = default; + }; + + // build_package_config + // + using build_package_config = + build_package_config_template<lazy_shared_ptr<build_public_key>>; + + using build_package_configs = + build_package_configs_template<lazy_shared_ptr<build_public_key>>; + + #pragma db value(build_package_config) definition + + #pragma db member(build_package_config::builds) transient + #pragma db member(build_package_config::constraints) transient + #pragma db member(build_package_config::auxiliaries) transient + #pragma db member(build_package_config::bot_keys) transient + + // build_package_bot_keys + // + using build_package_bot_keys = vector<lazy_shared_ptr<build_public_key>>; + using build_package_bot_key_key = odb::nested_key<build_package_bot_keys>; + + using build_package_bot_keys_map = + std::map<build_package_bot_key_key, lazy_shared_ptr<build_public_key>>; + + #pragma db value(build_package_bot_key_key) + #pragma db member(build_package_bot_key_key::outer) column("config_index") + #pragma db member(build_package_bot_key_key::inner) column("index") + // Forward declarations. // class build_package; @@ -107,6 +159,8 @@ namespace brep // Foreign object that is mapped to a subset of the package object. // + // Note: table created manually thus assign table name explicitly. + // #pragma db object table("build_package") pointer(shared_ptr) readonly session class build_package { @@ -132,21 +186,26 @@ namespace brep lazy_shared_ptr<build_repository> internal_repository; bool buildable; + optional<bool> custom_bot; // Mapped to the package object builds, build_constraints, - // build_auxiliaries, and build_configs members using the PostgreSQL - // foreign table mechanism. + // build_auxiliaries, bot_keys, and build_configs members using the + // PostgreSQL foreign table mechanism. // - build_class_exprs builds; - build_constraints constraints; - build_auxiliaries auxiliaries; - build_package_configs configs; - - // Group the builds and constraints members of this object as well as of - // the nested configs entries for an explicit load. Note that the configs - // top-level members are loaded implicitly. + build_class_exprs builds; + build_constraints constraints; + build_auxiliaries auxiliaries; + build_package_bot_keys bot_keys; + build_package_configs configs; + + // Group the builds/constraints, auxiliaries, and bot_keys members of this + // object together with their respective nested configs entries into the + // separate sections for an explicit load. Note that the configs top-level + // members are loaded implicitly. // odb::section constraints_section; + odb::section auxiliaries_section; + odb::section bot_keys_section; bool internal () const noexcept {return internal_repository != nullptr;} @@ -194,7 +253,7 @@ namespace brep #pragma db member(requirements_tests_section) load(lazy) update(always) - // builds, constraints, and auxiliaries + // builds, constraints, auxiliaries, and bot_keys // #pragma db member(builds) id_column("") value_column("") \ section(constraints_section) @@ -203,13 +262,16 @@ namespace brep section(constraints_section) #pragma db member(auxiliaries) id_column("") value_column("") \ - section(constraints_section) + section(auxiliaries_section) + + #pragma db member(bot_keys) id_column("") value_column("key_") \ + section(bot_keys_section) // configs // - // Note that build_package_config::{builds,constraints,auxiliaries} are - // persisted/loaded via the separate nested containers (see commons.hxx - // for details). + // Note that build_package_config::{builds,constraints,auxiliaries,bot_keys} + // are persisted/loaded via the separate nested containers (see + // commons.hxx for details). // #pragma db member(configs) id_column("") value_column("config_") @@ -244,9 +306,24 @@ namespace brep odb::nested_set (as, std::move (?)); \ move (as).to_configs (this.configs)) \ id_column("") key_column("") value_column("") \ - section(constraints_section) + section(auxiliaries_section) + + #pragma db member(config_bot_keys) \ + virtual(build_package_bot_keys_map) \ + after(config_auxiliaries) \ + get(odb::nested_get ( \ + brep::build_package_config_bot_keys< \ + lazy_shared_ptr<brep::build_public_key>> (this.configs))) \ + set(brep::build_package_config_bot_keys< \ + lazy_shared_ptr<brep::build_public_key>> bks; \ + odb::nested_set (bks, std::move (?)); \ + move (bks).to_configs (this.configs)) \ + id_column("") key_column("") value_column("key_") \ + section(bot_keys_section) #pragma db member(constraints_section) load(lazy) update(always) + #pragma db member(auxiliaries_section) load(lazy) update(always) + #pragma db member(bot_keys_section) load(lazy) update(always) private: friend class odb::access; diff --git a/libbrep/build.hxx b/libbrep/build.hxx index 4c470cd..af49c03 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -28,7 +28,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 26, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 27, closed) // We have to keep these mappings at the global scope instead of inside the // brep namespace because they need to be also effective in the bbot namespace diff --git a/libbrep/build.xml b/libbrep/build.xml index 815c915..1eba85a 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,6 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1"> + <changeset version="27"/> + <changeset version="26"/> <changeset version="25"> diff --git a/libbrep/common.cxx b/libbrep/common.cxx index c97a346..4f729a3 100644 --- a/libbrep/common.cxx +++ b/libbrep/common.cxx @@ -32,14 +32,4 @@ namespace brep else if (r == "unbuildable") return unbuildable_reason::unbuildable; else throw invalid_argument ("invalid unbuildable reason '" + r + '\''); } - - build_package_config* - find (const string& name, build_package_configs& cs) - { - auto i (find_if (cs.begin (), cs.end (), - [&name] (const build_package_config& c) - {return c.name == name;})); - - return i != cs.end () ? &*i : nullptr; - } } diff --git a/libbrep/common.hxx b/libbrep/common.hxx index ea18fd4..1433c8c 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -326,6 +326,19 @@ namespace brep : tenant (move (t)), canonical_name (move (n)) {} }; + // public_key_id + // + #pragma db value + struct public_key_id + { + string tenant; + string fingerprint; + + public_key_id () = default; + public_key_id (string t, string f) + : tenant (move (t)), fingerprint (move (f)) {} + }; + // build_class_expr // using bpkg::build_class_expr; @@ -370,31 +383,38 @@ namespace brep #pragma db value(email) definition #pragma db member(email::value) virtual(string) before access(this) column("") - // build_package_config + // build_package_config_template // - using build_package_config = bpkg::build_package_config; - - #pragma db value(build_package_config) definition + using bpkg::build_package_config_template; // 1 for the default configuration which is always present. // - using build_package_configs = small_vector<build_package_config, 1>; + template <typename K> + using build_package_configs_template = + small_vector<build_package_config_template<K>, 1>; // Return the address of the configuration object with the specified name, // if present, and NULL otherwise. // - build_package_config* - find (const string& name, build_package_configs&); + template <typename K> + inline build_package_config_template<K>* + find (const string& name, build_package_configs_template<K>& cs) + { + auto i (find_if (cs.begin (), cs.end (), + [&name] (const build_package_config_template<K>& c) + {return c.name == name;})); + + return i != cs.end () ? &*i : nullptr; + } // Note that ODB doesn't support containers of value types which contain // containers. Thus, we will persist/load - // package_build_config::{builds,constraint,auxiliaries} via the separate - // nested containers using the adapter classes. - // - // build_package_config::builds + // build_package_config_template<K>::{builds,constraint,auxiliaries,bot_keys} + // via the separate nested containers using the adapter classes. // - #pragma db member(build_package_config::builds) transient + // build_package_config_template<K>::builds + // using build_class_expr_key = odb::nested_key<build_class_exprs>; using build_class_exprs_map = std::map<build_class_expr_key, build_class_expr>; @@ -402,24 +422,27 @@ namespace brep #pragma db member(build_class_expr_key::outer) column("config_index") #pragma db member(build_class_expr_key::inner) column("index") - // Adapter for build_package_config::builds. + // Adapter for build_package_config_template<K>::builds. + // + // Note: 1 as for build_package_configs_template. // - class build_package_config_builds: - public small_vector<build_class_exprs, 1> // 1 as for build_package_configs. + class build_package_config_builds: public small_vector<build_class_exprs, 1> { public: build_package_config_builds () = default; + template <typename K> explicit - build_package_config_builds (const build_package_configs& cs) + build_package_config_builds (const build_package_configs_template<K>& cs) { reserve (cs.size ()); - for (const build_package_config& c: cs) + for (const build_package_config_template<K>& c: cs) push_back (c.builds); } + template <typename K> void - to_configs (build_package_configs& cs) && + to_configs (build_package_configs_template<K>& cs) && { // Note that the empty trailing entries will be missing (see ODB's // nested-container.hxx for details). @@ -432,10 +455,8 @@ namespace brep } }; - // build_package_config::constraints + // build_package_config_template<K>::constraints // - #pragma db member(build_package_config::constraints) transient - using build_constraint_key = odb::nested_key<build_constraints>; using build_constraints_map = std::map<build_constraint_key, build_constraint>; @@ -443,24 +464,29 @@ namespace brep #pragma db member(build_constraint_key::outer) column("config_index") #pragma db member(build_constraint_key::inner) column("index") - // Adapter for build_package_config::constraints. + // Adapter for build_package_config_template<K>::constraints. + // + // Note: 1 as for build_package_configs_template. // class build_package_config_constraints: - public small_vector<build_constraints, 1> // 1 as for build_package_configs. + public small_vector<build_constraints, 1> { public: build_package_config_constraints () = default; + template <typename K> explicit - build_package_config_constraints (const build_package_configs& cs) + build_package_config_constraints ( + const build_package_configs_template<K>& cs) { reserve (cs.size ()); - for (const build_package_config& c: cs) + for (const build_package_config_template<K>& c: cs) push_back (c.constraints); } + template <typename K> void - to_configs (build_package_configs& cs) && + to_configs (build_package_configs_template<K>& cs) && { // Note that the empty trailing entries will be missing (see ODB's // nested-container.hxx for details). @@ -473,10 +499,8 @@ namespace brep } }; - // build_package_config::auxiliaries + // build_package_config_template<K>::auxiliaries // - #pragma db member(build_package_config::auxiliaries) transient - using build_auxiliary_key = odb::nested_key<build_auxiliaries>; using build_auxiliaries_map = std::map<build_auxiliary_key, build_auxiliary>; @@ -484,24 +508,29 @@ namespace brep #pragma db member(build_auxiliary_key::outer) column("config_index") #pragma db member(build_auxiliary_key::inner) column("index") - // Adapter for build_package_config::auxiliaries. + // Adapter for build_package_config_template<K>::auxiliaries. + // + // Note: 1 as for build_package_configs_template. // class build_package_config_auxiliaries: - public small_vector<build_auxiliaries, 1> // 1 as for build_package_configs. + public small_vector<build_auxiliaries, 1> { public: build_package_config_auxiliaries () = default; + template <typename K> explicit - build_package_config_auxiliaries (const build_package_configs& cs) + build_package_config_auxiliaries ( + const build_package_configs_template<K>& cs) { reserve (cs.size ()); - for (const build_package_config& c: cs) + for (const build_package_config_template<K>& c: cs) push_back (c.auxiliaries); } + template <typename K> void - to_configs (build_package_configs& cs) && + to_configs (build_package_configs_template<K>& cs) && { // Note that the empty trailing entries will be missing (see ODB's // nested-container.hxx for details). @@ -514,6 +543,40 @@ namespace brep } }; + // build_package_config_template<K>::bot_keys + // + // Adapter for build_package_config_template<K>::bot_keys. + // + // Note: 1 as for build_package_configs_template. + // + template <typename K> + class build_package_config_bot_keys: public small_vector<vector<K>, 1> + { + public: + build_package_config_bot_keys () = default; + + explicit + build_package_config_bot_keys (const build_package_configs_template<K>& cs) + { + this->reserve (cs.size ()); + for (const build_package_config_template<K>& c: cs) + this->push_back (c.bot_keys); + } + + void + to_configs (build_package_configs_template<K>& cs) && + { + // Note that the empty trailing entries will be missing (see ODB's + // nested-container.hxx for details). + // + assert (this->size () <= cs.size ()); + + auto i (cs.begin ()); + for (vector<K>& bks: *this) + i++->bot_keys = move (bks); + } + }; + // The primary reason why a package is unbuildable by the build bot // controller service. // @@ -611,13 +674,12 @@ namespace brep // Version comparison operators. // - // They allow comparing objects that have epoch, canonical_upstream, - // canonical_release, and revision data members. The idea is that this - // works for both query members of types version and canonical_version. - // Note, though, that the object revisions should be comparable (both - // optional, numeric, etc), so to compare version to query member or - // canonical_version you may need to explicitly convert the version object - // to canonical_version. + // Compare objects that have epoch, canonical_upstream, canonical_release, + // and revision data members. The idea is that this works for both query + // members of types version and canonical_version. Note, though, that the + // object revisions should be comparable (both optional, numeric, etc), so + // to compare version to query member or canonical_version you may need to + // explicitly convert the version object to canonical_version. // template <typename T1, typename T2> inline auto @@ -769,10 +831,9 @@ namespace brep return compare_version_lt (x.version, y.version, true); } - // They allow comparing objects that have tenant, name, and version data - // members. The idea is that this works for both query members of package id - // types (in particular in join conditions) as well as for values of - // package_id type. + // Compare objects that have tenant, name, and version data members. The + // idea is that this works for both query members of package id types (in + // particular in join conditions) as well as for values of package_id type. // template <typename T1, typename T2> inline auto @@ -852,10 +913,10 @@ namespace brep return x.canonical_name.compare (y.canonical_name) < 0; } - // They allow comparing objects that have tenant and canonical_name data - // members. The idea is that this works for both query members of repository - // id types (in particular in join conditions) as well as for values of - // repository_id type. + // Compare objects that have tenant and canonical_name data members. The + // idea is that this works for both query members of repository id types (in + // particular in join conditions) as well as for values of repository_id + // type. // template <typename T1, typename T2> inline auto @@ -872,6 +933,38 @@ namespace brep { return x.tenant != y.tenant || x.canonical_name != y.canonical_name; } + + // Public key id comparison operators. + // + inline bool + operator< (const public_key_id& x, const public_key_id& y) + { + if (int r = x.tenant.compare (y.tenant)) + return r < 0; + + return x.fingerprint.compare (y.fingerprint) < 0; + } + + // Compare objects that have tenant and fingerprint data members. The idea + // is that this works for both query members of public key id types (in + // particular in join conditions) as well as for values of public_key_id + // type. + // + template <typename T1, typename T2> + inline auto + operator== (const T1& x, const T2& y) + -> decltype (x.tenant == y.tenant && x.fingerprint == y.fingerprint) + { + return x.tenant == y.tenant && x.fingerprint == y.fingerprint; + } + + template <typename T1, typename T2> + inline auto + operator!= (const T1& x, const T2& y) + -> decltype (x.tenant == y.tenant && x.fingerprint == y.fingerprint) + { + return x.tenant != y.tenant || x.fingerprint != y.fingerprint; + } } #endif // LIBBREP_COMMON_HXX diff --git a/libbrep/odb.sh b/libbrep/odb.sh index 9ee11fa..608ca41 100755 --- a/libbrep/odb.sh +++ b/libbrep/odb.sh @@ -53,7 +53,7 @@ $odb "${inc[@]}" -d pgsql --std c++14 --generate-query \ --hxx-prologue '#include <libbrep/common-traits.hxx>' \ -DLIBODB_BUILD2 -DLIBODB_PGSQL_BUILD2 \ --include-with-brackets --include-prefix libbrep \ - --guard-prefix LIBBREP \ + --guard-prefix LIBBREP \ common.hxx $odb "${inc[@]}" -d pgsql --std c++14 --generate-query --generate-schema \ @@ -74,7 +74,7 @@ $odb "${inc[@]}" -d pgsql --std c++14 --generate-query --generate-schema \ --odb-epilogue '#include <libbrep/wrapper-traits.hxx>' \ --generate-prepared -DLIBODB_BUILD2 -DLIBODB_PGSQL_BUILD2 \ --include-with-brackets --include-prefix libbrep \ - --guard-prefix LIBBREP \ + --guard-prefix LIBBREP \ build.hxx $odb "${inc[@]}" -d pgsql --std c++14 --generate-query \ diff --git a/libbrep/package.cxx b/libbrep/package.cxx index 37795f0..4eb6fe8 100644 --- a/libbrep/package.cxx +++ b/libbrep/package.cxx @@ -82,7 +82,8 @@ namespace brep build_class_exprs bs, build_constraints_type bc, build_auxiliaries_type ac, - build_package_configs bcs, + package_build_bot_keys bk, + package_build_configs bcs, optional<path> lc, optional<string> fr, optional<string> sh, @@ -116,6 +117,7 @@ namespace brep builds (move (bs)), build_constraints (move (bc)), build_auxiliaries (move (ac)), + build_bot_keys (move (bk)), build_configs (move (bcs)), internal_repository (move (rp)), location (move (lc)), @@ -134,6 +136,31 @@ namespace brep buildable = !unbuildable_reason; + // If the package is buildable deduce the custom_bot flag. + // + if (buildable) + { + for (const package_build_config& bc: build_configs) + { + bool custom (!bc.effective_bot_keys (build_bot_keys).empty ()); + + if (!custom_bot) + { + custom_bot = custom; + } + // + // If both the custom and default bots are used by the package, then + // reset the custom_bot flag to nullopt and bail out from the build + // package configurations loop. + // + else if (*custom_bot != custom) + { + custom_bot = nullopt; + break; + } + } + } + assert (internal_repository->internal); } @@ -143,7 +170,7 @@ namespace brep build_class_exprs bs, build_constraints_type bc, build_auxiliaries_type ac, - build_package_configs bcs, + package_build_configs bcs, shared_ptr<repository_type> rp) : id (rp->tenant, move (nm), vr), tenant (id.tenant), diff --git a/libbrep/package.hxx b/libbrep/package.hxx index b8c3c33..45008d4 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -20,7 +20,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 32, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 33, closed) namespace brep { @@ -248,8 +248,8 @@ namespace brep string id; - // If true, display the packages in the web interface only in the tenant - // view mode. + // If this flag is true, then display the packages in the web interface + // only in the tenant view mode. // bool private_; // Note: foreign-mapped in build. @@ -456,6 +456,58 @@ namespace brep #pragma db member(text) column("") }; + // Tweak public_key_id mapping to include a constraint (this only affects the + // database schema). + // + #pragma db member(public_key_id::tenant) points_to(tenant) + + #pragma db object pointer(shared_ptr) session + class public_key: public string + { + public: + public_key (string tenant, string fingerprint, string key) + : string (move (key)), id (move (tenant), move (fingerprint)) {} + + public_key_id id; + + // Database mapping. + // + #pragma db member(id) id column("") + + #pragma db member(data) virtual(string) access(this) + + private: + friend class odb::access; + public_key () = default; + }; + + // package_build_config + // + using package_build_config = + build_package_config_template<lazy_shared_ptr<public_key>>; + + using package_build_configs = + build_package_configs_template<lazy_shared_ptr<public_key>>; + + #pragma db value(package_build_config) definition + + #pragma db member(package_build_config::builds) transient + #pragma db member(package_build_config::constraints) transient + #pragma db member(package_build_config::auxiliaries) transient + #pragma db member(package_build_config::bot_keys) transient + + // package_build_bot_keys + // + using package_build_bot_keys = vector<lazy_shared_ptr<public_key>>; + using package_build_bot_key_key = odb::nested_key<package_build_bot_keys>; + + using package_build_bot_keys_map = std::map<package_build_bot_key_key, + lazy_shared_ptr<public_key>>; + + #pragma db value(package_build_bot_key_key) + #pragma db member(package_build_bot_key_key::outer) column("config_index") + #pragma db member(package_build_bot_key_key::inner) column("index") + // Tweak package_id mapping to include a constraint (this only affects the // database schema). // @@ -507,7 +559,8 @@ namespace brep build_class_exprs, build_constraints_type, build_auxiliaries_type, - build_package_configs, + package_build_bot_keys, + package_build_configs, optional<path> location, optional<string> fragment, optional<string> sha256sum, @@ -535,7 +588,7 @@ namespace brep build_class_exprs, build_constraints_type, build_auxiliaries_type, - build_package_configs, + package_build_configs, shared_ptr<repository_type>); bool @@ -589,15 +642,25 @@ namespace brep requirements_type requirements; // Note: foreign-mapped in build. small_vector<test_dependency, 1> tests; // Note: foreign-mapped in build. - // Common build classes, constraints, and auxiliaries that apply to all - // configurations unless overridden. + // Common build classes, constraints, auxiliaries, and bot keys that apply + // to all configurations unless overridden. // build_class_exprs builds; // Note: foreign-mapped in build. build_constraints_type build_constraints; // Note: foreign-mapped in build. build_auxiliaries_type build_auxiliaries; // Note: foreign-mapped in build. + package_build_bot_keys build_bot_keys; // Note: foreign-mapped in build. + package_build_configs build_configs; // Note: foreign-mapped in build. - build_package_configs build_configs; // Note: foreign-mapped in build. - + // Group the build_configs, builds, and build_constraints members of this + // object together with their respective nested configs entries into the + // separate section for an explicit load. + // + // Note that while the build auxiliaries and bot keys are persisted via + // the newly created package objects, they are only used via the + // foreign-mapped build_package objects (see build-package.hxx for + // details). Thus, we add them to the never-loaded unused_section (see + // below). + // odb::section build_section; // Note that it is foreign-mapped in build. @@ -628,6 +691,18 @@ namespace brep bool buildable; // Note: foreign-mapped in build. optional<brep::unbuildable_reason> unbuildable_reason; + // If this flag is true, then all the package configurations are buildable + // with the custom build bots. If false, then all configurations are + // buildable with the default bots. If nullopt, then some configurations + // are buildable with the custom and some with the default build bots. + // + // Note: meaningless if buildable is false. + // + optional<bool> custom_bot; // Note: foreign-mapped in build. + + private: + odb::section unused_section; + // Database mapping. // #pragma db member(id) id column("") @@ -750,13 +825,19 @@ namespace brep // build_auxiliaries // #pragma db member(build_auxiliaries) id_column("") value_column("") \ - section(build_section) + section(unused_section) + + // build_bot_keys + // + #pragma db member(build_bot_keys) \ + id_column("") value_column("key_") value_not_null \ + section(unused_section) // build_configs // - // Note that build_package_config::{builds,constraints,auxiliaries} are - // persisted/loaded via the separate nested containers (see commons.hxx - // for details). + // Note that package_build_config::{builds,constraints,auxiliaries, + // bot_keys} are persisted/loaded via the separate nested containers (see + // commons.hxx for details). // #pragma db member(build_configs) id_column("") value_column("config_") \ section(build_section) @@ -792,9 +873,23 @@ namespace brep odb::nested_set (as, std::move (?)); \ move (as).to_configs (this.build_configs)) \ id_column("") key_column("") value_column("") \ - section(build_section) - - #pragma db member(build_section) load(lazy) update(always) + section(unused_section) + + #pragma db member(build_config_bot_keys) \ + virtual(package_build_bot_keys_map) \ + after(build_config_auxiliaries) \ + get(odb::nested_get ( \ + brep::build_package_config_bot_keys< \ + lazy_shared_ptr<brep::public_key>> (this.build_configs))) \ + set(brep::build_package_config_bot_keys< \ + lazy_shared_ptr<brep::public_key>> bks; \ + odb::nested_set (bks, std::move (?)); \ + move (bks).to_configs (this.build_configs)) \ + id_column("") key_column("") value_column("key_") value_not_null \ + section(unused_section) + + #pragma db member(build_section) load(lazy) update(always) + #pragma db member(unused_section) load(lazy) update(manual) // other_repositories // diff --git a/libbrep/package.xml b/libbrep/package.xml index b66a66d..96e93a7 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,139 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1"> + <changeset version="33"> + <add-table name="public_key" kind="object"> + <column name="tenant" type="TEXT" null="false"/> + <column name="fingerprint" type="TEXT" null="false"/> + <column name="data" type="TEXT" null="false"/> + <primary-key> + <column name="tenant"/> + <column name="fingerprint"/> + </primary-key> + <foreign-key name="tenant_fk" deferrable="DEFERRED"> + <column name="tenant"/> + <references table="tenant"> + <column name="id"/> + </references> + </foreign-key> + </add-table> + <alter-table name="package"> + <add-column name="custom_bot" type="BOOLEAN" null="true"/> + </alter-table> + <add-table name="package_build_bot_keys" kind="container"> + <column name="tenant" type="TEXT" null="false"/> + <column name="name" type="CITEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="key_tenant" type="TEXT" null="false"/> + <column name="key_fingerprint" type="TEXT" null="false"/> + <foreign-key name="tenant_fk" deferrable="DEFERRED"> + <column name="tenant"/> + <references table="tenant"> + <column name="id"/> + </references> + </foreign-key> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_build_bot_keys_object_id_i"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <index name="package_build_bot_keys_index_i"> + <column name="index"/> + </index> + <foreign-key name="key_tenant_fk" deferrable="DEFERRED"> + <column name="key_tenant"/> + <references table="tenant"> + <column name="id"/> + </references> + </foreign-key> + <foreign-key name="key_fk" deferrable="DEFERRED"> + <column name="key_tenant"/> + <column name="key_fingerprint"/> + <references table="public_key"> + <column name="tenant"/> + <column name="fingerprint"/> + </references> + </foreign-key> + </add-table> + <add-table name="package_build_config_bot_keys" kind="container"> + <column name="tenant" type="TEXT" null="false"/> + <column name="name" type="CITEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="config_index" type="BIGINT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="key_tenant" type="TEXT" null="false"/> + <column name="key_fingerprint" type="TEXT" null="false"/> + <foreign-key name="tenant_fk" deferrable="DEFERRED"> + <column name="tenant"/> + <references table="tenant"> + <column name="id"/> + </references> + </foreign-key> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_build_config_bot_keys_object_id_i"> + <column name="tenant"/> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <foreign-key name="key_tenant_fk" deferrable="DEFERRED"> + <column name="key_tenant"/> + <references table="tenant"> + <column name="id"/> + </references> + </foreign-key> + <foreign-key name="key_fk" deferrable="DEFERRED"> + <column name="key_tenant"/> + <column name="key_fingerprint"/> + <references table="public_key"> + <column name="tenant"/> + <column name="fingerprint"/> + </references> + </foreign-key> + </add-table> + </changeset> + <changeset version="32"> <alter-table name="tenant"> <add-column name="build_toolchain_name" type="TEXT" null="true"/> diff --git a/load/load.cli b/load/load.cli index b9aa769..99d76f6 100644 --- a/load/load.cli +++ b/load/load.cli @@ -167,6 +167,22 @@ class options this option to specify multiple package manager options." } + brep::path openssl = "openssl" + { + "<path>", + "The openssl program to be used for crypto operations. You can also + specify additional options that should be passed to the openssl program + with \cb{openssl-option}. If the openssl program is not explicitly + specified, then \cb{brep-load} will use \cb{openssl} by default." + } + + brep::strings openssl-option + { + "<opt>", + "Additional option to be passed to the openssl program (see \cb{openssl} + for details). Repeat this option to specify multiple openssl options." + } + std::string --pager // String to allow empty value. { "<path>", diff --git a/load/load.cxx b/load/load.cxx index 14b8374..474b443 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -20,6 +20,7 @@ #include <libbutl/pager.hxx> #include <libbutl/sha256.hxx> #include <libbutl/process.hxx> +#include <libbutl/openssl.hxx> #include <libbutl/fdstream.hxx> #include <libbutl/filesystem.hxx> #include <libbutl/tab-parser.hxx> @@ -364,7 +365,8 @@ repository_info (const options& lo, const string& rl, const cstrings& options) // the repository. Should be called once per repository. // static void -load_packages (const shared_ptr<repository>& rp, +load_packages (const options& lo, + const shared_ptr<repository>& rp, const repository_location& cl, database& db, bool ignore_unknown, @@ -421,10 +423,12 @@ load_packages (const shared_ptr<repository>& rp, using brep::dependency_alternative; using brep::dependency_alternatives; + const string& tenant (rp->tenant); + for (package_manifest& pm: pms) { shared_ptr<package> p ( - db.find<package> (package_id (rp->tenant, pm.name, pm.version))); + db.find<package> (package_id (tenant, pm.name, pm.version))); // sha256sum should always be present if the package manifest comes from // the packages.manifest file belonging to the pkg repository. @@ -433,21 +437,49 @@ load_packages (const shared_ptr<repository>& rp, if (p == nullptr) { - if (rp->internal) + // Apply the package manifest overrides. + // + if (!overrides.empty ()) + try { - if (!overrides.empty ()) - try - { - pm.override (overrides, overrides_name); - } - catch (const manifest_parsing& e) - { - cerr << "error: unable to override " << p << " manifest: " << e - << endl; + pm.override (overrides, overrides_name); + } + catch (const manifest_parsing& e) + { + cerr << "error: unable to override " << pm.name << ' ' << pm.version + << " manifest: " << e << endl; - throw failed (); + throw failed (); + } + + // Convert the package manifest build configurations (contain public + // keys data) into the brep's build package configurations (contain + // public key object lazy pointers). Keep the bot key lists empty if + // the package is not buildable. + // + package_build_configs build_configs; + + if (!pm.build_configs.empty ()) + { + build_configs.reserve (pm.build_configs.size ()); + + for (bpkg::build_package_config& c: pm.build_configs) + { + build_configs.emplace_back (move (c.name), + move (c.arguments), + move (c.comment), + move (c.builds), + move (c.constraints), + move (c.auxiliaries), + package_build_bot_keys (), + move (c.email), + move (c.warning_email), + move (c.error_email)); } + } + if (rp->internal) + { // Create internal package object. // // Return nullopt if the text is in a file (can happen if the @@ -594,6 +626,107 @@ load_packages (const shared_ptr<repository>& rp, // package_name project (pm.effective_project ()); + // If the package is buildable, then save the package manifest's + // common and build configuration-specific bot keys into the database + // and translate the key data lists into the lists of the public key + // object lazy pointers. + // + package_build_bot_keys bot_keys; + + if (rp->buildable) + { + // Save the specified bot keys into the database as public key + // objects, unless they are already persisted. Translate these keys + // into the public key object lazy pointers. + // + auto keys_to_objects = [&lo, + &pm, + &tenant, + &db] (strings&& keys) + { + package_build_bot_keys r; + + if (keys.empty ()) + return r; + + r.reserve (keys.size ()); + + for (string& key: keys) + { + // Calculate the key fingerprint. + // + string fp; + + try + { + openssl os (path ("-"), path ("-"), 2, + lo.openssl (), + "pkey", + lo.openssl_option (), "-pubin", "-outform", "DER"); + + os.out << key; + os.out.close (); + + fp = sha256 (os.in).string (); + os.in.close (); + + if (!os.wait ()) + { + cerr << "process " << lo.openssl () << ' ' << *os.exit + << endl; + + throw io_error (""); + } + } + catch (const io_error&) + { + cerr << "error: unable to convert custom build bot public key " + << "for package " << pm.name << ' ' << pm.version << endl + << " info: key:" << endl + << key << endl; + + throw failed (); + } + catch (const process_error& e) + { + cerr << "error: unable to convert custom build bot public key " + << "for package " << pm.name << ' ' << pm.version << ": " + << e << endl; + + throw failed (); + } + + // Try to find the public_key object for the calculated + // fingerprint. If it doesn't exist, then create and persist the + // new object. + // + public_key_id id (tenant, move (fp)); + shared_ptr<public_key> k (db.find<public_key> (id)); + + if (k == nullptr) + { + k = make_shared<public_key> (move (id.tenant), + move (id.fingerprint), + move (key)); + + db.persist (k); + } + + r.push_back (move (k)); + } + + return r; + }; + + bot_keys = keys_to_objects (move (pm.build_bot_keys)); + + assert (build_configs.size () == pm.build_configs.size ()); + + for (size_t i (0); i != build_configs.size (); ++i) + build_configs[i].bot_keys = + keys_to_objects (move (pm.build_configs[i].bot_keys)); + } + p = make_shared<package> ( move (pm.name), move (pm.version), @@ -622,7 +755,8 @@ load_packages (const shared_ptr<repository>& rp, move (pm.builds), move (pm.build_constraints), move (pm.build_auxiliaries), - move (pm.build_configs), + move (bot_keys), + move (build_configs), move (pm.location), move (pm.fragment), move (pm.sha256sum), @@ -636,7 +770,7 @@ load_packages (const shared_ptr<repository>& rp, move (pm.builds), move (pm.build_constraints), move (pm.build_auxiliaries), - move (pm.build_configs), + move (build_configs), rp); db.persist (p); @@ -1018,7 +1152,8 @@ load_repositories (const options& lo, // We don't apply overrides to the external packages. // - load_packages (pr, + load_packages (lo, + pr, !pr->cache_location.empty () ? pr->cache_location : cl, db, ignore_unknown, @@ -1630,6 +1765,7 @@ try { db.erase_query<package> (); db.erase_query<repository> (); + db.erase_query<public_key> (); db.erase_query<tenant> (); } else // Multi-tenant mode. @@ -1642,6 +1778,9 @@ try db.erase_query<repository> ( query<repository>::id.tenant.in_range (ts.begin (), ts.end ())); + db.erase_query<public_key> ( + query<public_key>::id.tenant.in_range (ts.begin (), ts.end ())); + db.erase_query<tenant> ( query<tenant>::id.in_range (ts.begin (), ts.end ())); } @@ -1696,7 +1835,8 @@ try ir.buildable, priority++)); - load_packages (r, + load_packages (ops, + r, r->cache_location, db, ops.ignore_unknown (), diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx index 78661c3..c1630b0 100644 --- a/mod/build-config-module.hxx +++ b/mod/build-config-module.hxx @@ -36,8 +36,9 @@ namespace brep void init (const options::build&); + template <typename K> bool - exclude (const build_package_config& pc, + exclude (const build_package_config_template<K>& pc, const build_class_exprs& common_builds, const build_constraints& common_constraints, const build_target_config& tc, diff --git a/mod/build-result-module.cxx b/mod/build-result-module.cxx index 68fbe4c..9ac1390 100644 --- a/mod/build-result-module.cxx +++ b/mod/build-result-module.cxx @@ -3,11 +3,16 @@ #include <mod/build-result-module.hxx> +#include <odb/database.hxx> + #include <libbutl/openssl.hxx> #include <libbutl/fdstream.hxx> #include <libbutl/process-io.hxx> #include <libbutl/semantic-version.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> + namespace brep { using namespace std; @@ -230,54 +235,112 @@ namespace brep else { assert (b.agent_fingerprint && challenge); - auto i (bot_agent_key_map_->find (*b.agent_fingerprint)); - // The agent's key is recently replaced. + auto auth = [&challenge, + &b, + &o, + &fail, &trace, + &warn_auth, + this] (const path& key) + { + bool r (false); + + try + { + openssl os ([&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }, + path ("-"), fdstream_mode::text, 2, + process_env (o.openssl (), o.openssl_envvar ()), + use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", + o.openssl_option (), + use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", + "-pubin", + "-inkey", key); + + for (const auto& c: *challenge) + os.out.put (c); // Sets badbit on failure. + + os.out.close (); + + string s; + getline (os.in, s); + + bool v (os.in.eof ()); + os.in.close (); + + if (os.wait () && v) + { + r = (s == *b.agent_challenge); + + if (!r) + warn_auth ("challenge mismatched"); + } + else // The signature is presumably meaningless. + warn_auth ("unable to verify challenge"); + } + catch (const system_error& e) + { + fail << "unable to verify challenge: " << e; + } + + return r; + }; + + const string& fp (*b.agent_fingerprint); + auto i (bot_agent_key_map_->find (fp)); + + // Note that it is possible that the default vs custom bot + // classification has changed since the task request time. It feels that + // there is nothing wrong with that and we will handle that + // automatically. // - if (i == bot_agent_key_map_->end ()) + if (i != bot_agent_key_map_->end ()) // Default bot? { - warn_auth ("agent's public key not found"); + r = auth (i->second); } - else - try + else // Custom bot. { - openssl os ([&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }, - path ("-"), fdstream_mode::text, 2, - process_env (o.openssl (), o.openssl_envvar ()), - use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", - o.openssl_option (), - use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", - "-pubin", - "-inkey", - i->second); - - for (const auto& c: *challenge) - os.out.put (c); // Sets badbit on failure. - - os.out.close (); - - string s; - getline (os.in, s); - - bool v (os.in.eof ()); - os.in.close (); - - if (os.wait () && v) - { - r = (s == *b.agent_challenge); + shared_ptr<build_public_key> k ( + build_db_->find<build_public_key> (public_key_id (b.tenant, fp))); - if (!r) - warn_auth ("challenge mismatched"); + if (k != nullptr) + { + // Temporarily save the key data to disk (note that it's the + // challenge which is passed via stdin to openssl). Hopefully /tmp + // is using tmpfs. + // + auto_rmfile arm; + + try + { + arm = auto_rmfile (path::temp_path ("brep-custom-bot-key")); + } + catch (const system_error& e) + { + fail << "unable to obtain temporary file: " << e; + } + + try + { + ofdstream os (arm.path); + os << *k; + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to '" << arm.path << "': " << e; + } + + r = auth (arm.path); + } + else + { + // The agent's key is recently replaced. + // + warn_auth ("agent's public key not found"); } - else // The signature is presumably meaningless. - warn_auth ("unable to verify challenge"); - } - catch (const system_error& e) - { - fail << "unable to verify challenge: " << e; } } diff --git a/mod/build-target-config.cxx b/mod/build-target-config.cxx index a30cf07..a30e281 100644 --- a/mod/build-target-config.cxx +++ b/mod/build-target-config.cxx @@ -21,17 +21,13 @@ namespace brep {"all"}, '+', "All."); bool - exclude (const build_package_config& pc, - const build_class_exprs& cbs, - const build_constraints& ccs, + exclude (const build_class_exprs& exprs, + const build_constraints& constrs, const build_target_config& tc, const map<string, string>& class_inheritance_map, string* reason, bool default_all_ucs) { - const build_class_exprs& exprs (pc.effective_builds (cbs)); - const build_constraints& constrs (pc.effective_constraints (ccs)); - // Save the first sentence of the reason, lower-case the first letter if // the beginning looks like a word (all subsequent characters until a // whitespace are lower-case letters). diff --git a/mod/build-target-config.hxx b/mod/build-target-config.hxx index 180ca80..60d159c 100644 --- a/mod/build-target-config.hxx +++ b/mod/build-target-config.hxx @@ -32,14 +32,31 @@ namespace brep // configuration set needlessly). // bool - exclude (const build_package_config&, - const build_class_exprs& common_builds, - const build_constraints& common_constraints, + exclude (const build_class_exprs& builds, + const build_constraints& constraints, const build_target_config&, const std::map<string, string>& class_inheritance_map, string* reason = nullptr, bool default_all_ucs = false); + template <typename K> + inline bool + exclude (const build_package_config_template<K>& pc, + const build_class_exprs& common_builds, + const build_constraints& common_constraints, + const build_target_config& tc, + const std::map<string, string>& class_inheritance_map, + string* reason = nullptr, + bool default_all_ucs = false) + { + return exclude (pc.effective_builds (common_builds), + pc.effective_constraints (common_constraints), + tc, + class_inheritance_map, + reason, + default_all_ucs); + } + // Convert dash-separated components (target, build target configuration // name, machine name) or a pattern thereof into a path, replacing dashes // with slashes (directory separators), `**` with `*/**/*`, and appending diff --git a/mod/database-module.cxx b/mod/database-module.cxx index 07babc6..bbb3e59 100644 --- a/mod/database-module.cxx +++ b/mod/database-module.cxx @@ -76,7 +76,7 @@ namespace brep throw; } - void database_module:: + optional<string> database_module:: update_tenant_service_state ( const connection_ptr& conn, const string& tid, @@ -88,6 +88,8 @@ namespace brep // assert (build_db_ != nullptr); + optional<string> r; + for (size_t retry (retry_);; ) { try @@ -104,6 +106,8 @@ namespace brep { s.data = move (*data); build_db_->update (t); + + r = move (s.data); } } @@ -121,7 +125,11 @@ namespace brep HANDLER_DIAG; l1 ([&]{trace << e << "; " << retry + 1 << " tenant service " << "state update retries left";}); + + r = nullopt; // Prepare for the next iteration. } } + + return r; } } diff --git a/mod/database-module.hxx b/mod/database-module.hxx index 910cb35..298afbf 100644 --- a/mod/database-module.hxx +++ b/mod/database-module.hxx @@ -57,7 +57,8 @@ namespace brep // Update the tenant-associated service state if the specified // notification callback-returned function (expected to be not NULL) - // returns the new state data. + // returns the new state data. Return the service state data, if updated, + // and nullopt otherwise. // // Specifically, start the database transaction, query the service state, // and call the callback-returned function on this state. If this call @@ -65,7 +66,7 @@ namespace brep // state with this data and persist the change. Repeat all the above steps // on the recoverable database failures (deadlocks, etc). // - void + optional<string> update_tenant_service_state ( const odb::core::connection_ptr&, const string& tid, diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 773d041..07aff8d 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -44,6 +44,28 @@ using namespace odb::core; static thread_local mt19937 rand_gen (random_device {} ()); +// The challenge (nonce) is randomly generated for every build task if brep is +// configured to authenticate bbot agents. +// +// Nonce generator must guarantee a probabilistically insignificant chance +// of repeating a previously generated value. The common approach is to use +// counters or random number generators (alone or in combination), that +// produce values of the sufficient length. 64-bit non-repeating and +// 512-bit random numbers are considered to be more than sufficient for +// most practical purposes. +// +// We will produce the challenge as the sha256sum of the 512-bit random +// number and the 64-bit current timestamp combination. The latter is +// not really a non-repeating counter and can't be used alone. However +// adding it is a good and cheap uniqueness improvement. +// +// Note that since generating a challenge is not exactly cheap/fast, we will +// generate it in advance for every task request, out of the database +// transaction, and will cache it if it turns out that it wasn't used (no +// package configuration to (re-)build, etc). +// +static thread_local optional<string> challenge; + // Generate a random number in the specified range (max value is included). // static inline size_t @@ -132,7 +154,8 @@ init (scanner& s) // template <typename T> static inline query<T> -package_query (brep::params::build_task& params, +package_query (bool custom_bot, + brep::params::build_task& params, interactive_mode imode, uint64_t queued_expiration_ns) { @@ -141,9 +164,39 @@ package_query (brep::params::build_task& params, query q (!query::build_tenant::archived); + if (custom_bot) + { + // Note that we could potentially only query the packages which refer to + // this custom bot key in one of their build configurations. For that we + // would need to additionally join the current query tables with the bot + // fingerprint-containing build_package_bot_keys and + // build_package_config_bot_keys tables and use the SELECT DISTINCT + // clause. The problem is that we also use the ORDER BY clause and in this + // case PostgreSQL requires all the ORDER BY clause expressions to also be + // present in the SELECT DISTINCT clause and fails with the 'for SELECT + // DISTINCT, ORDER BY expressions must appear in select list' error if + // that's not the case. Also note that in the ODB-generated code the + // 'build_package.project::TEXT' expression in the SELECT DISTINCT clause + // (see the CITEXT type mapping for details in libbrep/common.hxx) would + // not match the 'build_package.name' expression in the ORDER BY clause + // and so we will end up with the mentioned error. One (hackish) way to + // fix that would be to add a dummy member of the string type for the + // build_package.name column. This all sounds quite hairy at the moment + // and it also feels that this can potentially pessimize querying the + // packages built with the default bots only. Thus let's keep it simple + // for now and filter packages by the bot fingerprint at the program + // level. + // + q = q && (query::build_package::custom_bot.is_null () || + query::build_package::custom_bot); + } + else + q = q && (query::build_package::custom_bot.is_null () || + !query::build_package::custom_bot); + // Filter by repositories canonical names (if requested). // - const vector<string>& rp (params.repository ()); + const strings& rp (params.repository ()); if (!rp.empty ()) q = q && @@ -213,20 +266,28 @@ handle (request& rq, response& rs) throw invalid_request (400, e.what ()); } - // Obtain the agent's public key fingerprint if requested. If the fingerprint - // is requested but is not present in the request or is unknown, then respond - // with 401 HTTP code (unauthorized). + // Obtain the agent's public key fingerprint if requested. If the + // fingerprint is requested but is not present in the request, then respond + // with 401 HTTP code (unauthorized). If a key with the specified + // fingerprint is not present in the build bot agent keys directory, then + // assume that this is a custom build bot. + // + // Note that if the agent authentication is not configured (the agent keys + // directory is not specified), then the bot can never be custom and its + // fingerprint is ignored, if present. // optional<string> agent_fp; + bool custom_bot (false); if (bot_agent_key_map_ != nullptr) { - if (!tqm.fingerprint || - bot_agent_key_map_->find (*tqm.fingerprint) == - bot_agent_key_map_->end ()) + if (!tqm.fingerprint) throw invalid_request (401, "unauthorized"); agent_fp = move (tqm.fingerprint); + + custom_bot = (bot_agent_key_map_->find (*agent_fp) == + bot_agent_key_map_->end ()); } // The resulting task manifest and the related build, package, and @@ -578,63 +639,6 @@ handle (request& rq, response& rs) : optional<size_t> ()), options_->build_hard_rebuild_timeout ())); - // Return the challenge (nonce) if brep is configured to authenticate bbot - // agents. Return nullopt otherwise. - // - // Nonce generator must guarantee a probabilistically insignificant chance - // of repeating a previously generated value. The common approach is to use - // counters or random number generators (alone or in combination), that - // produce values of the sufficient length. 64-bit non-repeating and - // 512-bit random numbers are considered to be more than sufficient for - // most practical purposes. - // - // We will produce the challenge as the sha256sum of the 512-bit random - // number and the 64-bit current timestamp combination. The latter is - // not really a non-repeating counter and can't be used alone. However - // adding it is a good and cheap uniqueness improvement. - // - auto challenge = [&agent_fp, &now, &fail, &trace, this] () - { - optional<string> r; - - if (agent_fp) - { - try - { - auto print_args = [&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }; - - openssl os (print_args, - nullfd, path ("-"), 2, - process_env (options_->openssl (), - options_->openssl_envvar ()), - "rand", - options_->openssl_option (), 64); - - vector<char> nonce (os.in.read_binary ()); - os.in.close (); - - if (!os.wait () || nonce.size () != 64) - fail << "unable to generate nonce"; - - uint64_t t (chrono::duration_cast<chrono::nanoseconds> ( - now.time_since_epoch ()).count ()); - - sha256 cs (nonce.data (), nonce.size ()); - cs.append (&t, sizeof (t)); - r = cs.string (); - } - catch (const system_error& e) - { - fail << "unable to generate nonce: " << e; - } - } - - return r; - }; - // Convert butl::standard_version type to brep::version. // brep::version toolchain_version (tqm.toolchain_version.string ()); @@ -659,7 +663,8 @@ handle (request& rq, response& rs) using pkg_query = query<buildable_package>; using prep_pkg_query = prepared_query<buildable_package>; - pkg_query pq (package_query<buildable_package> (params, + pkg_query pq (package_query<buildable_package> (custom_bot, + params, imode, queued_expiration_ns)); @@ -815,7 +820,8 @@ handle (request& rq, response& rs) { using query = query<buildable_package_count>; - query q (package_query<buildable_package_count> (params, + query q (package_query<buildable_package_count> (custom_bot, + params, imode, queued_expiration_ns)); @@ -1241,7 +1247,8 @@ handle (request& rq, response& rs) // small_vector<bpkg::test_dependency, 1> tests; - build_db_->load (*p, p->requirements_tests_section); + if (!p->requirements_tests_section.loaded ()) + build_db_->load (*p, p->requirements_tests_section); for (const build_test_dependency& td: p->tests) { @@ -1293,6 +1300,8 @@ handle (request& rq, response& rs) true /* default_all_ucs */)) continue; + build_db_->load (*tp, tp->auxiliaries_section); + for (const build_auxiliary& ba: tpc->effective_auxiliaries (tp->auxiliaries)) { @@ -1312,20 +1321,56 @@ handle (request& rq, response& rs) vector<auxiliary_machine> tms; vector<build_machine> bms; - tms.reserve (picked_machines.size ()); - bms.reserve (picked_machines.size ()); - - for (pair<auxiliary_config_machine, string>& pm: picked_machines) + if (size_t n = picked_machines.size ()) { - const machine_header_manifest& m (*pm.first.machine); - tms.push_back (auxiliary_machine {m.name, move (pm.second)}); - bms.push_back (build_machine {m.name, m.summary}); + tms.reserve (n); + bms.reserve (n); + + for (pair<auxiliary_config_machine, string>& pm: picked_machines) + { + const machine_header_manifest& m (*pm.first.machine); + tms.push_back (auxiliary_machine {m.name, move (pm.second)}); + bms.push_back (build_machine {m.name, m.summary}); + } } return collect_auxiliaries_result { move (tms), move (bms), move (tests)}; }; + if (agent_fp && !challenge) + try + { + auto print_args = [&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }; + + openssl os (print_args, + nullfd, path ("-"), 2, + process_env (options_->openssl (), + options_->openssl_envvar ()), + "rand", + options_->openssl_option (), 64); + + vector<char> nonce (os.in.read_binary ()); + os.in.close (); + + if (!os.wait () || nonce.size () != 64) + fail << "unable to generate nonce"; + + uint64_t t (chrono::duration_cast<chrono::nanoseconds> ( + now.time_since_epoch ()).count ()); + + sha256 cs (nonce.data (), nonce.size ()); + cs.append (&t, sizeof (t)); + challenge = cs.string (); + } + catch (const system_error& e) + { + fail << "unable to generate nonce: " << e; + } + // While at it, collect the aborted for various reasons builds // (interactive builds in multiple configurations, builds with too many // auxiliary machines, etc) to send the notification emails at the end @@ -1603,8 +1648,40 @@ handle (request& rq, response& rs) // bool package_built (false); + build_db_->load (*p, p->bot_keys_section); + for (const build_package_config& pc: p->configs) { + // If this is a custom bot, then skip this configuration if it + // doesn't contain this bot's public key in its custom bot keys + // list. Otherwise (this is a default bot), skip this + // configuration if its custom bot keys list is not empty. + // + { + const build_package_bot_keys& bks ( + pc.effective_bot_keys (p->bot_keys)); + + if (custom_bot) + { + assert (agent_fp); // Wouldn't be here otherwise. + + if (find_if ( + bks.begin (), bks.end (), + [&agent_fp] (const lazy_shared_ptr<build_public_key>& k) + { + return k.object_id ().fingerprint == *agent_fp; + }) == bks.end ()) + { + continue; + } + } + else + { + if (!bks.empty ()) + continue; + } + } + pkg_config = pc.name; // Iterate through the built configurations and erase them from the @@ -1647,35 +1724,38 @@ handle (request& rq, response& rs) // the package configuration and for which all the requested // auxiliary machines can be provided. // - auto i (configs.begin ()); - auto e (configs.end ()); + const config_machine* cm (nullptr); + optional<collect_auxiliaries_result> aux; build_db_->load (*p, p->constraints_section); - optional<collect_auxiliaries_result> aux; - for (; i != e; ++i) + for (auto i (configs.begin ()), e (configs.end ()); i != e; ++i) { - const build_target_config& tc (*i->second.config); + cm = &i->second; + const build_target_config& tc (*cm->config); + + if (!exclude (pc, p->builds, p->constraints, tc)) + { + if (!p->auxiliaries_section.loaded ()) + build_db_->load (*p, p->auxiliaries_section); - if (!exclude (pc, p->builds, p->constraints, tc) && - (aux = collect_auxiliaries (p, pc, tc))) - break; + if ((aux = collect_auxiliaries (p, pc, tc))) + break; + } } - if (i != e) + if (aux) { - config_machine& cm (i->second); - machine_header_manifest& mh (*cm.machine); + machine_header_manifest& mh (*cm->machine); build_id bid (move (id), - cm.config->target, - cm.config->name, + cm->config->target, + cm->config->name, move (pkg_config), move (toolchain_name), toolchain_version); shared_ptr<build> b (build_db_->find<build> (bid)); - optional<string> cl (challenge ()); // Move the interactive build login information into the build // object, if the package to be built interactively. @@ -1700,12 +1780,14 @@ handle (request& rq, response& rs) move (toolchain_version), move (login), move (agent_fp), - move (cl), + move (challenge), build_machine { mh.name, move (mh.summary)}, move (aux->build_auxiliary_machines), - controller_checksum (*cm.config), - machine_checksum (*cm.machine)); + controller_checksum (*cm->config), + machine_checksum (*cm->machine)); + + challenge = nullopt; build_db_->persist (b); } @@ -1742,7 +1824,10 @@ handle (request& rq, response& rs) } b->agent_fingerprint = move (agent_fp); - b->agent_challenge = move (cl); + + b->agent_challenge = move (challenge); + challenge = nullopt; + b->machine = build_machine {mh.name, move (mh.summary)}; // Mark the section as loaded, so auxiliary_machines are @@ -1753,8 +1838,8 @@ handle (request& rq, response& rs) b->auxiliary_machines = move (aux->build_auxiliary_machines); - string ccs (controller_checksum (*cm.config)); - string mcs (machine_checksum (*cm.machine)); + string ccs (controller_checksum (*cm->config)); + string mcs (machine_checksum (*cm->machine)); // Issue the hard rebuild if it is forced or the // configuration or machine has changed. @@ -1831,7 +1916,7 @@ handle (request& rq, response& rs) move (aux->tests), move (aux->task_auxiliary_machines), move (bp.interactive), - cm); + *cm); task_build = move (b); task_package = move (p); @@ -1911,8 +1996,6 @@ handle (request& rq, response& rs) sort (rebuilds.begin (), rebuilds.end (), cmp); - optional<string> cl (challenge ()); - // Pick the first build configuration from the ordered list. // // Note that the configurations and packages may not match the @@ -1971,13 +2054,17 @@ handle (request& rq, response& rs) (t->interactive.has_value () == (imode == interactive_mode::true_)))) { + const build_target_config& tc (*cm.config); + build_db_->load (*p, p->constraints_section); - const build_target_config& tc (*cm.config); + if (exclude (*pc, p->builds, p->constraints, tc)) + continue; + + build_db_->load (*p, p->auxiliaries_section); - optional<collect_auxiliaries_result> aux; - if (!exclude (*pc, p->builds, p->constraints, tc) && - (aux = collect_auxiliaries (p, *pc, tc))) + if (optional<collect_auxiliaries_result> aux = + collect_auxiliaries (p, *pc, tc)) { assert (b->status); @@ -1999,10 +2086,10 @@ handle (request& rq, response& rs) unforced = (b->force == force_state::unforced); - // Can't move from, as may need them on the next iteration. - // - b->agent_fingerprint = agent_fp; - b->agent_challenge = cl; + b->agent_fingerprint = move (agent_fp); + + b->agent_challenge = move (challenge); + challenge = nullopt; const machine_header_manifest& mh (*cm.machine); b->machine = build_machine {mh.name, mh.summary}; @@ -2095,9 +2182,12 @@ handle (request& rq, response& rs) } catch (const odb::deadlock&) { - // Just try with the next rebuild. But first, reset the task - // manifest and the session that we may have prepared. + // Just try with the next rebuild. But first, restore the agent's + // fingerprint and challenge and reset the task manifest and the + // session that we may have prepared. // + agent_fp = move (b->agent_fingerprint); + challenge = move (b->agent_challenge); task_response = task_response_manifest (); } @@ -2118,7 +2208,7 @@ handle (request& rq, response& rs) { assert (tss); // Wouldn't be here otherwise. - const tenant_service& ss (tss->first); + tenant_service& ss (tss->first); // If the task build has no initial state (is just created), then // temporarily move it into the list of the queued builds until the @@ -2141,7 +2231,11 @@ handle (request& rq, response& rs) nullopt /* initial_state */, qhs, log_writer_)) - update_tenant_service_state (conn, qbs.back ().tenant, f); + { + if (optional<string> data = + update_tenant_service_state (conn, qbs.back ().tenant, f)) + ss.data = move (data); + } } // Send the `queued` notification for the task build, unless it is @@ -2161,7 +2255,11 @@ handle (request& rq, response& rs) initial_state, qhs, log_writer_)) - update_tenant_service_state (conn, qbs.back ().tenant, f); + { + if (optional<string> data = + update_tenant_service_state (conn, qbs.back ().tenant, f)) + ss.data = move (data); + } } if (restore_build) @@ -2177,11 +2275,15 @@ handle (request& rq, response& rs) { assert (tss); // Wouldn't be here otherwise. - const tenant_service& ss (tss->first); + tenant_service& ss (tss->first); const build& b (*tss->second); if (auto f = tsb->build_building (ss, b, log_writer_)) - update_tenant_service_state (conn, b.tenant, f); + { + if (optional<string> data = + update_tenant_service_state (conn, b.tenant, f)) + ss.data = move (data); + } } // If the task manifest is prepared, then check that the number of the @@ -2290,11 +2392,15 @@ handle (request& rq, response& rs) { assert (tss); // Wouldn't be here otherwise. - const tenant_service& ss (tss->first); + tenant_service& ss (tss->first); const build& b (*tss->second); if (auto f = tsb->build_built (ss, b, log_writer_)) - update_tenant_service_state (conn, b.tenant, f); + { + if (optional<string> data = + update_tenant_service_state (conn, b.tenant, f)) + ss.data = move (data); + } } } diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx index 5cf0759..fcd50da 100644 --- a/mod/mod-package-details.cxx +++ b/mod/mod-package-details.cxx @@ -227,7 +227,7 @@ handle (request& rq, response& rs) << ~TABLE; } - auto pkg_count ( + size_t pkg_count ( package_db_->query_value<package_count> ( search_params<package_count> (squery, tenant, name))); diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 51b21c6..91923e5 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -541,7 +541,7 @@ handle (request& rq, response& rs) // builds = false; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { const build_class_exprs& exprs (pc.effective_builds (pkg->builds)); @@ -726,7 +726,7 @@ handle (request& rq, response& rs) s << H3 << "Builds" << ~H3 << DIV(ID="builds"); - auto exclude = [&pkg, this] (const build_package_config& pc, + auto exclude = [&pkg, this] (const package_build_config& pc, const build_target_config& tc, string* rs = nullptr) { @@ -767,7 +767,7 @@ handle (request& rq, response& rs) query sq (false); set<config_toolchain> unbuilt_configs; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& bc: *target_conf_map_) { @@ -886,7 +886,7 @@ handle (request& rq, response& rs) // if (!tn->interactive) { - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& tc: *target_conf_) { diff --git a/mod/mod-packages.cxx b/mod/mod-packages.cxx index 914a841..6026024 100644 --- a/mod/mod-packages.cxx +++ b/mod/mod-packages.cxx @@ -156,7 +156,7 @@ handle (request& rq, response& rs) session sn; transaction t (package_db_->begin ()); - auto pkg_count ( + size_t pkg_count ( package_db_->query_value<latest_package_count> ( search_param<latest_package_count> (squery, tn))); diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx index 2f49f81..42d481d 100644 --- a/monitor/monitor.cxx +++ b/monitor/monitor.cxx @@ -95,7 +95,7 @@ namespace brep // not. // void - add_delay (shared_ptr<build_delay>, bool report); + add_delay (shared_ptr<build_delay>, bool custom_bot, bool report); bool empty () const {return reported_delay_count_ == 0;} @@ -108,19 +108,39 @@ namespace brep print (const char* header, bool total, bool full) const; private: - // Maps delays to the report flag. + // Maps delays to the custom bot and report flag. // - map<shared_ptr<const build_delay>, bool, compare_delay> delays_; + struct delay_info + { + bool custom_bot; + bool report; + }; + + map<shared_ptr<const build_delay>, delay_info, compare_delay> delays_; size_t reported_delay_count_ = 0; + + // Number of reported/total delayed package configurations which need to + // be built with the custom build bots. + // + size_t custom_total_delay_count_ = 0; + size_t custom_reported_delay_count_ = 0; }; void delay_report:: - add_delay (shared_ptr<build_delay> delay, bool report) + add_delay (shared_ptr<build_delay> delay, bool custom_bot, bool report) { - delays_.emplace (move (delay), report); + delays_.emplace (move (delay), delay_info {custom_bot, report}); + + if (custom_bot) + ++custom_total_delay_count_; if (report) + { ++reported_delay_count_; + + if (custom_bot) + ++custom_reported_delay_count_; + } } void delay_report:: @@ -134,6 +154,17 @@ namespace brep if (total) cerr << '/' << delays_.size (); + if (custom_reported_delay_count_ != 0 || + (total && custom_total_delay_count_ != 0)) + { + cerr << " including " << custom_reported_delay_count_; + + if (total) + cerr << '/' << custom_total_delay_count_; + + cerr << " custom"; + } + cerr << "):" << endl; // Group the printed delays by toolchain and target configuration. @@ -146,10 +177,15 @@ namespace brep size_t config_reported_delay_count (0); size_t config_total_delay_count (0); + size_t config_custom_reported_delay_count (0); + size_t config_custom_total_delay_count (0); + auto brief_config = [&target_config_name, &target, &config_reported_delay_count, &config_total_delay_count, + &config_custom_reported_delay_count, + &config_custom_total_delay_count, total] () { if (target_config_name != nullptr) @@ -166,21 +202,36 @@ namespace brep if (total) cerr << '/' << config_total_delay_count; + if (config_custom_reported_delay_count != 0 || + (total && config_custom_total_delay_count != 0)) + { + cerr << " including " << config_custom_reported_delay_count; + + if (total) + cerr << '/' << config_custom_total_delay_count; + + cerr << " custom"; + } + cerr << ')' << endl; } config_reported_delay_count = 0; config_total_delay_count = 0; + + config_custom_reported_delay_count = 0; + config_custom_total_delay_count = 0; } }; for (const auto& dr: delays_) { - bool report (dr.second); + bool report (dr.second.report); if (full && !report) continue; + bool custom_bot (dr.second.custom_bot); const shared_ptr<const build_delay>& d (dr.first); // Print the toolchain, if changed. @@ -239,6 +290,9 @@ namespace brep cerr << " " << d->package_name << '/' << d->package_version << ' ' << d->package_config_name; + if (custom_bot) + cerr << " (custom bot)"; + if (!d->tenant.empty ()) cerr << ' ' << d->tenant; @@ -247,9 +301,17 @@ namespace brep else { if (report) + { ++config_reported_delay_count; + if (custom_bot) + ++config_custom_reported_delay_count; + } + ++config_total_delay_count; + + if (custom_bot) + ++config_custom_total_delay_count; } } @@ -953,6 +1015,21 @@ namespace brep soft_delayed = false; } + // If there is a delay, then deduce if this package + // configuration needs to be built with a custom build bot. + // + // Note: only meaningful if there is a delay. + // + bool custom_bot (false); + + if (hard_delayed || soft_delayed) + { + if (!p->bot_keys_section.loaded ()) + db.load (*p, p->bot_keys_section); + + custom_bot = !pc.effective_bot_keys (p->bot_keys).empty (); + } + // Add hard/soft delays to the respective reports and // collect the delay for update, if it is reported. // @@ -983,7 +1060,7 @@ namespace brep reported = true; } - hard_delays_report.add_delay (d, report); + hard_delays_report.add_delay (d, custom_bot, report); } if (soft_delayed) @@ -997,7 +1074,7 @@ namespace brep reported = true; } - soft_delays_report.add_delay (d, report); + soft_delays_report.add_delay (d, custom_bot, report); } // If we don't consider the report timestamps for reporting |