diff options
| -rw-r--r-- | NEWS | 4 | ||||
| -rw-r--r-- | libbrep/build.cxx | 8 | ||||
| -rw-r--r-- | libbrep/build.hxx | 58 | ||||
| -rw-r--r-- | libbrep/build.xml | 11 | ||||
| -rw-r--r-- | libbrep/common-traits.hxx | 38 | ||||
| -rw-r--r-- | libbrep/common.hxx | 8 | ||||
| -rw-r--r-- | mod/build-config-module.cxx | 14 | ||||
| -rw-r--r-- | mod/build-config-module.hxx | 22 | ||||
| -rw-r--r-- | mod/build-config.hxx | 23 | ||||
| -rw-r--r-- | mod/build.cxx | 2 | ||||
| -rw-r--r-- | mod/mod-build-force.cxx | 16 | ||||
| -rw-r--r-- | mod/mod-build-log.cxx | 20 | ||||
| -rw-r--r-- | mod/mod-build-result.cxx | 23 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 45 | ||||
| -rw-r--r-- | mod/mod-builds.cxx | 94 | ||||
| -rw-r--r-- | mod/mod-package-version-details.cxx | 38 | ||||
| -rw-r--r-- | mod/module.cli | 4 | ||||
| -rw-r--r-- | monitor/monitor.cxx | 46 |
18 files changed, 318 insertions, 156 deletions
@@ -1,3 +1,7 @@ +Version 0.16.0 + + * Note that no brep_build schema migration from version 18 is supported. + Version 0.15.0 * Support for disabling build notification emails per toolchain. diff --git a/libbrep/build.cxx b/libbrep/build.cxx index 9cbad8f..03775c4 100644 --- a/libbrep/build.cxx +++ b/libbrep/build.cxx @@ -58,20 +58,22 @@ namespace brep package_name_type pnm, version pvr, string cfg, + target_triplet trg, string tnm, version tvr, optional<string> inr, optional<string> afp, optional<string> ach, string mnm, string msm, - butl::target_triplet trg, string ccs, string mcs) : id (package_id (move (tnt), move (pnm), pvr), move (cfg), + move (trg), move (tnm), tvr), tenant (id.package.tenant), package_name (id.package.name), package_version (move (pvr)), configuration (id.configuration), + target (id.target), toolchain_name (id.toolchain_name), toolchain_version (move (tvr)), state (build_state::building), @@ -81,7 +83,6 @@ namespace brep agent_fingerprint (move (afp)), agent_challenge (move (ach)), machine (move (mnm)), machine_summary (move (msm)), - target (move (trg)), controller_checksum (move (ccs)), machine_checksum (move (mcs)) { @@ -93,15 +94,18 @@ namespace brep build_delay (string tnt, package_name_type pnm, version pvr, string cfg, + target_triplet trg, string tnm, version tvr, timestamp ptm) : id (package_id (move (tnt), move (pnm), pvr), move (cfg), + move (trg), move (tnm), tvr), tenant (id.package.tenant), package_name (id.package.name), package_version (move (pvr)), configuration (id.configuration), + target (id.target), toolchain_name (id.toolchain_name), toolchain_version (move (tvr)), package_timestamp (ptm) diff --git a/libbrep/build.hxx b/libbrep/build.hxx index 120ed5c..ceea81a 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -9,8 +9,6 @@ #include <odb/core.hxx> #include <odb/section.hxx> -#include <libbutl/target-triplet.hxx> - #include <libbrep/types.hxx> #include <libbrep/utility.hxx> @@ -28,14 +26,13 @@ // Used by the data migration entries. // -#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 18 +#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 19 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 18, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 19, closed) -// We have to keep these mappings at the global scope instead of inside -// the brep namespace because they need to be also effective in the -// bbot namespace from which we "borrow" types (and some of them use the mapped -// types). +// We have to keep these mappings at the global scope instead of inside the +// brep namespace because they need to be also effective in the bbot namespace +// from which we "borrow" types (and some of them use the mapped types). // #pragma db map type(bbot::result_status) as(std::string) \ to(to_string (?)) \ @@ -48,13 +45,19 @@ namespace brep { package_id package; string configuration; + target_triplet target; string toolchain_name; canonical_version toolchain_version; build_id () = default; - build_id (package_id p, string c, string n, const brep::version& v) + build_id (package_id p, + string c, + target_triplet t, + string n, + const brep::version& v) : package (move (p)), configuration (move (c)), + target (move (t)), toolchain_name (move (n)), toolchain_version (v) {} }; @@ -68,13 +71,16 @@ namespace brep if (int r = x.configuration.compare (y.configuration)) return r < 0; + if (int r = x.target.compare (y.target)) + return r < 0; + if (int r = x.toolchain_name.compare (y.toolchain_name)) return r < 0; return compare_version_lt (x.toolchain_version, y.toolchain_version, true); } - // These allow comparing objects that have package, configuration, + // These allow comparing objects that have package, configuration, target, // toolchain_name, and toolchain_version data members to build_id values. // The idea is that this works for both query members of build id types as // well as for values of the build_id type. @@ -84,12 +90,14 @@ namespace brep operator== (const T& x, const build_id& y) -> decltype (x.package == y.package && x.configuration == y.configuration && + x.target == y.target && x.toolchain_name == y.toolchain_name && x.toolchain_version.epoch == y.toolchain_version.epoch) { - return x.package == y.package && - x.configuration == y.configuration && - x.toolchain_name == y.toolchain_name && + return x.package == y.package && + x.configuration == y.configuration && + x.target == y.target && + x.toolchain_name == y.toolchain_name && compare_version_eq (x.toolchain_version, y.toolchain_version, true); } @@ -98,12 +106,14 @@ namespace brep operator!= (const T& x, const build_id& y) -> decltype (x.package == y.package && x.configuration == y.configuration && + x.target == y.target && x.toolchain_name == y.toolchain_name && x.toolchain_version.epoch == y.toolchain_version.epoch) { - return x.package != y.package || - x.configuration != y.configuration || - x.toolchain_name != y.toolchain_name || + return x.package != y.package || + x.configuration != y.configuration || + x.target == y.target || + x.toolchain_name != y.toolchain_name || compare_version_ne (x.toolchain_version, y.toolchain_version, true); } @@ -162,12 +172,6 @@ namespace brep ? bbot::to_result_status (*(?)) \ : brep::optional_result_status ()) - // target_triplet - // - #pragma db map type(butl::target_triplet) as(string) \ - to((?).string ()) \ - from(butl::target_triplet (?)) - // operation_results // using bbot::operation_result; @@ -189,12 +193,12 @@ namespace brep package_name_type, version, string configuration, + target_triplet, string toolchain_name, version toolchain_version, optional<string> interactive, optional<string> agent_fingerprint, optional<string> agent_challenge, string machine, string machine_summary, - butl::target_triplet, string controller_checksum, string machine_checksum); @@ -204,6 +208,7 @@ namespace brep package_name_type& package_name; // Tracks id.package.name. upstream_version package_version; // Original of id.package.version. string& configuration; // Tracks id.configuration. + target_triplet& target; // Tracks id.target. string& toolchain_name; // Tracks id.toolchain_name. upstream_version toolchain_version; // Original of id.toolchain_version. @@ -253,7 +258,6 @@ namespace brep string machine; string machine_summary; - butl::target_triplet target; // Note that the logs are stored as std::string/TEXT which is Ok since // they are UTF-8 and our database is UTF-8. @@ -285,6 +289,7 @@ namespace brep #pragma db member(package_version) \ set(this.package_version.init (this.id.package.version, (?))) #pragma db member(configuration) transient + #pragma db member(target) transient #pragma db member(toolchain_name) transient #pragma db member(toolchain_version) \ set(this.toolchain_version.init (this.id.toolchain_version, (?))) @@ -308,6 +313,7 @@ namespace brep : tenant (id.package.tenant), package_name (id.package.name), configuration (id.configuration), + target (id.target), toolchain_name (id.toolchain_name) {} }; @@ -403,6 +409,7 @@ namespace brep build_delay (string tenant, package_name_type, version, string configuration, + target_triplet, string toolchain_name, version toolchain_version, timestamp package_timestamp); @@ -412,6 +419,7 @@ namespace brep package_name_type& package_name; // Tracks id.package.name. upstream_version package_version; // Original of id.package.version. string& configuration; // Tracks id.configuration. + target_triplet& target; // Tracks id.target. string& toolchain_name; // Tracks id.toolchain_name. upstream_version toolchain_version; // Original of id.toolchain_version. @@ -440,6 +448,7 @@ namespace brep #pragma db member(package_version) \ set(this.package_version.init (this.id.package.version, (?))) #pragma db member(configuration) transient + #pragma db member(target) transient #pragma db member(toolchain_name) transient #pragma db member(toolchain_version) \ set(this.toolchain_version.init (this.id.toolchain_version, (?))) @@ -451,6 +460,7 @@ namespace brep : tenant (id.package.tenant), package_name (id.package.name), configuration (id.configuration), + target (id.target), toolchain_name (id.toolchain_name) {} }; } diff --git a/libbrep/build.xml b/libbrep/build.xml index 821444c..7fe936d 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,5 +1,5 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1"> - <model version="18"> + <model version="19"> <table name="build" kind="object"> <column name="package_tenant" type="TEXT" null="false"/> <column name="package_name" type="CITEXT" null="false"/> @@ -8,6 +8,7 @@ <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> <column name="package_version_revision" type="INTEGER" null="false"/> <column name="configuration" type="TEXT" null="false"/> + <column name="target" type="TEXT" null="false"/> <column name="toolchain_name" type="TEXT" null="false"/> <column name="toolchain_version_epoch" type="INTEGER" null="false"/> <column name="toolchain_version_canonical_upstream" type="TEXT" null="false"/> @@ -28,7 +29,6 @@ <column name="agent_challenge" type="TEXT" null="true"/> <column name="machine" type="TEXT" null="false"/> <column name="machine_summary" type="TEXT" null="false"/> - <column name="target" type="TEXT" null="false"/> <column name="controller_checksum" type="TEXT" null="false"/> <column name="machine_checksum" type="TEXT" null="false"/> <column name="agent_checksum" type="TEXT" null="true"/> @@ -42,6 +42,7 @@ <column name="package_version_canonical_release"/> <column name="package_version_revision"/> <column name="configuration"/> + <column name="target"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> @@ -60,6 +61,7 @@ <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> <column name="package_version_revision" type="INTEGER" null="false"/> <column name="configuration" type="TEXT" null="false"/> + <column name="target" type="TEXT" null="false"/> <column name="toolchain_name" type="TEXT" null="false"/> <column name="toolchain_version_epoch" type="INTEGER" null="false"/> <column name="toolchain_version_canonical_upstream" type="TEXT" null="false"/> @@ -77,6 +79,7 @@ <column name="package_version_canonical_release"/> <column name="package_version_revision"/> <column name="configuration"/> + <column name="target"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> @@ -90,6 +93,7 @@ <column name="package_version_canonical_release"/> <column name="package_version_revision"/> <column name="configuration"/> + <column name="target"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> @@ -105,6 +109,7 @@ <column name="package_version_canonical_release"/> <column name="package_version_revision"/> <column name="configuration"/> + <column name="target"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> @@ -123,6 +128,7 @@ <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> <column name="package_version_revision" type="INTEGER" null="false"/> <column name="configuration" type="TEXT" null="false"/> + <column name="target" type="TEXT" null="false"/> <column name="toolchain_name" type="TEXT" null="false"/> <column name="toolchain_version_epoch" type="INTEGER" null="false"/> <column name="toolchain_version_canonical_upstream" type="TEXT" null="false"/> @@ -143,6 +149,7 @@ <column name="package_version_canonical_release"/> <column name="package_version_revision"/> <column name="configuration"/> + <column name="target"/> <column name="toolchain_name"/> <column name="toolchain_version_epoch"/> <column name="toolchain_version_canonical_upstream"/> diff --git a/libbrep/common-traits.hxx b/libbrep/common-traits.hxx index 99e8f3e..141a738 100644 --- a/libbrep/common-traits.hxx +++ b/libbrep/common-traits.hxx @@ -10,12 +10,50 @@ #include <odb/pgsql/traits.hxx> +#include <libbutl/target-triplet.hxx> + #include <libbpkg/package-name.hxx> namespace odb { namespace pgsql { + // target_triplet + // + template <> + class value_traits<butl::target_triplet, id_string>: + value_traits<std::string, id_string> + { + public: + using value_type = butl::target_triplet; + using query_type = butl::target_triplet; + using image_type = details::buffer; + + using base_type = value_traits<std::string, id_string>; + + static void + set_value (value_type& v, + const details::buffer& b, + std::size_t n, + bool is_null) + { + std::string s; + base_type::set_value (s, b, n, is_null); + v = !s.empty () ? value_type (s) : value_type (); + } + + static void + set_image (details::buffer& b, + std::size_t& n, + bool& is_null, + const value_type& v) + { + base_type::set_image (b, n, is_null, v.string ()); + } + }; + + // package_name + // template <> class value_traits<bpkg::package_name, id_string>: value_traits<std::string, id_string> diff --git a/libbrep/common.hxx b/libbrep/common.hxx index b9b6928..af34f6c 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -11,6 +11,8 @@ #include <odb/query.hxx> #include <odb/nested-container.hxx> +#include <libbutl/target-triplet.hxx> + #include <libbpkg/package-name.hxx> #include <libbrep/types.hxx> @@ -229,6 +231,12 @@ namespace brep // extern const version wildcard_version; + // target_triplet + // + using butl::target_triplet; + + #pragma db value(target_triplet) type("TEXT") + // package_name // using bpkg::package_name; diff --git a/mod/build-config-module.cxx b/mod/build-config-module.cxx index 1b1d53e..bf21fbb 100644 --- a/mod/build-config-module.cxx +++ b/mod/build-config-module.cxx @@ -137,22 +137,12 @@ namespace brep bot_agent_key_map_ = shared_bot_agent_keys (bo, bo.build_bot_agent_keys ()); - cstrings conf_names; - - using conf_map_type = map<const char*, - const build_config*, - compare_c_string>; - + using conf_map_type = map<build_config_id, const build_config*>; conf_map_type conf_map; for (const auto& c: *build_conf_) - { - const char* cn (c.name.c_str ()); - conf_map[cn] = &c; - conf_names.push_back (cn); - } + conf_map[build_config_id {c.name, c.target}] = &c; - build_conf_names_ = make_shared<cstrings> (move (conf_names)); build_conf_map_ = make_shared<conf_map_type> (move (conf_map)); } diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx index 51813c0..b276d6c 100644 --- a/mod/build-config-module.hxx +++ b/mod/build-config-module.hxx @@ -6,7 +6,7 @@ #include <map> -#include <libbutl/utility.hxx> // compare_c_string +#include <libbutl/target-triplet.hxx> #include <libbpkg/manifest.hxx> @@ -64,15 +64,16 @@ namespace brep return belongs (cfg, cls.c_str ()); } - // Configuration/toolchain combination that, in particular, can be used as - // a set value. + // Configuration/target/toolchain combination that, in particular, can be + // used as a set value. // - // Note: contains shallow references to the configuration, toolchain name, - // and version. + // Note: contains shallow references to the configuration, target, + // toolchain name, and version. // struct config_toolchain { const string& configuration; + const butl::target_triplet& target; const string& toolchain_name; const bpkg::version& toolchain_version; @@ -85,7 +86,10 @@ namespace brep if (toolchain_version != ct.toolchain_version) return toolchain_version > ct.toolchain_version; - return configuration.compare (ct.configuration) < 0; + if (int r = configuration.compare (ct.configuration)) + return r < 0; + + return target.compare (ct.target) < 0; } }; @@ -93,11 +97,9 @@ namespace brep // Build configurations. // shared_ptr<const bbot::build_configs> build_conf_; - shared_ptr<const cstrings> build_conf_names_; - shared_ptr<const std::map<const char*, - const bbot::build_config*, - butl::compare_c_string>> build_conf_map_; + shared_ptr<const std::map<build_config_id, const bbot::build_config*>> + build_conf_map_; // Map of build bot agent public keys fingerprints to the key file paths. // diff --git a/mod/build-config.hxx b/mod/build-config.hxx index e8dfe07..4ef01f6 100644 --- a/mod/build-config.hxx +++ b/mod/build-config.hxx @@ -6,6 +6,8 @@ #include <map> +#include <libbutl/target-triplet.hxx> + #include <libbpkg/manifest.hxx> #include <libbbot/build-config.hxx> @@ -44,6 +46,27 @@ namespace brep // path dash_components_to_path (const string&); + + // Build configuration name/target combination that, in particular, + // identifies configurations in the buildtab and thus can be used as a + // set/map key. + // + // Note: contains shallow references to the configuration name and target. + // + struct build_config_id + { + reference_wrapper<const string> name; + reference_wrapper<const butl::target_triplet> target; + + bool + operator< (const build_config_id& x) const + { + if (int r = name.get ().compare (x.name.get ())) + return r < 0; + + return target.get ().compare (x.target.get ()) < 0; + } + }; } #endif // MOD_BUILD_CONFIG diff --git a/mod/build.cxx b/mod/build.cxx index 5b9d8aa..3b82aed 100644 --- a/mod/build.cxx +++ b/mod/build.cxx @@ -24,6 +24,7 @@ namespace brep mime_url_encode (b.package_name.string (), false) + '/' + b.package_version.string () + "/log/" + mime_url_encode (b.configuration, false /* query */) + '/' + + mime_url_encode (b.target.string (), false /* query */) + '/' + mime_url_encode (b.toolchain_name, false /* query */) + '/' + b.toolchain_version.string ()); @@ -48,6 +49,7 @@ namespace brep "?build-force&pn=" + mime_url_encode (b.package_name.string ()) + "&pv=" + b.package_version.string () + "&cf=" + mime_url_encode (b.configuration) + + "&tg=" + mime_url_encode (b.target.string ()) + "&tn=" + mime_url_encode (b.toolchain_name) + "&tv=" + b.toolchain_version.string () + "&reason="; diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx index bd172e3..281c76c 100644 --- a/mod/mod-build-force.cxx +++ b/mod/mod-build-force.cxx @@ -120,6 +120,17 @@ handle (request& rq, response& rs) if (config.empty ()) throw invalid_argument ("no configuration name"); + target_triplet target; + + try + { + target = target_triplet (params.target ()); + } + catch (const invalid_argument& e) + { + throw invalid_argument (string ("invalid target: ") + e.what ()); + } + string& toolchain_name (params.toolchain_name ()); if (toolchain_name.empty ()) @@ -130,6 +141,7 @@ handle (request& rq, response& rs) id = build_id (package_id (move (tenant), move (p), package_version), move (config), + move (target), move (toolchain_name), toolchain_version); } @@ -149,7 +161,7 @@ handle (request& rq, response& rs) // Make sure the build configuration still exists. // - if (build_conf_map_->find (id.configuration.c_str ()) == + if (build_conf_map_->find (build_config_id {id.configuration, id.target}) == build_conf_map_->end ()) config_expired ("no configuration"); @@ -177,7 +189,7 @@ handle (request& rq, response& rs) l1 ([&]{trace << "force rebuild for " << b->tenant << ' ' << b->package_name << '/' << b->package_version << ' ' - << b->configuration << ' ' + << b->configuration << '/' << b->target << ' ' << b->toolchain_name << '-' << b->toolchain_version << ": " << reason;}); } diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx index f106d4f..948c9c3 100644 --- a/mod/mod-build-log.cxx +++ b/mod/mod-build-log.cxx @@ -68,7 +68,7 @@ handle (request& rq, response& rs) // // Note that the URL path must be in the following form: // - // <pkg-name>/<pkg-version>/log/<cfg-name>/<toolchain-name>/<toolchain-version>[/<operation>] + // <pkg-name>/<pkg-version>/log/<cfg-name>/<target>/<toolchain-name>/<toolchain-version>[/<operation>] // // Also note that the presence of the first 3 components is guaranteed by // the repository_root module. @@ -132,6 +132,19 @@ handle (request& rq, response& rs) throw invalid_argument ("empty configuration name"); if (i == lpath.end ()) + throw invalid_argument ("no target"); + + target_triplet target; + try + { + target = target_triplet (*i++); + } + catch (const invalid_argument& e) + { + throw invalid_argument (string ("invalid target: ") + e.what ()); + } + + if (i == lpath.end ()) throw invalid_argument ("no toolchain name"); string toolchain_name (*i++); @@ -146,6 +159,7 @@ handle (request& rq, response& rs) id = build_id (package_id (tenant, move (name), package_version), move (config), + move (target), move (toolchain_name), toolchain_version); @@ -190,7 +204,7 @@ handle (request& rq, response& rs) // Make sure the build configuration still exists. // - if (build_conf_map_->find (id.configuration.c_str ()) == + if (build_conf_map_->find (build_config_id {id.configuration, id.target}) == build_conf_map_->end ()) config_expired ("no configuration"); @@ -233,9 +247,9 @@ handle (request& rq, response& rs) << "toolchain: " << b->toolchain_name << '-' << b->toolchain_version << endl << "config: " << b->configuration << endl + << "target: " << b->target << endl << "machine: " << b->machine << " (" << b->machine_summary << ")" << endl - << "target: " << b->target.string () << endl << "timestamp: "; butl::to_stream (os, diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index a55b41f..7eefe95 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -193,13 +193,29 @@ handle (request& rq, response&) p = s.find ('/', b); // End of configuration name. if (p == string::npos) - throw invalid_argument ("no toolchain name"); + throw invalid_argument ("no target"); string config (s, b, p - b); if (config.empty ()) throw invalid_argument ("empty configuration name"); + b = p + 1; // Start of target. + p = s.find ('/', b); // End of target. + + if (p == string::npos) + throw invalid_argument ("no toolchain name"); + + target_triplet target; + try + { + target = target_triplet (string (s, b, p - b)); + } + catch (const invalid_argument& e) + { + throw invalid_argument (string ("invalid target: ") + e.what ()); + } + b = p + 1; // Start of toolchain name. p = s.find ('/', b); // End of toolchain name. @@ -221,6 +237,7 @@ handle (request& rq, response&) id = build_id (package_id (move (tenant), move (name), package_version), move (config), + move (target), move (toolchain_name), toolchain_version); @@ -263,7 +280,8 @@ handle (request& rq, response&) // const bbot::build_config* cfg; { - auto i (build_conf_map_->find (id.configuration.c_str ())); + auto i (build_conf_map_->find (build_config_id {id.configuration, + id.target})); if (i == build_conf_map_->end ()) { @@ -541,6 +559,7 @@ handle (request& rq, response&) bld->package_name.string () + '/' + bld->package_version.string () + '/' + bld->configuration + '/' + + bld->target.string () + '/' + bld->toolchain_name + '-' + bld->toolchain_version.string ()); // Send notification emails to the interested parties. diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 29c048d..137cf6a 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -13,7 +13,6 @@ #include <libbutl/regex.hxx> #include <libbutl/sha256.hxx> -#include <libbutl/utility.hxx> // compare_c_string #include <libbutl/openssl.hxx> #include <libbutl/fdstream.hxx> // nullfd #include <libbutl/process-io.hxx> @@ -157,8 +156,7 @@ handle (request& rq, response& rs) task_response_manifest tsm; // Map build configurations to machines that are capable of building them. - // The first matching machine is selected for each configuration. Also - // create the configuration name list for use in database queries. + // The first matching machine is selected for each configuration. // struct config_machine { @@ -166,10 +164,9 @@ handle (request& rq, response& rs) machine_header_manifest* machine; }; - using config_machines = map<const char*, config_machine, compare_c_string>; + using config_machines = map<build_config_id, config_machine>; - cstrings cfg_names; - config_machines cfg_machines; + config_machines conf_machines; for (const auto& c: *build_conf_) { @@ -182,10 +179,9 @@ handle (request& rq, response& rs) if (path_match (dash_components_to_path (m.name), dash_components_to_path (c.machine_pattern), dir_path () /* start */, - path_match_flags::match_absent) && - cfg_machines.insert ( - make_pair (c.name.c_str (), config_machine ({&c, &m}))).second) - cfg_names.push_back (c.name.c_str ()); + path_match_flags::match_absent)) + conf_machines.emplace (build_config_id {c.name, c.target}, + config_machine {&c, &m}); } catch (const invalid_path&) {} } @@ -203,7 +199,7 @@ handle (request& rq, response& rs) // rebuild. The rebuild preference is given in the following order: the // greater force state, the greater overall status, the lower timestamp. // - if (!cfg_machines.empty ()) + if (!conf_machines.empty ()) { vector<shared_ptr<build>> rebuilds; @@ -223,6 +219,7 @@ handle (request& rq, response& rs) b->package_name.string () + '/' + b->package_version.string () + '/' + b->configuration + '/' + + b->target.string () + '/' + b->toolchain_name + '/' + b->toolchain_version.string () + '/' + to_string (ts)); @@ -599,12 +596,14 @@ handle (request& rq, response& rs) package_id id; + bld_query sq (false); + for (const auto& cm: conf_machines) + sq = sq || (bld_query::id.configuration == cm.first.name && + bld_query::id.target == cm.first.target); + bld_query bq ( equal<build> (bld_query::id.package, id) && - - bld_query::id.configuration.in_range (cfg_names.begin (), - cfg_names.end ()) && - + sq && bld_query::id.toolchain_name == tqm.toolchain_name && compare_version_eq (bld_query::id.toolchain_version, @@ -714,12 +713,13 @@ handle (request& rq, response& rs) // Also save the built package configurations for which it's time to // be rebuilt. // - config_machines configs (cfg_machines); // Make a copy for this pkg. + config_machines configs (conf_machines); // Make a copy for this pkg. auto pkg_builds (bld_prep_query.execute ()); for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i) { - auto j (configs.find (i->id.configuration.c_str ())); + auto j (configs.find (build_config_id {i->id.configuration, + i->id.target})); // Outdated configurations are already excluded with the database // query. @@ -758,6 +758,7 @@ handle (request& rq, response& rs) build_id bid (move (id), cm.config->name, + cm.config->target, move (tqm.toolchain_name), toolchain_version); @@ -784,6 +785,7 @@ handle (request& rq, response& rs) move (bid.package.name), move (bp.version), move (bid.configuration), + move (bid.target), move (bid.toolchain_name), move (toolchain_version), move (login), @@ -791,7 +793,6 @@ handle (request& rq, response& rs) move (cl), mh.name, move (mh.summary), - cm.config->target, controller_checksum (*cm.config), machine_checksum (*cm.machine)); @@ -823,7 +824,6 @@ handle (request& rq, response& rs) b->agent_challenge = move (cl); b->machine = mh.name; b->machine_summary = move (mh.summary); - b->target = cm.config->target; string ccs (controller_checksum (*cm.config)); string mcs (machine_checksum (*cm.machine)); @@ -911,11 +911,12 @@ handle (request& rq, response& rs) b->state == build_state::built && needs_rebuild (*b)) { - auto i (cfg_machines.find (b->id.configuration.c_str ())); + auto i (conf_machines.find (build_config_id {b->configuration, + b->target})); // Only actual package configurations are loaded (see above). // - assert (i != cfg_machines.end ()); + assert (i != conf_machines.end ()); const config_machine& cm (i->second); // Rebuild the package if still present, is buildable, doesn't @@ -962,8 +963,6 @@ handle (request& rq, response& rs) b->machine = mh.name; b->machine_summary = mh.summary; - b->target = cm.config->target; - // Issue the hard rebuild if the timeout expired, rebuild is // forced, or the configuration or machine has changed. // diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index 596a5ef..b9d841d 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -11,6 +11,7 @@ #include <odb/database.hxx> #include <odb/transaction.hxx> +#include <libbutl/utility.hxx> // compare_c_string #include <libbutl/timestamp.hxx> // to_string() #include <libbutl/path-pattern.hxx> @@ -137,7 +138,7 @@ match (const C qc, const string& pattern) // template <typename T> static inline query<T> -build_query (const brep::cstrings* configs, +build_query (const brep::vector<brep::build_config_id>* config_ids, const brep::params::builds& params, const brep::optional<brep::string>& tenant, const brep::optional<bool>& archived) @@ -154,9 +155,15 @@ build_query (const brep::cstrings* configs, if (archived) q = q && qt::archived == *archived; - if (configs != nullptr) - q = q && qb::id.configuration.in_range (configs->begin (), - configs->end ()); + if (config_ids != nullptr) + { + query sq (false); + for (const auto& id: *config_ids) + sq = sq || (qb::id.configuration == id.name && + qb::id.target == id.target); + + q = q && sq; + } // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no package builds match such a @@ -219,7 +226,7 @@ build_query (const brep::cstrings* configs, // Build target. // if (!params.target ().empty ()) - q = q && match<T> (qb::target, params.target ()); + q = q && match<T> (qb::id.target, params.target ()); // Build result. // @@ -461,15 +468,21 @@ handle (request& rq, response& rs) << DATALIST(ID="configs") << *OPTION(VALUE="*"); - for (const auto& c: *build_conf_names_) - s << *OPTION(VALUE=c); + // Print unique config names from the config map. + // + set<const char*, butl::compare_c_string> conf_names; + for (const auto& c: *build_conf_map_) + { + if (conf_names.insert (c.first.name.get ().c_str ()).second) + s << *OPTION(VALUE=c.first.name.get ()); + } s << ~DATALIST << ~TD << ~TR - << TR_INPUT ("machine", "mn", params.machine (), "*") << TR_INPUT ("target", "tg", params.target (), "*") + << TR_INPUT ("machine", "mn", params.machine (), "*") << TR_SELECT ("result", "rs", params.result (), build_results) << ~TBODY << ~TABLE @@ -497,18 +510,14 @@ handle (request& rq, response& rs) bool exclude_hidden (params.configuration ().empty () || path_pattern (params.configuration ())); - cstrings conf_names; + vector<build_config_id> conf_ids; + conf_ids.reserve (build_conf_map_->size ()); - if (exclude_hidden) + for (const auto& c: *build_conf_map_) { - for (const auto& c: *build_conf_map_) - { - if (belongs (*c.second, "all")) - conf_names.push_back (c.first); - } + if (!exclude_hidden || belongs (*c.second, "all")) + conf_ids.push_back (c.first); } - else - conf_names = *build_conf_names_; size_t count; size_t page (params.page ()); @@ -534,7 +543,7 @@ handle (request& rq, response& rs) using prep_query = prepared_query<package_build>; query q (build_query<package_build> ( - &conf_names, params, tn, nullopt /* archived */)); + &conf_ids, params, tn, nullopt /* archived */)); // Specify the portion. Note that we will be querying builds in chunks, // not to hold locks for too long. @@ -549,7 +558,7 @@ handle (request& rq, response& rs) // first). // q += "ORDER BY" + query::build::timestamp + "DESC" + - "OFFSET" + query::_ref (offset) + "LIMIT 500"; + "OFFSET" + query::_ref (offset) + "LIMIT 500"; connection_ptr conn (build_db_->connection ()); @@ -591,7 +600,8 @@ handle (request& rq, response& rs) { shared_ptr<build>& b (pb.build); - auto i (build_conf_map_->find (b->configuration.c_str ())); + auto i (build_conf_map_->find (build_config_id {b->configuration, + b->target})); assert (i != build_conf_map_->end ()); // Match the configuration against the package build @@ -610,8 +620,7 @@ handle (request& rq, response& rs) // same build multiple times. Let's skip the duplicates. Note: // we don't increment the counter in this case. // - if (find_if (builds.begin (), - builds.end (), + if (find_if (builds.begin (), builds.end (), [&b] (const shared_ptr<build>& pb) { return b->id == pb->id; @@ -672,8 +681,8 @@ handle (request& rq, response& rs) b.toolchain_name + '-' + b.toolchain_version.string ()) << TR_VALUE ("config", b.configuration) - << TR_VALUE ("machine", b.machine) << TR_VALUE ("target", b.target.string ()) + << TR_VALUE ("machine", b.machine) << TR_VALUE ("timestamp", ts); if (b.interactive) // Note: can only be present for the building state. @@ -766,7 +775,8 @@ handle (request& rq, response& rs) // Filter by toolchain. // if (tc == "*" || (t.first == tc_name && t.second == tc_version)) - config_toolchains.insert ({c.name, t.first, t.second}); + config_toolchains.insert ( + config_toolchain {c.name, c.target, t.first, t.second}); } } } @@ -784,15 +794,14 @@ handle (request& rq, response& rs) // due to the build configuration target or class set change. We should // deduct such builds count from the number of existing package builds. // - size_t nmax ( - config_toolchains.size () * - build_db_->query_value<buildable_package_count> ( - package_query<buildable_package_count> ( - params, tn, false /* archived */))); + size_t nmax (config_toolchains.size () * + build_db_->query_value<buildable_package_count> ( + package_query<buildable_package_count> ( + params, tn, false /* archived */))); size_t ncur = build_db_->query_value<package_build_count> ( build_query<package_build_count> ( - &conf_names, bld_params, tn, false /* archived */)); + &conf_ids, bld_params, tn, false /* archived */)); // From now we will be using specific package name and version for each // build database query. @@ -812,21 +821,23 @@ handle (request& rq, response& rs) package_id id; string config; + target_triplet target; const auto& bid (bld_query::build::id); bld_query bq ( equal<package_build_count> (bid.package, id) && bid.configuration == bld_query::_ref (config) && + bid.target == bld_query::_ref (target) && // Note that the query already constrains configurations via the - // configuration name. + // configuration name and target. // // Also note that while the query already constrains the tenant via // the build package id, we still need to pass the tenant not to // erroneously filter out the private tenants. // - build_query<package_build_count> (nullptr /* configs */, + build_query<package_build_count> (nullptr /* config_ids */, bld_params, tn, false /* archived */)); @@ -858,6 +869,7 @@ handle (request& rq, response& rs) nmax -= nt; config = c->name; + target = c->target; ncur -= bld_prep_query.execute_value (); } } @@ -882,6 +894,7 @@ handle (request& rq, response& rs) // 4: toolchain name // 5: toolchain version (descending) // 6: configuration name + // 7: configuration target // // Prepare the build package prepared query. // @@ -941,7 +954,7 @@ handle (request& rq, response& rs) // filter out the private tenants. // build_query<package_build> ( - &conf_names, bld_params, tn, false /* archived */)); + &conf_ids, bld_params, tn, false /* archived */)); prep_bld_query bld_prep_query ( conn->prepare_query<package_build> ("mod-builds-build-query", bq)); @@ -982,7 +995,9 @@ handle (request& rq, response& rs) for (const auto& ct: config_toolchains) { - auto i (build_conf_map_->find (ct.configuration.c_str ())); + auto i (build_conf_map_->find (build_config_id {ct.configuration, + ct.target})); + assert (i != build_conf_map_->end ()); if (!exclude (p->builds, p->constraints, *i->second)) @@ -997,8 +1012,10 @@ handle (request& rq, response& rs) { const build& b (*pb.build); - unbuilt_configs.erase ({ - b.id.configuration, b.toolchain_name, b.toolchain_version}); + unbuilt_configs.erase (config_toolchain {b.id.configuration, + b.id.target, + b.toolchain_name, + b.toolchain_version}); } // Print unbuilt package configurations. @@ -1011,9 +1028,6 @@ handle (request& rq, response& rs) continue; } - auto i (build_conf_map_->find (ct.configuration.c_str ())); - assert (i != build_conf_map_->end ()); - s << TABLE(CLASS="proplist build") << TBODY << TR_NAME (id.name, string (), root, id.tenant) @@ -1022,7 +1036,7 @@ handle (request& rq, response& rs) string (ct.toolchain_name) + '-' + ct.toolchain_version.string ()) << TR_VALUE ("config", ct.configuration) - << TR_VALUE ("target", i->second->target.string ()); + << TR_VALUE ("target", ct.target.string ()); // In the global view mode add the tenant builds link. Note that // the global view (and the link) makes sense only in the diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 4a1f0c1..8ea0b01 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -568,10 +568,13 @@ handle (request& rq, response& rs) toolchains.emplace_back (move (t.name), move (t.version)); } - // Collect configuration names and unbuilt configurations, skipping those - // that are hidden or excluded by the package. + // Compose the configuration filtering sub-query and collect unbuilt + // configurations, skipping those that are hidden or excluded by the + // package. // - cstrings conf_names; + using query = query<build>; + + query sq (false); set<config_toolchain> unbuilt_configs; for (const auto& c: *build_conf_map_) @@ -580,26 +583,24 @@ handle (request& rq, response& rs) if (belongs (cfg, "all") && !exclude (cfg)) { - conf_names.push_back (c.first); + const build_config_id& id (c.first); + + sq = sq || (query::id.configuration == id.name && + query::id.target == id.target); // Note: we will erase built configurations from the unbuilt // configurations set later (see below). // for (const auto& t: toolchains) - unbuilt_configs.insert ({cfg.name, t.first, t.second}); + unbuilt_configs.insert ( + config_toolchain {cfg.name, cfg.target, t.first, t.second}); } } // Print the package built configurations in the time-descending order. // - using query = query<build>; - for (auto& b: build_db_->query<build> ( - (query::id.package == pkg->id && - - query::id.configuration.in_range (conf_names.begin (), - conf_names.end ())) + - + (query::id.package == pkg->id && sq) + "ORDER BY" + query::timestamp + "DESC")) { string ts (butl::to_string (b.timestamp, @@ -630,9 +631,10 @@ handle (request& rq, response& rs) // While at it, erase the built configuration from the unbuilt // configurations set. // - unbuilt_configs.erase ({b.id.configuration, - b.toolchain_name, - b.toolchain_version}); + unbuilt_configs.erase (config_toolchain {b.configuration, + b.target, + b.toolchain_name, + b.toolchain_version}); } // Print the package unbuilt configurations with the following sort @@ -644,17 +646,13 @@ handle (request& rq, response& rs) // for (const auto& ct: unbuilt_configs) { - auto i (build_conf_map_->find (ct.configuration.c_str ())); - assert (i != build_conf_map_->end ()); - s << TABLE(CLASS="proplist build") << TBODY << TR_VALUE ("toolchain", ct.toolchain_name + '-' + ct.toolchain_version.string ()) << TR_VALUE ("config", - ct.configuration + " / " + - i->second->target.string ()) + ct.configuration + " / " + ct.target.string ()) << TR_VALUE ("result", "unbuilt") << ~TBODY << ~TABLE; diff --git a/mod/module.cli b/mod/module.cli index bd61873..3ac8b30 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -769,6 +769,10 @@ namespace brep // string configuration | cf; + // Package build target. + // + string target | tg; + // Toolchain name. // string toolchain_name | tn; diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx index 7c20efe..0d5545e 100644 --- a/monitor/monitor.cxx +++ b/monitor/monitor.cxx @@ -14,7 +14,6 @@ #include <odb/pgsql/database.hxx> #include <libbutl/pager.hxx> -#include <libbutl/utility.hxx> // compare_c_string #include <libbbot/build-config.hxx> @@ -52,9 +51,10 @@ namespace brep // 1: toolchain name // 2: toolchain version (descending) // 3: configuration name - // 4: tenant - // 5: package name - // 6: package version (descending) + // 4: configuration target + // 5: tenant + // 6: package name + // 7: package version (descending) // struct compare_delay { @@ -71,6 +71,9 @@ namespace brep if (int r = x->configuration.compare (y->configuration)) return r < 0; + if (int r = x->target.compare (y->target)) + return r < 0; + if (int r = x->tenant.compare (y->tenant)) return r < 0; @@ -137,25 +140,29 @@ namespace brep // Group the printed delays by toolchain and configuration. // - const string* toolchain_name (nullptr); - const version* toolchain_version (nullptr); - const string* configuration (nullptr); + const string* toolchain_name (nullptr); + const version* toolchain_version (nullptr); + const string* configuration (nullptr); + const target_triplet* target (nullptr); size_t config_reported_delay_count (0); size_t config_total_delay_count (0); auto brief_config = [&configuration, + &target, &config_reported_delay_count, &config_total_delay_count, total] () { if (configuration != nullptr) { + assert (target != nullptr); + // Only print configurations with delays that needs to be reported. // if (config_reported_delay_count != 0) { - cerr << " " << *configuration << " (" + cerr << " " << *configuration << '/' << *target << " (" << config_reported_delay_count; if (total) @@ -200,23 +207,27 @@ namespace brep toolchain_name = &d->toolchain_name; toolchain_version = &d->toolchain_version; configuration = nullptr; + target = nullptr; } // Print the configuration, if changed. // - if (configuration == nullptr || d->configuration != *configuration) + if (configuration == nullptr || + d->configuration != *configuration || + d->target != *target) { if (full) { if (configuration != nullptr) cerr << endl; - cerr << " " << d->configuration << endl; + cerr << " " << d->configuration << '/' << d->target << endl; } else brief_config (); configuration = &d->configuration; + target = &d->target; } // Print the delayed build package in the full report mode and count @@ -494,13 +505,11 @@ namespace brep // if (ops.clean ()) { - using config_map = map<const char*, - const build_config*, - compare_c_string>; + using config_map = map<build_config_id, const build_config*>; config_map conf_map; for (const build_config& c: configs) - conf_map[c.name.c_str ()] = &c; + conf_map[build_config_id {c.name, c.target}] = &c; // Prepare the build delay prepared query. // @@ -524,6 +533,7 @@ namespace brep order_by_version (query::id.package.version, false /* first */) + "," + query::id.configuration + "," + + query::id.target + "," + query::id.toolchain_name + order_by_version (query::id.toolchain_version, false /* first */) + @@ -569,7 +579,8 @@ namespace brep // // Check that the build configuration is still present. // - (ci = conf_map.find (d.configuration.c_str ())) == + (ci = conf_map.find (build_config_id {d.configuration, + d.target})) == conf_map.end ()); // Check that the package still present, is buildable and doesn't @@ -647,10 +658,12 @@ namespace brep using prep_bquery = prepared_query<package_build>; build_id id; + const auto& bid (bquery::build::id); bquery bq ((equal<package_build> (bid.package, id.package) && bid.configuration == bquery::_ref (id.configuration) && + bid.target == bquery::_ref (id.target) && bid.toolchain_name == bquery::_ref (id.toolchain_name)) + "ORDER BY" + bquery::build::soft_timestamp + "DESC, " + @@ -803,7 +816,7 @@ namespace brep for (const pair<string, version>& t: toolchains) { - id = build_id (p->id, c.name, t.first, t.second); + id = build_id (p->id, c.name, c.target, t.first, t.second); // If the toolchain version is unspecified then search for the // latest build across all toolchain versions and search for a @@ -868,6 +881,7 @@ namespace brep move (id.package.name), p->version, move (id.configuration), + move (id.target), move (id.toolchain_name), t.second, pts); |