diff options
Diffstat (limited to 'monitor')
| -rw-r--r-- | monitor/monitor.cli | 36 | ||||
| -rw-r--r-- | monitor/monitor.cxx | 974 |
2 files changed, 621 insertions, 389 deletions
diff --git a/monitor/monitor.cli b/monitor/monitor.cli index edfc004..3a58a1d 100644 --- a/monitor/monitor.cli +++ b/monitor/monitor.cli @@ -60,18 +60,26 @@ namespace brep { "\h|OPTIONS|" - std::size_t --build-timeout + std::size_t --soft-rebuild-timeout { "<seconds>", - "Time to wait (in seconds) before considering a package build as + "Time to wait (in seconds) before considering a package soft (re)build as delayed. If unspecified, it is the sum of the package rebuild timeout - (normal rebuild timeout if the alternative timeout is unspecified and - the maximum of two otherwise) and the build result timeout (see the - \cb{build-normal-rebuild-timeout}, \cb{build-alt-rebuild-*}, and - \cb{build-result-timeout} \c{brep} module configuration options - for details). - - Note that a package that was not built before it was archived is + (soft rebuild timeout if the alternative timeout is unspecified and + the maximum of two otherwise) and the build result timeout (see + the \cb{build-soft-rebuild-timeout}, \cb{build-alt-soft-rebuild-*}, + and \cb{build-result-timeout} \cb{brep} module configuration options + for details). The special zero value disables monitoring of soft + rebuilds. + + Note that if both soft and hard rebuilds are disabled in the + \cb{brep} module configuration, then \cb{brep-monitor} is unable to + come up with a reasonable build timeout on its own. In this case, to + monitor the initial package build delays, you may need to specify + either \cb{--soft-rebuild-timeout} or \cb{--hard-rebuild-timeout} + explicitly. + + Also note that a package that was not built before it was archived is always considered as delayed. However, to distinguish this case from a situation where a package was archived before a configuration have been added, \cb{brep-monitor} needs to observe the package as @@ -81,6 +89,16 @@ namespace brep timeout." } + std::size_t --hard-rebuild-timeout + { + "<seconds>", + "Time to wait (in seconds) before considering a package hard (re)build + as delayed. If unspecified, it is calculated in the same way as for + \cb{--soft-rebuild-timeout} but using the + \cb{build-hard-rebuild-timeout} and \cb{build-alt-hard-rebuild-*} + \cb{brep} module configuration options." + } + std::size_t --report-timeout { "<seconds>", diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx index bbab0a5..2f49f81 100644 --- a/monitor/monitor.cxx +++ b/monitor/monitor.cxx @@ -5,7 +5,6 @@ #include <set> #include <chrono> #include <iostream> -#include <algorithm> // find_if() #include <odb/database.hxx> #include <odb/transaction.hxx> @@ -13,10 +12,7 @@ #include <odb/pgsql/database.hxx> -#include <libbutl/pager.mxx> -#include <libbutl/utility.mxx> // compare_c_string - -#include <libbbot/build-config.hxx> +#include <libbutl/pager.hxx> #include <libbrep/build.hxx> #include <libbrep/common.hxx> @@ -25,14 +21,13 @@ #include <libbrep/build-package-odb.hxx> #include <libbrep/database-lock.hxx> -#include <mod/build-config.hxx> +#include <mod/build-target-config.hxx> #include <monitor/module-options.hxx> #include <monitor/monitor-options.hxx> using namespace std; using namespace butl; -using namespace bbot; using namespace odb::core; namespace brep @@ -41,6 +36,227 @@ namespace brep // struct failed {}; + // We will collect and report build delays as separate steps not to hold + // database locks while printing to stderr. Also we need to order delays + // properly, so while printing reports we could group delays by toolchain + // and target configuration. + // + // To achieve that, we will iterate through all possible package builds + // creating the list of delays with the following sort priority: + // + // 1: toolchain name + // 2: toolchain version (descending) + // 3: target configuration name + // 4: target + // 5: tenant + // 6: package name + // 7: package version (descending) + // 8: package configuration name + // + struct compare_delay + { + bool + operator() (const shared_ptr<const build_delay>& x, + const shared_ptr<const build_delay>& y) const + { + if (int r = x->toolchain_name.compare (y->toolchain_name)) + return r < 0; + + if (int r = x->toolchain_version.compare (y->toolchain_version)) + return r > 0; + + if (int r = x->target_config_name.compare (y->target_config_name)) + return r < 0; + + if (int r = x->target.compare (y->target)) + return r < 0; + + if (int r = x->tenant.compare (y->tenant)) + return r < 0; + + if (int r = x->package_name.compare (y->package_name)) + return r < 0; + + if (int r = x->package_version.compare (y->package_version)) + return r > 0; + + return x->package_config_name.compare (y->package_config_name) < 0; + } + }; + + // The ordered list of delays to report. + // + class delay_report + { + public: + // Note that in the brief mode we also need to print the total number of + // delays (reported or not) per target configuration. Thus, we add all + // delays to the report object, marking them if we need to report them or + // not. + // + void + add_delay (shared_ptr<build_delay>, bool report); + + bool + empty () const {return reported_delay_count_ == 0;} + + // In the brief mode (if full is false) print the number of reported/total + // (if total is true) delayed package configuration builds per target + // configuration rather than the package configurations themselves. + // + void + print (const char* header, bool total, bool full) const; + + private: + // Maps delays to the report flag. + // + map<shared_ptr<const build_delay>, bool, compare_delay> delays_; + size_t reported_delay_count_ = 0; + }; + + void delay_report:: + add_delay (shared_ptr<build_delay> delay, bool report) + { + delays_.emplace (move (delay), report); + + if (report) + ++reported_delay_count_; + } + + void delay_report:: + print (const char* header, bool total, bool full) const + { + if (empty ()) + return; + + cerr << header << " (" << reported_delay_count_; + + if (total) + cerr << '/' << delays_.size (); + + cerr << "):" << endl; + + // Group the printed delays by toolchain and target configuration. + // + const string* toolchain_name (nullptr); + const version* toolchain_version (nullptr); + const string* target_config_name (nullptr); + const target_triplet* target (nullptr); + + size_t config_reported_delay_count (0); + size_t config_total_delay_count (0); + + auto brief_config = [&target_config_name, + &target, + &config_reported_delay_count, + &config_total_delay_count, + total] () + { + if (target_config_name != nullptr) + { + assert (target != nullptr); + + // Only print configurations with delays that needs to be reported. + // + if (config_reported_delay_count != 0) + { + cerr << " " << *target_config_name << '/' << *target << " (" + << config_reported_delay_count; + + if (total) + cerr << '/' << config_total_delay_count; + + cerr << ')' << endl; + } + + config_reported_delay_count = 0; + config_total_delay_count = 0; + } + }; + + for (const auto& dr: delays_) + { + bool report (dr.second); + + if (full && !report) + continue; + + const shared_ptr<const build_delay>& d (dr.first); + + // Print the toolchain, if changed. + // + if (toolchain_name == nullptr || + d->toolchain_name != *toolchain_name || + d->toolchain_version != *toolchain_version) + { + if (!full) + brief_config (); + + if (toolchain_name != nullptr) + cerr << endl; + + cerr << " " << d->toolchain_name; + + if (!d->toolchain_version.empty ()) + cerr << "/" << d->toolchain_version; + + cerr << endl; + + toolchain_name = &d->toolchain_name; + toolchain_version = &d->toolchain_version; + target_config_name = nullptr; + target = nullptr; + } + + // Print the configuration, if changed. + // + if (target_config_name == nullptr || + d->target_config_name != *target_config_name || + d->target != *target) + { + if (full) + { + if (target_config_name != nullptr) + cerr << endl; + + cerr << " " << d->target_config_name << '/' << d->target << endl; + } + else + brief_config (); + + target_config_name = &d->target_config_name; + target = &d->target; + } + + // Print the delayed build package configuration in the full report mode + // and count configuration builds otherwise. + // + if (full) + { + // We can potentially extend this information with the archived flag + // or the delay duration. + // + cerr << " " << d->package_name << '/' << d->package_version + << ' ' << d->package_config_name; + + if (!d->tenant.empty ()) + cerr << ' ' << d->tenant; + + cerr << endl; + } + else + { + if (report) + ++config_reported_delay_count; + + ++config_total_delay_count; + } + } + + if (!full) + brief_config (); + } + static const char* help_info ( " info: run 'brep-monitor --help' for more information"); @@ -141,12 +357,24 @@ namespace brep return 1; } - if (mod_ops.build_alt_rebuild_start_specified () != - mod_ops.build_alt_rebuild_stop_specified ()) + auto bad_alt = [&f] (const char* what) + { + cerr << "build-alt-" << what << "-rebuild-start and build-alt-" + << what << "-rebuild-stop configuration options must both be " + << "either specified or not in '" << f << "'" << endl; + }; + + if (mod_ops.build_alt_hard_rebuild_start_specified () != + mod_ops.build_alt_hard_rebuild_stop_specified ()) + { + bad_alt("hard"); + return 1; + } + + if (mod_ops.build_alt_soft_rebuild_start_specified () != + mod_ops.build_alt_soft_rebuild_stop_specified ()) { - cerr << "build-alt-rebuild-start and build-alt-rebuild-stop " - << "configuration options must both be either specified or not " - << "in '" << f << "'" << endl; + bad_alt("soft"); return 1; } } @@ -221,11 +449,11 @@ namespace brep return 0; } - build_configs configs; + build_target_configs configs; try { - configs = parse_buildtab (mod_ops.build_config ()); + configs = bbot::parse_buildtab (mod_ops.build_config ()); } catch (const tab_parsing& e) { @@ -276,13 +504,12 @@ namespace brep // if (ops.clean ()) { - using config_map = map<const char*, - const build_config*, - compare_c_string>; + using config_map = map<build_target_config_id, + const build_target_config*>; config_map conf_map; - for (const build_config& c: configs) - conf_map[c.name.c_str ()] = &c; + for (const build_target_config& c: configs) + conf_map[build_target_config_id {c.target, c.name}] = &c; // Prepare the build delay prepared query. // @@ -301,15 +528,17 @@ namespace brep size_t offset (0); query q ("ORDER BY" + - query::id.package.tenant + "," + - query::id.package.name + + query::id.package.tenant + "," + + query::id.package.name + order_by_version (query::id.package.version, false /* first */) + "," + - query::id.configuration + "," + - query::id.toolchain_name + + query::id.target + "," + + query::id.target_config_name + "," + + query::id.package_config_name + "," + + query::id.toolchain_name + order_by_version (query::id.toolchain_version, - false /* first */) + - "OFFSET" + query::_ref (offset) + "LIMIT 100"); + false /* first */) + + "OFFSET" + query::_ref (offset) + "LIMIT 2000"); connection_ptr conn (db.connection ()); @@ -351,7 +580,9 @@ namespace brep // // Check that the build configuration is still present. // - (ci = conf_map.find (d.configuration.c_str ())) == + (ci = conf_map.find ( + build_target_config_id {d.target, + d.target_config_name})) == conf_map.end ()); // Check that the package still present, is buildable and doesn't @@ -365,12 +596,23 @@ namespace brep p = db.find<build_package> (pid); } - cleanup = (p == nullptr || - !p->buildable || - exclude (p->builds, - p->constraints, - *ci->second, - configs.class_inheritance_map)); + const build_package_config* pc (p != nullptr + ? find (d.package_config_name, + p->configs) + : nullptr); + + cleanup = (pc == nullptr || !p->buildable); + + if (!cleanup) + { + db.load (*p, p->constraints_section); + + cleanup = exclude (*pc, + p->builds, + p->constraints, + *ci->second, + configs.class_inheritance_map); + } } if (cleanup) @@ -384,59 +626,16 @@ namespace brep } } - // Collect and report delays as separate steps not to hold database locks - // while printing to stderr. Also we need to properly order delays for - // printing. - // - // Iterate through all possible package builds creating the list of delays - // with the following sort priority: - // - // 1: toolchain name - // 2: toolchain version (descending) - // 3: configuration name - // 4: tenant - // 5: package name - // 6: package version (descending) - // - // Such ordering will allow us to group build delays by toolchain and - // configuration while printing the report. - // - struct compare_delay - { - bool - operator() (const shared_ptr<const build_delay>& x, - const shared_ptr<const build_delay>& y) const - { - if (int r = x->toolchain_name.compare (y->toolchain_name)) - return r < 0; - - if (int r = x->toolchain_version.compare (y->toolchain_version)) - return r > 0; - - if (int r = x->configuration.compare (y->configuration)) - return r < 0; - - if (int r = x->tenant.compare (y->tenant)) - return r < 0; - - if (int r = x->package_name.compare (y->package_name)) - return r < 0; - - return x->package_version.compare (y->package_version) > 0; - } - }; - - size_t reported_delay_count (0); - size_t total_delay_count (0); - - set<shared_ptr<const build_delay>, compare_delay> delays; + delay_report hard_delays_report; + delay_report soft_delays_report; + set<shared_ptr<const build_delay>, compare_delay> update_delays; { connection_ptr conn (db.connection ()); // Prepare the buildable package prepared query. // - // Query buildable packages in chunks in order not to hold locks for - // too long. + // Query buildable packages in chunks in order not to hold locks for too + // long. // using pquery = query<buildable_package>; using prep_pquery = prepared_query<buildable_package>; @@ -456,80 +655,157 @@ namespace brep conn->prepare_query<buildable_package> ("buildable-package-query", pq)); - // Prepare the package build prepared query. + // Prepare the package configuration build prepared queries. // + using bquery = query<build>; + using prep_bquery = prepared_query<build>; + + build_id id; + // This query will only be used for toolchains that have no version // specified on the command line to obtain the latest completed build // across all toolchain versions, if present, and the latest incomplete // build otherwise. // - using bquery = query<package_build>; - using prep_bquery = prepared_query<package_build>; - - build_id id; - const auto& bid (bquery::build::id); - - bquery bq ((equal<package_build> (bid.package, id.package) && - bid.configuration == bquery::_ref (id.configuration) && - bid.toolchain_name == bquery::_ref (id.toolchain_name)) + - "ORDER BY" + - bquery::build::completion_timestamp + "DESC, " + - bquery::build::timestamp + "DESC" + - "LIMIT 1"); - - prep_bquery pbq ( - conn->prepare_query<package_build> ("package-build-query", bq)); + // Why don't we pick the latest toolchain version? We don't want to + // stuck with it on the toolchain rollback. Instead we prefer the + // toolchain that built the package last and if there are none, pick the + // one for which the build task was issued last. + // + // @@ TMP Check if we can optimize this query by adding index for + // soft_timestamp and/or by setting enable_nestloop=off (or some + // such) as we do in mod/mod-builds.cxx. + // + bquery lbq ((equal<build> (bquery::id, + id, + false /* toolchain_version */) && + bquery::state != "queued") + + "ORDER BY" + + bquery::soft_timestamp + "DESC, " + + bquery::timestamp + "DESC" + + "LIMIT 1"); + + prep_bquery plbq ( + conn->prepare_query<build> ("package-latest-build-query", lbq)); + + // This query will only be used to retrieve a specific build by id. + // + bquery bq (equal<build> (bquery::id, id) && bquery::state != "queued"); + prep_bquery pbq (conn->prepare_query<build> ("package-build-query", bq)); - duration build_timeout; + timestamp now (system_clock::now ()); - // If the build timeout is not specified explicitly, then calculate it - // as the sum of the package rebuild timeout (normal rebuild timeout if - // the alternative timeout is unspecified and the maximum of two - // otherwise) and the build result timeout. + // Calculate the build/rebuild expiration time, based on the respective + // --{soft,hard}-rebuild-timeout monitor options and the + // build-{soft,hard}-rebuild-timeout and + // build-alt-{soft,hard}-rebuild-{start,stop,timeout} brep module + // configuration options. + // + // If the --*-rebuild-timeout monitor option is zero or is not specified + // and the respective build-*-rebuild-timeout brep's configuration + // option is zero, then return timestamp_unknown to indicate 'never + // expire'. Note that this value is less than any build timestamp value, + // including timestamp_nonexistent. // - if (!ops.build_timeout_specified ()) + // NOTE: there is a similar code in mod/mod-build-task.cxx. + // + auto build_expiration = [&now, &mod_ops] ( + optional<size_t> rebuild_timeout, + const optional<pair<duration, duration>>& alt_interval, + optional<size_t> alt_timeout, + size_t normal_timeout) { - duration normal_rebuild_timeout ( - chrono::seconds (mod_ops.build_normal_rebuild_timeout ())); + duration t; - if (mod_ops.build_alt_rebuild_start_specified ()) + // If the rebuild timeout is not specified explicitly, then calculate + // it as the sum of the package rebuild timeout (normal rebuild + // timeout if the alternative timeout is unspecified and the maximum + // of two otherwise) and the build result timeout. + // + if (!rebuild_timeout) { - // Calculate the alternative rebuild timeout as the time interval - // lenght, unless it is specified explicitly. - // - if (!mod_ops.build_alt_rebuild_timeout_specified ()) + if (normal_timeout == 0) + return timestamp_unknown; + + chrono::seconds nt (normal_timeout); + + if (alt_interval) { - const duration& start (mod_ops.build_alt_rebuild_start ()); - const duration& stop (mod_ops.build_alt_rebuild_stop ()); + // Calculate the alternative timeout, unless it is specified + // explicitly. + // + if (!alt_timeout) + { + const duration& start (alt_interval->first); + const duration& stop (alt_interval->second); + + // Note that if the stop time is less than the start time then + // the interval extends through the midnight. + // + t = start <= stop ? (stop - start) : ((24h - start) + stop); + + // If the normal rebuild time out is greater than 24 hours, then + // increase the default alternative timeout by (normal - 24h) + // (see build-alt-soft-rebuild-timeout configuration option for + // details). + // + if (nt > 24h) + t += nt - 24h; + } + else + t = chrono::seconds (*alt_timeout); - // Note that if the stop time is less than the start time then the - // interval extends through the midnight. + // Take the maximum of the alternative and normal rebuild + // timeouts. // - build_timeout = start <= stop - ? stop - start - : (24h - start) + stop; + if (t < nt) + t = nt; } else - build_timeout = - chrono::seconds (mod_ops.build_alt_rebuild_timeout ()); + t = nt; - // Take the maximum of the alternative and normal rebuild timeouts. + // Summarize the rebuild and build result timeouts. // - if (build_timeout < normal_rebuild_timeout) - build_timeout = normal_rebuild_timeout; + t += chrono::seconds (mod_ops.build_result_timeout ()); } else - build_timeout = normal_rebuild_timeout; + { + if (*rebuild_timeout == 0) + return timestamp_unknown; - // Summarize the rebuild and build result timeouts. - // - build_timeout += chrono::seconds (mod_ops.build_result_timeout ()); - } - else - build_timeout = chrono::seconds (ops.build_timeout ()); + t = chrono::seconds (*rebuild_timeout); + } - timestamp now (system_clock::now ()); - timestamp build_expiration (now - build_timeout); + return now - t; + }; + + timestamp hard_rebuild_expiration ( + build_expiration ( + (ops.hard_rebuild_timeout_specified () + ? ops.hard_rebuild_timeout () + : optional<size_t> ()), + (mod_ops.build_alt_hard_rebuild_start_specified () + ? make_pair (mod_ops.build_alt_hard_rebuild_start (), + mod_ops.build_alt_hard_rebuild_stop ()) + : optional<pair<duration, duration>> ()), + (mod_ops.build_alt_hard_rebuild_timeout_specified () + ? mod_ops.build_alt_hard_rebuild_timeout () + : optional<size_t> ()), + mod_ops.build_hard_rebuild_timeout ())); + + timestamp soft_rebuild_expiration ( + build_expiration ( + (ops.soft_rebuild_timeout_specified () + ? ops.soft_rebuild_timeout () + : optional<size_t> ()), + (mod_ops.build_alt_soft_rebuild_start_specified () + ? make_pair (mod_ops.build_alt_soft_rebuild_start (), + mod_ops.build_alt_soft_rebuild_stop ()) + : optional<pair<duration, duration>> ()), + (mod_ops.build_alt_soft_rebuild_timeout_specified () + ? mod_ops.build_alt_soft_rebuild_timeout () + : optional<size_t> ()), + mod_ops.build_soft_rebuild_timeout ())); timestamp report_expiration ( now - chrono::seconds (ops.report_timeout ())); @@ -548,139 +824,190 @@ namespace brep for (auto& bp: bps) { - shared_ptr<build_package> p (db.load<build_package> (bp.id)); + shared_ptr<build_package>& p (bp.package); - for (const build_config& c: configs) - { - if (exclude (p->builds, - p->constraints, - c, - configs.class_inheritance_map)) - continue; + db.load (*p, p->constraints_section); - for (const pair<string, version>& t: toolchains) + for (const build_package_config& pc: p->configs) + { + for (const build_target_config& tc: configs) { - id = build_id (p->id, c.name, t.first, t.second); - - // If the toolchain version is unspecified then search for the - // latest build across all toolchain versions and search for a - // specific build otherwise. + // Note that we also don't build a package configuration if we + // are unable to assign all the required auxiliary machines + // for the build (see mod/mod-build-task.cxx for details). + // That means that we will also report delays which happen due + // to such an inability, which can potentially be not only + // because of the infrastructural problem but also because of + // an error in the package manifest (build auxiliary + // configuration pattern doesn't match any machine + // configuration anymore, etc). It doesn't seem easy to + // distinguish here which type of problem causes a delay. + // Thus, for now let's wait and see if it ever becomes a + // problem. // - shared_ptr<build> b; - - if (id.toolchain_version.empty ()) + if (exclude (pc, + p->builds, + p->constraints, + tc, + configs.class_inheritance_map)) + continue; + + for (const pair<string, version>& t: toolchains) { - auto pbs (pbq.execute ()); + id = build_id (p->id, + tc.target, tc.name, + pc.name, + t.first, t.second); + + // If the toolchain version is unspecified then search for + // the latest build across all toolchain versions and search + // for a specific build otherwise. + // + shared_ptr<build> b (id.toolchain_version.empty () + ? plbq.execute_one () + : pbq.execute_one ()); + + // Note that we consider a build as delayed if it is not + // completed in the expected timeframe. So even if the build + // task have been issued recently we may still consider the + // build as delayed. + // + timestamp bht (b != nullptr + ? b->hard_timestamp + : timestamp_nonexistent); - if (!pbs.empty ()) - b = move (pbs.begin ()->build); - } - else - b = db.find<build> (id); + timestamp bst (b != nullptr + ? b->soft_timestamp + : timestamp_nonexistent); - // Note that we consider a build as delayed if it is not - // completed in the expected timeframe. So even if the build - // task have been issued recently we may still consider the - // build as delayed. - // - timestamp bct (b != nullptr - ? b->completion_timestamp - : timestamp_nonexistent); + // Create the delay object to record a timestamp when the + // package configuration build could have potentially been + // started, unless it already exists. + // + shared_ptr<build_delay> d (db.find<build_delay> (id)); - // Create the delay object to record a timestamp when the - // package build could have potentially been started, unless - // it already exists. - // - shared_ptr<build_delay> d (db.find<build_delay> (id)); + if (d == nullptr) + { + // If the archived package has no build nor build delay + // for this configuration, then we assume that the + // configuration was added after the package tenant has + // been archived and so the package could have never been + // built for this configuration. Thus, we don't consider + // this build as delayed and so skip it. + // + if (bp.archived && b == nullptr) + continue; + + // Use the build hard, soft, or status change timestamp + // (see the timestamps description for their ordering + // information) as the build delay tracking starting point + // and fallback to the current time if there is no build + // yet. + // + timestamp pts (b == nullptr ? now : + bht != timestamp_nonexistent ? bht : + bst != timestamp_nonexistent ? bst : + b->timestamp); + + d = make_shared<build_delay> (move (id.package.tenant), + move (id.package.name), + p->version, + move (id.target), + move (id.target_config_name), + move (id.package_config_name), + move (id.toolchain_name), + t.second, + pts); + db.persist (d); + } - if (d == nullptr) - { - // If the archived package has no build nor build delay - // for this configuration, then we assume that the - // configuration was added after the package tenant has - // been archived and so the package could have never been - // built for this configuration. Thus, we don't consider - // this build as delayed and so skip it. + // Handle package builds differently based on their tenant's + // archive status. // - if (bp.archived && b == nullptr) - continue; - - // Use the build completion or build status change - // timestamp, whichever is earlier, as the build delay - // tracking starting point and fallback to the current time - // if there is no build yet. + // If the package is not archived then consider it as + // delayed if it is not (re-)built by the expiration + // time. Otherwise, consider it as delayed if it is unbuilt. // - timestamp pts ( - b == nullptr ? now : - bct != timestamp_nonexistent && bct < b->timestamp ? bct : - b->timestamp); - - d = make_shared<build_delay> (move (id.package.tenant), - move (id.package.name), - p->version, - move (id.configuration), - move (id.toolchain_name), - t.second, - pts); - db.persist (d); - } - - // Handle package builds differently based on their tenant's - // archive status. - // - // If the package is not archived then consider it as delayed - // if it is not (re-)built by the expiration time. Otherwise, - // consider it as delayed if it is unbuilt. - // - bool delayed; - - if (!bp.archived) - { - timestamp bts (bct != timestamp_nonexistent - ? bct - : d->package_timestamp); + // We also don't need to report an unbuilt archived package + // twice, as both soft and hard build delays. + // + bool hard_delayed; + bool soft_delayed; - delayed = (bts <= build_expiration); - } - else - delayed = (bct == timestamp_nonexistent); + if (!bp.archived) + { + auto delayed = [&d] (timestamp bt, timestamp be) + { + timestamp t (bt != timestamp_nonexistent + ? bt + : d->package_timestamp); + return t <= be; + }; + + hard_delayed = delayed (bht, hard_rebuild_expiration); + soft_delayed = delayed (bst, soft_rebuild_expiration); + } + else + { + hard_delayed = (bst == timestamp_nonexistent); + soft_delayed = false; + } - if (delayed) - { - // If the report timeout is zero then report the delay - // unconditionally. Otherwise, report the active package - // build delay if the report timeout is expired and the - // archived package build delay if it was never reported. - // Note that fixing the building infrastructure won't help - // building an archived package, so reporting its build - // delays repeatedly is meaningless. + // Add hard/soft delays to the respective reports and + // collect the delay for update, if it is reported. // - if (ops.report_timeout () == 0 || - (!bp.archived - ? d->report_timestamp <= report_expiration - : d->report_timestamp == timestamp_nonexistent)) + // Note that we update the delay objects persistent state + // later, after we successfully print the reports. + // + bool reported (false); + + if (hard_delayed) { - // Note that we update the delay objects persistent state - // later, after we successfully print the report. + // If the report timeout is zero then report the delay + // unconditionally. Otherwise, report the active package + // build delay if the report timeout is expired and the + // archived package build delay if it was never reported. + // Note that fixing the building infrastructure won't help + // building an archived package, so reporting its build + // delays repeatedly is meaningless. // - d->report_timestamp = now; - delays.insert (move (d)); + bool report ( + ops.report_timeout () == 0 || + (!bp.archived + ? d->report_hard_timestamp <= report_expiration + : d->report_hard_timestamp == timestamp_nonexistent)); + + if (report) + { + d->report_hard_timestamp = now; + reported = true; + } - ++reported_delay_count; + hard_delays_report.add_delay (d, report); } - // - // In the brief mode also collect unreported delays to - // deduce and print the total number of delays per - // configuration. Mark such delays with the - // timestamp_nonexistent report timestamp. - // - else if (!ops.full_report ()) + + if (soft_delayed) { - d->report_timestamp = timestamp_nonexistent; - delays.insert (move (d)); + bool report (ops.report_timeout () == 0 || + d->report_soft_timestamp <= report_expiration); + + if (report) + { + d->report_soft_timestamp = now; + reported = true; + } + + soft_delays_report.add_delay (d, report); } - ++total_delay_count; + // If we don't consider the report timestamps for reporting + // delays, it seems natural not to update these timestamps + // either. Note that reporting all delays and still updating + // the report timestamps can be achieved by specifying the + // zero report timeout. + // + if (reported && ops.report_timeout_specified ()) + update_delays.insert (move (d)); } } } @@ -691,161 +1018,48 @@ namespace brep } } - // Report package build delays, if any. + // Print delay reports, if not empty. // - if (reported_delay_count != 0) + if (!hard_delays_report.empty () || !soft_delays_report.empty ()) try { - // Print the report. - // cerr.exceptions (ostream::badbit | ostream::failbit); // Don't print the total delay count if the report timeout is zero since // all delays are reported in this case. // - bool print_total_delay_count (ops.report_timeout () != 0); - - cerr << "Package build delays (" << reported_delay_count; + bool total (ops.report_timeout () != 0); - if (print_total_delay_count) - cerr << '/' << total_delay_count; - - cerr << "):" << endl; - - // Group the printed delays by toolchain and configuration. - // - const string* toolchain_name (nullptr); - const version* toolchain_version (nullptr); - const string* configuration (nullptr); + hard_delays_report.print ("Package hard rebuild delays", + total, + ops.full_report ()); - // In the brief report mode print the number of reported/total delayed - // package builds per configuration rather than the packages themselves. + // Separate reports with an empty line. // - size_t config_reported_delay_count (0); - size_t config_total_delay_count (0); + if (!hard_delays_report.empty () && !soft_delays_report.empty ()) + cerr << endl; - auto brief_config = [&configuration, - &config_reported_delay_count, - &config_total_delay_count, - print_total_delay_count] () - { - if (configuration != nullptr) - { - // Only print configurations with delays that needs to be reported. - // - if (config_reported_delay_count != 0) - { - cerr << " " << *configuration << " (" - << config_reported_delay_count; - - if (print_total_delay_count) - cerr << '/' << config_total_delay_count; - - cerr << ')' << endl; - } - - config_reported_delay_count = 0; - config_total_delay_count = 0; - } - }; - - for (shared_ptr<const build_delay> d: delays) - { - // Print the toolchain, if changed. - // - if (toolchain_name == nullptr || - d->toolchain_name != *toolchain_name || - d->toolchain_version != *toolchain_version) - { - if (!ops.full_report ()) - brief_config (); - - if (toolchain_name != nullptr) - cerr << endl; - - cerr << " " << d->toolchain_name; - - if (!d->toolchain_version.empty ()) - cerr << "/" << d->toolchain_version; - - cerr << endl; - - toolchain_name = &d->toolchain_name; - toolchain_version = &d->toolchain_version; - configuration = nullptr; - } - - // Print the configuration, if changed. - // - if (configuration == nullptr || d->configuration != *configuration) - { - if (ops.full_report ()) - { - if (configuration != nullptr) - cerr << endl; - - cerr << " " << d->configuration << endl; - } - else - brief_config (); - - configuration = &d->configuration; - } - - // Print the delayed build package in the full report mode and count - // configuration builds otherwise. - // - if (ops.full_report ()) - { - // We can potentially extend this information with the archived flag - // or the delay duration. - // - cerr << " " << d->package_name << "/" << d->package_version; - - if (!d->tenant.empty ()) - cerr << " " << d->tenant; - - cerr << endl; - } - else - { - if (d->report_timestamp != timestamp_nonexistent) - ++config_reported_delay_count; - - ++config_total_delay_count; - } - } - - if (!ops.full_report ()) - brief_config (); - - // Persist the delay report timestamps. - // - // If we don't consider the report timestamps for reporting delays, it - // seems natural not to update these timestamps either. Note that - // reporting all delays and still updating the report timestamps can be - // achieved by specifying the zero report timeout. - // - if (ops.report_timeout_specified ()) - { - transaction t (db.begin ()); - - for (shared_ptr<const build_delay> d: delays) - { - // Only update timestamps for delays that needs to be reported. - // - if (d->report_timestamp != timestamp_nonexistent) - db.update (d); - } - - t.commit (); - } + soft_delays_report.print ("Package soft rebuild delays", + total, + ops.full_report ()); } catch (const io_error&) { return 1; // Not much we can do on stderr writing failure. } + // Persist the delay report timestamps. + // + if (!update_delays.empty ()) + { + transaction t (db.begin ()); + + for (shared_ptr<const build_delay> d: update_delays) + db.update (d); + + t.commit (); + } + return 0; } catch (const database_locked&) |