diff options
Diffstat (limited to 'monitor/monitor.cxx')
| -rw-r--r-- | monitor/monitor.cxx | 688 |
1 files changed, 427 insertions, 261 deletions
diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx index 5c4d451..7c20efe 100644 --- a/monitor/monitor.cxx +++ b/monitor/monitor.cxx @@ -41,6 +41,212 @@ namespace brep // struct failed {}; + // We will collect and report build delays as separate steps not to hold + // database locks while printing to stderr. Also we need to order delays + // properly, so while printing reports we could group delays by toolchain + // and configuration. + // + // To achieve that, we will iterate through all possible package builds + // creating the list of delays with the following sort priority: + // + // 1: toolchain name + // 2: toolchain version (descending) + // 3: configuration name + // 4: tenant + // 5: package name + // 6: package version (descending) + // + struct compare_delay + { + bool + operator() (const shared_ptr<const build_delay>& x, + const shared_ptr<const build_delay>& y) const + { + if (int r = x->toolchain_name.compare (y->toolchain_name)) + return r < 0; + + if (int r = x->toolchain_version.compare (y->toolchain_version)) + return r > 0; + + if (int r = x->configuration.compare (y->configuration)) + return r < 0; + + if (int r = x->tenant.compare (y->tenant)) + return r < 0; + + if (int r = x->package_name.compare (y->package_name)) + return r < 0; + + return x->package_version.compare (y->package_version) > 0; + } + }; + + // The ordered list of delays to report. + // + class delay_report + { + public: + // Note that in the brief mode we also need to print the total number of + // delays (reported or not) per configuration. Thus, we add all delays to + // the report object, marking them if we need to report them or not. + // + void + add_delay (shared_ptr<build_delay>, bool report); + + bool + empty () const + { + return reported_delay_count_ == 0; + } + + // In the brief mode (if full is false) print the number of reported/total + // (if total is true) delayed package builds per configuration rather than + // the packages themselves. + // + void + print (const char* header, bool total, bool full) const; + + private: + // Maps delays to the report flag. + // + map<shared_ptr<const build_delay>, bool, compare_delay> delays_; + size_t reported_delay_count_ = 0; + }; + + void delay_report:: + add_delay (shared_ptr<build_delay> delay, bool report) + { + delays_.emplace (move (delay), report); + + if (report) + ++reported_delay_count_; + } + + void delay_report:: + print (const char* header, bool total, bool full) const + { + if (empty ()) + return; + + cerr << header << " (" << reported_delay_count_; + + if (total) + cerr << '/' << delays_.size (); + + cerr << "):" << endl; + + // Group the printed delays by toolchain and configuration. + // + const string* toolchain_name (nullptr); + const version* toolchain_version (nullptr); + const string* configuration (nullptr); + + size_t config_reported_delay_count (0); + size_t config_total_delay_count (0); + + auto brief_config = [&configuration, + &config_reported_delay_count, + &config_total_delay_count, + total] () + { + if (configuration != nullptr) + { + // Only print configurations with delays that needs to be reported. + // + if (config_reported_delay_count != 0) + { + cerr << " " << *configuration << " (" + << config_reported_delay_count; + + if (total) + cerr << '/' << config_total_delay_count; + + cerr << ')' << endl; + } + + config_reported_delay_count = 0; + config_total_delay_count = 0; + } + }; + + for (const auto& dr: delays_) + { + bool report (dr.second); + + if (full && !report) + continue; + + const shared_ptr<const build_delay>& d (dr.first); + + // Print the toolchain, if changed. + // + if (toolchain_name == nullptr || + d->toolchain_name != *toolchain_name || + d->toolchain_version != *toolchain_version) + { + if (!full) + brief_config (); + + if (toolchain_name != nullptr) + cerr << endl; + + cerr << " " << d->toolchain_name; + + if (!d->toolchain_version.empty ()) + cerr << "/" << d->toolchain_version; + + cerr << endl; + + toolchain_name = &d->toolchain_name; + toolchain_version = &d->toolchain_version; + configuration = nullptr; + } + + // Print the configuration, if changed. + // + if (configuration == nullptr || d->configuration != *configuration) + { + if (full) + { + if (configuration != nullptr) + cerr << endl; + + cerr << " " << d->configuration << endl; + } + else + brief_config (); + + configuration = &d->configuration; + } + + // Print the delayed build package in the full report mode and count + // configuration builds otherwise. + // + if (full) + { + // We can potentially extend this information with the archived flag + // or the delay duration. + // + cerr << " " << d->package_name << "/" << d->package_version; + + if (!d->tenant.empty ()) + cerr << " " << d->tenant; + + cerr << endl; + } + else + { + if (report) + ++config_reported_delay_count; + + ++config_total_delay_count; + } + } + + if (!full) + brief_config (); + } + static const char* help_info ( " info: run 'brep-monitor --help' for more information"); @@ -141,12 +347,24 @@ namespace brep return 1; } - if (mod_ops.build_alt_rebuild_start_specified () != - mod_ops.build_alt_rebuild_stop_specified ()) + auto bad_alt = [&f] (const char* what) + { + cerr << "build-alt-" << what << "-rebuild-start and build-alt-" + << what << "-rebuild-stop configuration options must both be " + << "either specified or not in '" << f << "'" << endl; + }; + + if (mod_ops.build_alt_hard_rebuild_start_specified () != + mod_ops.build_alt_hard_rebuild_stop_specified ()) + { + bad_alt("hard"); + return 1; + } + + if (mod_ops.build_alt_soft_rebuild_start_specified () != + mod_ops.build_alt_soft_rebuild_stop_specified ()) { - cerr << "build-alt-rebuild-start and build-alt-rebuild-stop " - << "configuration options must both be either specified or not " - << "in '" << f << "'" << endl; + bad_alt("soft"); return 1; } } @@ -384,59 +602,16 @@ namespace brep } } - // Collect and report delays as separate steps not to hold database locks - // while printing to stderr. Also we need to properly order delays for - // printing. - // - // Iterate through all possible package builds creating the list of delays - // with the following sort priority: - // - // 1: toolchain name - // 2: toolchain version (descending) - // 3: configuration name - // 4: tenant - // 5: package name - // 6: package version (descending) - // - // Such ordering will allow us to group build delays by toolchain and - // configuration while printing the report. - // - struct compare_delay - { - bool - operator() (const shared_ptr<const build_delay>& x, - const shared_ptr<const build_delay>& y) const - { - if (int r = x->toolchain_name.compare (y->toolchain_name)) - return r < 0; - - if (int r = x->toolchain_version.compare (y->toolchain_version)) - return r > 0; - - if (int r = x->configuration.compare (y->configuration)) - return r < 0; - - if (int r = x->tenant.compare (y->tenant)) - return r < 0; - - if (int r = x->package_name.compare (y->package_name)) - return r < 0; - - return x->package_version.compare (y->package_version) > 0; - } - }; - - size_t reported_delay_count (0); - size_t total_delay_count (0); - - set<shared_ptr<const build_delay>, compare_delay> delays; + delay_report hard_delays_report; + delay_report soft_delays_report; + set<shared_ptr<const build_delay>, compare_delay> update_delays; { connection_ptr conn (db.connection ()); // Prepare the buildable package prepared query. // - // Query buildable packages in chunks in order not to hold locks for - // too long. + // Query buildable packages in chunks in order not to hold locks for too + // long. // using pquery = query<buildable_package>; using prep_pquery = prepared_query<buildable_package>; @@ -463,6 +638,11 @@ namespace brep // across all toolchain versions, if present, and the latest incomplete // build otherwise. // + // Why don't we pick the latest toolchain version? We don't want to + // stuck with it on the toolchain rollback. Instead we prefer the + // toolchain that built the package last and if there are none, pick the + // one for which the build task was issued last. + // using bquery = query<package_build>; using prep_bquery = prepared_query<package_build>; @@ -473,63 +653,126 @@ namespace brep bid.configuration == bquery::_ref (id.configuration) && bid.toolchain_name == bquery::_ref (id.toolchain_name)) + "ORDER BY" + - bquery::build::completion_timestamp + "DESC, " + + bquery::build::soft_timestamp + "DESC, " + bquery::build::timestamp + "DESC" + "LIMIT 1"); prep_bquery pbq ( conn->prepare_query<package_build> ("package-build-query", bq)); - duration build_timeout; + timestamp now (system_clock::now ()); - // If the build timeout is not specified explicitly, then calculate it - // as the sum of the package rebuild timeout (normal rebuild timeout if - // the alternative timeout is unspecified and the maximum of two - // otherwise) and the build result timeout. + // Calculate the build/rebuild expiration time, based on the respective + // --{soft,hard}-rebuild-timeout monitor options and the + // build-{soft,hard}-rebuild-timeout and + // build-alt-{soft,hard}-rebuild-{start,stop,timeout} brep module + // configuration options. + // + // If the --*-rebuild-timeout monitor option is zero or is not specified + // and the respective build-*-rebuild-timeout brep's configuration + // option is zero, then return timestamp_unknown to indicate 'never + // expire'. Note that this value is less than any build timestamp value, + // including timestamp_nonexistent. // - if (!ops.build_timeout_specified ()) + // NOTE: there is a similar code in mod/mod-build-task.cxx. + // + auto build_expiration = [&now, &mod_ops] ( + optional<size_t> rebuild_timeout, + const optional<pair<duration, duration>>& alt_interval, + optional<size_t> alt_timeout, + size_t normal_timeout) { - duration normal_rebuild_timeout ( - chrono::seconds (mod_ops.build_normal_rebuild_timeout ())); + duration t; - if (mod_ops.build_alt_rebuild_start_specified ()) + // If the rebuild timeout is not specified explicitly, then calculate + // it as the sum of the package rebuild timeout (normal rebuild + // timeout if the alternative timeout is unspecified and the maximum + // of two otherwise) and the build result timeout. + // + if (!rebuild_timeout) { - // Calculate the alternative rebuild timeout as the time interval - // lenght, unless it is specified explicitly. - // - if (!mod_ops.build_alt_rebuild_timeout_specified ()) + if (normal_timeout == 0) + return timestamp_unknown; + + chrono::seconds nt (normal_timeout); + + if (alt_interval) { - const duration& start (mod_ops.build_alt_rebuild_start ()); - const duration& stop (mod_ops.build_alt_rebuild_stop ()); + // Calculate the alternative timeout, unless it is specified + // explicitly. + // + if (!alt_timeout) + { + const duration& start (alt_interval->first); + const duration& stop (alt_interval->second); - // Note that if the stop time is less than the start time then the - // interval extends through the midnight. + // Note that if the stop time is less than the start time then + // the interval extends through the midnight. + // + t = start <= stop ? (stop - start) : ((24h - start) + stop); + + // If the normal rebuild time out is greater than 24 hours, then + // increase the default alternative timeout by (normal - 24h) + // (see build-alt-soft-rebuild-timeout configuration option for + // details). + // + if (nt > 24h) + t += nt - 24h; + } + else + t = chrono::seconds (*alt_timeout); + + // Take the maximum of the alternative and normal rebuild + // timeouts. // - build_timeout = start <= stop - ? stop - start - : (24h - start) + stop; + if (t < nt) + t = nt; } else - build_timeout = - chrono::seconds (mod_ops.build_alt_rebuild_timeout ()); + t = nt; - // Take the maximum of the alternative and normal rebuild timeouts. + // Summarize the rebuild and build result timeouts. // - if (build_timeout < normal_rebuild_timeout) - build_timeout = normal_rebuild_timeout; + t += chrono::seconds (mod_ops.build_result_timeout ()); } else - build_timeout = normal_rebuild_timeout; + { + if (*rebuild_timeout == 0) + return timestamp_unknown; - // Summarize the rebuild and build result timeouts. - // - build_timeout += chrono::seconds (mod_ops.build_result_timeout ()); - } - else - build_timeout = chrono::seconds (ops.build_timeout ()); + t = chrono::seconds (*rebuild_timeout); + } - timestamp now (system_clock::now ()); - timestamp build_expiration (now - build_timeout); + return now - t; + }; + + timestamp hard_rebuild_expiration ( + build_expiration ( + (ops.hard_rebuild_timeout_specified () + ? ops.hard_rebuild_timeout () + : optional<size_t> ()), + (mod_ops.build_alt_hard_rebuild_start_specified () + ? make_pair (mod_ops.build_alt_hard_rebuild_start (), + mod_ops.build_alt_hard_rebuild_stop ()) + : optional<pair<duration, duration>> ()), + (mod_ops.build_alt_hard_rebuild_timeout_specified () + ? mod_ops.build_alt_hard_rebuild_timeout () + : optional<size_t> ()), + mod_ops.build_hard_rebuild_timeout ())); + + timestamp soft_rebuild_expiration ( + build_expiration ( + (ops.soft_rebuild_timeout_specified () + ? ops.soft_rebuild_timeout () + : optional<size_t> ()), + (mod_ops.build_alt_soft_rebuild_start_specified () + ? make_pair (mod_ops.build_alt_soft_rebuild_start (), + mod_ops.build_alt_soft_rebuild_stop ()) + : optional<pair<duration, duration>> ()), + (mod_ops.build_alt_soft_rebuild_timeout_specified () + ? mod_ops.build_alt_soft_rebuild_timeout () + : optional<size_t> ()), + mod_ops.build_soft_rebuild_timeout ())); timestamp report_expiration ( now - chrono::seconds (ops.report_timeout ())); @@ -583,8 +826,12 @@ namespace brep // task have been issued recently we may still consider the // build as delayed. // - timestamp bct (b != nullptr - ? b->completion_timestamp + timestamp bht (b != nullptr + ? b->hard_timestamp + : timestamp_nonexistent); + + timestamp bst (b != nullptr + ? b->soft_timestamp : timestamp_nonexistent); // Create the delay object to record a timestamp when the @@ -605,15 +852,17 @@ namespace brep if (bp.archived && b == nullptr) continue; - // Use the build completion or build status change - // timestamp, whichever is earlier, as the build delay - // tracking starting point and fallback to the current time - // if there is no build yet. + // Use the build hard, soft, or status change timestamp (see + // the timestamps description for their ordering + // information) as the build delay tracking starting point + // and fallback to the current time if there is no build + // yet. // timestamp pts ( - b == nullptr ? now : - bct != timestamp_nonexistent && bct < b->timestamp ? bct : - b->timestamp); + b == nullptr ? now : + bht != timestamp_nonexistent ? bht : + bst != timestamp_nonexistent ? bst : + b->timestamp); d = make_shared<build_delay> (move (id.package.tenant), move (id.package.name), @@ -632,20 +881,40 @@ namespace brep // if it is not (re-)built by the expiration time. Otherwise, // consider it as delayed if it is unbuilt. // - bool delayed; + // We also don't need to report an unbuilt archived package + // twice, as both soft and hard build delays. + // + bool hard_delayed; + bool soft_delayed; if (!bp.archived) { - timestamp bts (bct != timestamp_nonexistent - ? bct + auto delayed = [&d] (timestamp bt, timestamp be) + { + timestamp t (bt != timestamp_nonexistent + ? bt : d->package_timestamp); + return t <= be; + }; - delayed = (bts <= build_expiration); + hard_delayed = delayed (bht, hard_rebuild_expiration); + soft_delayed = delayed (bst, soft_rebuild_expiration); } else - delayed = (bct == timestamp_nonexistent); + { + hard_delayed = (bst == timestamp_nonexistent); + soft_delayed = false; + } - if (delayed) + // Add hard/soft delays to the respective reports and collect + // the delay for update, if it is reported. + // + // Note that we update the delay objects persistent state + // later, after we successfully print the reports. + // + bool reported (false); + + if (hard_delayed) { // If the report timeout is zero then report the delay // unconditionally. Otherwise, report the active package @@ -655,33 +924,43 @@ namespace brep // building an archived package, so reporting its build // delays repeatedly is meaningless. // - if (ops.report_timeout () == 0 || - (!bp.archived - ? d->report_timestamp <= report_expiration - : d->report_timestamp == timestamp_nonexistent)) - { - // Note that we update the delay objects persistent state - // later, after we successfully print the report. - // - d->report_timestamp = now; - delays.insert (move (d)); + bool report ( + ops.report_timeout () == 0 || + (!bp.archived + ? d->report_hard_timestamp <= report_expiration + : d->report_hard_timestamp == timestamp_nonexistent)); - ++reported_delay_count; + if (report) + { + d->report_hard_timestamp = now; + reported = true; } - // - // In the brief mode also collect unreported delays to - // deduce and print the total number of delays per - // configuration. Mark such delays with the - // timestamp_nonexistent report timestamp. - // - else if (!ops.full_report ()) + + hard_delays_report.add_delay (d, report); + } + + if (soft_delayed) + { + bool report (ops.report_timeout () == 0 || + d->report_soft_timestamp <= report_expiration); + + if (report) { - d->report_timestamp = timestamp_nonexistent; - delays.insert (move (d)); + d->report_soft_timestamp = now; + reported = true; } - ++total_delay_count; + soft_delays_report.add_delay (d, report); } + + // If we don't consider the report timestamps for reporting + // delays, it seems natural not to update these timestamps + // either. Note that reporting all delays and still updating + // the report timestamps can be achieved by specifying the + // zero report timeout. + // + if (reported && ops.report_timeout_specified ()) + update_delays.insert (move (d)); } } } @@ -691,161 +970,48 @@ namespace brep } } - // Report package build delays, if any. + // Print delay reports, if not empty. // - if (reported_delay_count != 0) + if (!hard_delays_report.empty () || !soft_delays_report.empty ()) try { - // Print the report. - // cerr.exceptions (ostream::badbit | ostream::failbit); // Don't print the total delay count if the report timeout is zero since // all delays are reported in this case. // - bool print_total_delay_count (ops.report_timeout () != 0); - - cerr << "Package build delays (" << reported_delay_count; - - if (print_total_delay_count) - cerr << '/' << total_delay_count; - - cerr << "):" << endl; - - // Group the printed delays by toolchain and configuration. - // - const string* toolchain_name (nullptr); - const version* toolchain_version (nullptr); - const string* configuration (nullptr); - - // In the brief report mode print the number of reported/total delayed - // package builds per configuration rather than the packages themselves. - // - size_t config_reported_delay_count (0); - size_t config_total_delay_count (0); - - auto brief_config = [&configuration, - &config_reported_delay_count, - &config_total_delay_count, - print_total_delay_count] () - { - if (configuration != nullptr) - { - // Only print configurations with delays that needs to be reported. - // - if (config_reported_delay_count != 0) - { - cerr << " " << *configuration << " (" - << config_reported_delay_count; + bool total (ops.report_timeout () != 0); - if (print_total_delay_count) - cerr << '/' << config_total_delay_count; + hard_delays_report.print ("Package hard rebuild delays", + total, + ops.full_report ()); - cerr << ')' << endl; - } - - config_reported_delay_count = 0; - config_total_delay_count = 0; - } - }; - - for (shared_ptr<const build_delay> d: delays) - { - // Print the toolchain, if changed. - // - if (toolchain_name == nullptr || - d->toolchain_name != *toolchain_name || - d->toolchain_version != *toolchain_version) - { - if (!ops.full_report ()) - brief_config (); - - if (toolchain_name != nullptr) - cerr << endl; - - cerr << " " << d->toolchain_name; - - if (!d->toolchain_version.empty ()) - cerr << "/" << d->toolchain_version; - - cerr << endl; - - toolchain_name = &d->toolchain_name; - toolchain_version = &d->toolchain_version; - configuration = nullptr; - } - - // Print the configuration, if changed. - // - if (configuration == nullptr || d->configuration != *configuration) - { - if (ops.full_report ()) - { - if (configuration != nullptr) - cerr << endl; - - cerr << " " << d->configuration << endl; - } - else - brief_config (); - - configuration = &d->configuration; - } - - // Print the delayed build package in the full report mode and count - // configuration builds otherwise. - // - if (ops.full_report ()) - { - // We can potentially extend this information with the archived flag - // or the delay duration. - // - cerr << " " << d->package_name << "/" << d->package_version; - - if (!d->tenant.empty ()) - cerr << " " << d->tenant; - - cerr << endl; - } - else - { - if (d->report_timestamp != timestamp_nonexistent) - ++config_reported_delay_count; - - ++config_total_delay_count; - } - } - - if (!ops.full_report ()) - brief_config (); - - // Persist the delay report timestamps. - // - // If we don't consider the report timestamps for reporting delays, it - // seems natural not to update these timestamps either. Note that - // reporting all delays and still updating the report timestamps can be - // achieved by specifying the zero report timeout. + // Separate reports with an empty line. // - if (ops.report_timeout_specified ()) - { - transaction t (db.begin ()); + if (!hard_delays_report.empty () && !soft_delays_report.empty ()) + cerr << endl; - for (shared_ptr<const build_delay> d: delays) - { - // Only update timestamps for delays that needs to be reported. - // - if (d->report_timestamp != timestamp_nonexistent) - db.update (d); - } - - t.commit (); - } + soft_delays_report.print ("Package soft rebuild delays", + total, + ops.full_report ()); } catch (const io_error&) { return 1; // Not much we can do on stderr writing failure. } + // Persist the delay report timestamps. + // + if (!update_delays.empty ()) + { + transaction t (db.begin ()); + + for (shared_ptr<const build_delay> d: update_delays) + db.update (d); + + t.commit (); + } + return 0; } catch (const database_locked&) |