aboutsummaryrefslogtreecommitdiff
path: root/monitor/monitor.cxx
diff options
context:
space:
mode:
Diffstat (limited to 'monitor/monitor.cxx')
-rw-r--r--monitor/monitor.cxx577
1 files changed, 351 insertions, 226 deletions
diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx
index 7c20efe..42d481d 100644
--- a/monitor/monitor.cxx
+++ b/monitor/monitor.cxx
@@ -5,7 +5,6 @@
#include <set>
#include <chrono>
#include <iostream>
-#include <algorithm> // find_if()
#include <odb/database.hxx>
#include <odb/transaction.hxx>
@@ -14,9 +13,6 @@
#include <odb/pgsql/database.hxx>
#include <libbutl/pager.hxx>
-#include <libbutl/utility.hxx> // compare_c_string
-
-#include <libbbot/build-config.hxx>
#include <libbrep/build.hxx>
#include <libbrep/common.hxx>
@@ -25,14 +21,13 @@
#include <libbrep/build-package-odb.hxx>
#include <libbrep/database-lock.hxx>
-#include <mod/build-config.hxx>
+#include <mod/build-target-config.hxx>
#include <monitor/module-options.hxx>
#include <monitor/monitor-options.hxx>
using namespace std;
using namespace butl;
-using namespace bbot;
using namespace odb::core;
namespace brep
@@ -44,17 +39,19 @@ namespace brep
// We will collect and report build delays as separate steps not to hold
// database locks while printing to stderr. Also we need to order delays
// properly, so while printing reports we could group delays by toolchain
- // and configuration.
+ // and target configuration.
//
// To achieve that, we will iterate through all possible package builds
// creating the list of delays with the following sort priority:
//
// 1: toolchain name
// 2: toolchain version (descending)
- // 3: configuration name
- // 4: tenant
- // 5: package name
- // 6: package version (descending)
+ // 3: target configuration name
+ // 4: target
+ // 5: tenant
+ // 6: package name
+ // 7: package version (descending)
+ // 8: package configuration name
//
struct compare_delay
{
@@ -68,7 +65,10 @@ namespace brep
if (int r = x->toolchain_version.compare (y->toolchain_version))
return r > 0;
- if (int r = x->configuration.compare (y->configuration))
+ if (int r = x->target_config_name.compare (y->target_config_name))
+ return r < 0;
+
+ if (int r = x->target.compare (y->target))
return r < 0;
if (int r = x->tenant.compare (y->tenant))
@@ -77,7 +77,10 @@ namespace brep
if (int r = x->package_name.compare (y->package_name))
return r < 0;
- return x->package_version.compare (y->package_version) > 0;
+ if (int r = x->package_version.compare (y->package_version))
+ return r > 0;
+
+ return x->package_config_name.compare (y->package_config_name) < 0;
}
};
@@ -87,39 +90,57 @@ namespace brep
{
public:
// Note that in the brief mode we also need to print the total number of
- // delays (reported or not) per configuration. Thus, we add all delays to
- // the report object, marking them if we need to report them or not.
+ // delays (reported or not) per target configuration. Thus, we add all
+ // delays to the report object, marking them if we need to report them or
+ // not.
//
void
- add_delay (shared_ptr<build_delay>, bool report);
+ add_delay (shared_ptr<build_delay>, bool custom_bot, bool report);
bool
- empty () const
- {
- return reported_delay_count_ == 0;
- }
+ empty () const {return reported_delay_count_ == 0;}
// In the brief mode (if full is false) print the number of reported/total
- // (if total is true) delayed package builds per configuration rather than
- // the packages themselves.
+ // (if total is true) delayed package configuration builds per target
+ // configuration rather than the package configurations themselves.
//
void
print (const char* header, bool total, bool full) const;
private:
- // Maps delays to the report flag.
+ // Maps delays to the custom bot and report flag.
//
- map<shared_ptr<const build_delay>, bool, compare_delay> delays_;
+ struct delay_info
+ {
+ bool custom_bot;
+ bool report;
+ };
+
+ map<shared_ptr<const build_delay>, delay_info, compare_delay> delays_;
size_t reported_delay_count_ = 0;
+
+ // Number of reported/total delayed package configurations which need to
+ // be built with the custom build bots.
+ //
+ size_t custom_total_delay_count_ = 0;
+ size_t custom_reported_delay_count_ = 0;
};
void delay_report::
- add_delay (shared_ptr<build_delay> delay, bool report)
+ add_delay (shared_ptr<build_delay> delay, bool custom_bot, bool report)
{
- delays_.emplace (move (delay), report);
+ delays_.emplace (move (delay), delay_info {custom_bot, report});
+
+ if (custom_bot)
+ ++custom_total_delay_count_;
if (report)
+ {
++reported_delay_count_;
+
+ if (custom_bot)
+ ++custom_reported_delay_count_;
+ }
}
void delay_report::
@@ -133,49 +154,84 @@ namespace brep
if (total)
cerr << '/' << delays_.size ();
+ if (custom_reported_delay_count_ != 0 ||
+ (total && custom_total_delay_count_ != 0))
+ {
+ cerr << " including " << custom_reported_delay_count_;
+
+ if (total)
+ cerr << '/' << custom_total_delay_count_;
+
+ cerr << " custom";
+ }
+
cerr << "):" << endl;
- // Group the printed delays by toolchain and configuration.
+ // Group the printed delays by toolchain and target configuration.
//
- const string* toolchain_name (nullptr);
- const version* toolchain_version (nullptr);
- const string* configuration (nullptr);
+ const string* toolchain_name (nullptr);
+ const version* toolchain_version (nullptr);
+ const string* target_config_name (nullptr);
+ const target_triplet* target (nullptr);
size_t config_reported_delay_count (0);
size_t config_total_delay_count (0);
- auto brief_config = [&configuration,
+ size_t config_custom_reported_delay_count (0);
+ size_t config_custom_total_delay_count (0);
+
+ auto brief_config = [&target_config_name,
+ &target,
&config_reported_delay_count,
&config_total_delay_count,
+ &config_custom_reported_delay_count,
+ &config_custom_total_delay_count,
total] ()
{
- if (configuration != nullptr)
+ if (target_config_name != nullptr)
{
+ assert (target != nullptr);
+
// Only print configurations with delays that needs to be reported.
//
if (config_reported_delay_count != 0)
{
- cerr << " " << *configuration << " ("
+ cerr << " " << *target_config_name << '/' << *target << " ("
<< config_reported_delay_count;
if (total)
cerr << '/' << config_total_delay_count;
+ if (config_custom_reported_delay_count != 0 ||
+ (total && config_custom_total_delay_count != 0))
+ {
+ cerr << " including " << config_custom_reported_delay_count;
+
+ if (total)
+ cerr << '/' << config_custom_total_delay_count;
+
+ cerr << " custom";
+ }
+
cerr << ')' << endl;
}
config_reported_delay_count = 0;
config_total_delay_count = 0;
+
+ config_custom_reported_delay_count = 0;
+ config_custom_total_delay_count = 0;
}
};
for (const auto& dr: delays_)
{
- bool report (dr.second);
+ bool report (dr.second.report);
if (full && !report)
continue;
+ bool custom_bot (dr.second.custom_bot);
const shared_ptr<const build_delay>& d (dr.first);
// Print the toolchain, if changed.
@@ -197,49 +253,65 @@ namespace brep
cerr << endl;
- toolchain_name = &d->toolchain_name;
- toolchain_version = &d->toolchain_version;
- configuration = nullptr;
+ toolchain_name = &d->toolchain_name;
+ toolchain_version = &d->toolchain_version;
+ target_config_name = nullptr;
+ target = nullptr;
}
// Print the configuration, if changed.
//
- if (configuration == nullptr || d->configuration != *configuration)
+ if (target_config_name == nullptr ||
+ d->target_config_name != *target_config_name ||
+ d->target != *target)
{
if (full)
{
- if (configuration != nullptr)
+ if (target_config_name != nullptr)
cerr << endl;
- cerr << " " << d->configuration << endl;
+ cerr << " " << d->target_config_name << '/' << d->target << endl;
}
else
brief_config ();
- configuration = &d->configuration;
+ target_config_name = &d->target_config_name;
+ target = &d->target;
}
- // Print the delayed build package in the full report mode and count
- // configuration builds otherwise.
+ // Print the delayed build package configuration in the full report mode
+ // and count configuration builds otherwise.
//
if (full)
{
// We can potentially extend this information with the archived flag
// or the delay duration.
//
- cerr << " " << d->package_name << "/" << d->package_version;
+ cerr << " " << d->package_name << '/' << d->package_version
+ << ' ' << d->package_config_name;
+
+ if (custom_bot)
+ cerr << " (custom bot)";
if (!d->tenant.empty ())
- cerr << " " << d->tenant;
+ cerr << ' ' << d->tenant;
cerr << endl;
}
else
{
if (report)
+ {
++config_reported_delay_count;
+ if (custom_bot)
+ ++config_custom_reported_delay_count;
+ }
+
++config_total_delay_count;
+
+ if (custom_bot)
+ ++config_custom_total_delay_count;
}
}
@@ -439,11 +511,11 @@ namespace brep
return 0;
}
- build_configs configs;
+ build_target_configs configs;
try
{
- configs = parse_buildtab (mod_ops.build_config ());
+ configs = bbot::parse_buildtab (mod_ops.build_config ());
}
catch (const tab_parsing& e)
{
@@ -494,13 +566,12 @@ namespace brep
//
if (ops.clean ())
{
- using config_map = map<const char*,
- const build_config*,
- compare_c_string>;
+ using config_map = map<build_target_config_id,
+ const build_target_config*>;
config_map conf_map;
- for (const build_config& c: configs)
- conf_map[c.name.c_str ()] = &c;
+ for (const build_target_config& c: configs)
+ conf_map[build_target_config_id {c.target, c.name}] = &c;
// Prepare the build delay prepared query.
//
@@ -519,15 +590,17 @@ namespace brep
size_t offset (0);
query q ("ORDER BY" +
- query::id.package.tenant + "," +
- query::id.package.name +
+ query::id.package.tenant + "," +
+ query::id.package.name +
order_by_version (query::id.package.version,
false /* first */) + "," +
- query::id.configuration + "," +
- query::id.toolchain_name +
+ query::id.target + "," +
+ query::id.target_config_name + "," +
+ query::id.package_config_name + "," +
+ query::id.toolchain_name +
order_by_version (query::id.toolchain_version,
- false /* first */) +
- "OFFSET" + query::_ref (offset) + "LIMIT 100");
+ false /* first */) +
+ "OFFSET" + query::_ref (offset) + "LIMIT 2000");
connection_ptr conn (db.connection ());
@@ -569,7 +642,9 @@ namespace brep
//
// Check that the build configuration is still present.
//
- (ci = conf_map.find (d.configuration.c_str ())) ==
+ (ci = conf_map.find (
+ build_target_config_id {d.target,
+ d.target_config_name})) ==
conf_map.end ());
// Check that the package still present, is buildable and doesn't
@@ -583,12 +658,23 @@ namespace brep
p = db.find<build_package> (pid);
}
- cleanup = (p == nullptr ||
- !p->buildable ||
- exclude (p->builds,
- p->constraints,
- *ci->second,
- configs.class_inheritance_map));
+ const build_package_config* pc (p != nullptr
+ ? find (d.package_config_name,
+ p->configs)
+ : nullptr);
+
+ cleanup = (pc == nullptr || !p->buildable);
+
+ if (!cleanup)
+ {
+ db.load (*p, p->constraints_section);
+
+ cleanup = exclude (*pc,
+ p->builds,
+ p->constraints,
+ *ci->second,
+ configs.class_inheritance_map);
+ }
}
if (cleanup)
@@ -631,8 +717,13 @@ namespace brep
conn->prepare_query<buildable_package> ("buildable-package-query",
pq));
- // Prepare the package build prepared query.
+ // Prepare the package configuration build prepared queries.
//
+ using bquery = query<build>;
+ using prep_bquery = prepared_query<build>;
+
+ build_id id;
+
// This query will only be used for toolchains that have no version
// specified on the command line to obtain the latest completed build
// across all toolchain versions, if present, and the latest incomplete
@@ -643,22 +734,26 @@ namespace brep
// toolchain that built the package last and if there are none, pick the
// one for which the build task was issued last.
//
- using bquery = query<package_build>;
- using prep_bquery = prepared_query<package_build>;
-
- build_id id;
- const auto& bid (bquery::build::id);
-
- bquery bq ((equal<package_build> (bid.package, id.package) &&
- bid.configuration == bquery::_ref (id.configuration) &&
- bid.toolchain_name == bquery::_ref (id.toolchain_name)) +
- "ORDER BY" +
- bquery::build::soft_timestamp + "DESC, " +
- bquery::build::timestamp + "DESC" +
- "LIMIT 1");
-
- prep_bquery pbq (
- conn->prepare_query<package_build> ("package-build-query", bq));
+ // @@ TMP Check if we can optimize this query by adding index for
+ // soft_timestamp and/or by setting enable_nestloop=off (or some
+ // such) as we do in mod/mod-builds.cxx.
+ //
+ bquery lbq ((equal<build> (bquery::id,
+ id,
+ false /* toolchain_version */) &&
+ bquery::state != "queued") +
+ "ORDER BY" +
+ bquery::soft_timestamp + "DESC, " +
+ bquery::timestamp + "DESC" +
+ "LIMIT 1");
+
+ prep_bquery plbq (
+ conn->prepare_query<build> ("package-latest-build-query", lbq));
+
+ // This query will only be used to retrieve a specific build by id.
+ //
+ bquery bq (equal<build> (bquery::id, id) && bquery::state != "queued");
+ prep_bquery pbq (conn->prepare_query<build> ("package-build-query", bq));
timestamp now (system_clock::now ());
@@ -791,176 +886,206 @@ namespace brep
for (auto& bp: bps)
{
- shared_ptr<build_package> p (db.load<build_package> (bp.id));
+ shared_ptr<build_package>& p (bp.package);
- for (const build_config& c: configs)
- {
- if (exclude (p->builds,
- p->constraints,
- c,
- configs.class_inheritance_map))
- continue;
+ db.load (*p, p->constraints_section);
- for (const pair<string, version>& t: toolchains)
+ for (const build_package_config& pc: p->configs)
+ {
+ for (const build_target_config& tc: configs)
{
- id = build_id (p->id, c.name, t.first, t.second);
-
- // If the toolchain version is unspecified then search for the
- // latest build across all toolchain versions and search for a
- // specific build otherwise.
+ // Note that we also don't build a package configuration if we
+ // are unable to assign all the required auxiliary machines
+ // for the build (see mod/mod-build-task.cxx for details).
+ // That means that we will also report delays which happen due
+ // to such an inability, which can potentially be not only
+ // because of the infrastructural problem but also because of
+ // an error in the package manifest (build auxiliary
+ // configuration pattern doesn't match any machine
+ // configuration anymore, etc). It doesn't seem easy to
+ // distinguish here which type of problem causes a delay.
+ // Thus, for now let's wait and see if it ever becomes a
+ // problem.
//
- shared_ptr<build> b;
-
- if (id.toolchain_version.empty ())
+ if (exclude (pc,
+ p->builds,
+ p->constraints,
+ tc,
+ configs.class_inheritance_map))
+ continue;
+
+ for (const pair<string, version>& t: toolchains)
{
- auto pbs (pbq.execute ());
-
- if (!pbs.empty ())
- b = move (pbs.begin ()->build);
- }
- else
- b = db.find<build> (id);
+ id = build_id (p->id,
+ tc.target, tc.name,
+ pc.name,
+ t.first, t.second);
+
+ // If the toolchain version is unspecified then search for
+ // the latest build across all toolchain versions and search
+ // for a specific build otherwise.
+ //
+ shared_ptr<build> b (id.toolchain_version.empty ()
+ ? plbq.execute_one ()
+ : pbq.execute_one ());
+
+ // Note that we consider a build as delayed if it is not
+ // completed in the expected timeframe. So even if the build
+ // task have been issued recently we may still consider the
+ // build as delayed.
+ //
+ timestamp bht (b != nullptr
+ ? b->hard_timestamp
+ : timestamp_nonexistent);
- // Note that we consider a build as delayed if it is not
- // completed in the expected timeframe. So even if the build
- // task have been issued recently we may still consider the
- // build as delayed.
- //
- timestamp bht (b != nullptr
- ? b->hard_timestamp
- : timestamp_nonexistent);
+ timestamp bst (b != nullptr
+ ? b->soft_timestamp
+ : timestamp_nonexistent);
- timestamp bst (b != nullptr
- ? b->soft_timestamp
- : timestamp_nonexistent);
+ // Create the delay object to record a timestamp when the
+ // package configuration build could have potentially been
+ // started, unless it already exists.
+ //
+ shared_ptr<build_delay> d (db.find<build_delay> (id));
- // Create the delay object to record a timestamp when the
- // package build could have potentially been started, unless
- // it already exists.
- //
- shared_ptr<build_delay> d (db.find<build_delay> (id));
+ if (d == nullptr)
+ {
+ // If the archived package has no build nor build delay
+ // for this configuration, then we assume that the
+ // configuration was added after the package tenant has
+ // been archived and so the package could have never been
+ // built for this configuration. Thus, we don't consider
+ // this build as delayed and so skip it.
+ //
+ if (bp.archived && b == nullptr)
+ continue;
+
+ // Use the build hard, soft, or status change timestamp
+ // (see the timestamps description for their ordering
+ // information) as the build delay tracking starting point
+ // and fallback to the current time if there is no build
+ // yet.
+ //
+ timestamp pts (b == nullptr ? now :
+ bht != timestamp_nonexistent ? bht :
+ bst != timestamp_nonexistent ? bst :
+ b->timestamp);
+
+ d = make_shared<build_delay> (move (id.package.tenant),
+ move (id.package.name),
+ p->version,
+ move (id.target),
+ move (id.target_config_name),
+ move (id.package_config_name),
+ move (id.toolchain_name),
+ t.second,
+ pts);
+ db.persist (d);
+ }
- if (d == nullptr)
- {
- // If the archived package has no build nor build delay
- // for this configuration, then we assume that the
- // configuration was added after the package tenant has
- // been archived and so the package could have never been
- // built for this configuration. Thus, we don't consider
- // this build as delayed and so skip it.
+ // Handle package builds differently based on their tenant's
+ // archive status.
//
- if (bp.archived && b == nullptr)
- continue;
-
- // Use the build hard, soft, or status change timestamp (see
- // the timestamps description for their ordering
- // information) as the build delay tracking starting point
- // and fallback to the current time if there is no build
- // yet.
+ // If the package is not archived then consider it as
+ // delayed if it is not (re-)built by the expiration
+ // time. Otherwise, consider it as delayed if it is unbuilt.
//
- timestamp pts (
- b == nullptr ? now :
- bht != timestamp_nonexistent ? bht :
- bst != timestamp_nonexistent ? bst :
- b->timestamp);
-
- d = make_shared<build_delay> (move (id.package.tenant),
- move (id.package.name),
- p->version,
- move (id.configuration),
- move (id.toolchain_name),
- t.second,
- pts);
- db.persist (d);
- }
+ // We also don't need to report an unbuilt archived package
+ // twice, as both soft and hard build delays.
+ //
+ bool hard_delayed;
+ bool soft_delayed;
- // Handle package builds differently based on their tenant's
- // archive status.
- //
- // If the package is not archived then consider it as delayed
- // if it is not (re-)built by the expiration time. Otherwise,
- // consider it as delayed if it is unbuilt.
- //
- // We also don't need to report an unbuilt archived package
- // twice, as both soft and hard build delays.
- //
- bool hard_delayed;
- bool soft_delayed;
+ if (!bp.archived)
+ {
+ auto delayed = [&d] (timestamp bt, timestamp be)
+ {
+ timestamp t (bt != timestamp_nonexistent
+ ? bt
+ : d->package_timestamp);
+ return t <= be;
+ };
+
+ hard_delayed = delayed (bht, hard_rebuild_expiration);
+ soft_delayed = delayed (bst, soft_rebuild_expiration);
+ }
+ else
+ {
+ hard_delayed = (bst == timestamp_nonexistent);
+ soft_delayed = false;
+ }
- if (!bp.archived)
- {
- auto delayed = [&d] (timestamp bt, timestamp be)
+ // If there is a delay, then deduce if this package
+ // configuration needs to be built with a custom build bot.
+ //
+ // Note: only meaningful if there is a delay.
+ //
+ bool custom_bot (false);
+
+ if (hard_delayed || soft_delayed)
{
- timestamp t (bt != timestamp_nonexistent
- ? bt
- : d->package_timestamp);
- return t <= be;
- };
-
- hard_delayed = delayed (bht, hard_rebuild_expiration);
- soft_delayed = delayed (bst, soft_rebuild_expiration);
- }
- else
- {
- hard_delayed = (bst == timestamp_nonexistent);
- soft_delayed = false;
- }
+ if (!p->bot_keys_section.loaded ())
+ db.load (*p, p->bot_keys_section);
- // Add hard/soft delays to the respective reports and collect
- // the delay for update, if it is reported.
- //
- // Note that we update the delay objects persistent state
- // later, after we successfully print the reports.
- //
- bool reported (false);
+ custom_bot = !pc.effective_bot_keys (p->bot_keys).empty ();
+ }
- if (hard_delayed)
- {
- // If the report timeout is zero then report the delay
- // unconditionally. Otherwise, report the active package
- // build delay if the report timeout is expired and the
- // archived package build delay if it was never reported.
- // Note that fixing the building infrastructure won't help
- // building an archived package, so reporting its build
- // delays repeatedly is meaningless.
+ // Add hard/soft delays to the respective reports and
+ // collect the delay for update, if it is reported.
//
- bool report (
- ops.report_timeout () == 0 ||
- (!bp.archived
- ? d->report_hard_timestamp <= report_expiration
- : d->report_hard_timestamp == timestamp_nonexistent));
+ // Note that we update the delay objects persistent state
+ // later, after we successfully print the reports.
+ //
+ bool reported (false);
- if (report)
+ if (hard_delayed)
{
- d->report_hard_timestamp = now;
- reported = true;
+ // If the report timeout is zero then report the delay
+ // unconditionally. Otherwise, report the active package
+ // build delay if the report timeout is expired and the
+ // archived package build delay if it was never reported.
+ // Note that fixing the building infrastructure won't help
+ // building an archived package, so reporting its build
+ // delays repeatedly is meaningless.
+ //
+ bool report (
+ ops.report_timeout () == 0 ||
+ (!bp.archived
+ ? d->report_hard_timestamp <= report_expiration
+ : d->report_hard_timestamp == timestamp_nonexistent));
+
+ if (report)
+ {
+ d->report_hard_timestamp = now;
+ reported = true;
+ }
+
+ hard_delays_report.add_delay (d, custom_bot, report);
}
- hard_delays_report.add_delay (d, report);
- }
+ if (soft_delayed)
+ {
+ bool report (ops.report_timeout () == 0 ||
+ d->report_soft_timestamp <= report_expiration);
- if (soft_delayed)
- {
- bool report (ops.report_timeout () == 0 ||
- d->report_soft_timestamp <= report_expiration);
+ if (report)
+ {
+ d->report_soft_timestamp = now;
+ reported = true;
+ }
- if (report)
- {
- d->report_soft_timestamp = now;
- reported = true;
+ soft_delays_report.add_delay (d, custom_bot, report);
}
- soft_delays_report.add_delay (d, report);
+ // If we don't consider the report timestamps for reporting
+ // delays, it seems natural not to update these timestamps
+ // either. Note that reporting all delays and still updating
+ // the report timestamps can be achieved by specifying the
+ // zero report timeout.
+ //
+ if (reported && ops.report_timeout_specified ())
+ update_delays.insert (move (d));
}
-
- // If we don't consider the report timestamps for reporting
- // delays, it seems natural not to update these timestamps
- // either. Note that reporting all delays and still updating
- // the report timestamps can be achieved by specifying the
- // zero report timeout.
- //
- if (reported && ops.report_timeout_specified ())
- update_delays.insert (move (d));
}
}
}
generated by cgit v1.1 at 2024-04-30 07:06:35 +0000