aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--etc/brep-module.conf40
-rw-r--r--libbrep/build.cxx8
-rw-r--r--libbrep/build.hxx65
-rw-r--r--libbrep/build.xml18
-rw-r--r--migrate/migrate.cxx32
-rw-r--r--mod/buildfile2
-rw-r--r--mod/mod-build-result.cxx99
-rw-r--r--mod/mod-build-task.cxx262
-rw-r--r--mod/module.cli90
-rw-r--r--monitor/monitor.cli36
-rw-r--r--monitor/monitor.cxx688
11 files changed, 940 insertions, 400 deletions
diff --git a/etc/brep-module.conf b/etc/brep-module.conf
index 57d4865..9acef26 100644
--- a/etc/brep-module.conf
+++ b/etc/brep-module.conf
@@ -140,16 +140,20 @@ menu About=?about
# build-forced-rebuild-timeout 600
-# Time to wait before considering a package for a normal rebuild. Must be
-# specified in seconds. Default is 24 hours.
+# Time to wait before considering a package for a soft rebuild (only to be
+# performed if the build environment or any of the package dependencies have
+# changed). Must be specified in seconds. The special zero value disables soft
+# rebuilds. Default is 24 hours.
#
-# build-normal-rebuild-timeout 86400
+# build-soft-rebuild-timeout 86400
-# Alternative package rebuild timeout to use instead of the normal rebuild
-# timeout (see the build-normal-rebuild-timeout option for details) during
-# the specified time interval. Must be specified in seconds. Default is the
-# time interval length.
+# Alternative package soft rebuild timeout to use instead of the soft rebuild
+# timeout (see the build-soft-rebuild-timeout option for details) during the
+# specified time interval. Must be specified in seconds. Default is the time
+# interval length plus (build-soft-rebuild-timeout - 24h) if soft rebuild
+# timeout is greater than 24 hours (thus the rebuild is only triggered within
+# the last 24 hours of the build-soft-rebuild-timeout expiration).
#
# The alternative rebuild timeout can be used to "pull" the rebuild window to
# the specified time of day, for example, to optimize load and/or power
@@ -169,9 +173,25 @@ menu About=?about
# times must both be either specified or absent. If unspecified, then no
# alternative rebuild timeout will be used.
#
-# build-alt-rebuild-timeout
-# build-alt-rebuild-start
-# build-alt-rebuild-stop
+# build-alt-soft-rebuild-timeout
+# build-alt-soft-rebuild-start
+# build-alt-soft-rebuild-stop
+
+
+# Time to wait before considering a package for a hard rebuild (to be
+# performed unconditionally). Must be specified in seconds. The special zero
+# value disables hard rebuilds. Default is 7 days.
+#
+# build-hard-rebuild-timeout 604800
+
+
+# Alternative package hard rebuild timeout. The semantics is the same as for
+# the build-alt-soft-rebuild-* options but for the build-hard-rebuild-timeout
+# option.
+#
+# build-alt-hard-rebuild-timeout
+# build-alt-hard-rebuild-start
+# build-alt-hard-rebuild-stop
# The maximum size of the build task request manifest accepted. Note that the
diff --git a/libbrep/build.cxx b/libbrep/build.cxx
index 5f8cd71..9cbad8f 100644
--- a/libbrep/build.cxx
+++ b/libbrep/build.cxx
@@ -62,7 +62,9 @@ namespace brep
optional<string> inr,
optional<string> afp, optional<string> ach,
string mnm, string msm,
- butl::target_triplet trg)
+ butl::target_triplet trg,
+ string ccs,
+ string mcs)
: id (package_id (move (tnt), move (pnm), pvr),
move (cfg),
move (tnm), tvr),
@@ -79,7 +81,9 @@ namespace brep
agent_fingerprint (move (afp)), agent_challenge (move (ach)),
machine (move (mnm)),
machine_summary (move (msm)),
- target (move (trg))
+ target (move (trg)),
+ controller_checksum (move (ccs)),
+ machine_checksum (move (mcs))
{
}
diff --git a/libbrep/build.hxx b/libbrep/build.hxx
index 151f8dc..fbab13d 100644
--- a/libbrep/build.hxx
+++ b/libbrep/build.hxx
@@ -30,7 +30,7 @@
//
#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 12
-#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 14, closed)
+#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 15, closed)
// We have to keep these mappings at the global scope instead of inside
// the brep namespace because they need to be also effective in the
@@ -194,7 +194,9 @@ namespace brep
optional<string> agent_fingerprint,
optional<string> agent_challenge,
string machine, string machine_summary,
- butl::target_triplet);
+ butl::target_triplet,
+ string controller_checksum,
+ string machine_checksum);
build_id id;
@@ -223,15 +225,26 @@ namespace brep
//
optional<result_status> status;
- // Time of setting the result status that can be considered as the build
- // task completion (currently all the result_status values). Initialized
- // with timestamp_nonexistent by default.
+ // Times of the last soft/hard completed (re)builds. Used to decide when
+ // to perform soft and hard rebuilds, respectively.
+ //
+ // The soft timestamp is updated whenever we receive a task result.
+ //
+ // The hard timestamp is updated whenever we receive a task result with
+ // a status other than skip.
+ //
+ // Also note that whenever hard_timestamp is updated, soft_timestamp is
+ // updated as well and whenever soft_timestamp is updated, timestamp is
+ // updated as well. Thus the following condition is always true:
//
- // Note that in the future we may not consider abort and abnormal as the
- // task completion and, for example, proceed with automatic rebuild (the
- // flake monitor idea).
+ // hard_timestamp <= soft_timestamp <= timestamp
//
- timestamp_type completion_timestamp;
+ // Note that the "completed" above means that we may analyze the task
+ // result/log and deem it as not completed and proceed with automatic
+ // rebuild (the flake monitor idea).
+ //
+ timestamp_type soft_timestamp;
+ timestamp_type hard_timestamp;
// May be present only for the building state.
//
@@ -248,6 +261,21 @@ namespace brep
operation_results results;
odb::section results_section;
+ // Checksums of entities involved in the build.
+ //
+ // Optional checksums are provided by the external entities (agent and
+ // worker). All are absent initially.
+ //
+ // Note that the agent checksum can also be absent after the hard rebuild
+ // task is issued and the worker and dependency checksums - after a failed
+ // rebuild (error result status or worse).
+ //
+ string controller_checksum;
+ string machine_checksum;
+ optional<string> agent_checksum;
+ optional<string> worker_checksum;
+ optional<string> dependency_checksum;
+
// Database mapping.
//
#pragma db member(id) id column("")
@@ -265,15 +293,6 @@ namespace brep
//
#pragma db member(timestamp) index
- // This is not required since 0.14.0. Note however, that just dropping
- // this line won't pan out since this would require migration which odb is
- // currently unable to handle automatically, advising to re-implement this
- // change by adding a new data member with the desired default value,
- // migrating the data, and deleting the old data member. This sounds a bit
- // hairy, so let's keep it for now.
- //
- #pragma db member(completion_timestamp) default(0)
-
#pragma db member(results) id_column("") value_column("") \
section(results_section)
@@ -396,10 +415,14 @@ namespace brep
string& toolchain_name; // Tracks id.toolchain_name.
upstream_version toolchain_version; // Original of id.toolchain_version.
- // Time of the latest delay report. Initialized with timestamp_nonexistent
- // by default.
+ // Times of the latest soft and hard rebuild delay reports. Initialized
+ // with timestamp_nonexistent by default.
+ //
+ // Note that both reports notify about initial build delays (at their
+ // respective time intervals).
//
- timestamp report_timestamp;
+ timestamp report_soft_timestamp;
+ timestamp report_hard_timestamp;
// Time when the package is initially considered as buildable for this
// configuration and toolchain. It is used to track the build delay if the
diff --git a/libbrep/build.xml b/libbrep/build.xml
index 0ca362a..6a2301b 100644
--- a/libbrep/build.xml
+++ b/libbrep/build.xml
@@ -1,4 +1,22 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1">
+ <changeset version="15">
+ <alter-table name="build">
+ <add-column name="soft_timestamp" type="BIGINT" null="false"/>
+ <add-column name="hard_timestamp" type="BIGINT" null="false"/>
+ <add-column name="controller_checksum" type="TEXT" null="false"/>
+ <add-column name="machine_checksum" type="TEXT" null="false"/>
+ <add-column name="agent_checksum" type="TEXT" null="true"/>
+ <add-column name="worker_checksum" type="TEXT" null="true"/>
+ <add-column name="dependency_checksum" type="TEXT" null="true"/>
+ <drop-column name="completion_timestamp"/>
+ </alter-table>
+ <alter-table name="build_delay">
+ <add-column name="report_soft_timestamp" type="BIGINT" null="false"/>
+ <add-column name="report_hard_timestamp" type="BIGINT" null="false"/>
+ <drop-column name="report_timestamp"/>
+ </alter-table>
+ </changeset>
+
<changeset version="14"/>
<changeset version="13">
diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx
index 1093b7c..468d411 100644
--- a/migrate/migrate.cxx
+++ b/migrate/migrate.cxx
@@ -14,6 +14,8 @@
#include <libbutl/pager.hxx>
+#include <libbrep/build.hxx>
+#include <libbrep/build-odb.hxx>
#include <libbrep/package.hxx>
#include <libbrep/package-odb.hxx>
#include <libbrep/database-lock.hxx>
@@ -224,6 +226,36 @@ package_migrate_v20 ([] (database& db)
});
#endif
+// Register the data migration functions for the build database schema.
+//
+template <schema_version v>
+using build_migration_entry_base =
+ data_migration_entry<v, LIBBREP_BUILD_SCHEMA_VERSION_BASE>;
+
+template <schema_version v>
+struct build_migration_entry: build_migration_entry_base<v>
+{
+ build_migration_entry (void (*f) (database& db))
+ : build_migration_entry_base<v> (f, "build") {}
+};
+
+static const build_migration_entry<15>
+build_migrate_v15 ([] (database& db)
+{
+ // Setting proper checksums here feels a bit hairy. Let's assign them
+ // naturally on the first rebuild.
+ //
+ db.execute ("UPDATE build SET "
+ "soft_timestamp = completion_timestamp, "
+ "hard_timestamp = completion_timestamp, "
+ "controller_checksum = '', "
+ "machine_checksum = ''");
+
+ db.execute ("UPDATE build_delay SET "
+ "report_soft_timestamp = report_timestamp, "
+ "report_hard_timestamp = report_timestamp");
+});
+
// main() function
//
int
diff --git a/mod/buildfile b/mod/buildfile
index ff9cd60..58a3caf 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -50,7 +50,7 @@ if $cli.configured
cli.options += --std c++11 -I $src_root --include-with-brackets \
--include-prefix mod --guard-prefix MOD --generate-specifier \
--cxx-prologue "#include <mod/types-parsers.hxx>" \
---cli-namespace brep::cli --generate-file-scanner --option-length 45 \
+--cli-namespace brep::cli --generate-file-scanner --option-length 46 \
--generate-modifier --generate-description --option-prefix ""
# Include the generated cli files into the distribution and don't remove
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 3ae9f0f..1c46fc1 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -287,7 +287,6 @@ handle (request& rq, response&)
//
shared_ptr<build> bld;
- optional<result_status> prev_status;
bool build_notify (false);
bool unforced (true);
@@ -382,6 +381,58 @@ handle (request& rq, response&)
if (auth)
{
+ // Verify the result status/checksums.
+ //
+ // Specifically, if the result status is skip, then it can only be in
+ // response to the soft rebuild task (all checksums are present in the
+ // build object) and the result checksums must match the build object
+ // checksums. On verification failure respond with the bad request
+ // HTTP code (400).
+ //
+ if (rqm.result.status == result_status::skip)
+ {
+ if (!b->agent_checksum ||
+ !b->worker_checksum ||
+ !b->dependency_checksum)
+ throw invalid_request (400, "unexpected skip result status");
+
+ // Can only be absent for initial build, in which case the checksums
+ // are also absent and we would end up with the above 400 response.
+ //
+ assert (b->status);
+
+ // Verify that the result checksum matches the build checksum and
+ // throw invalid_request(400) if that's not the case.
+ //
+ auto verify = [] (const string& build_checksum,
+ const optional<string>& result_checksum,
+ const char* what)
+ {
+ if (!result_checksum)
+ throw invalid_request (
+ 400,
+ string (what) +
+ " checksum is expected for skip result status");
+
+ if (*result_checksum != build_checksum)
+ throw invalid_request (
+ 400,
+ string (what) + " checksum '" + build_checksum +
+ "' is expected instead of '" + *result_checksum +
+ "' for skip result status");
+ };
+
+ verify (*b->agent_checksum, rqm.agent_checksum, "agent");
+
+ verify (*b->worker_checksum,
+ rqm.result.worker_checksum,
+ "worker");
+
+ verify (*b->dependency_checksum,
+ rqm.result.dependency_checksum,
+ "dependency");
+ }
+
unforced = b->force == force_state::unforced;
// Don't send email to the build-email address for the
@@ -392,10 +443,7 @@ handle (request& rq, response&)
*b->status == rqm.result.status &&
unforced);
- prev_status = move (b->status);
-
b->state = build_state::built;
- b->status = rqm.result.status;
b->force = force_state::unforced;
// Cleanup the interactive build login information.
@@ -407,22 +455,43 @@ handle (request& rq, response&)
b->agent_fingerprint = nullopt;
b->agent_challenge = nullopt;
- // Mark the section as loaded, so results are updated.
- //
- b->results_section.load ();
- b->results = move (rqm.result.results);
-
b->timestamp = system_clock::now ();
- b->completion_timestamp = b->timestamp;
+ b->soft_timestamp = b->timestamp;
+
+ // If the result status is other than skip, then save the status,
+ // results, and checksums and update the hard timestamp.
+ //
+ if (rqm.result.status != result_status::skip)
+ {
+ b->status = rqm.result.status;
+ b->hard_timestamp = b->soft_timestamp;
+
+ // Mark the section as loaded, so results are updated.
+ //
+ b->results_section.load ();
+ b->results = move (rqm.result.results);
+
+ // Save the checksums.
+ //
+ b->agent_checksum = move (rqm.agent_checksum);
+ b->worker_checksum = move (rqm.result.worker_checksum);
+ b->dependency_checksum = move (rqm.result.dependency_checksum);
+ }
build_db_->update (b);
- shared_ptr<build_package> p (
- build_db_->load<build_package> (b->id.package));
+ // Don't send the build notification email if the task result is
+ // `skip`, the configuration is hidden, or is now excluded by the
+ // package.
+ //
+ if (rqm.result.status != result_status::skip && belongs (*cfg, "all"))
+ {
+ shared_ptr<build_package> p (
+ build_db_->load<build_package> (b->id.package));
- if (belongs (*cfg, "all") &&
- !exclude (p->builds, p->constraints, *cfg))
- bld = move (b);
+ if (!exclude (p->builds, p->constraints, *cfg))
+ bld = move (b);
+ }
}
}
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 8656f5e..22d0110 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -61,13 +61,23 @@ init (scanner& s)
if (options_->build_config_specified ())
{
- // Verify that build-alt-rebuild-{start,stop} are both either specified or
- // not.
+ // Verify that build-alt-*-rebuild-{start,stop} are both either specified
+ // or not.
//
- if (options_->build_alt_rebuild_start_specified () !=
- options_->build_alt_rebuild_stop_specified ())
- fail << "build-alt-rebuild-start and build-alt-rebuild-stop "
- << "configuration options must both be either specified or not";
+ auto bad_alt = [&fail] (const char* what)
+ {
+ fail << "build-alt-" << what << "-rebuild-start and build-alt-" << what
+ << "-rebuild-stop configuration options must both be either "
+ << "specified or not";
+ };
+
+ if (options_->build_alt_soft_rebuild_start_specified () !=
+ options_->build_alt_soft_rebuild_stop_specified ())
+ bad_alt ("soft");
+
+ if (options_->build_alt_hard_rebuild_start_specified () !=
+ options_->build_alt_hard_rebuild_stop_specified ())
+ bad_alt ("hard");
database_module::init (*options_, options_->build_db_retry ());
@@ -277,22 +287,25 @@ handle (request& rq, response& rs)
move (fps),
move (p->requirements),
move (tests),
+ move (b->dependency_checksum),
cm.machine->name,
cm.config->target,
cm.config->environment,
cm.config->args,
belongs (*cm.config, module_pkg ? "build2" : "host"),
cm.config->warning_regexes,
- move (t->interactive));
+ move (t->interactive),
+ move (b->worker_checksum));
return task_response_manifest (move (session),
move (b->agent_challenge),
move (result_url),
+ move (b->agent_checksum),
move (task));
};
- // Calculate the build (building state) or rebuild (built state) expiration
- // time for package configurations
+ // Calculate the build (building state) or rebuild (built state)
+ // expiration time for package configurations.
//
timestamp now (system_clock::now ());
@@ -316,44 +329,95 @@ handle (request& rq, response& rs)
timestamp forced_rebuild_expiration (
expiration (options_->build_forced_rebuild_timeout ()));
- timestamp normal_rebuild_expiration;
-
- if (options_->build_alt_rebuild_start_specified ())
+ // Calculate the soft/hard rebuild expiration time, based on the
+ // respective build-{soft,hard}-rebuild-timeout and
+ // build-alt-{soft,hard}-rebuild-{start,stop,timeout} configuration
+ // options.
+ //
+ // If normal_timeout is zero, then return timestamp_unknown to indicate
+ // 'never expire'. Note that this value is less than any build timestamp
+ // value, including timestamp_nonexistent.
+ //
+ // NOTE: there is a similar code in monitor/monitor.cxx.
+ //
+ auto build_expiration = [&now] (
+ const optional<pair<duration, duration>>& alt_interval,
+ optional<size_t> alt_timeout,
+ size_t normal_timeout)
{
- const duration& start (options_->build_alt_rebuild_start ());
- const duration& stop (options_->build_alt_rebuild_stop ());
+ if (normal_timeout == 0)
+ return timestamp_unknown;
- duration dt (daytime (now));
+ timestamp r;
+ chrono::seconds nt (normal_timeout);
- // Note that if the stop time is less than the start time then the
- // interval extends through the midnight.
- //
- bool alt_timeout (start <= stop
- ? dt >= start && dt < stop
- : dt >= start || dt < stop);
-
- // If we out of the alternative rebuild timeout interval, then fall back
- // to using the normal rebuild timeout.
- //
- if (alt_timeout)
+ if (alt_interval)
{
- if (!options_->build_alt_rebuild_timeout_specified ())
+ const duration& start (alt_interval->first);
+ const duration& stop (alt_interval->second);
+
+ duration dt (daytime (now));
+
+ // Note that if the stop time is less than the start time then the
+ // interval extends through the midnight.
+ //
+ bool use_alt_timeout (start <= stop
+ ? dt >= start && dt < stop
+ : dt >= start || dt < stop);
+
+ // If we out of the alternative rebuild timeout interval, then fall
+ // back to using the normal rebuild timeout.
+ //
+ if (use_alt_timeout)
{
- duration interval_len (start <= stop
- ? stop - start
- : (24h - start) + stop);
+ // Calculate the alternative timeout, unless it is specified
+ // explicitly.
+ //
+ duration t;
+
+ if (!alt_timeout)
+ {
+ t = start <= stop ? (stop - start) : ((24h - start) + stop);
- normal_rebuild_expiration = now - interval_len;
+ // If the normal rebuild timeout is greater than 24 hours, then
+ // increase the default alternative timeout by (normal - 24h) (see
+ // build-alt-soft-rebuild-timeout configuration option for
+ // details).
+ //
+ if (nt > 24h)
+ t += nt - 24h;
+ }
+ else
+ t = chrono::seconds (*alt_timeout);
+
+ r = now - t;
}
- else
- normal_rebuild_expiration =
- expiration (options_->build_alt_rebuild_timeout ());
}
- }
- if (normal_rebuild_expiration == timestamp_nonexistent)
- normal_rebuild_expiration =
- expiration (options_->build_normal_rebuild_timeout ());
+ return r != timestamp_nonexistent ? r : (now - nt);
+ };
+
+ timestamp soft_rebuild_expiration (
+ build_expiration (
+ (options_->build_alt_soft_rebuild_start_specified ()
+ ? make_pair (options_->build_alt_soft_rebuild_start (),
+ options_->build_alt_soft_rebuild_stop ())
+ : optional<pair<duration, duration>> ()),
+ (options_->build_alt_soft_rebuild_timeout_specified ()
+ ? options_->build_alt_soft_rebuild_timeout ()
+ : optional<size_t> ()),
+ options_->build_soft_rebuild_timeout ()));
+
+ timestamp hard_rebuild_expiration (
+ build_expiration (
+ (options_->build_alt_hard_rebuild_start_specified ()
+ ? make_pair (options_->build_alt_hard_rebuild_start (),
+ options_->build_alt_hard_rebuild_stop ())
+ : optional<pair<duration, duration>> ()),
+ (options_->build_alt_hard_rebuild_timeout_specified ()
+ ? options_->build_alt_hard_rebuild_timeout ()
+ : optional<size_t> ()),
+ options_->build_hard_rebuild_timeout ()));
// Return the challenge (nonce) if brep is configured to authenticate bbot
// agents. Return nullopt otherwise.
@@ -555,6 +619,68 @@ handle (request& rq, response& rs)
prep_bld_query bld_prep_query (
conn->prepare_query<build> ("mod-build-task-build-query", bq));
+ // Return true if a package needs to be rebuilt.
+ //
+ auto needs_rebuild = [&forced_rebuild_expiration,
+ &soft_rebuild_expiration,
+ &hard_rebuild_expiration] (const build& b)
+ {
+ assert (b.state == build_state::built);
+
+ return (b.force == force_state::forced &&
+ b.soft_timestamp <= forced_rebuild_expiration) ||
+ b.soft_timestamp <= soft_rebuild_expiration ||
+ b.hard_timestamp <= hard_rebuild_expiration;
+ };
+
+ // Convert a build to the hard rebuild, resetting the agent checksum and
+ // dropping the previous build task result.
+ //
+ // Note that since the checksums are hierarchical, the agent checksum
+ // reset will trigger resets of the "subordinate" checksums up to the
+ // dependency checksum and so the package will be rebuilt.
+ //
+ // Also note that there is no sense to keep the build task result since we
+ // don't accept the skip result for the hard rebuild task. We, however,
+ // keep the status intact (see below for the reasoning).
+ //
+ auto convert_to_hard = [] (const shared_ptr<build>& b)
+ {
+ b->agent_checksum = nullopt;
+
+ // Mark the section as loaded, so results are updated.
+ //
+ b->results_section.load ();
+ b->results.clear ();
+ };
+
+ // Return SHA256 checksum of the controller logic and the configuration
+ // target, environment, arguments, and warning-detecting regular
+ // expressions.
+ //
+ auto controller_checksum = [] (const build_config& c)
+ {
+ sha256 cs ("1"); // Hash the logic version.
+
+ cs.append (c.target.string ());
+ cs.append (c.environment ? *c.environment : "");
+
+ for (const string& a: c.args)
+ cs.append (a);
+
+ for (const string& re: c.warning_regexes)
+ cs.append (re);
+
+ return string (cs.string ());
+ };
+
+ // Return the machine id as a machine checksum.
+ //
+ auto machine_checksum = [] (const machine_header_manifest& m)
+ {
+ return m.id;
+ };
+
while (tsm.session.empty ())
{
transaction t (conn->begin ());
@@ -604,9 +730,7 @@ handle (request& rq, response& rs)
{
assert (i->force != force_state::forcing);
- if (i->timestamp <= (i->force == force_state::forced
- ? forced_rebuild_expiration
- : normal_rebuild_expiration))
+ if (needs_rebuild (*i))
rebuilds.emplace_back (i.load ());
}
}
@@ -666,14 +790,15 @@ handle (request& rq, response& rs)
move (cl),
mh.name,
move (mh.summary),
- cm.config->target);
+ cm.config->target,
+ controller_checksum (*cm.config),
+ machine_checksum (*cm.machine));
build_db_->persist (b);
}
else
{
- // The package configuration is in the building state, and there
- // are no results.
+ // The package configuration is in the building state.
//
// Note that in both cases we keep the status intact to be able
// to compare it with the final one in the result request
@@ -681,12 +806,7 @@ handle (request& rq, response& rs)
// email. The same is true for the forced flag (in the sense
// that we don't set the force state to unforced).
//
- // Load the section to assert the above statement.
- //
- build_db_->load (*b, b->results_section);
-
- assert (b->state == build_state::building &&
- b->results.empty ());
+ assert (b->state == build_state::building);
b->state = build_state::building;
b->interactive = move (login);
@@ -703,6 +823,22 @@ handle (request& rq, response& rs)
b->machine = mh.name;
b->machine_summary = move (mh.summary);
b->target = cm.config->target;
+
+ string ccs (controller_checksum (*cm.config));
+ string mcs (machine_checksum (*cm.machine));
+
+ // Issue the hard rebuild if it is forced or the configuration
+ // or machine has changed.
+ //
+ if (b->hard_timestamp <= hard_rebuild_expiration ||
+ b->force == force_state::forced ||
+ b->controller_checksum != ccs ||
+ b->machine_checksum != mcs)
+ convert_to_hard (b);
+
+ b->controller_checksum = move (ccs);
+ b->machine_checksum = move (mcs);
+
b->timestamp = system_clock::now ();
build_db_->update (b);
@@ -770,10 +906,9 @@ handle (request& rq, response& rs)
b = build_db_->find<build> (b->id);
- if (b != nullptr && b->state == build_state::built &&
- b->timestamp <= (b->force == force_state::forced
- ? forced_rebuild_expiration
- : normal_rebuild_expiration))
+ if (b != nullptr &&
+ b->state == build_state::built &&
+ needs_rebuild (*b))
{
auto i (cfg_machines.find (b->id.configuration.c_str ()));
@@ -828,10 +963,23 @@ handle (request& rq, response& rs)
b->target = cm.config->target;
- // Mark the section as loaded, so results are updated.
+ // Issue the hard rebuild if the timeout expired, rebuild is
+ // forced, or the configuration or machine has changed.
//
- b->results_section.load ();
- b->results.clear ();
+ // Note that we never reset the build status (see above for the
+ // reasoning).
+ //
+ string ccs (controller_checksum (*cm.config));
+ string mcs (machine_checksum (*cm.machine));
+
+ if (b->hard_timestamp <= hard_rebuild_expiration ||
+ b->force == force_state::forced ||
+ b->controller_checksum != ccs ||
+ b->machine_checksum != mcs)
+ convert_to_hard (b);
+
+ b->controller_checksum = move (ccs);
+ b->machine_checksum = move (mcs);
b->timestamp = system_clock::now ();
diff --git a/mod/module.cli b/mod/module.cli
index c95c20c..c2dce5b 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -195,45 +195,87 @@ namespace brep
be specified in seconds. Default is 10 minutes."
}
- size_t build-normal-rebuild-timeout = 86400
+ size_t build-soft-rebuild-timeout = 86400
{
"<seconds>",
- "Time to wait before considering a package for a normal rebuild. Must
- be specified in seconds. Default is 24 hours."
+ "Time to wait before considering a package for a soft rebuild (only to
+ be performed if the build environment or any of the package
+ dependencies have changed). Must be specified in seconds. The special
+ zero value disables soft rebuilds. Default is 24 hours"
}
- size_t build-alt-rebuild-timeout
+ size_t build-alt-soft-rebuild-timeout
{
"<seconds>",
- "Alternative package rebuild timeout to use instead of the normal
- rebuild timeout (see \cb{build-normal-rebuild-timeout} for details)
+ "Alternative package soft rebuild timeout to use instead of the soft
+ rebuild timeout (see \cb{build-soft-rebuild-timeout} for details)
during the time interval specified with the
- \cb{build-alt-rebuild-start} and \cb{build-alt-rebuild-stop} options.
- Must be specified in seconds. Default is the time interval length."
+ \cb{build-alt-soft-rebuild-start} and
+ \cb{build-alt-soft-rebuild-stop} options. Must be specified in
+ seconds. Default is the time interval length plus
+ \c{(\b{build-soft-rebuild-timeout} - 24h)} if soft rebuild timeout is
+ greater than 24 hours (thus the rebuild is only triggered within the
+ last 24 hours of the \cb{build-soft-rebuild-timeout} expiration)."
}
- duration build-alt-rebuild-start
+ duration build-alt-soft-rebuild-start
{
"<hours>:<minutes>",
- "The start time of the alternative package rebuild timeout (see
- \cb{build-alt-rebuild-timeout} for details). Must be specified as
- a time of day in the local timezone. The \cb{build-alt-rebuild-start}
- and \cb{build-alt-rebuild-stop} options must be either both specified
- or absent. If unspecified, then no alternative rebuild timeout will
- be used."
+ "The start time of the alternative package soft rebuild timeout (see
+ \cb{build-alt-soft-rebuild-timeout} for details). Must be specified
+ as a time of day in the local timezone. The
+ \cb{build-alt-soft-rebuild-start} and
+ \cb{build-alt-soft-rebuild-stop} options must be either both
+ specified or absent. If unspecified, then no alternative rebuild
+ timeout will be used."
}
- duration build-alt-rebuild-stop
+ duration build-alt-soft-rebuild-stop
{
"<hours>:<minutes>",
- "The end time of the alternative package rebuild timeout (see
- \cb{build-alt-rebuild-timeout} for details). Must be specified as
- a time of day in the local timezone. If it is less than the
- \cb{build-alt-rebuild-start} option value, then the time interval
- extends through midnight. The \cb{build-alt-rebuild-start} and
- \cb{build-alt-rebuild-stop} options must be either both specified or
- absent. If unspecified, then no alternative rebuild timeout will be
- used."
+ "The end time of the alternative package soft rebuild timeout (see
+ \cb{build-alt-soft-rebuild-timeout} for details). Must be specified
+ as a time of day in the local timezone. If it is less than the
+ \cb{build-alt-soft-rebuild-start} option value, then the time
+ interval extends through midnight. The
+ \cb{build-alt-soft-rebuild-start} and
+ \cb{build-alt-soft-rebuild-stop} options must be either both
+ specified or absent. If unspecified, then no alternative rebuild
+ timeout will be used."
+ }
+
+ size_t build-hard-rebuild-timeout = 604800
+ {
+ "<seconds>",
+ "Time to wait before considering a package for a hard rebuild (to be
+ performed unconditionally). Must be specified in seconds. The special
+ zero value disables hard rebuilds. Default is 7 days."
+ }
+
+ size_t build-alt-hard-rebuild-timeout
+ {
+ "<seconds>",
+ "Alternative package hard rebuild timeout. The semantics is the
+ same as for the \cb{build-alt-soft-rebuild-timeout} option but
+ for the \cb{build-hard-rebuild-timeout} option."
+ }
+
+ duration build-alt-hard-rebuild-start
+ {
+ "<hours>:<minutes>",
+ "The start time of the alternative package hard rebuild timeout (see
+ \cb{build-alt-hard-rebuild-timeout} for details). The semantics is
+ the same as for the \cb{build-alt-soft-rebuild-start} option but
+ for the \cb{build-hard-rebuild-timeout} option."
+ }
+
+ duration build-alt-hard-rebuild-stop
+ {
+ "<hours>:<minutes>",
+ "The end time of the alternative package hard rebuild timeout (see
+ \cb{build-alt-hard-rebuild-timeout} for details). The semantics is
+ the same as for the \cb{build-alt-soft-rebuild-stop} option but
+ for the \cb{build-hard-rebuild-timeout} option."
}
};
diff --git a/monitor/monitor.cli b/monitor/monitor.cli
index edfc004..3a58a1d 100644
--- a/monitor/monitor.cli
+++ b/monitor/monitor.cli
@@ -60,18 +60,26 @@ namespace brep
{
"\h|OPTIONS|"
- std::size_t --build-timeout
+ std::size_t --soft-rebuild-timeout
{
"<seconds>",
- "Time to wait (in seconds) before considering a package build as
+ "Time to wait (in seconds) before considering a package soft (re)build as
delayed. If unspecified, it is the sum of the package rebuild timeout
- (normal rebuild timeout if the alternative timeout is unspecified and
- the maximum of two otherwise) and the build result timeout (see the
- \cb{build-normal-rebuild-timeout}, \cb{build-alt-rebuild-*}, and
- \cb{build-result-timeout} \c{brep} module configuration options
- for details).
-
- Note that a package that was not built before it was archived is
+ (soft rebuild timeout if the alternative timeout is unspecified and
+ the maximum of two otherwise) and the build result timeout (see
+ the \cb{build-soft-rebuild-timeout}, \cb{build-alt-soft-rebuild-*},
+ and \cb{build-result-timeout} \cb{brep} module configuration options
+ for details). The special zero value disables monitoring of soft
+ rebuilds.
+
+ Note that if both soft and hard rebuilds are disabled in the
+ \cb{brep} module configuration, then \cb{brep-monitor} is unable to
+ come up with a reasonable build timeout on its own. In this case, to
+ monitor the initial package build delays, you may need to specify
+ either \cb{--soft-rebuild-timeout} or \cb{--hard-rebuild-timeout}
+ explicitly.
+
+ Also note that a package that was not built before it was archived is
always considered as delayed. However, to distinguish this case from
a situation where a package was archived before a configuration have
been added, \cb{brep-monitor} needs to observe the package as
@@ -81,6 +89,16 @@ namespace brep
timeout."
}
+ std::size_t --hard-rebuild-timeout
+ {
+ "<seconds>",
+ "Time to wait (in seconds) before considering a package hard (re)build
+ as delayed. If unspecified, it is calculated in the same way as for
+ \cb{--soft-rebuild-timeout} but using the
+ \cb{build-hard-rebuild-timeout} and \cb{build-alt-hard-rebuild-*}
+ \cb{brep} module configuration options."
+ }
+
std::size_t --report-timeout
{
"<seconds>",
diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx
index 5c4d451..7c20efe 100644
--- a/monitor/monitor.cxx
+++ b/monitor/monitor.cxx
@@ -41,6 +41,212 @@ namespace brep
//
struct failed {};
+ // We will collect and report build delays as separate steps not to hold
+ // database locks while printing to stderr. Also we need to order delays
+ // properly, so while printing reports we could group delays by toolchain
+ // and configuration.
+ //
+ // To achieve that, we will iterate through all possible package builds
+ // creating the list of delays with the following sort priority:
+ //
+ // 1: toolchain name
+ // 2: toolchain version (descending)
+ // 3: configuration name
+ // 4: tenant
+ // 5: package name
+ // 6: package version (descending)
+ //
+ struct compare_delay
+ {
+ bool
+ operator() (const shared_ptr<const build_delay>& x,
+ const shared_ptr<const build_delay>& y) const
+ {
+ if (int r = x->toolchain_name.compare (y->toolchain_name))
+ return r < 0;
+
+ if (int r = x->toolchain_version.compare (y->toolchain_version))
+ return r > 0;
+
+ if (int r = x->configuration.compare (y->configuration))
+ return r < 0;
+
+ if (int r = x->tenant.compare (y->tenant))
+ return r < 0;
+
+ if (int r = x->package_name.compare (y->package_name))
+ return r < 0;
+
+ return x->package_version.compare (y->package_version) > 0;
+ }
+ };
+
+ // The ordered list of delays to report.
+ //
+ class delay_report
+ {
+ public:
+ // Note that in the brief mode we also need to print the total number of
+ // delays (reported or not) per configuration. Thus, we add all delays to
+ // the report object, marking them if we need to report them or not.
+ //
+ void
+ add_delay (shared_ptr<build_delay>, bool report);
+
+ bool
+ empty () const
+ {
+ return reported_delay_count_ == 0;
+ }
+
+ // In the brief mode (if full is false) print the number of reported/total
+ // (if total is true) delayed package builds per configuration rather than
+ // the packages themselves.
+ //
+ void
+ print (const char* header, bool total, bool full) const;
+
+ private:
+ // Maps delays to the report flag.
+ //
+ map<shared_ptr<const build_delay>, bool, compare_delay> delays_;
+ size_t reported_delay_count_ = 0;
+ };
+
+ void delay_report::
+ add_delay (shared_ptr<build_delay> delay, bool report)
+ {
+ delays_.emplace (move (delay), report);
+
+ if (report)
+ ++reported_delay_count_;
+ }
+
+ void delay_report::
+ print (const char* header, bool total, bool full) const
+ {
+ if (empty ())
+ return;
+
+ cerr << header << " (" << reported_delay_count_;
+
+ if (total)
+ cerr << '/' << delays_.size ();
+
+ cerr << "):" << endl;
+
+ // Group the printed delays by toolchain and configuration.
+ //
+ const string* toolchain_name (nullptr);
+ const version* toolchain_version (nullptr);
+ const string* configuration (nullptr);
+
+ size_t config_reported_delay_count (0);
+ size_t config_total_delay_count (0);
+
+ auto brief_config = [&configuration,
+ &config_reported_delay_count,
+ &config_total_delay_count,
+ total] ()
+ {
+ if (configuration != nullptr)
+ {
+ // Only print configurations with delays that needs to be reported.
+ //
+ if (config_reported_delay_count != 0)
+ {
+ cerr << " " << *configuration << " ("
+ << config_reported_delay_count;
+
+ if (total)
+ cerr << '/' << config_total_delay_count;
+
+ cerr << ')' << endl;
+ }
+
+ config_reported_delay_count = 0;
+ config_total_delay_count = 0;
+ }
+ };
+
+ for (const auto& dr: delays_)
+ {
+ bool report (dr.second);
+
+ if (full && !report)
+ continue;
+
+ const shared_ptr<const build_delay>& d (dr.first);
+
+ // Print the toolchain, if changed.
+ //
+ if (toolchain_name == nullptr ||
+ d->toolchain_name != *toolchain_name ||
+ d->toolchain_version != *toolchain_version)
+ {
+ if (!full)
+ brief_config ();
+
+ if (toolchain_name != nullptr)
+ cerr << endl;
+
+ cerr << " " << d->toolchain_name;
+
+ if (!d->toolchain_version.empty ())
+ cerr << "/" << d->toolchain_version;
+
+ cerr << endl;
+
+ toolchain_name = &d->toolchain_name;
+ toolchain_version = &d->toolchain_version;
+ configuration = nullptr;
+ }
+
+ // Print the configuration, if changed.
+ //
+ if (configuration == nullptr || d->configuration != *configuration)
+ {
+ if (full)
+ {
+ if (configuration != nullptr)
+ cerr << endl;
+
+ cerr << " " << d->configuration << endl;
+ }
+ else
+ brief_config ();
+
+ configuration = &d->configuration;
+ }
+
+ // Print the delayed build package in the full report mode and count
+ // configuration builds otherwise.
+ //
+ if (full)
+ {
+ // We can potentially extend this information with the archived flag
+ // or the delay duration.
+ //
+ cerr << " " << d->package_name << "/" << d->package_version;
+
+ if (!d->tenant.empty ())
+ cerr << " " << d->tenant;
+
+ cerr << endl;
+ }
+ else
+ {
+ if (report)
+ ++config_reported_delay_count;
+
+ ++config_total_delay_count;
+ }
+ }
+
+ if (!full)
+ brief_config ();
+ }
+
static const char* help_info (
" info: run 'brep-monitor --help' for more information");
@@ -141,12 +347,24 @@ namespace brep
return 1;
}
- if (mod_ops.build_alt_rebuild_start_specified () !=
- mod_ops.build_alt_rebuild_stop_specified ())
+ auto bad_alt = [&f] (const char* what)
+ {
+ cerr << "build-alt-" << what << "-rebuild-start and build-alt-"
+ << what << "-rebuild-stop configuration options must both be "
+ << "either specified or not in '" << f << "'" << endl;
+ };
+
+ if (mod_ops.build_alt_hard_rebuild_start_specified () !=
+ mod_ops.build_alt_hard_rebuild_stop_specified ())
+ {
+ bad_alt("hard");
+ return 1;
+ }
+
+ if (mod_ops.build_alt_soft_rebuild_start_specified () !=
+ mod_ops.build_alt_soft_rebuild_stop_specified ())
{
- cerr << "build-alt-rebuild-start and build-alt-rebuild-stop "
- << "configuration options must both be either specified or not "
- << "in '" << f << "'" << endl;
+ bad_alt("soft");
return 1;
}
}
@@ -384,59 +602,16 @@ namespace brep
}
}
- // Collect and report delays as separate steps not to hold database locks
- // while printing to stderr. Also we need to properly order delays for
- // printing.
- //
- // Iterate through all possible package builds creating the list of delays
- // with the following sort priority:
- //
- // 1: toolchain name
- // 2: toolchain version (descending)
- // 3: configuration name
- // 4: tenant
- // 5: package name
- // 6: package version (descending)
- //
- // Such ordering will allow us to group build delays by toolchain and
- // configuration while printing the report.
- //
- struct compare_delay
- {
- bool
- operator() (const shared_ptr<const build_delay>& x,
- const shared_ptr<const build_delay>& y) const
- {
- if (int r = x->toolchain_name.compare (y->toolchain_name))
- return r < 0;
-
- if (int r = x->toolchain_version.compare (y->toolchain_version))
- return r > 0;
-
- if (int r = x->configuration.compare (y->configuration))
- return r < 0;
-
- if (int r = x->tenant.compare (y->tenant))
- return r < 0;
-
- if (int r = x->package_name.compare (y->package_name))
- return r < 0;
-
- return x->package_version.compare (y->package_version) > 0;
- }
- };
-
- size_t reported_delay_count (0);
- size_t total_delay_count (0);
-
- set<shared_ptr<const build_delay>, compare_delay> delays;
+ delay_report hard_delays_report;
+ delay_report soft_delays_report;
+ set<shared_ptr<const build_delay>, compare_delay> update_delays;
{
connection_ptr conn (db.connection ());
// Prepare the buildable package prepared query.
//
- // Query buildable packages in chunks in order not to hold locks for
- // too long.
+ // Query buildable packages in chunks in order not to hold locks for too
+ // long.
//
using pquery = query<buildable_package>;
using prep_pquery = prepared_query<buildable_package>;
@@ -463,6 +638,11 @@ namespace brep
// across all toolchain versions, if present, and the latest incomplete
// build otherwise.
//
+ // Why don't we pick the latest toolchain version? We don't want to
+ // stuck with it on the toolchain rollback. Instead we prefer the
+ // toolchain that built the package last and if there are none, pick the
+ // one for which the build task was issued last.
+ //
using bquery = query<package_build>;
using prep_bquery = prepared_query<package_build>;
@@ -473,63 +653,126 @@ namespace brep
bid.configuration == bquery::_ref (id.configuration) &&
bid.toolchain_name == bquery::_ref (id.toolchain_name)) +
"ORDER BY" +
- bquery::build::completion_timestamp + "DESC, " +
+ bquery::build::soft_timestamp + "DESC, " +
bquery::build::timestamp + "DESC" +
"LIMIT 1");
prep_bquery pbq (
conn->prepare_query<package_build> ("package-build-query", bq));
- duration build_timeout;
+ timestamp now (system_clock::now ());
- // If the build timeout is not specified explicitly, then calculate it
- // as the sum of the package rebuild timeout (normal rebuild timeout if
- // the alternative timeout is unspecified and the maximum of two
- // otherwise) and the build result timeout.
+ // Calculate the build/rebuild expiration time, based on the respective
+ // --{soft,hard}-rebuild-timeout monitor options and the
+ // build-{soft,hard}-rebuild-timeout and
+ // build-alt-{soft,hard}-rebuild-{start,stop,timeout} brep module
+ // configuration options.
+ //
+ // If the --*-rebuild-timeout monitor option is zero or is not specified
+ // and the respective build-*-rebuild-timeout brep's configuration
+ // option is zero, then return timestamp_unknown to indicate 'never
+ // expire'. Note that this value is less than any build timestamp value,
+ // including timestamp_nonexistent.
//
- if (!ops.build_timeout_specified ())
+ // NOTE: there is a similar code in mod/mod-build-task.cxx.
+ //
+ auto build_expiration = [&now, &mod_ops] (
+ optional<size_t> rebuild_timeout,
+ const optional<pair<duration, duration>>& alt_interval,
+ optional<size_t> alt_timeout,
+ size_t normal_timeout)
{
- duration normal_rebuild_timeout (
- chrono::seconds (mod_ops.build_normal_rebuild_timeout ()));
+ duration t;
- if (mod_ops.build_alt_rebuild_start_specified ())
+ // If the rebuild timeout is not specified explicitly, then calculate
+ // it as the sum of the package rebuild timeout (normal rebuild
+ // timeout if the alternative timeout is unspecified and the maximum
+ // of two otherwise) and the build result timeout.
+ //
+ if (!rebuild_timeout)
{
- // Calculate the alternative rebuild timeout as the time interval
- // lenght, unless it is specified explicitly.
- //
- if (!mod_ops.build_alt_rebuild_timeout_specified ())
+ if (normal_timeout == 0)
+ return timestamp_unknown;
+
+ chrono::seconds nt (normal_timeout);
+
+ if (alt_interval)
{
- const duration& start (mod_ops.build_alt_rebuild_start ());
- const duration& stop (mod_ops.build_alt_rebuild_stop ());
+ // Calculate the alternative timeout, unless it is specified
+ // explicitly.
+ //
+ if (!alt_timeout)
+ {
+ const duration& start (alt_interval->first);
+ const duration& stop (alt_interval->second);
- // Note that if the stop time is less than the start time then the
- // interval extends through the midnight.
+ // Note that if the stop time is less than the start time then
+ // the interval extends through the midnight.
+ //
+ t = start <= stop ? (stop - start) : ((24h - start) + stop);
+
+ // If the normal rebuild time out is greater than 24 hours, then
+ // increase the default alternative timeout by (normal - 24h)
+ // (see build-alt-soft-rebuild-timeout configuration option for
+ // details).
+ //
+ if (nt > 24h)
+ t += nt - 24h;
+ }
+ else
+ t = chrono::seconds (*alt_timeout);
+
+ // Take the maximum of the alternative and normal rebuild
+ // timeouts.
//
- build_timeout = start <= stop
- ? stop - start
- : (24h - start) + stop;
+ if (t < nt)
+ t = nt;
}
else
- build_timeout =
- chrono::seconds (mod_ops.build_alt_rebuild_timeout ());
+ t = nt;
- // Take the maximum of the alternative and normal rebuild timeouts.
+ // Summarize the rebuild and build result timeouts.
//
- if (build_timeout < normal_rebuild_timeout)
- build_timeout = normal_rebuild_timeout;
+ t += chrono::seconds (mod_ops.build_result_timeout ());
}
else
- build_timeout = normal_rebuild_timeout;
+ {
+ if (*rebuild_timeout == 0)
+ return timestamp_unknown;
- // Summarize the rebuild and build result timeouts.
- //
- build_timeout += chrono::seconds (mod_ops.build_result_timeout ());
- }
- else
- build_timeout = chrono::seconds (ops.build_timeout ());
+ t = chrono::seconds (*rebuild_timeout);
+ }
- timestamp now (system_clock::now ());
- timestamp build_expiration (now - build_timeout);
+ return now - t;
+ };
+
+ timestamp hard_rebuild_expiration (
+ build_expiration (
+ (ops.hard_rebuild_timeout_specified ()
+ ? ops.hard_rebuild_timeout ()
+ : optional<size_t> ()),
+ (mod_ops.build_alt_hard_rebuild_start_specified ()
+ ? make_pair (mod_ops.build_alt_hard_rebuild_start (),
+ mod_ops.build_alt_hard_rebuild_stop ())
+ : optional<pair<duration, duration>> ()),
+ (mod_ops.build_alt_hard_rebuild_timeout_specified ()
+ ? mod_ops.build_alt_hard_rebuild_timeout ()
+ : optional<size_t> ()),
+ mod_ops.build_hard_rebuild_timeout ()));
+
+ timestamp soft_rebuild_expiration (
+ build_expiration (
+ (ops.soft_rebuild_timeout_specified ()
+ ? ops.soft_rebuild_timeout ()
+ : optional<size_t> ()),
+ (mod_ops.build_alt_soft_rebuild_start_specified ()
+ ? make_pair (mod_ops.build_alt_soft_rebuild_start (),
+ mod_ops.build_alt_soft_rebuild_stop ())
+ : optional<pair<duration, duration>> ()),
+ (mod_ops.build_alt_soft_rebuild_timeout_specified ()
+ ? mod_ops.build_alt_soft_rebuild_timeout ()
+ : optional<size_t> ()),
+ mod_ops.build_soft_rebuild_timeout ()));
timestamp report_expiration (
now - chrono::seconds (ops.report_timeout ()));
@@ -583,8 +826,12 @@ namespace brep
// task have been issued recently we may still consider the
// build as delayed.
//
- timestamp bct (b != nullptr
- ? b->completion_timestamp
+ timestamp bht (b != nullptr
+ ? b->hard_timestamp
+ : timestamp_nonexistent);
+
+ timestamp bst (b != nullptr
+ ? b->soft_timestamp
: timestamp_nonexistent);
// Create the delay object to record a timestamp when the
@@ -605,15 +852,17 @@ namespace brep
if (bp.archived && b == nullptr)
continue;
- // Use the build completion or build status change
- // timestamp, whichever is earlier, as the build delay
- // tracking starting point and fallback to the current time
- // if there is no build yet.
+ // Use the build hard, soft, or status change timestamp (see
+ // the timestamps description for their ordering
+ // information) as the build delay tracking starting point
+ // and fallback to the current time if there is no build
+ // yet.
//
timestamp pts (
- b == nullptr ? now :
- bct != timestamp_nonexistent && bct < b->timestamp ? bct :
- b->timestamp);
+ b == nullptr ? now :
+ bht != timestamp_nonexistent ? bht :
+ bst != timestamp_nonexistent ? bst :
+ b->timestamp);
d = make_shared<build_delay> (move (id.package.tenant),
move (id.package.name),
@@ -632,20 +881,40 @@ namespace brep
// if it is not (re-)built by the expiration time. Otherwise,
// consider it as delayed if it is unbuilt.
//
- bool delayed;
+ // We also don't need to report an unbuilt archived package
+ // twice, as both soft and hard build delays.
+ //
+ bool hard_delayed;
+ bool soft_delayed;
if (!bp.archived)
{
- timestamp bts (bct != timestamp_nonexistent
- ? bct
+ auto delayed = [&d] (timestamp bt, timestamp be)
+ {
+ timestamp t (bt != timestamp_nonexistent
+ ? bt
: d->package_timestamp);
+ return t <= be;
+ };
- delayed = (bts <= build_expiration);
+ hard_delayed = delayed (bht, hard_rebuild_expiration);
+ soft_delayed = delayed (bst, soft_rebuild_expiration);
}
else
- delayed = (bct == timestamp_nonexistent);
+ {
+ hard_delayed = (bst == timestamp_nonexistent);
+ soft_delayed = false;
+ }
- if (delayed)
+ // Add hard/soft delays to the respective reports and collect
+ // the delay for update, if it is reported.
+ //
+ // Note that we update the delay objects persistent state
+ // later, after we successfully print the reports.
+ //
+ bool reported (false);
+
+ if (hard_delayed)
{
// If the report timeout is zero then report the delay
// unconditionally. Otherwise, report the active package
@@ -655,33 +924,43 @@ namespace brep
// building an archived package, so reporting its build
// delays repeatedly is meaningless.
//
- if (ops.report_timeout () == 0 ||
- (!bp.archived
- ? d->report_timestamp <= report_expiration
- : d->report_timestamp == timestamp_nonexistent))
- {
- // Note that we update the delay objects persistent state
- // later, after we successfully print the report.
- //
- d->report_timestamp = now;
- delays.insert (move (d));
+ bool report (
+ ops.report_timeout () == 0 ||
+ (!bp.archived
+ ? d->report_hard_timestamp <= report_expiration
+ : d->report_hard_timestamp == timestamp_nonexistent));
- ++reported_delay_count;
+ if (report)
+ {
+ d->report_hard_timestamp = now;
+ reported = true;
}
- //
- // In the brief mode also collect unreported delays to
- // deduce and print the total number of delays per
- // configuration. Mark such delays with the
- // timestamp_nonexistent report timestamp.
- //
- else if (!ops.full_report ())
+
+ hard_delays_report.add_delay (d, report);
+ }
+
+ if (soft_delayed)
+ {
+ bool report (ops.report_timeout () == 0 ||
+ d->report_soft_timestamp <= report_expiration);
+
+ if (report)
{
- d->report_timestamp = timestamp_nonexistent;
- delays.insert (move (d));
+ d->report_soft_timestamp = now;
+ reported = true;
}
- ++total_delay_count;
+ soft_delays_report.add_delay (d, report);
}
+
+ // If we don't consider the report timestamps for reporting
+ // delays, it seems natural not to update these timestamps
+ // either. Note that reporting all delays and still updating
+ // the report timestamps can be achieved by specifying the
+ // zero report timeout.
+ //
+ if (reported && ops.report_timeout_specified ())
+ update_delays.insert (move (d));
}
}
}
@@ -691,161 +970,48 @@ namespace brep
}
}
- // Report package build delays, if any.
+ // Print delay reports, if not empty.
//
- if (reported_delay_count != 0)
+ if (!hard_delays_report.empty () || !soft_delays_report.empty ())
try
{
- // Print the report.
- //
cerr.exceptions (ostream::badbit | ostream::failbit);
// Don't print the total delay count if the report timeout is zero since
// all delays are reported in this case.
//
- bool print_total_delay_count (ops.report_timeout () != 0);
-
- cerr << "Package build delays (" << reported_delay_count;
-
- if (print_total_delay_count)
- cerr << '/' << total_delay_count;
-
- cerr << "):" << endl;
-
- // Group the printed delays by toolchain and configuration.
- //
- const string* toolchain_name (nullptr);
- const version* toolchain_version (nullptr);
- const string* configuration (nullptr);
-
- // In the brief report mode print the number of reported/total delayed
- // package builds per configuration rather than the packages themselves.
- //
- size_t config_reported_delay_count (0);
- size_t config_total_delay_count (0);
-
- auto brief_config = [&configuration,
- &config_reported_delay_count,
- &config_total_delay_count,
- print_total_delay_count] ()
- {
- if (configuration != nullptr)
- {
- // Only print configurations with delays that needs to be reported.
- //
- if (config_reported_delay_count != 0)
- {
- cerr << " " << *configuration << " ("
- << config_reported_delay_count;
+ bool total (ops.report_timeout () != 0);
- if (print_total_delay_count)
- cerr << '/' << config_total_delay_count;
+ hard_delays_report.print ("Package hard rebuild delays",
+ total,
+ ops.full_report ());
- cerr << ')' << endl;
- }
-
- config_reported_delay_count = 0;
- config_total_delay_count = 0;
- }
- };
-
- for (shared_ptr<const build_delay> d: delays)
- {
- // Print the toolchain, if changed.
- //
- if (toolchain_name == nullptr ||
- d->toolchain_name != *toolchain_name ||
- d->toolchain_version != *toolchain_version)
- {
- if (!ops.full_report ())
- brief_config ();
-
- if (toolchain_name != nullptr)
- cerr << endl;
-
- cerr << " " << d->toolchain_name;
-
- if (!d->toolchain_version.empty ())
- cerr << "/" << d->toolchain_version;
-
- cerr << endl;
-
- toolchain_name = &d->toolchain_name;
- toolchain_version = &d->toolchain_version;
- configuration = nullptr;
- }
-
- // Print the configuration, if changed.
- //
- if (configuration == nullptr || d->configuration != *configuration)
- {
- if (ops.full_report ())
- {
- if (configuration != nullptr)
- cerr << endl;
-
- cerr << " " << d->configuration << endl;
- }
- else
- brief_config ();
-
- configuration = &d->configuration;
- }
-
- // Print the delayed build package in the full report mode and count
- // configuration builds otherwise.
- //
- if (ops.full_report ())
- {
- // We can potentially extend this information with the archived flag
- // or the delay duration.
- //
- cerr << " " << d->package_name << "/" << d->package_version;
-
- if (!d->tenant.empty ())
- cerr << " " << d->tenant;
-
- cerr << endl;
- }
- else
- {
- if (d->report_timestamp != timestamp_nonexistent)
- ++config_reported_delay_count;
-
- ++config_total_delay_count;
- }
- }
-
- if (!ops.full_report ())
- brief_config ();
-
- // Persist the delay report timestamps.
- //
- // If we don't consider the report timestamps for reporting delays, it
- // seems natural not to update these timestamps either. Note that
- // reporting all delays and still updating the report timestamps can be
- // achieved by specifying the zero report timeout.
+ // Separate reports with an empty line.
//
- if (ops.report_timeout_specified ())
- {
- transaction t (db.begin ());
+ if (!hard_delays_report.empty () && !soft_delays_report.empty ())
+ cerr << endl;
- for (shared_ptr<const build_delay> d: delays)
- {
- // Only update timestamps for delays that needs to be reported.
- //
- if (d->report_timestamp != timestamp_nonexistent)
- db.update (d);
- }
-
- t.commit ();
- }
+ soft_delays_report.print ("Package soft rebuild delays",
+ total,
+ ops.full_report ());
}
catch (const io_error&)
{
return 1; // Not much we can do on stderr writing failure.
}
+ // Persist the delay report timestamps.
+ //
+ if (!update_delays.empty ())
+ {
+ transaction t (db.begin ());
+
+ for (shared_ptr<const build_delay> d: update_delays)
+ db.update (d);
+
+ t.commit ();
+ }
+
return 0;
}
catch (const database_locked&)