aboutsummaryrefslogtreecommitdiff
path: root/mod
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2021-09-27 11:09:51 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2021-10-04 18:49:09 +0300
commitb7ff8f89cea055e75881e716d8358ffa4d7779af (patch)
tree38c41d7b144d86c23b42519a9e4e683be62ccbb4 /mod
parentac80e3c37f1fee068e46b6f5a2043581d413e992 (diff)
Add support for soft and hard rebuilds
Diffstat (limited to 'mod')
-rw-r--r--mod/buildfile2
-rw-r--r--mod/mod-build-result.cxx99
-rw-r--r--mod/mod-build-task.cxx262
-rw-r--r--mod/module.cli90
4 files changed, 356 insertions, 97 deletions
diff --git a/mod/buildfile b/mod/buildfile
index ff9cd60..58a3caf 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -50,7 +50,7 @@ if $cli.configured
cli.options += --std c++11 -I $src_root --include-with-brackets \
--include-prefix mod --guard-prefix MOD --generate-specifier \
--cxx-prologue "#include <mod/types-parsers.hxx>" \
---cli-namespace brep::cli --generate-file-scanner --option-length 45 \
+--cli-namespace brep::cli --generate-file-scanner --option-length 46 \
--generate-modifier --generate-description --option-prefix ""
# Include the generated cli files into the distribution and don't remove
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 3ae9f0f..1c46fc1 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -287,7 +287,6 @@ handle (request& rq, response&)
//
shared_ptr<build> bld;
- optional<result_status> prev_status;
bool build_notify (false);
bool unforced (true);
@@ -382,6 +381,58 @@ handle (request& rq, response&)
if (auth)
{
+ // Verify the result status/checksums.
+ //
+ // Specifically, if the result status is skip, then it can only be in
+ // response to the soft rebuild task (all checksums are present in the
+ // build object) and the result checksums must match the build object
+ // checksums. On verification failure respond with the bad request
+ // HTTP code (400).
+ //
+ if (rqm.result.status == result_status::skip)
+ {
+ if (!b->agent_checksum ||
+ !b->worker_checksum ||
+ !b->dependency_checksum)
+ throw invalid_request (400, "unexpected skip result status");
+
+ // Can only be absent for initial build, in which case the checksums
+ // are also absent and we would end up with the above 400 response.
+ //
+ assert (b->status);
+
+ // Verify that the result checksum matches the build checksum and
+ // throw invalid_request(400) if that's not the case.
+ //
+ auto verify = [] (const string& build_checksum,
+ const optional<string>& result_checksum,
+ const char* what)
+ {
+ if (!result_checksum)
+ throw invalid_request (
+ 400,
+ string (what) +
+ " checksum is expected for skip result status");
+
+ if (*result_checksum != build_checksum)
+ throw invalid_request (
+ 400,
+ string (what) + " checksum '" + build_checksum +
+ "' is expected instead of '" + *result_checksum +
+ "' for skip result status");
+ };
+
+ verify (*b->agent_checksum, rqm.agent_checksum, "agent");
+
+ verify (*b->worker_checksum,
+ rqm.result.worker_checksum,
+ "worker");
+
+ verify (*b->dependency_checksum,
+ rqm.result.dependency_checksum,
+ "dependency");
+ }
+
unforced = b->force == force_state::unforced;
// Don't send email to the build-email address for the
@@ -392,10 +443,7 @@ handle (request& rq, response&)
*b->status == rqm.result.status &&
unforced);
- prev_status = move (b->status);
-
b->state = build_state::built;
- b->status = rqm.result.status;
b->force = force_state::unforced;
// Cleanup the interactive build login information.
@@ -407,22 +455,43 @@ handle (request& rq, response&)
b->agent_fingerprint = nullopt;
b->agent_challenge = nullopt;
- // Mark the section as loaded, so results are updated.
- //
- b->results_section.load ();
- b->results = move (rqm.result.results);
-
b->timestamp = system_clock::now ();
- b->completion_timestamp = b->timestamp;
+ b->soft_timestamp = b->timestamp;
+
+ // If the result status is other than skip, then save the status,
+ // results, and checksums and update the hard timestamp.
+ //
+ if (rqm.result.status != result_status::skip)
+ {
+ b->status = rqm.result.status;
+ b->hard_timestamp = b->soft_timestamp;
+
+ // Mark the section as loaded, so results are updated.
+ //
+ b->results_section.load ();
+ b->results = move (rqm.result.results);
+
+ // Save the checksums.
+ //
+ b->agent_checksum = move (rqm.agent_checksum);
+ b->worker_checksum = move (rqm.result.worker_checksum);
+ b->dependency_checksum = move (rqm.result.dependency_checksum);
+ }
build_db_->update (b);
- shared_ptr<build_package> p (
- build_db_->load<build_package> (b->id.package));
+ // Don't send the build notification email if the task result is
+ // `skip`, the configuration is hidden, or is now excluded by the
+ // package.
+ //
+ if (rqm.result.status != result_status::skip && belongs (*cfg, "all"))
+ {
+ shared_ptr<build_package> p (
+ build_db_->load<build_package> (b->id.package));
- if (belongs (*cfg, "all") &&
- !exclude (p->builds, p->constraints, *cfg))
- bld = move (b);
+ if (!exclude (p->builds, p->constraints, *cfg))
+ bld = move (b);
+ }
}
}
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 8656f5e..22d0110 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -61,13 +61,23 @@ init (scanner& s)
if (options_->build_config_specified ())
{
- // Verify that build-alt-rebuild-{start,stop} are both either specified or
- // not.
+ // Verify that build-alt-*-rebuild-{start,stop} are both either specified
+ // or not.
//
- if (options_->build_alt_rebuild_start_specified () !=
- options_->build_alt_rebuild_stop_specified ())
- fail << "build-alt-rebuild-start and build-alt-rebuild-stop "
- << "configuration options must both be either specified or not";
+ auto bad_alt = [&fail] (const char* what)
+ {
+ fail << "build-alt-" << what << "-rebuild-start and build-alt-" << what
+ << "-rebuild-stop configuration options must both be either "
+ << "specified or not";
+ };
+
+ if (options_->build_alt_soft_rebuild_start_specified () !=
+ options_->build_alt_soft_rebuild_stop_specified ())
+ bad_alt ("soft");
+
+ if (options_->build_alt_hard_rebuild_start_specified () !=
+ options_->build_alt_hard_rebuild_stop_specified ())
+ bad_alt ("hard");
database_module::init (*options_, options_->build_db_retry ());
@@ -277,22 +287,25 @@ handle (request& rq, response& rs)
move (fps),
move (p->requirements),
move (tests),
+ move (b->dependency_checksum),
cm.machine->name,
cm.config->target,
cm.config->environment,
cm.config->args,
belongs (*cm.config, module_pkg ? "build2" : "host"),
cm.config->warning_regexes,
- move (t->interactive));
+ move (t->interactive),
+ move (b->worker_checksum));
return task_response_manifest (move (session),
move (b->agent_challenge),
move (result_url),
+ move (b->agent_checksum),
move (task));
};
- // Calculate the build (building state) or rebuild (built state) expiration
- // time for package configurations
+ // Calculate the build (building state) or rebuild (built state)
+ // expiration time for package configurations.
//
timestamp now (system_clock::now ());
@@ -316,44 +329,95 @@ handle (request& rq, response& rs)
timestamp forced_rebuild_expiration (
expiration (options_->build_forced_rebuild_timeout ()));
- timestamp normal_rebuild_expiration;
-
- if (options_->build_alt_rebuild_start_specified ())
+ // Calculate the soft/hard rebuild expiration time, based on the
+ // respective build-{soft,hard}-rebuild-timeout and
+ // build-alt-{soft,hard}-rebuild-{start,stop,timeout} configuration
+ // options.
+ //
+ // If normal_timeout is zero, then return timestamp_unknown to indicate
+ // 'never expire'. Note that this value is less than any build timestamp
+ // value, including timestamp_nonexistent.
+ //
+ // NOTE: there is a similar code in monitor/monitor.cxx.
+ //
+ auto build_expiration = [&now] (
+ const optional<pair<duration, duration>>& alt_interval,
+ optional<size_t> alt_timeout,
+ size_t normal_timeout)
{
- const duration& start (options_->build_alt_rebuild_start ());
- const duration& stop (options_->build_alt_rebuild_stop ());
+ if (normal_timeout == 0)
+ return timestamp_unknown;
- duration dt (daytime (now));
+ timestamp r;
+ chrono::seconds nt (normal_timeout);
- // Note that if the stop time is less than the start time then the
- // interval extends through the midnight.
- //
- bool alt_timeout (start <= stop
- ? dt >= start && dt < stop
- : dt >= start || dt < stop);
-
- // If we out of the alternative rebuild timeout interval, then fall back
- // to using the normal rebuild timeout.
- //
- if (alt_timeout)
+ if (alt_interval)
{
- if (!options_->build_alt_rebuild_timeout_specified ())
+ const duration& start (alt_interval->first);
+ const duration& stop (alt_interval->second);
+
+ duration dt (daytime (now));
+
+ // Note that if the stop time is less than the start time then the
+ // interval extends through the midnight.
+ //
+ bool use_alt_timeout (start <= stop
+ ? dt >= start && dt < stop
+ : dt >= start || dt < stop);
+
+ // If we out of the alternative rebuild timeout interval, then fall
+ // back to using the normal rebuild timeout.
+ //
+ if (use_alt_timeout)
{
- duration interval_len (start <= stop
- ? stop - start
- : (24h - start) + stop);
+ // Calculate the alternative timeout, unless it is specified
+ // explicitly.
+ //
+ duration t;
+
+ if (!alt_timeout)
+ {
+ t = start <= stop ? (stop - start) : ((24h - start) + stop);
- normal_rebuild_expiration = now - interval_len;
+ // If the normal rebuild timeout is greater than 24 hours, then
+ // increase the default alternative timeout by (normal - 24h) (see
+ // build-alt-soft-rebuild-timeout configuration option for
+ // details).
+ //
+ if (nt > 24h)
+ t += nt - 24h;
+ }
+ else
+ t = chrono::seconds (*alt_timeout);
+
+ r = now - t;
}
- else
- normal_rebuild_expiration =
- expiration (options_->build_alt_rebuild_timeout ());
}
- }
- if (normal_rebuild_expiration == timestamp_nonexistent)
- normal_rebuild_expiration =
- expiration (options_->build_normal_rebuild_timeout ());
+ return r != timestamp_nonexistent ? r : (now - nt);
+ };
+
+ timestamp soft_rebuild_expiration (
+ build_expiration (
+ (options_->build_alt_soft_rebuild_start_specified ()
+ ? make_pair (options_->build_alt_soft_rebuild_start (),
+ options_->build_alt_soft_rebuild_stop ())
+ : optional<pair<duration, duration>> ()),
+ (options_->build_alt_soft_rebuild_timeout_specified ()
+ ? options_->build_alt_soft_rebuild_timeout ()
+ : optional<size_t> ()),
+ options_->build_soft_rebuild_timeout ()));
+
+ timestamp hard_rebuild_expiration (
+ build_expiration (
+ (options_->build_alt_hard_rebuild_start_specified ()
+ ? make_pair (options_->build_alt_hard_rebuild_start (),
+ options_->build_alt_hard_rebuild_stop ())
+ : optional<pair<duration, duration>> ()),
+ (options_->build_alt_hard_rebuild_timeout_specified ()
+ ? options_->build_alt_hard_rebuild_timeout ()
+ : optional<size_t> ()),
+ options_->build_hard_rebuild_timeout ()));
// Return the challenge (nonce) if brep is configured to authenticate bbot
// agents. Return nullopt otherwise.
@@ -555,6 +619,68 @@ handle (request& rq, response& rs)
prep_bld_query bld_prep_query (
conn->prepare_query<build> ("mod-build-task-build-query", bq));
+ // Return true if a package needs to be rebuilt.
+ //
+ auto needs_rebuild = [&forced_rebuild_expiration,
+ &soft_rebuild_expiration,
+ &hard_rebuild_expiration] (const build& b)
+ {
+ assert (b.state == build_state::built);
+
+ return (b.force == force_state::forced &&
+ b.soft_timestamp <= forced_rebuild_expiration) ||
+ b.soft_timestamp <= soft_rebuild_expiration ||
+ b.hard_timestamp <= hard_rebuild_expiration;
+ };
+
+ // Convert a build to the hard rebuild, resetting the agent checksum and
+ // dropping the previous build task result.
+ //
+ // Note that since the checksums are hierarchical, the agent checksum
+ // reset will trigger resets of the "subordinate" checksums up to the
+ // dependency checksum and so the package will be rebuilt.
+ //
+ // Also note that there is no sense to keep the build task result since we
+ // don't accept the skip result for the hard rebuild task. We, however,
+ // keep the status intact (see below for the reasoning).
+ //
+ auto convert_to_hard = [] (const shared_ptr<build>& b)
+ {
+ b->agent_checksum = nullopt;
+
+ // Mark the section as loaded, so results are updated.
+ //
+ b->results_section.load ();
+ b->results.clear ();
+ };
+
+ // Return SHA256 checksum of the controller logic and the configuration
+ // target, environment, arguments, and warning-detecting regular
+ // expressions.
+ //
+ auto controller_checksum = [] (const build_config& c)
+ {
+ sha256 cs ("1"); // Hash the logic version.
+
+ cs.append (c.target.string ());
+ cs.append (c.environment ? *c.environment : "");
+
+ for (const string& a: c.args)
+ cs.append (a);
+
+ for (const string& re: c.warning_regexes)
+ cs.append (re);
+
+ return string (cs.string ());
+ };
+
+ // Return the machine id as a machine checksum.
+ //
+ auto machine_checksum = [] (const machine_header_manifest& m)
+ {
+ return m.id;
+ };
+
while (tsm.session.empty ())
{
transaction t (conn->begin ());
@@ -604,9 +730,7 @@ handle (request& rq, response& rs)
{
assert (i->force != force_state::forcing);
- if (i->timestamp <= (i->force == force_state::forced
- ? forced_rebuild_expiration
- : normal_rebuild_expiration))
+ if (needs_rebuild (*i))
rebuilds.emplace_back (i.load ());
}
}
@@ -666,14 +790,15 @@ handle (request& rq, response& rs)
move (cl),
mh.name,
move (mh.summary),
- cm.config->target);
+ cm.config->target,
+ controller_checksum (*cm.config),
+ machine_checksum (*cm.machine));
build_db_->persist (b);
}
else
{
- // The package configuration is in the building state, and there
- // are no results.
+ // The package configuration is in the building state.
//
// Note that in both cases we keep the status intact to be able
// to compare it with the final one in the result request
@@ -681,12 +806,7 @@ handle (request& rq, response& rs)
// email. The same is true for the forced flag (in the sense
// that we don't set the force state to unforced).
//
- // Load the section to assert the above statement.
- //
- build_db_->load (*b, b->results_section);
-
- assert (b->state == build_state::building &&
- b->results.empty ());
+ assert (b->state == build_state::building);
b->state = build_state::building;
b->interactive = move (login);
@@ -703,6 +823,22 @@ handle (request& rq, response& rs)
b->machine = mh.name;
b->machine_summary = move (mh.summary);
b->target = cm.config->target;
+
+ string ccs (controller_checksum (*cm.config));
+ string mcs (machine_checksum (*cm.machine));
+
+ // Issue the hard rebuild if it is forced or the configuration
+ // or machine has changed.
+ //
+ if (b->hard_timestamp <= hard_rebuild_expiration ||
+ b->force == force_state::forced ||
+ b->controller_checksum != ccs ||
+ b->machine_checksum != mcs)
+ convert_to_hard (b);
+
+ b->controller_checksum = move (ccs);
+ b->machine_checksum = move (mcs);
+
b->timestamp = system_clock::now ();
build_db_->update (b);
@@ -770,10 +906,9 @@ handle (request& rq, response& rs)
b = build_db_->find<build> (b->id);
- if (b != nullptr && b->state == build_state::built &&
- b->timestamp <= (b->force == force_state::forced
- ? forced_rebuild_expiration
- : normal_rebuild_expiration))
+ if (b != nullptr &&
+ b->state == build_state::built &&
+ needs_rebuild (*b))
{
auto i (cfg_machines.find (b->id.configuration.c_str ()));
@@ -828,10 +963,23 @@ handle (request& rq, response& rs)
b->target = cm.config->target;
- // Mark the section as loaded, so results are updated.
+ // Issue the hard rebuild if the timeout expired, rebuild is
+ // forced, or the configuration or machine has changed.
//
- b->results_section.load ();
- b->results.clear ();
+ // Note that we never reset the build status (see above for the
+ // reasoning).
+ //
+ string ccs (controller_checksum (*cm.config));
+ string mcs (machine_checksum (*cm.machine));
+
+ if (b->hard_timestamp <= hard_rebuild_expiration ||
+ b->force == force_state::forced ||
+ b->controller_checksum != ccs ||
+ b->machine_checksum != mcs)
+ convert_to_hard (b);
+
+ b->controller_checksum = move (ccs);
+ b->machine_checksum = move (mcs);
b->timestamp = system_clock::now ();
diff --git a/mod/module.cli b/mod/module.cli
index c95c20c..c2dce5b 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -195,45 +195,87 @@ namespace brep
be specified in seconds. Default is 10 minutes."
}
- size_t build-normal-rebuild-timeout = 86400
+ size_t build-soft-rebuild-timeout = 86400
{
"<seconds>",
- "Time to wait before considering a package for a normal rebuild. Must
- be specified in seconds. Default is 24 hours."
+ "Time to wait before considering a package for a soft rebuild (only to
+ be performed if the build environment or any of the package
+ dependencies have changed). Must be specified in seconds. The special
+ zero value disables soft rebuilds. Default is 24 hours"
}
- size_t build-alt-rebuild-timeout
+ size_t build-alt-soft-rebuild-timeout
{
"<seconds>",
- "Alternative package rebuild timeout to use instead of the normal
- rebuild timeout (see \cb{build-normal-rebuild-timeout} for details)
+ "Alternative package soft rebuild timeout to use instead of the soft
+ rebuild timeout (see \cb{build-soft-rebuild-timeout} for details)
during the time interval specified with the
- \cb{build-alt-rebuild-start} and \cb{build-alt-rebuild-stop} options.
- Must be specified in seconds. Default is the time interval length."
+ \cb{build-alt-soft-rebuild-start} and
+ \cb{build-alt-soft-rebuild-stop} options. Must be specified in
+ seconds. Default is the time interval length plus
+ \c{(\b{build-soft-rebuild-timeout} - 24h)} if soft rebuild timeout is
+ greater than 24 hours (thus the rebuild is only triggered within the
+ last 24 hours of the \cb{build-soft-rebuild-timeout} expiration)."
}
- duration build-alt-rebuild-start
+ duration build-alt-soft-rebuild-start
{
"<hours>:<minutes>",
- "The start time of the alternative package rebuild timeout (see
- \cb{build-alt-rebuild-timeout} for details). Must be specified as
- a time of day in the local timezone. The \cb{build-alt-rebuild-start}
- and \cb{build-alt-rebuild-stop} options must be either both specified
- or absent. If unspecified, then no alternative rebuild timeout will
- be used."
+ "The start time of the alternative package soft rebuild timeout (see
+ \cb{build-alt-soft-rebuild-timeout} for details). Must be specified
+ as a time of day in the local timezone. The
+ \cb{build-alt-soft-rebuild-start} and
+ \cb{build-alt-soft-rebuild-stop} options must be either both
+ specified or absent. If unspecified, then no alternative rebuild
+ timeout will be used."
}
- duration build-alt-rebuild-stop
+ duration build-alt-soft-rebuild-stop
{
"<hours>:<minutes>",
- "The end time of the alternative package rebuild timeout (see
- \cb{build-alt-rebuild-timeout} for details). Must be specified as
- a time of day in the local timezone. If it is less than the
- \cb{build-alt-rebuild-start} option value, then the time interval
- extends through midnight. The \cb{build-alt-rebuild-start} and
- \cb{build-alt-rebuild-stop} options must be either both specified or
- absent. If unspecified, then no alternative rebuild timeout will be
- used."
+ "The end time of the alternative package soft rebuild timeout (see
+ \cb{build-alt-soft-rebuild-timeout} for details). Must be specified
+ as a time of day in the local timezone. If it is less than the
+ \cb{build-alt-soft-rebuild-start} option value, then the time
+ interval extends through midnight. The
+ \cb{build-alt-soft-rebuild-start} and
+ \cb{build-alt-soft-rebuild-stop} options must be either both
+ specified or absent. If unspecified, then no alternative rebuild
+ timeout will be used."
+ }
+
+ size_t build-hard-rebuild-timeout = 604800
+ {
+ "<seconds>",
+ "Time to wait before considering a package for a hard rebuild (to be
+ performed unconditionally). Must be specified in seconds. The special
+ zero value disables hard rebuilds. Default is 7 days."
+ }
+
+ size_t build-alt-hard-rebuild-timeout
+ {
+ "<seconds>",
+ "Alternative package hard rebuild timeout. The semantics is the
+ same as for the \cb{build-alt-soft-rebuild-timeout} option but
+ for the \cb{build-hard-rebuild-timeout} option."
+ }
+
+ duration build-alt-hard-rebuild-start
+ {
+ "<hours>:<minutes>",
+ "The start time of the alternative package hard rebuild timeout (see
+ \cb{build-alt-hard-rebuild-timeout} for details). The semantics is
+ the same as for the \cb{build-alt-soft-rebuild-start} option but
+ for the \cb{build-hard-rebuild-timeout} option."
+ }
+
+ duration build-alt-hard-rebuild-stop
+ {
+ "<hours>:<minutes>",
+ "The end time of the alternative package hard rebuild timeout (see
+ \cb{build-alt-hard-rebuild-timeout} for details). The semantics is
+ the same as for the \cb{build-alt-soft-rebuild-stop} option but
+ for the \cb{build-hard-rebuild-timeout} option."
}
};