aboutsummaryrefslogtreecommitdiff
path: root/mod/mod-build-task.cxx
diff options
context:
space:
mode:
Diffstat (limited to 'mod/mod-build-task.cxx')
-rw-r--r--mod/mod-build-task.cxx278
1 files changed, 213 insertions, 65 deletions
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 1cb5386..1466077 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -127,22 +127,80 @@ handle (request& rq, response& rs)
// Go through packages until we find one that has no build configuration
// present in the database, or has the untested one, or in the testing state
- // but expired, or the one, which build failed abnormally and expired. If
- // such a package configuration is found then put it into the testing state,
- // set the current timestamp and respond with the task for building this
- // package configuration.
+ // but expired (collectively called unbuilt). If such a package
+ // configuration is found then put it into the testing state, set the
+ // current timestamp and respond with the task for building this package
+ // configuration.
+ //
+ // While trying to find a non-built package configuration we will also
+ // collect the list of the tested package configurations which it's time to
+ // rebuilt. So if no unbuilt package is found, we will pickup one to
+ // rebuild. The rebuild preference is given in the following order: the
+ // greater force flag, the greater overall status, the lower timestamp.
//
if (!cfg_machines.empty ())
{
+ vector<shared_ptr<build>> rebuilds;
+
+ // Create the task response manifest. The package must have the internal
+ // repository loaded.
+ //
+ auto task = [this] (shared_ptr<build>&& b,
+ shared_ptr<package>&& p,
+ const config_machine& cm) -> task_response_manifest
+ {
+ string session (b->package_name + '/' + b->package_version.string () +
+ '/' + b->configuration);
+
+ string result_url (options_->host () + options_->root ().string () +
+ "?build-result");
+
+ lazy_shared_ptr<repository> r (p->internal_repository);
+
+ strings fp;
+ if (r->certificate)
+ fp.emplace_back (move (r->certificate->fingerprint));
+
+ task_manifest task (move (b->package_name),
+ move (b->package_version),
+ move (r->location),
+ move (fp),
+ cm.machine->name,
+ cm.config->target,
+ cm.config->vars);
+
+ // @@ We don't support challenge at the moment.
+ //
+ return task_response_manifest (move (session),
+ nullopt,
+ move (result_url),
+ move (task));
+ };
+
// Calculate the expiration time for package configurations being in the
- // testing state or those, which build failed abnormally.
+ // testing (build expiration) or the tested (rebuild expiration) state.
//
- timestamp expiration (timestamp::clock::now () -
- chrono::seconds (options_->build_result_timeout ()));
+ timestamp now (timestamp::clock::now ());
+
+ auto expiration = [&now] (size_t timeout) -> timestamp
+ {
+ return now - chrono::seconds (timeout);
+ };
+
+ auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t
+ {
+ return chrono::duration_cast<chrono::nanoseconds> (
+ expiration (timeout).time_since_epoch ()).count ();
+ };
- uint64_t expiration_ns (
- std::chrono::duration_cast<std::chrono::nanoseconds> (
- expiration.time_since_epoch ()).count ());
+ uint64_t build_expiration_ns (
+ expiration_ns (options_->build_result_timeout ()));
+
+ timestamp forced_rebuild_expiration (
+ expiration (options_->build_forced_rebuild_timeout ()));
+
+ timestamp normal_rebuild_expiration (
+ expiration (options_->build_normal_rebuild_timeout ()));
// Prepare the package version prepared query.
//
@@ -188,10 +246,9 @@ handle (request& rq, response& rs)
// configurations that have already been acted upon (initially empty).
//
// This is why we query the database for package configurations that
- // should not be built (in the tested state with the build terminated
- // normally or not expired, or in the testing state and not expired).
- // Having such a list we will select the first build configuration that is
- // not in the list (if available) for the response.
+ // should not be built (in the tested state, or in the testing state and
+ // not expired). Having such a list we will select the first build
+ // configuration that is not in the list (if available) for the response.
//
using bld_query = query<build>;
using prep_bld_query = prepared_query<build>;
@@ -212,12 +269,9 @@ handle (request& rq, response& rs)
bld_query::id.configuration.in_range (cfg_names.begin (),
cfg_names.end ()) &&
- ((bld_query::state == "tested" &&
- ((bld_query::status != "abort" && bld_query::status != "abnormal") ||
- bld_query::timestamp > expiration_ns)) ||
-
+ (bld_query::state == "tested" ||
(bld_query::state == "testing" &&
- bld_query::timestamp > expiration_ns)));
+ bld_query::timestamp > build_expiration_ns)));
connection_ptr bld_conn (build_db_->connection ());
@@ -262,25 +316,33 @@ handle (request& rq, response& rs)
// configurations that remained can be built. We will take the first
// one, if present.
//
+ // Also save the tested package configurations for which it's time
+ // to be rebuilt.
+ //
config_machines configs (cfg_machines); // Make a copy for this pkg.
+ auto pkg_builds (bld_prep_query.execute ());
- for (const auto& pc: bld_prep_query.execute ())
+ for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i)
{
- auto i (configs.find (pc.id.configuration.c_str ()));
+ auto j (configs.find (i->id.configuration.c_str ()));
// Outdated configurations are already excluded with the database
// query.
//
- assert (i != configs.end ());
- configs.erase (i);
+ assert (j != configs.end ());
+ configs.erase (j);
+
+ if (i->state == build_state::tested &&
+ i->timestamp <= (i->forced
+ ? forced_rebuild_expiration
+ : normal_rebuild_expiration))
+ rebuilds.emplace_back (i.load ());
}
if (!configs.empty ())
{
config_machine& cm (configs.begin ()->second);
- const build_config& cfg (*cm.config);
-
- build_id bid (move (id), cfg.name);
+ build_id bid (move (id), cm.config->name);
shared_ptr<build> b (build_db_->find<build> (bid));
// If build configuration doesn't exist then create the new one
@@ -297,27 +359,30 @@ handle (request& rq, response& rs)
}
else
{
- // If the package configuration is in the tested state, then we
- // need to cleanup the status and results prior to the update.
- // Otherwise the status is already absent and there are no
- // results.
+ // The package configuration can be in the testing or untested
+ // state, so the forced flag is false and the status is absent,
+ // unless in the testing state (in which case they may or may not
+ // be set/exist), and there are no results.
+ //
+ // Note that in the testing state the status can be present if
+ // the rebuild task have been issued. In both cases we keep the
+ // status intact to be able to compare it with the final one in
+ // the result request handling in order to decide if to send the
+ // notification email. The same is true for the forced flag. We
+ // just assert that if the force flag is set, then the status
+ // exists.
//
- // Load the section to make sure results are updated for the
- // tested state, otherwise assert there are no results.
+ // Load the section to assert the above statement.
//
build_db_->load (*b, b->results_section);
- if (b->state == build_state::tested)
- {
- assert (b->status);
- b->status = nullopt;
+ assert (b->state != build_state::tested &&
+
+ ((!b->forced && !b->status) ||
+ (b->state == build_state::testing &&
+ (!b->forced || b->status))) &&
- b->results.clear ();
- }
- else
- {
- assert (!b->status && b->results.empty ());
- }
+ b->results.empty ());
b->state = build_state::testing;
b->timestamp = timestamp::clock::now ();
@@ -327,41 +392,21 @@ handle (request& rq, response& rs)
// Finally, prepare the task response manifest.
//
- tsm.session = b->package_name + '/' +
- b->package_version.string () + '/' + b->configuration;
-
- // @@ We don't support challenge at the moment, so leave it absent.
- //
-
- tsm.result_url = options_->host () + options_->root ().string () +
- "?build-result";
-
// Switch to the package database transaction to load the package.
//
transaction::current (pt);
shared_ptr<package> p (package_db_->load<package> (b->id.package));
- shared_ptr<repository> r (p->internal_repository.load ());
+ p->internal_repository.load ();
// Switch back to the build database transaction.
//
transaction::current (bt);
- strings fp;
- if (r->certificate)
- fp.emplace_back (move (r->certificate->fingerprint));
-
- tsm.task = task_manifest (
- move (b->package_name),
- move (b->package_version),
- move (r->location),
- move (fp),
- move (cm.machine->name),
- cfg.target,
- cfg.vars);
+ tsm = task (move (b), move (p), cm);
}
- // If task response manifest is filled, then can bail out from the
+ // If the task response manifest is prepared, then bail out from the
// package loop, commit transactions and respond.
//
if (!tsm.session.empty ())
@@ -374,6 +419,109 @@ handle (request& rq, response& rs)
transaction::current (pt); // Switch to the package database transaction.
pt.commit ();
}
+
+ // If we don't have an unbuilt package, then let's see if we have a
+ // package to rebuild.
+ //
+ if (tsm.session.empty () && !rebuilds.empty ())
+ {
+ // Sort the package configuration rebuild list with the following sort
+ // priority:
+ //
+ // 1: forced flag
+ // 2: overall status
+ // 3: timestamp (less is preferred)
+ //
+ auto cmp = [] (const shared_ptr<build>& x,
+ const shared_ptr<build>& y) -> bool
+ {
+ if (x->forced != y->forced)
+ return x->forced > y->forced; // Forced goes first.
+
+ assert (x->status && y->status); // Both tested.
+
+ if (x->status != y->status)
+ return x->status > y->status; // Larger status goes first.
+
+ return x->timestamp < y->timestamp; // Older goes first.
+ };
+
+ sort (rebuilds.begin (), rebuilds.end (), cmp);
+
+ // Pick the first package configuration from the ordered list.
+ //
+ // Note that the configurations may not match the required criteria
+ // anymore (as we have committed the database transactions that were
+ // used to collect this data) so we recheck. If we find one that matches
+ // then put it into the testing state, refresh the timestamp and
+ // update. Note that we don't amend the status and the force flag to
+ // have them available in the result request handling (see above).
+ //
+ for (auto& b: rebuilds)
+ {
+ try
+ {
+ transaction bt (build_db_->begin ());
+
+ b = build_db_->find<build> (b->id);
+
+ if (b != nullptr && b->state == build_state::tested &&
+ b->timestamp <= (b->forced
+ ? forced_rebuild_expiration
+ : normal_rebuild_expiration))
+ {
+ // Load the package (if still present).
+ //
+ transaction pt (package_db_->begin (), false);
+ transaction::current (pt);
+
+ shared_ptr<package> p (package_db_->find<package> (b->id.package));
+
+ if (p != nullptr)
+ p->internal_repository.load ();
+
+ // Commit the package database transaction and switch back to the
+ // build database transaction.
+ //
+ pt.commit ();
+ transaction::current (bt);
+
+ if (p != nullptr)
+ {
+ assert (b->status);
+
+ b->state = build_state::testing;
+
+ // Mark the section as loaded, so results are updated.
+ //
+ b->results_section.load ();
+ b->results.clear ();
+
+ b->timestamp = timestamp::clock::now ();
+
+ build_db_->update (b);
+
+ auto i (cfg_machines.find (b->id.configuration.c_str ()));
+
+ // Only actual package configurations are loaded (see above).
+ //
+ assert (i != cfg_machines.end ());
+
+ tsm = task (move (b), move (p), i->second);
+ }
+ }
+
+ bt.commit ();
+ }
+ catch (const odb::deadlock&) {} // Just try with the next rebuild.
+
+ // If the task response manifest is prepared, then bail out from the
+ // package configuration rebuilds loop and respond.
+ //
+ if (!tsm.session.empty ())
+ break;
+ }
+ }
}
// @@ Probably it would be a good idea to also send some cache control