diff options
Diffstat (limited to 'mod/mod-build-task.cxx')
| -rw-r--r-- | mod/mod-build-task.cxx | 278 |
1 files changed, 213 insertions, 65 deletions
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 1cb5386..1466077 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -127,22 +127,80 @@ handle (request& rq, response& rs) // Go through packages until we find one that has no build configuration // present in the database, or has the untested one, or in the testing state - // but expired, or the one, which build failed abnormally and expired. If - // such a package configuration is found then put it into the testing state, - // set the current timestamp and respond with the task for building this - // package configuration. + // but expired (collectively called unbuilt). If such a package + // configuration is found then put it into the testing state, set the + // current timestamp and respond with the task for building this package + // configuration. + // + // While trying to find a non-built package configuration we will also + // collect the list of the tested package configurations which it's time to + // rebuilt. So if no unbuilt package is found, we will pickup one to + // rebuild. The rebuild preference is given in the following order: the + // greater force flag, the greater overall status, the lower timestamp. // if (!cfg_machines.empty ()) { + vector<shared_ptr<build>> rebuilds; + + // Create the task response manifest. The package must have the internal + // repository loaded. + // + auto task = [this] (shared_ptr<build>&& b, + shared_ptr<package>&& p, + const config_machine& cm) -> task_response_manifest + { + string session (b->package_name + '/' + b->package_version.string () + + '/' + b->configuration); + + string result_url (options_->host () + options_->root ().string () + + "?build-result"); + + lazy_shared_ptr<repository> r (p->internal_repository); + + strings fp; + if (r->certificate) + fp.emplace_back (move (r->certificate->fingerprint)); + + task_manifest task (move (b->package_name), + move (b->package_version), + move (r->location), + move (fp), + cm.machine->name, + cm.config->target, + cm.config->vars); + + // @@ We don't support challenge at the moment. + // + return task_response_manifest (move (session), + nullopt, + move (result_url), + move (task)); + }; + // Calculate the expiration time for package configurations being in the - // testing state or those, which build failed abnormally. + // testing (build expiration) or the tested (rebuild expiration) state. // - timestamp expiration (timestamp::clock::now () - - chrono::seconds (options_->build_result_timeout ())); + timestamp now (timestamp::clock::now ()); + + auto expiration = [&now] (size_t timeout) -> timestamp + { + return now - chrono::seconds (timeout); + }; + + auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t + { + return chrono::duration_cast<chrono::nanoseconds> ( + expiration (timeout).time_since_epoch ()).count (); + }; - uint64_t expiration_ns ( - std::chrono::duration_cast<std::chrono::nanoseconds> ( - expiration.time_since_epoch ()).count ()); + uint64_t build_expiration_ns ( + expiration_ns (options_->build_result_timeout ())); + + timestamp forced_rebuild_expiration ( + expiration (options_->build_forced_rebuild_timeout ())); + + timestamp normal_rebuild_expiration ( + expiration (options_->build_normal_rebuild_timeout ())); // Prepare the package version prepared query. // @@ -188,10 +246,9 @@ handle (request& rq, response& rs) // configurations that have already been acted upon (initially empty). // // This is why we query the database for package configurations that - // should not be built (in the tested state with the build terminated - // normally or not expired, or in the testing state and not expired). - // Having such a list we will select the first build configuration that is - // not in the list (if available) for the response. + // should not be built (in the tested state, or in the testing state and + // not expired). Having such a list we will select the first build + // configuration that is not in the list (if available) for the response. // using bld_query = query<build>; using prep_bld_query = prepared_query<build>; @@ -212,12 +269,9 @@ handle (request& rq, response& rs) bld_query::id.configuration.in_range (cfg_names.begin (), cfg_names.end ()) && - ((bld_query::state == "tested" && - ((bld_query::status != "abort" && bld_query::status != "abnormal") || - bld_query::timestamp > expiration_ns)) || - + (bld_query::state == "tested" || (bld_query::state == "testing" && - bld_query::timestamp > expiration_ns))); + bld_query::timestamp > build_expiration_ns))); connection_ptr bld_conn (build_db_->connection ()); @@ -262,25 +316,33 @@ handle (request& rq, response& rs) // configurations that remained can be built. We will take the first // one, if present. // + // Also save the tested package configurations for which it's time + // to be rebuilt. + // config_machines configs (cfg_machines); // Make a copy for this pkg. + auto pkg_builds (bld_prep_query.execute ()); - for (const auto& pc: bld_prep_query.execute ()) + for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i) { - auto i (configs.find (pc.id.configuration.c_str ())); + auto j (configs.find (i->id.configuration.c_str ())); // Outdated configurations are already excluded with the database // query. // - assert (i != configs.end ()); - configs.erase (i); + assert (j != configs.end ()); + configs.erase (j); + + if (i->state == build_state::tested && + i->timestamp <= (i->forced + ? forced_rebuild_expiration + : normal_rebuild_expiration)) + rebuilds.emplace_back (i.load ()); } if (!configs.empty ()) { config_machine& cm (configs.begin ()->second); - const build_config& cfg (*cm.config); - - build_id bid (move (id), cfg.name); + build_id bid (move (id), cm.config->name); shared_ptr<build> b (build_db_->find<build> (bid)); // If build configuration doesn't exist then create the new one @@ -297,27 +359,30 @@ handle (request& rq, response& rs) } else { - // If the package configuration is in the tested state, then we - // need to cleanup the status and results prior to the update. - // Otherwise the status is already absent and there are no - // results. + // The package configuration can be in the testing or untested + // state, so the forced flag is false and the status is absent, + // unless in the testing state (in which case they may or may not + // be set/exist), and there are no results. + // + // Note that in the testing state the status can be present if + // the rebuild task have been issued. In both cases we keep the + // status intact to be able to compare it with the final one in + // the result request handling in order to decide if to send the + // notification email. The same is true for the forced flag. We + // just assert that if the force flag is set, then the status + // exists. // - // Load the section to make sure results are updated for the - // tested state, otherwise assert there are no results. + // Load the section to assert the above statement. // build_db_->load (*b, b->results_section); - if (b->state == build_state::tested) - { - assert (b->status); - b->status = nullopt; + assert (b->state != build_state::tested && + + ((!b->forced && !b->status) || + (b->state == build_state::testing && + (!b->forced || b->status))) && - b->results.clear (); - } - else - { - assert (!b->status && b->results.empty ()); - } + b->results.empty ()); b->state = build_state::testing; b->timestamp = timestamp::clock::now (); @@ -327,41 +392,21 @@ handle (request& rq, response& rs) // Finally, prepare the task response manifest. // - tsm.session = b->package_name + '/' + - b->package_version.string () + '/' + b->configuration; - - // @@ We don't support challenge at the moment, so leave it absent. - // - - tsm.result_url = options_->host () + options_->root ().string () + - "?build-result"; - // Switch to the package database transaction to load the package. // transaction::current (pt); shared_ptr<package> p (package_db_->load<package> (b->id.package)); - shared_ptr<repository> r (p->internal_repository.load ()); + p->internal_repository.load (); // Switch back to the build database transaction. // transaction::current (bt); - strings fp; - if (r->certificate) - fp.emplace_back (move (r->certificate->fingerprint)); - - tsm.task = task_manifest ( - move (b->package_name), - move (b->package_version), - move (r->location), - move (fp), - move (cm.machine->name), - cfg.target, - cfg.vars); + tsm = task (move (b), move (p), cm); } - // If task response manifest is filled, then can bail out from the + // If the task response manifest is prepared, then bail out from the // package loop, commit transactions and respond. // if (!tsm.session.empty ()) @@ -374,6 +419,109 @@ handle (request& rq, response& rs) transaction::current (pt); // Switch to the package database transaction. pt.commit (); } + + // If we don't have an unbuilt package, then let's see if we have a + // package to rebuild. + // + if (tsm.session.empty () && !rebuilds.empty ()) + { + // Sort the package configuration rebuild list with the following sort + // priority: + // + // 1: forced flag + // 2: overall status + // 3: timestamp (less is preferred) + // + auto cmp = [] (const shared_ptr<build>& x, + const shared_ptr<build>& y) -> bool + { + if (x->forced != y->forced) + return x->forced > y->forced; // Forced goes first. + + assert (x->status && y->status); // Both tested. + + if (x->status != y->status) + return x->status > y->status; // Larger status goes first. + + return x->timestamp < y->timestamp; // Older goes first. + }; + + sort (rebuilds.begin (), rebuilds.end (), cmp); + + // Pick the first package configuration from the ordered list. + // + // Note that the configurations may not match the required criteria + // anymore (as we have committed the database transactions that were + // used to collect this data) so we recheck. If we find one that matches + // then put it into the testing state, refresh the timestamp and + // update. Note that we don't amend the status and the force flag to + // have them available in the result request handling (see above). + // + for (auto& b: rebuilds) + { + try + { + transaction bt (build_db_->begin ()); + + b = build_db_->find<build> (b->id); + + if (b != nullptr && b->state == build_state::tested && + b->timestamp <= (b->forced + ? forced_rebuild_expiration + : normal_rebuild_expiration)) + { + // Load the package (if still present). + // + transaction pt (package_db_->begin (), false); + transaction::current (pt); + + shared_ptr<package> p (package_db_->find<package> (b->id.package)); + + if (p != nullptr) + p->internal_repository.load (); + + // Commit the package database transaction and switch back to the + // build database transaction. + // + pt.commit (); + transaction::current (bt); + + if (p != nullptr) + { + assert (b->status); + + b->state = build_state::testing; + + // Mark the section as loaded, so results are updated. + // + b->results_section.load (); + b->results.clear (); + + b->timestamp = timestamp::clock::now (); + + build_db_->update (b); + + auto i (cfg_machines.find (b->id.configuration.c_str ())); + + // Only actual package configurations are loaded (see above). + // + assert (i != cfg_machines.end ()); + + tsm = task (move (b), move (p), i->second); + } + } + + bt.commit (); + } + catch (const odb::deadlock&) {} // Just try with the next rebuild. + + // If the task response manifest is prepared, then bail out from the + // package configuration rebuilds loop and respond. + // + if (!tsm.session.empty ()) + break; + } + } } // @@ Probably it would be a good idea to also send some cache control |