diff options
Diffstat (limited to 'mod/mod-build-result.cxx')
| -rw-r--r-- | mod/mod-build-result.cxx | 407 |
1 files changed, 246 insertions, 161 deletions
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index 7806cda..ccce17f 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -6,12 +6,8 @@ #include <odb/database.hxx> #include <odb/transaction.hxx> -#include <libbutl/sendmail.hxx> -#include <libbutl/fdstream.hxx> -#include <libbutl/process-io.hxx> #include <libbutl/manifest-parser.hxx> #include <libbutl/manifest-serializer.hxx> -#include <libbutl/semantic-version.hxx> #include <libbbot/manifest.hxx> @@ -19,11 +15,12 @@ #include <libbrep/build.hxx> #include <libbrep/build-odb.hxx> -#include <libbrep/package.hxx> -#include <libbrep/package-odb.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> -#include <mod/build.hxx> // *_url() +#include <mod/build.hxx> // send_notification_email() #include <mod/module-options.hxx> +#include <mod/tenant-service.hxx> using namespace std; using namespace butl; @@ -31,14 +28,21 @@ using namespace bbot; using namespace brep::cli; using namespace odb::core; +brep::build_result:: +build_result (const tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} + // While currently the user-defined copy constructor is not required (we don't // need to deep copy nullptr's), it is a good idea to keep the placeholder // ready for less trivial cases. // brep::build_result:: -build_result (const build_result& r) +build_result (const build_result& r, const tenant_service_map& tsm) : build_result_module (r), - options_ (r.initialized_ ? r.options_ : nullptr) + options_ (r.initialized_ ? r.options_ : nullptr), + tenant_service_map_ (tsm) { } @@ -51,13 +55,8 @@ init (scanner& s) s, unknown_mode::fail, unknown_mode::fail); if (options_->build_config_specified ()) - { build_result_module::init (*options_, *options_); - database_module::init (static_cast<const options::package_db&> (*options_), - options_->package_db_retry ()); - } - if (options_->root ().empty ()) options_->root (dir_path ("/")); } @@ -132,9 +131,24 @@ handle (request& rq, response&) // if the session is valid. The thinking is that this is a problem with the // controller's setup (expires too fast), not with the agent's. // - auto warn_expired = [&rqm, &warn] (const string& d) + // Note, though, that there can be quite a common situation when a build + // machine is suspended by the bbot agent due to the build timeout. In this + // case the task result request may arrive anytime later (after the issue is + // investigated, etc) with the abort or abnormal status. By that arrival + // time a new build task may already be issued/completed for this package + // build configuration or this configuration may even be gone (brep has been + // reconfigured, package has gone, etc). We will log no warning in this + // case, assuming that such an expiration is not a problem with the + // controller's setup. + // + shared_ptr<build> b; + result_status rs (rqm.result.status); + + auto warn_expired = [&rqm, &warn, &b, &session, rs] (const string& d) { - warn << "session '" << rqm.session << "' expired: " << d; + if (!((b == nullptr || b->timestamp > session.timestamp) && + (rs == result_status::abort || rs == result_status::abnormal))) + warn << "session '" << rqm.session << "' expired: " << d; }; // Make sure the build configuration still exists. @@ -153,32 +167,6 @@ handle (request& rq, response&) tc = i->second; } - // Load the built package (if present). - // - // The only way not to deal with 2 databases simultaneously is to pull - // another bunch of the package fields into the build_package foreign - // object, which is a pain (see build_package.hxx for details). Doesn't seem - // worth it here: email members are really secondary and we don't need to - // switch transactions back and forth. - // - shared_ptr<package> pkg; - { - transaction t (package_db_->begin ()); - pkg = package_db_->find<package> (id.package); - t.commit (); - } - - if (pkg == nullptr) - { - warn_expired ("no package"); - return true; - } - - auto print_args = [&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }; - // Load and update the package build configuration (if present). // // NULL if the package build doesn't exist or is not updated for any reason @@ -187,20 +175,57 @@ handle (request& rq, response&) // shared_ptr<build> bld; + // The built package configuration. + // + // Not NULL if bld is not NULL. + // + shared_ptr<build_package> pkg; + build_package_config* cfg (nullptr); + + // Don't send email to the build-email address for the success-to-success + // status change, unless the build was forced. + // bool build_notify (false); bool unforced (true); + // If the package is built (result status differs from interrupt, etc) and + // the package tenant has a third-party service state associated with it, + // then check if the tenant_service_build_built callback is registered for + // the type of the associated service. If it is, then stash the state, the + // build object, and the callback pointer for the subsequent service `built` + // notification. Note that we send this notification for the skip result as + // well, since it is semantically equivalent to the previous build result + // with the actual build process being optimized out. + // + // If the package build is interrupted and the tenant_service_build_queued + // callback is associated with the package tenant, then stash the state, the + // build object, and the callback pointer and calculate the hints for the + // subsequent service `queued` notification. + // + const tenant_service_build_built* tsb (nullptr); + const tenant_service_build_queued* tsq (nullptr); + optional<pair<tenant_service, shared_ptr<build>>> tss; + tenant_service_build_queued::build_queued_hints qhs; + // Note that if the session authentication fails (probably due to the // authentication settings change), then we log this case with the warning // severity and respond with the 200 HTTP code as if the challenge is // valid. The thinking is that we shouldn't alarm a law-abaiding agent and // shouldn't provide any information to a malicious one. // + connection_ptr conn (build_db_->connection ()); { - transaction t (build_db_->begin ()); + transaction t (conn->begin ()); package_build pb; - shared_ptr<build> b; + + auto build_timestamp = [&b] () + { + return to_string ( + chrono::duration_cast<std::chrono::nanoseconds> ( + b->timestamp.time_since_epoch ()).count ()); + }; + if (!build_db_->query_one<package_build> ( query<package_build>::build::id == id, pb)) { @@ -208,19 +233,48 @@ handle (request& rq, response&) } else if ((b = move (pb.build))->state != build_state::building) { - warn_expired ("package configuration state is " + to_string (b->state)); + warn_expired ("package configuration state is " + to_string (b->state) + + ", force state " + to_string (b->force) + + ", timestamp " + build_timestamp ()); } else if (b->timestamp != session.timestamp) { - warn_expired ("non-matching timestamp"); + warn_expired ("non-matching timestamp " + build_timestamp ()); } else if (authenticate_session (*options_, rqm.challenge, *b, rqm.session)) { + const tenant_service_base* ts (nullptr); + + shared_ptr<build_tenant> t (build_db_->load<build_tenant> (b->tenant)); + + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + ts = i->second.get (); + } + // If the build is interrupted, then revert it to the original built - // state if this is a rebuild and delete it from the database otherwise. + // state if this is a rebuild. Otherwise (initial build), turn the build + // into the queued state if the tenant_service_build_queued callback is + // registered for the package tenant and delete it from the database + // otherwise. // - if (rqm.result.status == result_status::interrupt) + // Note that if the tenant_service_build_queued callback is registered, + // we always send the `queued` notification for the interrupted build, + // even when we reverse it to the original built state. We could also + // turn the build into the queued state in this case, but it feels that + // there is no harm in keeping the previous build information available + // for the user. + // + if (rs == result_status::interrupt) { + // Schedule the `queued` notification, if the + // tenant_service_build_queued callback is registered for the tenant. + // + tsq = dynamic_cast<const tenant_service_build_queued*> (ts); + if (b->status) // Is this a rebuild? { b->state = build_state::built; @@ -243,14 +297,77 @@ handle (request& rq, response&) // Note that we are unable to restore the pre-rebuild timestamp // since it has been overwritten when the build task was issued. // That, however, feels ok and we just keep it unchanged. + // + // Moreover, we actually use the fact that the build's timestamp is + // greater then its soft_timestamp as an indication that the build + // object represents the interrupted rebuild (see the build_task + // handler for details). + // + // @@ Actually, we also unable to restore the pre-rebuild machine + // and auxiliary machines, which are also displayed in the build + // log and may potentially be confusing. Should we drop them from + // the log in this case or replace with the "machine: unknown" + // record? build_db_->update (b); } - else - build_db_->erase (b); + else // Initial build. + { + if (tsq != nullptr) + { + // Since this is not a rebuild, there are no operation results and + // thus we don't need to load the results section to erase results + // from the database. + // + assert (b->results.empty ()); + + *b = build (move (b->tenant), + move (b->package_name), + move (b->package_version), + move (b->target), + move (b->target_config_name), + move (b->package_config_name), + move (b->toolchain_name), + move (b->toolchain_version)); + + build_db_->update (b); + } + else + build_db_->erase (b); + } + + // If we ought to call the tenant_service_build_queued::build_queued() + // callback, then also set the package tenant's queued timestamp to + // the current time to prevent the notifications race (see + // tenant::queued_timestamp for details). + // + if (tsq != nullptr) + { + // Calculate the tenant service hints. + // + buildable_package_count tpc ( + build_db_->query_value<buildable_package_count> ( + query<buildable_package_count>::build_tenant::id == t->id)); + + shared_ptr<build_package> p ( + build_db_->load<build_package> (b->id.package)); + + qhs = tenant_service_build_queued::build_queued_hints { + tpc == 1, p->configs.size () == 1}; + + // Set the package tenant's queued timestamp. + // + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + } } - else + else // Regular or skip build result. { + // Schedule the `built` notification, if the + // tenant_service_build_built callback is registered for the tenant. + // + tsb = dynamic_cast<const tenant_service_build_built*> (ts); + // Verify the result status/checksums. // // Specifically, if the result status is skip, then it can only be in @@ -259,7 +376,7 @@ handle (request& rq, response&) // checksums. On verification failure respond with the bad request // HTTP code (400). // - if (rqm.result.status == result_status::skip) + if (rs == result_status::skip) { if (!b->agent_checksum || !b->worker_checksum || @@ -303,14 +420,11 @@ handle (request& rq, response&) "dependency"); } - unforced = b->force == force_state::unforced; + unforced = (b->force == force_state::unforced); - // Don't send email to the build-email address for the - // success-to-success status change, unless the build was forced. - // - build_notify = !(rqm.result.status == result_status::success && - b->status && - *b->status == rqm.result.status && + build_notify = !(rs == result_status::success && + b->status && + *b->status == rs && unforced); b->state = build_state::built; @@ -329,11 +443,12 @@ handle (request& rq, response&) b->soft_timestamp = b->timestamp; // If the result status is other than skip, then save the status, - // results, and checksums and update the hard timestamp. + // results, and checksums and update the hard timestamp. Also stash + // the service notification information, if present. // - if (rqm.result.status != result_status::skip) + if (rs != result_status::skip) { - b->status = rqm.result.status; + b->status = rs; b->hard_timestamp = b->soft_timestamp; // Mark the section as loaded, so results are updated. @@ -350,129 +465,99 @@ handle (request& rq, response&) build_db_->update (b); - // Don't send the build notification email if the task result is - // `skip`, the configuration is hidden, or is now excluded by the - // package. + pkg = build_db_->load<build_package> (b->id.package); + cfg = find (b->package_config_name, pkg->configs); + + // The package configuration should be present (see mod-builds.cxx for + // details) but if it is not, let's log the warning. // - if (rqm.result.status != result_status::skip && belongs (*tc, "all")) + if (cfg != nullptr) { - shared_ptr<build_package> p ( - build_db_->load<build_package> (b->id.package)); - - // The package configuration should be present (see mod-builds.cxx - // for details) but if it is not, let's log the warning. + // Don't send the build notification email if the task result is + // `skip`, the configuration is hidden, or is now excluded by the + // package. // - if (const build_package_config* pc = find (b->package_config_name, - p->configs)) + if (rs != result_status::skip && !belongs (*tc, "hidden")) { - if (!exclude (*pc, p->builds, p->constraints, *tc)) - bld = move (b); + build_db_->load (*pkg, pkg->constraints_section); + + if (!exclude (*cfg, pkg->builds, pkg->constraints, *tc)) + bld = b; } - else - warn << "cannot find configuration '" << b->package_config_name - << "' for package " << p->id.name << '/' << p->version; } + else + warn << "cannot find configuration '" << b->package_config_name + << "' for package " << pkg->id.name << '/' << pkg->version; } + + // If required, stash the service notification information. + // + if (tsb != nullptr || tsq != nullptr) + tss = make_pair (move (*t->service), move (b)); } t.commit (); } - if (bld == nullptr) - return true; - - // Bail out if sending build notification emails is disabled for this - // toolchain. + // We either notify about the queued build or notify about the built package + // or don't notify at all. // - { - const map<string, bool>& tes (options_->build_toolchain_email ()); + assert (tsb == nullptr || tsq == nullptr); - auto i (tes.find (bld->id.toolchain_name)); - if (i != tes.end () && !i->second) - return true; - } - - string subj ((unforced ? "build " : "rebuild ") + - to_string (*bld->status) + ": " + - bld->package_name.string () + '/' + - bld->package_version.string () + ' ' + - bld->target_config_name + '/' + - bld->target.string () + ' ' + - bld->package_config_name + ' ' + - bld->toolchain_name + '-' + bld->toolchain_version.string ()); - - // Send notification emails to the interested parties. + // If the package build is interrupted and the tenant-associated third-party + // service needs to be notified about the queued builds, then call the + // tenant_service_build_queued::build_queued() callback function and update + // the service state, if requested. // - auto send_email = [&bld, &subj, &error, &trace, &print_args, this] - (const string& to) + if (tsq != nullptr) { - try - { - l2 ([&]{trace << "email '" << subj << "' to " << to;}); - - // Redirect the diagnostics to webserver error log. - // - // Note: if using this somewhere else, then need to factor out all this - // exit status handling code. - // - sendmail sm (print_args, - 2, - options_->email (), - subj, - {to}); - - if (bld->results.empty ()) - sm.out << "No operation results available." << endl; - else - { - const string& host (options_->host ()); - const dir_path& root (options_->root ()); - - ostream& os (sm.out); + assert (tss); // Wouldn't be here otherwise. - assert (bld->status); - os << "combined: " << *bld->status << endl << endl - << " " << build_log_url (host, root, *bld) << endl << endl; - - for (const auto& r: bld->results) - os << r.operation << ": " << r.status << endl << endl - << " " << build_log_url (host, root, *bld, &r.operation) - << endl << endl; - - os << "Force rebuild (enter the reason, use '+' instead of spaces):" - << endl << endl - << " " << build_force_url (host, root, *bld) << endl; - } + const tenant_service& ss (tss->first); - sm.out.close (); + vector<build> qbs; + qbs.push_back (move (*tss->second)); - if (!sm.wait ()) - error << "sendmail " << *sm.exit; - } - // Handle process_error and io_error (both derive from system_error). - // - catch (const system_error& e) - { - error << "sendmail error: " << e; - } - }; + if (auto f = tsq->build_queued (ss, + qbs, + build_state::building, + qhs, + log_writer_)) + update_tenant_service_state (conn, qbs.back ().tenant, f); + } - // Send the build notification email if a non-empty package build email is - // specified. + // If a third-party service needs to be notified about the built package, + // then call the tenant_service_build_built::build_built() callback function + // and update the service state, if requested. // - optional<email>& build_email (pkg->build_email); - if (build_notify && build_email && !build_email->empty ()) - send_email (*pkg->build_email); + if (tsb != nullptr) + { + assert (tss); // Wouldn't be here otherwise. - assert (bld->status); + const tenant_service& ss (tss->first); + const build& b (*tss->second); - // Send the build warning/error notification emails, if requested. - // - if (pkg->build_warning_email && *bld->status >= result_status::warning) - send_email (*pkg->build_warning_email); + if (auto f = tsb->build_built (ss, b, log_writer_)) + update_tenant_service_state (conn, b.tenant, f); + } - if (pkg->build_error_email && *bld->status >= result_status::error) - send_email (*pkg->build_error_email); + if (bld != nullptr) + { + // Don't sent the notification email for success-to-success status change, + // etc. + // + if (!build_notify) + (cfg->email ? cfg->email : pkg->build_email) = email (); + + send_notification_email (*options_, + conn, + *bld, + *pkg, + *cfg, + unforced ? "build" : "rebuild", + error, + verb_ >= 2 ? &trace : nullptr); + } return true; } |