diff options
Diffstat (limited to 'mod')
| -rw-r--r-- | mod/ci-common.cxx | 336 | ||||
| -rw-r--r-- | mod/ci-common.hxx | 43 | ||||
| -rw-r--r-- | mod/mod-build-force.cxx | 21 | ||||
| -rw-r--r-- | mod/mod-build-result.cxx | 28 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 151 | ||||
| -rw-r--r-- | mod/mod-ci.cxx | 63 | ||||
| -rw-r--r-- | mod/mod-ci.hxx | 24 | ||||
| -rw-r--r-- | mod/module.cli | 4 | ||||
| -rw-r--r-- | mod/page.cxx | 2 | ||||
| -rw-r--r-- | mod/tenant-service.hxx | 19 |
10 files changed, 622 insertions, 69 deletions
diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx index cb61e66..bd283f4 100644 --- a/mod/ci-common.cxx +++ b/mod/ci-common.cxx @@ -3,6 +3,9 @@ #include <mod/ci-common.hxx> +#include <odb/database.hxx> +#include <odb/transaction.hxx> + #include <libbutl/uuid.hxx> #include <libbutl/fdstream.hxx> #include <libbutl/sendmail.hxx> @@ -11,6 +14,9 @@ #include <libbutl/process-io.hxx> // operator<<(ostream, process_args) #include <libbutl/manifest-serializer.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> + #include <mod/external-handler.hxx> namespace brep @@ -38,13 +44,16 @@ namespace brep options_ = move (o); } - optional<ci_start::start_result> ci_start:: + static optional<ci_start::start_result> start (const basic_mark& error, const basic_mark& warn, const basic_mark* trace, + const options::ci_start& ops, + string&& request_id, optional<tenant_service>&& service, + bool service_load, const repository_location& repository, - const vector<package>& packages, + const vector<ci_start::package>& packages, const optional<string>& client_ip, const optional<string>& user_agent, const optional<string>& interactive, @@ -55,32 +64,15 @@ namespace brep using serializer = manifest_serializer; using serialization = manifest_serialization; - assert (options_ != nullptr); // Shouldn't be called otherwise. + using result = ci_start::start_result; // If the tenant service is specified, then its type may not be empty. // assert (!service || !service->type.empty ()); - // Generate the request id. - // - // Note that it will also be used as a CI result manifest reference, - // unless the latter is provided by the external handler. - // - string request_id; - - try - { - request_id = uuid::generate ().string (); - } - catch (const system_error& e) - { - error << "unable to generate request id: " << e; - return nullopt; - } - // Create the submission data directory. // - dir_path dd (options_->ci_data () / dir_path (request_id)); + dir_path dd (ops.ci_data () / dir_path (request_id)); try { @@ -103,10 +95,10 @@ namespace brep // auto client_error = [&request_id] (uint16_t status, string message) { - return start_result {status, - move (message), - request_id, - vector<pair<string, string>> ()}; + return result {status, + move (message), + request_id, + vector<pair<string, string>> ()}; }; // Serialize the CI request manifest to a stream. On the serialization @@ -119,6 +111,7 @@ namespace brep auto rqm = [&request_id, &ts, &service, + service_load, &repository, &packages, &client_ip, @@ -127,7 +120,7 @@ namespace brep &simulate, &custom_request, &client_error] (ostream& os, bool long_lines = false) - -> pair<bool, optional<start_result>> + -> pair<bool, optional<result>> { try { @@ -139,7 +132,7 @@ namespace brep s.next ("id", request_id); s.next ("repository", repository.string ()); - for (const package& p: packages) + for (const ci_start::package& p: packages) { if (!p.version) s.next ("package", p.name.string ()); @@ -178,6 +171,8 @@ namespace brep if (service->data) s.next ("service-data", *service->data); + + s.next ("service-action", service_load ? "load" : "start"); } // Serialize the request custom parameters. @@ -190,12 +185,12 @@ namespace brep s.next (nv.first, nv.second); s.next ("", ""); // End of manifest. - return make_pair (true, optional<start_result> ()); + return make_pair (true, optional<result> ()); } catch (const serialization& e) { return make_pair (false, - optional<start_result> ( + optional<result> ( client_error (400, string ("invalid parameter: ") + e.what ()))); @@ -209,7 +204,7 @@ namespace brep try { ofdstream os (rqf); - pair<bool, optional<start_result>> r (rqm (os)); + pair<bool, optional<result>> r (rqm (os)); os.close (); if (!r.first) @@ -228,7 +223,7 @@ namespace brep // auto ovm = [&overrides, &client_error] (ostream& os, bool long_lines = false) - -> pair<bool, optional<start_result>> + -> pair<bool, optional<result>> { try { @@ -240,12 +235,12 @@ namespace brep s.next (nv.first, nv.second); s.next ("", ""); // End of manifest. - return make_pair (true, optional<start_result> ()); + return make_pair (true, optional<result> ()); } catch (const serialization& e) { return make_pair (false, - optional<start_result> ( + optional<result> ( client_error ( 400, string ("invalid manifest override: ") + @@ -261,7 +256,7 @@ namespace brep try { ofdstream os (ovf); - pair<bool, optional<start_result>> r (ovm (os)); + pair<bool, optional<result>> r (ovm (os)); os.close (); if (!r.first) @@ -305,16 +300,16 @@ namespace brep // manifest from its stdout and parse it into the resulting manifest // object. Otherwise, create implied CI result manifest. // - start_result sr; + result sr; - if (options_->ci_handler_specified ()) + if (ops.ci_handler_specified ()) { using namespace external_handler; - optional<result_manifest> r (run (options_->ci_handler (), - options_->ci_handler_argument (), + optional<result_manifest> r (run (ops.ci_handler (), + ops.ci_handler_argument (), dd, - options_->ci_handler_timeout (), + ops.ci_handler_timeout (), error, warn, trace)); @@ -358,7 +353,7 @@ namespace brep { try { - serialize_manifest (sr, os, long_lines); + ci_start::serialize_manifest (sr, os, long_lines); return true; } catch (const serialization& e) @@ -424,7 +419,7 @@ namespace brep // assume that the web server error log is monitored and the email sending // failure will be noticed. // - if (options_->ci_email_specified () && !simulate) + if (ops.ci_email_specified () && !simulate) try { // Redirect the diagnostics to the web server error log. @@ -435,14 +430,13 @@ namespace brep *trace << process_args {args, n}; }, 2 /* stderr */, - options_->email (), + ops.email (), "CI request submission (" + sr.reference + ')', - {options_->ci_email ()}); + {ops.ci_email ()}); // Write the CI request manifest. // - pair<bool, optional<start_result>> r ( - rqm (sm.out, true /* long_lines */)); + pair<bool, optional<result>> r (rqm (sm.out, true /* long_lines */)); assert (r.first); // The serialization succeeded once, so can't fail now. @@ -473,7 +467,55 @@ namespace brep error << "sendmail error: " << e; } - return optional<start_result> (move (sr)); + return optional<result> (move (sr)); + } + + optional<ci_start::start_result> ci_start:: + start (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + optional<tenant_service>&& service, + const repository_location& repository, + const vector<package>& packages, + const optional<string>& client_ip, + const optional<string>& user_agent, + const optional<string>& interactive, + const optional<string>& simulate, + const vector<pair<string, string>>& custom_request, + const vector<pair<string, string>>& overrides) const + { + assert (options_ != nullptr); // Shouldn't be called otherwise. + + // Generate the request id. + // + // Note that it will also be used as a CI result manifest reference, + // unless the latter is provided by the external handler. + // + string request_id; + + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return nullopt; + } + + return brep::start (error, warn, trace, + *options_, + move (request_id), + move (service), + false /* service_load */, + repository, + packages, + client_ip, + user_agent, + interactive, + simulate, + custom_request, + overrides); } void ci_start:: @@ -491,4 +533,206 @@ namespace brep s.next ("", ""); // End of manifest. } + + optional<string> ci_start:: + create (const basic_mark& error, + const basic_mark&, + const basic_mark* trace, + odb::core::database& db, + tenant_service&& service) const + { + using namespace odb::core; + + // Generate the request id. + // + string request_id; + + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return nullopt; + } + + // Use the generated request id if the tenant service id is not specified. + // + if (service.id.empty ()) + service.id = request_id; + + // Delay the first load attempt for 10 seconds (see mod-build-task.cxx for + // details). + // + build_tenant t (move (request_id), + move (service), + system_clock::now () - chrono::seconds (40 - 10)); + + { + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + // Note that in contrast to brep-load, we know that the tenant id is + // unique and thus we don't try to remove a tenant with such an id. + // There is also not much reason to assume that we may have switched + // from the single-tenant mode here and remove the respective tenant, + // unless we are in the tenant-service functionality development mode. + // +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + cstrings ts ({""}); + + db.erase_query<build_package> ( + query<build_package>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_repository> ( + query<build_repository>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_public_key> ( + query<build_public_key>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_tenant> ( + query<build_tenant>::id.in_range (ts.begin (), ts.end ())); +#endif + + db.persist (t); + + tr.commit (); + } + + if (trace != nullptr) + *trace << "unloaded CI request " << t.id << " for service " + << t.service->id << ' ' << t.service->type << " is created"; + + return move (t.id); + } + + optional<ci_start::start_result> ci_start:: + load (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database& db, + tenant_service&& service, + const repository_location& repository) const + { + using namespace odb::core; + + string request_id; + { + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + using query = query<build_tenant>; + + shared_ptr<build_tenant> t ( + db.query_one<build_tenant> (query::service.id == service.id && + query::service.type == service.type)); + + if (t == nullptr) + { + error << "unable to find tenant for service " << service.id << ' ' + << service.type; + + return nullopt; + } + else if (t->archived) + { + error << "tenant " << t->id << " for service " << service.id << ' ' + << service.type << " is already archived"; + + return nullopt; + } + else if (!t->loaded_timestamp) + { + error << "tenant " << t->id << " for service " << service.id << ' ' + << service.type << " is already loaded"; + + return nullopt; + } + + t->loaded_timestamp = nullopt; + db.update (t); + + tr.commit (); + + request_id = move (t->id); + } + + assert (options_ != nullptr); // Shouldn't be called otherwise. + + optional<start_result> r (brep::start (error, warn, trace, + *options_, + move (request_id), + move (service), + true /* service_load */, + repository, + {} /* packages */, + nullopt /* client_ip */, + nullopt /* user_agent */, + nullopt /* interactive */, + nullopt /* simulate */, + {} /* custom_request */, + {} /* overrides */)); + + // Note: on error (r == nullopt) the diagnostics is already issued. + // + if (trace != nullptr && r) + *trace << "CI request for '" << repository << "' is " + << (r->status != 200 ? "not " : "") << "loaded: " + << r->message << " (reference: " << r->reference << ')'; + + return r; + } + + void ci_start:: + abandon (const basic_mark& error, + const basic_mark&, + const basic_mark* trace, + odb::core::database& db, + tenant_service&& service) const + { + using namespace odb::core; + + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + using query = query<build_tenant>; + + shared_ptr<build_tenant> t ( + db.query_one<build_tenant> (query::service.id == service.id && + query::service.type == service.type)); + + if (t == nullptr) + { + error << "unable to find tenant for service " << service.id << ' ' + << service.type; + + return; + } + else if (!t->loaded_timestamp) + { + error << "tenant " << t->id << " for service " << service.id << ' ' + << service.type << " is already loaded"; + + return; + } + + // We could probably remove the tenant from the database, but let's just + // archive it and keep for troubleshooting. + // + if (!t->archived) + { + t->archived = true; + db.update (t); + } + + tr.commit (); + + if (trace != nullptr) + *trace << "unloaded CI request " << t->id << " for service " + << service.id << ' ' << service.type << " is abandoned"; + } } diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx index 6f62c4b..45c959a 100644 --- a/mod/ci-common.hxx +++ b/mod/ci-common.hxx @@ -36,6 +36,7 @@ namespace brep package_name name; optional<brep::version> version; }; + // Note that the inability to generate the reference is an internal // error. Thus, it is not optional. // @@ -62,7 +63,47 @@ namespace brep const optional<string>& interactive = nullopt, const optional<string>& simulate = nullopt, const vector<pair<string, string>>& custom_request = {}, - const vector<pair<string, string>>& overrides = {}); + const vector<pair<string, string>>& overrides = {}) const; + + // Create an unloaded CI request returning start_result::reference on + // success and nullopt on an internal error. Such a request is not started + // until loaded with the load() function below. See also the + // build_unloaded() tenant services notification. + // + // Note: should be called out of the database transaction. + // + optional<string> + create (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + tenant_service&&) const; + + // Load (and start) previously created (as unloaded) CI request. Similarly + // to the start() function, return nullopt on an internal error. + // + // Note that tenant_service::id is used to identify the CI request tenant. + // + // Note: should be called out of the database transaction. + // + optional<start_result> + load (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + tenant_service&&, + const repository_location& repository) const; + + // Abandon previously created (as unloaded) CI request. + // + // Note: should be called out of the database transaction. + // + void + abandon (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + tenant_service&&) const; // Helpers. // diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx index bdae356..168a835 100644 --- a/mod/mod-build-force.cxx +++ b/mod/mod-build-force.cxx @@ -192,7 +192,14 @@ handle (request& rq, response& rs) optional<pair<tenant_service, shared_ptr<build>>> tss; tenant_service_build_queued::build_queued_hints qhs; + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // connection_ptr conn (build_db_->connection ()); + { transaction t (conn->begin ()); @@ -297,14 +304,28 @@ handle (request& rq, response& rs) vector<build> qbs; qbs.push_back (move (b)); + // Release the database connection since the build_queued() notification + // can potentially be time-consuming (e.g., it may perform an HTTP + // request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, build_state::building, qhs, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, qbs.back ().tenant, f); + } } + // Release the database connection prior to writing into the unbuffered + // response stream. + // + conn.reset (); + // We have all the data, so don't buffer the response content. // ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index ccce17f..64503aa 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -207,13 +207,20 @@ handle (request& rq, response&) optional<pair<tenant_service, shared_ptr<build>>> tss; tenant_service_build_queued::build_queued_hints qhs; + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // + connection_ptr conn (build_db_->connection ()); + // Note that if the session authentication fails (probably due to the // authentication settings change), then we log this case with the warning // severity and respond with the 200 HTTP code as if the challenge is // valid. The thinking is that we shouldn't alarm a law-abaiding agent and // shouldn't provide any information to a malicious one. // - connection_ptr conn (build_db_->connection ()); { transaction t (conn->begin ()); @@ -518,12 +525,20 @@ handle (request& rq, response&) vector<build> qbs; qbs.push_back (move (*tss->second)); + // Release the database connection since build_queued() notification can + // potentially be time-consuming (e.g., it may perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, build_state::building, qhs, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, qbs.back ().tenant, f); + } } // If a third-party service needs to be notified about the built package, @@ -537,8 +552,16 @@ handle (request& rq, response&) const tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since build_built() notification can + // potentially be time-consuming (e.g., it may perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_built (ss, b, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, b.tenant, f); + } } if (bld != nullptr) @@ -549,6 +572,9 @@ handle (request& rq, response&) if (!build_notify) (cfg->email ? cfg->email : pkg->build_email) = email (); + if (conn == nullptr) + conn = build_db_->connection (); + send_notification_email (*options_, conn, *bld, diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 07aff8d..0752603 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -399,6 +399,92 @@ handle (request& rq, response& rs) } } + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // + connection_ptr conn (build_db_->connection ()); + + timestamp now (system_clock::now ()); + + auto expiration = [&now] (size_t timeout) -> timestamp + { + return now - chrono::seconds (timeout); + }; + + auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t + { + return chrono::duration_cast<chrono::nanoseconds> ( + expiration (timeout).time_since_epoch ()).count (); + }; + + // Perform some housekeeping first. + // + // Notify a tenant-associated third-party service about the unloaded CI + // request, if present. + // + { + const tenant_service_build_unloaded* tsu (nullptr); + + transaction tr (conn->begin ()); + + using query = query<build_tenant>; + + // Pick the unloaded tenant with the earliest loaded timestamp, skipping + // those whose timestamp is less than 40 seconds ago. + // + // NOTE: don't forget to update ci_start::create() if changing the timeout + // here. + // + shared_ptr<build_tenant> t ( + build_db_->query_one<build_tenant> ( + (query::loaded_timestamp <= expiration_ns (40) && + !query::archived) + + "ORDER BY" + query::loaded_timestamp + + "LIMIT 1")); + + if (t != nullptr && t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + tsu = dynamic_cast<const tenant_service_build_unloaded*> ( + i->second.get ()); + + if (tsu != nullptr) + { + // If we ought to call the + // tenant_service_build_unloaded::build_unloaded() callback, then + // set the package tenant's loaded timestamp to the current time to + // prevent the notifications race. + // + t->loaded_timestamp = system_clock::now (); + build_db_->update (t); + } + } + } + + tr.commit (); + + if (tsu != nullptr) + { + // Release the database connection since the build_unloaded() + // notification can potentially be time-consuming (e.g., it may perform + // an HTTP request). + // + conn.reset (); + + if (auto f = tsu->build_unloaded (move (*t->service), log_writer_)) + { + conn = build_db_->connection (); + update_tenant_service_state (conn, t->id, f); + } + } + } + // Go through package build configurations until we find one that has no // build target configuration present in the database, or is in the building // state but expired (collectively called unbuilt). If such a target @@ -524,19 +610,6 @@ handle (request& rq, response& rs) // Calculate the build/rebuild (building/built state) and the `queued` // notifications expiration time for package configurations. // - timestamp now (system_clock::now ()); - - auto expiration = [&now] (size_t timeout) -> timestamp - { - return now - chrono::seconds (timeout); - }; - - auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t - { - return chrono::duration_cast<chrono::nanoseconds> ( - expiration (timeout).time_since_epoch ()).count (); - }; - uint64_t normal_result_expiration_ns ( expiration_ns (options_->build_result_timeout ())); @@ -825,7 +898,10 @@ handle (request& rq, response& rs) imode, queued_expiration_ns)); - transaction t (build_db_->begin ()); + if (conn == nullptr) + conn = build_db_->connection (); + + transaction t (conn->begin ()); // If there are any non-archived interactive build tenants, then the // chosen randomization approach doesn't really work since interactive @@ -886,7 +962,8 @@ handle (request& rq, response& rs) "OFFSET" + pkg_query::_ref (offset) + "LIMIT" + pkg_query::_ref (limit); - connection_ptr conn (build_db_->connection ()); + if (conn == nullptr) + conn = build_db_->connection (); prep_pkg_query pkg_prep_query ( conn->prepare_query<buildable_package> ( @@ -2226,12 +2303,20 @@ handle (request& rq, response& rs) if (!qbs.empty ()) { + // Release the database connection since the build_queued() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, nullopt /* initial_state */, qhs, log_writer_)) { + conn = build_db_->connection (); + if (optional<string> data = update_tenant_service_state (conn, qbs.back ().tenant, f)) ss.data = move (data); @@ -2250,12 +2335,20 @@ handle (request& rq, response& rs) qbs.push_back (move (b)); restore_build = true; + // Release the database connection since the build_queued() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, initial_state, qhs, log_writer_)) { + conn = build_db_->connection (); + if (optional<string> data = update_tenant_service_state (conn, qbs.back ().tenant, f)) ss.data = move (data); @@ -2278,8 +2371,16 @@ handle (request& rq, response& rs) tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since the build_building() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_building (ss, b, log_writer_)) { + conn = build_db_->connection (); + if (optional<string> data = update_tenant_service_state (conn, b.tenant, f)) ss.data = move (data); @@ -2306,6 +2407,9 @@ handle (request& rq, response& rs) const tenant_service_build_built* tsb (nullptr); optional<pair<tenant_service, shared_ptr<build>>> tss; { + if (conn == nullptr) + conn = build_db_->connection (); + transaction t (conn->begin ()); shared_ptr<build> b (build_db_->find<build> (task_build->id)); @@ -2395,8 +2499,16 @@ handle (request& rq, response& rs) tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since the build_built() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_built (ss, b, log_writer_)) { + conn = build_db_->connection (); + if (optional<string> data = update_tenant_service_state (conn, b.tenant, f)) ss.data = move (data); @@ -2407,6 +2519,10 @@ handle (request& rq, response& rs) // Send notification emails for all the aborted builds. // for (const aborted_build& ab: aborted_builds) + { + if (conn == nullptr) + conn = build_db_->connection (); + send_notification_email (*options_, conn, *ab.b, @@ -2415,9 +2531,14 @@ handle (request& rq, response& rs) ab.what, error, verb_ >= 2 ? &trace : nullptr); + } } } + // Release the database connection as soon as possible. + // + conn.reset (); + serialize_task_response_manifest (); return true; } diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx index 5974d45..18e38fc 100644 --- a/mod/mod-ci.cxx +++ b/mod/mod-ci.cxx @@ -36,7 +36,12 @@ ci (const ci& r, tenant_service_map& tsm) #else ci (const ci& r) #endif - : handler (r), + : +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED + handler (r), + #else + database_module (r), +#endif ci_start (r), options_ (r.initialized_ ? r.options_ : nullptr), form_ (r.initialized_ || r.form_ == nullptr @@ -100,6 +105,13 @@ init (scanner& s) } } +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + if (!options_->build_config_specified ()) + fail << "package building functionality must be enabled"; + + database_module::init (*options_, options_->build_db_retry ()); +#endif + if (options_->root ().empty ()) options_->root (dir_path ("/")); } @@ -347,6 +359,7 @@ handle (request& rq, response& rs) user_agent = h.value; } +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED optional<start_result> r (start (error, warn, verb_ ? &trace : nullptr, @@ -367,6 +380,23 @@ handle (request& rq, response& rs) : optional<string> ()), custom_request, overrides)); +#else + assert (build_db_ != nullptr); // Wouldn't be here otherwise. + + optional<start_result> r; + + if (optional<string> ref = create (error, + warn, + verb_ ? &trace : nullptr, + *build_db_, + tenant_service ("", "ci", rl.string ()))) + { + string msg ("unloaded CI request is created: " + + options_->host () + tenant_dir (root, *ref).string ()); + + r = start_result {200, move (msg), move (*ref), {}}; + } +#endif if (!r) return respond_error (); // The diagnostics is already issued. @@ -472,4 +502,35 @@ build_built (const tenant_service&, return ts.data ? *ts.data + ", " + s : s; }; } + +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED +function<optional<string> (const brep::tenant_service&)> brep::ci:: +build_unloaded (tenant_service&& ts, + const diag_epilogue& log_writer) const noexcept +{ + NOTIFICATION_DIAG (log_writer); + + assert (ts.data); // Repository location. + + try + { + repository_location rl (*ts.data); + + if (!load (error, warn, verb_ ? &trace : nullptr, + *build_db_, + move (ts), + rl)) + return nullptr; // The diagnostics is already issued. + } + catch (const invalid_argument& e) + { + error << "invalid repository location '" << *ts.data << "' stored for " + << "tenant service " << ts.id << ' ' << ts.type; + + return nullptr; + } + + return [] (const tenant_service& ts) {return "loaded " + *ts.data;}; +} +#endif #endif diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx index 1e2ee15..a83b9d3 100644 --- a/mod/mod-ci.hxx +++ b/mod/mod-ci.hxx @@ -17,18 +17,34 @@ #include <mod/ci-common.hxx> +#if defined(BREP_CI_TENANT_SERVICE_UNLOADED) && !defined(BREP_CI_TENANT_SERVICE) +# error BREP_CI_TENANT_SERVICE must be defined if BREP_CI_TENANT_SERVICE_UNLOADED is defined +#endif + #ifdef BREP_CI_TENANT_SERVICE # include <mod/tenant-service.hxx> + +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED +# include <mod/database-module.hxx> +#endif #endif namespace brep { - class ci: public handler, + class ci: +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED + public handler, +#else + public database_module, +#endif private ci_start #ifdef BREP_CI_TENANT_SERVICE , public tenant_service_build_queued, public tenant_service_build_building, public tenant_service_build_built +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + , tenant_service_build_unloaded +#endif #endif { public: @@ -74,6 +90,12 @@ namespace brep build_built (const tenant_service&, const build&, const diag_epilogue& log_writer) const noexcept override; + +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + virtual function<optional<string> (const tenant_service&)> + build_unloaded (tenant_service&&, + const diag_epilogue& log_writer) const noexcept override; +#endif #endif private: diff --git a/mod/module.cli b/mod/module.cli index a107ffe..5f63930 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -800,7 +800,7 @@ namespace brep { }; - class ci: ci_start, page, repository_url, handler + class ci: ci_start, build, build_db, page, repository_url, handler { // Classic CI-specific options. // @@ -815,7 +815,7 @@ namespace brep } }; - class ci_github: ci_start, ci_cancel, build_db, handler + class ci_github: ci_start, ci_cancel, build, build_db, handler { // GitHub CI-specific options (e.g., request timeout when invoking // GitHub APIs). diff --git a/mod/page.cxx b/mod/page.cxx index bc2e42d..177fb64 100644 --- a/mod/page.cxx +++ b/mod/page.cxx @@ -739,7 +739,7 @@ namespace brep << ~TR; } - // BUILD_RESULT + // TR_BUILD_RESULT // void TR_BUILD_RESULT:: operator() (serializer& s) const diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx index 9205f76..b7f5c02 100644 --- a/mod/tenant-service.hxx +++ b/mod/tenant-service.hxx @@ -21,7 +21,8 @@ namespace brep virtual ~tenant_service_base () = default; }; - // Possible build notifications: + // Possible build notifications (see also the unloaded special notification + // below): // // queued // building @@ -121,6 +122,22 @@ namespace brep const diag_epilogue& log_writer) const noexcept = 0; }; + // This notification is only made on unloaded CI requests created with the + // ci_start::create() call and until they are loaded with ci_start::load() + // or, alternatively, abandoned with ci_start::abandon(). + // + // Note: make sure the implementation of this notification does not take + // too long (currently 40 seconds) to avoid nested notifications. Note + // also that the first notification is delayed (currently 10 seconds). + // + class tenant_service_build_unloaded: public virtual tenant_service_base + { + public: + virtual function<optional<string> (const tenant_service&)> + build_unloaded (tenant_service&&, + const diag_epilogue& log_writer) const noexcept = 0; + }; + // Map of service type (tenant_service::type) to service. // using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>; |