aboutsummaryrefslogtreecommitdiff
path: root/mod
diff options
context:
space:
mode:
Diffstat (limited to 'mod')
-rw-r--r--mod/ci-common.cxx336
-rw-r--r--mod/ci-common.hxx43
-rw-r--r--mod/mod-build-force.cxx21
-rw-r--r--mod/mod-build-result.cxx28
-rw-r--r--mod/mod-build-task.cxx151
-rw-r--r--mod/mod-ci.cxx63
-rw-r--r--mod/mod-ci.hxx24
-rw-r--r--mod/module.cli4
-rw-r--r--mod/page.cxx2
-rw-r--r--mod/tenant-service.hxx19
10 files changed, 622 insertions, 69 deletions
diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx
index cb61e66..bd283f4 100644
--- a/mod/ci-common.cxx
+++ b/mod/ci-common.cxx
@@ -3,6 +3,9 @@
#include <mod/ci-common.hxx>
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
#include <libbutl/uuid.hxx>
#include <libbutl/fdstream.hxx>
#include <libbutl/sendmail.hxx>
@@ -11,6 +14,9 @@
#include <libbutl/process-io.hxx> // operator<<(ostream, process_args)
#include <libbutl/manifest-serializer.hxx>
+#include <libbrep/build-package.hxx>
+#include <libbrep/build-package-odb.hxx>
+
#include <mod/external-handler.hxx>
namespace brep
@@ -38,13 +44,16 @@ namespace brep
options_ = move (o);
}
- optional<ci_start::start_result> ci_start::
+ static optional<ci_start::start_result>
start (const basic_mark& error,
const basic_mark& warn,
const basic_mark* trace,
+ const options::ci_start& ops,
+ string&& request_id,
optional<tenant_service>&& service,
+ bool service_load,
const repository_location& repository,
- const vector<package>& packages,
+ const vector<ci_start::package>& packages,
const optional<string>& client_ip,
const optional<string>& user_agent,
const optional<string>& interactive,
@@ -55,32 +64,15 @@ namespace brep
using serializer = manifest_serializer;
using serialization = manifest_serialization;
- assert (options_ != nullptr); // Shouldn't be called otherwise.
+ using result = ci_start::start_result;
// If the tenant service is specified, then its type may not be empty.
//
assert (!service || !service->type.empty ());
- // Generate the request id.
- //
- // Note that it will also be used as a CI result manifest reference,
- // unless the latter is provided by the external handler.
- //
- string request_id;
-
- try
- {
- request_id = uuid::generate ().string ();
- }
- catch (const system_error& e)
- {
- error << "unable to generate request id: " << e;
- return nullopt;
- }
-
// Create the submission data directory.
//
- dir_path dd (options_->ci_data () / dir_path (request_id));
+ dir_path dd (ops.ci_data () / dir_path (request_id));
try
{
@@ -103,10 +95,10 @@ namespace brep
//
auto client_error = [&request_id] (uint16_t status, string message)
{
- return start_result {status,
- move (message),
- request_id,
- vector<pair<string, string>> ()};
+ return result {status,
+ move (message),
+ request_id,
+ vector<pair<string, string>> ()};
};
// Serialize the CI request manifest to a stream. On the serialization
@@ -119,6 +111,7 @@ namespace brep
auto rqm = [&request_id,
&ts,
&service,
+ service_load,
&repository,
&packages,
&client_ip,
@@ -127,7 +120,7 @@ namespace brep
&simulate,
&custom_request,
&client_error] (ostream& os, bool long_lines = false)
- -> pair<bool, optional<start_result>>
+ -> pair<bool, optional<result>>
{
try
{
@@ -139,7 +132,7 @@ namespace brep
s.next ("id", request_id);
s.next ("repository", repository.string ());
- for (const package& p: packages)
+ for (const ci_start::package& p: packages)
{
if (!p.version)
s.next ("package", p.name.string ());
@@ -178,6 +171,8 @@ namespace brep
if (service->data)
s.next ("service-data", *service->data);
+
+ s.next ("service-action", service_load ? "load" : "start");
}
// Serialize the request custom parameters.
@@ -190,12 +185,12 @@ namespace brep
s.next (nv.first, nv.second);
s.next ("", ""); // End of manifest.
- return make_pair (true, optional<start_result> ());
+ return make_pair (true, optional<result> ());
}
catch (const serialization& e)
{
return make_pair (false,
- optional<start_result> (
+ optional<result> (
client_error (400,
string ("invalid parameter: ") +
e.what ())));
@@ -209,7 +204,7 @@ namespace brep
try
{
ofdstream os (rqf);
- pair<bool, optional<start_result>> r (rqm (os));
+ pair<bool, optional<result>> r (rqm (os));
os.close ();
if (!r.first)
@@ -228,7 +223,7 @@ namespace brep
//
auto ovm = [&overrides, &client_error] (ostream& os,
bool long_lines = false)
- -> pair<bool, optional<start_result>>
+ -> pair<bool, optional<result>>
{
try
{
@@ -240,12 +235,12 @@ namespace brep
s.next (nv.first, nv.second);
s.next ("", ""); // End of manifest.
- return make_pair (true, optional<start_result> ());
+ return make_pair (true, optional<result> ());
}
catch (const serialization& e)
{
return make_pair (false,
- optional<start_result> (
+ optional<result> (
client_error (
400,
string ("invalid manifest override: ") +
@@ -261,7 +256,7 @@ namespace brep
try
{
ofdstream os (ovf);
- pair<bool, optional<start_result>> r (ovm (os));
+ pair<bool, optional<result>> r (ovm (os));
os.close ();
if (!r.first)
@@ -305,16 +300,16 @@ namespace brep
// manifest from its stdout and parse it into the resulting manifest
// object. Otherwise, create implied CI result manifest.
//
- start_result sr;
+ result sr;
- if (options_->ci_handler_specified ())
+ if (ops.ci_handler_specified ())
{
using namespace external_handler;
- optional<result_manifest> r (run (options_->ci_handler (),
- options_->ci_handler_argument (),
+ optional<result_manifest> r (run (ops.ci_handler (),
+ ops.ci_handler_argument (),
dd,
- options_->ci_handler_timeout (),
+ ops.ci_handler_timeout (),
error,
warn,
trace));
@@ -358,7 +353,7 @@ namespace brep
{
try
{
- serialize_manifest (sr, os, long_lines);
+ ci_start::serialize_manifest (sr, os, long_lines);
return true;
}
catch (const serialization& e)
@@ -424,7 +419,7 @@ namespace brep
// assume that the web server error log is monitored and the email sending
// failure will be noticed.
//
- if (options_->ci_email_specified () && !simulate)
+ if (ops.ci_email_specified () && !simulate)
try
{
// Redirect the diagnostics to the web server error log.
@@ -435,14 +430,13 @@ namespace brep
*trace << process_args {args, n};
},
2 /* stderr */,
- options_->email (),
+ ops.email (),
"CI request submission (" + sr.reference + ')',
- {options_->ci_email ()});
+ {ops.ci_email ()});
// Write the CI request manifest.
//
- pair<bool, optional<start_result>> r (
- rqm (sm.out, true /* long_lines */));
+ pair<bool, optional<result>> r (rqm (sm.out, true /* long_lines */));
assert (r.first); // The serialization succeeded once, so can't fail now.
@@ -473,7 +467,55 @@ namespace brep
error << "sendmail error: " << e;
}
- return optional<start_result> (move (sr));
+ return optional<result> (move (sr));
+ }
+
+ optional<ci_start::start_result> ci_start::
+ start (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ optional<tenant_service>&& service,
+ const repository_location& repository,
+ const vector<package>& packages,
+ const optional<string>& client_ip,
+ const optional<string>& user_agent,
+ const optional<string>& interactive,
+ const optional<string>& simulate,
+ const vector<pair<string, string>>& custom_request,
+ const vector<pair<string, string>>& overrides) const
+ {
+ assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+ // Generate the request id.
+ //
+ // Note that it will also be used as a CI result manifest reference,
+ // unless the latter is provided by the external handler.
+ //
+ string request_id;
+
+ try
+ {
+ request_id = uuid::generate ().string ();
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to generate request id: " << e;
+ return nullopt;
+ }
+
+ return brep::start (error, warn, trace,
+ *options_,
+ move (request_id),
+ move (service),
+ false /* service_load */,
+ repository,
+ packages,
+ client_ip,
+ user_agent,
+ interactive,
+ simulate,
+ custom_request,
+ overrides);
}
void ci_start::
@@ -491,4 +533,206 @@ namespace brep
s.next ("", ""); // End of manifest.
}
+
+ optional<string> ci_start::
+ create (const basic_mark& error,
+ const basic_mark&,
+ const basic_mark* trace,
+ odb::core::database& db,
+ tenant_service&& service) const
+ {
+ using namespace odb::core;
+
+ // Generate the request id.
+ //
+ string request_id;
+
+ try
+ {
+ request_id = uuid::generate ().string ();
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to generate request id: " << e;
+ return nullopt;
+ }
+
+ // Use the generated request id if the tenant service id is not specified.
+ //
+ if (service.id.empty ())
+ service.id = request_id;
+
+ // Delay the first load attempt for 10 seconds (see mod-build-task.cxx for
+ // details).
+ //
+ build_tenant t (move (request_id),
+ move (service),
+ system_clock::now () - chrono::seconds (40 - 10));
+
+ {
+ assert (!transaction::has_current ());
+
+ transaction tr (db.begin ());
+
+ // Note that in contrast to brep-load, we know that the tenant id is
+ // unique and thus we don't try to remove a tenant with such an id.
+ // There is also not much reason to assume that we may have switched
+ // from the single-tenant mode here and remove the respective tenant,
+ // unless we are in the tenant-service functionality development mode.
+ //
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+ cstrings ts ({""});
+
+ db.erase_query<build_package> (
+ query<build_package>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+ db.erase_query<build_repository> (
+ query<build_repository>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+ db.erase_query<build_public_key> (
+ query<build_public_key>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+ db.erase_query<build_tenant> (
+ query<build_tenant>::id.in_range (ts.begin (), ts.end ()));
+#endif
+
+ db.persist (t);
+
+ tr.commit ();
+ }
+
+ if (trace != nullptr)
+ *trace << "unloaded CI request " << t.id << " for service "
+ << t.service->id << ' ' << t.service->type << " is created";
+
+ return move (t.id);
+ }
+
+ optional<ci_start::start_result> ci_start::
+ load (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ odb::core::database& db,
+ tenant_service&& service,
+ const repository_location& repository) const
+ {
+ using namespace odb::core;
+
+ string request_id;
+ {
+ assert (!transaction::has_current ());
+
+ transaction tr (db.begin ());
+
+ using query = query<build_tenant>;
+
+ shared_ptr<build_tenant> t (
+ db.query_one<build_tenant> (query::service.id == service.id &&
+ query::service.type == service.type));
+
+ if (t == nullptr)
+ {
+ error << "unable to find tenant for service " << service.id << ' '
+ << service.type;
+
+ return nullopt;
+ }
+ else if (t->archived)
+ {
+ error << "tenant " << t->id << " for service " << service.id << ' '
+ << service.type << " is already archived";
+
+ return nullopt;
+ }
+ else if (!t->loaded_timestamp)
+ {
+ error << "tenant " << t->id << " for service " << service.id << ' '
+ << service.type << " is already loaded";
+
+ return nullopt;
+ }
+
+ t->loaded_timestamp = nullopt;
+ db.update (t);
+
+ tr.commit ();
+
+ request_id = move (t->id);
+ }
+
+ assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+ optional<start_result> r (brep::start (error, warn, trace,
+ *options_,
+ move (request_id),
+ move (service),
+ true /* service_load */,
+ repository,
+ {} /* packages */,
+ nullopt /* client_ip */,
+ nullopt /* user_agent */,
+ nullopt /* interactive */,
+ nullopt /* simulate */,
+ {} /* custom_request */,
+ {} /* overrides */));
+
+ // Note: on error (r == nullopt) the diagnostics is already issued.
+ //
+ if (trace != nullptr && r)
+ *trace << "CI request for '" << repository << "' is "
+ << (r->status != 200 ? "not " : "") << "loaded: "
+ << r->message << " (reference: " << r->reference << ')';
+
+ return r;
+ }
+
+ void ci_start::
+ abandon (const basic_mark& error,
+ const basic_mark&,
+ const basic_mark* trace,
+ odb::core::database& db,
+ tenant_service&& service) const
+ {
+ using namespace odb::core;
+
+ assert (!transaction::has_current ());
+
+ transaction tr (db.begin ());
+
+ using query = query<build_tenant>;
+
+ shared_ptr<build_tenant> t (
+ db.query_one<build_tenant> (query::service.id == service.id &&
+ query::service.type == service.type));
+
+ if (t == nullptr)
+ {
+ error << "unable to find tenant for service " << service.id << ' '
+ << service.type;
+
+ return;
+ }
+ else if (!t->loaded_timestamp)
+ {
+ error << "tenant " << t->id << " for service " << service.id << ' '
+ << service.type << " is already loaded";
+
+ return;
+ }
+
+ // We could probably remove the tenant from the database, but let's just
+ // archive it and keep for troubleshooting.
+ //
+ if (!t->archived)
+ {
+ t->archived = true;
+ db.update (t);
+ }
+
+ tr.commit ();
+
+ if (trace != nullptr)
+ *trace << "unloaded CI request " << t->id << " for service "
+ << service.id << ' ' << service.type << " is abandoned";
+ }
}
diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx
index 6f62c4b..45c959a 100644
--- a/mod/ci-common.hxx
+++ b/mod/ci-common.hxx
@@ -36,6 +36,7 @@ namespace brep
package_name name;
optional<brep::version> version;
};
+
// Note that the inability to generate the reference is an internal
// error. Thus, it is not optional.
//
@@ -62,7 +63,47 @@ namespace brep
const optional<string>& interactive = nullopt,
const optional<string>& simulate = nullopt,
const vector<pair<string, string>>& custom_request = {},
- const vector<pair<string, string>>& overrides = {});
+ const vector<pair<string, string>>& overrides = {}) const;
+
+ // Create an unloaded CI request returning start_result::reference on
+ // success and nullopt on an internal error. Such a request is not started
+ // until loaded with the load() function below. See also the
+ // build_unloaded() tenant services notification.
+ //
+ // Note: should be called out of the database transaction.
+ //
+ optional<string>
+ create (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ odb::core::database&,
+ tenant_service&&) const;
+
+ // Load (and start) previously created (as unloaded) CI request. Similarly
+ // to the start() function, return nullopt on an internal error.
+ //
+ // Note that tenant_service::id is used to identify the CI request tenant.
+ //
+ // Note: should be called out of the database transaction.
+ //
+ optional<start_result>
+ load (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ odb::core::database&,
+ tenant_service&&,
+ const repository_location& repository) const;
+
+ // Abandon previously created (as unloaded) CI request.
+ //
+ // Note: should be called out of the database transaction.
+ //
+ void
+ abandon (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ odb::core::database&,
+ tenant_service&&) const;
// Helpers.
//
diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx
index bdae356..168a835 100644
--- a/mod/mod-build-force.cxx
+++ b/mod/mod-build-force.cxx
@@ -192,7 +192,14 @@ handle (request& rq, response& rs)
optional<pair<tenant_service, shared_ptr<build>>> tss;
tenant_service_build_queued::build_queued_hints qhs;
+ // Acquire the database connection for the subsequent transactions.
+ //
+ // Note that we will release it prior to any potentially time-consuming
+ // operations (such as HTTP requests) and re-acquire it again afterwards,
+ // if required.
+ //
connection_ptr conn (build_db_->connection ());
+
{
transaction t (conn->begin ());
@@ -297,14 +304,28 @@ handle (request& rq, response& rs)
vector<build> qbs;
qbs.push_back (move (b));
+ // Release the database connection since the build_queued() notification
+ // can potentially be time-consuming (e.g., it may perform an HTTP
+ // request).
+ //
+ conn.reset ();
+
if (auto f = tsq->build_queued (ss,
qbs,
build_state::building,
qhs,
log_writer_))
+ {
+ conn = build_db_->connection ();
update_tenant_service_state (conn, qbs.back ().tenant, f);
+ }
}
+ // Release the database connection prior to writing into the unbuffered
+ // response stream.
+ //
+ conn.reset ();
+
// We have all the data, so don't buffer the response content.
//
ostream& os (rs.content (200, "text/plain;charset=utf-8", false));
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index ccce17f..64503aa 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -207,13 +207,20 @@ handle (request& rq, response&)
optional<pair<tenant_service, shared_ptr<build>>> tss;
tenant_service_build_queued::build_queued_hints qhs;
+ // Acquire the database connection for the subsequent transactions.
+ //
+ // Note that we will release it prior to any potentially time-consuming
+ // operations (such as HTTP requests) and re-acquire it again afterwards,
+ // if required.
+ //
+ connection_ptr conn (build_db_->connection ());
+
// Note that if the session authentication fails (probably due to the
// authentication settings change), then we log this case with the warning
// severity and respond with the 200 HTTP code as if the challenge is
// valid. The thinking is that we shouldn't alarm a law-abaiding agent and
// shouldn't provide any information to a malicious one.
//
- connection_ptr conn (build_db_->connection ());
{
transaction t (conn->begin ());
@@ -518,12 +525,20 @@ handle (request& rq, response&)
vector<build> qbs;
qbs.push_back (move (*tss->second));
+ // Release the database connection since build_queued() notification can
+ // potentially be time-consuming (e.g., it may perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsq->build_queued (ss,
qbs,
build_state::building,
qhs,
log_writer_))
+ {
+ conn = build_db_->connection ();
update_tenant_service_state (conn, qbs.back ().tenant, f);
+ }
}
// If a third-party service needs to be notified about the built package,
@@ -537,8 +552,16 @@ handle (request& rq, response&)
const tenant_service& ss (tss->first);
const build& b (*tss->second);
+ // Release the database connection since build_built() notification can
+ // potentially be time-consuming (e.g., it may perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsb->build_built (ss, b, log_writer_))
+ {
+ conn = build_db_->connection ();
update_tenant_service_state (conn, b.tenant, f);
+ }
}
if (bld != nullptr)
@@ -549,6 +572,9 @@ handle (request& rq, response&)
if (!build_notify)
(cfg->email ? cfg->email : pkg->build_email) = email ();
+ if (conn == nullptr)
+ conn = build_db_->connection ();
+
send_notification_email (*options_,
conn,
*bld,
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 07aff8d..0752603 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -399,6 +399,92 @@ handle (request& rq, response& rs)
}
}
+ // Acquire the database connection for the subsequent transactions.
+ //
+ // Note that we will release it prior to any potentially time-consuming
+ // operations (such as HTTP requests) and re-acquire it again afterwards,
+ // if required.
+ //
+ connection_ptr conn (build_db_->connection ());
+
+ timestamp now (system_clock::now ());
+
+ auto expiration = [&now] (size_t timeout) -> timestamp
+ {
+ return now - chrono::seconds (timeout);
+ };
+
+ auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t
+ {
+ return chrono::duration_cast<chrono::nanoseconds> (
+ expiration (timeout).time_since_epoch ()).count ();
+ };
+
+ // Perform some housekeeping first.
+ //
+ // Notify a tenant-associated third-party service about the unloaded CI
+ // request, if present.
+ //
+ {
+ const tenant_service_build_unloaded* tsu (nullptr);
+
+ transaction tr (conn->begin ());
+
+ using query = query<build_tenant>;
+
+ // Pick the unloaded tenant with the earliest loaded timestamp, skipping
+ // those whose timestamp is less than 40 seconds ago.
+ //
+ // NOTE: don't forget to update ci_start::create() if changing the timeout
+ // here.
+ //
+ shared_ptr<build_tenant> t (
+ build_db_->query_one<build_tenant> (
+ (query::loaded_timestamp <= expiration_ns (40) &&
+ !query::archived) +
+ "ORDER BY" + query::loaded_timestamp +
+ "LIMIT 1"));
+
+ if (t != nullptr && t->service)
+ {
+ auto i (tenant_service_map_.find (t->service->type));
+
+ if (i != tenant_service_map_.end ())
+ {
+ tsu = dynamic_cast<const tenant_service_build_unloaded*> (
+ i->second.get ());
+
+ if (tsu != nullptr)
+ {
+ // If we ought to call the
+ // tenant_service_build_unloaded::build_unloaded() callback, then
+ // set the package tenant's loaded timestamp to the current time to
+ // prevent the notifications race.
+ //
+ t->loaded_timestamp = system_clock::now ();
+ build_db_->update (t);
+ }
+ }
+ }
+
+ tr.commit ();
+
+ if (tsu != nullptr)
+ {
+ // Release the database connection since the build_unloaded()
+ // notification can potentially be time-consuming (e.g., it may perform
+ // an HTTP request).
+ //
+ conn.reset ();
+
+ if (auto f = tsu->build_unloaded (move (*t->service), log_writer_))
+ {
+ conn = build_db_->connection ();
+ update_tenant_service_state (conn, t->id, f);
+ }
+ }
+ }
+
// Go through package build configurations until we find one that has no
// build target configuration present in the database, or is in the building
// state but expired (collectively called unbuilt). If such a target
@@ -524,19 +610,6 @@ handle (request& rq, response& rs)
// Calculate the build/rebuild (building/built state) and the `queued`
// notifications expiration time for package configurations.
//
- timestamp now (system_clock::now ());
-
- auto expiration = [&now] (size_t timeout) -> timestamp
- {
- return now - chrono::seconds (timeout);
- };
-
- auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t
- {
- return chrono::duration_cast<chrono::nanoseconds> (
- expiration (timeout).time_since_epoch ()).count ();
- };
-
uint64_t normal_result_expiration_ns (
expiration_ns (options_->build_result_timeout ()));
@@ -825,7 +898,10 @@ handle (request& rq, response& rs)
imode,
queued_expiration_ns));
- transaction t (build_db_->begin ());
+ if (conn == nullptr)
+ conn = build_db_->connection ();
+
+ transaction t (conn->begin ());
// If there are any non-archived interactive build tenants, then the
// chosen randomization approach doesn't really work since interactive
@@ -886,7 +962,8 @@ handle (request& rq, response& rs)
"OFFSET" + pkg_query::_ref (offset) +
"LIMIT" + pkg_query::_ref (limit);
- connection_ptr conn (build_db_->connection ());
+ if (conn == nullptr)
+ conn = build_db_->connection ();
prep_pkg_query pkg_prep_query (
conn->prepare_query<buildable_package> (
@@ -2226,12 +2303,20 @@ handle (request& rq, response& rs)
if (!qbs.empty ())
{
+ // Release the database connection since the build_queued()
+ // notification can potentially be time-consuming (e.g., it may
+ // perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsq->build_queued (ss,
qbs,
nullopt /* initial_state */,
qhs,
log_writer_))
{
+ conn = build_db_->connection ();
+
if (optional<string> data =
update_tenant_service_state (conn, qbs.back ().tenant, f))
ss.data = move (data);
@@ -2250,12 +2335,20 @@ handle (request& rq, response& rs)
qbs.push_back (move (b));
restore_build = true;
+ // Release the database connection since the build_queued()
+ // notification can potentially be time-consuming (e.g., it may
+ // perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsq->build_queued (ss,
qbs,
initial_state,
qhs,
log_writer_))
{
+ conn = build_db_->connection ();
+
if (optional<string> data =
update_tenant_service_state (conn, qbs.back ().tenant, f))
ss.data = move (data);
@@ -2278,8 +2371,16 @@ handle (request& rq, response& rs)
tenant_service& ss (tss->first);
const build& b (*tss->second);
+ // Release the database connection since the build_building()
+ // notification can potentially be time-consuming (e.g., it may
+ // perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsb->build_building (ss, b, log_writer_))
{
+ conn = build_db_->connection ();
+
if (optional<string> data =
update_tenant_service_state (conn, b.tenant, f))
ss.data = move (data);
@@ -2306,6 +2407,9 @@ handle (request& rq, response& rs)
const tenant_service_build_built* tsb (nullptr);
optional<pair<tenant_service, shared_ptr<build>>> tss;
{
+ if (conn == nullptr)
+ conn = build_db_->connection ();
+
transaction t (conn->begin ());
shared_ptr<build> b (build_db_->find<build> (task_build->id));
@@ -2395,8 +2499,16 @@ handle (request& rq, response& rs)
tenant_service& ss (tss->first);
const build& b (*tss->second);
+ // Release the database connection since the build_built()
+ // notification can potentially be time-consuming (e.g., it may
+ // perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsb->build_built (ss, b, log_writer_))
{
+ conn = build_db_->connection ();
+
if (optional<string> data =
update_tenant_service_state (conn, b.tenant, f))
ss.data = move (data);
@@ -2407,6 +2519,10 @@ handle (request& rq, response& rs)
// Send notification emails for all the aborted builds.
//
for (const aborted_build& ab: aborted_builds)
+ {
+ if (conn == nullptr)
+ conn = build_db_->connection ();
+
send_notification_email (*options_,
conn,
*ab.b,
@@ -2415,9 +2531,14 @@ handle (request& rq, response& rs)
ab.what,
error,
verb_ >= 2 ? &trace : nullptr);
+ }
}
}
+ // Release the database connection as soon as possible.
+ //
+ conn.reset ();
+
serialize_task_response_manifest ();
return true;
}
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
index 5974d45..18e38fc 100644
--- a/mod/mod-ci.cxx
+++ b/mod/mod-ci.cxx
@@ -36,7 +36,12 @@ ci (const ci& r, tenant_service_map& tsm)
#else
ci (const ci& r)
#endif
- : handler (r),
+ :
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
+ handler (r),
+ #else
+ database_module (r),
+#endif
ci_start (r),
options_ (r.initialized_ ? r.options_ : nullptr),
form_ (r.initialized_ || r.form_ == nullptr
@@ -100,6 +105,13 @@ init (scanner& s)
}
}
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+ if (!options_->build_config_specified ())
+ fail << "package building functionality must be enabled";
+
+ database_module::init (*options_, options_->build_db_retry ());
+#endif
+
if (options_->root ().empty ())
options_->root (dir_path ("/"));
}
@@ -347,6 +359,7 @@ handle (request& rq, response& rs)
user_agent = h.value;
}
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
optional<start_result> r (start (error,
warn,
verb_ ? &trace : nullptr,
@@ -367,6 +380,23 @@ handle (request& rq, response& rs)
: optional<string> ()),
custom_request,
overrides));
+#else
+ assert (build_db_ != nullptr); // Wouldn't be here otherwise.
+
+ optional<start_result> r;
+
+ if (optional<string> ref = create (error,
+ warn,
+ verb_ ? &trace : nullptr,
+ *build_db_,
+ tenant_service ("", "ci", rl.string ())))
+ {
+ string msg ("unloaded CI request is created: " +
+ options_->host () + tenant_dir (root, *ref).string ());
+
+ r = start_result {200, move (msg), move (*ref), {}};
+ }
+#endif
if (!r)
return respond_error (); // The diagnostics is already issued.
@@ -472,4 +502,35 @@ build_built (const tenant_service&,
return ts.data ? *ts.data + ", " + s : s;
};
}
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_unloaded (tenant_service&& ts,
+ const diag_epilogue& log_writer) const noexcept
+{
+ NOTIFICATION_DIAG (log_writer);
+
+ assert (ts.data); // Repository location.
+
+ try
+ {
+ repository_location rl (*ts.data);
+
+ if (!load (error, warn, verb_ ? &trace : nullptr,
+ *build_db_,
+ move (ts),
+ rl))
+ return nullptr; // The diagnostics is already issued.
+ }
+ catch (const invalid_argument& e)
+ {
+ error << "invalid repository location '" << *ts.data << "' stored for "
+ << "tenant service " << ts.id << ' ' << ts.type;
+
+ return nullptr;
+ }
+
+ return [] (const tenant_service& ts) {return "loaded " + *ts.data;};
+}
+#endif
#endif
diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx
index 1e2ee15..a83b9d3 100644
--- a/mod/mod-ci.hxx
+++ b/mod/mod-ci.hxx
@@ -17,18 +17,34 @@
#include <mod/ci-common.hxx>
+#if defined(BREP_CI_TENANT_SERVICE_UNLOADED) && !defined(BREP_CI_TENANT_SERVICE)
+# error BREP_CI_TENANT_SERVICE must be defined if BREP_CI_TENANT_SERVICE_UNLOADED is defined
+#endif
+
#ifdef BREP_CI_TENANT_SERVICE
# include <mod/tenant-service.hxx>
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+# include <mod/database-module.hxx>
+#endif
#endif
namespace brep
{
- class ci: public handler,
+ class ci:
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
+ public handler,
+#else
+ public database_module,
+#endif
private ci_start
#ifdef BREP_CI_TENANT_SERVICE
, public tenant_service_build_queued,
public tenant_service_build_building,
public tenant_service_build_built
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+ , tenant_service_build_unloaded
+#endif
#endif
{
public:
@@ -74,6 +90,12 @@ namespace brep
build_built (const tenant_service&,
const build&,
const diag_epilogue& log_writer) const noexcept override;
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+ virtual function<optional<string> (const tenant_service&)>
+ build_unloaded (tenant_service&&,
+ const diag_epilogue& log_writer) const noexcept override;
+#endif
#endif
private:
diff --git a/mod/module.cli b/mod/module.cli
index a107ffe..5f63930 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -800,7 +800,7 @@ namespace brep
{
};
- class ci: ci_start, page, repository_url, handler
+ class ci: ci_start, build, build_db, page, repository_url, handler
{
// Classic CI-specific options.
//
@@ -815,7 +815,7 @@ namespace brep
}
};
- class ci_github: ci_start, ci_cancel, build_db, handler
+ class ci_github: ci_start, ci_cancel, build, build_db, handler
{
// GitHub CI-specific options (e.g., request timeout when invoking
// GitHub APIs).
diff --git a/mod/page.cxx b/mod/page.cxx
index bc2e42d..177fb64 100644
--- a/mod/page.cxx
+++ b/mod/page.cxx
@@ -739,7 +739,7 @@ namespace brep
<< ~TR;
}
- // BUILD_RESULT
+ // TR_BUILD_RESULT
//
void TR_BUILD_RESULT::
operator() (serializer& s) const
diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx
index 9205f76..b7f5c02 100644
--- a/mod/tenant-service.hxx
+++ b/mod/tenant-service.hxx
@@ -21,7 +21,8 @@ namespace brep
virtual ~tenant_service_base () = default;
};
- // Possible build notifications:
+ // Possible build notifications (see also the unloaded special notification
+ // below):
//
// queued
// building
@@ -121,6 +122,22 @@ namespace brep
const diag_epilogue& log_writer) const noexcept = 0;
};
+ // This notification is only made on unloaded CI requests created with the
+ // ci_start::create() call and until they are loaded with ci_start::load()
+ // or, alternatively, abandoned with ci_start::abandon().
+ //
+ // Note: make sure the implementation of this notification does not take
+ // too long (currently 40 seconds) to avoid nested notifications. Note
+ // also that the first notification is delayed (currently 10 seconds).
+ //
+ class tenant_service_build_unloaded: public virtual tenant_service_base
+ {
+ public:
+ virtual function<optional<string> (const tenant_service&)>
+ build_unloaded (tenant_service&&,
+ const diag_epilogue& log_writer) const noexcept = 0;
+ };
+
// Map of service type (tenant_service::type) to service.
//
using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>;