aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--brep/handler/ci/ci-load.in27
-rw-r--r--doc/manual.cli11
-rw-r--r--etc/brep-module.conf8
-rw-r--r--etc/private/install/brep-module.conf8
-rw-r--r--libbrep/build-extra.sql6
-rw-r--r--libbrep/build-package.hxx2
-rw-r--r--libbrep/build.cxx94
-rw-r--r--libbrep/build.hxx43
-rw-r--r--libbrep/build.xml2
-rw-r--r--libbrep/common.hxx30
-rw-r--r--libbrep/package.cxx8
-rw-r--r--libbrep/package.hxx43
-rw-r--r--libbrep/package.xml16
-rw-r--r--load/load.cli22
-rw-r--r--load/load.cxx55
-rw-r--r--mod/buildfile5
-rw-r--r--mod/ci-common.cxx494
-rw-r--r--mod/ci-common.hxx96
-rw-r--r--mod/database-module.cxx56
-rw-r--r--mod/database-module.hxx23
-rw-r--r--mod/external-handler.cxx3
-rw-r--r--mod/external-handler.hxx2
-rw-r--r--mod/mod-build-force.cxx78
-rw-r--r--mod/mod-build-force.hxx8
-rw-r--r--mod/mod-build-log.cxx2
-rw-r--r--mod/mod-build-result.cxx159
-rw-r--r--mod/mod-build-result.hxx8
-rw-r--r--mod/mod-build-task.cxx395
-rw-r--r--mod/mod-build-task.hxx8
-rw-r--r--mod/mod-builds.cxx10
-rw-r--r--mod/mod-ci.cxx585
-rw-r--r--mod/mod-ci.hxx49
-rw-r--r--mod/mod-package-version-details.cxx2
-rw-r--r--mod/mod-repository-root.cxx35
-rw-r--r--mod/mod-repository-root.hxx3
-rw-r--r--mod/module.cli57
-rw-r--r--mod/page.cxx1
-rw-r--r--mod/page.hxx7
-rw-r--r--mod/tenant-service.hxx107
-rw-r--r--monitor/monitor.cxx8
-rw-r--r--web/server/module.hxx3
41 files changed, 2041 insertions, 538 deletions
diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in
index dbdc450..3f04ea8 100644
--- a/brep/handler/ci/ci-load.in
+++ b/brep/handler/ci/ci-load.in
@@ -108,6 +108,13 @@ declare -A packages
#
spec=
+# Third party service information which, if specified, needs to be associated
+# with the being created tenant.
+#
+service_id=
+service_type=
+service_data=
+
while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
case "$n" in
repository) repository="$v" ;;
@@ -122,6 +129,10 @@ while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
fi
spec="$spec$v"
;;
+
+ service-id) service_id="$v" ;;
+ service-type) service_type="$v" ;;
+ service-data) service_data="$v" ;;
esac
done
@@ -141,6 +152,12 @@ if [[ -n "$simulate" && "$simulate" != "success" ]]; then
exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
fi
+# Use the generated reference if the tenant service id is not specified.
+#
+if [[ -n "$service_type" && -z "$service_id" ]]; then
+ service_id="$reference"
+fi
+
message_suffix=
if [[ -n "$result_url" ]]; then
message_suffix=": $result_url/@$reference" # Append the tenant id.
@@ -306,6 +323,16 @@ if [[ -n "$interactive" ]]; then
loader_options+=(--interactive "$interactive")
fi
+# Pass the tenant service information, if specified, to the loader.
+#
+if [[ -n "$service_id" ]]; then
+ loader_options+=(--service-id "$service_id" --service-type "$service_type")
+
+ if [[ -n "$service_data" ]]; then
+ loader_options+=(--service-data "$service_data")
+ fi
+fi
+
run "$loader" "${loader_options[@]}" "$loadtab"
# Remove the no longer needed CI request data directory.
diff --git a/doc/manual.cli b/doc/manual.cli
index 6dab404..2b96393 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -341,12 +341,23 @@ repository: <url>
timestamp: <date-time>
[client-ip]: <string>
[user-agent]: <string>
+[service-id]: <string>
+[service-type]: <string>
+[service-data]: <string>
\
The \c{package} value can be repeated multiple times. The \c{timestamp} value
is in the ISO-8601 \c{<YYYY>-<MM>-<DD>T<hh>:<mm>:<ss>Z} form (always
UTC). Note also that \c{client-ip} can be IPv4 or IPv6.
+Note that some CI service implementations may serve as backends for
+third-party services. The latter may initiate CI tasks, providing all the
+required information via some custom protocol, and expect the CI service to
+notify it about the progress. In this case the third-party service type as
+well as optionally the third-party id and custom state data can be
+communicated to the underlying CI handler program via the respective
+\c{service-*} manifest values.
+
\h#ci-overrides-manifest|CI Overrides Manifest|
diff --git a/etc/brep-module.conf b/etc/brep-module.conf
index 606c2d5..093d1f8 100644
--- a/etc/brep-module.conf
+++ b/etc/brep-module.conf
@@ -201,6 +201,14 @@ menu About=?about
# build-alt-hard-rebuild-stop
+# Time to wait before assuming the 'queued' notifications are delivered for
+# package CI requests submitted via third-party services (GitHub, etc). During
+# this time a package is not considered for a build. Must be specified in
+# seconds. Default is 30 seconds.
+#
+# build-queued-timeout 30
+
+
# The maximum size of the build task request manifest accepted. Note that the
# HTTP POST request body is cached to retry database transactions in the face
# of recoverable failures (deadlock, loss of connection, etc). Default is
diff --git a/etc/private/install/brep-module.conf b/etc/private/install/brep-module.conf
index df439f4..4e5eb87 100644
--- a/etc/private/install/brep-module.conf
+++ b/etc/private/install/brep-module.conf
@@ -201,6 +201,14 @@ menu About=?about
# build-alt-hard-rebuild-stop
+# Time to wait before assuming the 'queued' notifications are delivered for
+# package CI requests submitted via third-party services (GitHub, etc). During
+# this time a package is not considered for a build. Must be specified in
+# seconds. Default is 30 seconds.
+#
+# build-queued-timeout 30
+
+
# The maximum size of the build task request manifest accepted. Note that the
# HTTP POST request body is cached to retry database transactions in the face
# of recoverable failures (deadlock, loss of connection, etc). Default is
diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql
index 23015f4..c15ddc1 100644
--- a/libbrep/build-extra.sql
+++ b/libbrep/build-extra.sql
@@ -36,7 +36,11 @@ CREATE FOREIGN TABLE build_tenant (
id TEXT NOT NULL,
private BOOLEAN NOT NULL,
interactive TEXT NULL,
- archived BOOLEAN NOT NULL)
+ archived BOOLEAN NOT NULL,
+ service_id TEXT NULL,
+ service_type TEXT NULL,
+ service_data TEXT NULL,
+ queued_timestamp BIGINT NULL)
SERVER package_server OPTIONS (table_name 'tenant');
-- The foreign table for build_repository object.
diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx
index 6d7fb14..a0e1082 100644
--- a/libbrep/build-package.hxx
+++ b/libbrep/build-package.hxx
@@ -35,6 +35,8 @@ namespace brep
bool private_;
optional<string> interactive;
bool archived;
+ optional<tenant_service> service;
+ optional<timestamp> queued_timestamp;
// Database mapping.
//
diff --git a/libbrep/build.cxx b/libbrep/build.cxx
index c8a2cd1..8fadfa3 100644
--- a/libbrep/build.cxx
+++ b/libbrep/build.cxx
@@ -12,6 +12,7 @@ namespace brep
{
switch (s)
{
+ case build_state::queued: return "queued";
case build_state::building: return "building";
case build_state::built: return "built";
}
@@ -22,7 +23,8 @@ namespace brep
build_state
to_build_state (const string& s)
{
- if (s == "building") return build_state::building;
+ if (s == "queued") return build_state::queued;
+ else if (s == "building") return build_state::building;
else if (s == "built") return build_state::built;
else throw invalid_argument ("invalid build state '" + s + '\'');
}
@@ -91,6 +93,96 @@ namespace brep
{
}
+ build::
+ build (string tnt,
+ package_name_type pnm,
+ version pvr,
+ target_triplet trg,
+ string tcf,
+ string pcf,
+ string tnm, version tvr)
+ : id (package_id (move (tnt), move (pnm), pvr),
+ move (trg),
+ move (tcf),
+ move (pcf),
+ move (tnm), tvr),
+ tenant (id.package.tenant),
+ package_name (id.package.name),
+ package_version (move (pvr)),
+ target (id.target),
+ target_config_name (id.target_config_name),
+ package_config_name (id.package_config_name),
+ toolchain_name (id.toolchain_name),
+ toolchain_version (move (tvr)),
+ state (build_state::queued),
+ timestamp (timestamp_type::clock::now ()),
+ force (force_state::unforced)
+ {
+ }
+
+ build::
+ build (build&& b)
+ : id (move (b.id)),
+ tenant (id.package.tenant),
+ package_name (id.package.name),
+ package_version (move (b.package_version)),
+ target (id.target),
+ target_config_name (id.target_config_name),
+ package_config_name (id.package_config_name),
+ toolchain_name (id.toolchain_name),
+ toolchain_version (move (b.toolchain_version)),
+ state (b.state),
+ interactive (move (b.interactive)),
+ timestamp (b.timestamp),
+ force (b.force),
+ status (b.status),
+ soft_timestamp (b.soft_timestamp),
+ hard_timestamp (b.hard_timestamp),
+ agent_fingerprint (move (b.agent_fingerprint)),
+ agent_challenge (move (b.agent_challenge)),
+ machine (move (b.machine)),
+ machine_summary (move (b.machine_summary)),
+ results (move (b.results)),
+ results_section (move (b.results_section)),
+ controller_checksum (move (b.controller_checksum)),
+ machine_checksum (move (b.machine_checksum)),
+ agent_checksum (move (b.agent_checksum)),
+ worker_checksum (move (b.worker_checksum)),
+ dependency_checksum (move (b.dependency_checksum))
+ {
+ }
+
+ build& build::
+ operator= (build&& b)
+ {
+ if (this != &b)
+ {
+ id = move (b.id);
+ package_version = move (b.package_version);
+ toolchain_version = move (b.toolchain_version);
+ state = b.state;
+ interactive = move (b.interactive);
+ timestamp = b.timestamp;
+ force = b.force;
+ status = b.status;
+ soft_timestamp = b.soft_timestamp;
+ hard_timestamp = b.hard_timestamp;
+ agent_fingerprint = move (b.agent_fingerprint);
+ agent_challenge = move (b.agent_challenge);
+ machine = move (b.machine);
+ machine_summary = move (b.machine_summary);
+ results = move (b.results);
+ results_section = move (b.results_section);
+ controller_checksum = move (b.controller_checksum);
+ machine_checksum = move (b.machine_checksum);
+ agent_checksum = move (b.agent_checksum);
+ worker_checksum = move (b.worker_checksum);
+ dependency_checksum = move (b.dependency_checksum);
+ }
+
+ return *this;
+ }
+
// build_delay
//
build_delay::
diff --git a/libbrep/build.hxx b/libbrep/build.hxx
index adad535..236f73c 100644
--- a/libbrep/build.hxx
+++ b/libbrep/build.hxx
@@ -28,7 +28,7 @@
//
#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20
-#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 23, closed)
+#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 24, closed)
// We have to keep these mappings at the global scope instead of inside the
// brep namespace because they need to be also effective in the bbot namespace
@@ -129,8 +129,14 @@ namespace brep
// build_state
//
+ // The queued build state is semantically equivalent to a non-existent
+ // build. It is only used for those tenants, which have a third-party
+ // service associated that requires the `queued` notifications (see
+ // mod/tenant-service.hxx for background).
+ //
enum class build_state: std::uint8_t
{
+ queued,
building,
built
};
@@ -212,6 +218,23 @@ namespace brep
string controller_checksum,
string machine_checksum);
+ // Create the build object with the queued state.
+ //
+ build (string tenant,
+ package_name_type, version,
+ target_triplet,
+ string target_config_name,
+ string package_config_name,
+ string toolchain_name, version toolchain_version);
+
+ // Move-only type.
+ //
+ build (build&&);
+ build& operator= (build&&);
+
+ build (const build&) = delete;
+ build& operator= (const build&) = delete;
+
build_id id;
string& tenant; // Tracks id.package.tenant.
@@ -315,9 +338,6 @@ namespace brep
#pragma db member(results_section) load(lazy) update(always)
- build (const build&) = delete;
- build& operator= (const build&) = delete;
-
private:
friend class odb::access;
@@ -377,7 +397,7 @@ namespace brep
canonical_version version_;
};
- // Build of an existing buildable package.
+ // Builds of existing buildable packages.
//
#pragma db view \
object(build) \
@@ -408,6 +428,19 @@ namespace brep
#pragma db member(result) column("count(" + build::id.package.name + ")")
};
+ // Ids of existing buildable package builds.
+ //
+ #pragma db view object(build) \
+ object(build_package inner: \
+ brep::operator== (build::id.package, build_package::id) && \
+ build_package::buildable)
+ struct package_build_id
+ {
+ build_id id;
+
+ operator build_id& () {return id;}
+ };
+
// Used to track the package build delays since the last build or, if not
// present, since the first opportunity to build the package.
//
diff --git a/libbrep/build.xml b/libbrep/build.xml
index 0a25488..e757aba 100644
--- a/libbrep/build.xml
+++ b/libbrep/build.xml
@@ -1,4 +1,6 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1">
+ <changeset version="24"/>
+
<changeset version="23"/>
<changeset version="22"/>
diff --git a/libbrep/common.hxx b/libbrep/common.hxx
index fec22e8..11aae67 100644
--- a/libbrep/common.hxx
+++ b/libbrep/common.hxx
@@ -127,6 +127,20 @@ namespace brep
std::chrono::duration_cast<brep::timestamp::duration> ( \
std::chrono::nanoseconds (?))))
+ using optional_timestamp = optional<timestamp>;
+ using optional_uint64 = optional<uint64_t>;
+
+ #pragma db map type(optional_timestamp) as(brep::optional_uint64) \
+ to((?) \
+ ? std::chrono::duration_cast<std::chrono::nanoseconds> ( \
+ (?)->time_since_epoch ()).count () \
+ : brep::optional_uint64 ()) \
+ from((?) \
+ ? brep::timestamp ( \
+ std::chrono::duration_cast<brep::timestamp::duration> ( \
+ std::chrono::nanoseconds (*(?)))) \
+ : brep::optional_timestamp ())
+
// version
//
using bpkg::version;
@@ -517,6 +531,22 @@ namespace brep
#pragma db member(requirement_key::middle) column("alternative_index")
#pragma db member(requirement_key::inner) column("index")
+ // Third-party service state which may optionally be associated with a
+ // tenant (see also mod/tenant-service.hxx for background).
+ //
+ #pragma db value
+ struct tenant_service
+ {
+ string id;
+ string type;
+ optional<string> data;
+
+ tenant_service () = default;
+
+ tenant_service (string i, string t, optional<string> d = nullopt)
+ : id (move (i)), type (move (t)), data (move (d)) {}
+ };
+
// Version comparison operators.
//
// They allow comparing objects that have epoch, canonical_upstream,
diff --git a/libbrep/package.cxx b/libbrep/package.cxx
index e5e7767..2320547 100644
--- a/libbrep/package.cxx
+++ b/libbrep/package.cxx
@@ -40,11 +40,15 @@ namespace brep
// tenant
//
tenant::
- tenant (string i, bool p, optional<string> r)
+ tenant (string i,
+ bool p,
+ optional<string> r,
+ optional<tenant_service> s)
: id (move (i)),
private_ (p),
interactive (move (r)),
- creation_timestamp (timestamp::clock::now ())
+ creation_timestamp (timestamp::clock::now ()),
+ service (move (s))
{
}
diff --git a/libbrep/package.hxx b/libbrep/package.hxx
index 06e6335..9bb9af9 100644
--- a/libbrep/package.hxx
+++ b/libbrep/package.hxx
@@ -20,7 +20,7 @@
//
#define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27
-#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 29, closed)
+#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 30, closed)
namespace brep
{
@@ -241,31 +241,62 @@ namespace brep
// Create the tenant object with the timestamp set to now and the archived
// flag set to false.
//
- explicit
- tenant (string id, bool private_, optional<string> interactive);
+ tenant (string id,
+ bool private_,
+ optional<string> interactive,
+ optional<tenant_service>);
string id;
// If true, display the packages in the web interface only in the tenant
// view mode.
//
- bool private_; // Note: foreign-mapped in build.
+ bool private_; // Note: foreign-mapped in build.
// Interactive package build breakpoint.
//
// If present, then packages from this tenant will only be built
// interactively and only non-interactively otherwise.
//
- optional<string> interactive; // Note: foreign-mapped in build.
+ optional<string> interactive; // Note: foreign-mapped in build.
timestamp creation_timestamp;
- bool archived = false; // Note: foreign-mapped in build.
+ bool archived = false; // Note: foreign-mapped in build.
+
+ optional<tenant_service> service; // Note: foreign-mapped in build.
+
+ // Note that due to the implementation complexity and performance
+ // considerations, the service notifications are not synchronized. This
+ // leads to a potential race, so that before we have sent the `queued`
+ // notification for a package build, some other thread (potentially in a
+ // different process) could have already sent the `building` notification
+ // for it. It feels like there is no easy way to reliably fix that.
+ // Instead, we just decrease the probability of such a notifications
+ // sequence failure by delaying builds of the freshly queued packages for
+ // some time. Specifically, whenever the `queued` notification is ought
+ // to be sent (normally out of the database transaction, since it likely
+ // sends an HTTP request, etc) the tenant's queued_timestamp member is set
+ // to the current time. During the configured time interval since that
+ // time point the build tasks may not be issued for the tenant's packages.
+ //
+ // Also note that while there are similar potential races for other
+ // notification sequences, their probability is rather low due to the
+ // natural reasons (non-zero build task execution time, etc) and thus we
+ // just ignore them.
+ //
+ optional<timestamp> queued_timestamp; // Note: foreign-mapped in build.
// Database mapping.
//
#pragma db member(id) id
#pragma db member(private_)
+ #pragma db index("tenant_service_i") \
+ unique \
+ members(service.id, service.type)
+
+ #pragma db index member(service.id)
+
private:
friend class odb::access;
tenant () = default;
diff --git a/libbrep/package.xml b/libbrep/package.xml
index bf7915e..6852f75 100644
--- a/libbrep/package.xml
+++ b/libbrep/package.xml
@@ -1,4 +1,20 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1">
+ <changeset version="30">
+ <alter-table name="tenant">
+ <add-column name="service_id" type="TEXT" null="true"/>
+ <add-column name="service_type" type="TEXT" null="true"/>
+ <add-column name="service_data" type="TEXT" null="true"/>
+ <add-column name="queued_timestamp" type="BIGINT" null="true"/>
+ <add-index name="tenant_service_i" type="UNIQUE">
+ <column name="service_id"/>
+ <column name="service_type"/>
+ </add-index>
+ <add-index name="tenant_service_id_i">
+ <column name="service_id"/>
+ </add-index>
+ </alter-table>
+ </changeset>
+
<changeset version="29">
<alter-table name="package_tests">
<add-column name="test_enable" type="TEXT" null="true"/>
diff --git a/load/load.cli b/load/load.cli
index 16b5f9f..1b7c4c5 100644
--- a/load/load.cli
+++ b/load/load.cli
@@ -77,6 +77,28 @@ class options
breakpoint. Implies \cb{--private}."
};
+ std::string --service-id
+ {
+ "<id>",
+ "Third party service information to associate with the being created
+ tenant. Requires the \cb{--tenant} and \cb{--service-type} options to be
+ specified."
+ };
+
+ std::string --service-type
+ {
+ "<type>",
+ "Type of the service to associate with the being created tenant. Requires
+ the \cb{--service-id} option to be specified."
+ };
+
+ std::string --service-data
+ {
+ "<data>",
+ "Service data to associate with the being created tenant. Requires the
+ \cb{--service-id} option to be specified."
+ };
+
brep::path --overrides-file
{
"<file>",
diff --git a/load/load.cxx b/load/load.cxx
index b644a3a..56e4e19 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -1492,6 +1492,40 @@ try
throw failed ();
}
+ // Verify the --service-* options.
+ //
+ if (ops.service_id_specified ())
+ {
+ if (!ops.tenant_specified ())
+ {
+ cerr << "error: --service-id requires --tenant" << endl;
+ throw failed ();
+ }
+
+ if (ops.service_type ().empty ())
+ {
+ cerr << "error: --service-id requires --service-type"
+ << endl;
+ throw failed ();
+ }
+ }
+ else
+ {
+ if (ops.service_type_specified ())
+ {
+ cerr << "error: --service-type requires --service-id"
+ << endl;
+ throw failed ();
+ }
+
+ if (ops.service_data_specified ())
+ {
+ cerr << "error: --service-data requires --service-id"
+ << endl;
+ throw failed ();
+ }
+ }
+
// Parse and validate overrides, if specified.
//
// Note that here we make sure that the overrides manifest is valid.
@@ -1591,11 +1625,30 @@ try
// Persist the tenant.
//
+ // Note that if the tenant service is specified and some tenant with the
+ // same service id and type is already persisted, then we will end up with
+ // the `object already persistent` error and terminate with the exit code
+ // 1 (fatal error). We could potentially dedicate a special exit code for
+ // such a case, so that the caller may recognize it and behave accordingly
+ // (CI request handler can treat it as a client error rather than an
+ // internal error, etc). However, let's first see if it ever becomes a
+ // problem.
+ //
+ optional<tenant_service> service;
+
+ if (ops.service_id_specified ())
+ service = tenant_service (ops.service_id (),
+ ops.service_type (),
+ (ops.service_data_specified ()
+ ? ops.service_data ()
+ : optional<string> ()));
+
db.persist (tenant (tnt,
ops.private_ (),
(ops.interactive_specified ()
? ops.interactive ()
- : optional<string> ())));
+ : optional<string> ()),
+ move (service)));
// On the first pass over the internal repositories we load their
// certificate information and packages.
diff --git a/mod/buildfile b/mod/buildfile
index 6693a35..c3895dc 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -35,6 +35,11 @@ mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}} \
{hxx ixx txx cxx}{+{$libu_src} } \
$libs
+# Add support for tenant-associated service notifications to the CI module for
+# the debugging of the notifications machinery.
+#
+cxx.poptions += -DBREP_CI_TENANT_SERVICE
+
libus{mod}: ../web/xhtml/libus{xhtml}
libue{mod}: ../web/xhtml/libue{xhtml}
diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx
new file mode 100644
index 0000000..cb61e66
--- /dev/null
+++ b/mod/ci-common.cxx
@@ -0,0 +1,494 @@
+// file : mod/ci-common.cxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/ci-common.hxx>
+
+#include <libbutl/uuid.hxx>
+#include <libbutl/fdstream.hxx>
+#include <libbutl/sendmail.hxx>
+#include <libbutl/timestamp.hxx>
+#include <libbutl/filesystem.hxx>
+#include <libbutl/process-io.hxx> // operator<<(ostream, process_args)
+#include <libbutl/manifest-serializer.hxx>
+
+#include <mod/external-handler.hxx>
+
+namespace brep
+{
+ using namespace std;
+ using namespace butl;
+
+ void ci_start::
+ init (shared_ptr<options::ci_start> o)
+ {
+ // Verify the data directory satisfies the requirements.
+ //
+ const dir_path& d (o->ci_data ());
+
+ if (d.relative ())
+ throw runtime_error ("ci-data directory path must be absolute");
+
+ if (!dir_exists (d))
+ throw runtime_error ("ci-data directory '" + d.string () +
+ "' does not exist");
+
+ if (o->ci_handler_specified () && o->ci_handler ().relative ())
+ throw runtime_error ("ci-handler path must be absolute");
+
+ options_ = move (o);
+ }
+
+ optional<ci_start::start_result> ci_start::
+ start (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ optional<tenant_service>&& service,
+ const repository_location& repository,
+ const vector<package>& packages,
+ const optional<string>& client_ip,
+ const optional<string>& user_agent,
+ const optional<string>& interactive,
+ const optional<string>& simulate,
+ const vector<pair<string, string>>& custom_request,
+ const vector<pair<string, string>>& overrides)
+ {
+ using serializer = manifest_serializer;
+ using serialization = manifest_serialization;
+
+ assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+ // If the tenant service is specified, then its type may not be empty.
+ //
+ assert (!service || !service->type.empty ());
+
+ // Generate the request id.
+ //
+ // Note that it will also be used as a CI result manifest reference,
+ // unless the latter is provided by the external handler.
+ //
+ string request_id;
+
+ try
+ {
+ request_id = uuid::generate ().string ();
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to generate request id: " << e;
+ return nullopt;
+ }
+
+ // Create the submission data directory.
+ //
+ dir_path dd (options_->ci_data () / dir_path (request_id));
+
+ try
+ {
+ // It's highly unlikely but still possible that the directory already
+ // exists. This can only happen if the generated uuid is not unique.
+ //
+ if (try_mkdir (dd) == mkdir_status::already_exists)
+ throw_generic_error (EEXIST);
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to create directory '" << dd << "': " << e;
+ return nullopt;
+ }
+
+ auto_rmdir ddr (dd);
+
+ // Return the start_result object for the client errors (normally the bad
+ // request status code (400) for the client data serialization errors).
+ //
+ auto client_error = [&request_id] (uint16_t status, string message)
+ {
+ return start_result {status,
+ move (message),
+ request_id,
+ vector<pair<string, string>> ()};
+ };
+
+ // Serialize the CI request manifest to a stream. On the serialization
+ // error return false together with the start_result object containing the
+ // bad request (400) code and the error message. On the stream error pass
+ // through the io_error exception. Otherwise return true.
+ //
+ timestamp ts (system_clock::now ());
+
+ auto rqm = [&request_id,
+ &ts,
+ &service,
+ &repository,
+ &packages,
+ &client_ip,
+ &user_agent,
+ &interactive,
+ &simulate,
+ &custom_request,
+ &client_error] (ostream& os, bool long_lines = false)
+ -> pair<bool, optional<start_result>>
+ {
+ try
+ {
+ serializer s (os, "request", long_lines);
+
+ // Serialize the submission manifest header.
+ //
+ s.next ("", "1"); // Start of manifest.
+ s.next ("id", request_id);
+ s.next ("repository", repository.string ());
+
+ for (const package& p: packages)
+ {
+ if (!p.version)
+ s.next ("package", p.name.string ());
+ else
+ s.next ("package",
+ p.name.string () + '/' + p.version->string ());
+ }
+
+ if (interactive)
+ s.next ("interactive", *interactive);
+
+ if (simulate)
+ s.next ("simulate", *simulate);
+
+ s.next ("timestamp",
+ butl::to_string (ts,
+ "%Y-%m-%dT%H:%M:%SZ",
+ false /* special */,
+ false /* local */));
+
+ if (client_ip)
+ s.next ("client-ip", *client_ip);
+
+ if (user_agent)
+ s.next ("user-agent", *user_agent);
+
+ if (service)
+ {
+ // Note that if the service id is not specified, then the handler
+ // will use the generated reference instead.
+ //
+ if (!service->id.empty ())
+ s.next ("service-id", service->id);
+
+ s.next ("service-type", service->type);
+
+ if (service->data)
+ s.next ("service-data", *service->data);
+ }
+
+ // Serialize the request custom parameters.
+ //
+ // Note that the serializer constraints the custom parameter names
+ // (can't start with '#', can't contain ':' and the whitespaces,
+ // etc).
+ //
+ for (const pair<string, string>& nv: custom_request)
+ s.next (nv.first, nv.second);
+
+ s.next ("", ""); // End of manifest.
+ return make_pair (true, optional<start_result> ());
+ }
+ catch (const serialization& e)
+ {
+ return make_pair (false,
+ optional<start_result> (
+ client_error (400,
+ string ("invalid parameter: ") +
+ e.what ())));
+ }
+ };
+
+ // Serialize the CI request manifest to the submission directory.
+ //
+ path rqf (dd / "request.manifest");
+
+ try
+ {
+ ofdstream os (rqf);
+ pair<bool, optional<start_result>> r (rqm (os));
+ os.close ();
+
+ if (!r.first)
+ return move (*r.second);
+ }
+ catch (const io_error& e)
+ {
+ error << "unable to write to '" << rqf << "': " << e;
+ return nullopt;
+ }
+
+ // Serialize the CI overrides manifest to a stream. On the serialization
+ // error return false together with the start_result object containing the
+ // bad request (400) code and the error message. On the stream error pass
+ // through the io_error exception. Otherwise return true.
+ //
+ auto ovm = [&overrides, &client_error] (ostream& os,
+ bool long_lines = false)
+ -> pair<bool, optional<start_result>>
+ {
+ try
+ {
+ serializer s (os, "overrides", long_lines);
+
+ s.next ("", "1"); // Start of manifest.
+
+ for (const pair<string, string>& nv: overrides)
+ s.next (nv.first, nv.second);
+
+ s.next ("", ""); // End of manifest.
+ return make_pair (true, optional<start_result> ());
+ }
+ catch (const serialization& e)
+ {
+ return make_pair (false,
+ optional<start_result> (
+ client_error (
+ 400,
+ string ("invalid manifest override: ") +
+ e.what ())));
+ }
+ };
+
+ // Serialize the CI overrides manifest to the submission directory.
+ //
+ path ovf (dd / "overrides.manifest");
+
+ if (!overrides.empty ())
+ try
+ {
+ ofdstream os (ovf);
+ pair<bool, optional<start_result>> r (ovm (os));
+ os.close ();
+
+ if (!r.first)
+ return move (*r.second);
+ }
+ catch (const io_error& e)
+ {
+ error << "unable to write to '" << ovf << "': " << e;
+ return nullopt;
+ }
+
+ // Given that the submission data is now successfully persisted we are no
+ // longer in charge of removing it, except for the cases when the
+ // submission handler terminates with an error (see below for details).
+ //
+ ddr.cancel ();
+
+ // If the handler terminates with non-zero exit status or specifies 5XX
+ // (HTTP server error) submission result manifest status value, then we
+ // stash the submission data directory for troubleshooting. Otherwise, if
+ // it's the 4XX (HTTP client error) status value, then we remove the
+ // directory.
+ //
+ auto stash_submit_dir = [&dd, error] ()
+ {
+ if (dir_exists (dd))
+ try
+ {
+ mvdir (dd, dir_path (dd + ".fail"));
+ }
+ catch (const system_error& e)
+ {
+ // Not much we can do here. Let's just log the issue and bail out
+ // leaving the directory in place.
+ //
+ error << "unable to rename directory '" << dd << "': " << e;
+ }
+ };
+
+ // Run the submission handler, if specified, reading the CI result
+ // manifest from its stdout and parse it into the resulting manifest
+ // object. Otherwise, create implied CI result manifest.
+ //
+ start_result sr;
+
+ if (options_->ci_handler_specified ())
+ {
+ using namespace external_handler;
+
+ optional<result_manifest> r (run (options_->ci_handler (),
+ options_->ci_handler_argument (),
+ dd,
+ options_->ci_handler_timeout (),
+ error,
+ warn,
+ trace));
+ if (!r)
+ {
+ stash_submit_dir ();
+ return nullopt; // The diagnostics is already issued.
+ }
+
+ sr.status = r->status;
+
+ for (manifest_name_value& nv: r->values)
+ {
+ string& n (nv.name);
+ string& v (nv.value);
+
+ if (n == "message")
+ sr.message = move (v);
+ else if (n == "reference")
+ sr.reference = move (v);
+ else if (n != "status")
+ sr.custom_result.emplace_back (move (n), move (v));
+ }
+
+ if (sr.reference.empty ())
+ sr.reference = move (request_id);
+ }
+ else // Create the implied CI result manifest.
+ {
+ sr.status = 200;
+ sr.message = "CI request is queued";
+ sr.reference = move (request_id);
+ }
+
+ // Serialize the CI result manifest manifest to a stream. On the
+ // serialization error log the error description and return false, on the
+ // stream error pass through the io_error exception, otherwise return
+ // true.
+ //
+ auto rsm = [&sr, &error] (ostream& os, bool long_lines = false) -> bool
+ {
+ try
+ {
+ serialize_manifest (sr, os, long_lines);
+ return true;
+ }
+ catch (const serialization& e)
+ {
+ error << "ref " << sr.reference << ": unable to serialize handler's "
+ << "output: " << e;
+ return false;
+ }
+ };
+
+ // If the submission data directory still exists then perform an
+ // appropriate action on it, depending on the submission result status.
+ // Note that the handler could move or remove the directory.
+ //
+ if (dir_exists (dd))
+ {
+ // Remove the directory if the client error is detected.
+ //
+ if (sr.status >= 400 && sr.status < 500)
+ {
+ rmdir_r (dd);
+ }
+ //
+ // Otherwise, save the result manifest, into the directory. Also stash
+ // the directory for troubleshooting in case of the server error.
+ //
+ else
+ {
+ path rsf (dd / "result.manifest");
+
+ try
+ {
+ ofdstream os (rsf);
+
+ // Not being able to stash the result manifest is not a reason to
+ // claim the submission failed. The error is logged nevertheless.
+ //
+ rsm (os);
+
+ os.close ();
+ }
+ catch (const io_error& e)
+ {
+ // Not fatal (see above).
+ //
+ error << "unable to write to '" << rsf << "': " << e;
+ }
+
+ if (sr.status >= 500 && sr.status < 600)
+ stash_submit_dir ();
+ }
+ }
+
+ // Send email, if configured, and the CI request submission is not
+ // simulated. Use the long lines manifest serialization mode for the
+ // convenience of copying/clicking URLs they contain.
+ //
+ // Note that we don't consider the email sending failure to be a
+ // submission failure as the submission data is successfully persisted and
+ // the handler is successfully executed, if configured. One can argue that
+ // email can be essential for the submission processing and missing it
+ // would result in the incomplete submission. In this case it's natural to
+ // assume that the web server error log is monitored and the email sending
+ // failure will be noticed.
+ //
+ if (options_->ci_email_specified () && !simulate)
+ try
+ {
+ // Redirect the diagnostics to the web server error log.
+ //
+ sendmail sm ([trace] (const char* args[], size_t n)
+ {
+ if (trace != nullptr)
+ *trace << process_args {args, n};
+ },
+ 2 /* stderr */,
+ options_->email (),
+ "CI request submission (" + sr.reference + ')',
+ {options_->ci_email ()});
+
+ // Write the CI request manifest.
+ //
+ pair<bool, optional<start_result>> r (
+ rqm (sm.out, true /* long_lines */));
+
+ assert (r.first); // The serialization succeeded once, so can't fail now.
+
+ // Write the CI overrides manifest.
+ //
+ sm.out << "\n\n";
+
+ r = ovm (sm.out, true /* long_lines */);
+ assert (r.first); // The serialization succeeded once, so can't fail now.
+
+ // Write the CI result manifest.
+ //
+ sm.out << "\n\n";
+
+ // We don't care about the result (see above).
+ //
+ rsm (sm.out, true /* long_lines */);
+
+ sm.out.close ();
+
+ if (!sm.wait ())
+ error << "sendmail " << *sm.exit;
+ }
+ // Handle process_error and io_error (both derive from system_error).
+ //
+ catch (const system_error& e)
+ {
+ error << "sendmail error: " << e;
+ }
+
+ return optional<start_result> (move (sr));
+ }
+
+ void ci_start::
+ serialize_manifest (const start_result& r, ostream& os, bool long_lines)
+ {
+ manifest_serializer s (os, "result", long_lines);
+
+ s.next ("", "1"); // Start of manifest.
+ s.next ("status", to_string (r.status));
+ s.next ("message", r.message);
+ s.next ("reference", r.reference);
+
+ for (const pair<string, string>& nv: r.custom_result)
+ s.next (nv.first, nv.second);
+
+ s.next ("", ""); // End of manifest.
+ }
+}
diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx
new file mode 100644
index 0000000..6f62c4b
--- /dev/null
+++ b/mod/ci-common.hxx
@@ -0,0 +1,96 @@
+// file : mod/ci-common.hxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_CI_COMMON_HXX
+#define MOD_CI_COMMON_HXX
+
+#include <odb/forward.hxx> // database
+
+#include <libbrep/types.hxx>
+#include <libbrep/utility.hxx>
+
+#include <libbrep/common.hxx>
+
+#include <mod/diagnostics.hxx>
+#include <mod/module-options.hxx>
+
+namespace brep
+{
+ class ci_start
+ {
+ public:
+ void
+ init (shared_ptr<options::ci_start>);
+
+ // If the request handling has been performed normally, then return the
+ // information that corresponds to the CI result manifest (see CI Result
+ // Manifest in the manual). Otherwise (some internal has error occured),
+ // log the error and return nullopt.
+ //
+ // The arguments correspond to the CI request and overrides manifest
+ // values (see CI Request and Overrides Manifests in the manual). Note:
+ // request id and timestamp are generated by the implementation.
+ //
+ struct package
+ {
+ package_name name;
+ optional<brep::version> version;
+ };
+ // Note that the inability to generate the reference is an internal
+ // error. Thus, it is not optional.
+ //
+ struct start_result
+ {
+ uint16_t status;
+ string message;
+ string reference;
+ vector<pair<string, string>> custom_result;
+ };
+
+ // In the optional service information, if id is empty, then the generated
+ // reference is used instead.
+ //
+ optional<start_result>
+ start (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ optional<tenant_service>&&,
+ const repository_location& repository,
+ const vector<package>& packages,
+ const optional<string>& client_ip,
+ const optional<string>& user_agent,
+ const optional<string>& interactive = nullopt,
+ const optional<string>& simulate = nullopt,
+ const vector<pair<string, string>>& custom_request = {},
+ const vector<pair<string, string>>& overrides = {});
+
+ // Helpers.
+ //
+
+ // Serialize the start result as a CI result manifest.
+ //
+ static void
+ serialize_manifest (const start_result&, ostream&, bool long_lines = false);
+
+ private:
+ shared_ptr<options::ci_start> options_;
+ };
+
+ class ci_cancel
+ {
+ public:
+ void
+ init (shared_ptr<options::ci_cancel>, shared_ptr<odb::core::database>);
+
+ // @@ TODO Archive the tenant.
+ //
+ void
+ cancel (/*...*/);
+
+ private:
+ shared_ptr<options::ci_cancel> options_;
+ shared_ptr<odb::core::database> build_db_;
+ };
+}
+
+#endif // MOD_CI_COMMON_HXX
diff --git a/mod/database-module.cxx b/mod/database-module.cxx
index f598bfd..07babc6 100644
--- a/mod/database-module.cxx
+++ b/mod/database-module.cxx
@@ -3,13 +3,20 @@
#include <mod/database-module.hxx>
+#include <odb/database.hxx>
#include <odb/exceptions.hxx>
+#include <odb/transaction.hxx>
+
+#include <libbrep/build-package.hxx>
+#include <libbrep/build-package-odb.hxx>
#include <mod/database.hxx>
#include <mod/module-options.hxx>
namespace brep
{
+ using namespace odb::core;
+
// While currently the user-defined copy constructor is not required (we
// don't need to deep copy nullptr's), it is a good idea to keep the
// placeholder ready for less trivial cases.
@@ -68,4 +75,53 @@ namespace brep
throw;
}
+
+ void database_module::
+ update_tenant_service_state (
+ const connection_ptr& conn,
+ const string& tid,
+ const function<optional<string> (const tenant_service&)>& f)
+ {
+ assert (f != nullptr); // Shouldn't be called otherwise.
+
+ // Must be initialized via the init(options::build_db) function call.
+ //
+ assert (build_db_ != nullptr);
+
+ for (size_t retry (retry_);; )
+ {
+ try
+ {
+ transaction tr (conn->begin ());
+
+ shared_ptr<build_tenant> t (build_db_->find<build_tenant> (tid));
+
+ if (t != nullptr && t->service)
+ {
+ tenant_service& s (*t->service);
+
+ if (optional<string> data = f (s))
+ {
+ s.data = move (*data);
+ build_db_->update (t);
+ }
+ }
+
+ tr.commit ();
+
+ // Bail out if we have successfully updated the service state.
+ //
+ break;
+ }
+ catch (const odb::recoverable& e)
+ {
+ if (retry-- == 0)
+ throw;
+
+ HANDLER_DIAG;
+ l1 ([&]{trace << e << "; " << retry + 1 << " tenant service "
+ << "state update retries left";});
+ }
+ }
+ }
}
diff --git a/mod/database-module.hxx b/mod/database-module.hxx
index f72ba83..910cb35 100644
--- a/mod/database-module.hxx
+++ b/mod/database-module.hxx
@@ -4,7 +4,7 @@
#ifndef MOD_DATABASE_MODULE_HXX
#define MOD_DATABASE_MODULE_HXX
-#include <odb/forward.hxx> // database
+#include <odb/forward.hxx> // odb::core::database, odb::core::connection_ptr
#include <libbrep/types.hxx>
#include <libbrep/utility.hxx>
@@ -14,6 +14,8 @@
namespace brep
{
+ struct tenant_service;
+
// A handler that utilises the database. Specifically, it will retry the
// request in the face of recoverable database failures (deadlock, loss of
// connection, etc) up to a certain number of times.
@@ -50,6 +52,25 @@ namespace brep
virtual bool
handle (request&, response&) = 0;
+ // Helpers.
+ //
+
+ // Update the tenant-associated service state if the specified
+ // notification callback-returned function (expected to be not NULL)
+ // returns the new state data.
+ //
+ // Specifically, start the database transaction, query the service state,
+ // and call the callback-returned function on this state. If this call
+ // returns the data string (rather than nullopt), then update the service
+ // state with this data and persist the change. Repeat all the above steps
+ // on the recoverable database failures (deadlocks, etc).
+ //
+ void
+ update_tenant_service_state (
+ const odb::core::connection_ptr&,
+ const string& tid,
+ const function<optional<string> (const tenant_service&)>&);
+
protected:
size_t retry_ = 0; // Max of all retries.
diff --git a/mod/external-handler.cxx b/mod/external-handler.cxx
index dc4c0fd..3a85bd8 100644
--- a/mod/external-handler.cxx
+++ b/mod/external-handler.cxx
@@ -15,7 +15,8 @@
#include <libbutl/process.hxx>
#include <libbutl/fdstream.hxx>
-#include <libbutl/process-io.hxx> // operator<<(ostream, process_args)
+#include <libbutl/process-io.hxx> // operator<<(ostream, process_args)
+#include <libbutl/manifest-parser.hxx>
using namespace std;
using namespace butl;
diff --git a/mod/external-handler.hxx b/mod/external-handler.hxx
index be16e5b..0276a25 100644
--- a/mod/external-handler.hxx
+++ b/mod/external-handler.hxx
@@ -4,7 +4,7 @@
#ifndef MOD_EXTERNAL_HANDLER_HXX
#define MOD_EXTERNAL_HANDLER_HXX
-#include <libbutl/manifest-parser.hxx>
+#include <libbutl/manifest-types.hxx>
#include <libbrep/types.hxx>
#include <libbrep/utility.hxx>
diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx
index 04e1883..dea89de 100644
--- a/mod/mod-build-force.cxx
+++ b/mod/mod-build-force.cxx
@@ -12,20 +12,28 @@
#include <libbrep/build-odb.hxx>
#include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
using namespace std;
using namespace brep::cli;
using namespace odb::core;
+brep::build_force::
+build_force (const tenant_service_map& tsm)
+ : tenant_service_map_ (tsm)
+{
+}
+
// While currently the user-defined copy constructor is not required (we don't
// need to deep copy nullptr's), it is a good idea to keep the placeholder
// ready for less trivial cases.
//
brep::build_force::
-build_force (const build_force& r)
+build_force (const build_force& r, const tenant_service_map& tsm)
: database_module (r),
build_config_module (r),
- options_ (r.initialized_ ? r.options_ : nullptr)
+ options_ (r.initialized_ ? r.options_ : nullptr),
+ tenant_service_map_ (tsm)
{
}
@@ -173,15 +181,26 @@ handle (request& rq, response& rs)
// Load the package build configuration (if present), set the force flag and
// update the object's persistent state.
//
+ // If the incomplete package build is being forced to rebuild and the
+ // tenant_service_build_queued callback is associated with the package
+ // tenant, then stash the state, the build object, and the callback pointer
+ // for the subsequent service `queued` notification.
+ //
+ const tenant_service_build_queued* tsq (nullptr);
+ optional<pair<tenant_service, shared_ptr<build>>> tss;
+
+ connection_ptr conn (build_db_->connection ());
{
- transaction t (build_db_->begin ());
+ transaction t (conn->begin ());
package_build pb;
+ shared_ptr<build> b;
+
if (!build_db_->query_one<package_build> (
- query<package_build>::build::id == id, pb))
+ query<package_build>::build::id == id, pb) ||
+ (b = move (pb.build))->state == build_state::queued)
config_expired ("no package build");
- shared_ptr<build> b (pb.build);
force_state force (b->state == build_state::built
? force_state::forced
: force_state::forcing);
@@ -211,11 +230,60 @@ handle (request& rq, response& rs)
b->force = force;
build_db_->update (b);
+
+ if (force == force_state::forcing)
+ {
+ shared_ptr<build_tenant> t (build_db_->load<build_tenant> (b->tenant));
+
+ if (t->service)
+ {
+ auto i (tenant_service_map_.find (t->service->type));
+
+ if (i != tenant_service_map_.end ())
+ {
+ tsq = dynamic_cast<const tenant_service_build_queued*> (
+ i->second.get ());
+
+ if (tsq != nullptr)
+ {
+ // If we ought to call the
+ // tenant_service_build_queued::build_queued() callback, then
+ // also set the package tenant's queued timestamp to the current
+ // time to prevent the notifications race (see
+ // tenant::queued_timestamp for details).
+ //
+ t->queued_timestamp = system_clock::now ();
+ build_db_->update (t);
+
+ tss = make_pair (move (*t->service), move (b));
+ }
+ }
+ }
+ }
}
t.commit ();
}
+ // If the incomplete package build is being forced to rebuild and the
+ // tenant-associated third-party service needs to be notified about the
+ // queued builds, then call the tenant_service_build_queued::build_queued()
+ // callback function and update the service state, if requested.
+ //
+ if (tsq != nullptr)
+ {
+ assert (tss); // Wouldn't be here otherwise.
+
+ const tenant_service& ss (tss->first);
+ build& b (*tss->second);
+
+ vector<build> qbs;
+ qbs.push_back (move (b));
+
+ if (auto f = tsq->build_queued (ss, qbs, build_state::building))
+ update_tenant_service_state (conn, qbs.back ().tenant, f);
+ }
+
// We have all the data, so don't buffer the response content.
//
ostream& os (rs.content (200, "text/plain;charset=utf-8", false));
diff --git a/mod/mod-build-force.hxx b/mod/mod-build-force.hxx
index 22df383..ea9c141 100644
--- a/mod/mod-build-force.hxx
+++ b/mod/mod-build-force.hxx
@@ -8,6 +8,7 @@
#include <libbrep/utility.hxx>
#include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
#include <mod/database-module.hxx>
#include <mod/build-config-module.hxx>
@@ -16,13 +17,13 @@ namespace brep
class build_force: public database_module, private build_config_module
{
public:
- build_force () = default;
+ explicit
+ build_force (const tenant_service_map&);
// Create a shallow copy (handling instance) if initialized and a deep
// copy (context exemplar) otherwise.
//
- explicit
- build_force (const build_force&);
+ build_force (const build_force&, const tenant_service_map&);
virtual bool
handle (request&, response&);
@@ -39,6 +40,7 @@ namespace brep
private:
shared_ptr<options::build_force> options_;
+ const tenant_service_map& tenant_service_map_;
};
}
diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx
index 3841fad..fae506b 100644
--- a/mod/mod-build-log.cxx
+++ b/mod/mod-build-log.cxx
@@ -227,7 +227,7 @@ handle (request& rq, response& rs)
query<package_build>::build::id == id, pb))
config_expired ("no package build");
- b = pb.build;
+ b = move (pb.build);
if (b->state != build_state::built)
config_expired ("state is " + to_string (b->state));
else
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 24b518d..7023e39 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -24,6 +24,7 @@
#include <mod/build.hxx> // *_url()
#include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
using namespace std;
using namespace butl;
@@ -31,14 +32,21 @@ using namespace bbot;
using namespace brep::cli;
using namespace odb::core;
+brep::build_result::
+build_result (const tenant_service_map& tsm)
+ : tenant_service_map_ (tsm)
+{
+}
+
// While currently the user-defined copy constructor is not required (we don't
// need to deep copy nullptr's), it is a good idea to keep the placeholder
// ready for less trivial cases.
//
brep::build_result::
-build_result (const build_result& r)
+build_result (const build_result& r, const tenant_service_map& tsm)
: build_result_module (r),
- options_ (r.initialized_ ? r.options_ : nullptr)
+ options_ (r.initialized_ ? r.options_ : nullptr),
+ tenant_service_map_ (tsm)
{
}
@@ -186,14 +194,33 @@ handle (request& rq, response&)
bool build_notify (false);
bool unforced (true);
+ // If the package is built (result status differs from interrupt, etc) and
+ // the package tenant has a third-party service state associated with it,
+ // then check if the tenant_service_build_built callback is registered for
+ // the type of the associated service. If it is, then stash the state, the
+ // build object, and the callback pointer for the subsequent service `built`
+ // notification. Note that we send this notification for the skip result as
+ // well, since it is semantically equivalent to the previous build result
+ // with the actual build process being optimized out.
+ //
+ // If the package build is interrupted and the tenant_service_build_queued
+ // callback is associated with the package tenant, then stash the state, the
+ // build object, and the callback pointer for the subsequent service
+ // `queued` notification.
+ //
+ const tenant_service_build_built* tsb (nullptr);
+ const tenant_service_build_queued* tsq (nullptr);
+ optional<pair<tenant_service, shared_ptr<build>>> tss;
+
// Note that if the session authentication fails (probably due to the
// authentication settings change), then we log this case with the warning
// severity and respond with the 200 HTTP code as if the challenge is
// valid. The thinking is that we shouldn't alarm a law-abaiding agent and
// shouldn't provide any information to a malicious one.
//
+ connection_ptr conn (build_db_->connection ());
{
- transaction t (build_db_->begin ());
+ transaction t (conn->begin ());
package_build pb;
@@ -221,11 +248,38 @@ handle (request& rq, response&)
}
else if (authenticate_session (*options_, rqm.challenge, *b, rqm.session))
{
+ const tenant_service_base* ts (nullptr);
+
+ shared_ptr<build_tenant> t (build_db_->load<build_tenant> (b->tenant));
+
+ if (t->service)
+ {
+ auto i (tenant_service_map_.find (t->service->type));
+
+ if (i != tenant_service_map_.end ())
+ ts = i->second.get ();
+ }
+
// If the build is interrupted, then revert it to the original built
- // state if this is a rebuild and delete it from the database otherwise.
+ // state if this is a rebuild. Otherwise (initial build), turn the build
+ // into the queued state if the tenant_service_build_queued callback is
+ // registered for the package tenant and delete it from the database
+ // otherwise.
+ //
+ // Note that if the tenant_service_build_queued callback is registered,
+ // we always send the `queued` notification for the interrupted build,
+ // even when we reverse it to the original built state. We could also
+ // turn the build into the queued state in this case, but it feels that
+ // there is no harm in keeping the previous build information available
+ // for the user.
//
if (rs == result_status::interrupt)
{
+ // Schedule the `queued` notification, if the
+ // tenant_service_build_queued callback is registered for the tenant.
+ //
+ tsq = dynamic_cast<const tenant_service_build_queued*> (ts);
+
if (b->status) // Is this a rebuild?
{
b->state = build_state::built;
@@ -248,14 +302,57 @@ handle (request& rq, response&)
// Note that we are unable to restore the pre-rebuild timestamp
// since it has been overwritten when the build task was issued.
// That, however, feels ok and we just keep it unchanged.
+ //
+ // Moreover, we actually use the fact that the build's timestamp is
+ // greater then its soft_timestamp as an indication that the build
+ // object represents the interrupted rebuild (see the build_task
+ // handler for details).
build_db_->update (b);
}
- else
- build_db_->erase (b);
+ else // Initial build.
+ {
+ if (tsq != nullptr)
+ {
+ // Since this is not a rebuild, there are no operation results and
+ // thus we don't need to load the results section to erase results
+ // from the database.
+ //
+ assert (b->results.empty ());
+
+ *b = build (move (b->tenant),
+ move (b->package_name),
+ move (b->package_version),
+ move (b->target),
+ move (b->target_config_name),
+ move (b->package_config_name),
+ move (b->toolchain_name),
+ move (b->toolchain_version));
+
+ build_db_->update (b);
+ }
+ else
+ build_db_->erase (b);
+ }
+
+ // If we ought to call the tenant_service_build_queued::build_queued()
+ // callback, then also set the package tenant's queued timestamp to
+ // the current time to prevent the notifications race (see
+ // tenant::queued_timestamp for details).
+ //
+ if (tsq != nullptr)
+ {
+ t->queued_timestamp = system_clock::now ();
+ build_db_->update (t);
+ }
}
- else
+ else // Regular or skip build result.
{
+ // Schedule the `built` notification, if the
+ // tenant_service_build_built callback is registered for the tenant.
+ //
+ tsb = dynamic_cast<const tenant_service_build_built*> (ts);
+
// Verify the result status/checksums.
//
// Specifically, if the result status is skip, then it can only be in
@@ -334,7 +431,8 @@ handle (request& rq, response&)
b->soft_timestamp = b->timestamp;
// If the result status is other than skip, then save the status,
- // results, and checksums and update the hard timestamp.
+ // results, and checksums and update the hard timestamp. Also stash
+ // the service notification information, if present.
//
if (rs != result_status::skip)
{
@@ -372,18 +470,61 @@ handle (request& rq, response&)
build_db_->load (*pkg, pkg->constraints_section);
if (!exclude (*cfg, pkg->builds, pkg->constraints, *tc))
- bld = move (b);
+ bld = b;
}
}
else
warn << "cannot find configuration '" << b->package_config_name
<< "' for package " << pkg->id.name << '/' << pkg->version;
}
+
+ // If required, stash the service notification information.
+ //
+ if (tsb != nullptr || tsq != nullptr)
+ tss = make_pair (move (*t->service), move (b));
}
t.commit ();
}
+ // We either notify about the queued build or notify about the built package
+ // or don't notify at all.
+ //
+ assert (tsb == nullptr || tsq == nullptr);
+
+ // If the package build is interrupted and the tenant-associated third-party
+ // service needs to be notified about the queued builds, then call the
+ // tenant_service_build_queued::build_queued() callback function and update
+ // the service state, if requested.
+ //
+ if (tsq != nullptr)
+ {
+ assert (tss); // Wouldn't be here otherwise.
+
+ const tenant_service& ss (tss->first);
+
+ vector<build> qbs;
+ qbs.push_back (move (*tss->second));
+
+ if (auto f = tsq->build_queued (ss, qbs, build_state::building))
+ update_tenant_service_state (conn, qbs.back ().tenant, f);
+ }
+
+ // If a third-party service needs to be notified about the built package,
+ // then call the tenant_service_build_built::build_built() callback function
+ // and update the service state, if requested.
+ //
+ if (tsb != nullptr)
+ {
+ assert (tss); // Wouldn't be here otherwise.
+
+ const tenant_service& ss (tss->first);
+ const build& b (*tss->second);
+
+ if (auto f = tsb->build_built (ss, b))
+ update_tenant_service_state (conn, b.tenant, f);
+ }
+
if (bld == nullptr)
return true;
diff --git a/mod/mod-build-result.hxx b/mod/mod-build-result.hxx
index 87ef1f2..96449d5 100644
--- a/mod/mod-build-result.hxx
+++ b/mod/mod-build-result.hxx
@@ -8,6 +8,7 @@
#include <libbrep/utility.hxx>
#include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
#include <mod/build-result-module.hxx>
namespace brep
@@ -15,13 +16,13 @@ namespace brep
class build_result: public build_result_module
{
public:
- build_result () = default;
+ explicit
+ build_result (const tenant_service_map&);
// Create a shallow copy (handling instance) if initialized and a deep
// copy (context exemplar) otherwise.
//
- explicit
- build_result (const build_result&);
+ build_result (const build_result&, const tenant_service_map&);
virtual bool
handle (request&, response&);
@@ -35,6 +36,7 @@ namespace brep
private:
shared_ptr<options::build_result> options_;
+ const tenant_service_map& tenant_service_map_;
};
}
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 01d14cd..1a06ce1 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -58,15 +58,22 @@ rand (size_t min_val, size_t max_val)
static_cast<unsigned long long> (max_val)) (rand_gen));
}
+brep::build_task::
+build_task (const tenant_service_map& tsm)
+ : tenant_service_map_ (tsm)
+{
+}
+
// While currently the user-defined copy constructor is not required (we don't
// need to deep copy nullptr's), it is a good idea to keep the placeholder
// ready for less trivial cases.
//
brep::build_task::
-build_task (const build_task& r)
+build_task (const build_task& r, const tenant_service_map& tsm)
: database_module (r),
build_config_module (r),
- options_ (r.initialized_ ? r.options_ : nullptr)
+ options_ (r.initialized_ ? r.options_ : nullptr),
+ tenant_service_map_ (tsm)
{
}
@@ -117,9 +124,15 @@ init (scanner& s)
options_->root (dir_path ("/"));
}
+// Skip tenants with the freshly queued packages from the consideration (see
+// tenant::queued_timestamp for the details on the service notifications race
+// prevention).
+//
template <typename T>
static inline query<T>
-package_query (brep::params::build_task& params, interactive_mode imode)
+package_query (brep::params::build_task& params,
+ interactive_mode imode,
+ uint64_t queued_expiration_ns)
{
using namespace brep;
using query = query<T>;
@@ -153,7 +166,9 @@ package_query (brep::params::build_task& params, interactive_mode imode)
case interactive_mode::both: break;
}
- return q;
+ return q &&
+ (query::build_tenant::queued_timestamp.is_null () ||
+ query::build_tenant::queued_timestamp < queued_expiration_ns);
}
bool brep::build_task::
@@ -261,7 +276,7 @@ handle (request& rq, response& rs)
config_machines conf_machines;
- for (const auto& c: *target_conf_)
+ for (const build_target_config& c: *target_conf_)
{
for (auto& m: tqm.machines)
{
@@ -303,34 +318,34 @@ handle (request& rq, response& rs)
// Create the task response manifest. Must be called inside the build db
// transaction.
//
- auto task = [this] (shared_ptr<build>&& b,
- shared_ptr<build_package>&& p,
+ auto task = [this] (const build& b,
+ build_package&& p,
build_package_config&& pc,
optional<string>&& interactive,
const config_machine& cm) -> task_response_manifest
{
uint64_t ts (
chrono::duration_cast<std::chrono::nanoseconds> (
- b->timestamp.time_since_epoch ()).count ());
-
- string session (b->tenant + '/' +
- b->package_name.string () + '/' +
- b->package_version.string () + '/' +
- b->target.string () + '/' +
- b->target_config_name + '/' +
- b->package_config_name + '/' +
- b->toolchain_name + '/' +
- b->toolchain_version.string () + '/' +
+ b.timestamp.time_since_epoch ()).count ());
+
+ string session (b.tenant + '/' +
+ b.package_name.string () + '/' +
+ b.package_version.string () + '/' +
+ b.target.string () + '/' +
+ b.target_config_name + '/' +
+ b.package_config_name + '/' +
+ b.toolchain_name + '/' +
+ b.toolchain_version.string () + '/' +
to_string (ts));
- string tenant (tenant_dir (options_->root (), b->tenant).string ());
+ string tenant (tenant_dir (options_->root (), b.tenant).string ());
string result_url (options_->host () + tenant + "?build-result");
assert (transaction::has_current ());
- assert (p->internal ()); // The package is expected to be buildable.
+ assert (p.internal ()); // The package is expected to be buildable.
- shared_ptr<build_repository> r (p->internal_repository.load ());
+ shared_ptr<build_repository> r (p.internal_repository.load ());
strings fps;
if (r->certificate_fingerprint)
@@ -341,9 +356,9 @@ handle (request& rq, response& rs)
//
small_vector<bpkg::test_dependency, 1> tests;
- build_db_->load (*p, p->requirements_tests_section);
+ build_db_->load (p, p.requirements_tests_section);
- for (const build_test_dependency& td: p->tests)
+ for (const build_test_dependency& td: p.tests)
{
// Don't exclude unresolved external tests.
//
@@ -390,16 +405,17 @@ handle (request& rq, response& rs)
move (td.reflect));
}
- bool module_pkg (
- b->package_name.string ().compare (0, 10, "libbuild2-") == 0);
+ package_name& pn (p.id.name);
- task_manifest task (move (b->package_name),
- move (b->package_version),
+ bool module_pkg (pn.string ().compare (0, 10, "libbuild2-") == 0);
+
+ task_manifest task (move (pn),
+ move (p.version),
move (r->location),
move (fps),
- move (p->requirements),
+ move (p.requirements),
move (tests),
- move (b->dependency_checksum),
+ b.dependency_checksum,
cm.machine->name,
cm.config->target,
cm.config->environment,
@@ -408,7 +424,7 @@ handle (request& rq, response& rs)
belongs (*cm.config, module_pkg ? "build2" : "host"),
cm.config->warning_regexes,
move (interactive),
- move (b->worker_checksum));
+ b.worker_checksum);
// Collect the build artifacts upload URLs, skipping those which are
// excluded with the upload-*-exclude configuration options.
@@ -434,7 +450,7 @@ handle (request& rq, response& rs)
};
if (!exclude (options_->upload_toolchain_exclude (),
- b->toolchain_name) &&
+ b.toolchain_name) &&
!exclude (options_->upload_repository_exclude (),
r->canonical_name))
{
@@ -444,15 +460,15 @@ handle (request& rq, response& rs)
}
return task_response_manifest (move (session),
- move (b->agent_challenge),
+ b.agent_challenge,
move (result_url),
move (upload_urls),
- move (b->agent_checksum),
+ b.agent_checksum,
move (task));
};
- // Calculate the build (building state) or rebuild (built state)
- // expiration time for package configurations.
+ // Calculate the build/rebuild (building/built state) and the `queued`
+ // notifications expiration time for package configurations.
//
timestamp now (system_clock::now ());
@@ -476,6 +492,9 @@ handle (request& rq, response& rs)
timestamp forced_rebuild_expiration (
expiration (options_->build_forced_rebuild_timeout ()));
+ uint64_t queued_expiration_ns (
+ expiration_ns (options_->build_queued_timeout ()));
+
// Calculate the soft/hard rebuild expiration time, based on the
// respective build-{soft,hard}-rebuild-timeout and
// build-alt-{soft,hard}-rebuild-{start,stop,timeout} configuration
@@ -626,6 +645,7 @@ handle (request& rq, response& rs)
// Convert butl::standard_version type to brep::version.
//
brep::version toolchain_version (tqm.toolchain_version.string ());
+ string& toolchain_name (tqm.toolchain_name);
// Prepare the buildable package prepared query.
//
@@ -646,7 +666,9 @@ handle (request& rq, response& rs)
using pkg_query = query<buildable_package>;
using prep_pkg_query = prepared_query<buildable_package>;
- pkg_query pq (package_query<buildable_package> (params, imode));
+ pkg_query pq (package_query<buildable_package> (params,
+ imode,
+ queued_expiration_ns));
// Transform (in-place) the interactive login information into the actual
// login command, if specified in the manifest and the transformation
@@ -800,7 +822,9 @@ handle (request& rq, response& rs)
{
using query = query<buildable_package_count>;
- query q (package_query<buildable_package_count> (params, imode));
+ query q (package_query<buildable_package_count> (params,
+ imode,
+ queued_expiration_ns));
transaction t (build_db_->begin ());
@@ -896,17 +920,18 @@ handle (request& rq, response& rs)
equal<build> (bld_query::id.package, id) &&
bld_query::id.package_config_name == bld_query::_ref (pkg_config) &&
sq &&
- bld_query::id.toolchain_name == tqm.toolchain_name &&
+ bld_query::id.toolchain_name == toolchain_name &&
compare_version_eq (bld_query::id.toolchain_version,
canonical_version (toolchain_version),
true /* revision */) &&
- (bld_query::state == "built" ||
- (bld_query::force == "forcing" &&
- bld_query::timestamp > forced_result_expiration_ns) ||
- (bld_query::force != "forcing" && // Unforced or forced.
- bld_query::timestamp > normal_result_expiration_ns)));
+ (bld_query::state == "built" ||
+ (bld_query::state == "building" &&
+ ((bld_query::force == "forcing" &&
+ bld_query::timestamp > forced_result_expiration_ns) ||
+ (bld_query::force != "forcing" && // Unforced or forced.
+ bld_query::timestamp > normal_result_expiration_ns)))));
prep_bld_query bld_prep_query (
conn->prepare_query<build> ("mod-build-task-build-query", bq));
@@ -971,6 +996,107 @@ handle (request& rq, response& rs)
//
optional<string> start_tenant;
+ // If the build task is created and the tenant of the being built
+ // package has a third-party service state associated with it, then
+ // check if the tenant_service_build_building and/or
+ // tenant_service_build_queued callbacks are registered for the type of
+ // the associated service. If they are, then stash the state, the build
+ // object, and the callback pointers for the subsequent service
+ // notifications.
+ //
+ // Also, if the tenant_service_build_queued callback is registered, then
+ // create, persist, and stash the queued build objects for all the
+ // unbuilt by the current toolchain and not yet queued configurations of
+ // the package the build task is created for. Note that for the task
+ // build, we need to make sure that the third-party service receives the
+ // `queued` notification prior to the `building` notification (see
+ // mod/tenant-service.hxx for valid transitions). The `queued`
+ // notification is assumed to be already sent for the build if the
+ // respective object exists and any of the following is true for it:
+ //
+ // - It is in the queued state (initial_state is build_state::queued).
+ //
+ // - It is a user-forced rebuild of an incomplete build
+ // (rebuild_forced_build is true).
+ //
+ // - It is a rebuild of an interrupted rebuild (rebuild_forced_build is
+ // true).
+ //
+ const tenant_service_build_building* tsb (nullptr);
+ const tenant_service_build_queued* tsq (nullptr);
+ optional<pair<tenant_service, shared_ptr<build>>> tss;
+ vector<build> qbs;
+ optional<build_state> initial_state;
+ bool rebuild_forced_build (false);
+ bool rebuild_interrupted_rebuild (false);
+
+ // Create, persist, and return the queued build objects for all the
+ // unbuilt by the current toolchain and not yet queued configurations of
+ // the specified package.
+ //
+ // Note that the build object argument is only used for the toolchain
+ // information retrieval. Also note that the package constraints section
+ // is expected to be loaded.
+ //
+ auto queue_builds = [this] (const build_package& p, const build& b)
+ {
+ assert (p.constraints_section.loaded ());
+
+ // Query the existing build ids and stash them into the set.
+ //
+ set<build_id> existing_builds;
+
+ using query = query<package_build_id>;
+
+ query q (query::build::id.package == p.id &&
+ query::build::id.toolchain_name == b.toolchain_name &&
+ compare_version_eq (query::build::id.toolchain_version,
+ b.id.toolchain_version,
+ true /* revision */));
+
+ for (build_id& id: build_db_->query<package_build_id> (q))
+ existing_builds.emplace (move (id));
+
+ // Go through all the potential package builds and queue those which
+ // are not in the existing builds set.
+ //
+ vector<build> r;
+
+ for (const build_package_config& pc: p.configs)
+ {
+ for (const build_target_config& tc: *target_conf_)
+ {
+ if (!exclude (pc, p.builds, p.constraints, tc))
+ {
+ build_id id (p.id,
+ tc.target, tc.name,
+ pc.name,
+ b.toolchain_name, b.toolchain_version);
+
+ if (existing_builds.find (id) == existing_builds.end ())
+ {
+ r.emplace_back (move (id.package.tenant),
+ move (id.package.name),
+ p.version,
+ move (id.target),
+ move (id.target_config_name),
+ move (id.package_config_name),
+ move (id.toolchain_name),
+ b.toolchain_version);
+
+ // @@ TODO Persist the whole vector of builds with a single
+ // operation if/when bulk operations support is added
+ // for objects with containers.
+ //
+ build_db_->persist (r.back ());
+ }
+ }
+ }
+ }
+
+ return r;
+ };
+
for (bool done (false); tsm.session.empty () && !done; )
{
transaction t (conn->begin ());
@@ -1163,7 +1289,7 @@ handle (request& rq, response& rs)
tc.target,
tc.name,
pc,
- tqm.toolchain_name,
+ toolchain_name,
toolchain_version);
// Can there be any existing builds for such a tenant? Doesn't
@@ -1279,7 +1405,7 @@ handle (request& rq, response& rs)
cm.config->target,
cm.config->name,
move (pkg_config),
- move (tqm.toolchain_name),
+ move (toolchain_name),
toolchain_version);
shared_ptr<build> b (build_db_->find<build> (bid));
@@ -1318,16 +1444,20 @@ handle (request& rq, response& rs)
}
else
{
- // The build configuration is in the building state.
+ // The build configuration is in the building or queued
+ // state.
//
- // Note that in both cases we keep the status intact to be
- // able to compare it with the final one in the result
- // request handling in order to decide if to send the
- // notification email or to revert it to the built state if
- // interrupted. The same is true for the forced flag (in
- // the sense that we don't set the force state to unforced).
+ // Note that in both the building and built cases we keep
+ // the status intact to be able to compare it with the final
+ // one in the result request handling in order to decide if
+ // to send the notification email or to revert it to the
+ // built state if interrupted. The same is true for the
+ // forced flag (in the sense that we don't set the force
+ // state to unforced).
//
- assert (b->state == build_state::building);
+ assert (b->state != build_state::built);
+
+ initial_state = b->state;
b->state = build_state::building;
b->interactive = move (login);
@@ -1337,7 +1467,10 @@ handle (request& rq, response& rs)
// still recognize that the rebuild was forced.
//
if (b->force == force_state::forcing)
+ {
b->force = force_state::forced;
+ rebuild_forced_build = true;
+ }
b->agent_fingerprint = move (agent_fp);
b->agent_challenge = move (cl);
@@ -1364,21 +1497,58 @@ handle (request& rq, response& rs)
build_db_->update (b);
}
+ shared_ptr<build_tenant> t (
+ build_db_->load<build_tenant> (b->tenant));
+
// Archive an interactive tenant.
//
if (bp.interactive)
{
- shared_ptr<build_tenant> t (
- build_db_->load<build_tenant> (b->id.package.tenant));
-
t->archived = true;
build_db_->update (t);
}
- // Finally, prepare the task response manifest.
+ // Finally, stash the service notification information, if
+ // present, and prepare the task response manifest.
//
+ if (t->service)
+ {
+ auto i (tenant_service_map_.find (t->service->type));
+
+ if (i != tenant_service_map_.end ())
+ {
+ const tenant_service_base* s (i->second.get ());
+
+ tsb = dynamic_cast<const tenant_service_build_building*> (s);
+ tsq = dynamic_cast<const tenant_service_build_queued*> (s);
+
+ if (tsq != nullptr)
+ {
+ qbs = queue_builds (*p, *b);
+
+ // If we ought to call the
+ // tenant_service_build_queued::build_queued() callback,
+ // then also set the package tenant's queued timestamp
+ // to the current time to prevent the notifications race
+ // (see tenant::queued_timestamp for details).
+ //
+ if (!qbs.empty () ||
+ !initial_state ||
+ (*initial_state != build_state::queued &&
+ !rebuild_forced_build))
+ {
+ t->queued_timestamp = system_clock::now ();
+ build_db_->update (t);
+ }
+ }
+
+ if (tsb != nullptr || tsq != nullptr)
+ tss = make_pair (move (*t->service), b);
+ }
+ }
+
tsm = task (
- move (b), move (p), move (pc), move (bp.interactive), cm);
+ *b, move (*p), move (pc), move (bp.interactive), cm);
break; // Bail out from the package configurations loop.
}
@@ -1493,6 +1663,11 @@ handle (request& rq, response& rs)
{
assert (b->status);
+ initial_state = build_state::built;
+
+ rebuild_interrupted_rebuild =
+ (b->timestamp > b->soft_timestamp);
+
b->state = build_state::building;
// Save the interactive build login information into the
@@ -1535,8 +1710,45 @@ handle (request& rq, response& rs)
build_db_->update (b);
+ // Stash the service notification information, if present,
+ // and prepare the task response manifest.
+ //
+ if (t->service)
+ {
+ auto i (tenant_service_map_.find (t->service->type));
+
+ if (i != tenant_service_map_.end ())
+ {
+ const tenant_service_base* s (i->second.get ());
+
+ tsb = dynamic_cast<const tenant_service_build_building*> (s);
+ tsq = dynamic_cast<const tenant_service_build_queued*> (s);
+
+ if (tsq != nullptr)
+ {
+ qbs = queue_builds (*p, *b);
+
+ // If we ought to call the
+ // tenant_service_build_queued::build_queued()
+ // callback, then also set the package tenant's queued
+ // timestamp to the current time to prevent the
+ // notifications race (see tenant::queued_timestamp
+ // for details).
+ //
+ if (!qbs.empty () || !rebuild_interrupted_rebuild)
+ {
+ t->queued_timestamp = system_clock::now ();
+ build_db_->update (t);
+ }
+ }
+
+ if (tsb != nullptr || tsq != nullptr)
+ tss = make_pair (move (*t->service), b);
+ }
+ }
+
tsm = task (
- move (b), move (p), move (*pc), move (t->interactive), cm);
+ *b, move (*p), move (*pc), move (t->interactive), cm);
}
}
}
@@ -1558,6 +1770,73 @@ handle (request& rq, response& rs)
break;
}
}
+
+ // If the tenant-associated third-party service needs to be notified
+ // about the queued builds, then call the
+ // tenant_service_build_queued::build_queued() callback function and
+ // update the service state, if requested.
+ //
+ if (tsq != nullptr)
+ {
+ assert (tss); // Wouldn't be here otherwise.
+
+ const tenant_service& ss (tss->first);
+
+ // If the task build has no initial state (is just created), then
+ // temporarily move it into the list of the queued builds until the
+ // `queued` notification is delivered. Afterwards, restore it so that
+ // the `building` notification can also be sent.
+ //
+ build& b (*tss->second);
+ bool restore_build (false);
+
+ if (!initial_state)
+ {
+ qbs.push_back (move (b));
+ restore_build = true;
+ }
+
+ if (!qbs.empty ())
+ {
+ if (auto f = tsq->build_queued (ss, qbs, nullopt /* initial_state */))
+ update_tenant_service_state (conn, qbs.back ().tenant, f);
+ }
+
+ // Send the `queued` notification for the task build, unless it is
+ // already sent, and update the service state, if requested.
+ //
+ if (initial_state &&
+ *initial_state != build_state::queued &&
+ !rebuild_interrupted_rebuild &&
+ !rebuild_forced_build)
+ {
+ qbs.clear ();
+ qbs.push_back (move (b));
+ restore_build = true;
+
+ if (auto f = tsq->build_queued (ss, qbs, initial_state))
+ update_tenant_service_state (conn, qbs.back ().tenant, f);
+ }
+
+ if (restore_build)
+ b = move (qbs.back ());
+ }
+
+ // If a third-party service needs to be notified about the package
+ // build, then call the tenant_service_build_built::build_building()
+ // callback function and, if requested, update the tenant-associated
+ // service state.
+ //
+ if (tsb != nullptr)
+ {
+ assert (tss); // Wouldn't be here otherwise.
+
+ const tenant_service& ss (tss->first);
+ const build& b (*tss->second);
+
+ if (auto f = tsb->build_building (ss, b))
+ update_tenant_service_state (conn, b.tenant, f);
+ }
}
}
diff --git a/mod/mod-build-task.hxx b/mod/mod-build-task.hxx
index 7875db1..d0b3d44 100644
--- a/mod/mod-build-task.hxx
+++ b/mod/mod-build-task.hxx
@@ -8,6 +8,7 @@
#include <libbrep/utility.hxx>
#include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
#include <mod/database-module.hxx>
#include <mod/build-config-module.hxx>
@@ -16,13 +17,13 @@ namespace brep
class build_task: public database_module, private build_config_module
{
public:
- build_task () = default;
+ explicit
+ build_task (const tenant_service_map&);
// Create a shallow copy (handling instance) if initialized and a deep
// copy (context exemplar) otherwise.
//
- explicit
- build_task (const build_task&);
+ build_task (const build_task&, const tenant_service_map&);
virtual bool
handle (request&, response&);
@@ -36,6 +37,7 @@ namespace brep
private:
shared_ptr<options::build_task> options_;
+ const tenant_service_map& tenant_service_map_;
};
}
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index f260b72..b0de618 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -225,13 +225,19 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids,
// Build result.
//
const string& rs (params.result ());
+ bool add_state (true);
if (rs != "*")
{
if (rs == "pending")
+ {
q = q && qb::force != "unforced";
+ }
else if (rs == "building")
+ {
q = q && qb::state == "building";
+ add_state = false;
+ }
else
{
query sq (qb::status == rs);
@@ -259,8 +265,12 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids,
// well (rebuild).
//
q = q && qb::state == "built" && sq;
+ add_state = false;
}
}
+
+ if (add_state)
+ q = q && qb::state != "queued";
}
catch (const invalid_argument&)
{
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
index df2365a..fec603e 100644
--- a/mod/mod-ci.cxx
+++ b/mod/mod-ci.cxx
@@ -3,18 +3,11 @@
#include <mod/mod-ci.hxx>
-#include <ostream>
-
-#include <libbutl/uuid.hxx>
-#include <libbutl/sendmail.hxx>
#include <libbutl/fdstream.hxx>
-#include <libbutl/timestamp.hxx>
-#include <libbutl/filesystem.hxx>
-#include <libbutl/process-io.hxx> // operator<<(ostream, process_args)
#include <libbutl/manifest-parser.hxx>
#include <libbutl/manifest-serializer.hxx>
-#include <libbpkg/manifest.hxx>
+#include <libbpkg/manifest.hxx> // package_manifest
#include <libbpkg/package-name.hxx>
#include <web/server/module.hxx>
@@ -23,20 +16,35 @@
#include <mod/page.hxx>
#include <mod/module-options.hxx>
-#include <mod/external-handler.hxx>
using namespace std;
using namespace butl;
using namespace web;
using namespace brep::cli;
+#ifdef BREP_CI_TENANT_SERVICE
+brep::ci::
+ci (tenant_service_map& tsm)
+ : tenant_service_map_ (tsm)
+{
+}
+#endif
+
brep::ci::
+#ifdef BREP_CI_TENANT_SERVICE
+ci (const ci& r, tenant_service_map& tsm)
+#else
ci (const ci& r)
+#endif
: handler (r),
+ ci_start (r),
options_ (r.initialized_ ? r.options_ : nullptr),
form_ (r.initialized_ || r.form_ == nullptr
? r.form_
: make_shared<xhtml::fragment> (*r.form_))
+#ifdef BREP_CI_TENANT_SERVICE
+ , tenant_service_map_ (tsm)
+#endif
{
}
@@ -45,22 +53,25 @@ init (scanner& s)
{
HANDLER_DIAG;
+#ifdef BREP_CI_TENANT_SERVICE
+ {
+ shared_ptr<tenant_service_base> ts (
+ dynamic_pointer_cast<tenant_service_base> (shared_from_this ()));
+
+ assert (ts != nullptr); // By definition.
+
+ tenant_service_map_["ci"] = move (ts);
+ }
+#endif
+
options_ = make_shared<options::ci> (
s, unknown_mode::fail, unknown_mode::fail);
- // Verify that the CI request handling is setup properly, if configured.
+ // Prepare for the CI requests handling, if configured.
//
if (options_->ci_data_specified ())
{
- // Verify the data directory satisfies the requirements.
- //
- const dir_path& d (options_->ci_data ());
-
- if (d.relative ())
- fail << "ci-data directory path must be absolute";
-
- if (!dir_exists (d))
- fail << "ci-data directory '" << d << "' does not exist";
+ ci_start::init (make_shared<options::ci_start> (*options_));
// Parse XHTML5 form file, if configured.
//
@@ -87,10 +98,6 @@ init (scanner& s)
fail << "unable to read ci-form file '" << ci_form << "': " << e;
}
}
-
- if (options_->ci_handler_specified () &&
- options_->ci_handler ().relative ())
- fail << "ci-handler path must be absolute";
}
if (options_->root ().empty ())
@@ -130,9 +137,8 @@ handle (request& rq, response& rs)
//
// return respond_error (); // Request is handled with an error.
//
- string request_id; // Will be set later.
- auto respond_manifest = [&rs, &request_id] (status_code status,
- const string& message) -> bool
+ auto respond_manifest = [&rs] (status_code status,
+ const string& message) -> bool
{
serializer s (rs.content (status, "text/manifest;charset=utf-8"),
"response");
@@ -140,10 +146,6 @@ handle (request& rq, response& rs)
s.next ("", "1"); // Start of manifest.
s.next ("status", to_string (status));
s.next ("message", message);
-
- if (!request_id.empty ())
- s.next ("reference", request_id);
-
s.next ("", ""); // End of manifest.
return true;
};
@@ -234,9 +236,11 @@ handle (request& rq, response& rs)
if (rl.empty () || rl.local ())
return respond_manifest (400, "invalid repository location");
- // Verify the package name[/version] arguments.
+ // Parse the package name[/version] arguments.
//
- for (const string& s: params.package())
+ vector<package> packages;
+
+ for (const string& s: params.package ())
{
// Let's skip the potentially unfilled package form fields.
//
@@ -245,18 +249,21 @@ handle (request& rq, response& rs)
try
{
+ package pkg;
size_t p (s.find ('/'));
if (p != string::npos)
{
- package_name (string (s, 0, p));
+ pkg.name = package_name (string (s, 0, p));
// Not to confuse with module::version.
//
- bpkg::version (string (s, p + 1));
+ pkg.version = bpkg::version (string (s, p + 1));
}
else
- package_name p (s); // Not to confuse with the s variable declaration.
+ pkg.name = package_name (s);
+
+ packages.push_back (move (pkg));
}
catch (const invalid_argument&)
{
@@ -265,31 +272,49 @@ handle (request& rq, response& rs)
}
// Verify that unknown parameter values satisfy the requirements (contain
- // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n').
+ // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n') and
+ // stash them.
//
// Actually, the expected ones must satisfy too, so check them as well.
//
- string what;
- for (const name_value& nv: rps)
+ vector<pair<string, string>> custom_request;
{
- if (nv.value &&
- !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t"))
- return respond_manifest (400,
- "invalid parameter " + nv.name + ": " + what);
+ string what;
+ for (const name_value& nv: rps)
+ {
+ if (nv.value &&
+ !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t"))
+ return respond_manifest (400,
+ "invalid parameter " + nv.name + ": " + what);
+
+ const string& n (nv.name);
+
+ if (n != "repository" &&
+ n != "_" &&
+ n != "package" &&
+ n != "overrides" &&
+ n != "interactive" &&
+ n != "simulate")
+ custom_request.emplace_back (n, nv.value ? *nv.value : "");
+ }
}
// Parse and validate overrides, if present.
//
- vector<manifest_name_value> overrides;
+ vector<pair<string, string>> overrides;
if (params.overrides_specified ())
try
{
istream& is (rq.open_upload ("overrides"));
parser mp (is, "overrides");
- overrides = parse_manifest (mp);
+ vector<manifest_name_value> ovrs (parse_manifest (mp));
+
+ package_manifest::validate_overrides (ovrs, mp.name ());
- package_manifest::validate_overrides (overrides, mp.name ());
+ overrides.reserve (ovrs.size ());
+ for (manifest_name_value& nv: ovrs)
+ overrides.emplace_back (move (nv.name), move (nv.value));
}
// Note that invalid_argument (thrown by open_upload() function call) can
// mean both no overrides upload or multiple overrides uploads.
@@ -310,383 +335,127 @@ handle (request& rq, response& rs)
return respond_error ();
}
- try
- {
- // Note that from now on the result manifest we respond with will contain
- // the reference value.
- //
- request_id = uuid::generate ().string ();
- }
- catch (const system_error& e)
- {
- error << "unable to generate request id: " << e;
- return respond_error ();
- }
-
- // Create the submission data directory.
+ // Stash the User-Agent HTTP header and the client IP address.
//
- dir_path dd (options_->ci_data () / dir_path (request_id));
-
- try
+ optional<string> client_ip;
+ optional<string> user_agent;
+ for (const name_value& h: rq.headers ())
{
- // It's highly unlikely but still possible that the directory already
- // exists. This can only happen if the generated uuid is not unique.
- //
- if (try_mkdir (dd) == mkdir_status::already_exists)
- throw_generic_error (EEXIST);
+ if (icasecmp (h.name, ":Client-IP") == 0)
+ client_ip = h.value;
+ else if (icasecmp (h.name, "User-Agent") == 0)
+ user_agent = h.value;
}
- catch (const system_error& e)
- {
- error << "unable to create directory '" << dd << "': " << e;
- return respond_error ();
- }
-
- auto_rmdir ddr (dd);
-
- // Serialize the CI request manifest to a stream. On the serialization error
- // respond to the client with the manifest containing the bad request (400)
- // code and return false, on the stream error pass through the io_error
- // exception, otherwise return true.
- //
- timestamp ts (system_clock::now ());
-
- auto rqm = [&request_id,
- &rl,
- &ts,
- &simulate,
- &rq,
- &rps,
- &params,
- &respond_manifest]
- (ostream& os, bool long_lines = false) -> bool
- {
- try
- {
- serializer s (os, "request", long_lines);
- // Serialize the submission manifest header.
- //
- s.next ("", "1"); // Start of manifest.
- s.next ("id", request_id);
- s.next ("repository", rl.string ());
-
- for (const string& p: params.package ())
- {
- if (!p.empty ()) // Skip empty package names (see above for details).
- s.next ("package", p);
- }
-
- if (params.interactive_specified ())
- s.next ("interactive", params.interactive ());
-
- if (!simulate.empty ())
- s.next ("simulate", simulate);
-
- s.next ("timestamp",
- butl::to_string (ts,
- "%Y-%m-%dT%H:%M:%SZ",
- false /* special */,
- false /* local */));
-
- // Serialize the User-Agent HTTP header and the client IP address.
- //
- optional<string> ip;
- optional<string> ua;
- for (const name_value& h: rq.headers ())
- {
- if (icasecmp (h.name, ":Client-IP") == 0)
- ip = h.value;
- else if (icasecmp (h.name, "User-Agent") == 0)
- ua = h.value;
- }
-
- if (ip)
- s.next ("client-ip", *ip);
-
- if (ua)
- s.next ("user-agent", *ua);
-
- // Serialize the request parameters.
- //
- // Note that the serializer constraints the parameter names (can't start
- // with '#', can't contain ':' and the whitespaces, etc.).
- //
- for (const name_value& nv: rps)
- {
- const string& n (nv.name);
-
- if (n != "repository" &&
- n != "_" &&
- n != "package" &&
- n != "overrides" &&
- n != "interactive" &&
- n != "simulate")
- s.next (n, nv.value ? *nv.value : "");
- }
-
- s.next ("", ""); // End of manifest.
- return true;
- }
- catch (const serialization& e)
- {
- respond_manifest (400, string ("invalid parameter: ") + e.what ());
- return false;
- }
- };
-
- // Serialize the CI request manifest to the submission directory.
- //
- path rqf (dd / "request.manifest");
+ optional<start_result> r (start (error,
+ warn,
+ verb_ ? &trace : nullptr,
+#ifdef BREP_CI_TENANT_SERVICE
+ tenant_service ("", "ci"),
+#else
+ nullopt /* service */,
+#endif
+ rl,
+ packages,
+ client_ip,
+ user_agent,
+ (params.interactive_specified ()
+ ? params.interactive ()
+ : optional<string> ()),
+ (!simulate.empty ()
+ ? simulate
+ : optional<string> ()),
+ custom_request,
+ overrides));
+
+ if (!r)
+ return respond_error (); // The diagnostics is already issued.
try
{
- ofdstream os (rqf);
- bool r (rqm (os));
- os.close ();
-
- if (!r)
- return true; // The client is already responded with the manifest.
- }
- catch (const io_error& e)
- {
- error << "unable to write to '" << rqf << "': " << e;
- return respond_error ();
+ serialize_manifest (*r,
+ rs.content (r->status, "text/manifest;charset=utf-8"));
}
-
- // Serialize the CI overrides manifest to a stream. On the stream error pass
- // through the io_error exception.
- //
- // Note that it can't throw the serialization exception as the override
- // manifest is parsed from the stream and so verified.
- //
- auto ovm = [&overrides] (ostream& os, bool long_lines = false)
+ catch (const serialization& e)
{
- try
- {
- serializer s (os, "overrides", long_lines);
- serialize_manifest (s, overrides);
- }
- catch (const serialization&) {assert (false);} // See above.
- };
+ error << "ref " << r->reference << ": unable to serialize handler's "
+ << "output: " << e;
- // Serialize the CI overrides manifest to the submission directory.
- //
- path ovf (dd / "overrides.manifest");
-
- if (!overrides.empty ())
- try
- {
- ofdstream os (ovf);
- ovm (os);
- os.close ();
- }
- catch (const io_error& e)
- {
- error << "unable to write to '" << ovf << "': " << e;
return respond_error ();
}
- // Given that the submission data is now successfully persisted we are no
- // longer in charge of removing it, except for the cases when the submission
- // handler terminates with an error (see below for details).
- //
- ddr.cancel ();
-
- // If the handler terminates with non-zero exit status or specifies 5XX
- // (HTTP server error) submission result manifest status value, then we
- // stash the submission data directory for troubleshooting. Otherwise, if
- // it's the 4XX (HTTP client error) status value, then we remove the
- // directory.
- //
- auto stash_submit_dir = [&dd, error] ()
- {
- if (dir_exists (dd))
- try
- {
- mvdir (dd, dir_path (dd + ".fail"));
- }
- catch (const system_error& e)
- {
- // Not much we can do here. Let's just log the issue and bail out
- // leaving the directory in place.
- //
- error << "unable to rename directory '" << dd << "': " << e;
- }
- };
-
- // Run the submission handler, if specified, reading the result manifest
- // from its stdout and caching it as a name/value pair list for later use
- // (forwarding to the client, sending via email, etc). Otherwise, create
- // implied result manifest.
- //
- status_code sc;
- vector<manifest_name_value> rvs;
-
- if (options_->ci_handler_specified ())
- {
- using namespace external_handler;
-
- optional<result_manifest> r (run (options_->ci_handler (),
- options_->ci_handler_argument (),
- dd,
- options_->ci_handler_timeout (),
- error,
- warn,
- verb_ ? &trace : nullptr));
- if (!r)
- {
- stash_submit_dir ();
- return respond_error (); // The diagnostics is already issued.
- }
-
- sc = r->status;
- rvs = move (r->values);
- }
- else // Create the implied result manifest.
- {
- sc = 200;
-
- auto add = [&rvs] (string n, string v)
- {
- manifest_name_value nv {
- move (n), move (v),
- 0 /* name_line */, 0 /* name_column */,
- 0 /* value_line */, 0 /* value_column */,
- 0 /* start_pos */, 0 /* colon_pos */, 0 /* end_pos */};
-
- rvs.emplace_back (move (nv));
- };
-
- add ("status", "200");
- add ("message", "CI request is queued");
- add ("reference", request_id);
- }
-
- assert (!rvs.empty ()); // Produced by the handler or is implied.
-
- // Serialize the submission result manifest to a stream. On the
- // serialization error log the error description and return false, on the
- // stream error pass through the io_error exception, otherwise return true.
- //
- auto rsm = [&rvs, &error, &request_id] (ostream& os,
- bool long_lines = false) -> bool
- {
- try
- {
- serializer s (os, "result", long_lines);
- serialize_manifest (s, rvs);
- return true;
- }
- catch (const serialization& e)
- {
- error << "ref " << request_id << ": unable to serialize handler's "
- << "output: " << e;
- return false;
- }
- };
-
- // If the submission data directory still exists then perform an appropriate
- // action on it, depending on the submission result status. Note that the
- // handler could move or remove the directory.
- //
- if (dir_exists (dd))
- {
- // Remove the directory if the client error is detected.
- //
- if (sc >= 400 && sc < 500)
- {
- rmdir_r (dd);
- }
- //
- // Otherwise, save the result manifest, into the directory. Also stash the
- // directory for troubleshooting in case of the server error.
- //
- else
- {
- path rsf (dd / "result.manifest");
-
- try
- {
- ofdstream os (rsf);
-
- // Not being able to stash the result manifest is not a reason to
- // claim the submission failed. The error is logged nevertheless.
- //
- rsm (os);
-
- os.close ();
- }
- catch (const io_error& e)
- {
- // Not fatal (see above).
- //
- error << "unable to write to '" << rsf << "': " << e;
- }
-
- if (sc >= 500 && sc < 600)
- stash_submit_dir ();
- }
- }
-
- // Send email, if configured, and the CI request submission is not simulated.
- // Use the long lines manifest serialization mode for the convenience of
- // copying/clicking URLs they contain.
- //
- // Note that we don't consider the email sending failure to be a submission
- // failure as the submission data is successfully persisted and the handler
- // is successfully executed, if configured. One can argue that email can be
- // essential for the submission processing and missing it would result in
- // the incomplete submission. In this case it's natural to assume that the
- // web server error log is monitored and the email sending failure will be
- // noticed.
- //
- if (options_->ci_email_specified () && simulate.empty ())
- try
- {
- // Redirect the diagnostics to the web server error log.
- //
- sendmail sm ([&trace, this] (const char* args[], size_t n)
- {
- l2 ([&]{trace << process_args {args, n};});
- },
- 2 /* stderr */,
- options_->email (),
- "CI request submission (" + request_id + ')',
- {options_->ci_email ()});
-
- // Write the CI request manifest.
- //
- bool r (rqm (sm.out, true /* long_lines */));
- assert (r); // The serialization succeeded once, so can't fail now.
-
- // Write the CI overrides manifest.
- //
- sm.out << "\n\n";
-
- ovm (sm.out, true /* long_lines */);
-
- // Write the CI result manifest.
- //
- sm.out << "\n\n";
-
- // We don't care about the result (see above).
- //
- rsm (sm.out, true /* long_lines */);
-
- sm.out.close ();
+ return true;
+}
- if (!sm.wait ())
- error << "sendmail " << *sm.exit;
- }
- // Handle process_error and io_error (both derive from system_error).
- //
- catch (const system_error& e)
- {
- error << "sendmail error: " << e;
- }
+#ifdef BREP_CI_TENANT_SERVICE
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_queued (const tenant_service&,
+ const vector<build>& bs,
+ optional<build_state> initial_state) const
+{
+ return [&bs, initial_state] (const tenant_service& ts)
+ {
+ optional<string> r (ts.data);
+
+ for (const build& b: bs)
+ {
+ string s ((!initial_state
+ ? "queued "
+ : "queued " + to_string (*initial_state) + ' ') +
+ b.package_name.string () + '/' +
+ b.package_version.string () + '/' +
+ b.target.string () + '/' +
+ b.target_config_name + '/' +
+ b.package_config_name + '/' +
+ b.toolchain_name + '/' +
+ b.toolchain_version.string ());
+
+ if (r)
+ {
+ *r += ", ";
+ *r += s;
+ }
+ else
+ r = move (s);
+ }
+
+ return r;
+ };
+}
- if (!rsm (rs.content (sc, "text/manifest;charset=utf-8")))
- return respond_error (); // The error description is already logged.
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_building (const tenant_service&, const build& b) const
+{
+ return [&b] (const tenant_service& ts)
+ {
+ string s ("building " +
+ b.package_name.string () + '/' +
+ b.package_version.string () + '/' +
+ b.target.string () + '/' +
+ b.target_config_name + '/' +
+ b.package_config_name + '/' +
+ b.toolchain_name + '/' +
+ b.toolchain_version.string ());
+
+ return ts.data ? *ts.data + ", " + s : s;
+ };
+}
- return true;
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_built (const tenant_service&, const build& b) const
+{
+ return [&b] (const tenant_service& ts)
+ {
+ string s ("built " +
+ b.package_name.string () + '/' +
+ b.package_version.string () + '/' +
+ b.target.string () + '/' +
+ b.target_config_name + '/' +
+ b.package_config_name + '/' +
+ b.toolchain_name + '/' +
+ b.toolchain_version.string ());
+
+ return ts.data ? *ts.data + ", " + s : s;
+ };
}
+#endif
diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx
index 431f53b..3b1e1be 100644
--- a/mod/mod-ci.hxx
+++ b/mod/mod-ci.hxx
@@ -9,14 +9,39 @@
#include <libbrep/types.hxx>
#include <libbrep/utility.hxx>
+#include <libbrep/build.hxx>
+#include <libbrep/common.hxx> // tenant_service
+
#include <mod/module.hxx>
#include <mod/module-options.hxx>
+#include <mod/ci-common.hxx>
+
+#ifdef BREP_CI_TENANT_SERVICE
+# include <mod/tenant-service.hxx>
+#endif
+
namespace brep
{
- class ci: public handler
+ class ci: public handler,
+ private ci_start
+#ifdef BREP_CI_TENANT_SERVICE
+ , public tenant_service_build_queued,
+ public tenant_service_build_building,
+ public tenant_service_build_built
+#endif
{
public:
+
+#ifdef BREP_CI_TENANT_SERVICE
+ explicit
+ ci (tenant_service_map&);
+
+ // Create a shallow copy (handling instance) if initialized and a deep
+ // copy (context exemplar) otherwise.
+ //
+ ci (const ci&, tenant_service_map&);
+#else
ci () = default;
// Create a shallow copy (handling instance) if initialized and a deep
@@ -24,12 +49,26 @@ namespace brep
//
explicit
ci (const ci&);
+#endif
virtual bool
- handle (request&, response&);
+ handle (request&, response&) override;
virtual const cli::options&
- cli_options () const {return options::ci::description ();}
+ cli_options () const override {return options::ci::description ();}
+
+#ifdef BREP_CI_TENANT_SERVICE
+ virtual function<optional<string> (const tenant_service&)>
+ build_queued (const tenant_service&,
+ const vector<build>&,
+ optional<build_state> initial_state) const override;
+
+ virtual function<optional<string> (const tenant_service&)>
+ build_building (const tenant_service&, const build&) const override;
+
+ virtual function<optional<string> (const tenant_service&)>
+ build_built (const tenant_service&, const build&) const override;
+#endif
private:
virtual void
@@ -38,6 +77,10 @@ namespace brep
private:
shared_ptr<options::ci> options_;
shared_ptr<web::xhtml::fragment> form_;
+
+#ifdef BREP_CI_TENANT_SERVICE
+ tenant_service_map& tenant_service_map_;
+#endif
};
}
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index 35a1a22..51b21c6 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -802,7 +802,7 @@ handle (request& rq, response& rs)
// Print the package built configurations in the time-descending order.
//
for (auto& b: build_db_->query<build> (
- (query::id.package == pkg->id && sq) +
+ (query::id.package == pkg->id && query::state != "queued" && sq) +
"ORDER BY" + query::timestamp + "DESC"))
{
string ts (butl::to_string (b.timestamp,
diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx
index 1b18996..34b4007 100644
--- a/mod/mod-repository-root.cxx
+++ b/mod/mod-repository-root.cxx
@@ -108,18 +108,31 @@ namespace brep
//
repository_root::
repository_root ()
- : packages_ (make_shared<packages> ()),
+ :
+ //
+ // Only create and populate the tenant service map in the examplar
+ // passing a reference to it to all the sub-handler exemplars. Note
+ // that we dispatch the tenant service callbacks to the examplar
+ // without creating a new instance for each callback (thus the
+ // callbacks are const).
+ //
+ tenant_service_map_ (make_shared<tenant_service_map> ()),
+ packages_ (make_shared<packages> ()),
package_details_ (make_shared<package_details> ()),
package_version_details_ (make_shared<package_version_details> ()),
repository_details_ (make_shared<repository_details> ()),
- build_task_ (make_shared<build_task> ()),
- build_result_ (make_shared<build_result> ()),
- build_force_ (make_shared<build_force> ()),
+ build_task_ (make_shared<build_task> (*tenant_service_map_)),
+ build_result_ (make_shared<build_result> (*tenant_service_map_)),
+ build_force_ (make_shared<build_force> (*tenant_service_map_)),
build_log_ (make_shared<build_log> ()),
builds_ (make_shared<builds> ()),
build_configs_ (make_shared<build_configs> ()),
submit_ (make_shared<submit> ()),
+#ifdef BREP_CI_TENANT_SERVICE
+ ci_ (make_shared<ci> (*tenant_service_map_)),
+#else
ci_ (make_shared<ci> ()),
+#endif
upload_ (make_shared<upload> ())
{
}
@@ -127,6 +140,10 @@ namespace brep
repository_root::
repository_root (const repository_root& r)
: handler (r),
+ tenant_service_map_ (
+ r.initialized_
+ ? r.tenant_service_map_
+ : make_shared<tenant_service_map> ()),
//
// Deep/shallow-copy sub-handlers depending on whether this is an
// exemplar/handler.
@@ -151,15 +168,15 @@ namespace brep
build_task_ (
r.initialized_
? r.build_task_
- : make_shared<build_task> (*r.build_task_)),
+ : make_shared<build_task> (*r.build_task_, *tenant_service_map_)),
build_result_ (
r.initialized_
? r.build_result_
- : make_shared<build_result> (*r.build_result_)),
+ : make_shared<build_result> (*r.build_result_, *tenant_service_map_)),
build_force_ (
r.initialized_
? r.build_force_
- : make_shared<build_force> (*r.build_force_)),
+ : make_shared<build_force> (*r.build_force_, *tenant_service_map_)),
build_log_ (
r.initialized_
? r.build_log_
@@ -179,7 +196,11 @@ namespace brep
ci_ (
r.initialized_
? r.ci_
+#ifdef BREP_CI_TENANT_SERVICE
+ : make_shared<ci> (*r.ci_, *tenant_service_map_)),
+#else
: make_shared<ci> (*r.ci_)),
+#endif
upload_ (
r.initialized_
? r.upload_
diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx
index 4f40c94..aa60fda 100644
--- a/mod/mod-repository-root.hxx
+++ b/mod/mod-repository-root.hxx
@@ -9,6 +9,7 @@
#include <mod/module.hxx>
#include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
namespace brep
{
@@ -59,6 +60,8 @@ namespace brep
version ();
private:
+ shared_ptr<tenant_service_map> tenant_service_map_;
+
shared_ptr<packages> packages_;
shared_ptr<package_details> package_details_;
shared_ptr<package_version_details> package_version_details_;
diff --git a/mod/module.cli b/mod/module.cli
index 3fdd7e3..3e81b38 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -21,7 +21,7 @@ namespace brep
{
// Option groups.
//
- class handler
+ class repository_email
{
string email
{
@@ -29,7 +29,10 @@ namespace brep
"Repository email. This email is used for the \cb{From:} header in
emails send by \cb{brep} (for example, build failure notifications)."
}
+ };
+ class handler
+ {
string host
{
"<host>",
@@ -279,6 +282,15 @@ namespace brep
the same as for the \cb{build-alt-soft-rebuild-stop} option but
for the \cb{build-hard-rebuild-timeout} option."
}
+
+ size_t build-queued-timeout = 30
+ {
+ "<seconds>",
+ "Time to wait before assuming the \cb{queued} notifications are
+ delivered for package CI requests submitted via third-party services
+ (GitHub, etc). During this time a package is not considered for a
+ build. Must be specified in seconds. Default is 30 seconds."
+ }
};
class build_db
@@ -563,7 +575,7 @@ namespace brep
}
};
- class build_result: build, package_db, build_db, handler
+ class build_result: build, package_db, build_db, repository_email, handler
{
size_t build-result-request-max-size = 10485760
{
@@ -626,7 +638,7 @@ namespace brep
}
};
- class submit: page, handler
+ class submit: page, repository_email, handler
{
dir_path submit-data
{
@@ -706,7 +718,7 @@ namespace brep
}
};
- class ci: page, handler
+ class ci_start: repository_email
{
dir_path ci-data
{
@@ -721,15 +733,6 @@ namespace brep
granted to the user that runs the web server."
}
- path ci-form
- {
- "<file>",
- "The package CI form fragment. If specified, then its contents are
- treated as an XHTML5 fragment that is inserted into the <body>
- element of the CI page. If unspecified, then no CI page will be
- displayed. Note that the file path must be absolute."
- }
-
string ci-email
{
"<email>",
@@ -766,7 +769,33 @@ namespace brep
}
};
- class upload: build, build_db, build_upload, handler
+ class ci_cancel
+ {
+ };
+
+ class ci: ci_start, page, handler
+ {
+ // Classic CI-specific options.
+ //
+
+ path ci-form
+ {
+ "<file>",
+ "The package CI form fragment. If specified, then its contents are
+ treated as an XHTML5 fragment that is inserted into the <body>
+ element of the CI page. If unspecified, then no CI page will be
+ displayed. Note that the file path must be absolute."
+ }
+ };
+
+ class ci_github: ci_start, ci_cancel, build_db, handler
+ {
+ // GitHub CI-specific options (e.g., request timeout when invoking
+ // GitHub APIs).
+ //
+ };
+
+ class upload: build, build_db, build_upload, repository_email, handler
{
};
diff --git a/mod/page.cxx b/mod/page.cxx
index d844a89..5483183 100644
--- a/mod/page.cxx
+++ b/mod/page.cxx
@@ -761,7 +761,6 @@ namespace brep
}
else
{
-
// If no unsuccessful operation results available, then print the
// overall build status. If there are any operation results available,
// then also print unsuccessful operation statuses with the links to the
diff --git a/mod/page.hxx b/mod/page.hxx
index f3c27d5..cac2b8b 100644
--- a/mod/page.hxx
+++ b/mod/page.hxx
@@ -473,7 +473,12 @@ namespace brep
bool a,
const string& h,
const dir_path& r):
- build_ (b), archived_ (a), host_ (h), root_ (r) {}
+ build_ (b), archived_ (a), host_ (h), root_ (r)
+ {
+ // We don't expect a queued build to ever be displayed.
+ //
+ assert (build_.state != build_state::queued);
+ }
void
operator() (xml::serializer&) const;
diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx
new file mode 100644
index 0000000..a7bc941
--- /dev/null
+++ b/mod/tenant-service.hxx
@@ -0,0 +1,107 @@
+// file : mod/tenant-service.hxx -*- C++ -*-
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_TENANT_SERVICE_HXX
+#define MOD_TENANT_SERVICE_HXX
+
+#include <map>
+
+#include <libbrep/types.hxx>
+#include <libbrep/utility.hxx>
+
+#include <libbrep/build.hxx>
+
+namespace brep
+{
+ class tenant_service_base
+ {
+ public:
+ virtual ~tenant_service_base () = default;
+ };
+
+ // Possible build notifications:
+ //
+ // queued
+ // building
+ // built
+ //
+ // Possible transitions:
+ //
+ // -> queued
+ // queued -> building
+ // building -> queued (interrupted & re-queued due to higher priority task)
+ // building -> built
+ // built -> queued (periodic or user-forced rebuild)
+ //
+ // While the implementation tries to make sure the notifications arrive in
+ // the correct order, this is currently done by imposing delays (some
+ // natural, such as building->built, and some artificial, such as
+ // queued->building). As result, it is unlikely but possible to be notified
+ // about the state transitions in the wrong order, especially if the
+ // notifications take a long time. To minimize the chance of this happening,
+ // the service implementation should strive to batch the queued state
+ // notifications (or which there could be hundreds) in a single request if
+ // at all possible. Also, if supported by the third-party API, it makes
+ // sense for the implementation to protect against overwriting later states
+ // with earlier. For example, if it's possible to place a condition on a
+ // notification, it makes sense to only set the state to queued if none of
+ // the later states (e.g., building) are already in effect.
+ //
+ // Note also that it's possible for the build to get deleted at any stage
+ // without any further notifications. This can happen, for example, due to
+ // data retention timeout or because the build configuration (buildtab
+ // entry) is no longer present. There is no explicit `deleted` transition
+ // notification because such situations (i.e., when a notification sequence
+ // is abandoned half way) are not expected to arise ordinarily in a
+ // properly-configured brep instance. And the third-party service is
+ // expected to deal with them using some overall timeout/expiration
+ // mechanism which it presumably has.
+ //
+ // Each build notification is in its own interface since a service may not
+ // be interested in all of them while computing the information to pass is
+ // expensive.
+ //
+ class tenant_service_build_queued: public virtual tenant_service_base
+ {
+ public:
+ // If the returned function is not NULL, it is called to update the
+ // service data. It should return the new data or nullopt if no update is
+ // necessary. Note: tenant_service::data passed to the callback and to the
+ // returned function may not be the same. Also, the returned function may
+ // be called multiple times (on transaction retries).
+ //
+ // The passed initial_state indicates the logical initial state and is
+ // either absent, `building` (interrupted), or `built` (rebuild). Note
+ // that all the passed build objects have the same initial state.
+ //
+ // The implementation of this and the below functions should normally not
+ // need to make any decisions based on the passed build::state. Rather,
+ // the function name suffix (_queued, _building, _built) signify the
+ // logical end state.
+ //
+ virtual function<optional<string> (const tenant_service&)>
+ build_queued (const tenant_service&,
+ const vector<build>&,
+ optional<build_state> initial_state) const = 0;
+ };
+
+ class tenant_service_build_building: public virtual tenant_service_base
+ {
+ public:
+ virtual function<optional<string> (const tenant_service&)>
+ build_building (const tenant_service&, const build&) const = 0;
+ };
+
+ class tenant_service_build_built: public virtual tenant_service_base
+ {
+ public:
+ virtual function<optional<string> (const tenant_service&)>
+ build_built (const tenant_service&, const build&) const = 0;
+ };
+
+ // Map of service type (tenant_service::type) to service.
+ //
+ using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>;
+}
+
+#endif // MOD_TENANT_SERVICE_HXX
diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx
index 6d6bb99..77f387b 100644
--- a/monitor/monitor.cxx
+++ b/monitor/monitor.cxx
@@ -675,7 +675,8 @@ namespace brep
const auto& bid (bquery::build::id);
- bquery bq ((equal<package_build> (bid.package, id.package) &&
+ bquery bq ((bquery::build::state != "queued" &&
+ equal<package_build> (bid.package, id.package) &&
bid.target == bquery::_ref (id.target) &&
bid.target_config_name ==
bquery::_ref (id.target_config_name) &&
@@ -864,8 +865,13 @@ namespace brep
b = move (pbs.begin ()->build);
}
else
+ {
b = db.find<build> (id);
+ if (b->state == build_state::queued)
+ b = nullptr;
+ }
+
// Note that we consider a build as delayed if it is not
// completed in the expected timeframe. So even if the build
// task have been issued recently we may still consider the
diff --git a/web/server/module.hxx b/web/server/module.hxx
index f870163..20f6217 100644
--- a/web/server/module.hxx
+++ b/web/server/module.hxx
@@ -9,6 +9,7 @@
#include <vector>
#include <iosfwd>
#include <chrono>
+#include <memory> // enable_shared_from_this
#include <cstdint> // uint16_t
#include <cstddef> // size_t
#include <utility> // move()
@@ -236,7 +237,7 @@ namespace web
// directories (e.g., apache/) if you need to see the code that
// does this.
//
- class handler
+ class handler: public std::enable_shared_from_this<handler>
{
public:
virtual