diff options
| -rw-r--r-- | brep/handler/ci/ci-load.in | 57 | ||||
| -rw-r--r-- | doc/manual.cli | 12 | ||||
| -rw-r--r-- | libbrep/build-extra.sql | 2 | ||||
| -rw-r--r-- | libbrep/build-package.hxx | 2 | ||||
| -rw-r--r-- | libbrep/build.cxx | 2 | ||||
| -rw-r--r-- | libbrep/build.hxx | 10 | ||||
| -rw-r--r-- | libbrep/build.xml | 6 | ||||
| -rw-r--r-- | libbrep/package-extra.sql | 39 | ||||
| -rw-r--r-- | libbrep/package.cxx | 4 | ||||
| -rw-r--r-- | libbrep/package.hxx | 17 | ||||
| -rw-r--r-- | libbrep/package.xml | 7 | ||||
| -rw-r--r-- | load/buildfile | 4 | ||||
| -rw-r--r-- | load/load.cli | 13 | ||||
| -rw-r--r-- | load/load.cxx | 11 | ||||
| -rw-r--r-- | mod/buildfile | 2 | ||||
| -rw-r--r-- | mod/mod-build-result.cxx | 4 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 130 | ||||
| -rw-r--r-- | mod/mod-builds.cxx | 58 | ||||
| -rw-r--r-- | mod/mod-ci.cxx | 18 | ||||
| -rw-r--r-- | mod/mod-package-version-details.cxx | 43 | ||||
| -rw-r--r-- | mod/mod-packages.cxx | 6 | ||||
| -rw-r--r-- | mod/module.cli | 22 | ||||
| -rw-r--r-- | mod/types-parsers.cxx | 42 | ||||
| -rw-r--r-- | mod/types-parsers.hxx | 9 | ||||
| -rw-r--r-- | www/ci.xhtml | 4 |
25 files changed, 399 insertions, 125 deletions
diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in index cd2b879..d568341 100644 --- a/brep/handler/ci/ci-load.in +++ b/brep/handler/ci/ci-load.in @@ -25,7 +25,10 @@ verbose= #true fetch_timeout=60 trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. +set -o errtrace # Trap ERR in functions. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in the current shell. +shopt -s nullglob # Expand no-match globs to nothing rather than themselves. @import brep/handler/handler@ @import brep/handler/ci/ci@ @@ -33,7 +36,7 @@ set -o errtrace # Trap ERR in functions. # The handler's own options. # result_url= -while [ $# -gt 0 ]; do +while [[ "$#" -gt 0 ]]; do case $1 in --result-url) shift @@ -50,7 +53,7 @@ done # loader="$1" -if [ -z "$loader" ]; then +if [[ -z "$loader" ]]; then error "$usage" fi @@ -60,7 +63,7 @@ shift # options. # loader_options=() -while [ $# -gt 1 ]; do +while [[ "$#" -gt 1 ]]; do loader_options+=("$1") shift done @@ -69,11 +72,11 @@ done # data_dir="${1%/}" -if [ -z "$data_dir" ]; then +if [[ -z "$data_dir" ]]; then error "$usage" fi -if [ ! -d "$data_dir" ]; then +if [[ ! -d "$data_dir" ]]; then error "'$data_dir' does not exist or is not a directory" fi @@ -84,8 +87,9 @@ reference="$(basename "$data_dir")" # manifest_parser_start "$data_dir/request.manifest" -simulate= repository= +interactive= +simulate= # Package map. We first enter packages from the request manifest as keys and # setting the values to true. Then we go through the repository package list @@ -106,13 +110,14 @@ spec= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in - simulate) simulate="$v" ;; - repository) repository="$v" ;; + repository) repository="$v" ;; + interactive) interactive="$v" ;; + simulate) simulate="$v" ;; package) packages["$v"]=true - if [ -n "$spec" ]; then + if [[ -n "$spec" ]]; then spec="$spec," fi spec="$spec$v" @@ -122,22 +127,22 @@ done manifest_parser_finish -if [ -n "$spec" ]; then +if [[ -n "$spec" ]]; then spec="$spec@" fi spec="$spec$repository" -if [ -z "$repository" ]; then +if [[ -z "$repository" ]]; then error "repository manifest value expected" fi -if [ -n "$simulate" -a "$simulate" != "success" ]; then +if [[ -n "$simulate" && "$simulate" != "success" ]]; then exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" fi message_suffix= -if [ -n "$result_url" ]; then +if [[ -n "$result_url" ]]; then message_suffix=": $result_url/@$reference" # Append the tenant id. fi @@ -146,7 +151,7 @@ fi # Note that we can't assume a real repository URL is specified if simulating # so trying to query the repository info is not a good idea. # -if [ -n "$simulate" ]; then +if [[ -n "$simulate" ]]; then run rm -r "$data_dir" trace "CI request for '$spec' is simulated$message_suffix" @@ -188,9 +193,9 @@ manifest_values=() manifest_version= more=true -while [ "$more" ]; do +while [[ "$more" ]]; do - if [ -n "$manifest_version" ]; then + if [[ -n "$manifest_version" ]]; then manifest_names=("") manifest_values=("$manifest_version") fi @@ -237,8 +242,8 @@ while [ "$more" ]; do packages_manifest_names+=("${manifest_names[@]}") packages_manifest_values+=("${manifest_values[@]}") - if [ -z "$display_name" ]; then - if [ -n "$project" ]; then + if [[ -z "$display_name" ]]; then + if [[ -n "$project" ]]; then display_name="$project" else display_name="$name" @@ -252,7 +257,7 @@ manifest_parser_finish # the repository. # for p in "${!packages[@]}"; do - if [ "${packages[$p]}" ]; then + if [[ "${packages[$p]}" ]]; then exit_with_manifest 422 "unknown package $p" fi done @@ -260,7 +265,7 @@ done # Verify that the repository is not empty. Failed that, the repository display # name wouldn't be set. # -if [ -z "$display_name" ]; then +if [[ -z "$display_name" ]]; then exit_with_manifest 422 "no packages in repository" fi @@ -272,7 +277,7 @@ run mv "$cache_dir/packages.manifest" "$cache_dir/packages.manifest.orig" # manifest_serializer_start "$cache_dir/packages.manifest" -for ((i=0; i <= ${#packages_manifest_names[@]}; ++i)); do +for ((i=0; i <= "${#packages_manifest_names[@]}"; ++i)); do manifest_serialize "${packages_manifest_names[$i]}" \ "${packages_manifest_values[$i]}" done @@ -286,7 +291,7 @@ run echo "$repository $display_name cache:cache" >"$loadtab" # Apply overrides, if uploaded. # -if [ -f "$data_dir/overrides.manifest" ]; then +if [[ -f "$data_dir/overrides.manifest" ]]; then loader_options+=(--overrides-file "$data_dir/overrides.manifest") fi @@ -295,6 +300,12 @@ fi # loader_options+=(--force --shallow --tenant "$reference") +# Build the packages interactively, if requested. +# +if [[ -n "$interactive" ]]; then + loader_options+=(--interactive "$interactive") +fi + run "$loader" "${loader_options[@]}" "$loadtab" # Remove the no longer needed CI request data directory. diff --git a/doc/manual.cli b/doc/manual.cli index 71a25a5..d0a1923 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -306,6 +306,15 @@ Check violations that are explicitly mentioned above are always reported with the CI result manifest. Other errors (for example, internal server errors) might be reported with unformatted text, including HTML. +If the CI request contains the \c{interactive} parameter, then the CI service +provides the execution environment login information for each test and stops +them at the specified breakpoint. + +Pre-defined breakpoint ids are \c{error} and \c{warning}. The breakpoint id is +included into the CI request manifest and the CI service must at least handle +\c{error} but may recognize additional ids (build phase/command identifiers, +etc). + If the CI request contains the \c{simulate} parameter, then the CI service simulates the specified outcome of the CI process without actually performing any externally visible actions (e.g., testing the package, publishing the @@ -328,8 +337,9 @@ corresponding to the custom request parameters. id: <request-id> repository: <url> [package]: <name>[/<version>] -timestamp: <date-time> +[interactive]: <breakpoint> [simulate]: <outcome> +timestamp: <date-time> [client-ip]: <string> [user-agent]: <string> \ diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index ddc5961..a0cea97 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -22,6 +22,8 @@ DROP FOREIGN TABLE IF EXISTS build_tenant; -- CREATE FOREIGN TABLE build_tenant ( id TEXT NOT NULL, + private BOOLEAN NOT NULL, + interactive TEXT NULL, archived BOOLEAN NOT NULL) SERVER package_server OPTIONS (table_name 'tenant'); diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 09ec41d..0cfd609 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -30,6 +30,8 @@ namespace brep public: string id; + bool private_; + optional<string> interactive; bool archived; // Database mapping. diff --git a/libbrep/build.cxx b/libbrep/build.cxx index db5bda2..5f8cd71 100644 --- a/libbrep/build.cxx +++ b/libbrep/build.cxx @@ -59,6 +59,7 @@ namespace brep version pvr, string cfg, string tnm, version tvr, + optional<string> inr, optional<string> afp, optional<string> ach, string mnm, string msm, butl::target_triplet trg) @@ -72,6 +73,7 @@ namespace brep toolchain_name (id.toolchain_name), toolchain_version (move (tvr)), state (build_state::building), + interactive (move (inr)), timestamp (timestamp_type::clock::now ()), force (force_state::unforced), agent_fingerprint (move (afp)), agent_challenge (move (ach)), diff --git a/libbrep/build.hxx b/libbrep/build.hxx index 380b17b..49105d1 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -25,7 +25,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 12 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 12, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 13, closed) // We have to keep these mappings at the global scope instead of inside // the brep namespace because they need to be also effective in the @@ -178,13 +178,14 @@ namespace brep using package_name_type = brep::package_name; // Create the build object with the building state, non-existent status, - // the timestamp set to now and the force state set to unforced. + // the timestamp set to now, and the force state set to unforced. // build (string tenant, package_name_type, version, string configuration, string toolchain_name, version toolchain_version, + optional<string> interactive, optional<string> agent_fingerprint, optional<string> agent_challenge, string machine, string machine_summary, @@ -201,6 +202,11 @@ namespace brep build_state state; + // If present, the login information for the interactive build. May be + // present only in the building state. + // + optional<string> interactive; + // Time of the last state change (the creation time initially). // timestamp_type timestamp; diff --git a/libbrep/build.xml b/libbrep/build.xml index 3af7640..31bb4de 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,10 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1"> + <changeset version="13"> + <alter-table name="build"> + <add-column name="interactive" type="TEXT" null="true"/> + </alter-table> + </changeset> + <model version="12"> <table name="build" kind="object"> <column name="package_tenant" type="TEXT" null="false"/> diff --git a/libbrep/package-extra.sql b/libbrep/package-extra.sql index fe936ff..5c04147 100644 --- a/libbrep/package-extra.sql +++ b/libbrep/package-extra.sql @@ -38,16 +38,17 @@ DROP TYPE IF EXISTS weighted_text CASCADE; CREATE TYPE weighted_text AS (a TEXT, b TEXT, c TEXT, d TEXT); -- Return the latest versions of matching a tenant internal packages as a set --- of package rows. If tenant is NULL, then match all tenants. +-- of package rows. If tenant is NULL, then match all public tenants. -- CREATE FUNCTION latest_packages(IN tenant TEXT) RETURNS SETOF package AS $$ SELECT p1.* - FROM package p1 LEFT JOIN package p2 ON ( + FROM package p1 + LEFT JOIN package p2 ON ( p1.internal_repository_canonical_name IS NOT NULL AND - p1.tenant = p2.tenant AND - p1.name = p2.name AND + p1.tenant = p2.tenant AND + p1.name = p2.name AND p2.internal_repository_canonical_name IS NOT NULL AND (p1.version_epoch < p2.version_epoch OR p1.version_epoch = p2.version_epoch AND @@ -56,8 +57,12 @@ RETURNS SETOF package AS $$ (p1.version_canonical_release < p2.version_canonical_release OR p1.version_canonical_release = p2.version_canonical_release AND p1.version_revision < p2.version_revision)))) + JOIN tenant t ON (p1.tenant = t.id) WHERE - (latest_packages.tenant IS NULL OR p1.tenant = latest_packages.tenant) AND + CASE + WHEN latest_packages.tenant IS NULL THEN NOT t.private + ELSE p1.tenant = latest_packages.tenant + END AND p1.internal_repository_canonical_name IS NOT NULL AND p2.name IS NULL; $$ LANGUAGE SQL STABLE; @@ -83,7 +88,8 @@ $$ LANGUAGE SQL STABLE; -- Search for the latest version of an internal packages matching the -- specified search query and tenant. Return a set of rows containing the -- package id and search rank. If query is NULL, then match all packages and --- return 0 rank for all rows. If tenant is NULL, then match all tenants. +-- return 0 rank for all rows. If tenant is NULL, then match all public +-- tenants. -- CREATE FUNCTION search_latest_packages(IN query tsquery, @@ -107,9 +113,9 @@ RETURNS SETOF record AS $$ $$ LANGUAGE SQL STABLE; -- Search for packages matching the search query and tenant and having the --- specified name. Return a set of rows containing the package id and search +-- specified name. Return a set of rows containing the package id and search -- rank. If query is NULL, then match all packages and return 0 rank for all --- rows. If tenant is NULL, then match all tenants. +-- rows. If tenant is NULL, then match all public tenants. -- CREATE FUNCTION search_packages(IN query tsquery, @@ -121,19 +127,22 @@ search_packages(IN query tsquery, OUT version_revision INTEGER, OUT rank real) RETURNS SETOF record AS $$ - SELECT tenant, name, version_epoch, version_canonical_upstream, - version_canonical_release, version_revision, + SELECT p.tenant, p.name, p.version_epoch, p.version_canonical_upstream, + p.version_canonical_release, p.version_revision, CASE WHEN query IS NULL THEN 0 -- Weight mapping: D C B A ELSE ts_rank_cd('{0.05, 0.2, 0.9, 1.0}', search_index, query) END AS rank - FROM package + FROM package p JOIN tenant t ON (p.tenant = t.id) WHERE - (search_packages.tenant IS NULL OR tenant = search_packages.tenant) AND - name = search_packages.name AND - internal_repository_canonical_name IS NOT NULL AND - (query IS NULL OR search_index @@ query); + CASE + WHEN search_packages.tenant IS NULL THEN NOT t.private + ELSE p.tenant = search_packages.tenant + END AND + name = search_packages.name AND + internal_repository_canonical_name IS NOT NULL AND + (query IS NULL OR search_index @@ query); $$ LANGUAGE SQL STABLE; -- Parse weighted_text to tsvector. diff --git a/libbrep/package.cxx b/libbrep/package.cxx index 564fec7..65fa1ba 100644 --- a/libbrep/package.cxx +++ b/libbrep/package.cxx @@ -40,8 +40,10 @@ namespace brep // tenant // tenant:: - tenant (string i) + tenant (string i, bool p, optional<string> r) : id (move (i)), + private_ (p), + interactive (move (r)), creation_timestamp (timestamp::clock::now ()) { } diff --git a/libbrep/package.hxx b/libbrep/package.hxx index 33444a9..1619185 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -20,7 +20,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 19 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 19, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 20, closed) namespace brep { @@ -226,16 +226,29 @@ namespace brep // flag set to false. // explicit - tenant (string id); + tenant (string id, bool private_, optional<string> interactive); string id; + // If true, display the packages in the web interface only in the tenant + // view mode. + // + bool private_; // Note: foreign-mapped in build. + + // Interactive package build breakpoint. + // + // If present, then packages from this tenant will only be built + // interactively and only non-interactively otherwise. + // + optional<string> interactive; // Note: foreign-mapped in build. + timestamp creation_timestamp; bool archived = false; // Note: foreign-mapped in build. // Database mapping. // #pragma db member(id) id + #pragma db member(private_) default(false) // @@ TMP private: friend class odb::access; diff --git a/libbrep/package.xml b/libbrep/package.xml index 454cdbc..01597c2 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,11 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1"> + <changeset version="20"> + <alter-table name="tenant"> + <add-column name="private" type="BOOLEAN" null="false" default="FALSE"/> + <add-column name="interactive" type="TEXT" null="true"/> + </alter-table> + </changeset> + <model version="19"> <table name="tenant" kind="object"> <column name="id" type="TEXT" null="false"/> diff --git a/load/buildfile b/load/buildfile index b55489f..4278f20 100644 --- a/load/buildfile +++ b/load/buildfile @@ -23,8 +23,8 @@ if $cli.configured cli.options += --std c++11 -I $src_root --include-with-brackets \ --include-prefix load --guard-prefix LOAD --generate-specifier \ ---cxx-prologue "#include <load/types-parsers.hxx>" --page-usage print_ \ ---ansi-color --long-usage +--generate-modifier --cxx-prologue "#include <load/types-parsers.hxx>" \ +--page-usage print_ --ansi-color --long-usage # Include the generated cli files into the distribution and don't remove # them when cleaning in src (so that clean results in a state identical to diff --git a/load/load.cli b/load/load.cli index 05bbb11..16b5f9f 100644 --- a/load/load.cli +++ b/load/load.cli @@ -64,6 +64,19 @@ class options specified, then the single-tenant mode is assumed." }; + bool --private + { + "Display the tenant packages in the web interface only in the tenant view + mode." + }; + + std::string --interactive + { + "<bkp>", + "Build the tenant packages interactively, stopping builds at the specified + breakpoint. Implies \cb{--private}." + }; + brep::path --overrides-file { "<file>", diff --git a/load/load.cxx b/load/load.cxx index 31230a7..0d53a0d 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -1474,6 +1474,11 @@ try throw failed (); } + // Note: the interactive tenant implies private. + // + if (ops.interactive_specified ()) + ops.private_ (true); + // Load the description of all the internal repositories from the // configuration file. // @@ -1511,7 +1516,11 @@ try // Persist the tenant. // - db.persist (tenant (tnt)); + db.persist (tenant (tnt, + ops.private_ (), + (ops.interactive_specified () + ? ops.interactive () + : optional<string> ()))); // On the first pass over the internal repositories we load their // certificate information and packages. diff --git a/mod/buildfile b/mod/buildfile index 191d966..ff9cd60 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -50,7 +50,7 @@ if $cli.configured cli.options += --std c++11 -I $src_root --include-with-brackets \ --include-prefix mod --guard-prefix MOD --generate-specifier \ --cxx-prologue "#include <mod/types-parsers.hxx>" \ ---cli-namespace brep::cli --generate-file-scanner --option-length 41 \ +--cli-namespace brep::cli --generate-file-scanner --option-length 45 \ --generate-modifier --generate-description --option-prefix "" # Include the generated cli files into the distribution and don't remove diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index 8a684fe..bec362a 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -398,6 +398,10 @@ handle (request& rq, response&) b->status = rqm.result.status; b->force = force_state::unforced; + // Cleanup the interactive build login information. + // + b->interactive = nullopt; + // Cleanup the authentication data. // b->agent_fingerprint = nullopt; diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 04b2a36..ebf434a 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -4,12 +4,14 @@ #include <mod/mod-build-task.hxx> #include <map> +#include <regex> #include <chrono> #include <odb/database.hxx> #include <odb/transaction.hxx> #include <odb/schema-catalog.hxx> +#include <libbutl/regex.mxx> #include <libbutl/sha256.mxx> #include <libbutl/utility.mxx> // compare_c_string #include <libbutl/openssl.mxx> @@ -195,11 +197,12 @@ handle (request& rq, response& rs) { vector<shared_ptr<build>> rebuilds; - // Create the task response manifest. The package must have the internal - // repository loaded. + // Create the task response manifest. Must be called inside the build db + // transaction. // auto task = [this] (shared_ptr<build>&& b, shared_ptr<build_package>&& p, + shared_ptr<build_tenant>&& t, const config_machine& cm) -> task_response_manifest { uint64_t ts ( @@ -218,7 +221,11 @@ handle (request& rq, response& rs) tenant_dir (options_->root (), b->tenant).string () + "?build-result"); - lazy_shared_ptr<build_repository> r (p->internal_repository); + assert (transaction::has_current ()); + + assert (p->internal ()); // The package is expected to be buildable. + + lazy_shared_ptr<build_repository> r (p->internal_repository.load ()); strings fps; if (r->certificate_fingerprint) @@ -265,7 +272,8 @@ handle (request& rq, response& rs) cm.config->target, cm.config->environment, cm.config->args, - cm.config->warning_regexes); + cm.config->warning_regexes, + move (t->interactive)); return task_response_manifest (move (session), move (b->agent_challenge), @@ -430,12 +438,66 @@ handle (request& rq, response& rs) pkg_query::build_repository::id.canonical_name.in_range (rp.begin (), rp.end ()); + // Transform (in-place) the interactive login information into the actual + // login command, if specified in the manifest and the transformation + // regexes are specified in the configuration. + // + if (tqm.interactive_login && + options_->build_interactive_login_specified ()) + { + optional<string> lc; + string l (tqm.agent + ' ' + *tqm.interactive_login); + + // Use the first matching regex for the transformation. + // + for (const pair<regex, string>& rf: options_->build_interactive_login ()) + { + pair<string, bool> r (regex_replace_match (l, rf.first, rf.second)); + + if (r.second) + { + lc = move (r.first); + break; + } + } + + if (!lc) + throw invalid_request (400, "unable to match login info '" + l + "'"); + + tqm.interactive_login = move (lc); + } + + // If the interactive mode if false or true, then filter out the + // respective packages. Otherwise, order them so that packages from the + // interactive build tenants appear first. + // + interactive_mode imode (tqm.effective_interactive_mode ()); + + switch (imode) + { + case interactive_mode::false_: + { + pq = pq && pkg_query::build_tenant::interactive.is_null (); + break; + } + case interactive_mode::true_: + { + pq = pq && pkg_query::build_tenant::interactive.is_not_null (); + break; + } + case interactive_mode::both: break; // See below. + } + // Specify the portion. // size_t offset (0); - pq += "ORDER BY" + - pkg_query::build_package::id.tenant + "," + + pq += "ORDER BY"; + + if (imode == interactive_mode::both) + pq += pkg_query::build_tenant::interactive + "NULLS LAST,"; + + pq += pkg_query::build_package::id.tenant + "," + pkg_query::build_package::id.name + order_by_version (pkg_query::build_package::id.version, false) + "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"; @@ -512,8 +574,8 @@ handle (request& rq, response& rs) // configurations that remained can be built. We will take the first // one, if present. // - // Also save the built package configurations for which it's time to be - // rebuilt. + // Also save the built package configurations for which it's time to + // be rebuilt. // config_machines configs (cfg_machines); // Make a copy for this pkg. auto pkg_builds (bld_prep_query.execute ()); @@ -567,6 +629,16 @@ handle (request& rq, response& rs) shared_ptr<build> b (build_db_->find<build> (bid)); optional<string> cl (challenge ()); + shared_ptr<build_tenant> t ( + build_db_->load<build_tenant> (bid.package.tenant)); + + // Move the interactive build login information into the build + // object, if the package to be built interactively. + // + optional<string> login (t->interactive + ? move (tqm.interactive_login) + : nullopt); + // If build configuration doesn't exist then create the new one // and persist. Otherwise put it into the building state, refresh // the timestamp and update. @@ -579,6 +651,7 @@ handle (request& rq, response& rs) move (bid.configuration), move (bid.toolchain_name), move (toolchain_version), + move (login), move (agent_fp), move (cl), mh.name, @@ -606,6 +679,7 @@ handle (request& rq, response& rs) b->results.empty ()); b->state = build_state::building; + b->interactive = move (login); // Switch the force state not to reissue the task after the // forced rebuild timeout. Note that the result handler will @@ -626,13 +700,7 @@ handle (request& rq, response& rs) // Finally, prepare the task response manifest. // - // We iterate over buildable packages. - // - assert (p->internal ()); - - p->internal_repository.load (); - - tsm = task (move (b), move (p), cm); + tsm = task (move (b), move (p), move (t), cm); } } @@ -704,20 +772,40 @@ handle (request& rq, response& rs) assert (i != cfg_machines.end ()); const config_machine& cm (i->second); - // Rebuild the package if still present, is buildable and doesn't - // exclude the configuration. + // Rebuild the package if still present, is buildable, doesn't + // exclude the configuration, and matches the request's + // interactive mode. + // + // Note that while change of the latter seems rather far fetched, + // let's check it for good measure. // shared_ptr<build_package> p ( build_db_->find<build_package> (b->id.package)); - if (p != nullptr && - p->internal () && + shared_ptr<build_tenant> t ( + p != nullptr + ? build_db_->load<build_tenant> (p->id.tenant) + : nullptr); + + if (p != nullptr && + p->buildable && + (t->interactive.has_value () == + (imode != interactive_mode::false_)) && !exclude (p->builds, p->constraints, *cm.config)) { assert (b->status); b->state = build_state::building; + // Save the interactive build login information into the build + // object, if the package to be built interactively. + // + // Can't move from, as may need it on the next iteration. + // + b->interactive = t->interactive + ? tqm.interactive_login + : nullopt; + // Can't move from, as may need them on the next iteration. // b->agent_fingerprint = agent_fp; @@ -738,9 +826,7 @@ handle (request& rq, response& rs) build_db_->update (b); - p->internal_repository.load (); - - tsm = task (move (b), move (p), cm); + tsm = task (move (b), move (p), move (t), cm); } } diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index ab9e93e..5ffe6dd 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -133,6 +133,8 @@ match (const C qc, const string& pattern) return qc + "SIMILAR TO" + query<T>::_val (transform (pattern)); } +// If tenant is absent, then query builds from all the public tenants. +// template <typename T> static inline query<T> build_query (const brep::cstrings* configs, @@ -143,18 +145,18 @@ build_query (const brep::cstrings* configs, using namespace brep; using query = query<T>; using qb = typename query::build; - - query q (configs != nullptr - ? qb::id.configuration.in_range (configs->begin (), configs->end ()) - : query (true)); + using qt = typename query::build_tenant; const auto& pid (qb::id.package); - if (tenant) - q = q && pid.tenant == *tenant; + query q (tenant ? pid.tenant == *tenant : !qt::private_); if (archived) - q = q && query::build_tenant::archived == *archived; + q = q && qt::archived == *archived; + + if (configs != nullptr) + q = q && qb::id.configuration.in_range (configs->begin (), + configs->end ()); // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no package builds match such a @@ -267,6 +269,8 @@ build_query (const brep::cstrings* configs, return q; } +// If tenant is absent, then query packages from all the public tenants. +// template <typename T> static inline query<T> package_query (const brep::params::builds& params, @@ -276,14 +280,12 @@ package_query (const brep::params::builds& params, using namespace brep; using query = query<T>; using qp = typename query::build_package; + using qt = typename query::build_tenant; - query q (true); - - if (tenant) - q = q && qp::id.tenant == *tenant; + query q (tenant ? qp::id.tenant == *tenant : !qt::private_); if (archived) - q = q && query::build_tenant::archived == *archived; + q = q && qt::archived == *archived; // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no packages match such a query. @@ -383,7 +385,7 @@ handle (request& rq, response& rs) << DIV(ID="content"); // If the tenant is empty then we are in the global view and will display - // builds from all the tenants. + // builds from all the public tenants. // optional<string> tn; if (!tenant.empty ()) @@ -672,14 +674,18 @@ handle (request& rq, response& rs) << TR_VALUE ("config", b.configuration) << TR_VALUE ("machine", b.machine) << TR_VALUE ("target", b.target.string ()) - << TR_VALUE ("timestamp", ts) - << TR_BUILD_RESULT (b, host, root); + << TR_VALUE ("timestamp", ts); + + if (b.interactive) // Note: can only be present for the building state. + s << TR_VALUE ("login", *b.interactive); + + s << TR_BUILD_RESULT (b, host, root); // In the global view mode add the tenant builds link. Note that the // global view (and the link) makes sense only in the multi-tenant mode. // if (!tn && !b.tenant.empty ()) - s << TR_TENANT (tenant_name, "builds", root, b.tenant); + s << TR_TENANT (tenant_name, "builds", root, b.tenant); s << ~TBODY << ~TABLE; @@ -809,15 +815,20 @@ handle (request& rq, response& rs) const auto& bid (bld_query::build::id); - bld_query bq (equal<package_build_count> (bid.package, id) && - bid.configuration == bld_query::_ref (config) && + bld_query bq ( + equal<package_build_count> (bid.package, id) && + bid.configuration == bld_query::_ref (config) && // Note that the query already constrains configurations via the - // configuration name and the tenant via the build package id. + // configuration name. + // + // Also note that while the query already constrains the tenant via + // the build package id, we still need to pass the tenant not to + // erroneously filter out the private tenants. // build_query<package_build_count> (nullptr /* configs */, bld_params, - nullopt /* tenant */, + tn, false /* archived */)); prep_bld_query bld_prep_query ( @@ -925,11 +936,12 @@ handle (request& rq, response& rs) bld_query bq ( equal<package_build> (bld_query::build::id.package, id) && - // Note that the query already constrains the tenant via the build - // package id. + // Note that while the query already constrains the tenant via the build + // package id, we still need to pass the tenant not to erroneously + // filter out the private tenants. // build_query<package_build> ( - &conf_names, bld_params, nullopt /* tenant */, false /* archived */)); + &conf_names, bld_params, tn, false /* archived */)); prep_bld_query bld_prep_query ( conn->prepare_query<package_build> ("mod-builds-build-query", bq)); diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx index d2da93f..4da72b6 100644 --- a/mod/mod-ci.cxx +++ b/mod/mod-ci.cxx @@ -376,15 +376,18 @@ handle (request& rq, response& rs) s.next ("package", p); } + if (params.interactive_specified ()) + s.next ("interactive", params.interactive ()); + + if (!simulate.empty ()) + s.next ("simulate", simulate); + s.next ("timestamp", butl::to_string (ts, "%Y-%m-%dT%H:%M:%SZ", false /* special */, false /* local */)); - if (!simulate.empty ()) - s.next ("simulate", simulate); - // Serialize the User-Agent HTTP header and the client IP address. // optional<string> ip; @@ -412,10 +415,11 @@ handle (request& rq, response& rs) { const string& n (nv.name); - if (n != "repository" && - n != "_" && - n != "package" && - n != "overrides" && + if (n != "repository" && + n != "_" && + n != "package" && + n != "overrides" && + n != "interactive" && n != "simulate") s.next (n, nv.value ? *nv.value : ""); } diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index a7682ec..a23989e 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -472,7 +472,7 @@ handle (request& rq, response& rs) } } - bool archived (package_db_->load<brep::tenant> (tenant)->archived); + shared_ptr<brep::tenant> tn (package_db_->load<brep::tenant> (tenant)); t.commit (); @@ -504,7 +504,7 @@ handle (request& rq, response& rs) // vector<pair<string, version>> toolchains; - if (!archived) + if (!tn->archived) { using query = query<toolchain>; @@ -568,8 +568,12 @@ handle (request& rq, response& rs) b.toolchain_version.string ()) << TR_VALUE ("config", b.configuration + " / " + b.target.string ()) - << TR_VALUE ("timestamp", ts) - << TR_BUILD_RESULT (b, host, root) + << TR_VALUE ("timestamp", ts); + + if (b.interactive) // Note: can only be present for the building state. + s << TR_VALUE ("login", *b.interactive); + + s << TR_BUILD_RESULT (b, host, root) << ~TBODY << ~TABLE; @@ -606,22 +610,27 @@ handle (request& rq, response& rs) << ~TABLE; } - // Print the package build exclusions that belong to the 'default' class. + // Print the package build exclusions that belong to the 'default' class, + // unless the package is built interactively (normally for a single + // configuration). // - for (const auto& c: *build_conf_) + if (!tn->interactive) { - string reason; - if (belongs (c, "default") && exclude (c, &reason)) + for (const auto& c: *build_conf_) { - s << TABLE(CLASS="proplist build") - << TBODY - << TR_VALUE ("config", c.name + " / " + c.target.string ()) - << TR_VALUE ("result", - !reason.empty () - ? "excluded (" + reason + ')' - : "excluded") - << ~TBODY - << ~TABLE; + string reason; + if (belongs (c, "default") && exclude (c, &reason)) + { + s << TABLE(CLASS="proplist build") + << TBODY + << TR_VALUE ("config", c.name + " / " + c.target.string ()) + << TR_VALUE ("result", + !reason.empty () + ? "excluded (" + reason + ')' + : "excluded") + << ~TBODY + << ~TABLE; + } } } diff --git a/mod/mod-packages.cxx b/mod/mod-packages.cxx index 65c7c5b..222b817 100644 --- a/mod/mod-packages.cxx +++ b/mod/mod-packages.cxx @@ -49,8 +49,8 @@ init (scanner& s) options_->root (dir_path ("/")); // Check that the database 'package' schema matches the current one. It's - // enough to perform the check in just a single module implementation (and we - // don't do in the dispatcher because it doesn't use the database). + // enough to perform the check in just a single module implementation (and + // we don't do in the dispatcher because it doesn't use the database). // // Note that the failure can be reported by each web server worker process. // While it could be tempting to move the check to the @@ -137,7 +137,7 @@ handle (request& rq, response& rs) << DIV(ID="content"); // If the tenant is empty then we are in the global view and will display - // packages from all the tenants. + // packages from all the public tenants. // optional<string> tn; if (!tenant.empty ()) diff --git a/mod/module.cli b/mod/module.cli index b59158a..3ba6ecd 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -1,6 +1,8 @@ // file : mod/options.cli -*- C++ -*- // license : MIT; see accompanying LICENSE file +include <regex>; + include <libbpkg/manifest.hxx>; // repository_location include <web/xhtml/fragment.hxx>; @@ -307,7 +309,7 @@ namespace brep edge. The value is treated as an XHTML5 fragment." } - vector<page_menu> menu; + vector<page_menu> menu { "<label=link>", "Web page menu. Each entry is displayed in the page header in the @@ -341,7 +343,7 @@ namespace brep The default is 500 (~ 80 characters * 6 lines)." } - uint16_t package-changes = 5000; + uint16_t package-changes = 5000 { "<len>", "Number of package changes characters to display in brief pages. The @@ -394,6 +396,18 @@ namespace brep "Time to wait before considering the expected task result lost. Must be specified in seconds. The default is 3 hours." } + + vector<pair<std::regex, string>> build-interactive-login + { + "</regex/replacement/>", + "Regular expressions for transforming the interactive build login + information, for example, into the actual command that can be used + by the user. The regular expressions are matched against the + \"<agent>\ <interactive-login>\" string containing the respective + task request manifest values. The first matching expression is used + for the transformation. If no expression matches, then the task + request is considered invalid, unless no expressions are specified." + } }; class build_result: build, package_db, build_db, handler @@ -837,6 +851,10 @@ namespace brep // string overrides; + // Interactive build execution breakpoint. + // + string interactive; + // Submission simulation outcome. // string simulate; diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx index dc21e97..422b353 100644 --- a/mod/types-parsers.cxx +++ b/mod/types-parsers.cxx @@ -3,11 +3,15 @@ #include <mod/types-parsers.hxx> +#include <sstream> + +#include <libbutl/regex.mxx> #include <libbutl/timestamp.mxx> // from_string() #include <mod/module-options.hxx> using namespace std; +using namespace butl; using namespace bpkg; using namespace web::xhtml; @@ -75,9 +79,9 @@ namespace brep string t ("1970-01-01 "); t += v; - x = butl::from_string (t.c_str (), - "%Y-%m-%d %H:%M", - false /* local */).time_since_epoch (); + x = from_string (t.c_str (), + "%Y-%m-%d %H:%M", + false /* local */).time_since_epoch (); return; } catch (const invalid_argument&) {} @@ -181,5 +185,37 @@ namespace brep throw invalid_value (o, v); } } + + // Parse the '/regex/replacement/' string into the regex/replacement pair. + // + void parser<pair<std::regex, string>>:: + parse (pair<std::regex, string>& x, bool& xs, scanner& s) + { + xs = true; + const char* o (s.next ()); + + if (!s.more ()) + throw missing_value (o); + + const char* v (s.next ()); + + try + { + x = regex_replace_parse (v); + } + catch (const invalid_argument& e) + { + throw invalid_value (o, v, e.what ()); + } + catch (const regex_error& e) + { + // Sanitize the description. + // + ostringstream os; + os << e; + + throw invalid_value (o, v, os.str ()); + } + } } } diff --git a/mod/types-parsers.hxx b/mod/types-parsers.hxx index 6b851eb..05a7263 100644 --- a/mod/types-parsers.hxx +++ b/mod/types-parsers.hxx @@ -7,6 +7,8 @@ #ifndef MOD_TYPES_PARSERS_HXX #define MOD_TYPES_PARSERS_HXX +#include <regex> + #include <libbpkg/manifest.hxx> // repository_location #include <web/xhtml/fragment.hxx> @@ -75,6 +77,13 @@ namespace brep static void parse (web::xhtml::fragment&, bool&, scanner&); }; + + template <> + struct parser<pair<std::regex, string>> + { + static void + parse (pair<std::regex, string>&, bool&, scanner&); + }; } } diff --git a/www/ci.xhtml b/www/ci.xhtml index 185f08b..573cca7 100644 --- a/www/ci.xhtml +++ b/www/ci.xhtml @@ -13,6 +13,10 @@ <th>package</th> <td><input type="text" name="package"/></td> </tr> + <tr> + <th>interactive</th> + <td><input type="text" name="interactive"/></td> + </tr> </tbody> </table> <table class="form-table"> |