aboutsummaryrefslogtreecommitdiff
path: root/mod
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2021-03-13 16:09:48 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2021-03-26 20:41:21 +0300
commit6ccee38f43493f8f6e87bab549e9ef952244f39a (patch)
tree3c75f102175fc6a566234e904a818dcd74029755 /mod
parentf7327a0b3cd6723cb289819bad1d664cfd5d6618 (diff)
Add support for interactive CI mode
Diffstat (limited to 'mod')
-rw-r--r--mod/buildfile2
-rw-r--r--mod/mod-build-result.cxx4
-rw-r--r--mod/mod-build-task.cxx130
-rw-r--r--mod/mod-builds.cxx58
-rw-r--r--mod/mod-ci.cxx18
-rw-r--r--mod/mod-package-version-details.cxx43
-rw-r--r--mod/mod-packages.cxx6
-rw-r--r--mod/module.cli22
-rw-r--r--mod/types-parsers.cxx42
-rw-r--r--mod/types-parsers.hxx9
10 files changed, 256 insertions, 78 deletions
diff --git a/mod/buildfile b/mod/buildfile
index 191d966..ff9cd60 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -50,7 +50,7 @@ if $cli.configured
cli.options += --std c++11 -I $src_root --include-with-brackets \
--include-prefix mod --guard-prefix MOD --generate-specifier \
--cxx-prologue "#include <mod/types-parsers.hxx>" \
---cli-namespace brep::cli --generate-file-scanner --option-length 41 \
+--cli-namespace brep::cli --generate-file-scanner --option-length 45 \
--generate-modifier --generate-description --option-prefix ""
# Include the generated cli files into the distribution and don't remove
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 8a684fe..bec362a 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -398,6 +398,10 @@ handle (request& rq, response&)
b->status = rqm.result.status;
b->force = force_state::unforced;
+ // Cleanup the interactive build login information.
+ //
+ b->interactive = nullopt;
+
// Cleanup the authentication data.
//
b->agent_fingerprint = nullopt;
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 04b2a36..ebf434a 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -4,12 +4,14 @@
#include <mod/mod-build-task.hxx>
#include <map>
+#include <regex>
#include <chrono>
#include <odb/database.hxx>
#include <odb/transaction.hxx>
#include <odb/schema-catalog.hxx>
+#include <libbutl/regex.mxx>
#include <libbutl/sha256.mxx>
#include <libbutl/utility.mxx> // compare_c_string
#include <libbutl/openssl.mxx>
@@ -195,11 +197,12 @@ handle (request& rq, response& rs)
{
vector<shared_ptr<build>> rebuilds;
- // Create the task response manifest. The package must have the internal
- // repository loaded.
+ // Create the task response manifest. Must be called inside the build db
+ // transaction.
//
auto task = [this] (shared_ptr<build>&& b,
shared_ptr<build_package>&& p,
+ shared_ptr<build_tenant>&& t,
const config_machine& cm) -> task_response_manifest
{
uint64_t ts (
@@ -218,7 +221,11 @@ handle (request& rq, response& rs)
tenant_dir (options_->root (), b->tenant).string () +
"?build-result");
- lazy_shared_ptr<build_repository> r (p->internal_repository);
+ assert (transaction::has_current ());
+
+ assert (p->internal ()); // The package is expected to be buildable.
+
+ lazy_shared_ptr<build_repository> r (p->internal_repository.load ());
strings fps;
if (r->certificate_fingerprint)
@@ -265,7 +272,8 @@ handle (request& rq, response& rs)
cm.config->target,
cm.config->environment,
cm.config->args,
- cm.config->warning_regexes);
+ cm.config->warning_regexes,
+ move (t->interactive));
return task_response_manifest (move (session),
move (b->agent_challenge),
@@ -430,12 +438,66 @@ handle (request& rq, response& rs)
pkg_query::build_repository::id.canonical_name.in_range (rp.begin (),
rp.end ());
+ // Transform (in-place) the interactive login information into the actual
+ // login command, if specified in the manifest and the transformation
+ // regexes are specified in the configuration.
+ //
+ if (tqm.interactive_login &&
+ options_->build_interactive_login_specified ())
+ {
+ optional<string> lc;
+ string l (tqm.agent + ' ' + *tqm.interactive_login);
+
+ // Use the first matching regex for the transformation.
+ //
+ for (const pair<regex, string>& rf: options_->build_interactive_login ())
+ {
+ pair<string, bool> r (regex_replace_match (l, rf.first, rf.second));
+
+ if (r.second)
+ {
+ lc = move (r.first);
+ break;
+ }
+ }
+
+ if (!lc)
+ throw invalid_request (400, "unable to match login info '" + l + "'");
+
+ tqm.interactive_login = move (lc);
+ }
+
+ // If the interactive mode if false or true, then filter out the
+ // respective packages. Otherwise, order them so that packages from the
+ // interactive build tenants appear first.
+ //
+ interactive_mode imode (tqm.effective_interactive_mode ());
+
+ switch (imode)
+ {
+ case interactive_mode::false_:
+ {
+ pq = pq && pkg_query::build_tenant::interactive.is_null ();
+ break;
+ }
+ case interactive_mode::true_:
+ {
+ pq = pq && pkg_query::build_tenant::interactive.is_not_null ();
+ break;
+ }
+ case interactive_mode::both: break; // See below.
+ }
+
// Specify the portion.
//
size_t offset (0);
- pq += "ORDER BY" +
- pkg_query::build_package::id.tenant + "," +
+ pq += "ORDER BY";
+
+ if (imode == interactive_mode::both)
+ pq += pkg_query::build_tenant::interactive + "NULLS LAST,";
+
+ pq += pkg_query::build_package::id.tenant + "," +
pkg_query::build_package::id.name +
order_by_version (pkg_query::build_package::id.version, false) +
"OFFSET" + pkg_query::_ref (offset) + "LIMIT 50";
@@ -512,8 +574,8 @@ handle (request& rq, response& rs)
// configurations that remained can be built. We will take the first
// one, if present.
//
- // Also save the built package configurations for which it's time to be
- // rebuilt.
+ // Also save the built package configurations for which it's time to
+ // be rebuilt.
//
config_machines configs (cfg_machines); // Make a copy for this pkg.
auto pkg_builds (bld_prep_query.execute ());
@@ -567,6 +629,16 @@ handle (request& rq, response& rs)
shared_ptr<build> b (build_db_->find<build> (bid));
optional<string> cl (challenge ());
+ shared_ptr<build_tenant> t (
+ build_db_->load<build_tenant> (bid.package.tenant));
+
+ // Move the interactive build login information into the build
+ // object, if the package to be built interactively.
+ //
+ optional<string> login (t->interactive
+ ? move (tqm.interactive_login)
+ : nullopt);
+
// If build configuration doesn't exist then create the new one
// and persist. Otherwise put it into the building state, refresh
// the timestamp and update.
@@ -579,6 +651,7 @@ handle (request& rq, response& rs)
move (bid.configuration),
move (bid.toolchain_name),
move (toolchain_version),
+ move (login),
move (agent_fp),
move (cl),
mh.name,
@@ -606,6 +679,7 @@ handle (request& rq, response& rs)
b->results.empty ());
b->state = build_state::building;
+ b->interactive = move (login);
// Switch the force state not to reissue the task after the
// forced rebuild timeout. Note that the result handler will
@@ -626,13 +700,7 @@ handle (request& rq, response& rs)
// Finally, prepare the task response manifest.
//
- // We iterate over buildable packages.
- //
- assert (p->internal ());
-
- p->internal_repository.load ();
-
- tsm = task (move (b), move (p), cm);
+ tsm = task (move (b), move (p), move (t), cm);
}
}
@@ -704,20 +772,40 @@ handle (request& rq, response& rs)
assert (i != cfg_machines.end ());
const config_machine& cm (i->second);
- // Rebuild the package if still present, is buildable and doesn't
- // exclude the configuration.
+ // Rebuild the package if still present, is buildable, doesn't
+ // exclude the configuration, and matches the request's
+ // interactive mode.
+ //
+ // Note that while change of the latter seems rather far fetched,
+ // let's check it for good measure.
//
shared_ptr<build_package> p (
build_db_->find<build_package> (b->id.package));
- if (p != nullptr &&
- p->internal () &&
+ shared_ptr<build_tenant> t (
+ p != nullptr
+ ? build_db_->load<build_tenant> (p->id.tenant)
+ : nullptr);
+
+ if (p != nullptr &&
+ p->buildable &&
+ (t->interactive.has_value () ==
+ (imode != interactive_mode::false_)) &&
!exclude (p->builds, p->constraints, *cm.config))
{
assert (b->status);
b->state = build_state::building;
+ // Save the interactive build login information into the build
+ // object, if the package to be built interactively.
+ //
+ // Can't move from, as may need it on the next iteration.
+ //
+ b->interactive = t->interactive
+ ? tqm.interactive_login
+ : nullopt;
+
// Can't move from, as may need them on the next iteration.
//
b->agent_fingerprint = agent_fp;
@@ -738,9 +826,7 @@ handle (request& rq, response& rs)
build_db_->update (b);
- p->internal_repository.load ();
-
- tsm = task (move (b), move (p), cm);
+ tsm = task (move (b), move (p), move (t), cm);
}
}
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index ab9e93e..5ffe6dd 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -133,6 +133,8 @@ match (const C qc, const string& pattern)
return qc + "SIMILAR TO" + query<T>::_val (transform (pattern));
}
+// If tenant is absent, then query builds from all the public tenants.
+//
template <typename T>
static inline query<T>
build_query (const brep::cstrings* configs,
@@ -143,18 +145,18 @@ build_query (const brep::cstrings* configs,
using namespace brep;
using query = query<T>;
using qb = typename query::build;
-
- query q (configs != nullptr
- ? qb::id.configuration.in_range (configs->begin (), configs->end ())
- : query (true));
+ using qt = typename query::build_tenant;
const auto& pid (qb::id.package);
- if (tenant)
- q = q && pid.tenant == *tenant;
+ query q (tenant ? pid.tenant == *tenant : !qt::private_);
if (archived)
- q = q && query::build_tenant::archived == *archived;
+ q = q && qt::archived == *archived;
+
+ if (configs != nullptr)
+ q = q && qb::id.configuration.in_range (configs->begin (),
+ configs->end ());
// Note that there is no error reported if the filter parameters parsing
// fails. Instead, it is considered that no package builds match such a
@@ -267,6 +269,8 @@ build_query (const brep::cstrings* configs,
return q;
}
+// If tenant is absent, then query packages from all the public tenants.
+//
template <typename T>
static inline query<T>
package_query (const brep::params::builds& params,
@@ -276,14 +280,12 @@ package_query (const brep::params::builds& params,
using namespace brep;
using query = query<T>;
using qp = typename query::build_package;
+ using qt = typename query::build_tenant;
- query q (true);
-
- if (tenant)
- q = q && qp::id.tenant == *tenant;
+ query q (tenant ? qp::id.tenant == *tenant : !qt::private_);
if (archived)
- q = q && query::build_tenant::archived == *archived;
+ q = q && qt::archived == *archived;
// Note that there is no error reported if the filter parameters parsing
// fails. Instead, it is considered that no packages match such a query.
@@ -383,7 +385,7 @@ handle (request& rq, response& rs)
<< DIV(ID="content");
// If the tenant is empty then we are in the global view and will display
- // builds from all the tenants.
+ // builds from all the public tenants.
//
optional<string> tn;
if (!tenant.empty ())
@@ -672,14 +674,18 @@ handle (request& rq, response& rs)
<< TR_VALUE ("config", b.configuration)
<< TR_VALUE ("machine", b.machine)
<< TR_VALUE ("target", b.target.string ())
- << TR_VALUE ("timestamp", ts)
- << TR_BUILD_RESULT (b, host, root);
+ << TR_VALUE ("timestamp", ts);
+
+ if (b.interactive) // Note: can only be present for the building state.
+ s << TR_VALUE ("login", *b.interactive);
+
+ s << TR_BUILD_RESULT (b, host, root);
// In the global view mode add the tenant builds link. Note that the
// global view (and the link) makes sense only in the multi-tenant mode.
//
if (!tn && !b.tenant.empty ())
- s << TR_TENANT (tenant_name, "builds", root, b.tenant);
+ s << TR_TENANT (tenant_name, "builds", root, b.tenant);
s << ~TBODY
<< ~TABLE;
@@ -809,15 +815,20 @@ handle (request& rq, response& rs)
const auto& bid (bld_query::build::id);
- bld_query bq (equal<package_build_count> (bid.package, id) &&
- bid.configuration == bld_query::_ref (config) &&
+ bld_query bq (
+ equal<package_build_count> (bid.package, id) &&
+ bid.configuration == bld_query::_ref (config) &&
// Note that the query already constrains configurations via the
- // configuration name and the tenant via the build package id.
+ // configuration name.
+ //
+ // Also note that while the query already constrains the tenant via
+ // the build package id, we still need to pass the tenant not to
+ // erroneously filter out the private tenants.
//
build_query<package_build_count> (nullptr /* configs */,
bld_params,
- nullopt /* tenant */,
+ tn,
false /* archived */));
prep_bld_query bld_prep_query (
@@ -925,11 +936,12 @@ handle (request& rq, response& rs)
bld_query bq (
equal<package_build> (bld_query::build::id.package, id) &&
- // Note that the query already constrains the tenant via the build
- // package id.
+ // Note that while the query already constrains the tenant via the build
+ // package id, we still need to pass the tenant not to erroneously
+ // filter out the private tenants.
//
build_query<package_build> (
- &conf_names, bld_params, nullopt /* tenant */, false /* archived */));
+ &conf_names, bld_params, tn, false /* archived */));
prep_bld_query bld_prep_query (
conn->prepare_query<package_build> ("mod-builds-build-query", bq));
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
index d2da93f..4da72b6 100644
--- a/mod/mod-ci.cxx
+++ b/mod/mod-ci.cxx
@@ -376,15 +376,18 @@ handle (request& rq, response& rs)
s.next ("package", p);
}
+ if (params.interactive_specified ())
+ s.next ("interactive", params.interactive ());
+
+ if (!simulate.empty ())
+ s.next ("simulate", simulate);
+
s.next ("timestamp",
butl::to_string (ts,
"%Y-%m-%dT%H:%M:%SZ",
false /* special */,
false /* local */));
- if (!simulate.empty ())
- s.next ("simulate", simulate);
-
// Serialize the User-Agent HTTP header and the client IP address.
//
optional<string> ip;
@@ -412,10 +415,11 @@ handle (request& rq, response& rs)
{
const string& n (nv.name);
- if (n != "repository" &&
- n != "_" &&
- n != "package" &&
- n != "overrides" &&
+ if (n != "repository" &&
+ n != "_" &&
+ n != "package" &&
+ n != "overrides" &&
+ n != "interactive" &&
n != "simulate")
s.next (n, nv.value ? *nv.value : "");
}
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index a7682ec..a23989e 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -472,7 +472,7 @@ handle (request& rq, response& rs)
}
}
- bool archived (package_db_->load<brep::tenant> (tenant)->archived);
+ shared_ptr<brep::tenant> tn (package_db_->load<brep::tenant> (tenant));
t.commit ();
@@ -504,7 +504,7 @@ handle (request& rq, response& rs)
//
vector<pair<string, version>> toolchains;
- if (!archived)
+ if (!tn->archived)
{
using query = query<toolchain>;
@@ -568,8 +568,12 @@ handle (request& rq, response& rs)
b.toolchain_version.string ())
<< TR_VALUE ("config",
b.configuration + " / " + b.target.string ())
- << TR_VALUE ("timestamp", ts)
- << TR_BUILD_RESULT (b, host, root)
+ << TR_VALUE ("timestamp", ts);
+
+ if (b.interactive) // Note: can only be present for the building state.
+ s << TR_VALUE ("login", *b.interactive);
+
+ s << TR_BUILD_RESULT (b, host, root)
<< ~TBODY
<< ~TABLE;
@@ -606,22 +610,27 @@ handle (request& rq, response& rs)
<< ~TABLE;
}
- // Print the package build exclusions that belong to the 'default' class.
+ // Print the package build exclusions that belong to the 'default' class,
+ // unless the package is built interactively (normally for a single
+ // configuration).
//
- for (const auto& c: *build_conf_)
+ if (!tn->interactive)
{
- string reason;
- if (belongs (c, "default") && exclude (c, &reason))
+ for (const auto& c: *build_conf_)
{
- s << TABLE(CLASS="proplist build")
- << TBODY
- << TR_VALUE ("config", c.name + " / " + c.target.string ())
- << TR_VALUE ("result",
- !reason.empty ()
- ? "excluded (" + reason + ')'
- : "excluded")
- << ~TBODY
- << ~TABLE;
+ string reason;
+ if (belongs (c, "default") && exclude (c, &reason))
+ {
+ s << TABLE(CLASS="proplist build")
+ << TBODY
+ << TR_VALUE ("config", c.name + " / " + c.target.string ())
+ << TR_VALUE ("result",
+ !reason.empty ()
+ ? "excluded (" + reason + ')'
+ : "excluded")
+ << ~TBODY
+ << ~TABLE;
+ }
}
}
diff --git a/mod/mod-packages.cxx b/mod/mod-packages.cxx
index 65c7c5b..222b817 100644
--- a/mod/mod-packages.cxx
+++ b/mod/mod-packages.cxx
@@ -49,8 +49,8 @@ init (scanner& s)
options_->root (dir_path ("/"));
// Check that the database 'package' schema matches the current one. It's
- // enough to perform the check in just a single module implementation (and we
- // don't do in the dispatcher because it doesn't use the database).
+ // enough to perform the check in just a single module implementation (and
+ // we don't do in the dispatcher because it doesn't use the database).
//
// Note that the failure can be reported by each web server worker process.
// While it could be tempting to move the check to the
@@ -137,7 +137,7 @@ handle (request& rq, response& rs)
<< DIV(ID="content");
// If the tenant is empty then we are in the global view and will display
- // packages from all the tenants.
+ // packages from all the public tenants.
//
optional<string> tn;
if (!tenant.empty ())
diff --git a/mod/module.cli b/mod/module.cli
index b59158a..3ba6ecd 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -1,6 +1,8 @@
// file : mod/options.cli -*- C++ -*-
// license : MIT; see accompanying LICENSE file
+include <regex>;
+
include <libbpkg/manifest.hxx>; // repository_location
include <web/xhtml/fragment.hxx>;
@@ -307,7 +309,7 @@ namespace brep
edge. The value is treated as an XHTML5 fragment."
}
- vector<page_menu> menu;
+ vector<page_menu> menu
{
"<label=link>",
"Web page menu. Each entry is displayed in the page header in the
@@ -341,7 +343,7 @@ namespace brep
The default is 500 (~ 80 characters * 6 lines)."
}
- uint16_t package-changes = 5000;
+ uint16_t package-changes = 5000
{
"<len>",
"Number of package changes characters to display in brief pages. The
@@ -394,6 +396,18 @@ namespace brep
"Time to wait before considering the expected task result lost. Must be
specified in seconds. The default is 3 hours."
}
+
+ vector<pair<std::regex, string>> build-interactive-login
+ {
+ "</regex/replacement/>",
+ "Regular expressions for transforming the interactive build login
+ information, for example, into the actual command that can be used
+ by the user. The regular expressions are matched against the
+ \"<agent>\ <interactive-login>\" string containing the respective
+ task request manifest values. The first matching expression is used
+ for the transformation. If no expression matches, then the task
+ request is considered invalid, unless no expressions are specified."
+ }
};
class build_result: build, package_db, build_db, handler
@@ -837,6 +851,10 @@ namespace brep
//
string overrides;
+ // Interactive build execution breakpoint.
+ //
+ string interactive;
+
// Submission simulation outcome.
//
string simulate;
diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx
index dc21e97..422b353 100644
--- a/mod/types-parsers.cxx
+++ b/mod/types-parsers.cxx
@@ -3,11 +3,15 @@
#include <mod/types-parsers.hxx>
+#include <sstream>
+
+#include <libbutl/regex.mxx>
#include <libbutl/timestamp.mxx> // from_string()
#include <mod/module-options.hxx>
using namespace std;
+using namespace butl;
using namespace bpkg;
using namespace web::xhtml;
@@ -75,9 +79,9 @@ namespace brep
string t ("1970-01-01 ");
t += v;
- x = butl::from_string (t.c_str (),
- "%Y-%m-%d %H:%M",
- false /* local */).time_since_epoch ();
+ x = from_string (t.c_str (),
+ "%Y-%m-%d %H:%M",
+ false /* local */).time_since_epoch ();
return;
}
catch (const invalid_argument&) {}
@@ -181,5 +185,37 @@ namespace brep
throw invalid_value (o, v);
}
}
+
+ // Parse the '/regex/replacement/' string into the regex/replacement pair.
+ //
+ void parser<pair<std::regex, string>>::
+ parse (pair<std::regex, string>& x, bool& xs, scanner& s)
+ {
+ xs = true;
+ const char* o (s.next ());
+
+ if (!s.more ())
+ throw missing_value (o);
+
+ const char* v (s.next ());
+
+ try
+ {
+ x = regex_replace_parse (v);
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_value (o, v, e.what ());
+ }
+ catch (const regex_error& e)
+ {
+ // Sanitize the description.
+ //
+ ostringstream os;
+ os << e;
+
+ throw invalid_value (o, v, os.str ());
+ }
+ }
}
}
diff --git a/mod/types-parsers.hxx b/mod/types-parsers.hxx
index 6b851eb..05a7263 100644
--- a/mod/types-parsers.hxx
+++ b/mod/types-parsers.hxx
@@ -7,6 +7,8 @@
#ifndef MOD_TYPES_PARSERS_HXX
#define MOD_TYPES_PARSERS_HXX
+#include <regex>
+
#include <libbpkg/manifest.hxx> // repository_location
#include <web/xhtml/fragment.hxx>
@@ -75,6 +77,13 @@ namespace brep
static void
parse (web::xhtml::fragment&, bool&, scanner&);
};
+
+ template <>
+ struct parser<pair<std::regex, string>>
+ {
+ static void
+ parse (pair<std::regex, string>&, bool&, scanner&);
+ };
}
}