diff options
Diffstat (limited to 'mod')
45 files changed, 5948 insertions, 1815 deletions
diff --git a/mod/build-config-module.cxx b/mod/build-config-module.cxx index eba753e..97c9f9e 100644 --- a/mod/build-config-module.cxx +++ b/mod/build-config-module.cxx @@ -71,7 +71,7 @@ namespace brep try { - for (const dir_entry& de: dir_iterator (d, false /* ignore_dangling */)) + for (const dir_entry& de: dir_iterator (d, dir_iterator::no_follow)) { if (de.path ().extension () == "pem" && de.type () == entry_type::regular) @@ -108,7 +108,7 @@ namespace brep catch (const system_error& e) { ostringstream os; - os<< "unable to iterate over agents keys directory '" << d << "'"; + os << "unable to iterate over agents keys directory '" << d << "'"; throw_generic_error (e.code ().value (), os.str ().c_str ()); } diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx index 78661c3..c1630b0 100644 --- a/mod/build-config-module.hxx +++ b/mod/build-config-module.hxx @@ -36,8 +36,9 @@ namespace brep void init (const options::build&); + template <typename K> bool - exclude (const build_package_config& pc, + exclude (const build_package_config_template<K>& pc, const build_class_exprs& common_builds, const build_constraints& common_constraints, const build_target_config& tc, diff --git a/mod/build-result-module.cxx b/mod/build-result-module.cxx new file mode 100644 index 0000000..9ac1390 --- /dev/null +++ b/mod/build-result-module.cxx @@ -0,0 +1,349 @@ +// file : mod/build-result-module.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <mod/build-result-module.hxx> + +#include <odb/database.hxx> + +#include <libbutl/openssl.hxx> +#include <libbutl/fdstream.hxx> +#include <libbutl/process-io.hxx> +#include <libbutl/semantic-version.hxx> + +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> + +namespace brep +{ + using namespace std; + using namespace butl; + + // While currently the user-defined copy constructor is not required (we + // don't need to deep copy nullptr's), it is a good idea to keep the + // placeholder ready for less trivial cases. + // + build_result_module:: + build_result_module (const build_result_module& r) + : database_module (r), + build_config_module (r), + use_openssl_pkeyutl_ (r.initialized_ ? r.use_openssl_pkeyutl_ : false) + { + } + + void build_result_module:: + init (const options::build& bo, const options::build_db& bdo) + { + HANDLER_DIAG; + + build_config_module::init (bo); + database_module::init (bdo, bdo.build_db_retry ()); + + try + { + optional<openssl_info> oi ( + openssl::info ([&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }, + 2, + bo.openssl ())); + + use_openssl_pkeyutl_ = oi && + oi->name == "OpenSSL" && + oi->version >= semantic_version {3, 0, 0}; + } + catch (const system_error& e) + { + fail << "unable to obtain openssl version: " << e; + } + } + + build_result_module::parse_session_result build_result_module:: + parse_session (const string& s) const + { + using brep::version; // Not to confuse with module::version. + + parse_session_result r; + + size_t p (s.find ('/')); // End of tenant. + + if (p == string::npos) + throw invalid_argument ("no package name"); + + if (tenant.compare (0, tenant.size (), s, 0, p) != 0) + throw invalid_argument ("tenant mismatch"); + + size_t b (p + 1); // Start of package name. + p = s.find ('/', b); // End of package name. + + if (p == b) + throw invalid_argument ("empty package name"); + + if (p == string::npos) + throw invalid_argument ("no package version"); + + package_name name; + + try + { + name = package_name (string (s, b, p - b)); + } + catch (const invalid_argument& e) + { + throw invalid_argument ( + string ("invalid package name : ") + e.what ()); + } + + b = p + 1; // Start of version. + p = s.find ('/', b); // End of version. + + if (p == string::npos) + throw invalid_argument ("no target"); + + auto parse_version = [&s, &b, &p] (const char* what) -> version + { + // Intercept exception handling to add the parsing error attribution. + // + try + { + return brep::version (string (s, b, p - b)); + } + catch (const invalid_argument& e) + { + throw invalid_argument ( + string ("invalid ") + what + ": " + e.what ()); + } + }; + + r.package_version = parse_version ("package version"); + + b = p + 1; // Start of target. + p = s.find ('/', b); // End of target. + + if (p == string::npos) + throw invalid_argument ("no target configuration name"); + + target_triplet target; + try + { + target = target_triplet (string (s, b, p - b)); + } + catch (const invalid_argument& e) + { + throw invalid_argument (string ("invalid target: ") + e.what ()); + } + + b = p + 1; // Start of target configuration name. + p = s.find ('/', b); // End of target configuration name. + + if (p == string::npos) + throw invalid_argument ("no package configuration name"); + + string target_config (s, b, p - b); + + if (target_config.empty ()) + throw invalid_argument ("empty target configuration name"); + + b = p + 1; // Start of package configuration name. + p = s.find ('/', b); // End of package configuration name. + + if (p == string::npos) + throw invalid_argument ("no toolchain name"); + + string package_config (s, b, p - b); + + if (package_config.empty ()) + throw invalid_argument ("empty package configuration name"); + + b = p + 1; // Start of toolchain name. + p = s.find ('/', b); // End of toolchain name. + + if (p == string::npos) + throw invalid_argument ("no toolchain version"); + + string toolchain_name (s, b, p - b); + + if (toolchain_name.empty ()) + throw invalid_argument ("empty toolchain name"); + + b = p + 1; // Start of toolchain version. + p = s.find ('/', b); // End of toolchain version. + + if (p == string::npos) + throw invalid_argument ("no timestamp"); + + r.toolchain_version = parse_version ("toolchain version"); + + r.id = build_id (package_id (move (tenant), move (name), r.package_version), + move (target), + move (target_config), + move (package_config), + move (toolchain_name), + r.toolchain_version); + + try + { + size_t tsn; + string ts (s, p + 1); + + r.timestamp = timestamp (chrono::duration_cast<timestamp::duration> ( + chrono::nanoseconds (stoull (ts, &tsn)))); + + if (tsn != ts.size ()) + throw invalid_argument ("trailing junk"); + } + // Handle invalid_argument or out_of_range (both derive from logic_error), + // that can be thrown by stoull(). + // + catch (const logic_error& e) + { + throw invalid_argument (string ("invalid timestamp: ") + e.what ()); + } + + return r; + } + + bool build_result_module:: + authenticate_session (const options::build& o, + const optional<vector<char>>& challenge, + const build& b, + const string& session) const + { + HANDLER_DIAG; + + auto warn_auth = [&session, &warn] (const string& d) + { + warn << "session '" << session << "' authentication failed: " << d; + }; + + bool r (false); + + // Must both be present or absent. + // + if (!b.agent_challenge != !challenge) + { + warn_auth (challenge ? "unexpected challenge": "challenge is expected"); + } + else if (bot_agent_key_map_ == nullptr) // Authentication is disabled. + { + r = true; + } + else if (!b.agent_challenge) // Authentication is recently enabled. + { + warn_auth ("challenge is required now"); + } + else + { + assert (b.agent_fingerprint && challenge); + + auto auth = [&challenge, + &b, + &o, + &fail, &trace, + &warn_auth, + this] (const path& key) + { + bool r (false); + + try + { + openssl os ([&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }, + path ("-"), fdstream_mode::text, 2, + process_env (o.openssl (), o.openssl_envvar ()), + use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", + o.openssl_option (), + use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", + "-pubin", + "-inkey", key); + + for (const auto& c: *challenge) + os.out.put (c); // Sets badbit on failure. + + os.out.close (); + + string s; + getline (os.in, s); + + bool v (os.in.eof ()); + os.in.close (); + + if (os.wait () && v) + { + r = (s == *b.agent_challenge); + + if (!r) + warn_auth ("challenge mismatched"); + } + else // The signature is presumably meaningless. + warn_auth ("unable to verify challenge"); + } + catch (const system_error& e) + { + fail << "unable to verify challenge: " << e; + } + + return r; + }; + + const string& fp (*b.agent_fingerprint); + auto i (bot_agent_key_map_->find (fp)); + + // Note that it is possible that the default vs custom bot + // classification has changed since the task request time. It feels that + // there is nothing wrong with that and we will handle that + // automatically. + // + if (i != bot_agent_key_map_->end ()) // Default bot? + { + r = auth (i->second); + } + else // Custom bot. + { + shared_ptr<build_public_key> k ( + build_db_->find<build_public_key> (public_key_id (b.tenant, fp))); + + if (k != nullptr) + { + // Temporarily save the key data to disk (note that it's the + // challenge which is passed via stdin to openssl). Hopefully /tmp + // is using tmpfs. + // + auto_rmfile arm; + + try + { + arm = auto_rmfile (path::temp_path ("brep-custom-bot-key")); + } + catch (const system_error& e) + { + fail << "unable to obtain temporary file: " << e; + } + + try + { + ofdstream os (arm.path); + os << *k; + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to '" << arm.path << "': " << e; + } + + r = auth (arm.path); + } + else + { + // The agent's key is recently replaced. + // + warn_auth ("agent's public key not found"); + } + } + } + + return r; + } +} diff --git a/mod/build-result-module.hxx b/mod/build-result-module.hxx new file mode 100644 index 0000000..34466e4 --- /dev/null +++ b/mod/build-result-module.hxx @@ -0,0 +1,78 @@ +// file : mod/build-result-module.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_BUILD_RESULT_MODULE_HXX +#define MOD_BUILD_RESULT_MODULE_HXX + +#include <libbrep/types.hxx> +#include <libbrep/utility.hxx> + +#include <libbrep/build.hxx> + +#include <mod/module-options.hxx> +#include <mod/database-module.hxx> +#include <mod/build-config-module.hxx> + +namespace brep +{ + // Base class for modules that handle the build task results. + // + // Specifically, it loads build controller configuration, initializes the + // build database instance, and provides utilities for parsing and + // authenticating the build task session. + // + class build_result_module: public database_module, + protected build_config_module + { + protected: + build_result_module () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + build_result_module (const build_result_module&); + + void + init (const options::build&, const options::build_db&); + + using handler::init; // Unhide. + + // Parse the build task session and verify that the session matches the + // tenant. Throw invalid_argument on errors. + // + struct parse_session_result + { + build_id id; + brep::version package_version; + brep::version toolchain_version; + brep::timestamp timestamp; + }; + + parse_session_result + parse_session (const string&) const; + + // Return true if bbot agent authentication is disabled or the agent is + // recognized and challenge matches. If the session authentication fails + // (challenge is not expected, expected but doesn't match, etc), then log + // the failure reason with the warning severity and return false. + // + // Note that the session argument is used only for logging. + // + bool + authenticate_session (const options::build&, + const optional<vector<char>>& challenge, + const build&, + const string& session) const; + + protected: + // True if the openssl version is greater or equal to 3.0.0 and so pkeyutl + // needs to be used instead of rsautl. + // + // Note that openssl 3.0.0 deprecates rsautl in favor of pkeyutl. + // + bool use_openssl_pkeyutl_; + }; +} + +#endif // MOD_BUILD_RESULT_MODULE_HXX diff --git a/mod/build-target-config.cxx b/mod/build-target-config.cxx index a30cf07..a30e281 100644 --- a/mod/build-target-config.cxx +++ b/mod/build-target-config.cxx @@ -21,17 +21,13 @@ namespace brep {"all"}, '+', "All."); bool - exclude (const build_package_config& pc, - const build_class_exprs& cbs, - const build_constraints& ccs, + exclude (const build_class_exprs& exprs, + const build_constraints& constrs, const build_target_config& tc, const map<string, string>& class_inheritance_map, string* reason, bool default_all_ucs) { - const build_class_exprs& exprs (pc.effective_builds (cbs)); - const build_constraints& constrs (pc.effective_constraints (ccs)); - // Save the first sentence of the reason, lower-case the first letter if // the beginning looks like a word (all subsequent characters until a // whitespace are lower-case letters). diff --git a/mod/build-target-config.hxx b/mod/build-target-config.hxx index 180ca80..60d159c 100644 --- a/mod/build-target-config.hxx +++ b/mod/build-target-config.hxx @@ -32,14 +32,31 @@ namespace brep // configuration set needlessly). // bool - exclude (const build_package_config&, - const build_class_exprs& common_builds, - const build_constraints& common_constraints, + exclude (const build_class_exprs& builds, + const build_constraints& constraints, const build_target_config&, const std::map<string, string>& class_inheritance_map, string* reason = nullptr, bool default_all_ucs = false); + template <typename K> + inline bool + exclude (const build_package_config_template<K>& pc, + const build_class_exprs& common_builds, + const build_constraints& common_constraints, + const build_target_config& tc, + const std::map<string, string>& class_inheritance_map, + string* reason = nullptr, + bool default_all_ucs = false) + { + return exclude (pc.effective_builds (common_builds), + pc.effective_constraints (common_constraints), + tc, + class_inheritance_map, + reason, + default_all_ucs); + } + // Convert dash-separated components (target, build target configuration // name, machine name) or a pattern thereof into a path, replacing dashes // with slashes (directory separators), `**` with `*/**/*`, and appending diff --git a/mod/build.cxx b/mod/build.cxx index 4abd416..5c37acb 100644 --- a/mod/build.cxx +++ b/mod/build.cxx @@ -3,12 +3,22 @@ #include <mod/build.hxx> +#include <odb/database.hxx> +#include <odb/connection.hxx> +#include <odb/transaction.hxx> + +#include <libbutl/sendmail.hxx> +#include <libbutl/process-io.hxx> + #include <web/server/mime-url-encoding.hxx> +#include <libbrep/build-package-odb.hxx> + #include <mod/utility.hxx> namespace brep { + using namespace std; using namespace web; string @@ -57,4 +67,144 @@ namespace brep "&tv=" + b.toolchain_version.string () + "&reason="; } + + void + send_notification_email (const options::build_email_notification& o, + const odb::core::connection_ptr& conn, + const build& b, + const build_package& p, + const build_package_config& pc, + const string& what, + const basic_mark& error, + const basic_mark* trace) + { + using namespace odb::core; + using namespace butl; + + assert (b.state == build_state::built && b.status); + + // Bail out if sending build notification emails is disabled for this + // toolchain for this package. + // + { + const map<string, build_email>& tes (o.build_toolchain_email ()); + auto i (tes.find (b.id.toolchain_name)); + build_email mode (i != tes.end () ? i->second : build_email::latest); + + if (mode == build_email::none) + { + return; + } + else if (mode == build_email::latest) + { + transaction t (conn->begin ()); + database& db (t.database ()); + + const auto& id (query<buildable_package>::build_package::id); + + buildable_package lp ( + db.query_value<buildable_package> ( + (id.tenant == b.tenant && id.name == b.package_name) + + order_by_version_desc (id.version) + + "LIMIT 1")); + + t.commit (); + + if (lp.package->version != p.version) + return; + } + } + + string subj (what + ' ' + + to_string (*b.status) + ": " + + b.package_name.string () + '/' + + b.package_version.string () + ' ' + + b.target_config_name + '/' + + b.target.string () + ' ' + + b.package_config_name + ' ' + + b.toolchain_name + '-' + b.toolchain_version.string ()); + + // Send notification emails to the interested parties. + // + auto send_email = [&b, &subj, &o, &error, trace] (const string& to) + { + try + { + if (trace != nullptr) + *trace << "email '" << subj << "' to " << to; + + // Redirect the diagnostics to webserver error log. + // + sendmail sm ([trace] (const char* args[], size_t n) + { + if (trace != nullptr) + *trace << process_args {args, n}; + }, + 2, + o.email (), + subj, + {to}); + + if (b.results.empty ()) + { + sm.out << "No operation results available." << endl; + } + else + { + const string& host (o.host ()); + const dir_path& root (o.root ()); + + ostream& os (sm.out); + + os << "combined: " << *b.status << endl << endl + << " " << build_log_url (host, root, b) << endl << endl; + + for (const auto& r: b.results) + os << r.operation << ": " << r.status << endl << endl + << " " << build_log_url (host, root, b, &r.operation) + << endl << endl; + + os << "Force rebuild (enter the reason, use '+' instead of spaces):" + << endl << endl + << " " << build_force_url (host, root, b) << endl; + } + + sm.out.close (); + + if (!sm.wait ()) + error << "sendmail " << *sm.exit; + } + // Handle process_error and io_error (both derive from system_error). + // + catch (const system_error& e) + { + error << "sendmail error: " << e; + } + }; + + // Send the build notification email if a non-empty package build email is + // specified. + // + if (const optional<email>& e = pc.effective_email (p.build_email)) + { + if (!e->empty ()) + send_email (*e); + } + + // Send the build warning/error notification emails, if requested. + // + if (*b.status >= result_status::warning) + { + if (const optional<email>& e = + pc.effective_warning_email (p.build_warning_email)) + send_email (*e); + } + + if (*b.status >= result_status::error) + { + if (const optional<email>& e = + pc.effective_error_email (p.build_error_email)) + send_email (*e); + } + } } diff --git a/mod/build.hxx b/mod/build.hxx index f0846be..07e4411 100644 --- a/mod/build.hxx +++ b/mod/build.hxx @@ -4,10 +4,16 @@ #ifndef MOD_BUILD_HXX #define MOD_BUILD_HXX +#include <odb/forward.hxx> // odb::core::connection_ptr + #include <libbrep/types.hxx> #include <libbrep/utility.hxx> #include <libbrep/build.hxx> +#include <libbrep/build-package.hxx> + +#include <mod/diagnostics.hxx> +#include <mod/module-options.hxx> // Various package build-related utilities. // @@ -25,6 +31,19 @@ namespace brep // string build_force_url (const string& host, const dir_path& root, const build&); + + // Send the notification email for the specified package configuration + // build. The build is expected to be in the built state. + // + void + send_notification_email (const options::build_email_notification&, + const odb::core::connection_ptr&, + const build&, + const build_package&, + const build_package_config&, + const string& what, // build, rebuild, etc. + const basic_mark& error, + const basic_mark* trace); } #endif // MOD_BUILD_HXX diff --git a/mod/buildfile b/mod/buildfile index 6693a35..c3895dc 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -35,6 +35,11 @@ mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}} \ {hxx ixx txx cxx}{+{$libu_src} } \ $libs +# Add support for tenant-associated service notifications to the CI module for +# the debugging of the notifications machinery. +# +cxx.poptions += -DBREP_CI_TENANT_SERVICE + libus{mod}: ../web/xhtml/libus{xhtml} libue{mod}: ../web/xhtml/libue{xhtml} diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx new file mode 100644 index 0000000..cb61e66 --- /dev/null +++ b/mod/ci-common.cxx @@ -0,0 +1,494 @@ +// file : mod/ci-common.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <mod/ci-common.hxx> + +#include <libbutl/uuid.hxx> +#include <libbutl/fdstream.hxx> +#include <libbutl/sendmail.hxx> +#include <libbutl/timestamp.hxx> +#include <libbutl/filesystem.hxx> +#include <libbutl/process-io.hxx> // operator<<(ostream, process_args) +#include <libbutl/manifest-serializer.hxx> + +#include <mod/external-handler.hxx> + +namespace brep +{ + using namespace std; + using namespace butl; + + void ci_start:: + init (shared_ptr<options::ci_start> o) + { + // Verify the data directory satisfies the requirements. + // + const dir_path& d (o->ci_data ()); + + if (d.relative ()) + throw runtime_error ("ci-data directory path must be absolute"); + + if (!dir_exists (d)) + throw runtime_error ("ci-data directory '" + d.string () + + "' does not exist"); + + if (o->ci_handler_specified () && o->ci_handler ().relative ()) + throw runtime_error ("ci-handler path must be absolute"); + + options_ = move (o); + } + + optional<ci_start::start_result> ci_start:: + start (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + optional<tenant_service>&& service, + const repository_location& repository, + const vector<package>& packages, + const optional<string>& client_ip, + const optional<string>& user_agent, + const optional<string>& interactive, + const optional<string>& simulate, + const vector<pair<string, string>>& custom_request, + const vector<pair<string, string>>& overrides) + { + using serializer = manifest_serializer; + using serialization = manifest_serialization; + + assert (options_ != nullptr); // Shouldn't be called otherwise. + + // If the tenant service is specified, then its type may not be empty. + // + assert (!service || !service->type.empty ()); + + // Generate the request id. + // + // Note that it will also be used as a CI result manifest reference, + // unless the latter is provided by the external handler. + // + string request_id; + + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return nullopt; + } + + // Create the submission data directory. + // + dir_path dd (options_->ci_data () / dir_path (request_id)); + + try + { + // It's highly unlikely but still possible that the directory already + // exists. This can only happen if the generated uuid is not unique. + // + if (try_mkdir (dd) == mkdir_status::already_exists) + throw_generic_error (EEXIST); + } + catch (const system_error& e) + { + error << "unable to create directory '" << dd << "': " << e; + return nullopt; + } + + auto_rmdir ddr (dd); + + // Return the start_result object for the client errors (normally the bad + // request status code (400) for the client data serialization errors). + // + auto client_error = [&request_id] (uint16_t status, string message) + { + return start_result {status, + move (message), + request_id, + vector<pair<string, string>> ()}; + }; + + // Serialize the CI request manifest to a stream. On the serialization + // error return false together with the start_result object containing the + // bad request (400) code and the error message. On the stream error pass + // through the io_error exception. Otherwise return true. + // + timestamp ts (system_clock::now ()); + + auto rqm = [&request_id, + &ts, + &service, + &repository, + &packages, + &client_ip, + &user_agent, + &interactive, + &simulate, + &custom_request, + &client_error] (ostream& os, bool long_lines = false) + -> pair<bool, optional<start_result>> + { + try + { + serializer s (os, "request", long_lines); + + // Serialize the submission manifest header. + // + s.next ("", "1"); // Start of manifest. + s.next ("id", request_id); + s.next ("repository", repository.string ()); + + for (const package& p: packages) + { + if (!p.version) + s.next ("package", p.name.string ()); + else + s.next ("package", + p.name.string () + '/' + p.version->string ()); + } + + if (interactive) + s.next ("interactive", *interactive); + + if (simulate) + s.next ("simulate", *simulate); + + s.next ("timestamp", + butl::to_string (ts, + "%Y-%m-%dT%H:%M:%SZ", + false /* special */, + false /* local */)); + + if (client_ip) + s.next ("client-ip", *client_ip); + + if (user_agent) + s.next ("user-agent", *user_agent); + + if (service) + { + // Note that if the service id is not specified, then the handler + // will use the generated reference instead. + // + if (!service->id.empty ()) + s.next ("service-id", service->id); + + s.next ("service-type", service->type); + + if (service->data) + s.next ("service-data", *service->data); + } + + // Serialize the request custom parameters. + // + // Note that the serializer constraints the custom parameter names + // (can't start with '#', can't contain ':' and the whitespaces, + // etc). + // + for (const pair<string, string>& nv: custom_request) + s.next (nv.first, nv.second); + + s.next ("", ""); // End of manifest. + return make_pair (true, optional<start_result> ()); + } + catch (const serialization& e) + { + return make_pair (false, + optional<start_result> ( + client_error (400, + string ("invalid parameter: ") + + e.what ()))); + } + }; + + // Serialize the CI request manifest to the submission directory. + // + path rqf (dd / "request.manifest"); + + try + { + ofdstream os (rqf); + pair<bool, optional<start_result>> r (rqm (os)); + os.close (); + + if (!r.first) + return move (*r.second); + } + catch (const io_error& e) + { + error << "unable to write to '" << rqf << "': " << e; + return nullopt; + } + + // Serialize the CI overrides manifest to a stream. On the serialization + // error return false together with the start_result object containing the + // bad request (400) code and the error message. On the stream error pass + // through the io_error exception. Otherwise return true. + // + auto ovm = [&overrides, &client_error] (ostream& os, + bool long_lines = false) + -> pair<bool, optional<start_result>> + { + try + { + serializer s (os, "overrides", long_lines); + + s.next ("", "1"); // Start of manifest. + + for (const pair<string, string>& nv: overrides) + s.next (nv.first, nv.second); + + s.next ("", ""); // End of manifest. + return make_pair (true, optional<start_result> ()); + } + catch (const serialization& e) + { + return make_pair (false, + optional<start_result> ( + client_error ( + 400, + string ("invalid manifest override: ") + + e.what ()))); + } + }; + + // Serialize the CI overrides manifest to the submission directory. + // + path ovf (dd / "overrides.manifest"); + + if (!overrides.empty ()) + try + { + ofdstream os (ovf); + pair<bool, optional<start_result>> r (ovm (os)); + os.close (); + + if (!r.first) + return move (*r.second); + } + catch (const io_error& e) + { + error << "unable to write to '" << ovf << "': " << e; + return nullopt; + } + + // Given that the submission data is now successfully persisted we are no + // longer in charge of removing it, except for the cases when the + // submission handler terminates with an error (see below for details). + // + ddr.cancel (); + + // If the handler terminates with non-zero exit status or specifies 5XX + // (HTTP server error) submission result manifest status value, then we + // stash the submission data directory for troubleshooting. Otherwise, if + // it's the 4XX (HTTP client error) status value, then we remove the + // directory. + // + auto stash_submit_dir = [&dd, error] () + { + if (dir_exists (dd)) + try + { + mvdir (dd, dir_path (dd + ".fail")); + } + catch (const system_error& e) + { + // Not much we can do here. Let's just log the issue and bail out + // leaving the directory in place. + // + error << "unable to rename directory '" << dd << "': " << e; + } + }; + + // Run the submission handler, if specified, reading the CI result + // manifest from its stdout and parse it into the resulting manifest + // object. Otherwise, create implied CI result manifest. + // + start_result sr; + + if (options_->ci_handler_specified ()) + { + using namespace external_handler; + + optional<result_manifest> r (run (options_->ci_handler (), + options_->ci_handler_argument (), + dd, + options_->ci_handler_timeout (), + error, + warn, + trace)); + if (!r) + { + stash_submit_dir (); + return nullopt; // The diagnostics is already issued. + } + + sr.status = r->status; + + for (manifest_name_value& nv: r->values) + { + string& n (nv.name); + string& v (nv.value); + + if (n == "message") + sr.message = move (v); + else if (n == "reference") + sr.reference = move (v); + else if (n != "status") + sr.custom_result.emplace_back (move (n), move (v)); + } + + if (sr.reference.empty ()) + sr.reference = move (request_id); + } + else // Create the implied CI result manifest. + { + sr.status = 200; + sr.message = "CI request is queued"; + sr.reference = move (request_id); + } + + // Serialize the CI result manifest manifest to a stream. On the + // serialization error log the error description and return false, on the + // stream error pass through the io_error exception, otherwise return + // true. + // + auto rsm = [&sr, &error] (ostream& os, bool long_lines = false) -> bool + { + try + { + serialize_manifest (sr, os, long_lines); + return true; + } + catch (const serialization& e) + { + error << "ref " << sr.reference << ": unable to serialize handler's " + << "output: " << e; + return false; + } + }; + + // If the submission data directory still exists then perform an + // appropriate action on it, depending on the submission result status. + // Note that the handler could move or remove the directory. + // + if (dir_exists (dd)) + { + // Remove the directory if the client error is detected. + // + if (sr.status >= 400 && sr.status < 500) + { + rmdir_r (dd); + } + // + // Otherwise, save the result manifest, into the directory. Also stash + // the directory for troubleshooting in case of the server error. + // + else + { + path rsf (dd / "result.manifest"); + + try + { + ofdstream os (rsf); + + // Not being able to stash the result manifest is not a reason to + // claim the submission failed. The error is logged nevertheless. + // + rsm (os); + + os.close (); + } + catch (const io_error& e) + { + // Not fatal (see above). + // + error << "unable to write to '" << rsf << "': " << e; + } + + if (sr.status >= 500 && sr.status < 600) + stash_submit_dir (); + } + } + + // Send email, if configured, and the CI request submission is not + // simulated. Use the long lines manifest serialization mode for the + // convenience of copying/clicking URLs they contain. + // + // Note that we don't consider the email sending failure to be a + // submission failure as the submission data is successfully persisted and + // the handler is successfully executed, if configured. One can argue that + // email can be essential for the submission processing and missing it + // would result in the incomplete submission. In this case it's natural to + // assume that the web server error log is monitored and the email sending + // failure will be noticed. + // + if (options_->ci_email_specified () && !simulate) + try + { + // Redirect the diagnostics to the web server error log. + // + sendmail sm ([trace] (const char* args[], size_t n) + { + if (trace != nullptr) + *trace << process_args {args, n}; + }, + 2 /* stderr */, + options_->email (), + "CI request submission (" + sr.reference + ')', + {options_->ci_email ()}); + + // Write the CI request manifest. + // + pair<bool, optional<start_result>> r ( + rqm (sm.out, true /* long_lines */)); + + assert (r.first); // The serialization succeeded once, so can't fail now. + + // Write the CI overrides manifest. + // + sm.out << "\n\n"; + + r = ovm (sm.out, true /* long_lines */); + assert (r.first); // The serialization succeeded once, so can't fail now. + + // Write the CI result manifest. + // + sm.out << "\n\n"; + + // We don't care about the result (see above). + // + rsm (sm.out, true /* long_lines */); + + sm.out.close (); + + if (!sm.wait ()) + error << "sendmail " << *sm.exit; + } + // Handle process_error and io_error (both derive from system_error). + // + catch (const system_error& e) + { + error << "sendmail error: " << e; + } + + return optional<start_result> (move (sr)); + } + + void ci_start:: + serialize_manifest (const start_result& r, ostream& os, bool long_lines) + { + manifest_serializer s (os, "result", long_lines); + + s.next ("", "1"); // Start of manifest. + s.next ("status", to_string (r.status)); + s.next ("message", r.message); + s.next ("reference", r.reference); + + for (const pair<string, string>& nv: r.custom_result) + s.next (nv.first, nv.second); + + s.next ("", ""); // End of manifest. + } +} diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx new file mode 100644 index 0000000..6f62c4b --- /dev/null +++ b/mod/ci-common.hxx @@ -0,0 +1,96 @@ +// file : mod/ci-common.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_CI_COMMON_HXX +#define MOD_CI_COMMON_HXX + +#include <odb/forward.hxx> // database + +#include <libbrep/types.hxx> +#include <libbrep/utility.hxx> + +#include <libbrep/common.hxx> + +#include <mod/diagnostics.hxx> +#include <mod/module-options.hxx> + +namespace brep +{ + class ci_start + { + public: + void + init (shared_ptr<options::ci_start>); + + // If the request handling has been performed normally, then return the + // information that corresponds to the CI result manifest (see CI Result + // Manifest in the manual). Otherwise (some internal has error occured), + // log the error and return nullopt. + // + // The arguments correspond to the CI request and overrides manifest + // values (see CI Request and Overrides Manifests in the manual). Note: + // request id and timestamp are generated by the implementation. + // + struct package + { + package_name name; + optional<brep::version> version; + }; + // Note that the inability to generate the reference is an internal + // error. Thus, it is not optional. + // + struct start_result + { + uint16_t status; + string message; + string reference; + vector<pair<string, string>> custom_result; + }; + + // In the optional service information, if id is empty, then the generated + // reference is used instead. + // + optional<start_result> + start (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + optional<tenant_service>&&, + const repository_location& repository, + const vector<package>& packages, + const optional<string>& client_ip, + const optional<string>& user_agent, + const optional<string>& interactive = nullopt, + const optional<string>& simulate = nullopt, + const vector<pair<string, string>>& custom_request = {}, + const vector<pair<string, string>>& overrides = {}); + + // Helpers. + // + + // Serialize the start result as a CI result manifest. + // + static void + serialize_manifest (const start_result&, ostream&, bool long_lines = false); + + private: + shared_ptr<options::ci_start> options_; + }; + + class ci_cancel + { + public: + void + init (shared_ptr<options::ci_cancel>, shared_ptr<odb::core::database>); + + // @@ TODO Archive the tenant. + // + void + cancel (/*...*/); + + private: + shared_ptr<options::ci_cancel> options_; + shared_ptr<odb::core::database> build_db_; + }; +} + +#endif // MOD_CI_COMMON_HXX diff --git a/mod/database-module.cxx b/mod/database-module.cxx index f598bfd..bbb3e59 100644 --- a/mod/database-module.cxx +++ b/mod/database-module.cxx @@ -3,13 +3,20 @@ #include <mod/database-module.hxx> +#include <odb/database.hxx> #include <odb/exceptions.hxx> +#include <odb/transaction.hxx> + +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> #include <mod/database.hxx> #include <mod/module-options.hxx> namespace brep { + using namespace odb::core; + // While currently the user-defined copy constructor is not required (we // don't need to deep copy nullptr's), it is a good idea to keep the // placeholder ready for less trivial cases. @@ -68,4 +75,61 @@ namespace brep throw; } + + optional<string> database_module:: + update_tenant_service_state ( + const connection_ptr& conn, + const string& tid, + const function<optional<string> (const tenant_service&)>& f) + { + assert (f != nullptr); // Shouldn't be called otherwise. + + // Must be initialized via the init(options::build_db) function call. + // + assert (build_db_ != nullptr); + + optional<string> r; + + for (size_t retry (retry_);; ) + { + try + { + transaction tr (conn->begin ()); + + shared_ptr<build_tenant> t (build_db_->find<build_tenant> (tid)); + + if (t != nullptr && t->service) + { + tenant_service& s (*t->service); + + if (optional<string> data = f (s)) + { + s.data = move (*data); + build_db_->update (t); + + r = move (s.data); + } + } + + tr.commit (); + + // Bail out if we have successfully updated the service state. + // + break; + } + catch (const odb::recoverable& e) + { + if (retry-- == 0) + throw; + + HANDLER_DIAG; + l1 ([&]{trace << e << "; " << retry + 1 << " tenant service " + << "state update retries left";}); + + r = nullopt; // Prepare for the next iteration. + } + } + + return r; + } } diff --git a/mod/database-module.hxx b/mod/database-module.hxx index f72ba83..298afbf 100644 --- a/mod/database-module.hxx +++ b/mod/database-module.hxx @@ -4,7 +4,7 @@ #ifndef MOD_DATABASE_MODULE_HXX #define MOD_DATABASE_MODULE_HXX -#include <odb/forward.hxx> // database +#include <odb/forward.hxx> // odb::core::database, odb::core::connection_ptr #include <libbrep/types.hxx> #include <libbrep/utility.hxx> @@ -14,6 +14,8 @@ namespace brep { + struct tenant_service; + // A handler that utilises the database. Specifically, it will retry the // request in the face of recoverable database failures (deadlock, loss of // connection, etc) up to a certain number of times. @@ -50,6 +52,26 @@ namespace brep virtual bool handle (request&, response&) = 0; + // Helpers. + // + + // Update the tenant-associated service state if the specified + // notification callback-returned function (expected to be not NULL) + // returns the new state data. Return the service state data, if updated, + // and nullopt otherwise. + // + // Specifically, start the database transaction, query the service state, + // and call the callback-returned function on this state. If this call + // returns the data string (rather than nullopt), then update the service + // state with this data and persist the change. Repeat all the above steps + // on the recoverable database failures (deadlocks, etc). + // + optional<string> + update_tenant_service_state ( + const odb::core::connection_ptr&, + const string& tid, + const function<optional<string> (const tenant_service&)>&); + protected: size_t retry_ = 0; // Max of all retries. diff --git a/mod/database.cxx b/mod/database.cxx index c88555a..02d521d 100644 --- a/mod/database.cxx +++ b/mod/database.cxx @@ -24,10 +24,10 @@ namespace brep operator< (const db_key& x, const db_key& y) { int r; - if ((r = x.user.compare (y.user)) != 0 || - (r = x.role.compare (y.role)) != 0 || + if ((r = x.user.compare (y.user)) != 0 || + (r = x.role.compare (y.role)) != 0 || (r = x.password.compare (y.password)) != 0 || - (r = x.name.compare (y.name)) != 0 || + (r = x.name.compare (y.name)) != 0 || (r = x.host.compare (y.host))) return r < 0; diff --git a/mod/diagnostics.hxx b/mod/diagnostics.hxx index 37ab25e..f83e1de 100644 --- a/mod/diagnostics.hxx +++ b/mod/diagnostics.hxx @@ -109,7 +109,7 @@ namespace brep uncaught_ (r.uncaught_), #endif data_ (move (r.data_)), - os_ (move (r.os_)), + os_ (move (r.os_)), // Note: can throw. epilogue_ (r.epilogue_) { r.data_.clear (); // Empty. diff --git a/mod/external-handler.cxx b/mod/external-handler.cxx index dc4c0fd..3a85bd8 100644 --- a/mod/external-handler.cxx +++ b/mod/external-handler.cxx @@ -15,7 +15,8 @@ #include <libbutl/process.hxx> #include <libbutl/fdstream.hxx> -#include <libbutl/process-io.hxx> // operator<<(ostream, process_args) +#include <libbutl/process-io.hxx> // operator<<(ostream, process_args) +#include <libbutl/manifest-parser.hxx> using namespace std; using namespace butl; diff --git a/mod/external-handler.hxx b/mod/external-handler.hxx index be16e5b..0276a25 100644 --- a/mod/external-handler.hxx +++ b/mod/external-handler.hxx @@ -4,7 +4,7 @@ #ifndef MOD_EXTERNAL_HANDLER_HXX #define MOD_EXTERNAL_HANDLER_HXX -#include <libbutl/manifest-parser.hxx> +#include <libbutl/manifest-types.hxx> #include <libbrep/types.hxx> #include <libbrep/utility.hxx> diff --git a/mod/mod-build-configs.cxx b/mod/mod-build-configs.cxx index 79c47f7..9282544 100644 --- a/mod/mod-build-configs.cxx +++ b/mod/mod-build-configs.cxx @@ -37,6 +37,9 @@ init (scanner& s) if (options_->build_config_specified ()) build_config_module::init (*options_); + + if (options_->root ().empty ()) + options_->root (dir_path ("/")); } bool brep::build_configs:: @@ -54,6 +57,8 @@ handle (request& rq, response& rs) params::build_configs params; + string& selected_class (params.class_name ()); // Note: can be empty. + try { name_value_scanner s (rq.parameters (1024)); @@ -64,8 +69,7 @@ handle (request& rq, response& rs) // character (that is otherwise forbidden in a class name) to the plus // character. // - string& cn (params.class_name ()); - replace (cn.begin (), cn.end (), ' ', '+'); + replace (selected_class.begin (), selected_class.end (), ' ', '+'); } catch (const cli::exception& e) { @@ -90,7 +94,7 @@ handle (request& rq, response& rs) { string r (tenant_dir (root, tenant).string () + "?build-configs"); - if (cls != "all") + if (!cls.empty ()) { r += '='; @@ -125,26 +129,36 @@ handle (request& rq, response& rs) for (auto b (cls.begin ()), i (b), e (cls.end ()); i != e; ++i) { - if (i != b) - s << ' '; - + // Skip the 'hidden' class. + // const string& c (*i); - print_class_name (c, c == params.class_name ()); - // Append the base class, if present. - // - auto j (im.find (c)); - if (j != im.end ()) + if (c != "hidden") { - s << ':'; - print_class_name (j->second); + // Note that here we rely on the fact that the first class in the list + // can never be 'hidden' (is always 'all'). + // + if (i != b) + s << ' '; + + print_class_name (c, c == selected_class); + + // Append the base class, if present. + // + auto j (im.find (c)); + if (j != im.end ()) + { + s << ':'; + print_class_name (j->second); + } } } s << ~P; } - // Print build configurations that belong to the selected class. + // Print build configurations that belong to the selected class (all + // configurations if no class is selected) and are not hidden. // // We will calculate the total configuration count and cache configurations // for printing (skipping an appropriate number of them for page number @@ -159,7 +173,8 @@ handle (request& rq, response& rs) size_t print (page_configs); for (const build_target_config& c: *target_conf_) { - if (belongs (c, params.class_name ())) + if ((selected_class.empty () || belongs (c, selected_class)) && + !belongs (c, "hidden")) { if (skip != 0) --skip; @@ -214,7 +229,7 @@ handle (request& rq, response& rs) count, page_configs, options_->build_config_pages (), - url (params.class_name ())) + url (selected_class)) << ~DIV << ~BODY << ~HTML; diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx index af4b8e9..bdae356 100644 --- a/mod/mod-build-force.cxx +++ b/mod/mod-build-force.cxx @@ -10,22 +10,32 @@ #include <libbrep/build.hxx> #include <libbrep/build-odb.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> #include <mod/module-options.hxx> +#include <mod/tenant-service.hxx> using namespace std; using namespace brep::cli; using namespace odb::core; +brep::build_force:: +build_force (const tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} + // While currently the user-defined copy constructor is not required (we don't // need to deep copy nullptr's), it is a good idea to keep the placeholder // ready for less trivial cases. // brep::build_force:: -build_force (const build_force& r) +build_force (const build_force& r, const tenant_service_map& tsm) : database_module (r), build_config_module (r), - options_ (r.initialized_ ? r.options_ : nullptr) + options_ (r.initialized_ ? r.options_ : nullptr), + tenant_service_map_ (tsm) { } @@ -173,36 +183,128 @@ handle (request& rq, response& rs) // Load the package build configuration (if present), set the force flag and // update the object's persistent state. // + // If the incomplete package build is being forced to rebuild and the + // tenant_service_build_queued callback is associated with the package + // tenant, then stash the state, the build object, and the callback pointer + // and calculate the hints for the subsequent service `queued` notification. + // + const tenant_service_build_queued* tsq (nullptr); + optional<pair<tenant_service, shared_ptr<build>>> tss; + tenant_service_build_queued::build_queued_hints qhs; + + connection_ptr conn (build_db_->connection ()); { - transaction t (build_db_->begin ()); + transaction t (conn->begin ()); package_build pb; + shared_ptr<build> b; + if (!build_db_->query_one<package_build> ( - query<package_build>::build::id == id, pb)) + query<package_build>::build::id == id, pb) || + (b = move (pb.build))->state == build_state::queued) config_expired ("no package build"); - shared_ptr<build> b (pb.build); force_state force (b->state == build_state::built ? force_state::forced : force_state::forcing); if (b->force != force) { + // Log the force rebuild with the warning severity, truncating the + // reason if too long. + // + diag_record dr (warn); + dr << "force rebuild for "; + + if (!b->tenant.empty ()) + dr << b->tenant << ' '; + + dr << b->package_name << '/' << b->package_version << ' ' + << b->target_config_name << '/' << b->target << ' ' + << b->package_config_name << ' ' + << b->toolchain_name << '-' << b->toolchain_version + << " (state: " << to_string (b->state) << ' ' << to_string (b->force) + << "): "; + + if (reason.size () < 50) + dr << reason; + else + dr << string (reason, 0, 50) << "..."; + b->force = force; build_db_->update (b); - l1 ([&]{trace << "force rebuild for " - << b->tenant << ' ' - << b->package_name << '/' << b->package_version << ' ' - << b->target_config_name << '/' << b->target << ' ' - << b->package_config_name << ' ' - << b->toolchain_name << '-' << b->toolchain_version - << ": " << reason;}); + if (force == force_state::forcing) + { + shared_ptr<build_tenant> t (build_db_->load<build_tenant> (b->tenant)); + + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + tsq = dynamic_cast<const tenant_service_build_queued*> ( + i->second.get ()); + + // If we ought to call the + // tenant_service_build_queued::build_queued() callback, then also + // set the package tenant's queued timestamp to the current time + // to prevent the notifications race (see tenant::queued_timestamp + // for details). + // + if (tsq != nullptr) + { + // Calculate the tenant service hints. + // + buildable_package_count tpc ( + build_db_->query_value<buildable_package_count> ( + query<buildable_package_count>::build_tenant::id == t->id)); + + shared_ptr<build_package> p ( + build_db_->load<build_package> (b->id.package)); + + qhs = tenant_service_build_queued::build_queued_hints { + tpc == 1, p->configs.size () == 1}; + + // Set the package tenant's queued timestamp. + // + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + + tss = make_pair (move (*t->service), move (b)); + } + } + } + } } t.commit (); } + // If the incomplete package build is being forced to rebuild and the + // tenant-associated third-party service needs to be notified about the + // queued builds, then call the tenant_service_build_queued::build_queued() + // callback function and update the service state, if requested. + // + if (tsq != nullptr) + { + assert (tss); // Wouldn't be here otherwise. + + const tenant_service& ss (tss->first); + build& b (*tss->second); + + vector<build> qbs; + qbs.push_back (move (b)); + + if (auto f = tsq->build_queued (ss, + qbs, + build_state::building, + qhs, + log_writer_)) + update_tenant_service_state (conn, qbs.back ().tenant, f); + } + // We have all the data, so don't buffer the response content. // ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); diff --git a/mod/mod-build-force.hxx b/mod/mod-build-force.hxx index 22df383..ea9c141 100644 --- a/mod/mod-build-force.hxx +++ b/mod/mod-build-force.hxx @@ -8,6 +8,7 @@ #include <libbrep/utility.hxx> #include <mod/module-options.hxx> +#include <mod/tenant-service.hxx> #include <mod/database-module.hxx> #include <mod/build-config-module.hxx> @@ -16,13 +17,13 @@ namespace brep class build_force: public database_module, private build_config_module { public: - build_force () = default; + explicit + build_force (const tenant_service_map&); // Create a shallow copy (handling instance) if initialized and a deep // copy (context exemplar) otherwise. // - explicit - build_force (const build_force&); + build_force (const build_force&, const tenant_service_map&); virtual bool handle (request&, response&); @@ -39,6 +40,7 @@ namespace brep private: shared_ptr<options::build_force> options_; + const tenant_service_map& tenant_service_map_; }; } diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx index 3841fad..c8e803b 100644 --- a/mod/mod-build-log.cxx +++ b/mod/mod-build-log.cxx @@ -227,11 +227,16 @@ handle (request& rq, response& rs) query<package_build>::build::id == id, pb)) config_expired ("no package build"); - b = pb.build; + b = move (pb.build); if (b->state != build_state::built) + { config_expired ("state is " + to_string (b->state)); + } else + { build_db_->load (*b, b->results_section); + build_db_->load (*b, b->auxiliary_machines_section); + } t.commit (); } @@ -250,14 +255,20 @@ handle (request& rq, response& rs) if (!b->tenant.empty ()) os << options_->tenant_name () << ": " << b->tenant << endl << endl; - os << "package: " << b->package_name << endl - << "version: " << b->package_version << endl - << "toolchain: " << b->toolchain_name << '-' << b->toolchain_version << endl - << "target: " << b->target << endl - << "tgt config: " << b->target_config_name << endl - << "pkg config: " << b->package_config_name << endl - << "machine: " << b->machine << " (" << b->machine_summary << ")" << endl - << "timestamp: "; + os << "package: " << b->package_name << endl + << "version: " << b->package_version << endl + << "toolchain: " << b->toolchain_name << '-' + << b->toolchain_version << endl + << "target: " << b->target << endl + << "target config: " << b->target_config_name << endl + << "package config: " << b->package_config_name << endl + << "build machine: " << b->machine.name << " -- " + << b->machine.summary << endl; + + for (const build_machine& m: b->auxiliary_machines) + os << "auxiliary machine: " << m.name << " -- " << m.summary << endl; + + os << "timestamp: "; butl::to_stream (os, b->timestamp, diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index 6f40a0b..ccce17f 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -6,13 +6,8 @@ #include <odb/database.hxx> #include <odb/transaction.hxx> -#include <libbutl/openssl.hxx> -#include <libbutl/sendmail.hxx> -#include <libbutl/fdstream.hxx> -#include <libbutl/process-io.hxx> #include <libbutl/manifest-parser.hxx> #include <libbutl/manifest-serializer.hxx> -#include <libbutl/semantic-version.hxx> #include <libbbot/manifest.hxx> @@ -20,11 +15,12 @@ #include <libbrep/build.hxx> #include <libbrep/build-odb.hxx> -#include <libbrep/package.hxx> -#include <libbrep/package-odb.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> -#include <mod/build.hxx> // *_url() +#include <mod/build.hxx> // send_notification_email() #include <mod/module-options.hxx> +#include <mod/tenant-service.hxx> using namespace std; using namespace butl; @@ -32,16 +28,21 @@ using namespace bbot; using namespace brep::cli; using namespace odb::core; +brep::build_result:: +build_result (const tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} + // While currently the user-defined copy constructor is not required (we don't // need to deep copy nullptr's), it is a good idea to keep the placeholder // ready for less trivial cases. // brep::build_result:: -build_result (const build_result& r) - : database_module (r), - build_config_module (r), +build_result (const build_result& r, const tenant_service_map& tsm) + : build_result_module (r), options_ (r.initialized_ ? r.options_ : nullptr), - use_openssl_pkeyutl_ (r.initialized_ ? r.use_openssl_pkeyutl_ : false) + tenant_service_map_ (tsm) { } @@ -53,35 +54,8 @@ init (scanner& s) options_ = make_shared<options::build_result> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (static_cast<const options::package_db&> (*options_), - options_->package_db_retry ()); - if (options_->build_config_specified ()) - { - database_module::init (static_cast<const options::build_db&> (*options_), - options_->build_db_retry ()); - - build_config_module::init (*options_); - } - - try - { - optional<openssl_info> oi ( - openssl::info ([&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }, - 2, - options_->openssl ())); - - use_openssl_pkeyutl_ = oi && - oi->name == "OpenSSL" && - oi->version >= semantic_version {3, 0, 0}; - } - catch (const system_error& e) - { - fail << "unable to obtain openssl version: " << e; - } + build_result_module::init (*options_, *options_); if (options_->root ().empty ()) options_->root (dir_path ("/")); @@ -129,149 +103,23 @@ handle (request& rq, response&) throw invalid_request (400, e.what ()); } - // Parse the task response session to obtain the build id and the timestamp, - // and to make sure the session matches tenant and the result manifest's - // package name, and version. + // Parse the task response session and make sure the session matches tenant + // and the result manifest's package name, and version. // - build_id id; - timestamp session_timestamp; + parse_session_result session; + const build_id& id (session.id); try { - const string& s (rqm.session); - - size_t p (s.find ('/')); // End of tenant. - - if (p == string::npos) - throw invalid_argument ("no package name"); - - if (tenant.compare (0, tenant.size (), s, 0, p) != 0) - throw invalid_argument ("tenant mismatch"); - - size_t b (p + 1); // Start of package name. - p = s.find ('/', b); // End of package name. - - if (p == b) - throw invalid_argument ("empty package name"); - - if (p == string::npos) - throw invalid_argument ("no package version"); - - package_name& name (rqm.result.name); - { - const string& n (name.string ()); - if (n.compare (0, n.size (), s, b, p - b) != 0) - throw invalid_argument ("package name mismatch"); - } - - b = p + 1; // Start of version. - p = s.find ('/', b); // End of version. - - if (p == string::npos) - throw invalid_argument ("no configuration name"); - - auto parse_version = [&s, &b, &p] (const char* what) -> version - { - // Intercept exception handling to add the parsing error attribution. - // - try - { - return brep::version (string (s, b, p - b)); - } - catch (const invalid_argument& e) - { - throw invalid_argument (string ("invalid ") + what + ": " + e.what ()); - } - }; + // Note: also verifies that the tenant matches the session. + // + session = parse_session (rqm.session); - version package_version (parse_version ("package version")); + if (rqm.result.name != id.package.name) + throw invalid_argument ("package name mismatch"); - if (package_version != rqm.result.version) + if (rqm.result.version != session.package_version) throw invalid_argument ("package version mismatch"); - - b = p + 1; // Start of target. - p = s.find ('/', b); // End of target. - - if (p == string::npos) - throw invalid_argument ("no target configuration name"); - - target_triplet target; - try - { - target = target_triplet (string (s, b, p - b)); - } - catch (const invalid_argument& e) - { - throw invalid_argument (string ("invalid target: ") + e.what ()); - } - - b = p + 1; // Start of target configuration name. - p = s.find ('/', b); // End of target configuration name. - - if (p == string::npos) - throw invalid_argument ("no package configuration name"); - - string target_config (s, b, p - b); - - if (target_config.empty ()) - throw invalid_argument ("empty target configuration name"); - - b = p + 1; // Start of package configuration name. - p = s.find ('/', b); // End of package configuration name. - - if (p == string::npos) - throw invalid_argument ("no toolchain name"); - - string package_config (s, b, p - b); - - if (package_config.empty ()) - throw invalid_argument ("empty package configuration name"); - - b = p + 1; // Start of toolchain name. - p = s.find ('/', b); // End of toolchain name. - - if (p == string::npos) - throw invalid_argument ("no toolchain version"); - - string toolchain_name (s, b, p - b); - - if (toolchain_name.empty ()) - throw invalid_argument ("empty toolchain name"); - - b = p + 1; // Start of toolchain version. - p = s.find ('/', b); // End of toolchain version. - - if (p == string::npos) - throw invalid_argument ("no timestamp"); - - version toolchain_version (parse_version ("toolchain version")); - - id = build_id (package_id (move (tenant), move (name), package_version), - move (target), - move (target_config), - move (package_config), - move (toolchain_name), - toolchain_version); - - try - { - size_t tsn; - string ts (s, p + 1); - - session_timestamp = timestamp ( - chrono::duration_cast<timestamp::duration> ( - chrono::nanoseconds (stoull (ts, &tsn)))); - - if (tsn != ts.size ()) - throw invalid_argument ("trailing junk"); - } - // Handle invalid_argument or out_of_range (both derive from logic_error), - // that can be thrown by stoull(). - // - catch (const logic_error& e) - { - throw invalid_argument (string ("invalid timestamp: ") + e.what ()); - } } catch (const invalid_argument& e) { @@ -283,9 +131,24 @@ handle (request& rq, response&) // if the session is valid. The thinking is that this is a problem with the // controller's setup (expires too fast), not with the agent's. // - auto warn_expired = [&rqm, &warn] (const string& d) + // Note, though, that there can be quite a common situation when a build + // machine is suspended by the bbot agent due to the build timeout. In this + // case the task result request may arrive anytime later (after the issue is + // investigated, etc) with the abort or abnormal status. By that arrival + // time a new build task may already be issued/completed for this package + // build configuration or this configuration may even be gone (brep has been + // reconfigured, package has gone, etc). We will log no warning in this + // case, assuming that such an expiration is not a problem with the + // controller's setup. + // + shared_ptr<build> b; + result_status rs (rqm.result.status); + + auto warn_expired = [&rqm, &warn, &b, &session, rs] (const string& d) { - warn << "session '" << rqm.session << "' expired: " << d; + if (!((b == nullptr || b->timestamp > session.timestamp) && + (rs == result_status::abort || rs == result_status::abnormal))) + warn << "session '" << rqm.session << "' expired: " << d; }; // Make sure the build configuration still exists. @@ -304,32 +167,6 @@ handle (request& rq, response&) tc = i->second; } - // Load the built package (if present). - // - // The only way not to deal with 2 databases simultaneously is to pull - // another bunch of the package fields into the build_package foreign - // object, which is a pain (see build_package.hxx for details). Doesn't seem - // worth it here: email members are really secondary and we don't need to - // switch transactions back and forth. - // - shared_ptr<package> pkg; - { - transaction t (package_db_->begin ()); - pkg = package_db_->find<package> (id.package); - t.commit (); - } - - if (pkg == nullptr) - { - warn_expired ("no package"); - return true; - } - - auto print_args = [&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }; - // Load and update the package build configuration (if present). // // NULL if the package build doesn't exist or is not updated for any reason @@ -338,103 +175,199 @@ handle (request& rq, response&) // shared_ptr<build> bld; + // The built package configuration. + // + // Not NULL if bld is not NULL. + // + shared_ptr<build_package> pkg; + build_package_config* cfg (nullptr); + + // Don't send email to the build-email address for the success-to-success + // status change, unless the build was forced. + // bool build_notify (false); bool unforced (true); + // If the package is built (result status differs from interrupt, etc) and + // the package tenant has a third-party service state associated with it, + // then check if the tenant_service_build_built callback is registered for + // the type of the associated service. If it is, then stash the state, the + // build object, and the callback pointer for the subsequent service `built` + // notification. Note that we send this notification for the skip result as + // well, since it is semantically equivalent to the previous build result + // with the actual build process being optimized out. + // + // If the package build is interrupted and the tenant_service_build_queued + // callback is associated with the package tenant, then stash the state, the + // build object, and the callback pointer and calculate the hints for the + // subsequent service `queued` notification. + // + const tenant_service_build_built* tsb (nullptr); + const tenant_service_build_queued* tsq (nullptr); + optional<pair<tenant_service, shared_ptr<build>>> tss; + tenant_service_build_queued::build_queued_hints qhs; + + // Note that if the session authentication fails (probably due to the + // authentication settings change), then we log this case with the warning + // severity and respond with the 200 HTTP code as if the challenge is + // valid. The thinking is that we shouldn't alarm a law-abaiding agent and + // shouldn't provide any information to a malicious one. + // + connection_ptr conn (build_db_->connection ()); { - transaction t (build_db_->begin ()); + transaction t (conn->begin ()); package_build pb; - shared_ptr<build> b; + + auto build_timestamp = [&b] () + { + return to_string ( + chrono::duration_cast<std::chrono::nanoseconds> ( + b->timestamp.time_since_epoch ()).count ()); + }; + if (!build_db_->query_one<package_build> ( query<package_build>::build::id == id, pb)) + { warn_expired ("no package build"); + } else if ((b = move (pb.build))->state != build_state::building) - warn_expired ("package configuration state is " + to_string (b->state)); - else if (b->timestamp != session_timestamp) - warn_expired ("non-matching timestamp"); - else { - // Check the challenge. - // - // If the challenge doesn't match expectations (probably due to the - // authentication settings change), then we log this case with the - // warning severity and respond with the 200 HTTP code as if the - // challenge is valid. The thinking is that we shouldn't alarm a - // law-abaiding agent and shouldn't provide any information to a - // malicious one. - // - auto warn_auth = [&rqm, &warn] (const string& d) + warn_expired ("package configuration state is " + to_string (b->state) + + ", force state " + to_string (b->force) + + ", timestamp " + build_timestamp ()); + } + else if (b->timestamp != session.timestamp) + { + warn_expired ("non-matching timestamp " + build_timestamp ()); + } + else if (authenticate_session (*options_, rqm.challenge, *b, rqm.session)) + { + const tenant_service_base* ts (nullptr); + + shared_ptr<build_tenant> t (build_db_->load<build_tenant> (b->tenant)); + + if (t->service) { - warn << "session '" << rqm.session << "' authentication failed: " << d; - }; + auto i (tenant_service_map_.find (t->service->type)); - bool auth (false); + if (i != tenant_service_map_.end ()) + ts = i->second.get (); + } - // Must both be present or absent. + // If the build is interrupted, then revert it to the original built + // state if this is a rebuild. Otherwise (initial build), turn the build + // into the queued state if the tenant_service_build_queued callback is + // registered for the package tenant and delete it from the database + // otherwise. + // + // Note that if the tenant_service_build_queued callback is registered, + // we always send the `queued` notification for the interrupted build, + // even when we reverse it to the original built state. We could also + // turn the build into the queued state in this case, but it feels that + // there is no harm in keeping the previous build information available + // for the user. // - if (!b->agent_challenge != !rqm.challenge) - warn_auth (rqm.challenge - ? "unexpected challenge" - : "challenge is expected"); - else if (bot_agent_key_map_ == nullptr) // Authentication is disabled. - auth = true; - else if (!b->agent_challenge) // Authentication is recently enabled. - warn_auth ("challenge is required now"); - else + if (rs == result_status::interrupt) { - assert (b->agent_fingerprint && rqm.challenge); - auto i (bot_agent_key_map_->find (*b->agent_fingerprint)); - - // The agent's key is recently replaced. + // Schedule the `queued` notification, if the + // tenant_service_build_queued callback is registered for the tenant. // - if (i == bot_agent_key_map_->end ()) - warn_auth ("agent's public key not found"); - else + tsq = dynamic_cast<const tenant_service_build_queued*> (ts); + + if (b->status) // Is this a rebuild? { - try - { - openssl os (print_args, - path ("-"), fdstream_mode::text, 2, - process_env (options_->openssl (), - options_->openssl_envvar ()), - use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", - options_->openssl_option (), - use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", - "-pubin", - "-inkey", - i->second); - - for (const auto& c: *rqm.challenge) - os.out.put (c); // Sets badbit on failure. - - os.out.close (); - - string s; - getline (os.in, s); - - bool v (os.in.eof ()); - os.in.close (); - - if (os.wait () && v) - { - auth = s == *b->agent_challenge; - - if (!auth) - warn_auth ("challenge mismatched"); - } - else // The signature is presumably meaningless. - warn_auth ("unable to verify challenge"); - } - catch (const system_error& e) + b->state = build_state::built; + + // Keep the force rebuild indication. Note that the forcing state is + // only valid for the building state. + // + if (b->force == force_state::forcing) + b->force = force_state::forced; + + // Cleanup the interactive build login information. + // + b->interactive = nullopt; + + // Cleanup the authentication data. + // + b->agent_fingerprint = nullopt; + b->agent_challenge = nullopt; + + // Note that we are unable to restore the pre-rebuild timestamp + // since it has been overwritten when the build task was issued. + // That, however, feels ok and we just keep it unchanged. + // + // Moreover, we actually use the fact that the build's timestamp is + // greater then its soft_timestamp as an indication that the build + // object represents the interrupted rebuild (see the build_task + // handler for details). + // + // @@ Actually, we also unable to restore the pre-rebuild machine + // and auxiliary machines, which are also displayed in the build + // log and may potentially be confusing. Should we drop them from + // the log in this case or replace with the "machine: unknown" + // record? + + build_db_->update (b); + } + else // Initial build. + { + if (tsq != nullptr) { - fail << "unable to verify challenge: " << e; + // Since this is not a rebuild, there are no operation results and + // thus we don't need to load the results section to erase results + // from the database. + // + assert (b->results.empty ()); + + *b = build (move (b->tenant), + move (b->package_name), + move (b->package_version), + move (b->target), + move (b->target_config_name), + move (b->package_config_name), + move (b->toolchain_name), + move (b->toolchain_version)); + + build_db_->update (b); } + else + build_db_->erase (b); } - } - if (auth) + // If we ought to call the tenant_service_build_queued::build_queued() + // callback, then also set the package tenant's queued timestamp to + // the current time to prevent the notifications race (see + // tenant::queued_timestamp for details). + // + if (tsq != nullptr) + { + // Calculate the tenant service hints. + // + buildable_package_count tpc ( + build_db_->query_value<buildable_package_count> ( + query<buildable_package_count>::build_tenant::id == t->id)); + + shared_ptr<build_package> p ( + build_db_->load<build_package> (b->id.package)); + + qhs = tenant_service_build_queued::build_queued_hints { + tpc == 1, p->configs.size () == 1}; + + // Set the package tenant's queued timestamp. + // + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + } + } + else // Regular or skip build result. { + // Schedule the `built` notification, if the + // tenant_service_build_built callback is registered for the tenant. + // + tsb = dynamic_cast<const tenant_service_build_built*> (ts); + // Verify the result status/checksums. // // Specifically, if the result status is skip, then it can only be in @@ -443,15 +376,16 @@ handle (request& rq, response&) // checksums. On verification failure respond with the bad request // HTTP code (400). // - if (rqm.result.status == result_status::skip) + if (rs == result_status::skip) { if (!b->agent_checksum || !b->worker_checksum || !b->dependency_checksum) throw invalid_request (400, "unexpected skip result status"); - // Can only be absent for initial build, in which case the checksums - // are also absent and we would end up with the above 400 response. + // Can only be absent for initial build, in which case the + // checksums are also absent and we would end up with the above + // 400 response. // assert (b->status); @@ -465,8 +399,7 @@ handle (request& rq, response&) if (!result_checksum) throw invalid_request ( 400, - string (what) + - " checksum is expected for skip result status"); + string (what) + " checksum is expected for skip result status"); if (*result_checksum != build_checksum) throw invalid_request ( @@ -487,14 +420,11 @@ handle (request& rq, response&) "dependency"); } - unforced = b->force == force_state::unforced; + unforced = (b->force == force_state::unforced); - // Don't send email to the build-email address for the - // success-to-success status change, unless the build was forced. - // - build_notify = !(rqm.result.status == result_status::success && - b->status && - *b->status == rqm.result.status && + build_notify = !(rs == result_status::success && + b->status && + *b->status == rs && unforced); b->state = build_state::built; @@ -513,11 +443,12 @@ handle (request& rq, response&) b->soft_timestamp = b->timestamp; // If the result status is other than skip, then save the status, - // results, and checksums and update the hard timestamp. + // results, and checksums and update the hard timestamp. Also stash + // the service notification information, if present. // - if (rqm.result.status != result_status::skip) + if (rs != result_status::skip) { - b->status = rqm.result.status; + b->status = rs; b->hard_timestamp = b->soft_timestamp; // Mark the section as loaded, so results are updated. @@ -534,129 +465,99 @@ handle (request& rq, response&) build_db_->update (b); - // Don't send the build notification email if the task result is - // `skip`, the configuration is hidden, or is now excluded by the - // package. + pkg = build_db_->load<build_package> (b->id.package); + cfg = find (b->package_config_name, pkg->configs); + + // The package configuration should be present (see mod-builds.cxx for + // details) but if it is not, let's log the warning. // - if (rqm.result.status != result_status::skip && belongs (*tc, "all")) + if (cfg != nullptr) { - shared_ptr<build_package> p ( - build_db_->load<build_package> (b->id.package)); - - // The package configuration should be present (see mod-builds.cxx - // for details) but if it is not, let's log the warning. + // Don't send the build notification email if the task result is + // `skip`, the configuration is hidden, or is now excluded by the + // package. // - if (const build_package_config* pc = find (b->package_config_name, - p->configs)) + if (rs != result_status::skip && !belongs (*tc, "hidden")) { - if (!exclude (*pc, p->builds, p->constraints, *tc)) - bld = move (b); + build_db_->load (*pkg, pkg->constraints_section); + + if (!exclude (*cfg, pkg->builds, pkg->constraints, *tc)) + bld = b; } - else - warn << "cannot find configuration '" << b->package_config_name - << "' for package " << p->id.name << '/' << p->version; } + else + warn << "cannot find configuration '" << b->package_config_name + << "' for package " << pkg->id.name << '/' << pkg->version; } + + // If required, stash the service notification information. + // + if (tsb != nullptr || tsq != nullptr) + tss = make_pair (move (*t->service), move (b)); } t.commit (); } - if (bld == nullptr) - return true; - - // Bail out if sending build notification emails is disabled for this - // toolchain. + // We either notify about the queued build or notify about the built package + // or don't notify at all. // - { - const map<string, bool>& tes (options_->build_toolchain_email ()); - - auto i (tes.find (bld->id.toolchain_name)); - if (i != tes.end () && !i->second) - return true; - } - - string subj ((unforced ? "build " : "rebuild ") + - to_string (*bld->status) + ": " + - bld->package_name.string () + '/' + - bld->package_version.string () + ' ' + - bld->target_config_name + '/' + - bld->target.string () + ' ' + - bld->package_config_name + ' ' + - bld->toolchain_name + '-' + bld->toolchain_version.string ()); + assert (tsb == nullptr || tsq == nullptr); - // Send notification emails to the interested parties. + // If the package build is interrupted and the tenant-associated third-party + // service needs to be notified about the queued builds, then call the + // tenant_service_build_queued::build_queued() callback function and update + // the service state, if requested. // - auto send_email = [&bld, &subj, &error, &trace, &print_args, this] - (const string& to) + if (tsq != nullptr) { - try - { - l2 ([&]{trace << "email '" << subj << "' to " << to;}); - - // Redirect the diagnostics to webserver error log. - // - // Note: if using this somewhere else, then need to factor out all this - // exit status handling code. - // - sendmail sm (print_args, - 2, - options_->email (), - subj, - {to}); - - if (bld->results.empty ()) - sm.out << "No operation results available." << endl; - else - { - const string& host (options_->host ()); - const dir_path& root (options_->root ()); - - ostream& os (sm.out); + assert (tss); // Wouldn't be here otherwise. - assert (bld->status); - os << "combined: " << *bld->status << endl << endl - << " " << build_log_url (host, root, *bld) << endl << endl; + const tenant_service& ss (tss->first); - for (const auto& r: bld->results) - os << r.operation << ": " << r.status << endl << endl - << " " << build_log_url (host, root, *bld, &r.operation) - << endl << endl; + vector<build> qbs; + qbs.push_back (move (*tss->second)); - os << "Force rebuild (enter the reason, use '+' instead of spaces):" - << endl << endl - << " " << build_force_url (host, root, *bld) << endl; - } - - sm.out.close (); - - if (!sm.wait ()) - error << "sendmail " << *sm.exit; - } - // Handle process_error and io_error (both derive from system_error). - // - catch (const system_error& e) - { - error << "sendmail error: " << e; - } - }; + if (auto f = tsq->build_queued (ss, + qbs, + build_state::building, + qhs, + log_writer_)) + update_tenant_service_state (conn, qbs.back ().tenant, f); + } - // Send the build notification email if a non-empty package build email is - // specified. + // If a third-party service needs to be notified about the built package, + // then call the tenant_service_build_built::build_built() callback function + // and update the service state, if requested. // - optional<email>& build_email (pkg->build_email); - if (build_notify && build_email && !build_email->empty ()) - send_email (*pkg->build_email); + if (tsb != nullptr) + { + assert (tss); // Wouldn't be here otherwise. - assert (bld->status); + const tenant_service& ss (tss->first); + const build& b (*tss->second); - // Send the build warning/error notification emails, if requested. - // - if (pkg->build_warning_email && *bld->status >= result_status::warning) - send_email (*pkg->build_warning_email); + if (auto f = tsb->build_built (ss, b, log_writer_)) + update_tenant_service_state (conn, b.tenant, f); + } - if (pkg->build_error_email && *bld->status >= result_status::error) - send_email (*pkg->build_error_email); + if (bld != nullptr) + { + // Don't sent the notification email for success-to-success status change, + // etc. + // + if (!build_notify) + (cfg->email ? cfg->email : pkg->build_email) = email (); + + send_notification_email (*options_, + conn, + *bld, + *pkg, + *cfg, + unforced ? "build" : "rebuild", + error, + verb_ >= 2 ? &trace : nullptr); + } return true; } diff --git a/mod/mod-build-result.hxx b/mod/mod-build-result.hxx index 1b32ad4..96449d5 100644 --- a/mod/mod-build-result.hxx +++ b/mod/mod-build-result.hxx @@ -8,21 +8,21 @@ #include <libbrep/utility.hxx> #include <mod/module-options.hxx> -#include <mod/database-module.hxx> -#include <mod/build-config-module.hxx> +#include <mod/tenant-service.hxx> +#include <mod/build-result-module.hxx> namespace brep { - class build_result: public database_module, private build_config_module + class build_result: public build_result_module { public: - build_result () = default; + explicit + build_result (const tenant_service_map&); // Create a shallow copy (handling instance) if initialized and a deep // copy (context exemplar) otherwise. // - explicit - build_result (const build_result&); + build_result (const build_result&, const tenant_service_map&); virtual bool handle (request&, response&); @@ -36,13 +36,7 @@ namespace brep private: shared_ptr<options::build_result> options_; - - // True if the openssl version is greater or equal to 3.0.0 and so pkeyutl - // needs to be used instead of rsautl. - // - // Note that openssl 3.0.0 deprecates rsautl in favor of pkeyutl. - // - bool use_openssl_pkeyutl_; + const tenant_service_map& tenant_service_map_; }; } diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index a15ff90..07aff8d 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -6,11 +6,14 @@ #include <map> #include <regex> #include <chrono> +#include <random> #include <odb/database.hxx> #include <odb/transaction.hxx> #include <odb/schema-catalog.hxx> +#include <libbutl/ft/lang.hxx> // thread_local + #include <libbutl/regex.hxx> #include <libbutl/sha256.hxx> #include <libbutl/openssl.hxx> @@ -29,9 +32,9 @@ #include <libbrep/build-package.hxx> #include <libbrep/build-package-odb.hxx> -#include <mod/build-target-config.hxx> - +#include <mod/build.hxx> // send_notification_email() #include <mod/module-options.hxx> +#include <mod/build-target-config.hxx> using namespace std; using namespace butl; @@ -39,15 +42,62 @@ using namespace bbot; using namespace brep::cli; using namespace odb::core; +static thread_local mt19937 rand_gen (random_device {} ()); + +// The challenge (nonce) is randomly generated for every build task if brep is +// configured to authenticate bbot agents. +// +// Nonce generator must guarantee a probabilistically insignificant chance +// of repeating a previously generated value. The common approach is to use +// counters or random number generators (alone or in combination), that +// produce values of the sufficient length. 64-bit non-repeating and +// 512-bit random numbers are considered to be more than sufficient for +// most practical purposes. +// +// We will produce the challenge as the sha256sum of the 512-bit random +// number and the 64-bit current timestamp combination. The latter is +// not really a non-repeating counter and can't be used alone. However +// adding it is a good and cheap uniqueness improvement. +// +// Note that since generating a challenge is not exactly cheap/fast, we will +// generate it in advance for every task request, out of the database +// transaction, and will cache it if it turns out that it wasn't used (no +// package configuration to (re-)build, etc). +// +static thread_local optional<string> challenge; + +// Generate a random number in the specified range (max value is included). +// +static inline size_t +rand (size_t min_val, size_t max_val) +{ + // Note that size_t is not whitelisted as a type the + // uniform_int_distribution class template can be instantiated with. + // + return min_val == max_val + ? min_val + : static_cast<size_t> ( + uniform_int_distribution<unsigned long long> ( + static_cast<unsigned long long> (min_val), + static_cast<unsigned long long> (max_val)) (rand_gen)); +} + +brep::build_task:: +build_task (const tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} + // While currently the user-defined copy constructor is not required (we don't // need to deep copy nullptr's), it is a good idea to keep the placeholder // ready for less trivial cases. // brep::build_task:: -build_task (const build_task& r) +build_task (const build_task& r, const tenant_service_map& tsm) : database_module (r), build_config_module (r), - options_ (r.initialized_ ? r.options_ : nullptr) + options_ (r.initialized_ ? r.options_ : nullptr), + tenant_service_map_ (tsm) { } @@ -98,6 +148,84 @@ init (scanner& s) options_->root (dir_path ("/")); } +// Skip tenants with the freshly queued packages from the consideration (see +// tenant::queued_timestamp for the details on the service notifications race +// prevention). +// +template <typename T> +static inline query<T> +package_query (bool custom_bot, + brep::params::build_task& params, + interactive_mode imode, + uint64_t queued_expiration_ns) +{ + using namespace brep; + using query = query<T>; + + query q (!query::build_tenant::archived); + + if (custom_bot) + { + // Note that we could potentially only query the packages which refer to + // this custom bot key in one of their build configurations. For that we + // would need to additionally join the current query tables with the bot + // fingerprint-containing build_package_bot_keys and + // build_package_config_bot_keys tables and use the SELECT DISTINCT + // clause. The problem is that we also use the ORDER BY clause and in this + // case PostgreSQL requires all the ORDER BY clause expressions to also be + // present in the SELECT DISTINCT clause and fails with the 'for SELECT + // DISTINCT, ORDER BY expressions must appear in select list' error if + // that's not the case. Also note that in the ODB-generated code the + // 'build_package.project::TEXT' expression in the SELECT DISTINCT clause + // (see the CITEXT type mapping for details in libbrep/common.hxx) would + // not match the 'build_package.name' expression in the ORDER BY clause + // and so we will end up with the mentioned error. One (hackish) way to + // fix that would be to add a dummy member of the string type for the + // build_package.name column. This all sounds quite hairy at the moment + // and it also feels that this can potentially pessimize querying the + // packages built with the default bots only. Thus let's keep it simple + // for now and filter packages by the bot fingerprint at the program + // level. + // + q = q && (query::build_package::custom_bot.is_null () || + query::build_package::custom_bot); + } + else + q = q && (query::build_package::custom_bot.is_null () || + !query::build_package::custom_bot); + + // Filter by repositories canonical names (if requested). + // + const strings& rp (params.repository ()); + + if (!rp.empty ()) + q = q && + query::build_repository::id.canonical_name.in_range (rp.begin (), + rp.end ()); + + // If the interactive mode is false or true, then filter out the respective + // packages. + // + switch (imode) + { + case interactive_mode::false_: + { + q = q && query::build_tenant::interactive.is_null (); + break; + } + case interactive_mode::true_: + { + q = q && query::build_tenant::interactive.is_not_null (); + break; + } + case interactive_mode::both: break; + } + + return q && + (query::build_tenant::queued_timestamp.is_null () || + query::build_tenant::queued_timestamp < queued_expiration_ns); +} + bool brep::build_task:: handle (request& rq, response& rs) { @@ -138,23 +266,72 @@ handle (request& rq, response& rs) throw invalid_request (400, e.what ()); } - // Obtain the agent's public key fingerprint if requested. If the fingerprint - // is requested but is not present in the request or is unknown, then respond - // with 401 HTTP code (unauthorized). + // Obtain the agent's public key fingerprint if requested. If the + // fingerprint is requested but is not present in the request, then respond + // with 401 HTTP code (unauthorized). If a key with the specified + // fingerprint is not present in the build bot agent keys directory, then + // assume that this is a custom build bot. + // + // Note that if the agent authentication is not configured (the agent keys + // directory is not specified), then the bot can never be custom and its + // fingerprint is ignored, if present. // optional<string> agent_fp; + bool custom_bot (false); if (bot_agent_key_map_ != nullptr) { - if (!tqm.fingerprint || - bot_agent_key_map_->find (*tqm.fingerprint) == - bot_agent_key_map_->end ()) + if (!tqm.fingerprint) throw invalid_request (401, "unauthorized"); agent_fp = move (tqm.fingerprint); + + custom_bot = (bot_agent_key_map_->find (*agent_fp) == + bot_agent_key_map_->end ()); } - task_response_manifest tsm; + // The resulting task manifest and the related build, package, and + // configuration objects. Note that the latter 3 are only meaningful if the + // the task manifest is present. + // + task_response_manifest task_response; + shared_ptr<build> task_build; + shared_ptr<build_package> task_package; + const build_package_config* task_config; + + auto serialize_task_response_manifest = [&task_response, &rs] () + { + // @@ Probably it would be a good idea to also send some cache control + // headers to avoid caching by HTTP proxies. That would require + // extension of the web::response interface. + // + + manifest_serializer s (rs.content (200, "text/manifest;charset=utf-8"), + "task_response_manifest"); + task_response.serialize (s); + }; + + interactive_mode imode (tqm.effective_interactive_mode ()); + + // Restict the interactive mode (specified by the task request manifest) if + // the interactive parameter is specified and is other than "both". If + // values specified by the parameter and manifest are incompatible (false vs + // true), then just bail out responding with the manifest with an empty + // session. + // + if (params.interactive () != interactive_mode::both) + { + if (imode != interactive_mode::both) + { + if (params.interactive () != imode) + { + serialize_task_response_manifest (); + return true; + } + } + else + imode = params.interactive (); // Can only change both to true or false. + } // Map build target configurations to machines that are capable of building // them. The first matching machine is selected for each configuration. @@ -169,25 +346,59 @@ handle (request& rq, response& rs) config_machines conf_machines; - for (const auto& c: *target_conf_) + for (const build_target_config& c: *target_conf_) { - for (auto& m: tqm.machines) + for (machine_header_manifest& m: tqm.machines) { - // The same story as in exclude() from build-config.cxx. - // + if (m.effective_role () == machine_role::build) try { + // The same story as in exclude() from build-target-config.cxx. + // if (path_match (dash_components_to_path (m.name), dash_components_to_path (c.machine_pattern), dir_path () /* start */, path_match_flags::match_absent)) + { conf_machines.emplace (build_target_config_id {c.target, c.name}, - config_machine {&c, &m}); + config_machine {&c, &m}); + break; + } } catch (const invalid_path&) {} } } + // Collect the auxiliary configurations/machines available for the build. + // + struct auxiliary_config_machine + { + string config; + const machine_header_manifest* machine; + }; + + vector<auxiliary_config_machine> auxiliary_config_machines; + + for (const machine_header_manifest& m: tqm.machines) + { + if (m.effective_role () == machine_role::auxiliary) + { + // Derive the auxiliary configuration name by stripping the first + // (architecture) component from the machine name. + // + size_t p (m.name.find ('-')); + + if (p == string::npos || p == 0 || p == m.name.size () - 1) + throw invalid_request (400, + (string ("no ") + + (p == 0 ? "architecture" : "OS") + + " component in machine name '" + m.name + "'")); + + auxiliary_config_machines.push_back ( + auxiliary_config_machine {string (m.name, p + 1), &m}); + } + } + // Go through package build configurations until we find one that has no // build target configuration present in the database, or is in the building // state but expired (collectively called unbuilt). If such a target @@ -208,118 +419,110 @@ handle (request& rq, response& rs) // Create the task response manifest. Must be called inside the build db // transaction. // - auto task = [this] (shared_ptr<build>&& b, - shared_ptr<build_package>&& p, - build_package_config&& pc, - shared_ptr<build_tenant>&& t, + auto task = [this] (const build& b, + const build_package& p, + const build_package_config& pc, + small_vector<bpkg::test_dependency, 1>&& tests, + vector<auxiliary_machine>&& ams, + optional<string>&& interactive, const config_machine& cm) -> task_response_manifest { uint64_t ts ( chrono::duration_cast<std::chrono::nanoseconds> ( - b->timestamp.time_since_epoch ()).count ()); - - string session (b->tenant + '/' + - b->package_name.string () + '/' + - b->package_version.string () + '/' + - b->target.string () + '/' + - b->target_config_name + '/' + - b->package_config_name + '/' + - b->toolchain_name + '/' + - b->toolchain_version.string () + '/' + + b.timestamp.time_since_epoch ()).count ()); + + string session (b.tenant + '/' + + b.package_name.string () + '/' + + b.package_version.string () + '/' + + b.target.string () + '/' + + b.target_config_name + '/' + + b.package_config_name + '/' + + b.toolchain_name + '/' + + b.toolchain_version.string () + '/' + to_string (ts)); - string result_url (options_->host () + - tenant_dir (options_->root (), b->tenant).string () + - "?build-result"); + string tenant (tenant_dir (options_->root (), b.tenant).string ()); + string result_url (options_->host () + tenant + "?build-result"); assert (transaction::has_current ()); - assert (p->internal ()); // The package is expected to be buildable. + assert (p.internal ()); // The package is expected to be buildable. - lazy_shared_ptr<build_repository> r (p->internal_repository.load ()); + shared_ptr<build_repository> r (p.internal_repository.load ()); strings fps; if (r->certificate_fingerprint) fps.emplace_back (move (*r->certificate_fingerprint)); - // Exclude external test packages which exclude the task build - // configuration. - // - small_vector<bpkg::test_dependency, 1> tests; - - for (const build_test_dependency& td: p->tests) - { - // Don't exclude unresolved external tests. - // - // Note that this may result in the build task failure. However, - // silently excluding such tests could end up with missed software - // bugs which feels much worse. - // - if (td.package != nullptr) - { - shared_ptr<build_package> p (td.package.load ()); - - // Use the default test package configuration. - // - // Note that potentially the test package default configuration may - // contain some (bpkg) arguments associated, but we currently don't - // provide build bot worker with such information. This, however, is - // probably too far fetched so let's keep it simple for now. - // - const build_package_config* pc (find ("default", p->configs)); - assert (pc != nullptr); // Must always be present. - - // Use the `all` class as a least restrictive default underlying - // build class set. Note that we should only apply the explicit - // build restrictions to the external test packages (think about - // the `builds: all` and `builds: -windows` manifest values for - // the primary and external test packages, respectively). - // - if (exclude (*pc, - p->builds, - p->constraints, - *cm.config, - nullptr /* reason */, - true /* default_all_ucs */)) - continue; - } - - tests.emplace_back (move (td.name), - td.type, - td.buildtime, - move (td.constraint), - move (td.reflect)); - } + const package_name& pn (p.id.name); - bool module_pkg ( - b->package_name.string ().compare (0, 10, "libbuild2-") == 0); + bool module_pkg (pn.string ().compare (0, 10, "libbuild2-") == 0); - task_manifest task (move (b->package_name), - move (b->package_version), + // Note that the auxiliary environment is crafted by the bbot agent + // after the auxiliary machines are booted. + // + task_manifest task (pn, + p.version, move (r->location), move (fps), - move (p->requirements), + p.requirements, move (tests), - move (b->dependency_checksum), + b.dependency_checksum, cm.machine->name, + move (ams), cm.config->target, cm.config->environment, + nullopt /* auxiliary_environment */, cm.config->args, - move (pc.arguments), + pc.arguments, belongs (*cm.config, module_pkg ? "build2" : "host"), cm.config->warning_regexes, - move (t->interactive), - move (b->worker_checksum)); + move (interactive), + b.worker_checksum); + + // Collect the build artifacts upload URLs, skipping those which are + // excluded with the upload-*-exclude configuration options. + // + vector<upload_url> upload_urls; + + for (const auto& ud: options_->upload_data ()) + { + const string& t (ud.first); + + auto exclude = [&t] (const multimap<string, string>& mm, + const string& v) + { + auto range (mm.equal_range (t)); + + for (auto i (range.first); i != range.second; ++i) + { + if (i->second == v) + return true; + } + + return false; + }; + + if (!exclude (options_->upload_toolchain_exclude (), + b.toolchain_name) && + !exclude (options_->upload_repository_exclude (), + r->canonical_name)) + { + upload_urls.emplace_back (options_->host () + tenant + "?upload=" + t, + t); + } + } return task_response_manifest (move (session), - move (b->agent_challenge), + b.agent_challenge, move (result_url), - move (b->agent_checksum), + move (upload_urls), + b.agent_checksum, move (task)); }; - // Calculate the build (building state) or rebuild (built state) - // expiration time for package configurations. + // Calculate the build/rebuild (building/built state) and the `queued` + // notifications expiration time for package configurations. // timestamp now (system_clock::now ()); @@ -343,6 +546,9 @@ handle (request& rq, response& rs) timestamp forced_rebuild_expiration ( expiration (options_->build_forced_rebuild_timeout ())); + uint64_t queued_expiration_ns ( + expiration_ns (options_->build_queued_timeout ())); + // Calculate the soft/hard rebuild expiration time, based on the // respective build-{soft,hard}-rebuild-timeout and // build-alt-{soft,hard}-rebuild-{start,stop,timeout} configuration @@ -433,66 +639,10 @@ handle (request& rq, response& rs) : optional<size_t> ()), options_->build_hard_rebuild_timeout ())); - // Return the challenge (nonce) if brep is configured to authenticate bbot - // agents. Return nullopt otherwise. - // - // Nonce generator must guarantee a probabilistically insignificant chance - // of repeating a previously generated value. The common approach is to use - // counters or random number generators (alone or in combination), that - // produce values of the sufficient length. 64-bit non-repeating and - // 512-bit random numbers are considered to be more than sufficient for - // most practical purposes. - // - // We will produce the challenge as the sha256sum of the 512-bit random - // number and the 64-bit current timestamp combination. The latter is - // not really a non-repeating counter and can't be used alone. However - // adding it is a good and cheap uniqueness improvement. - // - auto challenge = [&agent_fp, &now, &fail, &trace, this] () - { - optional<string> r; - - if (agent_fp) - { - try - { - auto print_args = [&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }; - - openssl os (print_args, - nullfd, path ("-"), 2, - process_env (options_->openssl (), - options_->openssl_envvar ()), - "rand", - options_->openssl_option (), 64); - - vector<char> nonce (os.in.read_binary ()); - os.in.close (); - - if (!os.wait () || nonce.size () != 64) - fail << "unable to generate nonce"; - - uint64_t t (chrono::duration_cast<chrono::nanoseconds> ( - now.time_since_epoch ()).count ()); - - sha256 cs (nonce.data (), nonce.size ()); - cs.append (&t, sizeof (t)); - r = cs.string (); - } - catch (const system_error& e) - { - fail << "unable to generate nonce: " << e; - } - } - - return r; - }; - // Convert butl::standard_version type to brep::version. // brep::version toolchain_version (tqm.toolchain_version.string ()); + string& toolchain_name (tqm.toolchain_name); // Prepare the buildable package prepared query. // @@ -513,18 +663,10 @@ handle (request& rq, response& rs) using pkg_query = query<buildable_package>; using prep_pkg_query = prepared_query<buildable_package>; - // Exclude archived tenants. - // - pkg_query pq (!pkg_query::build_tenant::archived); - - // Filter by repositories canonical names (if requested). - // - const vector<string>& rp (params.repository ()); - - if (!rp.empty ()) - pq = pq && - pkg_query::build_repository::id.canonical_name.in_range (rp.begin (), - rp.end ()); + pkg_query pq (package_query<buildable_package> (custom_bot, + params, + imode, + queued_expiration_ns)); // Transform (in-place) the interactive login information into the actual // login command, if specified in the manifest and the transformation @@ -555,493 +697,1727 @@ handle (request& rq, response& rs) tqm.interactive_login = move (lc); } - // If the interactive mode if false or true, then filter out the - // respective packages. Otherwise, order them so that packages from the - // interactive build tenants appear first. + // In the random package ordering mode iterate over the packages list by + // starting from the random offset and wrapping around when reaching the + // end. // - interactive_mode imode (tqm.effective_interactive_mode ()); - - switch (imode) - { - case interactive_mode::false_: - { - pq = pq && pkg_query::build_tenant::interactive.is_null (); - break; - } - case interactive_mode::true_: - { - pq = pq && pkg_query::build_tenant::interactive.is_not_null (); - break; - } - case interactive_mode::both: break; // See below. - } - - // Specify the portion. + // Note, however, that since there can be some packages which are already + // built for all configurations and are not archived yet, picking an + // unbuilt package this way may not work as desired. Think of the + // following case with 5 packages in 3 non-archived tenants: + // + // 0: A - unbuilt, tenant 1 + // 1: B - built, tenant 2 + // 2: C - built, tenant 2 + // 3: D - built, tenant 2 + // 4: E - unbuilt, tenant 3 + // + // If we just pick a random starting offset in the [0, 4] range, then we + // will build A package with probability 0.2 and E with probability 0.8. + // + // To fix that we will only try to build a package from a tenant that the + // random starting offset refers to. Failed that, we will randomly pick + // new starting offset and retry. To make sure we don't retry indefinitely + // when there are no more packages to build (and also for the sake of + // optimization; see below), we will track positions of packages which we + // (unsuccessfully) have already tried to build and skip them while + // generating the random starting offsets and while iterating over + // packages. // - size_t offset (0); + // Also note that since we iterate over packages in chunks, each queried + // in a separate transaction, the number of packages may potentially + // increase or decrease while iterating over them. Thus, to keep things + // consistent, we may need to update our tried positions tracking state + // accordingly (not to cycle, not to refer to an entry out of the list + // boundaries, etc). Generally, regardless whether the number of packages + // has changed or not, the offsets and position statuses may now refer to + // some different packages. The only sensible thing we can do in such + // cases (without trying to detect this situation and restart from + // scratch) is to serve the request and issue some build task, if + // possible. + // + bool random (options_->build_package_order () == build_order::random); + size_t start_offset (0); - pq += "ORDER BY"; + // List of "tried to build" package statuses. True entries denote + // positions of packages which we have tried to build. Initially all + // entries are false. + // + vector<bool> tried_positions; - if (imode == interactive_mode::both) - pq += pkg_query::build_tenant::interactive + "NULLS LAST,"; + // Number of false entries in the above vector. Used merely as an + // optimization to bail out. + // + size_t untried_positions_count (0); - pq += pkg_query::build_package::id.tenant + "," + - pkg_query::build_package::id.name + - order_by_version (pkg_query::build_package::id.version, false) + - "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"; + // Return a random position of a package that we have not yet tried to + // build, if present, and nullopt otherwise. + // + auto rand_position = [&tried_positions, + &untried_positions_count] () -> optional<size_t> + { + assert (untried_positions_count <= tried_positions.size ()); - connection_ptr conn (build_db_->connection ()); + if (untried_positions_count == 0) + return nullopt; - prep_pkg_query pkg_prep_query ( - conn->prepare_query<buildable_package> ( - "mod-build-task-package-query", pq)); + size_t r; + while (tried_positions[r = rand (0, tried_positions.size () - 1)]) ; + return r; + }; - // Prepare the build prepared query. - // - // Note that we can not query the database for configurations that a - // package was not built with, as the database contains only those build - // configurations that have already been acted upon (initially empty). + // Mark the package at specified position as tried to build. Assume that + // it is not yet been tried to build. // - // This is why we query the database for configurations that should not be - // built (in the built state, or in the building state and not - // expired). Having such a list we will select the first build - // configuration that is not in the list (if available) for the response. - // - using bld_query = query<build>; - using prep_bld_query = prepared_query<build>; - - package_id id; - string pkg_config_name; - - bld_query sq (false); - for (const auto& cm: conf_machines) - sq = sq || (bld_query::id.target == cm.first.target && - bld_query::id.target_config_name == cm.first.config && - bld_query::id.package_config_name == - bld_query::_ref (pkg_config_name)); - - bld_query bq ( - equal<build> (bld_query::id.package, id) && - sq && - bld_query::id.toolchain_name == tqm.toolchain_name && - - compare_version_eq (bld_query::id.toolchain_version, - canonical_version (toolchain_version), - true /* revision */) && - - (bld_query::state == "built" || - (bld_query::force == "forcing" && - bld_query::timestamp > forced_result_expiration_ns) || - (bld_query::force != "forcing" && // Unforced or forced. - bld_query::timestamp > normal_result_expiration_ns))); - - prep_bld_query bld_prep_query ( - conn->prepare_query<build> ("mod-build-task-build-query", bq)); - - // Return true if a package needs to be rebuilt. - // - auto needs_rebuild = [&forced_rebuild_expiration, - &soft_rebuild_expiration, - &hard_rebuild_expiration] (const build& b) + auto position_tried = [&tried_positions, + &untried_positions_count] (size_t i) { - assert (b.state == build_state::built); + assert (i < tried_positions.size () && + !tried_positions[i] && + untried_positions_count != 0); - return (b.force == force_state::forced && - b.timestamp <= forced_rebuild_expiration) || - b.soft_timestamp <= soft_rebuild_expiration || - b.hard_timestamp <= hard_rebuild_expiration; + tried_positions[i] = true; + --untried_positions_count; }; - // Convert a build to the hard rebuild, resetting the agent checksum and - // dropping the previous build task result. + // Resize the tried positions list and update the untried positions + // counter accordingly if the package number has changed. // - // Note that since the checksums are hierarchical, the agent checksum - // reset will trigger resets of the "subordinate" checksums up to the - // dependency checksum and so the package will be rebuilt. + // For simplicity, assume that packages are added/removed to/from the end + // of the list. Note that misguessing in such a rare cases are possible + // but not harmful (see above for the reasoning). // - // Also note that there is no sense to keep the build task result since we - // don't accept the skip result for the hard rebuild task. We, however, - // keep the status intact (see below for the reasoning). - // - auto convert_to_hard = [] (const shared_ptr<build>& b) + auto resize_tried_positions = [&tried_positions, &untried_positions_count] + (size_t n) { - b->agent_checksum = nullopt; + if (n > tried_positions.size ()) // Packages added? + { + untried_positions_count += n - tried_positions.size (); + tried_positions.resize (n, false); + } + else if (n < tried_positions.size ()) // Packages removed? + { + for (size_t i (n); i != tried_positions.size (); ++i) + { + if (!tried_positions[i]) + { + assert (untried_positions_count != 0); + --untried_positions_count; + } + } - // Mark the section as loaded, so results are updated. - // - b->results_section.load (); - b->results.clear (); + tried_positions.resize (n); + } + else + { + // Not supposed to be called if the number of packages didn't change. + // + assert (false); + } }; - // Return SHA256 checksum of the controller logic and the configuration - // target, environment, arguments, and warning-detecting regular - // expressions. - // - auto controller_checksum = [] (const build_target_config& c) + if (random) { - sha256 cs ("1"); // Hash the logic version. + using query = query<buildable_package_count>; - cs.append (c.target.string ()); - cs.append (c.environment ? *c.environment : ""); + query q (package_query<buildable_package_count> (custom_bot, + params, + imode, + queued_expiration_ns)); - for (const string& a: c.args) - cs.append (a); + transaction t (build_db_->begin ()); - for (const string& re: c.warning_regexes) - cs.append (re); + // If there are any non-archived interactive build tenants, then the + // chosen randomization approach doesn't really work since interactive + // tenants must be preferred over non-interactive ones, which is + // achieved by proper ordering of the package query result (see below). + // Thus, we just disable randomization if there are any interactive + // tenants. + // + // But shouldn't we randomize the order between packages in multiple + // interactive tenants? Given that such a tenant may only contain a + // single package and can only be built in a single configuration that + // is probably not important. However, we may assume that the + // randomization still happens naturally due to the random nature of the + // tenant id, which is used as a primary sorting criteria (see below). + // + size_t interactive_package_count ( + build_db_->query_value<buildable_package_count> ( + q && query::build_tenant::interactive.is_not_null ())); - return string (cs.string ()); - }; + if (interactive_package_count == 0) + { + untried_positions_count = + build_db_->query_value<buildable_package_count> (q); + } + else + random = false; - // Return the machine id as a machine checksum. - // - auto machine_checksum = [] (const machine_header_manifest& m) - { - return m.id; - }; + t.commit (); + + if (untried_positions_count != 0) + { + tried_positions.resize (untried_positions_count, false); + + optional<size_t> so (rand_position ()); + assert (so); // Wouldn't be here otherwise. + start_offset = *so; + } + } - while (tsm.session.empty ()) + if (!random || !tried_positions.empty ()) { - transaction t (conn->begin ()); + // Specify the portion. + // + size_t offset (start_offset); + size_t limit (50); + + pq += "ORDER BY"; - // Query (and cache) buildable packages. + // If the interactive mode is both, then order the packages so that ones + // from the interactive build tenants appear first. // - auto packages (pkg_prep_query.execute ()); + if (imode == interactive_mode::both) + pq += pkg_query::build_tenant::interactive + "NULLS LAST,"; + + pq += pkg_query::build_package::id.tenant + "," + + pkg_query::build_package::id.name + + order_by_version (pkg_query::build_package::id.version, false) + + "OFFSET" + pkg_query::_ref (offset) + + "LIMIT" + pkg_query::_ref (limit); + + connection_ptr conn (build_db_->connection ()); - // Bail out if there is nothing left. + prep_pkg_query pkg_prep_query ( + conn->prepare_query<buildable_package> ( + "mod-build-task-package-query", pq)); + + // Prepare the build prepared query. + // + // Note that we can not query the database for configurations that a + // package was not built with, as the database contains only those build + // configurations that have already been acted upon (initially empty). + // + // This is why we query the database for configurations that should not + // be built (in the built state, or in the building state and not + // expired). Having such a list we will select the first build + // configuration that is not in the list (if available) for the + // response. + // + using bld_query = query<build>; + using prep_bld_query = prepared_query<build>; + + package_id id; + string pkg_config; + + bld_query sq (false); + for (const auto& cm: conf_machines) + sq = sq || (bld_query::id.target == cm.first.target && + bld_query::id.target_config_name == cm.first.config); + + bld_query bq ( + equal<build> (bld_query::id.package, id) && + bld_query::id.package_config_name == bld_query::_ref (pkg_config) && + sq && + bld_query::id.toolchain_name == toolchain_name && + + compare_version_eq (bld_query::id.toolchain_version, + canonical_version (toolchain_version), + true /* revision */) && + + (bld_query::state == "built" || + (bld_query::state == "building" && + ((bld_query::force == "forcing" && + bld_query::timestamp > forced_result_expiration_ns) || + (bld_query::force != "forcing" && // Unforced or forced. + bld_query::timestamp > normal_result_expiration_ns))))); + + prep_bld_query bld_prep_query ( + conn->prepare_query<build> ("mod-build-task-build-query", bq)); + + // Return true if a package needs to be rebuilt. // - if (packages.empty ()) + auto needs_rebuild = [&forced_rebuild_expiration, + &soft_rebuild_expiration, + &hard_rebuild_expiration] (const build& b) { - t.commit (); - break; - } + assert (b.state == build_state::built); + + return (b.force == force_state::forced && + b.timestamp <= forced_rebuild_expiration) || + b.soft_timestamp <= soft_rebuild_expiration || + b.hard_timestamp <= hard_rebuild_expiration; + }; + + // Convert a build to the hard rebuild, resetting the agent checksum. + // + // Note that since the checksums are hierarchical, the agent checksum + // reset will trigger resets of the "subordinate" checksums up to the + // dependency checksum and so the package will be rebuilt. + // + // Also note that we keep the previous build task result and status + // intact since we may still need to revert the build into the built + // state if the task execution is interrupted. + // + auto convert_to_hard = [] (const shared_ptr<build>& b) + { + b->agent_checksum = nullopt; + }; + + // Return SHA256 checksum of the controller logic and the configuration + // target, environment, arguments, and warning-detecting regular + // expressions. + // + auto controller_checksum = [] (const build_target_config& c) + { + sha256 cs ("1"); // Hash the logic version. + + cs.append (c.target.string ()); + cs.append (c.environment ? *c.environment : ""); + + for (const string& a: c.args) + cs.append (a); + + for (const string& re: c.warning_regexes) + cs.append (re); + + return string (cs.string ()); + }; - offset += packages.size (); + // Return the machine id as a machine checksum. + // + // Note that we don't include auxiliary machine ids into this checksum + // since a different machine will most likely get picked for a pattern. + // And we view all auxiliary machines that match a pattern as equal for + // testing purposes (in other words, pattern is not the way to get + // coverage). + // + auto machine_checksum = [] (const machine_header_manifest& m) + { + return m.id; + }; - // Iterate over packages until we find one that needs building. + // Tenant that the start offset refers to. + // + optional<string> start_tenant; + + // If the build task is created and the tenant of the being built + // package has a third-party service state associated with it, then + // check if the tenant_service_build_building and/or + // tenant_service_build_queued callbacks are registered for the type of + // the associated service. If they are, then stash the state, the build + // object, and the callback pointers for the subsequent service + // notifications. + // + // Also, if the tenant_service_build_queued callback is registered, then + // create, persist, and stash the queued build objects for all the + // unbuilt by the current toolchain and not yet queued configurations of + // the package the build task is created for and calculate the hints. + // Note that for the task build, we need to make sure that the + // third-party service receives the `queued` notification prior to the + // `building` notification (see mod/tenant-service.hxx for valid + // transitions). The `queued` notification is assumed to be already sent + // for the build if the respective object exists and any of the + // following is true for it: + // + // - It is in the queued state (initial_state is build_state::queued). + // + // - It is a user-forced rebuild of an incomplete build + // (rebuild_forced_build is true). // - for (auto& bp: packages) + // - It is a rebuild of an interrupted rebuild (rebuild_forced_build is + // true). + // + const tenant_service_build_building* tsb (nullptr); + const tenant_service_build_queued* tsq (nullptr); + optional<pair<tenant_service, shared_ptr<build>>> tss; + vector<build> qbs; + tenant_service_build_queued::build_queued_hints qhs; + optional<build_state> initial_state; + bool rebuild_forced_build (false); + bool rebuild_interrupted_rebuild (false); + + // Create, persist, and return the queued build objects for all the + // unbuilt by the current toolchain and not yet queued configurations of + // the specified package. + // + // Note that the build object argument is only used for the toolchain + // information retrieval. Also note that the package constraints section + // is expected to be loaded. + // + auto queue_builds = [this] (const build_package& p, const build& b) { - id = move (bp.id); + assert (p.constraints_section.loaded ()); + + // Query the existing build ids and stash them into the set. + // + set<build_id> existing_builds; + + using query = query<package_build_id>; + + query q (query::build::id.package == p.id && + query::build::id.toolchain_name == b.toolchain_name && + compare_version_eq (query::build::id.toolchain_version, + b.id.toolchain_version, + true /* revision */)); + + for (build_id& id: build_db_->query<package_build_id> (q)) + existing_builds.emplace (move (id)); - shared_ptr<build_package> p (build_db_->load<build_package> (id)); + // Go through all the potential package builds and queue those which + // are not in the existing builds set. + // + vector<build> r; - for (build_package_config& pc: p->configs) + for (const build_package_config& pc: p.configs) { - pkg_config_name = pc.name; + for (const build_target_config& tc: *target_conf_) + { + if (!exclude (pc, p.builds, p.constraints, tc)) + { + build_id id (p.id, + tc.target, tc.name, + pc.name, + b.toolchain_name, b.toolchain_version); - // Iterate through the built configurations and erase them from the - // build configuration map. All those configurations that remained - // can be built. We will take the first one, if present. + if (existing_builds.find (id) == existing_builds.end ()) + { + r.emplace_back (move (id.package.tenant), + move (id.package.name), + p.version, + move (id.target), + move (id.target_config_name), + move (id.package_config_name), + move (id.toolchain_name), + b.toolchain_version); + + // @@ TODO Persist the whole vector of builds with a single + // operation if/when bulk operations support is added + // for objects with containers. + // + build_db_->persist (r.back ()); + } + } + } + } + + return r; + }; + + auto queue_hints = [this] (const build_package& p) + { + buildable_package_count tpc ( + build_db_->query_value<buildable_package_count> ( + query<buildable_package_count>::build_tenant::id == p.id.tenant)); + + return tenant_service_build_queued::build_queued_hints { + tpc == 1, p.configs.size () == 1}; + }; + + // Collect the auxiliary machines required for testing of the specified + // package configuration and the external test packages, if present for + // the specified target configuration (task_auxiliary_machines), + // together with the auxiliary machines information that needs to be + // persisted in the database as a part of the build object + // (build_auxiliary_machines, which is parallel to + // task_auxiliary_machines). While at it collect the involved test + // dependencies. Return nullopt if any auxiliary configuration patterns + // may not be resolved to the auxiliary machines (no matching + // configuration, auxiliary machines RAM limit is exceeded, etc). + // + // Note that if the same auxiliary environment name is used for multiple + // packages (for example, for the main and tests packages or for the + // tests and examples packages, etc), then a shared auxiliary machine is + // used for all these packages. In this case all the respective + // configuration patterns must match the configuration derived from this + // machine name. If they don't, then return nullopt. The thinking here + // is that on the next task request a machine whose derived + // configuration matches all the patterns can potentially be picked. + // + struct collect_auxiliaries_result + { + vector<auxiliary_machine> task_auxiliary_machines; + vector<build_machine> build_auxiliary_machines; + small_vector<bpkg::test_dependency, 1> tests; + }; + + auto collect_auxiliaries = [&tqm, &auxiliary_config_machines, this] + (const shared_ptr<build_package>& p, + const build_package_config& pc, + const build_target_config& tc) + -> optional<collect_auxiliaries_result> + { + // The list of the picked build auxiliary machines together with the + // environment names they have been picked for. + // + vector<pair<auxiliary_config_machine, string>> picked_machines; + + // Try to randomly pick the auxiliary machine that matches the + // specified pattern and which can be supplied with the minimum + // required RAM, if specified. Return false if such a machine is not + // available. If a machine is already picked for the specified + // environment name, then return true if the machine's configuration + // matches the specified pattern and false otherwise. + // + auto pick_machine = + [&tqm, + &picked_machines, + used_ram = uint64_t (0), + available_machines = auxiliary_config_machines] + (const build_auxiliary& ba) mutable -> bool + { + vector<size_t> ams; // Indexes of the available matching machines. + optional<uint64_t> ar (tqm.auxiliary_ram); + + // If the machine configuration name pattern (which is legal) or any + // of the machine configuration names (illegal) are invalid paths, + // then we assume we cannot pick the machine. // - // Also save the built configurations for which it's time to be - // rebuilt. + try + { + // The same story as in exclude() from build-target-config.cxx. + // + auto match = [pattern = dash_components_to_path (ba.config)] + (const string& config) + { + return path_match (dash_components_to_path (config), + pattern, + dir_path () /* start */, + path_match_flags::match_absent); + }; + + // Check if a machine is already picked for the specified + // environment name. + // + for (const auto& m: picked_machines) + { + if (m.second == ba.environment_name) + return match (m.first.config); + } + + // Collect the matching machines from the list of the available + // machines and bail out if there are none. + // + for (size_t i (0); i != available_machines.size (); ++i) + { + const auxiliary_config_machine& m (available_machines[i]); + optional<uint64_t> mr (m.machine->ram_minimum); + + if (match (m.config) && (!mr || !ar || used_ram + *mr <= *ar)) + ams.push_back (i); + } + + if (ams.empty ()) + return false; + } + catch (const invalid_path&) + { + return false; + } + + // Pick the matching machine randomly. // - config_machines configs (conf_machines); // Make a copy for this pkg. - auto pkg_builds (bld_prep_query.execute ()); + size_t i (ams[rand (0, ams.size () - 1)]); + auxiliary_config_machine& cm (available_machines[i]); + + // Bump the used RAM. + // + if (optional<uint64_t> r = cm.machine->ram_minimum) + used_ram += *r; + + // Move out the picked machine from the available machines list. + // + picked_machines.emplace_back (move (cm), ba.environment_name); + available_machines.erase (available_machines.begin () + i); + return true; + }; + + // Collect auxiliary machines for the main package build configuration. + // + for (const build_auxiliary& ba: + pc.effective_auxiliaries (p->auxiliaries)) + { + if (!pick_machine (ba)) + return nullopt; // No matched auxiliary machine. + } - for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i) + // Collect the test packages and the auxiliary machines for their + // default build configurations. Exclude external test packages which + // exclude the current target configuration. + // + small_vector<bpkg::test_dependency, 1> tests; + + if (!p->requirements_tests_section.loaded ()) + build_db_->load (*p, p->requirements_tests_section); + + for (const build_test_dependency& td: p->tests) + { + // Don't exclude unresolved external tests. + // + // Note that this may result in the build task failure. However, + // silently excluding such tests could end up with missed software + // bugs which feels much worse. + // + if (td.package != nullptr) { - auto j ( - configs.find (build_target_config_id {i->id.target, - i->id.target_config_name})); + shared_ptr<build_package> tp (td.package.load ()); + + // Try to use the test package configuration named the same as the + // current configuration of the main package. If there is no such + // a configuration, then fallback to using the default + // configuration (which must exist). If the selected test package + // configuration excludes the current target configuration, then + // exclude this external test package from the build task. + // + // Note that potentially the selected test package configuration + // may contain some (bpkg) arguments associated, but we currently + // don't provide build bot worker with such information. This, + // however, is probably too far fetched so let's keep it simple + // for now. + // + const build_package_config* tpc (find (pc.name, tp->configs)); - // Outdated configurations are already excluded with the database - // query. + if (tpc == nullptr) + { + tpc = find ("default", tp->configs); + + assert (tpc != nullptr); // Must always be present. + } + + // Use the `all` class as a least restrictive default underlying + // build class set. Note that we should only apply the explicit + // build restrictions to the external test packages (think about + // the `builds: all` and `builds: -windows` manifest values for + // the primary and external test packages, respectively). // - assert (j != configs.end ()); - configs.erase (j); + build_db_->load (*tp, tp->constraints_section); + + if (exclude (*tpc, + tp->builds, + tp->constraints, + tc, + nullptr /* reason */, + true /* default_all_ucs */)) + continue; + + build_db_->load (*tp, tp->auxiliaries_section); + + for (const build_auxiliary& ba: + tpc->effective_auxiliaries (tp->auxiliaries)) + { + if (!pick_machine (ba)) + return nullopt; // No matched auxiliary machine. + } + } + + tests.emplace_back (td.name, + td.type, + td.buildtime, + td.constraint, + td.enable, + td.reflect); + } + + vector<auxiliary_machine> tms; + vector<build_machine> bms; + + if (size_t n = picked_machines.size ()) + { + tms.reserve (n); + bms.reserve (n); + + for (pair<auxiliary_config_machine, string>& pm: picked_machines) + { + const machine_header_manifest& m (*pm.first.machine); + tms.push_back (auxiliary_machine {m.name, move (pm.second)}); + bms.push_back (build_machine {m.name, m.summary}); + } + } + + return collect_auxiliaries_result { + move (tms), move (bms), move (tests)}; + }; + + if (agent_fp && !challenge) + try + { + auto print_args = [&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }; + + openssl os (print_args, + nullfd, path ("-"), 2, + process_env (options_->openssl (), + options_->openssl_envvar ()), + "rand", + options_->openssl_option (), 64); + + vector<char> nonce (os.in.read_binary ()); + os.in.close (); + + if (!os.wait () || nonce.size () != 64) + fail << "unable to generate nonce"; + + uint64_t t (chrono::duration_cast<chrono::nanoseconds> ( + now.time_since_epoch ()).count ()); + + sha256 cs (nonce.data (), nonce.size ()); + cs.append (&t, sizeof (t)); + challenge = cs.string (); + } + catch (const system_error& e) + { + fail << "unable to generate nonce: " << e; + } + + // While at it, collect the aborted for various reasons builds + // (interactive builds in multiple configurations, builds with too many + // auxiliary machines, etc) to send the notification emails at the end + // of the request handling. + // + struct aborted_build + { + shared_ptr<build> b; + shared_ptr<build_package> p; + const build_package_config* pc; + const char* what; + }; + vector<aborted_build> aborted_builds; + + // Note: is only used for crafting of the notification email subjects. + // + bool unforced (true); + + for (bool done (false); !task_response.task && !done; ) + { + transaction tr (conn->begin ()); + + // We need to be careful in the random package ordering mode not to + // miss the end after having wrapped around. + // + done = (start_offset != 0 && + offset < start_offset && + offset + limit >= start_offset); + + if (done) + limit = start_offset - offset; + + // Query (and cache) buildable packages. + // + auto packages (pkg_prep_query.execute ()); + + size_t chunk_size (packages.size ()); + size_t next_offset (offset + chunk_size); + + // If we are in the random package ordering mode, then also check if + // the package number has changed and, if that's the case, resize the + // tried positions list accordingly. + // + if (random && + (next_offset > tried_positions.size () || + (next_offset < tried_positions.size () && chunk_size < limit))) + { + resize_tried_positions (next_offset); + } + + // Bail out if there is nothing left, unless we need to wrap around in + // the random package ordering mode. + // + if (chunk_size == 0) + { + tr.commit (); + + if (start_offset != 0 && offset >= start_offset) + offset = 0; + else + done = true; + + continue; + } + + size_t position (offset); // Current package position. + offset = next_offset; - if (i->state == build_state::built) + // Iterate over packages until we find one that needs building or have + // to bail out in the random package ordering mode for some reason (no + // more untried positions, need to restart, etc). + // + // Note that it is not uncommon for the sequentially examined packages + // to belong to the same tenant (single tenant mode, etc). Thus, we + // will cache the loaded tenant objects. + // + shared_ptr<build_tenant> t; + + for (auto& bp: packages) + { + shared_ptr<build_package>& p (bp.package); + + id = p->id; + + // Reset the tenant cache if the current package belongs to a + // different tenant. + // + if (t != nullptr && t->id != id.tenant) + t = nullptr; + + // If we are in the random package ordering mode, then cache the + // tenant the start offset refers to, if not cached yet, and check + // if we are still iterating over packages from this tenant + // otherwise. If the latter is not the case, then restart from a new + // random untried offset, if present, and bail out otherwise. + // + if (random) + { + if (!start_tenant) { - assert (i->force != force_state::forcing); + start_tenant = id.tenant; + } + else if (*start_tenant != id.tenant) + { + if (optional<size_t> so = rand_position ()) + { + start_offset = *so; + offset = start_offset; + start_tenant = nullopt; + limit = 50; + done = false; + } + else + done = true; - if (needs_rebuild (*i)) - rebuilds.emplace_back (i.load ()); + break; } + + size_t pos (position++); + + // Should have been resized, if required. + // + assert (pos < tried_positions.size ()); + + // Skip the position if it has already been tried. + // + if (tried_positions[pos]) + continue; + + position_tried (pos); } - if (!configs.empty ()) + // Note that a request to interactively build a package in multiple + // configurations is most likely a mistake than a deliberate choice. + // Thus, for the interactive tenant let's check if the package can + // be built in multiple configurations. If that's the case then we + // will put all the potential builds into the aborted state and + // continue iterating looking for another package. Otherwise, just + // proceed for this package normally. + // + // It also feels like a good idea to archive an interactive tenant + // after a build object is created for it, regardless if the build + // task is issued or not. This way we make sure that an interactive + // build is never performed multiple times for such a tenant for any + // reason (multiple toolchains, buildtab change, etc). Note that the + // build result will still be accepted for an archived build. + // + if (bp.interactive) { - // Find the first build configuration that is not excluded by the - // package configuration. + // Note that the tenant can be archived via some other package on + // some previous iteration. Skip the package if that's the case. // - auto i (configs.begin ()); - auto e (configs.end ()); + // Also note that if bp.archived is false, then we need to + // (re-)load the tenant object to re-check the archived flag. + // + if (!bp.archived) + { + if (t == nullptr) + t = build_db_->load<build_tenant> (id.tenant); + + bp.archived = t->archived; + } - for (; - i != e && - exclude (pc, p->builds, p->constraints, *i->second.config); - ++i) ; + if (bp.archived) + continue; - if (i != e) + assert (t != nullptr); // Wouldn't be here otherwise. + + // Collect the potential build configurations as all combinations + // of the tenant's packages build configurations and the + // non-excluded (by the packages) build target + // configurations. Note that here we ignore the machines from the + // task request. + // + struct build_config { - config_machine& cm (i->second); - machine_header_manifest& mh (*cm.machine); + shared_ptr<build_package> p; + const build_package_config* pc; + const build_target_config* tc; + }; - build_id bid (move (id), - cm.config->target, - cm.config->name, - move (pkg_config_name), - move (tqm.toolchain_name), - toolchain_version); + small_vector<build_config, 1> build_configs; - shared_ptr<build> b (build_db_->find<build> (bid)); - optional<string> cl (challenge ()); + // Note that we don't bother creating a prepared query here, since + // its highly unlikely to encounter multiple interactive tenants + // per task request. Given that we archive such tenants + // immediately, as a common case there will be none. + // + pkg_query pq (pkg_query::build_tenant::id == id.tenant); + for (auto& tp: build_db_->query<buildable_package> (pq)) + { + shared_ptr<build_package>& p (tp.package); - shared_ptr<build_tenant> t ( - build_db_->load<build_tenant> (bid.package.tenant)); + build_db_->load (*p, p->constraints_section); - // Move the interactive build login information into the build - // object, if the package to be built interactively. + for (build_package_config& pc: p->configs) + { + for (const auto& tc: *target_conf_) + { + if (!exclude (pc, p->builds, p->constraints, tc)) + build_configs.push_back (build_config {p, &pc, &tc}); + } + } + } + + // If multiple build configurations are collected, then abort all + // the potential builds and continue iterating over the packages. + // + if (build_configs.size () > 1) + { + // Abort the builds. // - optional<string> login (t->interactive - ? move (tqm.interactive_login) - : nullopt); + for (build_config& c: build_configs) + { + shared_ptr<build_package>& p (c.p); + const string& pc (c.pc->name); + const build_target_config& tc (*c.tc); + + build_id bid (p->id, + tc.target, + tc.name, + pc, + toolchain_name, + toolchain_version); + + // Can there be any existing builds for such a tenant? Doesn't + // seem so, unless due to some manual intervention into the + // database. Anyway, let's just leave such a build alone. + // + shared_ptr<build> b (build_db_->find<build> (bid)); + + if (b == nullptr) + { + b = make_shared<build> (move (bid.package.tenant), + move (bid.package.name), + p->version, + move (bid.target), + move (bid.target_config_name), + move (bid.package_config_name), + move (bid.toolchain_name), + toolchain_version, + result_status::abort, + operation_results ({ + operation_result { + "configure", + result_status::abort, + "error: multiple configurations " + "for interactive build\n"}}), + build_machine { + "brep", "build task module"}); + + build_db_->persist (b); + + // Schedule the build notification email. + // + aborted_builds.push_back (aborted_build { + move (b), move (p), c.pc, "build"}); + } + } - // If build configuration doesn't exist then create the new one - // and persist. Otherwise put it into the building state, refresh - // the timestamp and update. + // Archive the tenant. // - if (b == nullptr) + t->archived = true; + build_db_->update (t); + + continue; // Skip the package. + } + } + + // If true, then the package is (being) built for some + // configurations. + // + // Note that since we only query the built and forced rebuild + // objects there can be false negatives. + // + bool package_built (false); + + build_db_->load (*p, p->bot_keys_section); + + for (const build_package_config& pc: p->configs) + { + // If this is a custom bot, then skip this configuration if it + // doesn't contain this bot's public key in its custom bot keys + // list. Otherwise (this is a default bot), skip this + // configuration if its custom bot keys list is not empty. + // + { + const build_package_bot_keys& bks ( + pc.effective_bot_keys (p->bot_keys)); + + if (custom_bot) { - b = make_shared<build> (move (bid.package.tenant), - move (bid.package.name), - move (bp.version), - move (bid.target), - move (bid.target_config_name), - move (bid.package_config_name), - move (bid.toolchain_name), - move (toolchain_version), - move (login), - move (agent_fp), - move (cl), - mh.name, - move (mh.summary), - controller_checksum (*cm.config), - machine_checksum (*cm.machine)); - - build_db_->persist (b); + assert (agent_fp); // Wouldn't be here otherwise. + + if (find_if ( + bks.begin (), bks.end (), + [&agent_fp] (const lazy_shared_ptr<build_public_key>& k) + { + return k.object_id ().fingerprint == *agent_fp; + }) == bks.end ()) + { + continue; + } } else { - // The build configuration is in the building state. - // - // Note that in both cases we keep the status intact to be - // able to compare it with the final one in the result request - // handling in order to decide if to send the notification - // email. The same is true for the forced flag (in the sense - // that we don't set the force state to unforced). - // - assert (b->state == build_state::building); - - b->state = build_state::building; - b->interactive = move (login); + if (!bks.empty ()) + continue; + } + } - // Switch the force state not to reissue the task after the - // forced rebuild timeout. Note that the result handler will - // still recognize that the rebuild was forced. - // - if (b->force == force_state::forcing) - b->force = force_state::forced; + pkg_config = pc.name; - b->agent_fingerprint = move (agent_fp); - b->agent_challenge = move (cl); - b->machine = mh.name; - b->machine_summary = move (mh.summary); + // Iterate through the built configurations and erase them from the + // build configuration map. All those configurations that remained + // can be built. We will take the first one, if present. + // + // Also save the built configurations for which it's time to be + // rebuilt. + // + config_machines configs (conf_machines); // Make copy for this pkg. + auto pkg_builds (bld_prep_query.execute ()); - string ccs (controller_checksum (*cm.config)); - string mcs (machine_checksum (*cm.machine)); + if (!package_built && !pkg_builds.empty ()) + package_built = true; - // Issue the hard rebuild if it is forced or the configuration - // or machine has changed. - // - if (b->hard_timestamp <= hard_rebuild_expiration || - b->force == force_state::forced || - b->controller_checksum != ccs || - b->machine_checksum != mcs) - convert_to_hard (b); + for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i) + { + auto j ( + configs.find (build_target_config_id { + i->id.target, i->id.target_config_name})); - b->controller_checksum = move (ccs); - b->machine_checksum = move (mcs); + // Outdated configurations are already excluded with the + // database query. + // + assert (j != configs.end ()); + configs.erase (j); - b->timestamp = system_clock::now (); + if (i->state == build_state::built) + { + assert (i->force != force_state::forcing); - build_db_->update (b); + if (needs_rebuild (*i)) + rebuilds.emplace_back (i.load ()); } + } - // Finally, prepare the task response manifest. + if (!configs.empty ()) + { + // Find the first build configuration that is not excluded by + // the package configuration and for which all the requested + // auxiliary machines can be provided. // - tsm = task (move (b), move (p), move (pc), move (t), cm); - break; // Bail out from the package configurations loop. + const config_machine* cm (nullptr); + optional<collect_auxiliaries_result> aux; + + build_db_->load (*p, p->constraints_section); + + for (auto i (configs.begin ()), e (configs.end ()); i != e; ++i) + { + cm = &i->second; + const build_target_config& tc (*cm->config); + + if (!exclude (pc, p->builds, p->constraints, tc)) + { + if (!p->auxiliaries_section.loaded ()) + build_db_->load (*p, p->auxiliaries_section); + + if ((aux = collect_auxiliaries (p, pc, tc))) + break; + } + } + + if (aux) + { + machine_header_manifest& mh (*cm->machine); + + build_id bid (move (id), + cm->config->target, + cm->config->name, + move (pkg_config), + move (toolchain_name), + toolchain_version); + + shared_ptr<build> b (build_db_->find<build> (bid)); + + // Move the interactive build login information into the build + // object, if the package to be built interactively. + // + optional<string> login (bp.interactive + ? move (tqm.interactive_login) + : nullopt); + + // If build configuration doesn't exist then create the new + // one and persist. Otherwise put it into the building state, + // refresh the timestamp and update. + // + if (b == nullptr) + { + b = make_shared<build> (move (bid.package.tenant), + move (bid.package.name), + p->version, + move (bid.target), + move (bid.target_config_name), + move (bid.package_config_name), + move (bid.toolchain_name), + move (toolchain_version), + move (login), + move (agent_fp), + move (challenge), + build_machine { + mh.name, move (mh.summary)}, + move (aux->build_auxiliary_machines), + controller_checksum (*cm->config), + machine_checksum (*cm->machine)); + + challenge = nullopt; + + build_db_->persist (b); + } + else + { + // The build configuration is in the building or queued + // state. + // + // Note that in both the building and built cases we keep + // the status intact to be able to compare it with the final + // one in the result request handling in order to decide if + // to send the notification email or to revert it to the + // built state if interrupted. The same is true for the + // forced flag (in the sense that we don't set the force + // state to unforced). + // + assert (b->state != build_state::built); + + initial_state = b->state; + + b->state = build_state::building; + b->interactive = move (login); + + unforced = (b->force == force_state::unforced); + + // Switch the force state not to reissue the task after the + // forced rebuild timeout. Note that the result handler will + // still recognize that the rebuild was forced. + // + if (b->force == force_state::forcing) + { + b->force = force_state::forced; + rebuild_forced_build = true; + } + + b->agent_fingerprint = move (agent_fp); + + b->agent_challenge = move (challenge); + challenge = nullopt; + + b->machine = build_machine {mh.name, move (mh.summary)}; + + // Mark the section as loaded, so auxiliary_machines are + // updated. + // + b->auxiliary_machines_section.load (); + + b->auxiliary_machines = + move (aux->build_auxiliary_machines); + + string ccs (controller_checksum (*cm->config)); + string mcs (machine_checksum (*cm->machine)); + + // Issue the hard rebuild if it is forced or the + // configuration or machine has changed. + // + if (b->hard_timestamp <= hard_rebuild_expiration || + b->force == force_state::forced || + b->controller_checksum != ccs || + b->machine_checksum != mcs) + convert_to_hard (b); + + b->controller_checksum = move (ccs); + b->machine_checksum = move (mcs); + + b->timestamp = system_clock::now (); + + build_db_->update (b); + } + + if (t == nullptr) + t = build_db_->load<build_tenant> (b->tenant); + + // Archive an interactive tenant. + // + if (bp.interactive) + { + t->archived = true; + build_db_->update (t); + } + + // Finally, stash the service notification information, if + // present, and prepare the task response manifest. + // + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + const tenant_service_base* s (i->second.get ()); + + tsb = dynamic_cast<const tenant_service_build_building*> (s); + tsq = dynamic_cast<const tenant_service_build_queued*> (s); + + if (tsq != nullptr) + { + qbs = queue_builds (*p, *b); + + // If we ought to call the + // tenant_service_build_queued::build_queued() callback, + // then also set the package tenant's queued timestamp + // to the current time to prevent the notifications race + // (see tenant::queued_timestamp for details). + // + if (!qbs.empty () || + !initial_state || + (*initial_state != build_state::queued && + !rebuild_forced_build)) + { + qhs = queue_hints (*p); + + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + } + } + + if (tsb != nullptr || tsq != nullptr) + tss = make_pair (*t->service, b); + } + } + + task_response = task (*b, + *p, + pc, + move (aux->tests), + move (aux->task_auxiliary_machines), + move (bp.interactive), + *cm); + + task_build = move (b); + task_package = move (p); + task_config = &pc; + + package_built = true; + + break; // Bail out from the package configurations loop. + } + } + } + + // If the task manifest is prepared, then bail out from the package + // loop, commit the transaction and respond. Otherwise, stash the + // build toolchain into the tenant, unless it is already stashed or + // the current package already has some configurations (being) + // built. + // + if (!task_response.task) + { + // Note that since there can be false negatives for the + // package_built flag (see above), there can be redundant tenant + // queries which, however, seems harmless (query uses the primary + // key and the object memory footprint is small). + // + if (!package_built) + { + if (t == nullptr) + t = build_db_->load<build_tenant> (p->id.tenant); + + if (!t->toolchain) + { + t->toolchain = build_toolchain {toolchain_name, + toolchain_version}; + + build_db_->update (t); + } } } + else + break; } - // If the task response manifest is prepared, then bail out from the - // package loop, commit the transaction and respond. - // - if (!tsm.session.empty ()) - break; + tr.commit (); } - t.commit (); - } - - // If we don't have an unbuilt package, then let's see if we have a build - // configuration to rebuild. - // - if (tsm.session.empty () && !rebuilds.empty ()) - { - // Sort the configuration rebuild list with the following sort priority: + // If we don't have an unbuilt package, then let's see if we have a + // build configuration to rebuild. // - // 1: force state - // 2: overall status - // 3: timestamp (less is preferred) + if (!task_response.task && !rebuilds.empty ()) + { + // Sort the configuration rebuild list with the following sort + // priority: + // + // 1: force state + // 2: overall status + // 3: timestamp (less is preferred) + // + auto cmp = [] (const shared_ptr<build>& x, const shared_ptr<build>& y) + { + if (x->force != y->force) + return x->force > y->force; // Forced goes first. + + assert (x->status && y->status); // Both built. + + if (x->status != y->status) + return x->status > y->status; // Larger status goes first. + + // Older build completion goes first. + // + // Note that a completed build can have the state change timestamp + // (timestamp member) newer than the completion timestamp + // (soft_timestamp member) if the build was interrupted. + // + return x->soft_timestamp < y->soft_timestamp; + }; + + sort (rebuilds.begin (), rebuilds.end (), cmp); + + // Pick the first build configuration from the ordered list. + // + // Note that the configurations and packages may not match the + // required criteria anymore (as we have committed the database + // transactions that were used to collect this data) so we recheck. If + // we find one that matches then put it into the building state, + // refresh the timestamp and update. Note that we don't amend the + // status and the force state to have them available in the result + // request handling (see above). + // + for (auto& b: rebuilds) + { + try + { + transaction t (conn->begin ()); + + b = build_db_->find<build> (b->id); + + if (b != nullptr && + b->state == build_state::built && + needs_rebuild (*b)) + { + auto i (conf_machines.find ( + build_target_config_id { + b->target, b->target_config_name})); + + // Only actual package configurations are loaded (see above). + // + assert (i != conf_machines.end ()); + const config_machine& cm (i->second); + + // Rebuild the package configuration if still present, is + // buildable, doesn't exclude the target configuration, can be + // provided with all the requested auxiliary machines, and + // matches the request's interactive mode. + // + // Note that while change of the latter seems rather far fetched, + // let's check it for good measure. + // + shared_ptr<build_package> p ( + build_db_->find<build_package> (b->id.package)); + + shared_ptr<build_tenant> t ( + p != nullptr + ? build_db_->load<build_tenant> (p->id.tenant) + : nullptr); + + build_package_config* pc (p != nullptr + ? find (b->package_config_name, + p->configs) + : nullptr); + + if (pc != nullptr && + p->buildable && + (imode == interactive_mode::both || + (t->interactive.has_value () == + (imode == interactive_mode::true_)))) + { + const build_target_config& tc (*cm.config); + + build_db_->load (*p, p->constraints_section); + + if (exclude (*pc, p->builds, p->constraints, tc)) + continue; + + build_db_->load (*p, p->auxiliaries_section); + + if (optional<collect_auxiliaries_result> aux = + collect_auxiliaries (p, *pc, tc)) + { + assert (b->status); + + initial_state = build_state::built; + + rebuild_interrupted_rebuild = + (b->timestamp > b->soft_timestamp); + + b->state = build_state::building; + + // Save the interactive build login information into the + // build object, if the package to be built interactively. + // + // Can't move from, as may need it on the next iteration. + // + b->interactive = t->interactive + ? tqm.interactive_login + : nullopt; + + unforced = (b->force == force_state::unforced); + + b->agent_fingerprint = move (agent_fp); + + b->agent_challenge = move (challenge); + challenge = nullopt; + + const machine_header_manifest& mh (*cm.machine); + b->machine = build_machine {mh.name, mh.summary}; + + // Mark the section as loaded, so auxiliary_machines are + // updated. + // + b->auxiliary_machines_section.load (); + + b->auxiliary_machines = + move (aux->build_auxiliary_machines); + + // Issue the hard rebuild if the timeout expired, rebuild is + // forced, or the configuration or machine has changed. + // + // Note that we never reset the build status (see above for + // the reasoning). + // + string ccs (controller_checksum (*cm.config)); + string mcs (machine_checksum (*cm.machine)); + + if (b->hard_timestamp <= hard_rebuild_expiration || + b->force == force_state::forced || + b->controller_checksum != ccs || + b->machine_checksum != mcs) + convert_to_hard (b); + + b->controller_checksum = move (ccs); + b->machine_checksum = move (mcs); + + b->timestamp = system_clock::now (); + + build_db_->update (b); + + // Stash the service notification information, if present, + // and prepare the task response manifest. + // + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + const tenant_service_base* s (i->second.get ()); + + tsb = dynamic_cast<const tenant_service_build_building*> (s); + tsq = dynamic_cast<const tenant_service_build_queued*> (s); + + if (tsq != nullptr) + { + qbs = queue_builds (*p, *b); + + // If we ought to call the + // tenant_service_build_queued::build_queued() + // callback, then also set the package tenant's queued + // timestamp to the current time to prevent the + // notifications race (see tenant::queued_timestamp + // for details). + // + if (!qbs.empty () || !rebuild_interrupted_rebuild) + { + qhs = queue_hints (*p); + + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + } + } + + if (tsb != nullptr || tsq != nullptr) + tss = make_pair (move (*t->service), b); + } + } + + task_response = task (*b, + *p, + *pc, + move (aux->tests), + move (aux->task_auxiliary_machines), + move (t->interactive), + cm); + + task_build = move (b); + task_package = move (p); + task_config = pc; + } + } + } + + t.commit (); + } + catch (const odb::deadlock&) + { + // Just try with the next rebuild. But first, restore the agent's + // fingerprint and challenge and reset the task manifest and the + // session that we may have prepared. + // + agent_fp = move (b->agent_fingerprint); + challenge = move (b->agent_challenge); + task_response = task_response_manifest (); + } + + // If the task manifest is prepared, then bail out from the package + // configuration rebuilds loop and respond. + // + if (task_response.task) + break; + } + } + + // If the tenant-associated third-party service needs to be notified + // about the queued builds, then call the + // tenant_service_build_queued::build_queued() callback function and + // update the service state, if requested. // - auto cmp = [] (const shared_ptr<build>& x, const shared_ptr<build>& y) + if (tsq != nullptr) { - if (x->force != y->force) - return x->force > y->force; // Forced goes first. + assert (tss); // Wouldn't be here otherwise. - assert (x->status && y->status); // Both built. + tenant_service& ss (tss->first); - if (x->status != y->status) - return x->status > y->status; // Larger status goes first. + // If the task build has no initial state (is just created), then + // temporarily move it into the list of the queued builds until the + // `queued` notification is delivered. Afterwards, restore it so that + // the `building` notification can also be sent. + // + build& b (*tss->second); + bool restore_build (false); - return x->timestamp < y->timestamp; // Older goes first. - }; + if (!initial_state) + { + qbs.push_back (move (b)); + restore_build = true; + } + + if (!qbs.empty ()) + { + if (auto f = tsq->build_queued (ss, + qbs, + nullopt /* initial_state */, + qhs, + log_writer_)) + { + if (optional<string> data = + update_tenant_service_state (conn, qbs.back ().tenant, f)) + ss.data = move (data); + } + } - sort (rebuilds.begin (), rebuilds.end (), cmp); + // Send the `queued` notification for the task build, unless it is + // already sent, and update the service state, if requested. + // + if (initial_state && + *initial_state != build_state::queued && + !rebuild_interrupted_rebuild && + !rebuild_forced_build) + { + qbs.clear (); + qbs.push_back (move (b)); + restore_build = true; + + if (auto f = tsq->build_queued (ss, + qbs, + initial_state, + qhs, + log_writer_)) + { + if (optional<string> data = + update_tenant_service_state (conn, qbs.back ().tenant, f)) + ss.data = move (data); + } + } - optional<string> cl (challenge ()); + if (restore_build) + b = move (qbs.back ()); + } - // Pick the first build configuration from the ordered list. + // If a third-party service needs to be notified about the package + // build, then call the tenant_service_build_built::build_building() + // callback function and, if requested, update the tenant-associated + // service state. // - // Note that the configurations and packages may not match the required - // criteria anymore (as we have committed the database transactions that - // were used to collect this data) so we recheck. If we find one that - // matches then put it into the building state, refresh the timestamp and - // update. Note that we don't amend the status and the force state to - // have them available in the result request handling (see above). + if (tsb != nullptr) + { + assert (tss); // Wouldn't be here otherwise. + + tenant_service& ss (tss->first); + const build& b (*tss->second); + + if (auto f = tsb->build_building (ss, b, log_writer_)) + { + if (optional<string> data = + update_tenant_service_state (conn, b.tenant, f)) + ss.data = move (data); + } + } + + // If the task manifest is prepared, then check that the number of the + // build auxiliary machines is less than 10. If that's not the case, + // then turn the build into the built state with the abort status. // - for (auto& b: rebuilds) + if (task_response.task && + task_response.task->auxiliary_machines.size () > 9) { - try + // Respond with the no-task manifest. + // + task_response = task_response_manifest (); + + // If the package tenant has a third-party service state associated + // with it, then check if the tenant_service_build_built callback is + // registered for the type of the associated service. If it is, then + // stash the state, the build object, and the callback pointer for the + // subsequent service `built` notification. + // + const tenant_service_build_built* tsb (nullptr); + optional<pair<tenant_service, shared_ptr<build>>> tss; { - transaction t (build_db_->begin ()); + transaction t (conn->begin ()); - b = build_db_->find<build> (b->id); + shared_ptr<build> b (build_db_->find<build> (task_build->id)); - if (b != nullptr && - b->state == build_state::built && - needs_rebuild (*b)) + // For good measure, check that the build object is in the building + // state and has not been updated. + // + if (b->state == build_state::building && + b->timestamp == task_build->timestamp) { - auto i (conf_machines.find ( - build_target_config_id {b->target, - b->target_config_name})); + b->state = build_state::built; + b->status = result_status::abort; + b->force = force_state::unforced; - // Only actual package configurations are loaded (see above). + // Cleanup the interactive build login information. // - assert (i != conf_machines.end ()); - const config_machine& cm (i->second); + b->interactive = nullopt; - // Rebuild the package if still present, is buildable, doesn't - // exclude the configuration, and matches the request's - // interactive mode. - // - // Note that while change of the latter seems rather far fetched, - // let's check it for good measure. + // Cleanup the authentication data. // - shared_ptr<build_package> p ( - build_db_->find<build_package> (b->id.package)); + b->agent_fingerprint = nullopt; + b->agent_challenge = nullopt; - shared_ptr<build_tenant> t ( - p != nullptr - ? build_db_->load<build_tenant> (p->id.tenant) - : nullptr); - - build_package_config* pc (p != nullptr - ? find (b->package_config_name, - p->configs) - : nullptr); - - if (pc != nullptr && - p->buildable && - (imode == interactive_mode::both || - (t->interactive.has_value () == - (imode == interactive_mode::true_))) && - !exclude (*pc, p->builds, p->constraints, *cm.config)) - { - assert (b->status); - - b->state = build_state::building; - - // Save the interactive build login information into the build - // object, if the package to be built interactively. - // - // Can't move from, as may need it on the next iteration. - // - b->interactive = t->interactive - ? tqm.interactive_login - : nullopt; + b->timestamp = system_clock::now (); + b->soft_timestamp = b->timestamp; + b->hard_timestamp = b->soft_timestamp; - // Can't move from, as may need them on the next iteration. - // - b->agent_fingerprint = agent_fp; - b->agent_challenge = cl; + // Mark the section as loaded, so results are updated. + // + b->results_section.load (); - const machine_header_manifest& mh (*cm.machine); - b->machine = mh.name; - b->machine_summary = mh.summary; + b->results = operation_results ({ + operation_result { + "configure", + result_status::abort, + "error: not more than 9 auxiliary machines are allowed"}}); - // Issue the hard rebuild if the timeout expired, rebuild is - // forced, or the configuration or machine has changed. - // - // Note that we never reset the build status (see above for the - // reasoning). - // - string ccs (controller_checksum (*cm.config)); - string mcs (machine_checksum (*cm.machine)); + b->agent_checksum = nullopt; + b->worker_checksum = nullopt; + b->dependency_checksum = nullopt; - if (b->hard_timestamp <= hard_rebuild_expiration || - b->force == force_state::forced || - b->controller_checksum != ccs || - b->machine_checksum != mcs) - convert_to_hard (b); + build_db_->update (b); - b->controller_checksum = move (ccs); - b->machine_checksum = move (mcs); + // Schedule the `built` notification, if the + // tenant_service_build_built callback is registered for the + // tenant. + // + shared_ptr<build_tenant> t ( + build_db_->load<build_tenant> (b->tenant)); - b->timestamp = system_clock::now (); + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); - build_db_->update (b); + if (i != tenant_service_map_.end ()) + { + tsb = dynamic_cast<const tenant_service_build_built*> ( + i->second.get ()); - tsm = task (move (b), move (p), move (*pc), move (t), cm); + // If required, stash the service notification information. + // + if (tsb != nullptr) + tss = make_pair (move (*t->service), b); + } } + + // Schedule the build notification email. + // + aborted_builds.push_back ( + aborted_build {move (b), + move (task_package), + task_config, + unforced ? "build" : "rebuild"}); } t.commit (); } - catch (const odb::deadlock&) {} // Just try with the next rebuild. - // If the task response manifest is prepared, then bail out from the - // package configuration rebuilds loop and respond. + // If a third-party service needs to be notified about the built + // package, then call the tenant_service_build_built::build_built() + // callback function and update the service state, if requested. // - if (!tsm.session.empty ()) - break; + if (tsb != nullptr) + { + assert (tss); // Wouldn't be here otherwise. + + tenant_service& ss (tss->first); + const build& b (*tss->second); + + if (auto f = tsb->build_built (ss, b, log_writer_)) + { + if (optional<string> data = + update_tenant_service_state (conn, b.tenant, f)) + ss.data = move (data); + } + } } + + // Send notification emails for all the aborted builds. + // + for (const aborted_build& ab: aborted_builds) + send_notification_email (*options_, + conn, + *ab.b, + *ab.p, + *ab.pc, + ab.what, + error, + verb_ >= 2 ? &trace : nullptr); } } - // @@ Probably it would be a good idea to also send some cache control - // headers to avoid caching by HTTP proxies. That would require extension - // of the web::response interface. - // - - manifest_serializer s (rs.content (200, "text/manifest;charset=utf-8"), - "task_response_manifest"); - tsm.serialize (s); - + serialize_task_response_manifest (); return true; } diff --git a/mod/mod-build-task.hxx b/mod/mod-build-task.hxx index 7875db1..d0b3d44 100644 --- a/mod/mod-build-task.hxx +++ b/mod/mod-build-task.hxx @@ -8,6 +8,7 @@ #include <libbrep/utility.hxx> #include <mod/module-options.hxx> +#include <mod/tenant-service.hxx> #include <mod/database-module.hxx> #include <mod/build-config-module.hxx> @@ -16,13 +17,13 @@ namespace brep class build_task: public database_module, private build_config_module { public: - build_task () = default; + explicit + build_task (const tenant_service_map&); // Create a shallow copy (handling instance) if initialized and a deep // copy (context exemplar) otherwise. // - explicit - build_task (const build_task&); + build_task (const build_task&, const tenant_service_map&); virtual bool handle (request&, response&); @@ -36,6 +37,7 @@ namespace brep private: shared_ptr<options::build_task> options_; + const tenant_service_map& tenant_service_map_; }; } diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index 095dee3..30562f3 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -138,8 +138,7 @@ template <typename T> static inline query<T> build_query (const brep::vector<brep::build_target_config_id>* config_ids, const brep::params::builds& params, - const brep::optional<brep::string>& tenant, - const brep::optional<bool>& archived) + const brep::optional<brep::string>& tenant) { using namespace brep; using query = query<T>; @@ -150,9 +149,6 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids, query q (tenant ? pid.tenant == *tenant : !qt::private_); - if (archived) - q = q && qt::archived == *archived; - if (config_ids != nullptr) { query sq (false); @@ -229,13 +225,19 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids, // Build result. // const string& rs (params.result ()); + bool add_state (true); if (rs != "*") { if (rs == "pending") + { q = q && qb::force != "unforced"; + } else if (rs == "building") + { q = q && qb::state == "building"; + add_state = false; + } else { query sq (qb::status == rs); @@ -263,8 +265,12 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids, // well (rebuild). // q = q && qb::state == "built" && sq; + add_state = false; } } + + if (add_state) + q = q && qb::state != "queued"; } catch (const invalid_argument&) { @@ -279,8 +285,7 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids, template <typename T> static inline query<T> package_query (const brep::params::builds& params, - const brep::optional<brep::string>& tenant, - const brep::optional<bool>& archived) + const brep::optional<brep::string>& tenant) { using namespace brep; using query = query<T>; @@ -289,9 +294,6 @@ package_query (const brep::params::builds& params, query q (tenant ? qp::id.tenant == *tenant : !qt::private_); - if (archived) - q = q && qt::archived == *archived; - // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no packages match such a query. // @@ -391,8 +393,11 @@ handle (request& rq, response& rs) if (!tenant.empty ()) tn = tenant; - // Return the list of distinct toolchain name/version pairs. The build db - // transaction must be started. + // Return the list of distinct toolchain name/version pairs. If no builds + // are present for the tenant, then fallback to the toolchain recorded in + // the tenant object, if present. + // + // Note: the build db transaction must be started. // using toolchains = vector<pair<string, version>>; @@ -408,11 +413,19 @@ handle (request& rq, response& rs) false /* first */))) r.emplace_back (move (t.name), move (t.version)); + if (r.empty ()) + { + shared_ptr<build_tenant> t (build_db_->find<build_tenant> (tenant)); + + if (t != nullptr && t->toolchain) + r.emplace_back (t->toolchain->name, t->toolchain->version); + } + return r; }; auto print_form = [&s, ¶ms, this] (const toolchains& toolchains, - size_t build_count) + optional<size_t> build_count) { // Print the package builds filter form on the first page only. // @@ -519,11 +532,11 @@ handle (request& rq, response& rs) for (const auto& c: *target_conf_map_) { - if (!exclude_hidden || belongs (*c.second, "all")) + if (!exclude_hidden || !belongs (*c.second, "hidden")) conf_ids.push_back (c.first); } - size_t count; + optional<size_t> count; size_t page (params.page ()); if (params.result () != "unbuilt") // Print package build configurations. @@ -538,37 +551,22 @@ handle (request& rq, response& rs) // printing the builds. // count = 0; - vector<shared_ptr<build>> builds; + vector<package_build> builds; builds.reserve (page_configs); - // Prepare the package build prepared query. + // Prepare the package build query. // using query = query<package_build>; - using prep_query = prepared_query<package_build>; - query q (build_query<package_build> ( - &conf_ids, params, tn, nullopt /* archived */)); - - // Specify the portion. Note that we will be querying builds in chunks, - // not to hold locks for too long. - // - // Also note that for each build we also load the corresponding - // package. Nevertheless, we use a fairly large portion to speed-up the - // builds traversal but also cache the package objects (see below). - // - size_t offset (0); + query q (build_query<package_build> (&conf_ids, params, tn)); // Print package build configurations ordered by the timestamp (later goes // first). // - q += "ORDER BY" + query::build::timestamp + "DESC" + - "OFFSET" + query::_ref (offset) + "LIMIT 500"; + q += "ORDER BY" + query::build::timestamp + "DESC"; connection_ptr conn (build_db_->connection ()); - prep_query pq ( - conn->prepare_query<package_build> ("mod-builds-query", q)); - // Note that we can't skip the proper number of builds in the database // query for a page numbers greater than one. So we will query builds from // the very beginning and skip the appropriate number of them while @@ -584,101 +582,102 @@ handle (request& rq, response& rs) // session sn; - for (bool ne (true); ne; ) + transaction t (conn->begin ()); + + // For some reason PostgreSQL (as of 9.4) picks the nested loop join + // strategy for the below package_build query, which executes quite slow + // even for reasonably small number of builds. Thus, we just discourage + // PostgreSQL from using this strategy in the current transaction. + // + // @@ TMP Re-check for the later PostgreSQL versions if we can drop this + // hint. If drop, then also grep for other places where this hint + // is used. + // + conn->execute ("SET LOCAL enable_nestloop=off"); + + // Iterate over builds and cache build objects that should be printed. + // Skip the appropriate number of them (for page number greater than + // one). + // + for (auto& pb: build_db_->query<package_build> (q)) { - transaction t (conn->begin ()); + shared_ptr<build>& b (pb.build); - // Query package builds (and cache the result). - // - auto bs (pq.execute ()); + auto i ( + target_conf_map_->find ( + build_target_config_id {b->target, b->target_config_name})); - if ((ne = !bs.empty ())) - { - offset += bs.size (); + assert (i != target_conf_map_->end ()); - // Iterate over builds and cache build objects that should be printed. - // Skip the appropriate number of them (for page number greater than - // one). - // - for (auto& pb: bs) - { - shared_ptr<build>& b (pb.build); + // Match the target configuration against the package build + // configuration expressions/constraints. + // + shared_ptr<build_package> p ( + build_db_->load<build_package> (b->id.package)); - auto i (target_conf_map_->find ( - build_target_config_id {b->target, - b->target_config_name})); + const build_package_config* pc (find (b->package_config_name, + p->configs)); - assert (i != target_conf_map_->end ()); + // The package configuration should be present since the configurations + // set cannot change if the package version doesn't change. If that's + // not the case, then the database has probably been manually amended. + // In this case let's just skip such a build as if it excluded and log + // the warning. + // + if (pc == nullptr) + { + warn << "cannot find configuration '" << b->package_config_name + << "' for package " << p->id.name << '/' << p->version; - // Match the target configuration against the package build - // configuration expressions/constraints. - // - shared_ptr<build_package> p ( - build_db_->load<build_package> (b->id.package)); + continue; + } - const build_package_config* pc (find (b->package_config_name, - p->configs)); + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); - // The package configuration should be present since the - // configurations set cannot change if the package version doesn't - // change. If that's not the case, then the database has probably - // been manually amended. In this case let's just skip such a build - // as if it excluded and log the warning. + if (!exclude (*pc, p->builds, p->constraints, *i->second)) + { + if (skip != 0) + --skip; + else if (print != 0) + { + // As we query builds in multiple transactions we may see the same + // build multiple times. Let's skip the duplicates. Note: we don't + // increment the counter in this case. // - if (pc == nullptr) - { - warn << "cannot find configuration '" << b->package_config_name - << "' for package " << p->id.name << '/' << p->version; - + if (find_if (builds.begin (), builds.end (), + [&b] (const package_build& pb) + { + return b->id == pb.build->id; + }) != builds.end ()) continue; - } - if (!exclude (*pc, p->builds, p->constraints, *i->second)) + if (b->state == build_state::built) { - if (skip != 0) - --skip; - else if (print != 0) - { - // As we query builds in multiple transactions we may see the - // same build multiple times. Let's skip the duplicates. Note: - // we don't increment the counter in this case. - // - if (find_if (builds.begin (), builds.end (), - [&b] (const shared_ptr<build>& pb) - { - return b->id == pb->id; - }) != builds.end ()) - continue; - - if (b->state == build_state::built) - { - build_db_->load (*b, b->results_section); + build_db_->load (*b, b->results_section); - // Let's clear unneeded result logs for builds being cached. - // - for (operation_result& r: b->results) - r.log.clear (); - } + // Let's clear unneeded result logs for builds being cached. + // + for (operation_result& r: b->results) + r.log.clear (); + } - builds.push_back (move (b)); + builds.push_back (move (pb)); - --print; - } - - ++count; - } + --print; } - } - // - // Print the filter form after the build count is calculated. Note: - // query_toolchains() must be called inside the build db transaction. - // - else - print_form (query_toolchains (), count); - t.commit (); + ++(*count); + } } + // Print the filter form after the build count is calculated. Note: + // query_toolchains() must be called inside the build db transaction. + // + print_form (query_toolchains (), count); + + t.commit (); + // Finally, print the cached package build configurations. // timestamp now (system_clock::now ()); @@ -686,19 +685,24 @@ handle (request& rq, response& rs) // Enclose the subsequent tables to be able to use nth-child CSS selector. // s << DIV; - for (const shared_ptr<build>& pb: builds) + for (const package_build& pb: builds) { - const build& b (*pb); + const build& b (*pb.build); string ts (butl::to_string (b.timestamp, "%Y-%m-%d %H:%M:%S %Z", true /* special */, true /* local */) + - " (" + butl::to_string (now - b.timestamp, false) + " ago)"); + " (" + butl::to_string (now - b.timestamp, false) + " ago"); + + if (pb.archived) + ts += ", archived"; + + ts += ')'; s << TABLE(CLASS="proplist build") << TBODY - << TR_NAME (b.package_name, string (), root, b.tenant) + << TR_NAME (b.package_name, root, b.tenant) << TR_VERSION (b.package_name, b.package_version, root, b.tenant) << TR_VALUE ("toolchain", b.toolchain_name + '-' + @@ -711,7 +715,7 @@ handle (request& rq, response& rs) if (b.interactive) // Note: can only be present for the building state. s << TR_VALUE ("login", *b.interactive); - s << TR_BUILD_RESULT (b, host, root); + s << TR_BUILD_RESULT (b, pb.archived, host, root); // In the global view mode add the tenant builds link. Note that the // global view (and the link) makes sense only in the multi-tenant mode. @@ -752,9 +756,27 @@ handle (request& rq, response& rs) const bpkg::version& toolchain_version; }; + // Cache the build package objects that would otherwise be loaded twice: + // first time during calculating the builds count and then during printing + // the builds. Note that the build package is a subset of the package + // object and normally has a small memory footprint. + // + // @@ TMP It feels that we can try to combine the mentioned steps and + // improve the performance a bit. We won't need the session in this + // case. + // + session sn; + + connection_ptr conn (build_db_->connection ()); + transaction t (conn->begin ()); + + // Discourage PostgreSQL from using the nested loop join strategy in the + // current transaction (see above for details). + // + conn->execute ("SET LOCAL enable_nestloop=off"); + vector<target_config_toolchain> config_toolchains; { - transaction t (build_db_->begin ()); toolchains = query_toolchains (); string th_name; @@ -797,7 +819,7 @@ handle (request& rq, response& rs) // (tgt.empty () || path_match (c.target.string (), tgt)) && - (!exclude_hidden || belongs (c, "all"))) // Filter hidden. + (!exclude_hidden || !belongs (c, "hidden"))) // Filter hidden. { target_configs.push_back (&c); @@ -829,9 +851,8 @@ handle (request& rq, response& rs) // size_t npos (0); - size_t ncur = build_db_->query_value<package_build_count> ( - build_query<package_build_count> ( - &conf_ids, bld_params, tn, false /* archived */)); + size_t ncur (build_db_->query_value<package_build_count> ( + build_query<package_build_count> (&conf_ids, bld_params, tn))); // From now we will be using specific values for the below filters for // each build database query. Note that the toolchain is the only @@ -873,8 +894,7 @@ handle (request& rq, response& rs) // build_query<package_build_count> (nullptr /* config_ids */, bld_params, - tn, - false /* archived */)); + tn)); prep_bld_query bld_prep_query ( build_db_->prepare_query<package_build_count> ( @@ -889,14 +909,16 @@ handle (request& rq, response& rs) // be easy as the cached values depend on the filter form parameters. // query<buildable_package> q ( - package_query<buildable_package> (params, tn, false /* archived */)); + package_query<buildable_package> (params, tn)); for (auto& bp: build_db_->query<buildable_package> (q)) { - id = move (bp.id); + shared_ptr<build_package>& p (bp.package); - shared_ptr<build_package> p (build_db_->load<build_package> (id)); + id = p->id; + // Note: load the constrains section lazily. + // for (const build_package_config& c: p->configs) { // Filter by package config name. @@ -905,6 +927,9 @@ handle (request& rq, response& rs) { for (const auto& tc: target_configs) { + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); + if (exclude (c, p->builds, p->constraints, *tc)) { target = tc->target; @@ -923,9 +948,7 @@ handle (request& rq, response& rs) count = npos - ncur; } else - count = 0; - - t.commit (); + count = nullopt; // Unknown count. } // Print the filter form. @@ -943,7 +966,7 @@ handle (request& rq, response& rs) // 7: target configuration name // 8: package configuration name // - // Prepare the build package prepared query. + // Prepare the build package query. // // Note that we can't skip the proper number of packages in the database // query for a page numbers greater than one. So we will query packages @@ -958,28 +981,14 @@ handle (request& rq, response& rs) // URL query parameter. Alternatively, we can invent the page number cap. // using pkg_query = query<buildable_package>; - using prep_pkg_query = prepared_query<buildable_package>; - pkg_query pq ( - package_query<buildable_package> (params, tn, false /* archived */)); - - // Specify the portion. Note that we will still be querying packages in - // chunks, not to hold locks for too long. For each package we will query - // its builds, so let's keep the portion small. - // - size_t offset (0); + pkg_query pq (package_query<buildable_package> (params, tn)); pq += "ORDER BY" + pkg_query::build_package::id.name + order_by_version_desc (pkg_query::build_package::id.version, false /* first */) + "," + - pkg_query::build_package::id.tenant + - "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"; - - connection_ptr conn (build_db_->connection ()); - - prep_pkg_query pkg_prep_query ( - conn->prepare_query<buildable_package> ("mod-builds-package-query", pq)); + pkg_query::build_package::id.tenant; // Prepare the build prepared query. // @@ -999,8 +1008,7 @@ handle (request& rq, response& rs) // via the build package id, we still need to pass the // tenant not to erroneously filter out the private tenants. // - build_query<package_build> ( - &conf_ids, bld_params, tn, false /* archived */)); + build_query<package_build> (&conf_ids, bld_params, tn)); prep_bld_query bld_prep_query ( conn->prepare_query<package_build> ("mod-builds-build-query", bq)); @@ -1011,115 +1019,115 @@ handle (request& rq, response& rs) // Enclose the subsequent tables to be able to use nth-child CSS selector. // s << DIV; - while (print != 0) - { - transaction t (conn->begin ()); - // Query (and cache) buildable packages. - // - auto packages (pkg_prep_query.execute ()); + // Query (and cache) buildable packages. + // + auto packages (build_db_->query<buildable_package> (pq)); - if (packages.empty ()) - print = 0; - else + if (packages.empty ()) + print = 0; + else + { + // Iterate over packages and print unbuilt configurations. Skip the + // appropriate number of them first (for page number greater than one). + // + for (auto& bp: packages) { - offset += packages.size (); + shared_ptr<build_package>& p (bp.package); - // Iterate over packages and print unbuilt configurations. Skip the - // appropriate number of them first (for page number greater than one). - // - for (auto& p: packages) - { - id = move (p.id); + id = p->id; - shared_ptr<build_package> bp (build_db_->load<build_package> (id)); + // Copy configuration/toolchain combinations for this package, + // skipping excluded configurations. + // + set<config_toolchain> unbuilt_configs; - // Copy configuration/toolchain combinations for this package, - // skipping excluded configurations. + // Load the constrains section lazily. + // + for (const build_package_config& pc: p->configs) + { + // Filter by package config name. // - set<config_toolchain> unbuilt_configs; - - for (const build_package_config& pc: bp->configs) + if (pkg_cfg.empty () || path_match (pc.name, pkg_cfg)) { - // Filter by package config name. - // - if (pkg_cfg.empty () || path_match (pc.name, pkg_cfg)) + for (const target_config_toolchain& ct: config_toolchains) { - for (const target_config_toolchain& ct: config_toolchains) - { - auto i ( - target_conf_map_->find ( - build_target_config_id {ct.target, ct.target_config})); - - assert (i != target_conf_map_->end ()); - - if (!exclude (pc, bp->builds, bp->constraints, *i->second)) - unbuilt_configs.insert ( - config_toolchain {ct.target, - ct.target_config, - pc.name, - ct.toolchain_name, - ct.toolchain_version}); - } + auto i ( + target_conf_map_->find ( + build_target_config_id {ct.target, ct.target_config})); + + assert (i != target_conf_map_->end ()); + + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); + + if (!exclude (pc, p->builds, p->constraints, *i->second)) + unbuilt_configs.insert ( + config_toolchain {ct.target, + ct.target_config, + pc.name, + ct.toolchain_name, + ct.toolchain_version}); } } + } - // Iterate through the package configuration builds and erase them - // from the unbuilt configurations set. - // - for (const auto& pb: bld_prep_query.execute ()) - { - const build& b (*pb.build); + // Iterate through the package configuration builds and erase them + // from the unbuilt configurations set. + // + for (const auto& pb: bld_prep_query.execute ()) + { + const build& b (*pb.build); - unbuilt_configs.erase (config_toolchain {b.target, - b.target_config_name, - b.package_config_name, - b.toolchain_name, - b.toolchain_version}); - } + unbuilt_configs.erase (config_toolchain {b.target, + b.target_config_name, + b.package_config_name, + b.toolchain_name, + b.toolchain_version}); + } - // Print unbuilt package configurations. - // - for (const auto& ct: unbuilt_configs) + // Print unbuilt package configurations. + // + for (const auto& ct: unbuilt_configs) + { + if (skip != 0) { - if (skip != 0) - { - --skip; - continue; - } - - s << TABLE(CLASS="proplist build") - << TBODY - << TR_NAME (id.name, string (), root, id.tenant) - << TR_VERSION (id.name, p.version, root, id.tenant) - << TR_VALUE ("toolchain", - string (ct.toolchain_name) + '-' + - ct.toolchain_version.string ()) - << TR_VALUE ("target", ct.target.string ()) - << TR_VALUE ("tgt config", ct.target_config) - << TR_VALUE ("pkg config", ct.package_config); - - // In the global view mode add the tenant builds link. Note that - // the global view (and the link) makes sense only in the - // multi-tenant mode. - // - if (!tn && !id.tenant.empty ()) - s << TR_TENANT (tenant_name, "builds", root, id.tenant); + --skip; + continue; + } - s << ~TBODY - << ~TABLE; + s << TABLE(CLASS="proplist build") + << TBODY + << TR_NAME (id.name, root, id.tenant) + << TR_VERSION (id.name, p->version, root, id.tenant) + << TR_VALUE ("toolchain", + string (ct.toolchain_name) + '-' + + ct.toolchain_version.string ()) + << TR_VALUE ("target", ct.target.string ()) + << TR_VALUE ("tgt config", ct.target_config) + << TR_VALUE ("pkg config", ct.package_config); + + // In the global view mode add the tenant builds link. Note that + // the global view (and the link) makes sense only in the + // multi-tenant mode. + // + if (!tn && !id.tenant.empty ()) + s << TR_TENANT (tenant_name, "builds", root, id.tenant); - if (--print == 0) // Bail out the configuration loop. - break; - } + s << ~TBODY + << ~TABLE; - if (print == 0) // Bail out the package loop. + if (--print == 0) // Bail out the configuration loop. break; } - } - t.commit (); + if (print == 0) // Bail out the package loop. + break; + } } + + t.commit (); + s << ~DIV; } @@ -1151,7 +1159,11 @@ handle (request& rq, response& rs) add_filter ("pc", pkg_cfg); add_filter ("rs", params.result (), "*"); - s << DIV_PAGER (page, count, page_configs, options_->build_pages (), u) + s << DIV_PAGER (page, + count ? *count : 0, + page_configs, + options_->build_pages (), + u) << ~DIV << ~BODY << ~HTML; diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx index 68bb9be..5974d45 100644 --- a/mod/mod-ci.cxx +++ b/mod/mod-ci.cxx @@ -3,18 +3,11 @@ #include <mod/mod-ci.hxx> -#include <ostream> - -#include <libbutl/uuid.hxx> -#include <libbutl/sendmail.hxx> #include <libbutl/fdstream.hxx> -#include <libbutl/timestamp.hxx> -#include <libbutl/filesystem.hxx> -#include <libbutl/process-io.hxx> // operator<<(ostream, process_args) #include <libbutl/manifest-parser.hxx> #include <libbutl/manifest-serializer.hxx> -#include <libbpkg/manifest.hxx> +#include <libbpkg/manifest.hxx> // package_manifest #include <libbpkg/package-name.hxx> #include <web/server/module.hxx> @@ -23,20 +16,35 @@ #include <mod/page.hxx> #include <mod/module-options.hxx> -#include <mod/external-handler.hxx> using namespace std; using namespace butl; using namespace web; using namespace brep::cli; +#ifdef BREP_CI_TENANT_SERVICE +brep::ci:: +ci (tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} +#endif + brep::ci:: +#ifdef BREP_CI_TENANT_SERVICE +ci (const ci& r, tenant_service_map& tsm) +#else ci (const ci& r) +#endif : handler (r), + ci_start (r), options_ (r.initialized_ ? r.options_ : nullptr), form_ (r.initialized_ || r.form_ == nullptr ? r.form_ : make_shared<xhtml::fragment> (*r.form_)) +#ifdef BREP_CI_TENANT_SERVICE + , tenant_service_map_ (tsm) +#endif { } @@ -45,22 +53,25 @@ init (scanner& s) { HANDLER_DIAG; +#ifdef BREP_CI_TENANT_SERVICE + { + shared_ptr<tenant_service_base> ts ( + dynamic_pointer_cast<tenant_service_base> (shared_from_this ())); + + assert (ts != nullptr); // By definition. + + tenant_service_map_["ci"] = move (ts); + } +#endif + options_ = make_shared<options::ci> ( s, unknown_mode::fail, unknown_mode::fail); - // Verify that the CI request handling is setup properly, if configured. + // Prepare for the CI requests handling, if configured. // if (options_->ci_data_specified ()) { - // Verify the data directory satisfies the requirements. - // - const dir_path& d (options_->ci_data ()); - - if (d.relative ()) - fail << "ci-data directory path must be absolute"; - - if (!dir_exists (d)) - fail << "ci-data directory '" << d << "' does not exist"; + ci_start::init (make_shared<options::ci_start> (*options_)); // Parse XHTML5 form file, if configured. // @@ -87,10 +98,6 @@ init (scanner& s) fail << "unable to read ci-form file '" << ci_form << "': " << e; } } - - if (options_->ci_handler_specified () && - options_->ci_handler ().relative ()) - fail << "ci-handler path must be absolute"; } if (options_->root ().empty ()) @@ -130,9 +137,8 @@ handle (request& rq, response& rs) // // return respond_error (); // Request is handled with an error. // - string request_id; // Will be set later. - auto respond_manifest = [&rs, &request_id] (status_code status, - const string& message) -> bool + auto respond_manifest = [&rs] (status_code status, + const string& message) -> bool { serializer s (rs.content (status, "text/manifest;charset=utf-8"), "response"); @@ -140,10 +146,6 @@ handle (request& rq, response& rs) s.next ("", "1"); // Start of manifest. s.next ("status", to_string (status)); s.next ("message", message); - - if (!request_id.empty ()) - s.next ("reference", request_id); - s.next ("", ""); // End of manifest. return true; }; @@ -234,9 +236,11 @@ handle (request& rq, response& rs) if (rl.empty () || rl.local ()) return respond_manifest (400, "invalid repository location"); - // Verify the package name[/version] arguments. + // Parse the package name[/version] arguments. // - for (const string& s: params.package()) + vector<package> packages; + + for (const string& s: params.package ()) { // Let's skip the potentially unfilled package form fields. // @@ -245,18 +249,21 @@ handle (request& rq, response& rs) try { + package pkg; size_t p (s.find ('/')); if (p != string::npos) { - package_name (string (s, 0, p)); + pkg.name = package_name (string (s, 0, p)); // Not to confuse with module::version. // - bpkg::version (string (s, p + 1)); + pkg.version = bpkg::version (string (s, p + 1)); } else - package_name p (s); // Not to confuse with the s variable declaration. + pkg.name = package_name (s); + + packages.push_back (move (pkg)); } catch (const invalid_argument&) { @@ -265,31 +272,49 @@ handle (request& rq, response& rs) } // Verify that unknown parameter values satisfy the requirements (contain - // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n'). + // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n') and + // stash them. // // Actually, the expected ones must satisfy too, so check them as well. // - string what; - for (const name_value& nv: rps) + vector<pair<string, string>> custom_request; { - if (nv.value && - !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t")) - return respond_manifest (400, - "invalid parameter " + nv.name + ": " + what); + string what; + for (const name_value& nv: rps) + { + if (nv.value && + !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t")) + return respond_manifest (400, + "invalid parameter " + nv.name + ": " + what); + + const string& n (nv.name); + + if (n != "repository" && + n != "_" && + n != "package" && + n != "overrides" && + n != "interactive" && + n != "simulate") + custom_request.emplace_back (n, nv.value ? *nv.value : ""); + } } // Parse and validate overrides, if present. // - vector<manifest_name_value> overrides; + vector<pair<string, string>> overrides; if (params.overrides_specified ()) try { istream& is (rq.open_upload ("overrides")); parser mp (is, "overrides"); - overrides = parse_manifest (mp); + vector<manifest_name_value> ovrs (parse_manifest (mp)); + + package_manifest::validate_overrides (ovrs, mp.name ()); - package_manifest::validate_overrides (overrides, mp.name ()); + overrides.reserve (ovrs.size ()); + for (manifest_name_value& nv: ovrs) + overrides.emplace_back (move (nv.name), move (nv.value)); } // Note that invalid_argument (thrown by open_upload() function call) can // mean both no overrides upload or multiple overrides uploads. @@ -310,385 +335,141 @@ handle (request& rq, response& rs) return respond_error (); } - try - { - // Note that from now on the result manifest we respond with will contain - // the reference value. - // - request_id = uuid::generate ().string (); - } - catch (const system_error& e) - { - error << "unable to generate request id: " << e; - return respond_error (); - } - - // Create the submission data directory. + // Stash the User-Agent HTTP header and the client IP address. // - dir_path dd (options_->ci_data () / dir_path (request_id)); - - try + optional<string> client_ip; + optional<string> user_agent; + for (const name_value& h: rq.headers ()) { - // It's highly unlikely but still possible that the directory already - // exists. This can only happen if the generated uuid is not unique. - // - if (try_mkdir (dd) == mkdir_status::already_exists) - throw_generic_error (EEXIST); + if (icasecmp (h.name, ":Client-IP") == 0) + client_ip = h.value; + else if (icasecmp (h.name, "User-Agent") == 0) + user_agent = h.value; } - catch (const system_error& e) - { - error << "unable to create directory '" << dd << "': " << e; - return respond_error (); - } - - auto_rmdir ddr (dd); - - // Serialize the CI request manifest to a stream. On the serialization error - // respond to the client with the manifest containing the bad request (400) - // code and return false, on the stream error pass through the io_error - // exception, otherwise return true. - // - timestamp ts (system_clock::now ()); - - auto rqm = [&request_id, - &rl, - &ts, - &simulate, - &rq, - &rps, - ¶ms, - &respond_manifest] - (ostream& os, bool long_lines = false) -> bool - { - try - { - serializer s (os, "request", long_lines); - // Serialize the submission manifest header. - // - s.next ("", "1"); // Start of manifest. - s.next ("id", request_id); - s.next ("repository", rl.string ()); - - for (const string& p: params.package ()) - { - if (!p.empty ()) // Skip empty package names (see above for details). - s.next ("package", p); - } - - if (params.interactive_specified ()) - s.next ("interactive", params.interactive ()); - - if (!simulate.empty ()) - s.next ("simulate", simulate); - - s.next ("timestamp", - butl::to_string (ts, - "%Y-%m-%dT%H:%M:%SZ", - false /* special */, - false /* local */)); - - // Serialize the User-Agent HTTP header and the client IP address. - // - optional<string> ip; - optional<string> ua; - for (const name_value& h: rq.headers ()) - { - if (icasecmp (h.name, ":Client-IP") == 0) - ip = h.value; - else if (icasecmp (h.name, "User-Agent") == 0) - ua = h.value; - } - - if (ip) - s.next ("client-ip", *ip); - - if (ua) - s.next ("user-agent", *ua); - - // Serialize the request parameters. - // - // Note that the serializer constraints the parameter names (can't start - // with '#', can't contain ':' and the whitespaces, etc.). - // - for (const name_value& nv: rps) - { - const string& n (nv.name); - - if (n != "repository" && - n != "_" && - n != "package" && - n != "overrides" && - n != "interactive" && - n != "simulate") - s.next (n, nv.value ? *nv.value : ""); - } - - s.next ("", ""); // End of manifest. - return true; - } - catch (const serialization& e) - { - respond_manifest (400, string ("invalid parameter: ") + e.what ()); - return false; - } - }; - - // Serialize the CI request manifest to the submission directory. - // - path rqf (dd / "request.manifest"); + optional<start_result> r (start (error, + warn, + verb_ ? &trace : nullptr, +#ifdef BREP_CI_TENANT_SERVICE + tenant_service ("", "ci"), +#else + nullopt /* service */, +#endif + rl, + packages, + client_ip, + user_agent, + (params.interactive_specified () + ? params.interactive () + : optional<string> ()), + (!simulate.empty () + ? simulate + : optional<string> ()), + custom_request, + overrides)); + + if (!r) + return respond_error (); // The diagnostics is already issued. try { - ofdstream os (rqf); - bool r (rqm (os)); - os.close (); - - if (!r) - return true; // The client is already responded with the manifest. - } - catch (const io_error& e) - { - error << "unable to write to '" << rqf << "': " << e; - return respond_error (); + serialize_manifest (*r, + rs.content (r->status, "text/manifest;charset=utf-8")); } - - // Serialize the CI overrides manifest to a stream. On the stream error pass - // through the io_error exception. - // - // Note that it can't throw the serialization exception as the override - // manifest is parsed from the stream and so verified. - // - auto ovm = [&overrides] (ostream& os, bool long_lines = false) + catch (const serialization& e) { - try - { - serializer s (os, "overrides", long_lines); - serialize_manifest (s, overrides); - } - catch (const serialization&) {assert (false);} // See above. - }; - - // Serialize the CI overrides manifest to the submission directory. - // - path ovf (dd / "overrides.manifest"); + error << "ref " << r->reference << ": unable to serialize handler's " + << "output: " << e; - if (!overrides.empty ()) - try - { - ofdstream os (ovf); - ovm (os); - os.close (); - } - catch (const io_error& e) - { - error << "unable to write to '" << ovf << "': " << e; return respond_error (); } - // Given that the submission data is now successfully persisted we are no - // longer in charge of removing it, except for the cases when the submission - // handler terminates with an error (see below for details). - // - ddr.cancel (); - - // If the handler terminates with non-zero exit status or specifies 5XX - // (HTTP server error) submission result manifest status value, then we - // stash the submission data directory for troubleshooting. Otherwise, if - // it's the 4XX (HTTP client error) status value, then we remove the - // directory. - // - // Note that leaving the directory in place in case of a submission error - // would have prevent the user from re-submitting until we research the - // issue and manually remove the directory. - // - auto stash_submit_dir = [&dd, error] () - { - if (dir_exists (dd)) - try - { - mvdir (dd, dir_path (dd + ".fail")); - } - catch (const system_error& e) - { - // Not much we can do here. Let's just log the issue and bail out - // leaving the directory in place. - // - error << "unable to rename directory '" << dd << "': " << e; - } - }; - - // Run the submission handler, if specified, reading the result manifest - // from its stdout and caching it as a name/value pair list for later use - // (forwarding to the client, sending via email, etc.). Otherwise, create - // implied result manifest. - // - status_code sc; - vector<manifest_name_value> rvs; - - if (options_->ci_handler_specified ()) - { - using namespace external_handler; - - optional<result_manifest> r (run (options_->ci_handler (), - options_->ci_handler_argument (), - dd, - options_->ci_handler_timeout (), - error, - warn, - verb_ ? &trace : nullptr)); - if (!r) - { - stash_submit_dir (); - return respond_error (); // The diagnostics is already issued. - } - - sc = r->status; - rvs = move (r->values); - } - else // Create the implied result manifest. - { - sc = 200; - - auto add = [&rvs] (string n, string v) - { - manifest_name_value nv { - move (n), move (v), - 0 /* name_line */, 0 /* name_column */, - 0 /* value_line */, 0 /* value_column */, - 0 /* start_pos */, 0 /* colon_pos */, 0 /* end_pos */}; - - rvs.emplace_back (move (nv)); - }; - - add ("status", "200"); - add ("message", "CI request is queued"); - add ("reference", request_id); - } - - assert (!rvs.empty ()); // Produced by the handler or is implied. - - // Serialize the submission result manifest to a stream. On the - // serialization error log the error description and return false, on the - // stream error pass through the io_error exception, otherwise return true. - // - auto rsm = [&rvs, &error, &request_id] (ostream& os, - bool long_lines = false) -> bool - { - try - { - serializer s (os, "result", long_lines); - serialize_manifest (s, rvs); - return true; - } - catch (const serialization& e) - { - error << "ref " << request_id << ": unable to serialize handler's " - << "output: " << e; - return false; - } - }; - - // If the submission data directory still exists then perform an appropriate - // action on it, depending on the submission result status. Note that the - // handler could move or remove the directory. - // - if (dir_exists (dd)) - { - // Remove the directory if the client error is detected. - // - if (sc >= 400 && sc < 500) - rmdir_r (dd); - - // Otherwise, save the result manifest, into the directory. Also stash the - // directory for troubleshooting in case of the server error. - // - else - { - path rsf (dd / "result.manifest"); - - try - { - ofdstream os (rsf); - - // Not being able to stash the result manifest is not a reason to - // claim the submission failed. The error is logged nevertheless. - // - rsm (os); - - os.close (); - } - catch (const io_error& e) - { - // Not fatal (see above). - // - error << "unable to write to '" << rsf << "': " << e; - } - - if (sc >= 500 && sc < 600) - stash_submit_dir (); - } - } - - // Send email, if configured, and the CI request submission is not simulated. - // Use the long lines manifest serialization mode for the convenience of - // copying/clicking URLs they contain. - // - // Note that we don't consider the email sending failure to be a submission - // failure as the submission data is successfully persisted and the handler - // is successfully executed, if configured. One can argue that email can be - // essential for the submission processing and missing it would result in - // the incomplete submission. In this case it's natural to assume that the - // web server error log is monitored and the email sending failure will be - // noticed. - // - if (options_->ci_email_specified () && simulate.empty ()) - try - { - // Redirect the diagnostics to the web server error log. - // - sendmail sm ([&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }, - 2 /* stderr */, - options_->email (), - "CI request submission (" + request_id + ')', - {options_->ci_email ()}); - - // Write the CI request manifest. - // - bool r (rqm (sm.out, true /* long_lines */)); - assert (r); // The serialization succeeded once, so can't fail now. - - // Write the CI overrides manifest. - // - sm.out << "\n\n"; - - ovm (sm.out, true /* long_lines */); - - // Write the CI result manifest. - // - sm.out << "\n\n"; - - // We don't care about the result (see above). - // - rsm (sm.out, true /* long_lines */); - - sm.out.close (); + return true; +} - if (!sm.wait ()) - error << "sendmail " << *sm.exit; - } - // Handle process_error and io_error (both derive from system_error). - // - catch (const system_error& e) - { - error << "sendmail error: " << e; - } +#ifdef BREP_CI_TENANT_SERVICE +function<optional<string> (const brep::tenant_service&)> brep::ci:: +build_queued (const tenant_service&, + const vector<build>& bs, + optional<build_state> initial_state, + const build_queued_hints& hints, + const diag_epilogue& log_writer) const noexcept +{ + NOTIFICATION_DIAG (log_writer); + + l2 ([&]{trace << "initial_state: " + << (initial_state ? to_string (*initial_state) : "none") + << ", hints " + << hints.single_package_version << ' ' + << hints.single_package_config;}); + + return [&bs, initial_state] (const tenant_service& ts) + { + optional<string> r (ts.data); + + for (const build& b: bs) + { + string s ((!initial_state + ? "queued " + : "queued " + to_string (*initial_state) + ' ') + + b.package_name.string () + '/' + + b.package_version.string () + '/' + + b.target.string () + '/' + + b.target_config_name + '/' + + b.package_config_name + '/' + + b.toolchain_name + '/' + + b.toolchain_version.string ()); + + if (r) + { + *r += ", "; + *r += s; + } + else + r = move (s); + } + + return r; + }; +} - if (!rsm (rs.content (sc, "text/manifest;charset=utf-8"))) - return respond_error (); // The error description is already logged. +function<optional<string> (const brep::tenant_service&)> brep::ci:: +build_building (const tenant_service&, + const build& b, + const diag_epilogue&) const noexcept +{ + return [&b] (const tenant_service& ts) + { + string s ("building " + + b.package_name.string () + '/' + + b.package_version.string () + '/' + + b.target.string () + '/' + + b.target_config_name + '/' + + b.package_config_name + '/' + + b.toolchain_name + '/' + + b.toolchain_version.string ()); + + return ts.data ? *ts.data + ", " + s : s; + }; +} - return true; +function<optional<string> (const brep::tenant_service&)> brep::ci:: +build_built (const tenant_service&, + const build& b, + const diag_epilogue&) const noexcept +{ + return [&b] (const tenant_service& ts) + { + string s ("built " + + b.package_name.string () + '/' + + b.package_version.string () + '/' + + b.target.string () + '/' + + b.target_config_name + '/' + + b.package_config_name + '/' + + b.toolchain_name + '/' + + b.toolchain_version.string ()); + + return ts.data ? *ts.data + ", " + s : s; + }; } +#endif diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx index 431f53b..1e2ee15 100644 --- a/mod/mod-ci.hxx +++ b/mod/mod-ci.hxx @@ -9,14 +9,39 @@ #include <libbrep/types.hxx> #include <libbrep/utility.hxx> +#include <libbrep/build.hxx> +#include <libbrep/common.hxx> // tenant_service + #include <mod/module.hxx> #include <mod/module-options.hxx> +#include <mod/ci-common.hxx> + +#ifdef BREP_CI_TENANT_SERVICE +# include <mod/tenant-service.hxx> +#endif + namespace brep { - class ci: public handler + class ci: public handler, + private ci_start +#ifdef BREP_CI_TENANT_SERVICE + , public tenant_service_build_queued, + public tenant_service_build_building, + public tenant_service_build_built +#endif { public: + +#ifdef BREP_CI_TENANT_SERVICE + explicit + ci (tenant_service_map&); + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + ci (const ci&, tenant_service_map&); +#else ci () = default; // Create a shallow copy (handling instance) if initialized and a deep @@ -24,20 +49,44 @@ namespace brep // explicit ci (const ci&); +#endif virtual bool - handle (request&, response&); + handle (request&, response&) override; virtual const cli::options& - cli_options () const {return options::ci::description ();} + cli_options () const override {return options::ci::description ();} + +#ifdef BREP_CI_TENANT_SERVICE + virtual function<optional<string> (const tenant_service&)> + build_queued (const tenant_service&, + const vector<build>&, + optional<build_state> initial_state, + const build_queued_hints&, + const diag_epilogue& log_writer) const noexcept override; + + virtual function<optional<string> (const tenant_service&)> + build_building (const tenant_service&, + const build&, + const diag_epilogue& log_writer) const noexcept override; + + virtual function<optional<string> (const tenant_service&)> + build_built (const tenant_service&, + const build&, + const diag_epilogue& log_writer) const noexcept override; +#endif private: virtual void - init (cli::scanner&); + init (cli::scanner&) override; private: shared_ptr<options::ci> options_; shared_ptr<web::xhtml::fragment> form_; + +#ifdef BREP_CI_TENANT_SERVICE + tenant_service_map& tenant_service_map_; +#endif }; } diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx index 13e6422..fcd50da 100644 --- a/mod/mod-package-details.cxx +++ b/mod/mod-package-details.cxx @@ -183,20 +183,20 @@ handle (request& rq, response& rs) // s << H2 << pkg->summary << ~H2; - if (const optional<string>& d = pkg->description) + if (const optional<typed_text>& d = pkg->package_description + ? pkg->package_description + : pkg->description) { const string id ("description"); const string what (name.string () + " description"); s << (full ? DIV_TEXT (*d, - *pkg->description_type, true /* strip_title */, id, what, error) : DIV_TEXT (*d, - *pkg->description_type, true /* strip_title */, options_->package_description (), url (!full, squery, page, id), @@ -227,7 +227,7 @@ handle (request& rq, response& rs) << ~TABLE; } - auto pkg_count ( + size_t pkg_count ( package_db_->query_value<package_count> ( search_params<package_count> (squery, tenant, name))); diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index b158228..91923e5 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -9,6 +9,8 @@ #include <odb/database.hxx> #include <odb/transaction.hxx> +#include <libbutl/filesystem.hxx> // dir_iterator, dir_entry + #include <web/server/module.hxx> #include <web/server/mime-url-encoding.hxx> @@ -47,6 +49,12 @@ init (scanner& s) options_ = make_shared<options::package_version_details> ( s, unknown_mode::fail, unknown_mode::fail); + // Verify that the bindist-url option is specified when necessary. + // + if (options_->bindist_root_specified () && + !options_->bindist_url_specified ()) + fail << "bindist-url must be specified if bindist-root is specified"; + database_module::init (static_cast<const options::package_db&> (*options_), options_->package_db_retry ()); @@ -181,20 +189,20 @@ handle (request& rq, response& rs) s << H2 << pkg->summary << ~H2; - if (const optional<string>& d = pkg->description) + if (const optional<typed_text>& d = pkg->package_description + ? pkg->package_description + : pkg->description) { const string id ("description"); const string what (title + " description"); s << (full - ? DIV_TEXT (*d, * - pkg->description_type, + ? DIV_TEXT (*d, true /* strip_title */, id, what, error) : DIV_TEXT (*d, - *pkg->description_type, true /* strip_title */, options_->package_description (), url (!full, id), @@ -445,7 +453,10 @@ handle (request& rq, response& rs) // // Print test dependencies of the specific type. // - auto print_tests = [&pkg, &s, &print_dependency] (test_dependency_type dt) + auto print_tests = [&pkg, + &s, + &print_dependency, + full] (test_dependency_type dt) { string id; @@ -484,6 +495,20 @@ handle (request& rq, response& rs) print_dependency (td); + if (td.enable || td.reflect) + { + if (full) + { + if (td.enable) + s << " ? (" << *td.enable << ')'; + + if (td.reflect) + s << ' ' << *td.reflect; + } + else + s << " ..."; + } + s << ~SPAN << ~TD << ~TR; @@ -509,16 +534,24 @@ handle (request& rq, response& rs) { package_db_->load (*pkg, pkg->build_section); - // If the package has a singe build configuration class expression with - // exactly one underlying class and the class is none, then we just drop - // the page builds section altogether. + // If all package build configurations has a singe effective build + // configuration class expression with exactly one underlying class and + // the class is none, then we just drop the page builds section + // altogether. // - if (pkg->builds.size () == 1) + builds = false; + + for (const package_build_config& pc: pkg->build_configs) { - const build_class_expr& be (pkg->builds[0]); + const build_class_exprs& exprs (pc.effective_builds (pkg->builds)); - builds = be.underlying_classes.size () != 1 || - be.underlying_classes[0] != "none"; + if (exprs.size () != 1 || + exprs[0].underlying_classes.size () != 1 || + exprs[0].underlying_classes[0] != "none") + { + builds = true; + break; + } } } @@ -526,12 +559,174 @@ handle (request& rq, response& rs) t.commit (); + // Display the binary distribution packages for this tenant, package, and + // version, if present. Print the archive distributions last. + // + if (options_->bindist_root_specified ()) + { + // Collect all the available package configurations by iterating over the + // <distribution> and <os-release> subdirectories and the <package-config> + // symlinks in the following filesystem hierarchy: + // + // [<tenant>/]<distribution>/<os-release>/<project>/<package>/<version>/<package-config> + // + // Note that it is possible that new directories and symlinks are created + // and/or removed while we iterate over the filesystem entries in the + // above hierarchy, which may result with system_error exceptions. If that + // happens, we just ignore such exceptions, trying to collect what we can. + // + const dir_path& br (options_->bindist_root ()); + + dir_path d (br); + + if (!tenant.empty ()) + d /= tenant; + + // Note that distribution and os_release are simple paths and the + // config_symlink and config_dir are relative to the bindist root + // directory. + // + struct bindist_config + { + dir_path distribution; // debian, fedora, archive + dir_path os_release; // fedora37, windows10 + path symlink; // .../x86_64, .../x86_64-release + dir_path directory; // .../x86_64-2023-05-11T10:13:43Z + + bool + operator< (const bindist_config& v) + { + if (int r = distribution.compare (v.distribution)) + return distribution.string () == "archive" ? false : + v.distribution.string () == "archive" ? true : + r < 0; + + if (int r = os_release.compare (v.os_release)) + return r < 0; + + return symlink < v.symlink; + } + }; + + vector<bindist_config> configs; + + if (dir_exists (d)) + try + { + for (const dir_entry& de: dir_iterator (d, dir_iterator::ignore_dangling)) + { + if (de.type () != entry_type::directory) + continue; + + // Distribution directory. + // + dir_path dd (path_cast<dir_path> (de.path ())); + + try + { + dir_path fdd (d / dd); + + for (const dir_entry& re: + dir_iterator (fdd, dir_iterator::ignore_dangling)) + { + if (re.type () != entry_type::directory) + continue; + + // OS release directory. + // + dir_path rd (path_cast<dir_path> (re.path ())); + + // Package version directory. + // + dir_path vd (fdd / + rd / + dir_path (pkg->project.string ()) / + dir_path (pn.string ()) / + dir_path (sver)); + + try + { + for (const dir_entry& ce: + dir_iterator (vd, dir_iterator::ignore_dangling)) + { + if (ce.ltype () != entry_type::symlink) + continue; + + // Skip the "hidden" symlinks which may potentially be used by + // the upload handlers until they expose the finalized upload + // directory. + // + const path& cl (ce.path ()); + if (cl.string () [0] == '.') + continue; + + try + { + path fcl (vd / cl); + dir_path cd (path_cast<dir_path> (followsymlink (fcl))); + + if (cd.sub (br)) + configs.push_back ( + bindist_config {dd, rd, fcl.leaf (br), cd.leaf (br)}); + } + catch (const system_error&) {} + } + } + catch (const system_error&) {} + } + } + catch (const system_error&) {} + } + } + catch (const system_error&) {} + + // Sort and print collected package configurations, if any. + // + if (!configs.empty ()) + { + sort (configs.begin (), configs.end ()); + + s << H3 << "Binaries" << ~H3 + << TABLE(ID="binaries") + << TBODY; + + for (const bindist_config& c: configs) + { + s << TR(CLASS="binaries") + << TD << SPAN(CLASS="value") << c.distribution << ~SPAN << ~TD + << TD << SPAN(CLASS="value") << c.os_release << ~SPAN << ~TD + << TD + << SPAN(CLASS="value") + << A + << HREF + << options_->bindist_url () << '/' << c.symlink + << ~HREF + << c.symlink.leaf () + << ~A + << " (" + << A + << HREF + << options_->bindist_url () << '/' << c.directory + << ~HREF + << "snapshot" + << ~A + << ")" + << ~SPAN + << ~TD + << ~TR; + } + + s << ~TBODY + << ~TABLE; + } + } + if (builds) { s << H3 << "Builds" << ~H3 << DIV(ID="builds"); - auto exclude = [&pkg, this] (const build_package_config& pc, + auto exclude = [&pkg, this] (const package_build_config& pc, const build_target_config& tc, string* rs = nullptr) { @@ -547,13 +742,7 @@ handle (request& rq, response& rs) // Query toolchains seen for the package tenant to produce a list of the // unbuilt configuration/toolchain combinations. // - // Note that it only make sense to print those unbuilt configurations that - // may still be built. That's why we leave the toolchains list empty if - // the package tenant is achieved. - // vector<pair<string, version>> toolchains; - - if (!tn->archived) { using query = query<toolchain>; @@ -564,7 +753,9 @@ handle (request& rq, response& rs) "ORDER BY" + query::build::id.toolchain_name + order_by_version_desc (query::build::id.toolchain_version, false /* first */))) + { toolchains.emplace_back (move (t.name), move (t.version)); + } } // Compose the configuration filtering sub-query and collect unbuilt @@ -576,13 +767,13 @@ handle (request& rq, response& rs) query sq (false); set<config_toolchain> unbuilt_configs; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& bc: *target_conf_map_) { const build_target_config& tc (*bc.second); - if (belongs (tc, "all") && !exclude (pc, tc)) + if (!belongs (tc, "hidden") && !exclude (pc, tc)) { const build_target_config_id& id (bc.first); @@ -611,15 +802,24 @@ handle (request& rq, response& rs) // Print the package built configurations in the time-descending order. // for (auto& b: build_db_->query<build> ( - (query::id.package == pkg->id && sq) + + (query::id.package == pkg->id && query::state != "queued" && sq) + "ORDER BY" + query::timestamp + "DESC")) { string ts (butl::to_string (b.timestamp, "%Y-%m-%d %H:%M:%S %Z", true /* special */, true /* local */) + - " (" + butl::to_string (now - b.timestamp, false) + " ago)"); + " (" + butl::to_string (now - b.timestamp, false) + " ago"); + + if (tn->archived) + ts += ", archived"; + + ts += ')'; + // @@ Note that here we also load result logs which we don't need. + // Probably we should invent some table view to only load operation + // names and statuses. + // if (b.state == build_state::built) build_db_->load (b, b.results_section); @@ -639,7 +839,7 @@ handle (request& rq, response& rs) if (b.interactive) // Note: can only be present for the building state. s << TR_VALUE ("login", *b.interactive); - s << TR_BUILD_RESULT (b, host, root) + s << TR_BUILD_RESULT (b, tn->archived, host, root) << ~TBODY << ~TABLE; @@ -686,7 +886,7 @@ handle (request& rq, response& rs) // if (!tn->interactive) { - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& tc: *target_conf_) { @@ -717,19 +917,25 @@ handle (request& rq, response& rs) s << ~DIV; } - const string& ch (pkg->changes); - - if (!ch.empty ()) + if (const optional<typed_text>& c = pkg->changes) { const string id ("changes"); + const string what (title + " changes"); s << H3 << "Changes" << ~H3 << (full - ? PRE_TEXT (ch, id) - : PRE_TEXT (ch, + ? DIV_TEXT (*c, + false /* strip_title */, + id, + what, + error) + : DIV_TEXT (*c, + false /* strip_title */, options_->package_changes (), - url (!full, "changes"), - id)); + url (!full, id), + id, + what, + error)); } s << ~DIV diff --git a/mod/mod-packages.cxx b/mod/mod-packages.cxx index f09d7fa..6026024 100644 --- a/mod/mod-packages.cxx +++ b/mod/mod-packages.cxx @@ -136,6 +136,16 @@ handle (request& rq, response& rs) << DIV_HEADER (options_->logo (), options_->menu (), root, tenant) << DIV(ID="content"); + // On the first page print the search page description, if specified. + // + if (page == 0) + { + const web::xhtml::fragment& desc (options_->search_description ()); + + if (!desc.empty ()) + s << DIV(ID="search-description") << desc << ~DIV; + } + // If the tenant is empty then we are in the global view and will display // packages from all the public tenants. // @@ -146,7 +156,7 @@ handle (request& rq, response& rs) session sn; transaction t (package_db_->begin ()); - auto pkg_count ( + size_t pkg_count ( package_db_->query_value<latest_package_count> ( search_param<latest_package_count> (squery, tn))); @@ -167,7 +177,7 @@ handle (request& rq, response& rs) s << TABLE(CLASS="proplist package") << TBODY - << TR_NAME (p->name, equery, root, p->tenant) + << TR_NAME (p->name, root, p->tenant) << TR_SUMMARY (p->summary) << TR_LICENSE (p->license_alternatives) << TR_DEPENDS (p->dependencies, root, p->tenant); diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index f00e80e..34b4007 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -16,6 +16,7 @@ #include <mod/mod-ci.hxx> #include <mod/mod-submit.hxx> +#include <mod/mod-upload.hxx> #include <mod/mod-builds.hxx> #include <mod/mod-packages.hxx> #include <mod/mod-build-log.hxx> @@ -107,24 +108,42 @@ namespace brep // repository_root:: repository_root () - : packages_ (make_shared<packages> ()), + : + // + // Only create and populate the tenant service map in the examplar + // passing a reference to it to all the sub-handler exemplars. Note + // that we dispatch the tenant service callbacks to the examplar + // without creating a new instance for each callback (thus the + // callbacks are const). + // + tenant_service_map_ (make_shared<tenant_service_map> ()), + packages_ (make_shared<packages> ()), package_details_ (make_shared<package_details> ()), package_version_details_ (make_shared<package_version_details> ()), repository_details_ (make_shared<repository_details> ()), - build_task_ (make_shared<build_task> ()), - build_result_ (make_shared<build_result> ()), - build_force_ (make_shared<build_force> ()), + build_task_ (make_shared<build_task> (*tenant_service_map_)), + build_result_ (make_shared<build_result> (*tenant_service_map_)), + build_force_ (make_shared<build_force> (*tenant_service_map_)), build_log_ (make_shared<build_log> ()), builds_ (make_shared<builds> ()), build_configs_ (make_shared<build_configs> ()), submit_ (make_shared<submit> ()), - ci_ (make_shared<ci> ()) +#ifdef BREP_CI_TENANT_SERVICE + ci_ (make_shared<ci> (*tenant_service_map_)), +#else + ci_ (make_shared<ci> ()), +#endif + upload_ (make_shared<upload> ()) { } repository_root:: repository_root (const repository_root& r) : handler (r), + tenant_service_map_ ( + r.initialized_ + ? r.tenant_service_map_ + : make_shared<tenant_service_map> ()), // // Deep/shallow-copy sub-handlers depending on whether this is an // exemplar/handler. @@ -149,15 +168,15 @@ namespace brep build_task_ ( r.initialized_ ? r.build_task_ - : make_shared<build_task> (*r.build_task_)), + : make_shared<build_task> (*r.build_task_, *tenant_service_map_)), build_result_ ( r.initialized_ ? r.build_result_ - : make_shared<build_result> (*r.build_result_)), + : make_shared<build_result> (*r.build_result_, *tenant_service_map_)), build_force_ ( r.initialized_ ? r.build_force_ - : make_shared<build_force> (*r.build_force_)), + : make_shared<build_force> (*r.build_force_, *tenant_service_map_)), build_log_ ( r.initialized_ ? r.build_log_ @@ -177,7 +196,15 @@ namespace brep ci_ ( r.initialized_ ? r.ci_ +#ifdef BREP_CI_TENANT_SERVICE + : make_shared<ci> (*r.ci_, *tenant_service_map_)), +#else : make_shared<ci> (*r.ci_)), +#endif + upload_ ( + r.initialized_ + ? r.upload_ + : make_shared<upload> (*r.upload_)), options_ ( r.initialized_ ? r.options_ @@ -204,6 +231,7 @@ namespace brep append (r, build_configs_->options ()); append (r, submit_->options ()); append (r, ci_->options ()); + append (r, upload_->options ()); return r; } @@ -249,6 +277,7 @@ namespace brep sub_init (*build_configs_, "build_configs"); sub_init (*submit_, "submit"); sub_init (*ci_, "ci"); + sub_init (*upload_, "upload"); // Parse own configuration options. // @@ -444,6 +473,13 @@ namespace brep return handle ("ci", param); } + else if (func == "upload") + { + if (handler_ == nullptr) + handler_.reset (new upload (*upload_)); + + return handle ("upload", param); + } else return nullopt; }; diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx index 9e28797..aa60fda 100644 --- a/mod/mod-repository-root.hxx +++ b/mod/mod-repository-root.hxx @@ -9,6 +9,7 @@ #include <mod/module.hxx> #include <mod/module-options.hxx> +#include <mod/tenant-service.hxx> namespace brep { @@ -24,6 +25,7 @@ namespace brep class build_configs; class submit; class ci; + class upload; class repository_root: public handler { @@ -58,6 +60,8 @@ namespace brep version (); private: + shared_ptr<tenant_service_map> tenant_service_map_; + shared_ptr<packages> packages_; shared_ptr<package_details> package_details_; shared_ptr<package_version_details> package_version_details_; @@ -70,6 +74,7 @@ namespace brep shared_ptr<build_configs> build_configs_; shared_ptr<submit> submit_; shared_ptr<ci> ci_; + shared_ptr<upload> upload_; shared_ptr<options::repository_root> options_; diff --git a/mod/mod-submit.cxx b/mod/mod-submit.cxx index b73c96e..5ee358a 100644 --- a/mod/mod-submit.cxx +++ b/mod/mod-submit.cxx @@ -163,7 +163,7 @@ handle (request& rq, response& rs) if (!options_->submit_data_specified ()) return respond_manifest (404, "submission disabled"); - // Parse the request form data and verifying the submission size limit. + // Parse the request form data and verify the submission size limit. // // Note that if it is exceeded, then there are parameters and this is the // submission rather than the form request, and so we respond with the @@ -292,8 +292,8 @@ handle (request& rq, response& rs) // However, using the abbreviated checksum can be helpful for // troubleshooting. // - td = dir_path (options_->submit_temp () / - dir_path (path::traits_type::temp_name (ref))); + td = options_->submit_temp () / + dir_path (path::traits_type::temp_name (ref)); // It's highly unlikely but still possible that the temporary directory // already exists. This can only happen due to the unclean web server @@ -553,7 +553,7 @@ handle (request& rq, response& rs) // Run the submission handler, if specified, reading the result manifest // from its stdout and caching it as a name/value pair list for later use - // (forwarding to the client, sending via email, etc.). Otherwise, create + // (forwarding to the client, sending via email, etc). Otherwise, create // implied result manifest. // status_code sc; diff --git a/mod/mod-upload.cxx b/mod/mod-upload.cxx new file mode 100644 index 0000000..9f8b9de --- /dev/null +++ b/mod/mod-upload.cxx @@ -0,0 +1,763 @@ +// file : mod/mod-upload.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <mod/mod-upload.hxx> + +#include <odb/database.hxx> +#include <odb/transaction.hxx> + +#include <libbutl/uuid.hxx> +#include <libbutl/base64.hxx> +#include <libbutl/sha256.hxx> +#include <libbutl/sendmail.hxx> +#include <libbutl/fdstream.hxx> +#include <libbutl/timestamp.hxx> +#include <libbutl/filesystem.hxx> +#include <libbutl/process-io.hxx> // operator<<(ostream, process_args) +#include <libbutl/manifest-types.hxx> +#include <libbutl/manifest-serializer.hxx> + +#include <web/server/module.hxx> + +#include <libbrep/build.hxx> +#include <libbrep/build-odb.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> + +#include <mod/module-options.hxx> +#include <mod/external-handler.hxx> + +using namespace std; +using namespace butl; +using namespace brep::cli; +using namespace odb::core; + +// While currently the user-defined copy constructor is not required (we don't +// need to deep copy nullptr's), it is a good idea to keep the placeholder +// ready for less trivial cases. +// +brep::upload:: +upload (const upload& r) + : build_result_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::upload:: +init (scanner& s) +{ + HANDLER_DIAG; + + options_ = make_shared<options::upload> ( + s, unknown_mode::fail, unknown_mode::fail); + + // Verify that the upload handling is setup properly, if configured. + // + for (const auto& ud: options_->upload_data ()) + { + const string& t (ud.first); + + if (t.empty ()) + fail << "empty upload type in upload-data configuration option"; + + if (ud.second.relative ()) + fail << t << " upload-data path '" << ud.second << "' is relative"; + + if (!dir_exists (ud.second)) + fail << t << " upload-data directory '" << ud.second + << "' does not exist"; + + const map<string, path>& uh (options_->upload_handler ()); + auto i (uh.find (t)); + + if (i != uh.end () && i->second.relative ()) + fail << t << " upload-handler path '" << i->second << "' is relative"; + } + + if (options_->upload_data_specified ()) + { + if (!options_->build_config_specified ()) + fail << "upload functionality is enabled but package building " + << "functionality is disabled"; + + build_result_module::init (*options_, *options_); + } +} + +bool brep::upload:: +handle (request& rq, response& rs) +{ + using brep::version; // Not to confuse with module::version. + + using serializer = manifest_serializer; + using serialization = manifest_serialization; + + HANDLER_DIAG; + + // We will respond with the manifest to the upload protocol violations and + // with a plain text message on the internal errors. In the latter case we + // will always respond with the same neutral message for security reason, + // logging the error details. Note that descriptions of exceptions caught by + // the web server are returned to the client (see web/module.hxx for + // details), and we want to avoid this when there is a danger of exposing + // sensitive data. + // + // Also we will pass through exceptions thrown by the underlying API, unless + // we need to handle them or add details for the description, in which case + // we will fallback to one of the above mentioned response methods. + // + // Note that both respond_manifest() and respond_error() are normally called + // right before the end of the request handling. They both always return + // true to allow bailing out with a single line, for example: + // + // return respond_error (); // Request is handled with an error. + // + string request_id; // Will be set later. + auto respond_manifest = [&rs, &request_id] (status_code status, + const string& message) -> bool + { + serializer s (rs.content (status, "text/manifest;charset=utf-8"), + "response"); + + s.next ("", "1"); // Start of manifest. + s.next ("status", to_string (status)); + s.next ("message", message); + + if (!request_id.empty ()) + s.next ("reference", request_id); + + s.next ("", ""); // End of manifest. + return true; + }; + + auto respond_error = [&rs] (status_code status = 500) -> bool + { + rs.content (status, "text/plain;charset=utf-8") + << "upload handling failed" << endl; + + return true; + }; + + // Check if the upload functionality is enabled. + // + // Note that this is not an upload protocol violation but it feels right to + // respond with the manifest, to help the client a bit. + // + if (!options_->upload_data_specified ()) + return respond_manifest (404, "upload disabled"); + + // Parse the request data and verify the upload size limit. + // + // Note that the size limit is upload type-specific. Thus, first, we need to + // determine the upload type which we expect to be specified in the URL as a + // value of the upload parameter. + // + string type; + dir_path dir; + + try + { + name_value_scanner s (rq.parameters (0 /* limit */, true /* url_only */)); + + // We only expect the upload=<type> parameter in URL. + // + params::upload params ( + params::upload (s, unknown_mode::fail, unknown_mode::fail)); + + type = move (params.type ()); + + if (type.empty ()) + return respond_manifest (400, "upload type expected"); + + // Check if this upload type is enabled. While at it, cache the upload + // data directory path. + // + const map<string, dir_path>& ud (options_->upload_data ()); + auto i (ud.find (type)); + + if (i == ud.end ()) + return respond_manifest (404, type + " upload disabled"); + + dir = i->second; + } + catch (const cli::exception&) + { + return respond_manifest (400, "invalid parameter"); + } + + try + { + const map<string, size_t>& us (options_->upload_max_size ()); + auto i (us.find (type)); + rq.parameters (i != us.end () ? i->second : 10485760); // 10M by default. + } + catch (const invalid_request& e) + { + if (e.status == 413) // Payload too large? + return respond_manifest (e.status, type + " upload size exceeds limit"); + + throw; + } + + // The request parameters are now parsed and the limit doesn't really matter. + // + const name_values& rps (rq.parameters (0 /* limit */)); + + // Verify the upload parameters we expect. The unknown ones will be + // serialized to the upload manifest. + // + params::upload params; + + try + { + name_value_scanner s (rps); + params = params::upload (s, unknown_mode::skip, unknown_mode::skip); + } + catch (const cli::exception&) + { + return respond_manifest (400, "invalid parameter"); + } + + const string& session (params.session ()); + const string& instance (params.instance ()); + const string& archive (params.archive ()); + const string& sha256sum (params.sha256sum ()); + + if (session.empty ()) + return respond_manifest (400, "upload session expected"); + + optional<vector<char>> challenge; + + if (params.challenge_specified ()) + try + { + challenge = base64_decode (params.challenge ()); + } + catch (const invalid_argument&) + { + return respond_manifest (400, "invalid challenge"); + } + + if (instance.empty ()) + return respond_manifest (400, "upload instance expected"); + + if (archive.empty ()) + return respond_manifest (400, "upload archive expected"); + + if (sha256sum.empty ()) + return respond_manifest (400, "upload archive checksum expected"); + + if (sha256sum.size () != 64) + return respond_manifest (400, "invalid upload archive checksum"); + + // Verify that unknown parameter values satisfy the requirements (contain + // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n'). + // + // Actually, the expected ones must satisfy too, so check them as well. + // + string what; + for (const name_value& nv: rps) + { + if (nv.value && + !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t")) + return respond_manifest (400, + "invalid parameter " + nv.name + ": " + what); + } + + parse_session_result sess; + + try + { + sess = parse_session (session); + } + catch (const invalid_argument& e) + { + return respond_manifest (400, string ("invalid session: ") + e.what ()); + } + + // If the session expired (no such configuration, etc) then, similar to the + // build result module, we log this case with the warning severity and + // respond with manifest with the 200 status as if the session is valid (see + // the build result module for the reasoning). + // + auto warn_expired = [&session, &warn] (const string& d) + { + warn << "session '" << session << "' expired: " << d; + }; + + const build_id& id (sess.id); + + // Make sure the build configuration still exists. + // + const build_target_config* tc; + { + auto i (target_conf_map_->find ( + build_target_config_id {id.target, id.target_config_name})); + + if (i == target_conf_map_->end ()) + { + warn_expired ("no build configuration"); + return respond_manifest (200, type + " upload is queued"); + } + + tc = i->second; + } + + // Note that if the session authentication fails (probably due to the + // authentication settings change), then we log this case with the warning + // severity and respond with manifest with the 200 status as if the + // challenge is valid (see the build result module for the reasoning). + // + shared_ptr<build> bld; + shared_ptr<build_package> pkg; + shared_ptr<build_repository> rep; + { + transaction t (build_db_->begin ()); + + package_build pb; + shared_ptr<build> b; + if (!build_db_->query_one<package_build> ( + query<package_build>::build::id == id, pb)) + { + warn_expired ("no package build"); + } + else if ((b = move (pb.build))->state != build_state::building) + { + warn_expired ("package configuration state is " + to_string (b->state)); + } + else if (b->timestamp != sess.timestamp) + { + warn_expired ("non-matching timestamp"); + } + else if (authenticate_session (*options_, challenge, *b, session)) + { + bld = move (b); + pkg = build_db_->load<build_package> (id.package); + rep = pkg->internal_repository.load (); + } + + t.commit (); + } + + // Note that from now on the result manifest we respond with will contain + // the reference value. + // + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return respond_error (); + } + + if (bld == nullptr) + return respond_manifest (200, type + " upload is queued"); + + // Create the upload data directory. + // + dir_path dd (dir / dir_path (request_id)); + + try + { + // It's highly unlikely but still possible that the directory already + // exists. This can only happen if the generated uuid is not unique. + // + if (try_mkdir (dd) == mkdir_status::already_exists) + throw_generic_error (EEXIST); + } + catch (const system_error& e) + { + error << "unable to create directory '" << dd << "': " << e; + return respond_error (); + } + + auto_rmdir ddr (dd); + + // Save the package archive into the temporary directory and verify its + // checksum. + // + // Note that the archive file name can potentially contain directory path in + // the POSIX form, so let's strip it if that's the case. + // + path a; + path af; + + try + { + size_t n (archive.find_last_of ('/')); + a = path (n != string::npos ? string (archive, n + 1) : archive); + af = dd / a; + } + catch (const invalid_path&) + { + return respond_manifest (400, "invalid package archive name"); + } + + try + { + istream& is (rq.open_upload ("archive")); + + // Note that istream::read() sets failbit if unable to read the requested + // number of bytes. + // + is.exceptions (istream::badbit); + + sha256 sha; + char buf[8192]; + ofdstream os (af, fdopen_mode::binary); + + while (!eof (is)) + { + is.read (buf, sizeof (buf)); + + if (size_t n = is.gcount ()) + { + sha.append (buf, n); + os.write (buf, n); + } + } + + os.close (); + + // Respond with the unprocessable entity (422) code for the archive + // checksum mismatch. + // + if (sha.string () != sha256sum) + return respond_manifest (422, "upload archive checksum mismatch"); + } + // Note that invalid_argument (thrown by open_upload() function call) can + // mean both no archive upload or multiple archive uploads. + // + catch (const invalid_argument&) + { + return respond_manifest (400, "archive upload expected"); + } + catch (const io_error& e) + { + error << "unable to write package archive '" << af << "': " << e; + return respond_error (); + } + + // Serialize the upload request manifest to a stream. On the serialization + // error respond to the client with the manifest containing the bad request + // (400) code and return false, on the stream error pass through the + // io_error exception, otherwise return true. + // + timestamp ts (system_clock::now ()); + + auto rqm = [&request_id, + &ts, + &rps, + &session, + &instance, + &a, + &sha256sum, + &id, + &bld, + &pkg, + &rep, + &tc, + &sess, + &respond_manifest, + this] (ostream& os, bool long_lines = false) -> bool + { + try + { + serializer s (os, "request", long_lines); + + // Serialize the upload manifest header. + // + s.next ("", "1"); // Start of manifest. + s.next ("id", request_id); + s.next ("session", session); + s.next ("instance", instance); + s.next ("archive", a.string ()); + s.next ("sha256sum", sha256sum); + + s.next ("timestamp", + butl::to_string (ts, + "%Y-%m-%dT%H:%M:%SZ", + false /* special */, + false /* local */)); + + s.next ("name", id.package.name.string ()); + s.next ("version", pkg->version.string ()); + s.next ("project", pkg->project.string ()); + s.next ("target-config", tc->name); + s.next ("package-config", id.package_config_name); + s.next ("target", tc->target.string ()); + + if (!tenant.empty ()) + s.next ("tenant", tenant); + + s.next ("toolchain-name", id.toolchain_name); + s.next ("toolchain-version", sess.toolchain_version.string ()); + s.next ("repository-name", rep->canonical_name); + + s.next ("machine-name", bld->machine.name); + s.next ("machine-summary", bld->machine.summary); + + // Serialize the request parameters. + // + // Note that the serializer constraints the parameter names (can't start + // with '#', can't contain ':' and the whitespaces, etc.). + // + for (const name_value& nv: rps) + { + // Note that the upload parameter is renamed to '_' by the root + // handler (see the request_proxy class for details). + // + const string& n (nv.name); + if (n != "_" && + n != "session" && + n != "challenge" && + n != "instance" && + n != "archive" && + n != "sha256sum") + s.next (n, nv.value ? *nv.value : ""); + } + + s.next ("", ""); // End of manifest. + return true; + } + catch (const serialization& e) + { + respond_manifest (400, string ("invalid parameter: ") + e.what ()); + return false; + } + }; + + // Serialize the upload request manifest to the upload directory. + // + path rqf (dd / "request.manifest"); + + try + { + ofdstream os (rqf); + bool r (rqm (os)); + os.close (); + + if (!r) + return true; // The client is already responded with the manifest. + } + catch (const io_error& e) + { + error << "unable to write to '" << rqf << "': " << e; + return respond_error (); + } + + // Given that the upload data is now successfully persisted we are no longer + // in charge of removing it, except for the cases when the upload + // handler terminates with an error (see below for details). + // + ddr.cancel (); + + // If the handler terminates with non-zero exit status or specifies 5XX + // (HTTP server error) upload result manifest status value, then we stash + // the upload data directory for troubleshooting. Otherwise, if it's the 4XX + // (HTTP client error) status value, then we remove the directory. + // + auto stash_upload_dir = [&dd, error] () + { + if (dir_exists (dd)) + try + { + mvdir (dd, dir_path (dd + ".fail")); + } + catch (const system_error& e) + { + // Not much we can do here. Let's just log the issue and bail out + // leaving the directory in place. + // + error << "unable to rename directory '" << dd << "': " << e; + } + }; + + // Run the upload handler, if specified, reading the result manifest from + // its stdout and caching it as a name/value pair list for later use + // (forwarding to the client, sending via email, etc). Otherwise, create + // implied result manifest. + // + status_code sc; + vector<manifest_name_value> rvs; + + const map<string, path>& uh (options_->upload_handler ()); + auto hi (uh.find (type)); + + if (hi != uh.end ()) + { + auto range (options_->upload_handler_argument ().equal_range (type)); + + strings args; + for (auto i (range.first); i != range.second; ++i) + args.push_back (i->second); + + const map<string, size_t>& ht (options_->upload_handler_timeout ()); + auto i (ht.find (type)); + + optional<external_handler::result_manifest> r ( + external_handler::run (hi->second, + args, + dd, + i != ht.end () ? i->second : 0, + error, + warn, + verb_ ? &trace : nullptr)); + + if (!r) + { + stash_upload_dir (); + return respond_error (); // The diagnostics is already issued. + } + + sc = r->status; + rvs = move (r->values); + } + else // Create the implied result manifest. + { + sc = 200; + + auto add = [&rvs] (string n, string v) + { + manifest_name_value nv { + move (n), move (v), + 0 /* name_line */, 0 /* name_column */, + 0 /* value_line */, 0 /* value_column */, + 0 /* start_pos */, 0 /* colon_pos */, 0 /* end_pos */}; + + rvs.emplace_back (move (nv)); + }; + + add ("status", "200"); + add ("message", type + " upload is queued"); + add ("reference", request_id); + } + + assert (!rvs.empty ()); // Produced by the handler or is implied. + + // Serialize the upload result manifest to a stream. On the serialization + // error log the error description and return false, on the stream error + // pass through the io_error exception, otherwise return true. + // + auto rsm = [&rvs, + &error, + &request_id, + &type] (ostream& os, bool long_lines = false) -> bool + { + try + { + serializer s (os, "result", long_lines); + serialize_manifest (s, rvs); + return true; + } + catch (const serialization& e) + { + error << "ref " << request_id << ": unable to serialize " << type + << " upload handler's output: " << e; + return false; + } + }; + + // If the upload data directory still exists then perform an appropriate + // action on it, depending on the upload result status. Note that the + // handler could move or remove the directory. + // + if (dir_exists (dd)) + { + // Remove the directory if the client error is detected. + // + if (sc >= 400 && sc < 500) + { + rmdir_r (dd); + } + // + // Otherwise, save the result manifest, into the directory. Also stash the + // directory for troubleshooting in case of the server error. + // + else + { + path rsf (dd / "result.manifest"); + + try + { + ofdstream os (rsf); + + // Not being able to stash the result manifest is not a reason to + // claim the upload failed. The error is logged nevertheless. + // + rsm (os); + + os.close (); + } + catch (const io_error& e) + { + // Not fatal (see above). + // + error << "unable to write to '" << rsf << "': " << e; + } + + if (sc >= 500 && sc < 600) + stash_upload_dir (); + } + } + + // Send email, if configured. Use the long lines manifest serialization mode + // for the convenience of copying/clicking URLs they contain. + // + // Note that we don't consider the email sending failure to be an upload + // failure as the upload data is successfully persisted and the handler is + // successfully executed, if configured. One can argue that email can be + // essential for the upload processing and missing it would result in the + // incomplete upload. In this case it's natural to assume that the web + // server error log is monitored and the email sending failure will be + // noticed. + // + const map<string, string>& ue (options_->upload_email ()); + auto ei (ue.find (type)); + + if (ei != ue.end ()) + try + { + // Redirect the diagnostics to the web server error log. + // + sendmail sm ([&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }, + 2 /* stderr */, + options_->email (), + type + " upload (" + request_id + ')', + {ei->second}); + + // Write the upload request manifest. + // + bool r (rqm (sm.out, true /* long_lines */)); + assert (r); // The serialization succeeded once, so can't fail now. + + // Write the upload result manifest. + // + sm.out << "\n\n"; + + // We don't care about the result (see above). + // + rsm (sm.out, true /* long_lines */); + + sm.out.close (); + + if (!sm.wait ()) + error << "sendmail " << *sm.exit; + } + // Handle process_error and io_error (both derive from system_error). + // + catch (const system_error& e) + { + error << "sendmail error: " << e; + } + + if (!rsm (rs.content (sc, "text/manifest;charset=utf-8"))) + return respond_error (); // The error description is already logged. + + return true; +} diff --git a/mod/mod-upload.hxx b/mod/mod-upload.hxx new file mode 100644 index 0000000..6cc723b --- /dev/null +++ b/mod/mod-upload.hxx @@ -0,0 +1,41 @@ +// file : mod/mod-upload.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_UPLOAD_HXX +#define MOD_MOD_UPLOAD_HXX + +#include <libbrep/types.hxx> +#include <libbrep/utility.hxx> + +#include <mod/module-options.hxx> +#include <mod/build-result-module.hxx> + +namespace brep +{ + class upload: public build_result_module + { + public: + upload () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + upload (const upload&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const {return options::upload::description ();} + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr<options::upload> options_; + }; +} + +#endif // MOD_MOD_UPLOAD_HXX diff --git a/mod/module.cli b/mod/module.cli index 7c6e0b4..a107ffe 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -5,6 +5,7 @@ include <map>; include <regex>; include <libbpkg/manifest.hxx>; // repository_location +include <libbbot/manifest.hxx>; // interactive_mode include <web/xhtml/fragment.hxx>; @@ -20,7 +21,7 @@ namespace brep { // Option groups. // - class handler + class repository_email { string email { @@ -28,7 +29,10 @@ namespace brep "Repository email. This email is used for the \cb{From:} header in emails send by \cb{brep} (for example, build failure notifications)." } + }; + class repository_url + { string host { "<host>", @@ -47,7 +51,29 @@ namespace brep Specify '\cb{/}' to use the web server root (\cb{http://example.org/})." } + }; + class build_email_notification: repository_email, repository_url + { + std::map<string, build_email> build-toolchain-email + { + "<name>=<mode>", + "Enable or disable package build notification emails. The valid <mode> + values are \cb{none}, \cb{latest}, and \cb{all}. If \cb{all} is + specified for a toolchain name, then emails are sent according to the + \cb{build-*email} package manifest values when all versions of a + package are built with this toolchain. If \cb{latest} is specified, + then for this toolchain name the emails are only sent for the latest + version of a package. If \cb{none} is specified, then no emails are + sent for this toolchain name. By default the \cb{latest} mode is + assumed. Repeat this option to enable/disable emails for multiple + toolchains. See \l{bpkg#manifest-package Package Manifest} for + details on \cb{build-*email} values." + } + }; + + class handler + { string tenant-name = "tenant" { "<name>", @@ -105,14 +131,14 @@ namespace brep { "<user>", "Package database login user name. If not specified, then operating - system (login) name is used. See also \c{package-db-role}." + system (login) name is used. See also \cb{package-db-role}." } string package-db-role = "brep" { "<user>", "Package database execution user name. If not empty then the login - user will be switched (with \c{SET ROLE}) to this user prior to + user will be switched (with \cb{SET ROLE}) to this user prior to executing any statements. If not specified, then \cb{brep} is used." } @@ -214,9 +240,9 @@ namespace brep \cb{build-alt-soft-rebuild-start} and \cb{build-alt-soft-rebuild-stop} options. Must be specified in seconds. Default is the time interval length plus - \c{(\b{build-soft-rebuild-timeout} - 24h)} if soft rebuild timeout is - greater than 24 hours (thus the rebuild is only triggered within the - last 24 hours of the \cb{build-soft-rebuild-timeout} expiration)." + \c{(\b{build-soft-rebuild-timeout} - 24h)} if soft rebuild timeout + is greater than 24 hours (thus the rebuild is only triggered within + the last 24 hours of the \cb{build-soft-rebuild-timeout} expiration)." } duration build-alt-soft-rebuild-start @@ -278,6 +304,15 @@ namespace brep the same as for the \cb{build-alt-soft-rebuild-stop} option but for the \cb{build-hard-rebuild-timeout} option." } + + size_t build-queued-timeout = 30 + { + "<seconds>", + "Time to wait before assuming the \cb{queued} notifications are + delivered for package CI requests submitted via third-party services + (GitHub, etc). During this time a package is not considered for a + build. Must be specified in seconds. Default is 30 seconds." + } }; class build_db @@ -286,14 +321,14 @@ namespace brep { "<user>", "Build database login user name. If not specified, then operating - system (login) name is used. See also \c{build-db-role}." + system (login) name is used. See also \cb{build-db-role}." } string build-db-role = "brep" { "<user>", "Build database execution user name. If not empty then the login - user will be switched (with \c{SET ROLE}) to this user prior to + user will be switched (with \cb{SET ROLE}) to this user prior to executing any statements. If not specified, then \cb{brep} is used." } @@ -343,6 +378,82 @@ namespace brep } }; + class build_upload + { + std::map<string, dir_path> upload-data + { + "<type>=<dir>", + "The directory to save upload data to for the specified upload type. + If unspecified, the build artifacts upload functionality will be + disabled for this type. See \l{brep The \cb{build2} Repository + Interface Manual} for more information on build artifacts upload. + + Note that the directory path must be absolute and the directory + itself must exist and have read, write, and execute permissions + granted to the user that runs the web server." + } + + std::map<string, size_t> upload-max-size + { + "<type>=<bytes>", + "The maximum size of the upload data accepted for the specified upload + type. Note that currently the entire upload request is read into + memory. The default is 10M." + } + + std::map<string, string> upload-email + { + "<type>=<email>", + "The build artifacts upload email. If specified, the upload request + and result manifests will be sent to this address. See \l{brep The + \cb{build2} Repository Interface Manual} for more information." + } + + std::map<string, path> upload-handler + { + "<type>=<path>", + "The handler program to be executed on build artifacts upload of the + specified type. The handler is executed as part of the HTTP request + and is passed additional arguments that can be specified with + \cb{upload-handler-argument} followed by the absolute path to the + upload directory (\cb{upload-data}). See \l{brep The \cb{build2} + Repository Interface Manual} for more information. Note that the + program path must be absolute." + } + + std::multimap<string, string> upload-handler-argument + { + "<type>=<arg>", + "Additional arguments to be passed to the upload handler program for + the specified upload type (see \cb{upload-handler} for details). + Repeat this option to specify multiple arguments." + } + + std::map<string, size_t> upload-handler-timeout + { + "<type>=<seconds>", + "The upload handler program timeout in seconds for the specified + upload type. If specified and the handler does not exit in the + allotted time, then it is killed and its termination is treated as + abnormal." + } + + std::multimap<string, string> upload-toolchain-exclude + { + "<type>=<name>", + "Disable upload of the specified type for the specified toolchain + name. Repeat this option to disable uploads for multiple toolchains." + } + + std::multimap<string, string> upload-repository-exclude + { + "<type>=<name>", + "Disable upload of the specified type for packages from the repository + with the specified canonical name. Repeat this option to disable + uploads for multiple repositories." + } + }; + class page { web::xhtml::fragment logo @@ -397,7 +508,7 @@ namespace brep // Handler options. // - class packages: search, package_db, page, handler + class packages: search, package_db, page, repository_url, handler { string search-title = "Packages" { @@ -405,24 +516,63 @@ namespace brep "Package search page title. It is placed inside XHTML5 <title> element." } + + web::xhtml::fragment search-description + { + "<xhtml>", + "Package search page description. If specified, it is displayed + before the search form on the first page only. The value is + treated as an XHTML5 fragment." + } }; - class package_details: package, search, package_db, page, handler + class package_details: package, package_db, + search, + page, + repository_url, + handler { }; class package_version_details: package, package_db, build, build_db, page, + repository_url, handler { + dir_path bindist-root + { + "<dir>", + "The root directory where the uploaded binary distribution packages + are saved to under the following directory hierarchy: + + \ + [<tenant>/]<distribution>/<os-release>/<project>/<package>/<version>/<package-config> + \ + + The package configuration directory symlinks that match these paths + are mapped to web URLs based on the \cb{bindist-url} value and + displayed on the package version details page. If this option is + specified, then \cb{bindist-url} must be specified as well." + } + + string bindist-url + { + "<url>", + "The root URL of the directory specified with the \cb{bindist-root} + option. This option must be specified if \cb{bindist-root} is + specified." + } }; - class repository_details: package_db, page, handler + class repository_details: package_db, page, repository_url, handler { }; - class build_task: build, build_db, handler + class build_task: build, build_db, + build_upload, + build_email_notification, + handler { size_t build-task-request-max-size = 102400 { @@ -452,9 +602,20 @@ namespace brep request is considered invalid, unless no expressions are specified. Repeat this option to specify multiple expressions." } + + build_order build-package-order = build_order::stable + { + "<order>", + "Order in which packages are considered for build. The valid <order> + values are \cb{stable} and \cb{random}. If not specified, then + \cb{stable} is assumed. Note that interactive builds are always + preferred." + } }; - class build_result: build, package_db, build_db, handler + class build_result: build, build_db, + build_email_notification, + handler { size_t build-result-request-max-size = 10485760 { @@ -464,22 +625,9 @@ namespace brep face of recoverable failures (deadlock, loss of connection, etc). The default is 10M." } - - std::map<string, bool> build-toolchain-email - { - "<name>=<bool>", - "Enable or disable package build notification emails. If \cb{true} is - specified for a toolchain name, then emails are sent according to the - \cb{build-*email} package manifest values when the package is built - with this toolchain. If \cb{false} is specified, then no emails are - sent for this toolchain name. By default build notification emails - are enabled. Repeat this option to enable/disable emails for multiple - toolchains. See \l{bpkg#manifest-package Package Manifest} for - details on \cb{build-*email} values." - } }; - class build_log: build, build_db, handler + class build_log: build, build_db, repository_url, handler { }; @@ -487,7 +635,7 @@ namespace brep { }; - class builds: build, build_db, page, handler + class builds: build, build_db, page, repository_url, handler { uint16_t build-page-entries = 20 { @@ -502,7 +650,7 @@ namespace brep } }; - class build_configs: build, page, handler + class build_configs: build, page, repository_url, handler { uint16_t build-config-page-entries = 20 { @@ -517,7 +665,7 @@ namespace brep } }; - class submit: page, handler + class submit: page, repository_email, repository_url, handler { dir_path submit-data { @@ -597,7 +745,7 @@ namespace brep } }; - class ci: page, handler + class ci_start: repository_email { dir_path ci-data { @@ -612,15 +760,6 @@ namespace brep granted to the user that runs the web server." } - path ci-form - { - "<file>", - "The package CI form fragment. If specified, then its contents are - treated as an XHTML5 fragment that is inserted into the <body> - element of the CI page. If unspecified, then no CI page will be - displayed. Note that the file path must be absolute." - } - string ci-email { "<email>", @@ -657,14 +796,44 @@ namespace brep } }; - class repository_root: handler + class ci_cancel + { + }; + + class ci: ci_start, page, repository_url, handler + { + // Classic CI-specific options. + // + + path ci-form + { + "<file>", + "The package CI form fragment. If specified, then its contents are + treated as an XHTML5 fragment that is inserted into the <body> + element of the CI page. If unspecified, then no CI page will be + displayed. Note that the file path must be absolute." + } + }; + + class ci_github: ci_start, ci_cancel, build_db, handler + { + // GitHub CI-specific options (e.g., request timeout when invoking + // GitHub APIs). + // + }; + + class upload: build, build_db, build_upload, repository_email, handler + { + }; + + class repository_root: repository_url, handler { string root-global-view = "packages" { "<service>", "The default view to display for the global repository root. The - <service> argument is one of the supported services (\c{packages}, - \c{builds}, \c{submit}, \c{ci}, etc). The default service is + <service> argument is one of the supported services (\cb{packages}, + \cb{builds}, \cb{submit}, \cb{ci}, etc). The default service is packages." } @@ -672,8 +841,8 @@ namespace brep { "<service>", "The default view to display for the tenant repository root. The - <service> argument is one of the supported services (\c{packages}, - \c{builds}, \c{submit}, \c{ci}, etc). The default service is + <service> argument is one of the supported services (\cb{packages}, + \cb{builds}, \cb{submit}, \cb{ci}, etc). The default service is packages." } }; @@ -730,9 +899,14 @@ namespace brep class build_task { - // Package repository canonical name (note: including pkg: type). + // Only consider packages from repositories with these canonical names + // (note: including pkg: type). // vector<string> repository | r; + + // Only consider tenants with this interactive build mode. + // + bbot::interactive_mode interactive = bbot::interactive_mode::both; }; class build_result @@ -851,10 +1025,13 @@ namespace brep class build_configs { + // By default, display all build configurations except those which + // belong to the 'hidden' class. + // // Note that the build-configs parameter is renamed to '_' by the root // handler (see the request_proxy class for details). // - string class_name | _ = "all"; + string class_name | _; // Display build configurations list starting from this page. // @@ -921,5 +1098,45 @@ namespace brep // string simulate; }; + + // Parameters other than challenge must be all present. + // + // Note also that besides these parameters there can be others. We don't + // recognize their semantics and just save them to the upload request + // manifest. + // + class upload + { + // Upload type. + // + // Note that the upload parameter is renamed to '_' by the root handler + // (see the request_proxy class for details). + // + string type | _; + + // Session id as returned by brep in the task response. + // + string session; + + // Answer to the private key challenge as posed by brep in the task + // response. It must be present only if the challenge value was present + // in the task response. + // + string challenge; + + // Upload instance name. + // + string instance; + + // Package archive file name. Must be <input type="file"/>. + // + // Note that it can potentially be not just a name but a file path. + // + string archive; + + // Package archive file SHA256 checksum. + // + string sha256sum; + }; } } diff --git a/mod/module.cxx b/mod/module.cxx index 2fcadd2..c8d0595 100644 --- a/mod/module.cxx +++ b/mod/module.cxx @@ -241,23 +241,46 @@ namespace brep initialized_ = m.initialized_; } -// For function func declared like this: -// using B = std::string (*)(int); -// using A = B (*)(int,int); -// A func(B (*)(char),B (*)(wchar_t)); -// __PRETTY_FUNCTION__ looks like this: -// virtual std::string (* (* brep::search::func(std::string (* (*)(char))(int) -// ,std::string (* (*)(wchar_t))(int)) const)(int, int))(int) -// + // Here are examples of __PRETTY_FUNCTION__ for some function declarations: + // + // 1) virtual bool brep::search::handle (web::request&, web::response&); + // + // virtual bool brep::search::handle(web::request&, web::response&) + // + // 2) using B = std::string (*) (int); + // virtual B brep::search::func (); + // + // virtual std::string (* brep::search::func())(int) + // + // 3) using B = std::string (*) (int); + // using A = B (*) (int,int); + // virtual A brep::search::func (B (*) (char), B (*) (wchar_t)); + // + // virtual std::string (* (* brep::search::func(std::string (* (*)(char))(int), std::string (* (*)(wchar_t))(int)))(int, int))(int) + // + // 4) using X = std::function<butl::optional<std::string> (int)> (*) (std::function<butl::optional<std::string> (long)>); + // X brep::search::func (std::function<butl::optional<std::string> (char)> (*) (std::function<butl::optional<std::string> (wchar_t)>)); + // + // std::function<std::optional<std::__cxx11::basic_string<char> >(int)> (* brep::search::func(std::function<std::optional<std::__cxx11::basic_string<char> >(char)> (*)(std::function<std::optional<std::__cxx11::basic_string<char> >(wchar_t)>)))(std::function<std::optional<std::__cxx11::basic_string<char> >(long int)>) + // + // 5) using X = std::function<butl::optional<std::string> (int)> (*) (std::function<butl::optional<std::string> (long)>); + // using Y = X (*) (int); + // Y brep::search::func (const char*); + // + // std::function<std::optional<std::__cxx11::basic_string<char> >(int)> (* (* brep::search::func(const char*))(int))(std::function<std::optional<std::__cxx11::basic_string<char> >(long int)>) + // string handler:: func_name (const char* pretty_name) { - const char* e (strchr (pretty_name, ')')); + // Position at the last ')' character, which is either the end of the + // function's arguments list or the returned function type argument list. + // + const char* e (strrchr (pretty_name, ')')); if (e && e > pretty_name) { - // Position e at last matching '(' which is the beginning of the - // argument list.. + // Position e at the matching '(' character which is the beginning of + // the mentioned argument list. // size_t d (1); @@ -273,11 +296,15 @@ namespace brep if (!d && e > pretty_name) { - // Position e at the character following the function name. + // Position e at the character which follows the function name. + // + // Specifically, go further to the left and stop at the '(' character + // which is preceded by the character other than ' ', ')', of '>'. // - while (e > pretty_name && - (*e != '(' || *(e - 1) == ' ' || *(e - 1) == ')')) - --e; + for (char c; + e > pretty_name && + !(*e == '(' && (c = *(e - 1)) != ' ' && c != ')' && c != '>'); + --e) ; if (e > pretty_name) { diff --git a/mod/module.hxx b/mod/module.hxx index 6423e6d..f3e062e 100644 --- a/mod/module.hxx +++ b/mod/module.hxx @@ -194,7 +194,7 @@ namespace brep log* log_ {nullptr}; // Diagnostics backend provided by the web server. private: - // Extract function name from a __PRETTY_FUNCTION__. + // Extract the full-qualified function name from a __PRETTY_FUNCTION__. // Throw invalid_argument if fail to parse. // static string diff --git a/mod/options-types.hxx b/mod/options-types.hxx index 4aa573f..f2b059b 100644 --- a/mod/options-types.hxx +++ b/mod/options-types.hxx @@ -25,6 +25,19 @@ namespace brep page_menu () = default; page_menu (string b, string l): label (move (b)), link (move (l)) {} }; + + enum class build_order + { + stable, + random + }; + + enum class build_email + { + none, + latest, // Only send emails for the latest package versions. + all + }; } #endif // MOD_OPTIONS_TYPES_HXX diff --git a/mod/page.cxx b/mod/page.cxx index e2a8b84..bc2e42d 100644 --- a/mod/page.cxx +++ b/mod/page.cxx @@ -137,9 +137,17 @@ namespace brep void DIV_COUNTER:: operator() (serializer& s) const { - s << DIV(ID="count") - << count_ << " " - << (count_ % 10 == 1 && count_ % 100 != 11 ? singular_ : plural_) + s << DIV(ID="count"); + + if (count_) + s << *count_; + else + s << '?'; + + s << ' ' + << (count_ && *count_ % 10 == 1 && *count_ % 100 != 11 + ? singular_ + : plural_) << ~DIV; } @@ -237,15 +245,9 @@ namespace brep << A << HREF << tenant_dir (root_, tenant_) / - path (mime_url_encode (name_.string (), false)); - - // Propagate search criteria to the package details page. - // - if (!query_.empty ()) - s << "?q=" << query_; - - s << ~HREF - << name_ + path (mime_url_encode (name_.string (), false)) + << ~HREF + << name_ << ~A << ~SPAN << ~TD @@ -747,8 +749,24 @@ namespace brep << TD << SPAN(CLASS="value"); + // Print the ' | ' separator if this is not the first item and reset the + // `first` flag to false otherwise. + // + bool first (true); + auto separate = [&s, &first] () + { + if (first) + first = false; + else + s << " | "; + }; + if (build_.state == build_state::building) - s << SPAN(CLASS="building") << "building" << ~SPAN << " | "; + { + separate (); + + s << SPAN(CLASS="building") << "building" << ~SPAN; + } else { // If no unsuccessful operation results available, then print the @@ -761,7 +779,10 @@ namespace brep if (build_.results.empty () || *build_.status == result_status::success) { assert (build_.status); - s << SPAN_BUILD_RESULT_STATUS (*build_.status) << " | "; + + separate (); + + s << SPAN_BUILD_RESULT_STATUS (*build_.status); } if (!build_.results.empty ()) @@ -769,6 +790,9 @@ namespace brep for (const auto& r: build_.results) { if (r.status != result_status::success) + { + separate (); + s << SPAN_BUILD_RESULT_STATUS (r.status) << " (" << A << HREF @@ -776,26 +800,33 @@ namespace brep << ~HREF << r.operation << ~A - << ") | "; + << ")"; + } } + separate (); + s << A << HREF << build_log_url (host_, root_, build_) << ~HREF << "log" - << ~A - << " | "; + << ~A; } } - if (build_.force == (build_.state == build_state::building - ? force_state::forcing - : force_state::forced)) - s << SPAN(CLASS="pending") << "pending" << ~SPAN; - else - s << A - << HREF << build_force_url (host_, root_, build_) << ~HREF - << "rebuild" - << ~A; + if (!archived_) + { + separate (); + + if (build_.force == (build_.state == build_state::building + ? force_state::forcing + : force_state::forced)) + s << SPAN(CLASS="pending") << "pending" << ~SPAN; + else + s << A + << HREF << build_force_url (host_, root_, build_) << ~HREF + << "rebuild" + << ~A; + } s << ~SPAN << ~TD @@ -923,14 +954,16 @@ namespace brep void DIV_TEXT:: operator() (serializer& s) const { - switch (type_) + const string& t (text_.text); + + switch (text_.type) { case text_type::plain: { // To keep things regular we wrap the preformatted text into <div>. // s << DIV(ID=id_, CLASS="plain"); - serialize_pre_text (s, text_, length_, url_, "" /* id */); + serialize_pre_text (s, t, length_, url_, "" /* id */); s << ~DIV; break; } @@ -950,9 +983,9 @@ namespace brep // calls to fail is the inability to allocate memory. Unfortunately, // instead of reporting the failure to the caller, the API issues // diagnostics to stderr and aborts the process. Let's decrease the - // probability of such an event by limiting the text size to 64K. + // probability of such an event by limiting the text size to 1M. // - if (text_.size () > 64 * 1024) + if (t.size () > 1024 * 1024) { print_error (what_ + " is too long"); return; @@ -976,7 +1009,7 @@ namespace brep // Enable GitHub extensions in the parser, if requested. // - if (type_ == text_type::github_mark) + if (text_.type == text_type::github_mark) { auto add = [&parser] (const char* ext) { @@ -995,7 +1028,7 @@ namespace brep add ("autolink"); } - cmark_parser_feed (parser.get (), text_.c_str (), text_.size ()); + cmark_parser_feed (parser.get (), t.c_str (), t.size ()); unique_ptr<cmark_node, void (*)(cmark_node*)> doc ( cmark_parser_finish (parser.get ()), diff --git a/mod/page.hxx b/mod/page.hxx index acfbe5b..7329e2d 100644 --- a/mod/page.hxx +++ b/mod/page.hxx @@ -82,21 +82,24 @@ namespace brep // Generate counter element. // - // It could be redunant to distinguish between singular and plural word forms - // if it wouldn't be so cheap in English, and phrase '1 Packages' wouldn't - // look that ugly. + // If the count argument is nullopt, then it is assumed that the count is + // unknown and the '?' character is printed instead of the number. + // + // Note that it could be redunant to distinguish between singular and plural + // word forms if it wouldn't be so cheap in English, and phrase '1 Packages' + // wouldn't look that ugly. // class DIV_COUNTER { public: - DIV_COUNTER (size_t c, const char* s, const char* p) + DIV_COUNTER (optional<size_t> c, const char* s, const char* p) : count_ (c), singular_ (s), plural_ (p) {} void operator() (xml::serializer&) const; private: - size_t count_; + optional<size_t> count_; const char* singular_; const char* plural_; }; @@ -193,24 +196,19 @@ namespace brep const string& tenant_; }; - // Generate package name element with an optional search criteria. The - // search string should be url-encoded, if specified. + // Generate package name element. // class TR_NAME { public: - TR_NAME (const package_name& n, - const string& q, - const dir_path& r, - const string& t) - : name_ (n), query_ (q), root_ (r), tenant_ (t) {} + TR_NAME (const package_name& n, const dir_path& r, const string& t) + : name_ (n), root_ (r), tenant_ (t) {} void operator() (xml::serializer&) const; private: const package_name& name_; - const string& query_; const dir_path& root_; const string& tenant_; }; @@ -474,14 +472,23 @@ namespace brep class TR_BUILD_RESULT { public: - TR_BUILD_RESULT (const build& b, const string& h, const dir_path& r): - build_ (b), host_ (h), root_ (r) {} + TR_BUILD_RESULT (const build& b, + bool a, + const string& h, + const dir_path& r): + build_ (b), archived_ (a), host_ (h), root_ (r) + { + // We don't expect a queued build to ever be displayed. + // + assert (build_.state != build_state::queued); + } void operator() (xml::serializer&) const; private: const build& build_; + bool archived_; const string& host_; const dir_path& root_; }; @@ -587,16 +594,14 @@ namespace brep public: // Generate a full text element. // - DIV_TEXT (const string& t, - text_type tp, + DIV_TEXT (const typed_text& t, bool st, const string& id, const string& what, const basic_mark& diag) : text_ (t), - type_ (tp), strip_title_ (st), - length_ (t.size ()), + length_ (t.text.size ()), url_ (nullptr), id_ (id), what_ (what), @@ -606,8 +611,7 @@ namespace brep // Generate a brief text element. // - DIV_TEXT (const string& t, - text_type tp, + DIV_TEXT (const typed_text& t, bool st, size_t l, const string& u, @@ -615,7 +619,6 @@ namespace brep const string& what, const basic_mark& diag) : text_ (t), - type_ (tp), strip_title_ (st), length_ (l), url_ (&u), @@ -629,8 +632,7 @@ namespace brep operator() (xml::serializer&) const; private: - const string& text_; - text_type type_; + const typed_text& text_; bool strip_title_; size_t length_; const string* url_; // Full page url. diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx new file mode 100644 index 0000000..9205f76 --- /dev/null +++ b/mod/tenant-service.hxx @@ -0,0 +1,155 @@ +// file : mod/tenant-service.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_TENANT_SERVICE_HXX +#define MOD_TENANT_SERVICE_HXX + +#include <map> + +#include <libbrep/types.hxx> +#include <libbrep/utility.hxx> + +#include <libbrep/build.hxx> + +#include <mod/diagnostics.hxx> + +namespace brep +{ + class tenant_service_base + { + public: + virtual ~tenant_service_base () = default; + }; + + // Possible build notifications: + // + // queued + // building + // built + // + // Possible transitions: + // + // -> queued + // queued -> building + // building -> queued (interrupted & re-queued due to higher priority task) + // building -> built + // built -> queued (periodic or user-forced rebuild) + // + // While the implementation tries to make sure the notifications arrive in + // the correct order, this is currently done by imposing delays (some + // natural, such as building->built, and some artificial, such as + // queued->building). As result, it is unlikely but possible to be notified + // about the state transitions in the wrong order, especially if the + // notifications take a long time. To minimize the chance of this happening, + // the service implementation should strive to batch the queued state + // notifications (or which there could be hundreds) in a single request if + // at all possible. Also, if supported by the third-party API, it makes + // sense for the implementation to protect against overwriting later states + // with earlier. For example, if it's possible to place a condition on a + // notification, it makes sense to only set the state to queued if none of + // the later states (e.g., building) are already in effect. + // + // Note also that it's possible for the build to get deleted at any stage + // without any further notifications. This can happen, for example, due to + // data retention timeout or because the build configuration (buildtab + // entry) is no longer present. There is no explicit `deleted` transition + // notification because such situations (i.e., when a notification sequence + // is abandoned half way) are not expected to arise ordinarily in a + // properly-configured brep instance. And the third-party service is + // expected to deal with them using some overall timeout/expiration + // mechanism which it presumably has. + // + // Each build notification is in its own interface since a service may not + // be interested in all of them while computing the information to pass is + // expensive. + + class tenant_service_build_queued: public virtual tenant_service_base + { + public: + // If the returned function is not NULL, it is called to update the + // service data. It should return the new data or nullopt if no update is + // necessary. Note: tenant_service::data passed to the callback and to the + // returned function may not be the same. Also, the returned function may + // be called multiple times (on transaction retries). + // + // The passed initial_state indicates the logical initial state and is + // either absent, `building` (interrupted), or `built` (rebuild). Note + // that all the passed build objects are for the same package version and + // have the same initial state. + // + // The implementation of this and the below functions should normally not + // need to make any decisions based on the passed build::state. Rather, + // the function name suffix (_queued, _building, _built) signify the + // logical end state. + // + // The build_queued_hints can be used to omit certain components from the + // build id. If single_package_version is true, then this tenant contains + // a single (non-test) package version and this package name and package + // version can be omitted. If single_package_config is true, then the + // package version being built only has the default package configuration + // and thus it can be omitted. + // + struct build_queued_hints + { + bool single_package_version; + bool single_package_config; + }; + + virtual function<optional<string> (const tenant_service&)> + build_queued (const tenant_service&, + const vector<build>&, + optional<build_state> initial_state, + const build_queued_hints&, + const diag_epilogue& log_writer) const noexcept = 0; + }; + + class tenant_service_build_building: public virtual tenant_service_base + { + public: + virtual function<optional<string> (const tenant_service&)> + build_building (const tenant_service&, + const build&, + const diag_epilogue& log_writer) const noexcept = 0; + }; + + class tenant_service_build_built: public virtual tenant_service_base + { + public: + virtual function<optional<string> (const tenant_service&)> + build_built (const tenant_service&, + const build&, + const diag_epilogue& log_writer) const noexcept = 0; + }; + + // Map of service type (tenant_service::type) to service. + // + using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>; + + // Every notification callback function that needs to produce any + // diagnostics shall begin with: + // + // NOTIFICATION_DIAG (log_writer); + // + // This will instantiate the error, warn, info, and trace diagnostics + // streams with the function's name. + // + // Note that a callback function is not expected to throw any exceptions. + // This is, in particular, why this macro doesn't instantiate the fail + // diagnostics stream. + // +#define NOTIFICATION_DIAG(log_writer) \ + const basic_mark error (severity::error, \ + log_writer, \ + __PRETTY_FUNCTION__); \ + const basic_mark warn (severity::warning, \ + log_writer, \ + __PRETTY_FUNCTION__); \ + const basic_mark info (severity::info, \ + log_writer, \ + __PRETTY_FUNCTION__); \ + const basic_mark trace (severity::trace, \ + log_writer, \ + __PRETTY_FUNCTION__) +} + +#endif // MOD_TENANT_SERVICE_HXX diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx index 3868c32..f135608 100644 --- a/mod/types-parsers.cxx +++ b/mod/types-parsers.cxx @@ -13,6 +13,7 @@ using namespace std; using namespace butl; using namespace bpkg; +using namespace bbot; using namespace web::xhtml; namespace brep @@ -114,6 +115,29 @@ namespace brep } } + // Parse interactive_mode. + // + void parser<interactive_mode>:: + parse (interactive_mode& x, bool& xs, scanner& s) + { + xs = true; + const char* o (s.next ()); + + if (!s.more ()) + throw missing_value (o); + + const string v (s.next ()); + + try + { + x = to_interactive_mode (v); + } + catch (const invalid_argument&) + { + throw invalid_value (o, v); + } + } + // Parse page_form. // void parser<page_form>:: @@ -180,9 +204,9 @@ namespace brep { x = fragment (v, o); } - catch (const xml::parsing&) + catch (const xml::parsing& e) { - throw invalid_value (o, v); + throw invalid_value (o, v, e.what ()); } } @@ -217,5 +241,47 @@ namespace brep throw invalid_value (o, v, os.str ()); } } + + // Parse build_order. + // + void parser<build_order>:: + parse (build_order& x, bool& xs, scanner& s) + { + xs = true; + const char* o (s.next ()); + + if (!s.more ()) + throw missing_value (o); + + const string v (s.next ()); + if (v == "stable") + x = build_order::stable; + else if (v == "random") + x = build_order::random; + else + throw invalid_value (o, v); + } + + // Parse build_email. + // + void parser<build_email>:: + parse (build_email& x, bool& xs, scanner& s) + { + xs = true; + const char* o (s.next ()); + + if (!s.more ()) + throw missing_value (o); + + const string v (s.next ()); + if (v == "none") + x = build_email::none; + else if (v == "latest") + x = build_email::latest; + else if (v == "all") + x = build_email::all; + else + throw invalid_value (o, v); + } } } diff --git a/mod/types-parsers.hxx b/mod/types-parsers.hxx index 05a7263..d48ae0b 100644 --- a/mod/types-parsers.hxx +++ b/mod/types-parsers.hxx @@ -10,6 +10,7 @@ #include <regex> #include <libbpkg/manifest.hxx> // repository_location +#include <libbbot/manifest.hxx> // interactive_mode #include <web/xhtml/fragment.hxx> @@ -58,6 +59,13 @@ namespace brep }; template <> + struct parser<bbot::interactive_mode> + { + static void + parse (bbot::interactive_mode&, bool&, scanner&); + }; + + template <> struct parser<page_form> { static void @@ -84,6 +92,20 @@ namespace brep static void parse (pair<std::regex, string>&, bool&, scanner&); }; + + template <> + struct parser<build_order> + { + static void + parse (build_order&, bool&, scanner&); + }; + + template <> + struct parser<build_email> + { + static void + parse (build_email&, bool&, scanner&); + }; } } |