diff options
| -rw-r--r-- | libbrep/package.hxx | 13 | ||||
| -rw-r--r-- | mod/mod-builds.cxx | 739 |
2 files changed, 561 insertions, 191 deletions
diff --git a/libbrep/package.hxx b/libbrep/package.hxx index ee51778..4693a34 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -495,6 +495,19 @@ namespace brep // #pragma db member(version) set(this.version.init (this.id.version, (?))) }; + + #pragma db view object(package) \ + object(repository: package::internal_repository) + struct package_version_count + { + size_t result; + + operator size_t () const {return result;} + + // Database mapping. + // + #pragma db member(result) column("count(" + package::id.name + ")") + }; } #endif // LIBBREP_PACKAGE_HXX diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index da72e17..e90e68f 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -4,12 +4,15 @@ #include <mod/mod-builds.hxx> +#include <set> + #include <libstudxml/serializer.hxx> #include <odb/database.hxx> #include <odb/transaction.hxx> -#include <libbutl/timestamp.hxx> // to_string() +#include <libbutl/timestamp.hxx> // to_string() +#include <libbutl/filesystem.hxx> // path_match() #include <libbbot/manifest.hxx> // to_result_status(), to_string(result_status) @@ -27,6 +30,7 @@ #include <mod/build-config.hxx> // *_url() using namespace std; +using namespace butl; using namespace bbot; using namespace web; using namespace odb::core; @@ -62,38 +66,38 @@ init (scanner& s) options_->root (dir_path ("/")); } -template <typename T, typename C> -static inline query<T> -build_query (const C& configs, const brep::params::builds& params) +// Transform the wildcard to the LIKE-pattern. +// +static string +transform (const string& s) { - using namespace brep; - - using query = query<T>; + if (s.empty ()) + return "%"; - // Transform the wildcard to the LIKE-pattern. - // - auto transform = [] (const string& s) -> string + string r; + for (char c: s) { - if (s.empty ()) - return "%"; - - string r; - for (char c: s) + switch (c) { - switch (c) - { - case '*': c = '%'; break; - case '?': c = '_'; break; - case '\\': - case '%': - case '_': r += '\\'; break; - } - - r += c; + case '*': c = '%'; break; + case '?': c = '_'; break; + case '\\': + case '%': + case '_': r += '\\'; break; } - return r; - }; + r += c; + } + + return r; +} + +template <typename T> +static inline query<T> +build_query (const brep::cstrings& configs, const brep::params::builds& params) +{ + using namespace brep; + using query = query<T>; query q ( query::id.configuration.in_range (configs.begin (), configs.end ())); @@ -112,10 +116,9 @@ build_query (const C& configs, const brep::params::builds& params) // Package version. // if (!params.version ().empty () && params.version () != "*") - { - version v (params.version ()); // May throw invalid_argument. - q = q && compare_version_eq (query::id.package.version, v, true); - } + q = q && compare_version_eq (query::id.package.version, + version (params.version ()), // May throw. + true); // Build toolchain name/version. // @@ -199,7 +202,47 @@ build_query (const C& configs, const brep::params::builds& params) return q; } +template <typename T> +static inline query<T> +package_query (const brep::params::builds& params) +{ + using namespace brep; + using query = query<T>; + + // Skip external and stub packages. + // + query q (query::package::internal_repository.is_not_null () && + compare_version_ne (query::package::id.version, + wildcard_version, + false)); + + // Note that there is no error reported if the filter parameters parsing + // fails. Instead, it is considered that no packages match such a query. + // + try + { + // Package name. + // + if (!params.name ().empty ()) + q = q && query::package::id.name.like (transform (params.name ())); + + // Package version. + // + if (!params.version ().empty () && params.version () != "*") + q = q && compare_version_eq (query::package::id.version, + version (params.version ()), // May throw. + true); + } + catch (const invalid_argument&) + { + return query (false); + } + + return q; +} + static const vector<pair<string, string>> build_results ({ + {"unbuilt", "<unbuilt>"}, {"*", "*"}, {"pending", "pending"}, {"building", "building"}, @@ -212,6 +255,7 @@ static const vector<pair<string, string>> build_results ({ bool brep::builds:: handle (request& rq, response& rs) { + using brep::version; using namespace web::xhtml; MODULE_DIAG; @@ -236,7 +280,6 @@ handle (request& rq, response& rs) throw invalid_request (400, e.what ()); } - size_t page (params.page ()); const char* title ("Builds"); xml::serializer s (rs.content (), title); @@ -259,191 +302,505 @@ handle (request& rq, response& rs) << DIV_HEADER (root, options_->logo (), options_->menu ()) << DIV(ID="content"); - transaction t (build_db_->begin ()); - - // Having packages and packages configurations builds in different databases, - // we unable to filter out builds for non-existent packages at the query - // level. Doing that in the C++ code would complicate it significantly. So - // we will print all the builds, relying on the sorting algorithm, that will - // likely to place expired ones at the end of the query result. + // Return the list of distinct toolchain name/version pairs. The build db + // transaction must be started and be the current one. // - auto count ( - build_db_->query_value<build_count> ( - build_query<build_count> (*build_conf_names_, params))); + using toolchains = vector<pair<string, version>>; - // Print the package builds filter form on the first page only. - // - if (page == 0) + auto query_toolchains = [this] () -> toolchains + { + using query = query<toolchain>; + + toolchains r; + for (auto& t: build_db_->query<toolchain> ( + "ORDER BY" + query::toolchain_name + + order_by_version_desc (query::id.toolchain_version, false))) + r.emplace_back (move (t.name), move (t.version)); + + return r; + }; + + auto print_form = [&s, ¶ms, this] (const toolchains& toolchains, + size_t build_count) { - // Populate the toolchains list with the distinct list of toolchain - // name/version pairs from all the existing package builds. Make sure the - // selected toolchain still present in the database. Otherwise fallback to - // the * wildcard selection. + // Print the package builds filter form on the first page only. // - string tc ("*"); - vector<pair<string, string>> toolchains ({{"*", "*"}}); + if (params.page () == 0) { - using query = query<toolchain>; - - for (const auto& t: build_db_->query<toolchain> ( - "ORDER BY" + query::toolchain_name + - order_by_version_desc (query::id.toolchain_version, false))) + // Populate the toolchains list with the distinct list of toolchain + // name/version pairs from all the existing package builds. Make sure the + // selected toolchain still present in the database. Otherwise fallback + // to the * wildcard selection. + // + string ctc ("*"); + vector<pair<string, string>> toolchain_opts ({{"*", "*"}}); { - string s (t.name + '-' + t.version.string ()); - toolchains.emplace_back (s, s); + for (const auto& t: toolchains) + { + string tc (t.first + '-' + t.second.string ()); + toolchain_opts.emplace_back (tc, tc); - if (s == params.toolchain ()) - tc = move (s); + if (tc == params.toolchain ()) + ctc = move (tc); + } } + + // The 'action' attribute is optional in HTML5. While the standard + // doesn't specify browser behavior explicitly for the case the attribute + // is omitted, the only reasonable behavior is to default it to the + // current document URL. Note that we specify the function name using the + // "hidden" <input/> element since the action url must not contain the + // query part. + // + s << FORM + << *INPUT(TYPE="hidden", NAME="builds") + << TABLE(ID="filter", CLASS="proplist") + << TBODY + << TR_INPUT ("name", "pn", params.name (), "*", true) + << TR_INPUT ("version", "pv", params.version (), "*") + << TR_SELECT ("toolchain", "tc", ctc, toolchain_opts) + + << TR(CLASS="config") + << TH << "config" << ~TH + << TD + << *INPUT(TYPE="text", + NAME="cf", + VALUE=params.configuration (), + PLACEHOLDER="*", + LIST="configs") + << DATALIST(ID="configs") + << *OPTION(VALUE="*"); + + for (const auto& c: *build_conf_names_) + s << *OPTION(VALUE=c); + + s << ~DATALIST + << ~TD + << ~TR + + << TR_INPUT ("machine", "mn", params.machine (), "*") + << TR_INPUT ("target", "tg", params.target (), "<default>") + << TR_SELECT ("result", "rs", params.result (), build_results) + << ~TBODY + << ~TABLE + << TABLE(CLASS="form-table") + << TBODY + << TR + << TD(ID="build-count") + << DIV_COUNTER (build_count, "Build", "Builds") + << ~TD + << TD(ID="filter-btn") + << *INPUT(TYPE="submit", VALUE="Filter") + << ~TD + << ~TR + << ~TBODY + << ~TABLE + << ~FORM; } + else + s << DIV_COUNTER (build_count, "Build", "Builds"); + }; + + size_t count; + size_t page (params.page ()); - // The 'action' attribute is optional in HTML5. While the standard doesn't - // specify browser behavior explicitly for the case the attribute is - // omitted, the only reasonable behavior is to default it to the current - // document URL. Note that we specify the function name using the "hidden" - // <input/> element since the action url must not contain the query part. + if (params.result () != "unbuilt") + { + transaction t (build_db_->begin ()); + + // Having packages and packages configurations builds in different + // databases, we unable to filter out builds for non-existent packages at + // the query level. Doing that in the C++ code would complicate it + // significantly. So we will print all the builds, relying on the sorting + // algorithm, that will likely to place expired ones at the end of the + // query result. // - s << FORM - << *INPUT(TYPE="hidden", NAME="builds") - << TABLE(ID="filter", CLASS="proplist") - << TBODY - << TR_INPUT ("name", "pn", params.name (), "*", true) - << TR_INPUT ("version", "pv", params.version (), "*") - << TR_SELECT ("toolchain", "tc", tc, toolchains) - - << TR(CLASS="config") - << TH << "config" << ~TH - << TD - << *INPUT(TYPE="text", - NAME="cf", - VALUE=params.configuration (), - PLACEHOLDER="*", - LIST="configs") - << DATALIST(ID="configs") - << *OPTION(VALUE="*"); - - for (const auto& c: *build_conf_names_) - s << *OPTION(VALUE=c); - - s << ~DATALIST - << ~TD - << ~TR - - << TR_INPUT ("machine", "mn", params.machine (), "*") - << TR_INPUT ("target", "tg", params.target (), "<default>") - << TR_SELECT ("result", "rs", params.result (), build_results) - << ~TBODY - << ~TABLE - << TABLE(CLASS="form-table") - << TBODY - << TR - << TD(ID="build-count") - << DIV_COUNTER (count, "Build", "Builds") - << ~TD - << TD(ID="filter-btn") - << *INPUT(TYPE="submit", VALUE="Filter") - << ~TD - << ~TR - << ~TBODY - << ~TABLE - << ~FORM; - } - else - s << DIV_COUNTER (count, "Build", "Builds"); + count = build_db_->query_value<build_count> ( + build_query<build_count> (*build_conf_names_, params)); - timestamp now (timestamp::clock::now ()); + // Print the filter form. + // + print_form (query_toolchains (), count); - // Enclose the subsequent tables to be able to use nth-child CSS selector. - // - s << DIV; - for (auto& b: build_db_->query<build> ( - build_query<build> (*build_conf_names_, params) + - "ORDER BY" + query<build>::timestamp + "DESC" + - "OFFSET" + to_string (page * page_configs) + - "LIMIT" + to_string (page_configs))) + // Print package build configurations ordered by the timestamp (later goes + // first). + // + timestamp now (timestamp::clock::now ()); + + // Enclose the subsequent tables to be able to use nth-child CSS selector. + // + s << DIV; + for (auto& b: build_db_->query<build> ( + build_query<build> (*build_conf_names_, params) + + "ORDER BY" + query<build>::timestamp + "DESC" + + "OFFSET" + to_string (page * page_configs) + + "LIMIT" + to_string (page_configs))) + { + string ts (butl::to_string (b.timestamp, + "%Y-%m-%d %H:%M:%S %Z", + true, + true) + + " (" + butl::to_string (now - b.timestamp, false) + " ago)"); + + s << TABLE(CLASS="proplist build") + << TBODY + << TR_NAME (b.package_name, string (), root) + << TR_VERSION (b.package_name, b.package_version, root) + << TR_VALUE ("toolchain", + b.toolchain_name + '-' + + b.toolchain_version.string ()) + << TR_VALUE ("config", b.configuration) + << TR_VALUE ("machine", b.machine) + << TR_VALUE ("target", + b.target ? b.target->string () : "<default>") + << TR_VALUE ("timestamp", ts) + << TR(CLASS="result") + << TH << "result" << ~TH + << TD + << SPAN(CLASS="value"); + + if (b.state == build_state::building) + s << "building | "; + else + { + build_db_->load (b, b.results_section); + + // If no unsuccessful operations results available, then print the + // overall build status. If there are any operations results available, + // then also print unsuccessful operations statuses with the links to + // the respective logs, followed with a link to the operations combined + // log. Print the forced package rebuild link afterwards, unless the + // package build is already pending. + // + if (b.results.empty () || *b.status == result_status::success) + { + assert (b.status); + s << SPAN_BUILD_RESULT_STATUS (*b.status) << " | "; + } + + if (!b.results.empty ()) + { + for (const auto& r: b.results) + { + if (r.status != result_status::success) + s << SPAN_BUILD_RESULT_STATUS (r.status) << " (" + << A + << HREF + << build_log_url (host, root, b, &r.operation) + << ~HREF + << r.operation + << ~A + << ") | "; + } + + s << A + << HREF << build_log_url (host, root, b) << ~HREF + << "log" + << ~A + << " | "; + } + } + + if (b.force == (b.state == build_state::building + ? force_state::forcing + : force_state::forced)) + s << "pending"; + else + s << A + << HREF << force_rebuild_url (host, root, b) << ~HREF + << "rebuild" + << ~A; + + s << ~SPAN + << ~TD + << ~TR + << ~TBODY + << ~TABLE; + } + s << ~DIV; + + t.commit (); + } + else // Print unbuilt package configurations. { - string ts (butl::to_string (b.timestamp, - "%Y-%m-%d %H:%M:%S %Z", - true, - true) + - " (" + butl::to_string (now - b.timestamp, false) + " ago)"); - - s << TABLE(CLASS="proplist build") - << TBODY - << TR_NAME (b.package_name, string (), root) - << TR_VERSION (b.package_name, b.package_version, root) - << TR_VALUE ("toolchain", - b.toolchain_name + '-' + b.toolchain_version.string ()) - << TR_VALUE ("config", b.configuration) - << TR_VALUE ("machine", b.machine) - << TR_VALUE ("target", b.target ? b.target->string () : "<default>") - << TR_VALUE ("timestamp", ts) - << TR(CLASS="result") - << TH << "result" << ~TH - << TD - << SPAN(CLASS="value"); - - if (b.state == build_state::building) - s << "building | "; - else + // Parameters to use for package build configurations queries. Note that we + // cleanup the machine and the result filter arguments, as they are + // irrelevant for unbuilt configurations. + // + params::builds bld_params (params); + bld_params.machine ().clear (); + bld_params.result () = "*"; + + // Query toolchains, and the number of package configurations being present + // in the build database and satisfying the specified filter arguments. + // Later we will subtract it from the maximum number of unbuilt package + // configurations, to get the real number of them. + // + toolchains toolchains; + { + transaction t (build_db_->begin ()); + toolchains = query_toolchains (); + + count = build_db_->query_value<build_count> ( + build_query<build_count> (*build_conf_names_, bld_params)); + + t.commit (); + } + + // Filter build configurations and toolchains, and create the set of + // configuration/toolchain combinations, that we will print for packages. + // + struct config_toolchain { - build_db_->load (b, b.results_section); - - // If no unsuccessful operations results available, then print the - // overall build status. If there are any operations results available, - // then also print unsuccessful operations statuses with the links to the - // respective logs, followed with a link to the operations combined log. - // Print the forced package rebuild link afterwards, unless the package - // build is already pending. + const string& configuration; + const string& toolchain_name; + const version& toolchain_version; + + bool + operator< (const config_toolchain& ct) const + { + int r (configuration.compare (ct.configuration)); + if (r != 0) + return r < 0; + + r = toolchain_name.compare (ct.toolchain_name); + if (r != 0) + return r < 0; + + return toolchain_version > ct.toolchain_version; + } + }; + + string tc_name; + version tc_version; + const string& tc (params.toolchain ()); + + if (tc != "*") + try + { + size_t p (tc.find ('-')); + if (p == string::npos) // Invalid format. + throw invalid_argument (""); + + tc_name.assign (tc, 0, p); + tc_version = version (string (tc, p + 1)); // May throw invalid_argument. + } + catch (const invalid_argument&) + { + // This is unlikely to be the user fault, as he selects the toolchain + // from the list. // - if (b.results.empty () || *b.status == result_status::success) + throw invalid_request (400, "invalid toolchain"); + } + + set<config_toolchain> config_toolchains; + { + const string& pc (params.configuration ()); + const string& tg (params.target ()); + + for (const auto& c: *build_conf_) { - assert (b.status); - s << SPAN_BUILD_RESULT_STATUS (*b.status) << " | "; + if ((pc.empty () || path_match (pc, c.name)) && // Filter by name. + + + (tg.empty () // Filter by target. + ? !c.target + : tg == "*" || + (c.target && path_match (tg, c.target->string ())))) + { + if (tc != "*") // Filter by toolchain. + config_toolchains.insert ({c.name, tc_name, tc_version}); + else // Add all toolchains. + { + for (const auto& t: toolchains) + config_toolchains.insert ({c.name, t.first, t.second}); + } + } } - if (!b.results.empty ()) + // Calculate the number of unbuilt package configuration as a difference + // between the maximum possible number of unbuilt configurations and the + // number of configurations being in the built or building state (see + // above). + // + transaction t (package_db_->begin ()); + + size_t n (config_toolchains.size () * + package_db_->query_value<package_version_count> ( + package_query<package_version_count> (params))); + + count = n > count ? n - count : 0; + t.commit (); + } + + // Print the filter form. + // + print_form (toolchains, count); + + // Print unbuilt package configurations with the following sort priority: + // + // 1: package name + // 2: package version (descending) + // 3: configuration name + // 4: toolchain name + // 5: toolchain version (descending) + // + // Prepare the package version prepared query. + // + // Note that we can't skip the proper number of packages in the database + // query for a page numbers greater than one. So we will query packages + // from the very beginning and skip the appropriate number of them while + // iterating through the query result. + // + // Note that such an approach has a security implication. HTTP request + // with a large page number will be quite expensive to process, as it + // effectively results in traversing all package versions and all the built + // configurations. To address this problem we may consider to reduce the + // pager to just '<Prev' '1' 'Next>' links, and pass the offset as a URL + // query parameter. Alternatively, we may invent the page number cap. + // + using pkg_query = query<package_version>; + using prep_pkg_query = prepared_query<package_version>; + + pkg_query pq (package_query<package_version> (params)); + + // Specify the portion. Note that we will still be querying packages in + // chunks, not to hold locks for too long. + // + size_t offset (0); + + pq += "ORDER BY" + + pkg_query::package::id.name + + order_by_version_desc (pkg_query::package::id.version, false) + + "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"; + + connection_ptr pkg_conn (package_db_->connection ()); + + prep_pkg_query pkg_prep_query ( + pkg_conn->prepare_query<package_version> ( + "mod-builds-package-version-query", pq)); + + // Prepare the build prepared query. + // + using bld_query = query<build>; + using prep_bld_query = prepared_query<build>; + + package_id id; // See the build query. + + const auto& qv (bld_query::id.package.version); + + // We will use specific package name and version for each database query. + // + bld_params.name ().clear (); + bld_params.version ().clear (); + + bld_query bq ( + bld_query::id.package.name == bld_query::_ref (id.name) && + + qv.epoch == bld_query::_ref (id.version.epoch) && + qv.canonical_upstream == + bld_query::_ref (id.version.canonical_upstream) && + qv.canonical_release == + bld_query::_ref (id.version.canonical_release) && + qv.revision == bld_query::_ref (id.version.revision) && + build_query<build_count> (*build_conf_names_, bld_params)); + + connection_ptr bld_conn (build_db_->connection ()); + + prep_bld_query bld_prep_query ( + bld_conn->prepare_query<build> ( + "mod-builds-package-build-query", bq)); + + size_t skip (page * page_configs); + size_t print (page_configs); + + // Enclose the subsequent tables to be able to use nth-child CSS selector. + // + s << DIV; + for (bool prn (true); prn; ) + { + // Start the package database transaction. + // + transaction pt (pkg_conn->begin ()); + + // Query package versions. + // + auto package_versions (pkg_prep_query.execute ()); + + if ((prn = !package_versions.empty ())) { - for (const auto& r: b.results) + offset += package_versions.size (); + + // Start the build database transaction. + // + transaction bt (bld_conn->begin (), false); + transaction::current (bt); + + // Iterate over packages and print unbuilt configurations. Skip the + // appropriate number of them first (for page number greater than one). + // + for (auto& pv: package_versions) { - if (r.status != result_status::success) - s << SPAN_BUILD_RESULT_STATUS (r.status) << " (" - << A - << HREF - << build_log_url (host, root, b, &r.operation) - << ~HREF - << r.operation - << ~A - << ") | "; + id = move (pv.id); + + // Iterate through the package configurations builds and erase them + // from the unbuilt configurations set. + // + // Make a copy for this package. + // + auto unbuilt_configs (config_toolchains); + for (const auto& b: bld_prep_query.execute ()) + unbuilt_configs.erase ({ + b.id.configuration, b.toolchain_name, b.toolchain_version}); + + // Print unbuilt package configurations. + // + for (const auto& ct: unbuilt_configs) + { + if (skip > 0) + { + --skip; + continue; + } + + if (print-- == 0) + { + prn = false; + break; + } + + auto i (build_conf_map_->find (ct.configuration.c_str ())); + assert (i != build_conf_map_->end ()); + + const optional<target_triplet>& tg (i->second->target); + + s << TABLE(CLASS="proplist build") + << TBODY + << TR_NAME (id.name, string (), root) + << TR_VERSION (id.name, pv.version, root) + << TR_VALUE ("toolchain", + string (ct.toolchain_name) + '-' + + ct.toolchain_version.string ()) + << TR_VALUE ("config", ct.configuration) + << TR_VALUE ("target", tg ? tg->string () : "<default>") + << ~TBODY + << ~TABLE; + } + + if (!prn) + break; } - s << A - << HREF << build_log_url (host, root, b) << ~HREF - << "log" - << ~A - << " | "; + // Commit the build database transaction and switch to the package + // database transaction. + bt.commit (); + transaction::current (pt); } - } - if (b.force == (b.state == build_state::building - ? force_state::forcing - : force_state::forced)) - s << "pending"; - else - s << A - << HREF << force_rebuild_url (host, root, b) << ~HREF - << "rebuild" - << ~A; - - s << ~SPAN - << ~TD - << ~TR - << ~TBODY - << ~TABLE; + pt.commit (); + } + s << ~DIV; } - s << ~DIV; - - t.commit (); string u (root.string () + "?builds"); |