diff options
Diffstat (limited to 'mod/mod-builds.cxx')
-rw-r--r-- | mod/mod-builds.cxx | 759 |
1 files changed, 423 insertions, 336 deletions
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index 5ffe6dd..30562f3 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -4,15 +4,15 @@ #include <mod/mod-builds.hxx> #include <set> -#include <algorithm> // find_if() #include <libstudxml/serializer.hxx> #include <odb/database.hxx> #include <odb/transaction.hxx> -#include <libbutl/timestamp.mxx> // to_string() -#include <libbutl/path-pattern.mxx> +#include <libbutl/utility.hxx> // compare_c_string +#include <libbutl/timestamp.hxx> // to_string() +#include <libbutl/path-pattern.hxx> #include <libbbot/manifest.hxx> // to_result_status(), to_string(result_status) @@ -31,7 +31,6 @@ using namespace std; using namespace butl; -using namespace bbot; using namespace web; using namespace odb::core; using namespace brep::cli; @@ -137,10 +136,9 @@ match (const C qc, const string& pattern) // template <typename T> static inline query<T> -build_query (const brep::cstrings* configs, +build_query (const brep::vector<brep::build_target_config_id>* config_ids, const brep::params::builds& params, - const brep::optional<brep::string>& tenant, - const brep::optional<bool>& archived) + const brep::optional<brep::string>& tenant) { using namespace brep; using query = query<T>; @@ -151,12 +149,15 @@ build_query (const brep::cstrings* configs, query q (tenant ? pid.tenant == *tenant : !qt::private_); - if (archived) - q = q && qt::archived == *archived; + if (config_ids != nullptr) + { + query sq (false); + for (const auto& id: *config_ids) + sq = sq || (qb::id.target == id.target && + qb::id.target_config_name == id.config); - if (configs != nullptr) - q = q && qb::id.configuration.in_range (configs->begin (), - configs->end ()); + q = q && sq; + } // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no package builds match such a @@ -175,7 +176,7 @@ build_query (const brep::cstrings* configs, { // May throw invalid_argument. // - version v (params.version (), false /* fold_zero_revision */); + version v (params.version (), version::none); q = q && compare_version_eq (pid.version, canonical_version (v), @@ -184,11 +185,11 @@ build_query (const brep::cstrings* configs, // Build toolchain name/version. // - const string& tc (params.toolchain ()); + const string& th (params.toolchain ()); - if (tc != "*") + if (th != "*") { - size_t p (tc.find ('-')); + size_t p (th.find ('-')); if (p == string::npos) // Invalid format. throw invalid_argument (""); @@ -196,8 +197,8 @@ build_query (const brep::cstrings* configs, // the exact version revision, so an absent and zero revisions have the // same semantics and the zero revision is folded. // - string tn (tc, 0, p); - version tv (string (tc, p + 1)); // May throw invalid_argument. + string tn (th, 0, p); + version tv (string (th, p + 1)); // May throw invalid_argument. q = q && qb::id.toolchain_name == tn && @@ -206,38 +207,44 @@ build_query (const brep::cstrings* configs, true /* revision */); } - // Build configuration name. + // Build target. // - if (!params.configuration ().empty ()) - q = q && match<T> (qb::id.configuration, params.configuration ()); + if (!params.target ().empty ()) + q = q && match<T> (qb::id.target, params.target ()); - // Build machine name. + // Build target configuration name. // - if (!params.machine ().empty ()) - q = q && match<T> (qb::machine, params.machine ()); + if (!params.target_config ().empty ()) + q = q && match<T> (qb::id.target_config_name, params.target_config ()); - // Build target. + // Build package configuration name. // - if (!params.target ().empty ()) - q = q && match<T> (qb::target, params.target ()); + if (!params.package_config ().empty ()) + q = q && match<T> (qb::id.package_config_name, params.package_config ()); // Build result. // const string& rs (params.result ()); + bool add_state (true); if (rs != "*") { if (rs == "pending") + { q = q && qb::force != "unforced"; + } else if (rs == "building") + { q = q && qb::state == "building"; + add_state = false; + } else { query sq (qb::status == rs); // May throw invalid_argument. // - result_status st (to_result_status (rs)); + result_status st (bbot::to_result_status (rs)); if (st != result_status::success) { @@ -258,8 +265,12 @@ build_query (const brep::cstrings* configs, // well (rebuild). // q = q && qb::state == "built" && sq; + add_state = false; } } + + if (add_state) + q = q && qb::state != "queued"; } catch (const invalid_argument&) { @@ -274,8 +285,7 @@ build_query (const brep::cstrings* configs, template <typename T> static inline query<T> package_query (const brep::params::builds& params, - const brep::optional<brep::string>& tenant, - const brep::optional<bool>& archived) + const brep::optional<brep::string>& tenant) { using namespace brep; using query = query<T>; @@ -284,9 +294,6 @@ package_query (const brep::params::builds& params, query q (tenant ? qp::id.tenant == *tenant : !qt::private_); - if (archived) - q = q && qt::archived == *archived; - // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no packages match such a query. // @@ -303,7 +310,7 @@ package_query (const brep::params::builds& params, { // May throw invalid_argument. // - version v (params.version (), false /* fold_zero_revision */); + version v (params.version (), version::none); q = q && compare_version_eq (qp::id.version, canonical_version (v), @@ -357,11 +364,6 @@ handle (request& rq, response& rs) throw invalid_request (400, e.what ()); } - // Override the name parameter for the old URL (see options.cli for details). - // - if (params.name_legacy_specified ()) - params.name (params.name_legacy ()); - const char* title ("Builds"); xml::serializer s (rs.content (), title); @@ -391,8 +393,11 @@ handle (request& rq, response& rs) if (!tenant.empty ()) tn = tenant; - // Return the list of distinct toolchain name/version pairs. The build db - // transaction must be started. + // Return the list of distinct toolchain name/version pairs. If no builds + // are present for the tenant, then fallback to the toolchain recorded in + // the tenant object, if present. + // + // Note: the build db transaction must be started. // using toolchains = vector<pair<string, version>>; @@ -408,11 +413,19 @@ handle (request& rq, response& rs) false /* first */))) r.emplace_back (move (t.name), move (t.version)); + if (r.empty ()) + { + shared_ptr<build_tenant> t (build_db_->find<build_tenant> (tenant)); + + if (t != nullptr && t->toolchain) + r.emplace_back (t->toolchain->name, t->toolchain->version); + } + return r; }; auto print_form = [&s, ¶ms, this] (const toolchains& toolchains, - size_t build_count) + optional<size_t> build_count) { // Print the package builds filter form on the first page only. // @@ -423,16 +436,16 @@ handle (request& rq, response& rs) // the selected toolchain is still present in the database. Otherwise // fallback to the * wildcard selection. // - string ctc ("*"); + string cth ("*"); vector<pair<string, string>> toolchain_opts ({{"*", "*"}}); { for (const auto& t: toolchains) { - string tc (t.first + '-' + t.second.string ()); - toolchain_opts.emplace_back (tc, tc); + string th (t.first + '-' + t.second.string ()); + toolchain_opts.emplace_back (th, th); - if (tc == params.toolchain ()) - ctc = move (tc); + if (th == params.toolchain ()) + cth = move (th); } } @@ -448,28 +461,42 @@ handle (request& rq, response& rs) << TBODY << TR_INPUT ("name", "builds", params.name (), "*", true) << TR_INPUT ("version", "pv", params.version (), "*") - << TR_SELECT ("toolchain", "tc", ctc, toolchain_opts) + << TR_SELECT ("toolchain", "th", cth, toolchain_opts) + << TR_INPUT ("target", "tg", params.target (), "*") - << TR(CLASS="config") - << TH << "config" << ~TH + << TR(CLASS="tgt-config") + << TH << "tgt config" << ~TH << TD << *INPUT(TYPE="text", - NAME="cf", - VALUE=params.configuration (), + NAME="tc", + VALUE=params.target_config (), PLACEHOLDER="*", - LIST="configs") - << DATALIST(ID="configs") + LIST="target-configs") + << DATALIST(ID="target-configs") << *OPTION(VALUE="*"); - for (const auto& c: *build_conf_names_) - s << *OPTION(VALUE=c); + // Print unique config names from the target config map. + // + set<const char*, butl::compare_c_string> conf_names; + for (const auto& c: *target_conf_map_) + { + if (conf_names.insert (c.first.config.get ().c_str ()).second) + s << *OPTION(VALUE=c.first.config.get ()); + } s << ~DATALIST << ~TD << ~TR - << TR_INPUT ("machine", "mn", params.machine (), "*") - << TR_INPUT ("target", "tg", params.target (), "*") + << TR(CLASS="pkg-config") + << TH << "pkg config" << ~TH + << TD + << *INPUT(TYPE="text", + NAME="pc", + VALUE=params.package_config (), + PLACEHOLDER="*") + << ~TD + << ~TR << TR_SELECT ("result", "rs", params.result (), build_results) << ~TBODY << ~TABLE @@ -491,26 +518,25 @@ handle (request& rq, response& rs) s << DIV_COUNTER (build_count, "Build", "Builds"); }; + const string& tgt (params.target ()); + const string& tgt_cfg (params.target_config ()); + const string& pkg_cfg (params.package_config ()); + // We will not display hidden configurations, unless the configuration is // specified explicitly. // - bool exclude_hidden (params.configuration ().empty () || - path_pattern (params.configuration ())); + bool exclude_hidden (tgt_cfg.empty () || path_pattern (tgt_cfg)); - cstrings conf_names; + vector<build_target_config_id> conf_ids; + conf_ids.reserve (target_conf_map_->size ()); - if (exclude_hidden) + for (const auto& c: *target_conf_map_) { - for (const auto& c: *build_conf_map_) - { - if (belongs (*c.second, "all")) - conf_names.push_back (c.first); - } + if (!exclude_hidden || !belongs (*c.second, "hidden")) + conf_ids.push_back (c.first); } - else - conf_names = *build_conf_names_; - size_t count; + optional<size_t> count; size_t page (params.page ()); if (params.result () != "unbuilt") // Print package build configurations. @@ -525,37 +551,22 @@ handle (request& rq, response& rs) // printing the builds. // count = 0; - vector<shared_ptr<build>> builds; + vector<package_build> builds; builds.reserve (page_configs); - // Prepare the package build prepared query. + // Prepare the package build query. // using query = query<package_build>; - using prep_query = prepared_query<package_build>; - query q (build_query<package_build> ( - &conf_names, params, tn, nullopt /* archived */)); - - // Specify the portion. Note that we will be querying builds in chunks, - // not to hold locks for too long. - // - // Also note that for each build we also load the corresponding - // package. Nevertheless, we use a fairly large portion to speed-up the - // builds traversal but also cache the package objects (see below). - // - size_t offset (0); + query q (build_query<package_build> (&conf_ids, params, tn)); // Print package build configurations ordered by the timestamp (later goes // first). // - q += "ORDER BY" + query::build::timestamp + "DESC" + - "OFFSET" + query::_ref (offset) + "LIMIT 500"; + q += "ORDER BY" + query::build::timestamp + "DESC"; connection_ptr conn (build_db_->connection ()); - prep_query pq ( - conn->prepare_query<package_build> ("mod-builds-query", q)); - // Note that we can't skip the proper number of builds in the database // query for a page numbers greater than one. So we will query builds from // the very beginning and skip the appropriate number of them while @@ -571,81 +582,101 @@ handle (request& rq, response& rs) // session sn; - for (bool ne (true); ne; ) + transaction t (conn->begin ()); + + // For some reason PostgreSQL (as of 9.4) picks the nested loop join + // strategy for the below package_build query, which executes quite slow + // even for reasonably small number of builds. Thus, we just discourage + // PostgreSQL from using this strategy in the current transaction. + // + // @@ TMP Re-check for the later PostgreSQL versions if we can drop this + // hint. If drop, then also grep for other places where this hint + // is used. + // + conn->execute ("SET LOCAL enable_nestloop=off"); + + // Iterate over builds and cache build objects that should be printed. + // Skip the appropriate number of them (for page number greater than + // one). + // + for (auto& pb: build_db_->query<package_build> (q)) { - transaction t (conn->begin ()); + shared_ptr<build>& b (pb.build); + + auto i ( + target_conf_map_->find ( + build_target_config_id {b->target, b->target_config_name})); - // Query package builds (and cache the result). + assert (i != target_conf_map_->end ()); + + // Match the target configuration against the package build + // configuration expressions/constraints. // - auto bs (pq.execute ()); + shared_ptr<build_package> p ( + build_db_->load<build_package> (b->id.package)); + + const build_package_config* pc (find (b->package_config_name, + p->configs)); - if ((ne = !bs.empty ())) + // The package configuration should be present since the configurations + // set cannot change if the package version doesn't change. If that's + // not the case, then the database has probably been manually amended. + // In this case let's just skip such a build as if it excluded and log + // the warning. + // + if (pc == nullptr) { - offset += bs.size (); + warn << "cannot find configuration '" << b->package_config_name + << "' for package " << p->id.name << '/' << p->version; - // Iterate over builds and cache build objects that should be printed. - // Skip the appropriate number of them (for page number greater than - // one). - // - for (auto& pb: bs) - { - shared_ptr<build>& b (pb.build); + continue; + } - auto i (build_conf_map_->find (b->configuration.c_str ())); - assert (i != build_conf_map_->end ()); + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); - // Match the configuration against the package build - // expressions/constraints. + if (!exclude (*pc, p->builds, p->constraints, *i->second)) + { + if (skip != 0) + --skip; + else if (print != 0) + { + // As we query builds in multiple transactions we may see the same + // build multiple times. Let's skip the duplicates. Note: we don't + // increment the counter in this case. // - shared_ptr<build_package> p ( - build_db_->load<build_package> (b->id.package)); - - if (!exclude (p->builds, p->constraints, *i->second)) + if (find_if (builds.begin (), builds.end (), + [&b] (const package_build& pb) + { + return b->id == pb.build->id; + }) != builds.end ()) + continue; + + if (b->state == build_state::built) { - if (skip != 0) - --skip; - else if (print != 0) - { - // As we query builds in multiple transactions we may see the - // same build multiple times. Let's skip the duplicates. Note: - // we don't increment the counter in this case. - // - if (find_if (builds.begin (), - builds.end (), - [&b] (const shared_ptr<build>& pb) - { - return b->id == pb->id; - }) != builds.end ()) - continue; - - if (b->state == build_state::built) - { - build_db_->load (*b, b->results_section); - - // Let's clear unneeded result logs for builds being cached. - // - for (operation_result& r: b->results) - r.log.clear (); - } + build_db_->load (*b, b->results_section); - builds.push_back (move (b)); + // Let's clear unneeded result logs for builds being cached. + // + for (operation_result& r: b->results) + r.log.clear (); + } - --print; - } + builds.push_back (move (pb)); - ++count; - } + --print; } + + ++(*count); } + } - // Print the filter form after the build count is calculated. Note: - // query_toolchains() must be called inside the build db transaction. - // - else - print_form (query_toolchains (), count); + // Print the filter form after the build count is calculated. Note: + // query_toolchains() must be called inside the build db transaction. + // + print_form (query_toolchains (), count); - t.commit (); - } + t.commit (); // Finally, print the cached package build configurations. // @@ -654,32 +685,37 @@ handle (request& rq, response& rs) // Enclose the subsequent tables to be able to use nth-child CSS selector. // s << DIV; - for (const shared_ptr<build>& pb: builds) + for (const package_build& pb: builds) { - const build& b (*pb); + const build& b (*pb.build); string ts (butl::to_string (b.timestamp, "%Y-%m-%d %H:%M:%S %Z", true /* special */, true /* local */) + - " (" + butl::to_string (now - b.timestamp, false) + " ago)"); + " (" + butl::to_string (now - b.timestamp, false) + " ago"); + + if (pb.archived) + ts += ", archived"; + + ts += ')'; s << TABLE(CLASS="proplist build") << TBODY - << TR_NAME (b.package_name, string (), root, b.tenant) + << TR_NAME (b.package_name, root, b.tenant) << TR_VERSION (b.package_name, b.package_version, root, b.tenant) << TR_VALUE ("toolchain", b.toolchain_name + '-' + b.toolchain_version.string ()) - << TR_VALUE ("config", b.configuration) - << TR_VALUE ("machine", b.machine) << TR_VALUE ("target", b.target.string ()) + << TR_VALUE ("tgt config", b.target_config_name) + << TR_VALUE ("pkg config", b.package_config_name) << TR_VALUE ("timestamp", ts); if (b.interactive) // Note: can only be present for the building state. s << TR_VALUE ("login", *b.interactive); - s << TR_BUILD_RESULT (b, host, root); + s << TR_BUILD_RESULT (b, pb.archived, host, root); // In the global view mode add the tenant builds link. Note that the // global view (and the link) makes sense only in the multi-tenant mode. @@ -695,47 +731,73 @@ handle (request& rq, response& rs) else // Print unbuilt package configurations. { // Parameters to use for package build configurations queries. Note that - // we cleanup the machine and the result filter arguments, as they are - // irrelevant for unbuilt configurations. + // we cleanup the result filter argument, as it is irrelevant for unbuilt + // configurations. // params::builds bld_params (params); - bld_params.machine ().clear (); bld_params.result () = "*"; - // Query toolchains, filter build configurations and toolchains, and - // create the set of configuration/toolchain combinations, that we will - // print for packages. Also calculate the number of unbuilt package - // configurations. + // Query toolchains, filter build target configurations and toolchains, + // and create the set of target configuration/toolchain combinations, that + // we will print for package configurations. Also calculate the number of + // unbuilt package configurations. // toolchains toolchains; - // Note that config_toolchains contains shallow references to the - // toolchain names and versions. + // Target configuration/toolchain combination. + // + // Note: all members are the shallow references. // - set<config_toolchain> config_toolchains; + struct target_config_toolchain + { + const butl::target_triplet& target; + const string& target_config; + const string& toolchain_name; + const bpkg::version& toolchain_version; + }; + + // Cache the build package objects that would otherwise be loaded twice: + // first time during calculating the builds count and then during printing + // the builds. Note that the build package is a subset of the package + // object and normally has a small memory footprint. + // + // @@ TMP It feels that we can try to combine the mentioned steps and + // improve the performance a bit. We won't need the session in this + // case. + // + session sn; + + connection_ptr conn (build_db_->connection ()); + transaction t (conn->begin ()); + + // Discourage PostgreSQL from using the nested loop join strategy in the + // current transaction (see above for details). + // + conn->execute ("SET LOCAL enable_nestloop=off"); + + vector<target_config_toolchain> config_toolchains; { - transaction t (build_db_->begin ()); toolchains = query_toolchains (); - string tc_name; - version tc_version; - const string& tc (params.toolchain ()); + string th_name; + version th_version; + const string& th (params.toolchain ()); - if (tc != "*") + if (th != "*") try { - size_t p (tc.find ('-')); + size_t p (th.find ('-')); if (p == string::npos) // Invalid format. throw invalid_argument (""); - tc_name.assign (tc, 0, p); + th_name.assign (th, 0, p); // May throw invalid_argument. // // Note that an absent and zero revisions have the same semantics, // so the zero revision is folded (see above for details). // - tc_version = version (string (tc, p + 1)); + th_version = version (string (th, p + 1)); } catch (const invalid_argument&) { @@ -745,63 +807,63 @@ handle (request& rq, response& rs) throw invalid_request (400, "invalid toolchain"); } - const string& pc (params.configuration ()); - const string& tg (params.target ()); - vector<const build_config*> configs; + vector<const build_target_config*> target_configs; - for (const auto& c: *build_conf_) + for (const auto& c: *target_conf_) { - if ((pc.empty () || path_match (c.name, pc)) && // Filter by name. + // Filter by name. + // + if ((tgt_cfg.empty () || path_match (c.name, tgt_cfg)) && // Filter by target. // - (tg.empty () || path_match (c.target.string (), tg)) && + (tgt.empty () || path_match (c.target.string (), tgt)) && - (!exclude_hidden || belongs (c, "all"))) // Filter hidden. + (!exclude_hidden || !belongs (c, "hidden"))) // Filter hidden. { - configs.push_back (&c); + target_configs.push_back (&c); for (const auto& t: toolchains) { // Filter by toolchain. // - if (tc == "*" || (t.first == tc_name && t.second == tc_version)) - config_toolchains.insert ({c.name, t.first, t.second}); + if (th == "*" || (t.first == th_name && t.second == th_version)) + config_toolchains.push_back ( + target_config_toolchain {c.target, c.name, t.first, t.second}); } } } - // Calculate the number of unbuilt package configurations as a - // difference between the maximum possible number of unbuilt - // configurations and the number of existing package builds. - // - // Note that we also need to deduct the package-excluded configurations - // count from the maximum possible number of unbuilt configurations. The - // only way to achieve this is to traverse through the packages and - // match their build expressions/constraints against our configurations. - // - // Also note that some existing builds can now be excluded by packages - // due to the build configuration target or class set change. We should - // deduct such builds count from the number of existing package builds. - // - size_t nmax ( - config_toolchains.size () * - build_db_->query_value<buildable_package_count> ( - package_query<buildable_package_count> ( - params, tn, false /* archived */))); - - size_t ncur = build_db_->query_value<package_build_count> ( - build_query<package_build_count> ( - &conf_names, bld_params, tn, false /* archived */)); - - // From now we will be using specific package name and version for each - // build database query. - // - bld_params.name ().clear (); - bld_params.version ().clear (); - if (!config_toolchains.empty ()) { + // Calculate the number of unbuilt package configurations as a + // difference between the possible number of unbuilt configurations + // and the number of existing package builds. + // + // Note that some existing builds can now be excluded by package + // configurations due to the build target configuration class set + // change. We should deduct such builds count from the number of + // existing package configurations builds. + // + // The only way to calculate both numbers is to traverse through the + // package configurations and match their build + // expressions/constraints against our target configurations. + // + size_t npos (0); + + size_t ncur (build_db_->query_value<package_build_count> ( + build_query<package_build_count> (&conf_ids, bld_params, tn))); + + // From now we will be using specific values for the below filters for + // each build database query. Note that the toolchain is the only + // filter left in bld_params. + // + bld_params.name ().clear (); + bld_params.version ().clear (); + bld_params.target ().clear (); + bld_params.target_config ().clear (); + bld_params.package_config ().clear (); + // Prepare the build count prepared query. // // For each package-excluded configuration we will query the number of @@ -811,63 +873,82 @@ handle (request& rq, response& rs) using prep_bld_query = prepared_query<package_build_count>; package_id id; - string config; + target_triplet target; + string target_config_name; + string package_config_name; const auto& bid (bld_query::build::id); bld_query bq ( - equal<package_build_count> (bid.package, id) && - bid.configuration == bld_query::_ref (config) && + equal<package_build_count> (bid.package, id) && + bid.target == bld_query::_ref (target) && + bid.target_config_name == bld_query::_ref (target_config_name) && + bid.package_config_name == bld_query::_ref (package_config_name) && // Note that the query already constrains configurations via the - // configuration name. + // configuration name and target. // // Also note that while the query already constrains the tenant via // the build package id, we still need to pass the tenant not to // erroneously filter out the private tenants. // - build_query<package_build_count> (nullptr /* configs */, + build_query<package_build_count> (nullptr /* config_ids */, bld_params, - tn, - false /* archived */)); + tn)); prep_bld_query bld_prep_query ( build_db_->prepare_query<package_build_count> ( "mod-builds-build-count-query", bq)); - size_t nt (tc == "*" ? toolchains.size () : 1); + // Number of possible builds per package configuration. + // + size_t nt (th == "*" ? toolchains.size () : 1); // The number of packages can potentially be large, and we may // implement some caching in the future. However, the caching will not // be easy as the cached values depend on the filter form parameters. // query<buildable_package> q ( - package_query<buildable_package> ( - params, tn, false /* archived */)); + package_query<buildable_package> (params, tn)); for (auto& bp: build_db_->query<buildable_package> (q)) { - id = move (bp.id); + shared_ptr<build_package>& p (bp.package); - shared_ptr<build_package> p (build_db_->load<build_package> (id)); + id = p->id; - for (const auto& c: configs) + // Note: load the constrains section lazily. + // + for (const build_package_config& c: p->configs) { - if (exclude (p->builds, p->constraints, *c)) + // Filter by package config name. + // + if (pkg_cfg.empty () || path_match (c.name, pkg_cfg)) { - nmax -= nt; - - config = c->name; - ncur -= bld_prep_query.execute_value (); + for (const auto& tc: target_configs) + { + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); + + if (exclude (c, p->builds, p->constraints, *tc)) + { + target = tc->target; + target_config_name = tc->name; + package_config_name = c.name; + ncur -= bld_prep_query.execute_value (); + } + else + npos += nt; + } } } } - } - assert (nmax >= ncur); - count = nmax - ncur; - - t.commit (); + assert (npos >= ncur); + count = npos - ncur; + } + else + count = nullopt; // Unknown count. } // Print the filter form. @@ -881,9 +962,11 @@ handle (request& rq, response& rs) // 3: package tenant // 4: toolchain name // 5: toolchain version (descending) - // 6: configuration name + // 6: target + // 7: target configuration name + // 8: package configuration name // - // Prepare the build package prepared query. + // Prepare the build package query. // // Note that we can't skip the proper number of packages in the database // query for a page numbers greater than one. So we will query packages @@ -898,28 +981,14 @@ handle (request& rq, response& rs) // URL query parameter. Alternatively, we can invent the page number cap. // using pkg_query = query<buildable_package>; - using prep_pkg_query = prepared_query<buildable_package>; - - pkg_query pq ( - package_query<buildable_package> (params, tn, false /* archived */)); - // Specify the portion. Note that we will still be querying packages in - // chunks, not to hold locks for too long. For each package we will query - // its builds, so let's keep the portion small. - // - size_t offset (0); + pkg_query pq (package_query<buildable_package> (params, tn)); pq += "ORDER BY" + pkg_query::build_package::id.name + order_by_version_desc (pkg_query::build_package::id.version, false /* first */) + "," + - pkg_query::build_package::id.tenant + - "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"; - - connection_ptr conn (build_db_->connection ()); - - prep_pkg_query pkg_prep_query ( - conn->prepare_query<buildable_package> ("mod-builds-package-query", pq)); + pkg_query::build_package::id.tenant; // Prepare the build prepared query. // @@ -933,15 +1002,13 @@ handle (request& rq, response& rs) package_id id; - bld_query bq ( - equal<package_build> (bld_query::build::id.package, id) && + bld_query bq (equal<package_build> (bld_query::build::id.package, id) && - // Note that while the query already constrains the tenant via the build - // package id, we still need to pass the tenant not to erroneously - // filter out the private tenants. - // - build_query<package_build> ( - &conf_names, bld_params, tn, false /* archived */)); + // Note that while the query already constrains the tenant + // via the build package id, we still need to pass the + // tenant not to erroneously filter out the private tenants. + // + build_query<package_build> (&conf_ids, bld_params, tn)); prep_bld_query bld_prep_query ( conn->prepare_query<package_build> ("mod-builds-build-query", bq)); @@ -952,99 +1019,115 @@ handle (request& rq, response& rs) // Enclose the subsequent tables to be able to use nth-child CSS selector. // s << DIV; - while (print != 0) - { - transaction t (conn->begin ()); - // Query (and cache) buildable packages. - // - auto packages (pkg_prep_query.execute ()); + // Query (and cache) buildable packages. + // + auto packages (build_db_->query<buildable_package> (pq)); - if (packages.empty ()) - print = 0; - else + if (packages.empty ()) + print = 0; + else + { + // Iterate over packages and print unbuilt configurations. Skip the + // appropriate number of them first (for page number greater than one). + // + for (auto& bp: packages) { - offset += packages.size (); + shared_ptr<build_package>& p (bp.package); + + id = p->id; - // Iterate over packages and print unbuilt configurations. Skip the - // appropriate number of them first (for page number greater than one). + // Copy configuration/toolchain combinations for this package, + // skipping excluded configurations. // - for (auto& p: packages) - { - id = move (p.id); + set<config_toolchain> unbuilt_configs; - // Copy configuration/toolchain combinations for this package, - // skipping excluded configurations. + // Load the constrains section lazily. + // + for (const build_package_config& pc: p->configs) + { + // Filter by package config name. // - set<config_toolchain> unbuilt_configs; + if (pkg_cfg.empty () || path_match (pc.name, pkg_cfg)) { - shared_ptr<build_package> p (build_db_->load<build_package> (id)); - - for (const auto& ct: config_toolchains) + for (const target_config_toolchain& ct: config_toolchains) { - auto i (build_conf_map_->find (ct.configuration.c_str ())); - assert (i != build_conf_map_->end ()); - - if (!exclude (p->builds, p->constraints, *i->second)) - unbuilt_configs.insert (ct); + auto i ( + target_conf_map_->find ( + build_target_config_id {ct.target, ct.target_config})); + + assert (i != target_conf_map_->end ()); + + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); + + if (!exclude (pc, p->builds, p->constraints, *i->second)) + unbuilt_configs.insert ( + config_toolchain {ct.target, + ct.target_config, + pc.name, + ct.toolchain_name, + ct.toolchain_version}); } } + } - // Iterate through the package configuration builds and erase them - // from the unbuilt configurations set. - // - for (const auto& pb: bld_prep_query.execute ()) - { - const build& b (*pb.build); + // Iterate through the package configuration builds and erase them + // from the unbuilt configurations set. + // + for (const auto& pb: bld_prep_query.execute ()) + { + const build& b (*pb.build); - unbuilt_configs.erase ({ - b.id.configuration, b.toolchain_name, b.toolchain_version}); - } + unbuilt_configs.erase (config_toolchain {b.target, + b.target_config_name, + b.package_config_name, + b.toolchain_name, + b.toolchain_version}); + } - // Print unbuilt package configurations. - // - for (const auto& ct: unbuilt_configs) + // Print unbuilt package configurations. + // + for (const auto& ct: unbuilt_configs) + { + if (skip != 0) { - if (skip != 0) - { - --skip; - continue; - } - - auto i (build_conf_map_->find (ct.configuration.c_str ())); - assert (i != build_conf_map_->end ()); - - s << TABLE(CLASS="proplist build") - << TBODY - << TR_NAME (id.name, string (), root, id.tenant) - << TR_VERSION (id.name, p.version, root, id.tenant) - << TR_VALUE ("toolchain", - string (ct.toolchain_name) + '-' + - ct.toolchain_version.string ()) - << TR_VALUE ("config", ct.configuration) - << TR_VALUE ("target", i->second->target.string ()); - - // In the global view mode add the tenant builds link. Note that - // the global view (and the link) makes sense only in the - // multi-tenant mode. - // - if (!tn && !id.tenant.empty ()) - s << TR_TENANT (tenant_name, "builds", root, id.tenant); + --skip; + continue; + } - s << ~TBODY - << ~TABLE; + s << TABLE(CLASS="proplist build") + << TBODY + << TR_NAME (id.name, root, id.tenant) + << TR_VERSION (id.name, p->version, root, id.tenant) + << TR_VALUE ("toolchain", + string (ct.toolchain_name) + '-' + + ct.toolchain_version.string ()) + << TR_VALUE ("target", ct.target.string ()) + << TR_VALUE ("tgt config", ct.target_config) + << TR_VALUE ("pkg config", ct.package_config); + + // In the global view mode add the tenant builds link. Note that + // the global view (and the link) makes sense only in the + // multi-tenant mode. + // + if (!tn && !id.tenant.empty ()) + s << TR_TENANT (tenant_name, "builds", root, id.tenant); - if (--print == 0) // Bail out the configuration loop. - break; - } + s << ~TBODY + << ~TABLE; - if (print == 0) // Bail out the package loop. + if (--print == 0) // Bail out the configuration loop. break; } - } - t.commit (); + if (print == 0) // Bail out the package loop. + break; + } } + + t.commit (); + s << ~DIV; } @@ -1070,13 +1153,17 @@ handle (request& rq, response& rs) }; add_filter ("pv", params.version ()); - add_filter ("tc", params.toolchain (), "*"); - add_filter ("cf", params.configuration ()); - add_filter ("mn", params.machine ()); - add_filter ("tg", params.target ()); + add_filter ("th", params.toolchain (), "*"); + add_filter ("tg", tgt); + add_filter ("tc", tgt_cfg); + add_filter ("pc", pkg_cfg); add_filter ("rs", params.result (), "*"); - s << DIV_PAGER (page, count, page_configs, options_->build_pages (), u) + s << DIV_PAGER (page, + count ? *count : 0, + page_configs, + options_->build_pages (), + u) << ~DIV << ~BODY << ~HTML; |