diff options
Diffstat (limited to 'mod/mod-builds.cxx')
| -rw-r--r-- | mod/mod-builds.cxx | 472 |
1 files changed, 242 insertions, 230 deletions
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index ff27c65..30562f3 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -138,8 +138,7 @@ template <typename T> static inline query<T> build_query (const brep::vector<brep::build_target_config_id>* config_ids, const brep::params::builds& params, - const brep::optional<brep::string>& tenant, - const brep::optional<bool>& archived) + const brep::optional<brep::string>& tenant) { using namespace brep; using query = query<T>; @@ -150,9 +149,6 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids, query q (tenant ? pid.tenant == *tenant : !qt::private_); - if (archived) - q = q && qt::archived == *archived; - if (config_ids != nullptr) { query sq (false); @@ -229,13 +225,19 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids, // Build result. // const string& rs (params.result ()); + bool add_state (true); if (rs != "*") { if (rs == "pending") + { q = q && qb::force != "unforced"; + } else if (rs == "building") + { q = q && qb::state == "building"; + add_state = false; + } else { query sq (qb::status == rs); @@ -263,8 +265,12 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids, // well (rebuild). // q = q && qb::state == "built" && sq; + add_state = false; } } + + if (add_state) + q = q && qb::state != "queued"; } catch (const invalid_argument&) { @@ -279,8 +285,7 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids, template <typename T> static inline query<T> package_query (const brep::params::builds& params, - const brep::optional<brep::string>& tenant, - const brep::optional<bool>& archived) + const brep::optional<brep::string>& tenant) { using namespace brep; using query = query<T>; @@ -289,9 +294,6 @@ package_query (const brep::params::builds& params, query q (tenant ? qp::id.tenant == *tenant : !qt::private_); - if (archived) - q = q && qt::archived == *archived; - // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no packages match such a query. // @@ -391,8 +393,11 @@ handle (request& rq, response& rs) if (!tenant.empty ()) tn = tenant; - // Return the list of distinct toolchain name/version pairs. The build db - // transaction must be started. + // Return the list of distinct toolchain name/version pairs. If no builds + // are present for the tenant, then fallback to the toolchain recorded in + // the tenant object, if present. + // + // Note: the build db transaction must be started. // using toolchains = vector<pair<string, version>>; @@ -408,11 +413,19 @@ handle (request& rq, response& rs) false /* first */))) r.emplace_back (move (t.name), move (t.version)); + if (r.empty ()) + { + shared_ptr<build_tenant> t (build_db_->find<build_tenant> (tenant)); + + if (t != nullptr && t->toolchain) + r.emplace_back (t->toolchain->name, t->toolchain->version); + } + return r; }; auto print_form = [&s, ¶ms, this] (const toolchains& toolchains, - size_t build_count) + optional<size_t> build_count) { // Print the package builds filter form on the first page only. // @@ -519,11 +532,11 @@ handle (request& rq, response& rs) for (const auto& c: *target_conf_map_) { - if (!exclude_hidden || belongs (*c.second, "all")) + if (!exclude_hidden || !belongs (*c.second, "hidden")) conf_ids.push_back (c.first); } - size_t count; + optional<size_t> count; size_t page (params.page ()); if (params.result () != "unbuilt") // Print package build configurations. @@ -541,34 +554,19 @@ handle (request& rq, response& rs) vector<package_build> builds; builds.reserve (page_configs); - // Prepare the package build prepared query. + // Prepare the package build query. // using query = query<package_build>; - using prep_query = prepared_query<package_build>; - query q (build_query<package_build> ( - &conf_ids, params, tn, nullopt /* archived */)); - - // Specify the portion. Note that we will be querying builds in chunks, - // not to hold locks for too long. - // - // Also note that for each build we also load the corresponding - // package. Nevertheless, we use a fairly large portion to speed-up the - // builds traversal but also cache the package objects (see below). - // - size_t offset (0); + query q (build_query<package_build> (&conf_ids, params, tn)); // Print package build configurations ordered by the timestamp (later goes // first). // - q += "ORDER BY" + query::build::timestamp + "DESC" + - "OFFSET" + query::_ref (offset) + "LIMIT 500"; + q += "ORDER BY" + query::build::timestamp + "DESC"; connection_ptr conn (build_db_->connection ()); - prep_query pq ( - conn->prepare_query<package_build> ("mod-builds-query", q)); - // Note that we can't skip the proper number of builds in the database // query for a page numbers greater than one. So we will query builds from // the very beginning and skip the appropriate number of them while @@ -584,101 +582,102 @@ handle (request& rq, response& rs) // session sn; - for (bool ne (true); ne; ) + transaction t (conn->begin ()); + + // For some reason PostgreSQL (as of 9.4) picks the nested loop join + // strategy for the below package_build query, which executes quite slow + // even for reasonably small number of builds. Thus, we just discourage + // PostgreSQL from using this strategy in the current transaction. + // + // @@ TMP Re-check for the later PostgreSQL versions if we can drop this + // hint. If drop, then also grep for other places where this hint + // is used. + // + conn->execute ("SET LOCAL enable_nestloop=off"); + + // Iterate over builds and cache build objects that should be printed. + // Skip the appropriate number of them (for page number greater than + // one). + // + for (auto& pb: build_db_->query<package_build> (q)) { - transaction t (conn->begin ()); + shared_ptr<build>& b (pb.build); - // Query package builds (and cache the result). - // - auto bs (pq.execute ()); + auto i ( + target_conf_map_->find ( + build_target_config_id {b->target, b->target_config_name})); - if ((ne = !bs.empty ())) - { - offset += bs.size (); + assert (i != target_conf_map_->end ()); - // Iterate over builds and cache build objects that should be printed. - // Skip the appropriate number of them (for page number greater than - // one). - // - for (auto& pb: bs) - { - shared_ptr<build>& b (pb.build); + // Match the target configuration against the package build + // configuration expressions/constraints. + // + shared_ptr<build_package> p ( + build_db_->load<build_package> (b->id.package)); - auto i (target_conf_map_->find ( - build_target_config_id {b->target, - b->target_config_name})); + const build_package_config* pc (find (b->package_config_name, + p->configs)); - assert (i != target_conf_map_->end ()); + // The package configuration should be present since the configurations + // set cannot change if the package version doesn't change. If that's + // not the case, then the database has probably been manually amended. + // In this case let's just skip such a build as if it excluded and log + // the warning. + // + if (pc == nullptr) + { + warn << "cannot find configuration '" << b->package_config_name + << "' for package " << p->id.name << '/' << p->version; - // Match the target configuration against the package build - // configuration expressions/constraints. - // - shared_ptr<build_package> p ( - build_db_->load<build_package> (b->id.package)); + continue; + } - const build_package_config* pc (find (b->package_config_name, - p->configs)); + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); - // The package configuration should be present since the - // configurations set cannot change if the package version doesn't - // change. If that's not the case, then the database has probably - // been manually amended. In this case let's just skip such a build - // as if it excluded and log the warning. + if (!exclude (*pc, p->builds, p->constraints, *i->second)) + { + if (skip != 0) + --skip; + else if (print != 0) + { + // As we query builds in multiple transactions we may see the same + // build multiple times. Let's skip the duplicates. Note: we don't + // increment the counter in this case. // - if (pc == nullptr) - { - warn << "cannot find configuration '" << b->package_config_name - << "' for package " << p->id.name << '/' << p->version; - + if (find_if (builds.begin (), builds.end (), + [&b] (const package_build& pb) + { + return b->id == pb.build->id; + }) != builds.end ()) continue; - } - if (!exclude (*pc, p->builds, p->constraints, *i->second)) + if (b->state == build_state::built) { - if (skip != 0) - --skip; - else if (print != 0) - { - // As we query builds in multiple transactions we may see the - // same build multiple times. Let's skip the duplicates. Note: - // we don't increment the counter in this case. - // - if (find_if (builds.begin (), builds.end (), - [&b] (const package_build& pb) - { - return b->id == pb.build->id; - }) != builds.end ()) - continue; - - if (b->state == build_state::built) - { - build_db_->load (*b, b->results_section); + build_db_->load (*b, b->results_section); - // Let's clear unneeded result logs for builds being cached. - // - for (operation_result& r: b->results) - r.log.clear (); - } + // Let's clear unneeded result logs for builds being cached. + // + for (operation_result& r: b->results) + r.log.clear (); + } - builds.push_back (move (pb)); + builds.push_back (move (pb)); - --print; - } - - ++count; - } + --print; } - } - // - // Print the filter form after the build count is calculated. Note: - // query_toolchains() must be called inside the build db transaction. - // - else - print_form (query_toolchains (), count); - t.commit (); + ++(*count); + } } + // Print the filter form after the build count is calculated. Note: + // query_toolchains() must be called inside the build db transaction. + // + print_form (query_toolchains (), count); + + t.commit (); + // Finally, print the cached package build configurations. // timestamp now (system_clock::now ()); @@ -694,11 +693,16 @@ handle (request& rq, response& rs) "%Y-%m-%d %H:%M:%S %Z", true /* special */, true /* local */) + - " (" + butl::to_string (now - b.timestamp, false) + " ago)"); + " (" + butl::to_string (now - b.timestamp, false) + " ago"); + + if (pb.archived) + ts += ", archived"; + + ts += ')'; s << TABLE(CLASS="proplist build") << TBODY - << TR_NAME (b.package_name, string (), root, b.tenant) + << TR_NAME (b.package_name, root, b.tenant) << TR_VERSION (b.package_name, b.package_version, root, b.tenant) << TR_VALUE ("toolchain", b.toolchain_name + '-' + @@ -752,9 +756,27 @@ handle (request& rq, response& rs) const bpkg::version& toolchain_version; }; + // Cache the build package objects that would otherwise be loaded twice: + // first time during calculating the builds count and then during printing + // the builds. Note that the build package is a subset of the package + // object and normally has a small memory footprint. + // + // @@ TMP It feels that we can try to combine the mentioned steps and + // improve the performance a bit. We won't need the session in this + // case. + // + session sn; + + connection_ptr conn (build_db_->connection ()); + transaction t (conn->begin ()); + + // Discourage PostgreSQL from using the nested loop join strategy in the + // current transaction (see above for details). + // + conn->execute ("SET LOCAL enable_nestloop=off"); + vector<target_config_toolchain> config_toolchains; { - transaction t (build_db_->begin ()); toolchains = query_toolchains (); string th_name; @@ -797,7 +819,7 @@ handle (request& rq, response& rs) // (tgt.empty () || path_match (c.target.string (), tgt)) && - (!exclude_hidden || belongs (c, "all"))) // Filter hidden. + (!exclude_hidden || !belongs (c, "hidden"))) // Filter hidden. { target_configs.push_back (&c); @@ -829,9 +851,8 @@ handle (request& rq, response& rs) // size_t npos (0); - size_t ncur = build_db_->query_value<package_build_count> ( - build_query<package_build_count> ( - &conf_ids, bld_params, tn, false /* archived */)); + size_t ncur (build_db_->query_value<package_build_count> ( + build_query<package_build_count> (&conf_ids, bld_params, tn))); // From now we will be using specific values for the below filters for // each build database query. Note that the toolchain is the only @@ -873,8 +894,7 @@ handle (request& rq, response& rs) // build_query<package_build_count> (nullptr /* config_ids */, bld_params, - tn, - false /* archived */)); + tn)); prep_bld_query bld_prep_query ( build_db_->prepare_query<package_build_count> ( @@ -889,14 +909,16 @@ handle (request& rq, response& rs) // be easy as the cached values depend on the filter form parameters. // query<buildable_package> q ( - package_query<buildable_package> (params, tn, false /* archived */)); + package_query<buildable_package> (params, tn)); for (auto& bp: build_db_->query<buildable_package> (q)) { - id = move (bp.id); + shared_ptr<build_package>& p (bp.package); - shared_ptr<build_package> p (build_db_->load<build_package> (id)); + id = p->id; + // Note: load the constrains section lazily. + // for (const build_package_config& c: p->configs) { // Filter by package config name. @@ -905,6 +927,9 @@ handle (request& rq, response& rs) { for (const auto& tc: target_configs) { + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); + if (exclude (c, p->builds, p->constraints, *tc)) { target = tc->target; @@ -923,9 +948,7 @@ handle (request& rq, response& rs) count = npos - ncur; } else - count = 0; - - t.commit (); + count = nullopt; // Unknown count. } // Print the filter form. @@ -943,7 +966,7 @@ handle (request& rq, response& rs) // 7: target configuration name // 8: package configuration name // - // Prepare the build package prepared query. + // Prepare the build package query. // // Note that we can't skip the proper number of packages in the database // query for a page numbers greater than one. So we will query packages @@ -958,28 +981,14 @@ handle (request& rq, response& rs) // URL query parameter. Alternatively, we can invent the page number cap. // using pkg_query = query<buildable_package>; - using prep_pkg_query = prepared_query<buildable_package>; - pkg_query pq ( - package_query<buildable_package> (params, tn, false /* archived */)); - - // Specify the portion. Note that we will still be querying packages in - // chunks, not to hold locks for too long. For each package we will query - // its builds, so let's keep the portion small. - // - size_t offset (0); + pkg_query pq (package_query<buildable_package> (params, tn)); pq += "ORDER BY" + pkg_query::build_package::id.name + order_by_version_desc (pkg_query::build_package::id.version, false /* first */) + "," + - pkg_query::build_package::id.tenant + - "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"; - - connection_ptr conn (build_db_->connection ()); - - prep_pkg_query pkg_prep_query ( - conn->prepare_query<buildable_package> ("mod-builds-package-query", pq)); + pkg_query::build_package::id.tenant; // Prepare the build prepared query. // @@ -999,8 +1008,7 @@ handle (request& rq, response& rs) // via the build package id, we still need to pass the // tenant not to erroneously filter out the private tenants. // - build_query<package_build> ( - &conf_ids, bld_params, tn, false /* archived */)); + build_query<package_build> (&conf_ids, bld_params, tn)); prep_bld_query bld_prep_query ( conn->prepare_query<package_build> ("mod-builds-build-query", bq)); @@ -1011,115 +1019,115 @@ handle (request& rq, response& rs) // Enclose the subsequent tables to be able to use nth-child CSS selector. // s << DIV; - while (print != 0) - { - transaction t (conn->begin ()); - // Query (and cache) buildable packages. - // - auto packages (pkg_prep_query.execute ()); + // Query (and cache) buildable packages. + // + auto packages (build_db_->query<buildable_package> (pq)); - if (packages.empty ()) - print = 0; - else + if (packages.empty ()) + print = 0; + else + { + // Iterate over packages and print unbuilt configurations. Skip the + // appropriate number of them first (for page number greater than one). + // + for (auto& bp: packages) { - offset += packages.size (); + shared_ptr<build_package>& p (bp.package); - // Iterate over packages and print unbuilt configurations. Skip the - // appropriate number of them first (for page number greater than one). - // - for (auto& p: packages) - { - id = move (p.id); + id = p->id; - shared_ptr<build_package> bp (build_db_->load<build_package> (id)); + // Copy configuration/toolchain combinations for this package, + // skipping excluded configurations. + // + set<config_toolchain> unbuilt_configs; - // Copy configuration/toolchain combinations for this package, - // skipping excluded configurations. + // Load the constrains section lazily. + // + for (const build_package_config& pc: p->configs) + { + // Filter by package config name. // - set<config_toolchain> unbuilt_configs; - - for (const build_package_config& pc: bp->configs) + if (pkg_cfg.empty () || path_match (pc.name, pkg_cfg)) { - // Filter by package config name. - // - if (pkg_cfg.empty () || path_match (pc.name, pkg_cfg)) + for (const target_config_toolchain& ct: config_toolchains) { - for (const target_config_toolchain& ct: config_toolchains) - { - auto i ( - target_conf_map_->find ( - build_target_config_id {ct.target, ct.target_config})); - - assert (i != target_conf_map_->end ()); - - if (!exclude (pc, bp->builds, bp->constraints, *i->second)) - unbuilt_configs.insert ( - config_toolchain {ct.target, - ct.target_config, - pc.name, - ct.toolchain_name, - ct.toolchain_version}); - } + auto i ( + target_conf_map_->find ( + build_target_config_id {ct.target, ct.target_config})); + + assert (i != target_conf_map_->end ()); + + if (!p->constraints_section.loaded ()) + build_db_->load (*p, p->constraints_section); + + if (!exclude (pc, p->builds, p->constraints, *i->second)) + unbuilt_configs.insert ( + config_toolchain {ct.target, + ct.target_config, + pc.name, + ct.toolchain_name, + ct.toolchain_version}); } } + } - // Iterate through the package configuration builds and erase them - // from the unbuilt configurations set. - // - for (const auto& pb: bld_prep_query.execute ()) - { - const build& b (*pb.build); + // Iterate through the package configuration builds and erase them + // from the unbuilt configurations set. + // + for (const auto& pb: bld_prep_query.execute ()) + { + const build& b (*pb.build); - unbuilt_configs.erase (config_toolchain {b.target, - b.target_config_name, - b.package_config_name, - b.toolchain_name, - b.toolchain_version}); - } + unbuilt_configs.erase (config_toolchain {b.target, + b.target_config_name, + b.package_config_name, + b.toolchain_name, + b.toolchain_version}); + } - // Print unbuilt package configurations. - // - for (const auto& ct: unbuilt_configs) + // Print unbuilt package configurations. + // + for (const auto& ct: unbuilt_configs) + { + if (skip != 0) { - if (skip != 0) - { - --skip; - continue; - } - - s << TABLE(CLASS="proplist build") - << TBODY - << TR_NAME (id.name, string (), root, id.tenant) - << TR_VERSION (id.name, p.version, root, id.tenant) - << TR_VALUE ("toolchain", - string (ct.toolchain_name) + '-' + - ct.toolchain_version.string ()) - << TR_VALUE ("target", ct.target.string ()) - << TR_VALUE ("tgt config", ct.target_config) - << TR_VALUE ("pkg config", ct.package_config); - - // In the global view mode add the tenant builds link. Note that - // the global view (and the link) makes sense only in the - // multi-tenant mode. - // - if (!tn && !id.tenant.empty ()) - s << TR_TENANT (tenant_name, "builds", root, id.tenant); + --skip; + continue; + } - s << ~TBODY - << ~TABLE; + s << TABLE(CLASS="proplist build") + << TBODY + << TR_NAME (id.name, root, id.tenant) + << TR_VERSION (id.name, p->version, root, id.tenant) + << TR_VALUE ("toolchain", + string (ct.toolchain_name) + '-' + + ct.toolchain_version.string ()) + << TR_VALUE ("target", ct.target.string ()) + << TR_VALUE ("tgt config", ct.target_config) + << TR_VALUE ("pkg config", ct.package_config); + + // In the global view mode add the tenant builds link. Note that + // the global view (and the link) makes sense only in the + // multi-tenant mode. + // + if (!tn && !id.tenant.empty ()) + s << TR_TENANT (tenant_name, "builds", root, id.tenant); - if (--print == 0) // Bail out the configuration loop. - break; - } + s << ~TBODY + << ~TABLE; - if (print == 0) // Bail out the package loop. + if (--print == 0) // Bail out the configuration loop. break; } - } - t.commit (); + if (print == 0) // Bail out the package loop. + break; + } } + + t.commit (); + s << ~DIV; } @@ -1151,7 +1159,11 @@ handle (request& rq, response& rs) add_filter ("pc", pkg_cfg); add_filter ("rs", params.result (), "*"); - s << DIV_PAGER (page, count, page_configs, options_->build_pages (), u) + s << DIV_PAGER (page, + count ? *count : 0, + page_configs, + options_->build_pages (), + u) << ~DIV << ~BODY << ~HTML; |