diff options
Diffstat (limited to 'clean')
| -rw-r--r-- | clean/buildfile | 1 | ||||
| -rw-r--r-- | clean/clean.cli | 60 | ||||
| -rw-r--r-- | clean/clean.cxx | 573 | ||||
| -rw-r--r-- | clean/options-types.hxx | 18 | ||||
| -rw-r--r-- | clean/types-parsers.cxx | 60 | ||||
| -rw-r--r-- | clean/types-parsers.hxx | 28 |
6 files changed, 439 insertions, 301 deletions
diff --git a/clean/buildfile b/clean/buildfile index 31a03df..d46a874 100644 --- a/clean/buildfile +++ b/clean/buildfile @@ -20,7 +20,6 @@ if $cli.configured cli.options += -I $src_root --include-with-brackets --include-prefix clean \ --guard-prefix CLEAN --generate-specifier --page-usage print_ --ansi-color \ ---cxx-prologue "#include <clean/types-parsers.hxx>" \ --long-usage # Include the generated cli files into the distribution and don't remove diff --git a/clean/clean.cli b/clean/clean.cli index 01bc57c..0e36386 100644 --- a/clean/clean.cli +++ b/clean/clean.cli @@ -6,28 +6,41 @@ include <vector>; include <string>; include <cstdint>; // uint16_t -include <clean/options-types.hxx>; - "\section=1" "\name=brep-clean" -"\summary=clean brep build database" +"\summary=clean brep databases" { - "<options> <buildtab>", + "<options> <buildtab> <timeout> <name> <hours>", "\h|SYNOPSIS| \c{\b{brep-clean --help}\n \b{brep-clean --version}\n - \b{brep-clean} [<options>] <buildtab>} + \b{brep-clean} [<options>] builds <buildtab> [<timeout>...]\n + \b{brep-clean} [<options>] tenants <timeout>} \h|DESCRIPTION| \cb{brep-clean} deletes expired package builds from the brep \cb{build} - database. The build is considered expired if the package version is not - in the \cb{package} database, or the configuration is not listed in the - <buildtab> file, or the timestamp is older than the one specified for - this build's toolchain (see \cb{--stale-timeout}). + database or deletes/archives tenants from the brep \cb{package} database. + + The first form considers a build as expired if the corresponding package + version is not in the \cb{package} database, or the configuration is not + listed in the <buildtab> file, or its age is older than the specified + timeout for this build toolchain. + + Build <timeout>, if specified, should have the \c{[<name>=]<hours>} form. + Specify zero for <hours> to make builds for a toolchain to never expire. + Omit <name> (including \cb{=}) to specify the default timeout. It will + apply to all the toolchains that don't have a toolchain-specific timeout. + + The second form considers a tenant as expired if its age is older than the + specified <timeout>. + + If the \cb{--archive} option is specified, then the tenant is archived + rather than deleted. In this state the tenant packages (and their builds) + are still visible in \cb{brep} but are not (re-)built by build bots. Note that \cb{brep-clean} expects the \cb{build} and \cb{package} database schemas to have already been created using \l{brep-migrate(1)}." @@ -37,49 +50,44 @@ class options { "\h|OPTIONS|" - brep::toolchain_timeouts --stale-timeout + bool --archive { - "[<name>=]<days>", - "Number of days to wait before considering builds for the named toolchain - as stale. Specify zero for <days> to make builds for a toolchain never - expire. Omit <name> (including \cb{=}) to specify the default timeout. - It will apply to all the toolchains that don't have a timeout specified - explicitly. If unspecified, the default timeout is zero (never expire)." + "Archive old tenants." } std::string --db-user { "<user>", - "Build database user name. If not specified, then operating system (login) - name is used." + "Database user name. If not specified, then operating system (login) name + is used." } std::string --db-password { "<pass>", - "Build database password. If not specified, then login without password is + "Database password. If not specified, then login without password is expected to work." } - std::string --db-name = "brep_build" + std::string --db-name { "<name>", - "Build database name. If not specified, then \cb{brep_build} is used by - default." + "Database name. If not specified, then \cb{brep_build} is used for the + first form and \cb{brep_package} for the second." } std::string --db-host { "<host>", - "Build database host name, address, or socket. If not specified, then - connect to \cb{localhost} using the operating system-default mechanism - (Unix-domain socket, etc)." + "Database host name, address, or socket. If not specified, then connect to + \cb{localhost} using the operating system-default mechanism (Unix-domain + socket, etc)." } std::uint16_t --db-port = 0 { "<port>", - "Build database port number. If not specified, the default port is used." + "Database port number. If not specified, the default port is used." } std::string --pager // String to allow empty value. diff --git a/clean/clean.cxx b/clean/clean.cxx index 102b165..deba1db 100644 --- a/clean/clean.cxx +++ b/clean/clean.cxx @@ -2,7 +2,9 @@ // copyright : Copyright (c) 2014-2018 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file +#include <map> #include <set> +#include <chrono> #include <iostream> #include <odb/database.hxx> @@ -17,6 +19,8 @@ #include <libbrep/build.hxx> #include <libbrep/build-odb.hxx> +#include <libbrep/package.hxx> +#include <libbrep/package-odb.hxx> #include <libbrep/build-package.hxx> #include <libbrep/build-package-odb.hxx> #include <libbrep/database-lock.hxx> @@ -25,226 +29,459 @@ using namespace std; using namespace bbot; -using namespace brep; using namespace odb::core; -// Operation failed, diagnostics has already been issued. -// -struct failed {}; +namespace brep +{ + // Operation failed, diagnostics has already been issued. + // + struct failed {}; -static const char* help_info ( - " info: run 'brep-clean --help' for more information"); + static const char* help_info ( + " info: run 'brep-clean --help' for more information"); -int -main (int argc, char* argv[]) -try -{ - cli::argv_scanner scan (argc, argv, true); - options ops (scan); + static int + clean_builds (const options&, cli::argv_scanner&, odb::pgsql::database&); - // Version. - // - if (ops.version ()) + static int + clean_tenants (const options&, cli::argv_scanner&, odb::pgsql::database&); + + static int + main (int argc, char* argv[]) + try { - cout << "brep-clean " << BREP_VERSION_ID << endl - << "libbrep " << LIBBREP_VERSION_ID << endl - << "libbbot " << LIBBBOT_VERSION_ID << endl - << "libbpkg " << LIBBPKG_VERSION_ID << endl - << "libbutl " << LIBBUTL_VERSION_ID << endl - << "Copyright (c) 2014-2018 Code Synthesis Ltd" << endl - << "This is free software released under the MIT license." << endl; + cli::argv_scanner scan (argc, argv, true); + options ops (scan); - return 0; - } + // Version. + // + if (ops.version ()) + { + cout << "brep-clean " << BREP_VERSION_ID << endl + << "libbrep " << LIBBREP_VERSION_ID << endl + << "libbbot " << LIBBBOT_VERSION_ID << endl + << "libbpkg " << LIBBPKG_VERSION_ID << endl + << "libbutl " << LIBBUTL_VERSION_ID << endl + << "Copyright (c) 2014-2018 Code Synthesis Ltd" << endl + << "This is free software released under the MIT license." << endl; + + return 0; + } - // Help. - // - if (ops.help ()) - { - butl::pager p ("brep-clean help", - false, - ops.pager_specified () ? &ops.pager () : nullptr, - &ops.pager_option ()); + // Help. + // + if (ops.help ()) + { + butl::pager p ("brep-clean help", + false, + ops.pager_specified () ? &ops.pager () : nullptr, + &ops.pager_option ()); + + print_usage (p.stream ()); - print_usage (p.stream ()); + // If the pager failed, assume it has issued some diagnostics. + // + return p.wait () ? 0 : 1; + } - // If the pager failed, assume it has issued some diagnostics. + // Detect the mode. // - return p.wait () ? 0 : 1; - } + if (!scan.more ()) + { + cerr << "error: 'builds' or 'tenants' is expected" << endl + << help_info << endl; + return 1; + } - const toolchain_timeouts& timeouts (ops.stale_timeout ()); + const string mode (scan.next ()); - auto i (timeouts.find (string ())); - timestamp default_timeout (i != timeouts.end () - ? i->second - : timestamp_nonexistent); + if (mode != "builds" && mode != "tenants") + throw cli::unknown_argument (mode); - // Load configurations names. - // - if (!scan.more ()) - { - cerr << "error: configuration file expected" << endl - << help_info << endl; - return 1; - } + const string db_schema (mode == "builds" ? "build" : "package"); - set<string> configs; - for (auto& c: parse_buildtab (path (scan.next ()))) - configs.emplace (move (c.name)); + const string& db_name (!ops.db_name ().empty () + ? ops.db_name () + : "brep_" + db_schema); - if (scan.more ()) - { - cerr << "error: unexpected argument encountered" << endl - << help_info << endl; - return 1; - } + odb::pgsql::database db ( + ops.db_user (), + ops.db_password (), + db_name, + ops.db_host (), + ops.db_port (), + "options='-c default_transaction_isolation=serializable'"); - odb::pgsql::database build_db ( - ops.db_user (), - ops.db_password (), - ops.db_name (), - ops.db_host (), - ops.db_port (), - "options='-c default_transaction_isolation=serializable'"); + // Prevent several brep-clean/migrate instances from updating build + // database simultaneously. + // + database_lock l (db); - // Prevent several brep-clean/migrate instances from updating build database - // simultaneously. - // - database_lock l (build_db); + // Check that the database schema matches the current one. + // + if (schema_catalog::current_version (db, db_schema) != + db.schema_version (db_schema)) + { + cerr << "error: " << db_schema << " database schema differs from the " + << "current one" << endl + << " info: use brep-migrate to migrate the database" << endl; + return 1; + } - // Check that the build database schema matches the current one. + return mode == "builds" + ? clean_builds (ops, scan, db) + : clean_tenants (ops, scan, db); + } + catch (const database_locked&) + { + cerr << "brep-clean or brep-migrate is running" << endl; + return 2; + } + catch (const recoverable& e) + { + cerr << "recoverable database error: " << e << endl; + return 3; + } + catch (const cli::exception& e) + { + cerr << "error: " << e << endl << help_info << endl; + return 1; + } + catch (const failed&) + { + return 1; // Diagnostics has already been issued. + } + // Fully qualified to avoid ambiguity with odb exception. // - const string bs ("build"); - if (schema_catalog::current_version (build_db, bs) != - build_db.schema_version (bs)) + catch (const std::exception& e) { - cerr << "error: build database schema differs from the current one" - << endl << " info: use brep-migrate to migrate the database" << endl; + cerr << "error: " << e << endl; return 1; } - // Prepare the build prepared query. + // Convert timeout duration into the time point. Return + // timestamp_nonexistent (never expire) for zero argument. Return nullopt if + // the argument is invalid. // - // Query package builds in chunks in order not to hold locks for too long. - // Sort the result by package version to minimize number of queries to the - // package database. - // - using bld_query = query<build>; - using prep_bld_query = prepared_query<build>; + static optional<timestamp> + timeout (const string& tm) + { + char* e (nullptr); + uint64_t t (strtoull (tm.c_str (), &e, 10)); - size_t offset (0); - bld_query bq ("ORDER BY" + - bld_query::id.package.tenant + "," + - bld_query::id.package.name + - order_by_version_desc (bld_query::id.package.version, false) + - "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); + if (*e != '\0' || tm.empty ()) + return nullopt; - connection_ptr conn (build_db.connection ()); + if (t == 0) + return timestamp_nonexistent; - prep_bld_query bld_prep_query ( - conn->prepare_query<build> ("build-query", bq)); + return system_clock::now () - chrono::hours (t); + } - // Prepare the package version query. - // - // Query buildable packages every time the new package name is encountered - // during iterating over the package builds. Such a query will be made once - // per package name due to the builds query sorting criteria (see above). - // - using pkg_query = query<buildable_package>; - using prep_pkg_query = prepared_query<buildable_package>; + static int + clean_builds (const options& ops, + cli::argv_scanner& scan, + odb::pgsql::database& db) + { + // Load configurations names. + // + if (!scan.more ()) + { + cerr << "error: configuration file expected" << endl + << help_info << endl; + return 1; + } - package_name pkg_name; - set<version> package_versions; + path cp; - pkg_query pq ( - pkg_query::build_package::id.name == pkg_query::_ref (pkg_name)); + try + { + cp = path (scan.next ()); + } + catch (const invalid_path& e) + { + cerr << "error: configuration file expected instead of '" << e.path + << "'" << endl + << help_info << endl; + return 1; + } - prep_pkg_query pkg_prep_query ( - conn->prepare_query<buildable_package> ("package-query", pq)); + set<string> configs; - for (bool ne (true); ne; ) - { - transaction t (conn->begin ()); + try + { + for (auto& c: parse_buildtab (cp)) + configs.emplace (move (c.name)); + } + catch (const io_error& e) + { + cerr << "error: unable to read '" << cp << "': " << e << endl; + return 1; + } - // Query builds. + // Parse timestamps. // - auto builds (bld_prep_query.execute ()); + map<string, timestamp> timeouts; // Toolchain timeouts. + timestamp default_timeout; // timestamp_nonexistent - if ((ne = !builds.empty ())) + while (scan.more ()) { - for (const auto& b: builds) + string a (scan.next ()); + + string tc; + optional<timestamp> to; + + size_t p (a.find ('=')); + + if (p == string::npos) + to = timeout (a); + else if (p > 0) // Note: toolchain name can't be empty. { - auto i (timeouts.find (b.toolchain_name)); + tc = string (a, 0, p); + to = timeout (string (a, p + 1)); + } - timestamp et (i != timeouts.end () - ? i->second - : default_timeout); + // Note that the default timeout can't be zero. + // + if (!to || (*to == timestamp_nonexistent && tc.empty ())) + { + cerr << "error: timeout expected instead of '" << a << "'" << endl + << help_info << endl; + return 1; + } - bool cleanup ( - // Check that the build is not stale. - // - b.timestamp <= et || + if (tc.empty ()) + default_timeout = *to; - // Check that the build configuration is still present. - // - // Note that we unable to detect configuration changes and rely on - // periodic rebuilds to take care of that. - // - configs.find (b.configuration) == configs.end ()); + timeouts[move (tc)] = move (*to); + } - // Check that the build package still exists. - // - if (!cleanup) + // Prepare the build prepared query. + // + // Query package builds in chunks in order not to hold locks for too long. + // Sort the result by package version to minimize number of queries to the + // package database. + // + using bld_query = query<build>; + using prep_bld_query = prepared_query<build>; + + size_t offset (0); + bld_query bq ("ORDER BY" + + bld_query::id.package.tenant + "," + + bld_query::id.package.name + + order_by_version_desc (bld_query::id.package.version, + false) + + "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); + + connection_ptr conn (db.connection ()); + + prep_bld_query bld_prep_query ( + conn->prepare_query<build> ("build-query", bq)); + + // Prepare the package version query. + // + // Query buildable packages every time the new tenant or package name is + // encountered during iterating over the package builds. Such a query will + // be made once per tenant package name due to the builds query sorting + // criteria (see above). + // + using pkg_query = query<buildable_package>; + using prep_pkg_query = prepared_query<buildable_package>; + + string tnt; + package_name pkg_name; + set<version> package_versions; + + pkg_query pq ( + pkg_query::build_package::id.tenant == pkg_query::_ref (tnt) && + pkg_query::build_package::id.name == pkg_query::_ref (pkg_name)); + + prep_pkg_query pkg_prep_query ( + conn->prepare_query<buildable_package> ("package-query", pq)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + // Query builds. + // + auto builds (bld_prep_query.execute ()); + + if ((ne = !builds.empty ())) + { + for (const auto& b: builds) { - if (pkg_name != b.package_name) - { - pkg_name = b.package_name; - package_versions.clear (); + auto i (timeouts.find (b.toolchain_name)); - for (auto& p: pkg_prep_query.execute ()) - package_versions.emplace (move (p.version)); + timestamp et (i != timeouts.end () + ? i->second + : default_timeout); + + bool cleanup ( + // Check that the build is not stale. + // + b.timestamp <= et || + + // Check that the build configuration is still present. + // + // Note that we unable to detect configuration changes and rely on + // periodic rebuilds to take care of that. + // + configs.find (b.configuration) == configs.end ()); + + // Check that the build package still exists. + // + if (!cleanup) + { + if (tnt != b.tenant || pkg_name != b.package_name) + { + tnt = b.tenant; + pkg_name = b.package_name; + package_versions.clear (); + + for (auto& p: pkg_prep_query.execute ()) + package_versions.emplace (move (p.version)); + } + + cleanup = package_versions.find (b.package_version) == + package_versions.end (); } - cleanup = package_versions.find (b.package_version) == - package_versions.end (); + if (cleanup) + db.erase (b); + else + ++offset; } - - if (cleanup) - build_db.erase (b); - else - ++offset; } + + t.commit (); } - t.commit (); + return 0; } - return 0; -} -catch (const database_locked&) -{ - cerr << "brep-clean or brep-migrate is running" << endl; - return 2; -} -catch (const recoverable& e) -{ - cerr << "recoverable database error: " << e << endl; - return 3; -} -catch (const cli::exception& e) -{ - cerr << "error: " << e << endl << help_info << endl; - return 1; -} -catch (const failed&) -{ - return 1; // Diagnostics has already been issued. + static int + clean_tenants (const options& ops, + cli::argv_scanner& scan, + odb::pgsql::database& db) + { + if (!scan.more ()) + { + cerr << "error: timeout expected" << endl + << help_info << endl; + return 1; + } + + string a (scan.next ()); + optional<timestamp> to (timeout (a)); + + // Note that the timeout can't be zero. + // + if (!to || *to == timestamp_nonexistent) + { + cerr << "error: timeout expected instead of '" << a << "'" << endl + << help_info << endl; + return 1; + } + + if (scan.more ()) + { + cerr << "error: unexpected argument encountered" << endl + << help_info << endl; + return 1; + } + + uint64_t ns ( + chrono::duration_cast<chrono::nanoseconds> ( + to->time_since_epoch ()).count ()); + + // Query tenants in chunks in order not to hold locks for too long. + // + connection_ptr conn (db.connection ()); + + // Archive (rather then delete) old tenants, if requested. + // + if (ops.archive ()) + { + using query = query<tenant>; + using pquery = prepared_query<tenant>; + + query q ((query::creation_timestamp < ns && !query::archived) + + "LIMIT 100"); + + pquery pq (conn->prepare_query<tenant> ("tenant-query", q)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + auto tenants (pq.execute ()); + if ((ne = !tenants.empty ())) + { + for (auto& t: tenants) + { + t.archived = true; + db.update (t); + } + } + + t.commit (); + } + + return 0; + } + + // Delete old tenants. + // + // Note that we don't delete dangling builds for the deleted packages. + // Doing so would require to operate on two databases, complicating the + // code and the utility interface. Note that dangling builds are never + // considered in the web interface and are always deleted with the + // 'brep-clean builds' command. + // + using query = query<tenant_id>; + using pquery = prepared_query<tenant_id>; + + query q ((query::creation_timestamp < ns) + "LIMIT 100"); + pquery pq (conn->prepare_query<tenant_id> ("tenant-id-query", q)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + auto tenant_ids (pq.execute ()); + if ((ne = !tenant_ids.empty ())) + { + // Cache tenant ids and erase packages, repositories, and tenants at + // once. + // + strings tids; + tids.reserve (tenant_ids.size ()); + + for (auto& tid: tenant_ids) + tids.push_back (move (tid.value)); + + using odb::query; + + db.erase_query<package> ( + query<package>::id.tenant.in_range (tids.begin (), tids.end ())); + + db.erase_query<repository> ( + query<repository>::id.tenant.in_range (tids.begin (), tids.end ())); + + db.erase_query<tenant> ( + query<tenant>::id.in_range (tids.begin (), tids.end ())); + } + + t.commit (); + } + + return 0; + } } -// Fully qualified to avoid ambiguity with odb exception. -// -catch (const std::exception& e) + +int +main (int argc, char* argv[]) { - cerr << "error: " << e << endl; - return 1; + return brep::main (argc, argv); } diff --git a/clean/options-types.hxx b/clean/options-types.hxx deleted file mode 100644 index 7190396..0000000 --- a/clean/options-types.hxx +++ /dev/null @@ -1,18 +0,0 @@ -// file : clean/options-types.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef CLEAN_OPTIONS_TYPES_HXX -#define CLEAN_OPTIONS_TYPES_HXX - -#include <map> - -#include <libbrep/types.hxx> -#include <libbrep/utility.hxx> - -namespace brep -{ - struct toolchain_timeouts: std::map<string, timestamp> {}; -} - -#endif // CLEAN_OPTIONS_TYPES_HXX diff --git a/clean/types-parsers.cxx b/clean/types-parsers.cxx deleted file mode 100644 index 31f3a8d..0000000 --- a/clean/types-parsers.cxx +++ /dev/null @@ -1,60 +0,0 @@ -// file : clean/types-parsers.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include <chrono> -#include <string> // strtoull() - -#include <clean/types-parsers.hxx> - -#include <clean/options-types.hxx> -#include <clean/clean-options.hxx> // cli namespace - -using namespace std; -using namespace brep; - -namespace cli -{ - void parser<toolchain_timeouts>:: - parse (toolchain_timeouts& x, bool& xs, scanner& s) - { - const char* o (s.next ()); - - if (!s.more ()) - throw missing_value (o); - - string ov (s.next ()); - size_t p (ov.find ('=')); - - timestamp now (system_clock::now ()); - - // Convert timeout duration into the time point. - // - auto timeout = [o, &ov, &now] (const string& tm) -> timestamp - { - char* e (nullptr); - uint64_t t (strtoull (tm.c_str (), &e, 10)); - - if (*e != '\0' || tm.empty ()) - throw invalid_value (o, ov); - - if (t == 0) - return timestamp_nonexistent; - - return now - chrono::duration<uint64_t, ratio<86400>> (t); - }; - - if (p == string::npos) - x[string ()] = timeout (ov); // Default timeout. - else - { - string k (ov, 0, p); - if (k.empty ()) - throw invalid_value (o, ov); - - x[k] = timeout (string (ov, p + 1)); - } - - xs = true; - } -} diff --git a/clean/types-parsers.hxx b/clean/types-parsers.hxx deleted file mode 100644 index fe7c77c..0000000 --- a/clean/types-parsers.hxx +++ /dev/null @@ -1,28 +0,0 @@ -// file : clean/types-parsers.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -// CLI parsers, included into the generated source files. -// - -#ifndef CLEAN_TYPES_PARSERS_HXX -#define CLEAN_TYPES_PARSERS_HXX - -#include <clean/options-types.hxx> - -namespace cli -{ - class scanner; - - template <typename T> - struct parser; - - template <> - struct parser<brep::toolchain_timeouts> - { - static void - parse (brep::toolchain_timeouts&, bool&, scanner&); - }; -} - -#endif // CLEAN_TYPES_PARSERS_HXX |