From 873987793b05fc0d6e9908f5030b2bca145c4e6d Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Sun, 28 Oct 2018 01:01:53 +0300 Subject: Add tenant object --- clean/buildfile | 1 - clean/clean.cli | 60 ++--- clean/clean.cxx | 573 ++++++++++++++++++++++++++++++++++-------------- clean/options-types.hxx | 18 -- clean/types-parsers.cxx | 60 ----- clean/types-parsers.hxx | 28 --- 6 files changed, 439 insertions(+), 301 deletions(-) delete mode 100644 clean/options-types.hxx delete mode 100644 clean/types-parsers.cxx delete mode 100644 clean/types-parsers.hxx (limited to 'clean') diff --git a/clean/buildfile b/clean/buildfile index 31a03df..d46a874 100644 --- a/clean/buildfile +++ b/clean/buildfile @@ -20,7 +20,6 @@ if $cli.configured cli.options += -I $src_root --include-with-brackets --include-prefix clean \ --guard-prefix CLEAN --generate-specifier --page-usage print_ --ansi-color \ ---cxx-prologue "#include " \ --long-usage # Include the generated cli files into the distribution and don't remove diff --git a/clean/clean.cli b/clean/clean.cli index 01bc57c..0e36386 100644 --- a/clean/clean.cli +++ b/clean/clean.cli @@ -6,28 +6,41 @@ include ; include ; include ; // uint16_t -include ; - "\section=1" "\name=brep-clean" -"\summary=clean brep build database" +"\summary=clean brep databases" { - " ", + " ", "\h|SYNOPSIS| \c{\b{brep-clean --help}\n \b{brep-clean --version}\n - \b{brep-clean} [] } + \b{brep-clean} [] builds [...]\n + \b{brep-clean} [] tenants } \h|DESCRIPTION| \cb{brep-clean} deletes expired package builds from the brep \cb{build} - database. The build is considered expired if the package version is not - in the \cb{package} database, or the configuration is not listed in the - file, or the timestamp is older than the one specified for - this build's toolchain (see \cb{--stale-timeout}). + database or deletes/archives tenants from the brep \cb{package} database. + + The first form considers a build as expired if the corresponding package + version is not in the \cb{package} database, or the configuration is not + listed in the file, or its age is older than the specified + timeout for this build toolchain. + + Build , if specified, should have the \c{[=]} form. + Specify zero for to make builds for a toolchain to never expire. + Omit (including \cb{=}) to specify the default timeout. It will + apply to all the toolchains that don't have a toolchain-specific timeout. + + The second form considers a tenant as expired if its age is older than the + specified . + + If the \cb{--archive} option is specified, then the tenant is archived + rather than deleted. In this state the tenant packages (and their builds) + are still visible in \cb{brep} but are not (re-)built by build bots. Note that \cb{brep-clean} expects the \cb{build} and \cb{package} database schemas to have already been created using \l{brep-migrate(1)}." @@ -37,49 +50,44 @@ class options { "\h|OPTIONS|" - brep::toolchain_timeouts --stale-timeout + bool --archive { - "[=]", - "Number of days to wait before considering builds for the named toolchain - as stale. Specify zero for to make builds for a toolchain never - expire. Omit (including \cb{=}) to specify the default timeout. - It will apply to all the toolchains that don't have a timeout specified - explicitly. If unspecified, the default timeout is zero (never expire)." + "Archive old tenants." } std::string --db-user { "", - "Build database user name. If not specified, then operating system (login) - name is used." + "Database user name. If not specified, then operating system (login) name + is used." } std::string --db-password { "", - "Build database password. If not specified, then login without password is + "Database password. If not specified, then login without password is expected to work." } - std::string --db-name = "brep_build" + std::string --db-name { "", - "Build database name. If not specified, then \cb{brep_build} is used by - default." + "Database name. If not specified, then \cb{brep_build} is used for the + first form and \cb{brep_package} for the second." } std::string --db-host { "", - "Build database host name, address, or socket. If not specified, then - connect to \cb{localhost} using the operating system-default mechanism - (Unix-domain socket, etc)." + "Database host name, address, or socket. If not specified, then connect to + \cb{localhost} using the operating system-default mechanism (Unix-domain + socket, etc)." } std::uint16_t --db-port = 0 { "", - "Build database port number. If not specified, the default port is used." + "Database port number. If not specified, the default port is used." } std::string --pager // String to allow empty value. diff --git a/clean/clean.cxx b/clean/clean.cxx index 102b165..deba1db 100644 --- a/clean/clean.cxx +++ b/clean/clean.cxx @@ -2,7 +2,9 @@ // copyright : Copyright (c) 2014-2018 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file +#include #include +#include #include #include @@ -17,6 +19,8 @@ #include #include +#include +#include #include #include #include @@ -25,226 +29,459 @@ using namespace std; using namespace bbot; -using namespace brep; using namespace odb::core; -// Operation failed, diagnostics has already been issued. -// -struct failed {}; +namespace brep +{ + // Operation failed, diagnostics has already been issued. + // + struct failed {}; -static const char* help_info ( - " info: run 'brep-clean --help' for more information"); + static const char* help_info ( + " info: run 'brep-clean --help' for more information"); -int -main (int argc, char* argv[]) -try -{ - cli::argv_scanner scan (argc, argv, true); - options ops (scan); + static int + clean_builds (const options&, cli::argv_scanner&, odb::pgsql::database&); - // Version. - // - if (ops.version ()) + static int + clean_tenants (const options&, cli::argv_scanner&, odb::pgsql::database&); + + static int + main (int argc, char* argv[]) + try { - cout << "brep-clean " << BREP_VERSION_ID << endl - << "libbrep " << LIBBREP_VERSION_ID << endl - << "libbbot " << LIBBBOT_VERSION_ID << endl - << "libbpkg " << LIBBPKG_VERSION_ID << endl - << "libbutl " << LIBBUTL_VERSION_ID << endl - << "Copyright (c) 2014-2018 Code Synthesis Ltd" << endl - << "This is free software released under the MIT license." << endl; + cli::argv_scanner scan (argc, argv, true); + options ops (scan); - return 0; - } + // Version. + // + if (ops.version ()) + { + cout << "brep-clean " << BREP_VERSION_ID << endl + << "libbrep " << LIBBREP_VERSION_ID << endl + << "libbbot " << LIBBBOT_VERSION_ID << endl + << "libbpkg " << LIBBPKG_VERSION_ID << endl + << "libbutl " << LIBBUTL_VERSION_ID << endl + << "Copyright (c) 2014-2018 Code Synthesis Ltd" << endl + << "This is free software released under the MIT license." << endl; + + return 0; + } - // Help. - // - if (ops.help ()) - { - butl::pager p ("brep-clean help", - false, - ops.pager_specified () ? &ops.pager () : nullptr, - &ops.pager_option ()); + // Help. + // + if (ops.help ()) + { + butl::pager p ("brep-clean help", + false, + ops.pager_specified () ? &ops.pager () : nullptr, + &ops.pager_option ()); + + print_usage (p.stream ()); - print_usage (p.stream ()); + // If the pager failed, assume it has issued some diagnostics. + // + return p.wait () ? 0 : 1; + } - // If the pager failed, assume it has issued some diagnostics. + // Detect the mode. // - return p.wait () ? 0 : 1; - } + if (!scan.more ()) + { + cerr << "error: 'builds' or 'tenants' is expected" << endl + << help_info << endl; + return 1; + } - const toolchain_timeouts& timeouts (ops.stale_timeout ()); + const string mode (scan.next ()); - auto i (timeouts.find (string ())); - timestamp default_timeout (i != timeouts.end () - ? i->second - : timestamp_nonexistent); + if (mode != "builds" && mode != "tenants") + throw cli::unknown_argument (mode); - // Load configurations names. - // - if (!scan.more ()) - { - cerr << "error: configuration file expected" << endl - << help_info << endl; - return 1; - } + const string db_schema (mode == "builds" ? "build" : "package"); - set configs; - for (auto& c: parse_buildtab (path (scan.next ()))) - configs.emplace (move (c.name)); + const string& db_name (!ops.db_name ().empty () + ? ops.db_name () + : "brep_" + db_schema); - if (scan.more ()) - { - cerr << "error: unexpected argument encountered" << endl - << help_info << endl; - return 1; - } + odb::pgsql::database db ( + ops.db_user (), + ops.db_password (), + db_name, + ops.db_host (), + ops.db_port (), + "options='-c default_transaction_isolation=serializable'"); - odb::pgsql::database build_db ( - ops.db_user (), - ops.db_password (), - ops.db_name (), - ops.db_host (), - ops.db_port (), - "options='-c default_transaction_isolation=serializable'"); + // Prevent several brep-clean/migrate instances from updating build + // database simultaneously. + // + database_lock l (db); - // Prevent several brep-clean/migrate instances from updating build database - // simultaneously. - // - database_lock l (build_db); + // Check that the database schema matches the current one. + // + if (schema_catalog::current_version (db, db_schema) != + db.schema_version (db_schema)) + { + cerr << "error: " << db_schema << " database schema differs from the " + << "current one" << endl + << " info: use brep-migrate to migrate the database" << endl; + return 1; + } - // Check that the build database schema matches the current one. + return mode == "builds" + ? clean_builds (ops, scan, db) + : clean_tenants (ops, scan, db); + } + catch (const database_locked&) + { + cerr << "brep-clean or brep-migrate is running" << endl; + return 2; + } + catch (const recoverable& e) + { + cerr << "recoverable database error: " << e << endl; + return 3; + } + catch (const cli::exception& e) + { + cerr << "error: " << e << endl << help_info << endl; + return 1; + } + catch (const failed&) + { + return 1; // Diagnostics has already been issued. + } + // Fully qualified to avoid ambiguity with odb exception. // - const string bs ("build"); - if (schema_catalog::current_version (build_db, bs) != - build_db.schema_version (bs)) + catch (const std::exception& e) { - cerr << "error: build database schema differs from the current one" - << endl << " info: use brep-migrate to migrate the database" << endl; + cerr << "error: " << e << endl; return 1; } - // Prepare the build prepared query. + // Convert timeout duration into the time point. Return + // timestamp_nonexistent (never expire) for zero argument. Return nullopt if + // the argument is invalid. // - // Query package builds in chunks in order not to hold locks for too long. - // Sort the result by package version to minimize number of queries to the - // package database. - // - using bld_query = query; - using prep_bld_query = prepared_query; + static optional + timeout (const string& tm) + { + char* e (nullptr); + uint64_t t (strtoull (tm.c_str (), &e, 10)); - size_t offset (0); - bld_query bq ("ORDER BY" + - bld_query::id.package.tenant + "," + - bld_query::id.package.name + - order_by_version_desc (bld_query::id.package.version, false) + - "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); + if (*e != '\0' || tm.empty ()) + return nullopt; - connection_ptr conn (build_db.connection ()); + if (t == 0) + return timestamp_nonexistent; - prep_bld_query bld_prep_query ( - conn->prepare_query ("build-query", bq)); + return system_clock::now () - chrono::hours (t); + } - // Prepare the package version query. - // - // Query buildable packages every time the new package name is encountered - // during iterating over the package builds. Such a query will be made once - // per package name due to the builds query sorting criteria (see above). - // - using pkg_query = query; - using prep_pkg_query = prepared_query; + static int + clean_builds (const options& ops, + cli::argv_scanner& scan, + odb::pgsql::database& db) + { + // Load configurations names. + // + if (!scan.more ()) + { + cerr << "error: configuration file expected" << endl + << help_info << endl; + return 1; + } - package_name pkg_name; - set package_versions; + path cp; - pkg_query pq ( - pkg_query::build_package::id.name == pkg_query::_ref (pkg_name)); + try + { + cp = path (scan.next ()); + } + catch (const invalid_path& e) + { + cerr << "error: configuration file expected instead of '" << e.path + << "'" << endl + << help_info << endl; + return 1; + } - prep_pkg_query pkg_prep_query ( - conn->prepare_query ("package-query", pq)); + set configs; - for (bool ne (true); ne; ) - { - transaction t (conn->begin ()); + try + { + for (auto& c: parse_buildtab (cp)) + configs.emplace (move (c.name)); + } + catch (const io_error& e) + { + cerr << "error: unable to read '" << cp << "': " << e << endl; + return 1; + } - // Query builds. + // Parse timestamps. // - auto builds (bld_prep_query.execute ()); + map timeouts; // Toolchain timeouts. + timestamp default_timeout; // timestamp_nonexistent - if ((ne = !builds.empty ())) + while (scan.more ()) { - for (const auto& b: builds) + string a (scan.next ()); + + string tc; + optional to; + + size_t p (a.find ('=')); + + if (p == string::npos) + to = timeout (a); + else if (p > 0) // Note: toolchain name can't be empty. { - auto i (timeouts.find (b.toolchain_name)); + tc = string (a, 0, p); + to = timeout (string (a, p + 1)); + } - timestamp et (i != timeouts.end () - ? i->second - : default_timeout); + // Note that the default timeout can't be zero. + // + if (!to || (*to == timestamp_nonexistent && tc.empty ())) + { + cerr << "error: timeout expected instead of '" << a << "'" << endl + << help_info << endl; + return 1; + } - bool cleanup ( - // Check that the build is not stale. - // - b.timestamp <= et || + if (tc.empty ()) + default_timeout = *to; - // Check that the build configuration is still present. - // - // Note that we unable to detect configuration changes and rely on - // periodic rebuilds to take care of that. - // - configs.find (b.configuration) == configs.end ()); + timeouts[move (tc)] = move (*to); + } - // Check that the build package still exists. - // - if (!cleanup) + // Prepare the build prepared query. + // + // Query package builds in chunks in order not to hold locks for too long. + // Sort the result by package version to minimize number of queries to the + // package database. + // + using bld_query = query; + using prep_bld_query = prepared_query; + + size_t offset (0); + bld_query bq ("ORDER BY" + + bld_query::id.package.tenant + "," + + bld_query::id.package.name + + order_by_version_desc (bld_query::id.package.version, + false) + + "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); + + connection_ptr conn (db.connection ()); + + prep_bld_query bld_prep_query ( + conn->prepare_query ("build-query", bq)); + + // Prepare the package version query. + // + // Query buildable packages every time the new tenant or package name is + // encountered during iterating over the package builds. Such a query will + // be made once per tenant package name due to the builds query sorting + // criteria (see above). + // + using pkg_query = query; + using prep_pkg_query = prepared_query; + + string tnt; + package_name pkg_name; + set package_versions; + + pkg_query pq ( + pkg_query::build_package::id.tenant == pkg_query::_ref (tnt) && + pkg_query::build_package::id.name == pkg_query::_ref (pkg_name)); + + prep_pkg_query pkg_prep_query ( + conn->prepare_query ("package-query", pq)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + // Query builds. + // + auto builds (bld_prep_query.execute ()); + + if ((ne = !builds.empty ())) + { + for (const auto& b: builds) { - if (pkg_name != b.package_name) - { - pkg_name = b.package_name; - package_versions.clear (); + auto i (timeouts.find (b.toolchain_name)); - for (auto& p: pkg_prep_query.execute ()) - package_versions.emplace (move (p.version)); + timestamp et (i != timeouts.end () + ? i->second + : default_timeout); + + bool cleanup ( + // Check that the build is not stale. + // + b.timestamp <= et || + + // Check that the build configuration is still present. + // + // Note that we unable to detect configuration changes and rely on + // periodic rebuilds to take care of that. + // + configs.find (b.configuration) == configs.end ()); + + // Check that the build package still exists. + // + if (!cleanup) + { + if (tnt != b.tenant || pkg_name != b.package_name) + { + tnt = b.tenant; + pkg_name = b.package_name; + package_versions.clear (); + + for (auto& p: pkg_prep_query.execute ()) + package_versions.emplace (move (p.version)); + } + + cleanup = package_versions.find (b.package_version) == + package_versions.end (); } - cleanup = package_versions.find (b.package_version) == - package_versions.end (); + if (cleanup) + db.erase (b); + else + ++offset; } - - if (cleanup) - build_db.erase (b); - else - ++offset; } + + t.commit (); } - t.commit (); + return 0; } - return 0; -} -catch (const database_locked&) -{ - cerr << "brep-clean or brep-migrate is running" << endl; - return 2; -} -catch (const recoverable& e) -{ - cerr << "recoverable database error: " << e << endl; - return 3; -} -catch (const cli::exception& e) -{ - cerr << "error: " << e << endl << help_info << endl; - return 1; -} -catch (const failed&) -{ - return 1; // Diagnostics has already been issued. + static int + clean_tenants (const options& ops, + cli::argv_scanner& scan, + odb::pgsql::database& db) + { + if (!scan.more ()) + { + cerr << "error: timeout expected" << endl + << help_info << endl; + return 1; + } + + string a (scan.next ()); + optional to (timeout (a)); + + // Note that the timeout can't be zero. + // + if (!to || *to == timestamp_nonexistent) + { + cerr << "error: timeout expected instead of '" << a << "'" << endl + << help_info << endl; + return 1; + } + + if (scan.more ()) + { + cerr << "error: unexpected argument encountered" << endl + << help_info << endl; + return 1; + } + + uint64_t ns ( + chrono::duration_cast ( + to->time_since_epoch ()).count ()); + + // Query tenants in chunks in order not to hold locks for too long. + // + connection_ptr conn (db.connection ()); + + // Archive (rather then delete) old tenants, if requested. + // + if (ops.archive ()) + { + using query = query; + using pquery = prepared_query; + + query q ((query::creation_timestamp < ns && !query::archived) + + "LIMIT 100"); + + pquery pq (conn->prepare_query ("tenant-query", q)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + auto tenants (pq.execute ()); + if ((ne = !tenants.empty ())) + { + for (auto& t: tenants) + { + t.archived = true; + db.update (t); + } + } + + t.commit (); + } + + return 0; + } + + // Delete old tenants. + // + // Note that we don't delete dangling builds for the deleted packages. + // Doing so would require to operate on two databases, complicating the + // code and the utility interface. Note that dangling builds are never + // considered in the web interface and are always deleted with the + // 'brep-clean builds' command. + // + using query = query; + using pquery = prepared_query; + + query q ((query::creation_timestamp < ns) + "LIMIT 100"); + pquery pq (conn->prepare_query ("tenant-id-query", q)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + auto tenant_ids (pq.execute ()); + if ((ne = !tenant_ids.empty ())) + { + // Cache tenant ids and erase packages, repositories, and tenants at + // once. + // + strings tids; + tids.reserve (tenant_ids.size ()); + + for (auto& tid: tenant_ids) + tids.push_back (move (tid.value)); + + using odb::query; + + db.erase_query ( + query::id.tenant.in_range (tids.begin (), tids.end ())); + + db.erase_query ( + query::id.tenant.in_range (tids.begin (), tids.end ())); + + db.erase_query ( + query::id.in_range (tids.begin (), tids.end ())); + } + + t.commit (); + } + + return 0; + } } -// Fully qualified to avoid ambiguity with odb exception. -// -catch (const std::exception& e) + +int +main (int argc, char* argv[]) { - cerr << "error: " << e << endl; - return 1; + return brep::main (argc, argv); } diff --git a/clean/options-types.hxx b/clean/options-types.hxx deleted file mode 100644 index 7190396..0000000 --- a/clean/options-types.hxx +++ /dev/null @@ -1,18 +0,0 @@ -// file : clean/options-types.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef CLEAN_OPTIONS_TYPES_HXX -#define CLEAN_OPTIONS_TYPES_HXX - -#include - -#include -#include - -namespace brep -{ - struct toolchain_timeouts: std::map {}; -} - -#endif // CLEAN_OPTIONS_TYPES_HXX diff --git a/clean/types-parsers.cxx b/clean/types-parsers.cxx deleted file mode 100644 index 31f3a8d..0000000 --- a/clean/types-parsers.cxx +++ /dev/null @@ -1,60 +0,0 @@ -// file : clean/types-parsers.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include -#include // strtoull() - -#include - -#include -#include // cli namespace - -using namespace std; -using namespace brep; - -namespace cli -{ - void parser:: - parse (toolchain_timeouts& x, bool& xs, scanner& s) - { - const char* o (s.next ()); - - if (!s.more ()) - throw missing_value (o); - - string ov (s.next ()); - size_t p (ov.find ('=')); - - timestamp now (system_clock::now ()); - - // Convert timeout duration into the time point. - // - auto timeout = [o, &ov, &now] (const string& tm) -> timestamp - { - char* e (nullptr); - uint64_t t (strtoull (tm.c_str (), &e, 10)); - - if (*e != '\0' || tm.empty ()) - throw invalid_value (o, ov); - - if (t == 0) - return timestamp_nonexistent; - - return now - chrono::duration> (t); - }; - - if (p == string::npos) - x[string ()] = timeout (ov); // Default timeout. - else - { - string k (ov, 0, p); - if (k.empty ()) - throw invalid_value (o, ov); - - x[k] = timeout (string (ov, p + 1)); - } - - xs = true; - } -} diff --git a/clean/types-parsers.hxx b/clean/types-parsers.hxx deleted file mode 100644 index fe7c77c..0000000 --- a/clean/types-parsers.hxx +++ /dev/null @@ -1,28 +0,0 @@ -// file : clean/types-parsers.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -// CLI parsers, included into the generated source files. -// - -#ifndef CLEAN_TYPES_PARSERS_HXX -#define CLEAN_TYPES_PARSERS_HXX - -#include - -namespace cli -{ - class scanner; - - template - struct parser; - - template <> - struct parser - { - static void - parse (brep::toolchain_timeouts&, bool&, scanner&); - }; -} - -#endif // CLEAN_TYPES_PARSERS_HXX -- cgit v1.1