diff options
Diffstat (limited to 'clean')
| -rw-r--r-- | clean/.gitignore | 2 | ||||
| -rw-r--r-- | clean/buildfile | 30 | ||||
| -rw-r--r-- | clean/clean.cli | 164 | ||||
| -rw-r--r-- | clean/clean.cxx | 287 | ||||
| -rw-r--r-- | clean/options-types.hxx | 18 | ||||
| -rw-r--r-- | clean/types-parsers.cxx | 60 | ||||
| -rw-r--r-- | clean/types-parsers.hxx | 28 |
7 files changed, 589 insertions, 0 deletions
diff --git a/clean/.gitignore b/clean/.gitignore new file mode 100644 index 0000000..8e11bbd --- /dev/null +++ b/clean/.gitignore @@ -0,0 +1,2 @@ +*-options.?xx +brep-clean diff --git a/clean/buildfile b/clean/buildfile new file mode 100644 index 0000000..b4feada --- /dev/null +++ b/clean/buildfile @@ -0,0 +1,30 @@ +# file : clean/buildfile +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +import libs = libodb%lib{odb} +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libbutl%lib{butl} +import libs += libbbot%lib{bbot} + +include ../libbrep/ + +exe{brep-clean}: {hxx ixx cxx}{* -clean-options} \ + {hxx ixx cxx}{clean-options} \ + ../libbrep/lib{brep} $libs + +# Generated options parser. +# +if $cli.configured +{ + cli.cxx{clean-options}: cli{clean} + + cli.options += -I $src_root --include-with-brackets --include-prefix clean \ +--guard-prefix CLEAN --generate-specifier --page-usage print_ --ansi-color \ +--cxx-prologue "#include <clean/types-parsers.hxx>" \ +--long-usage + + # Include generated cli files into the distribution. + # + cli.cxx{clean-options}: dist = true +} diff --git a/clean/clean.cli b/clean/clean.cli new file mode 100644 index 0000000..3f4e2ea --- /dev/null +++ b/clean/clean.cli @@ -0,0 +1,164 @@ +// file : clean/clean.cli +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +include <vector>; +include <string>; +include <cstdint>; // uint16_t + +include <clean/options-types.hxx>; + +"\section=1" +"\name=brep-clean" +"\summary=clean brep build database" + +{ + "<options> <buildtab>", + + "\h|SYNOPSIS| + + \cb{brep-clean --help}\n + \cb{brep-clean --version}\n + \c{\b{brep-clean} [<options>] <buildtab>} + + \h|DESCRIPTION| + + \cb{brep-clean} deletes expired package builds from the brep \cb{build} + database. The build is considered expired if the package version is not + in the \cb{package} database, or the configuration is not listed in the + <buildtab> file, or the timestamp is older than the one specified for + this build's toolchain (see \cb{--stale-timeout}). + + Note that \cb{brep-clean} expects the \cb{build} and \cb{package} database + schemas to have already been created using \l{brep-migrate(1)}." +} + +class options +{ + "\h|OPTIONS|" + + brep::toolchain_timeouts --stale-timeout + { + "[<name>=]<days>", + "Number of days to wait before considering builds for the named toolchain + as stale. Specify zero for <days> to make builds for a toolchain never + expire. Omit <name> (including \cb{=}) to specify the default timeout. + It will apply to all the toolchains that don't have a timeout specified + explicitly. If unspecified, the default timeout is zero (never expire)." + } + + std::string --build-db-user + { + "<user>", + "Build database user name. If not specified, then operating system (login) + name is used." + } + + std::string --build-db-password + { + "<pass>", + "Build database password. If not specified, then login without password is + expected to work." + } + + std::string --build-db-name = "brep_build" + { + "<name>", + "Build database name. If not specified, then \cb{brep_build} is used by + default." + } + + std::string --build-db-host + { + "<host>", + "Build database host name, address, or socket. If not specified, then + connect to \cb{localhost} using the operating system-default mechanism + (Unix-domain socket, etc)." + } + + std::uint16_t --build-db-port = 0 + { + "<port>", + "Build database port number. If not specified, the default port is used." + } + + std::string --package-db-user + { + "<user>", + "Package database user name. If not specified, then operating system + (login) name is used." + } + + std::string --package-db-password + { + "<pass>", + "Package database password. If not specified, then login without password + is expected to work." + } + + std::string --package-db-name = "brep_package" + { + "<name>", + "Package database name. If not specified, then \cb{brep_package} is used by + default." + } + + std::string --package-db-host + { + "<host>", + "Package database host name, address, or socket. If not specified, then + connect to \cb{localhost} using the operating system-default mechanism + (Unix-domain socket, etc)." + } + + std::uint16_t --package-db-port = 0 + { + "<port>", + "Package database port number. If not specified, the default port is used." + } + + std::string --pager // String to allow empty value. + { + "<path>", + "The pager program to be used to show long text. Commonly used pager + programs are \cb{less} and \cb{more}. You can also specify additional + options that should be passed to the pager program with + \cb{--pager-option}. If an empty string is specified as the pager + program, then no pager will be used. If the pager program is not + explicitly specified, then \cb{brep-clean} will try to use \cb{less}. + If it is not available, then no pager will be used." + } + + std::vector<std::string> --pager-option + { + "<opt>", + "Additional option to be passed to the pager program. See \cb{--pager} + for more information on the pager program. Repeat this option to + specify multiple pager options." + } + + bool --help {"Print usage information and exit."} + bool --version {"Print version and exit."} +}; + +"\h|EXIT STATUS| + +\dl| + +\li|\cb{0} + +Success.| + +\li|\cb{1} + +Fatal error.| + +\li|\cb{2} + +An instance of \cb{brep-clean} or \l{brep-migrate(1)} is already running. Try +again.| + +\li|\cb{3} + +Recoverable database error. Try again.|| +" diff --git a/clean/clean.cxx b/clean/clean.cxx new file mode 100644 index 0000000..fbb4d23 --- /dev/null +++ b/clean/clean.cxx @@ -0,0 +1,287 @@ +// file : clean/clean.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <set> +#include <iostream> + +#include <odb/database.hxx> +#include <odb/transaction.hxx> +#include <odb/schema-catalog.hxx> + +#include <odb/pgsql/database.hxx> + +#include <libbutl/pager.hxx> + +#include <libbbot/build-config.hxx> + +#include <libbrep/build.hxx> +#include <libbrep/build-odb.hxx> +#include <libbrep/package.hxx> +#include <libbrep/package-odb.hxx> +#include <libbrep/database-lock.hxx> + +#include <clean/clean-options.hxx> + +using namespace std; +using namespace bbot; +using namespace brep; +using namespace odb::core; + +// Operation failed, diagnostics has already been issued. +// +struct failed {}; + +static const char* help_info ( + " info: run 'brep-clean --help' for more information"); + +int +main (int argc, char* argv[]) +try +{ + cli::argv_scanner scan (argc, argv, true); + options ops (scan); + + // Version. + // + if (ops.version ()) + { + cout << "brep-clean " << BREP_VERSION_ID << endl + << "libbrep " << LIBBREP_VERSION_ID << endl + << "libbbot " << LIBBBOT_VERSION_ID << endl + << "libbpkg " << LIBBPKG_VERSION_ID << endl + << "libbutl " << LIBBUTL_VERSION_ID << endl + << "Copyright (c) 2014-2017 Code Synthesis Ltd" << endl + << "This is free software released under the MIT license." << endl; + + return 0; + } + + // Help. + // + if (ops.help ()) + { + butl::pager p ("brep-clean help", + false, + ops.pager_specified () ? &ops.pager () : nullptr, + &ops.pager_option ()); + + print_usage (p.stream ()); + + // If the pager failed, assume it has issued some diagnostics. + // + return p.wait () ? 0 : 1; + } + + const toolchain_timeouts& timeouts (ops.stale_timeout ()); + + auto i (timeouts.find (string ())); + timestamp default_timeout (i != timeouts.end () + ? i->second + : timestamp_nonexistent); + + // Load configurations names. + // + if (!scan.more ()) + { + cerr << "error: configuration file expected" << endl + << help_info << endl; + return 1; + } + + set<string> configs; + for (auto& c: parse_buildtab (path (scan.next ()))) + configs.emplace (move (c.name)); + + if (scan.more ()) + { + cerr << "error: unexpected argument encountered" << endl + << help_info << endl; + return 1; + } + + odb::pgsql::database build_db ( + ops.build_db_user (), + ops.build_db_password (), + ops.build_db_name (), + ops.build_db_host (), + ops.build_db_port (), + "options='-c default_transaction_isolation=serializable'"); + + odb::pgsql::database package_db ( + ops.package_db_user (), + ops.package_db_password (), + ops.package_db_name (), + ops.package_db_host (), + ops.package_db_port (), + "options='-c default_transaction_isolation=serializable'"); + + // Prevent several brep-clean/migrate instances from updating build database + // simultaneously. + // + database_lock l (build_db); + + // Check that the build and package database schemas match the current ones. + // + const string bs ("build"); + if (schema_catalog::current_version (build_db, bs) != + build_db.schema_version (bs)) + { + cerr << "error: build database schema differs from the current one" + << endl << " info: use brep-migrate to migrate the database" << endl; + return 1; + } + + const string ps ("package"); + if (schema_catalog::current_version (package_db, ps) != + package_db.schema_version (ps)) + { + cerr << "error: package database schema differs from the current one" + << endl << " info: use brep-migrate to migrate the database" << endl; + return 1; + } + + // Prepare the build prepared query. + // + // Query package builds in chunks in order not to hold locks for too long. + // Sort the result by package version to minimize number of queries to the + // package database. + // + using bld_query = query<build>; + using prep_bld_query = prepared_query<build>; + + size_t offset (0); + bld_query bq ("ORDER BY" + bld_query::id.package.name + + order_by_version_desc (bld_query::id.package.version, false) + + "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); + + connection_ptr bld_conn (build_db.connection ()); + + prep_bld_query bld_prep_query ( + bld_conn->prepare_query<build> ("build-query", bq)); + + // Prepare the package version query. + // + // Query package versions every time the new package name is encountered + // during iterating over the package builds. Such a query will be made once + // per package name due to the builds query sorting criteria (see above). + // + using pkg_query = query<package_version>; + using prep_pkg_query = prepared_query<package_version>; + + string package_name; + set<version> package_versions; + + pkg_query pq (pkg_query::package::id.name == pkg_query::_ref (package_name)); + + connection_ptr pkg_conn (package_db.connection ()); + + prep_pkg_query pkg_prep_query ( + pkg_conn->prepare_query<package_version> ("package-version-query", pq)); + + while (true) + { + // Start the build database transaction. + // + transaction bt (bld_conn->begin ()); + + // Query builds. + // + auto builds (bld_prep_query.execute ()); + + if (!builds.empty ()) + { + // Start the package database transaction. + // + transaction pt (pkg_conn->begin (), false); + + for (const auto& b: builds) + { + auto i (timeouts.find (b.toolchain_name)); + + timestamp et (i != timeouts.end () + ? i->second + : default_timeout); + + bool cleanup ( + // Check that the build is not stale. + // + b.timestamp <= et || + + // Check that the build configuration is still present. + // + // Note that we unable to detect configuration changes and rely on + // periodic rebuilds to take care of that. + // + configs.find (b.configuration) == configs.end ()); + + // Check that the build package still exists. + // + if (!cleanup) + { + if (package_name != b.package_name) + { + // Switch to the package database transaction. + // + transaction::current (pt); + + package_name = b.package_name; + package_versions.clear (); + + for (auto& v: pkg_prep_query.execute ()) + package_versions.emplace (move (v.version)); + + // Switch back to the build database transaction. + // + transaction::current (bt); + } + + cleanup = package_versions.find (b.package_version) == + package_versions.end (); + } + + if (cleanup) + build_db.erase (b); + else + ++offset; + } + + // Commit the package database transaction. + // + pt.commit (); + } + + bt.commit (); + + if (builds.empty ()) + break; + } + + return 0; +} +catch (const database_locked&) +{ + cerr << "brep-clean or brep-migrate is running" << endl; + return 2; +} +catch (const recoverable& e) +{ + cerr << "recoverable database error: " << e << endl; + return 3; +} +catch (const cli::exception& e) +{ + cerr << "error: " << e << endl << help_info << endl; + return 1; +} +catch (const failed&) +{ + return 1; // Diagnostics has already been issued. +} +// Fully qualified to avoid ambiguity with odb exception. +// +catch (const std::exception& e) +{ + cerr << "error: " << e << endl; + return 1; +} diff --git a/clean/options-types.hxx b/clean/options-types.hxx new file mode 100644 index 0000000..183c0df --- /dev/null +++ b/clean/options-types.hxx @@ -0,0 +1,18 @@ +// file : clean/options-types.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef CLEAN_OPTIONS_TYPES_HXX +#define CLEAN_OPTIONS_TYPES_HXX + +#include <map> + +#include <libbrep/types.hxx> +#include <libbrep/utility.hxx> + +namespace brep +{ + struct toolchain_timeouts: std::map<string, timestamp> {}; +} + +#endif // CLEAN_OPTIONS_TYPES_HXX diff --git a/clean/types-parsers.cxx b/clean/types-parsers.cxx new file mode 100644 index 0000000..26b9a02 --- /dev/null +++ b/clean/types-parsers.cxx @@ -0,0 +1,60 @@ +// file : clean/types-parsers.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <chrono> +#include <string> // strtoull() + +#include <clean/types-parsers.hxx> + +#include <clean/options-types.hxx> +#include <clean/clean-options.hxx> // cli namespace + +using namespace std; +using namespace brep; + +namespace cli +{ + void parser<toolchain_timeouts>:: + parse (toolchain_timeouts& x, bool& xs, scanner& s) + { + const char* o (s.next ()); + + if (!s.more ()) + throw missing_value (o); + + string ov (s.next ()); + size_t p (ov.find ('=')); + + timestamp now (timestamp::clock::now ()); + + // Convert timeout duration into the time point. + // + auto timeout = [o, &ov, &now] (const string& tm) -> timestamp + { + char* e (nullptr); + uint64_t t (strtoull (tm.c_str (), &e, 10)); + + if (*e != '\0' || tm.empty ()) + throw invalid_value (o, ov); + + if (t == 0) + return timestamp_nonexistent; + + return now - chrono::duration<uint64_t, ratio<86400>> (t); + }; + + if (p == string::npos) + x[string ()] = timeout (ov); // Default timeout. + else + { + string k (ov, 0, p); + if (k.empty ()) + throw invalid_value (o, ov); + + x[k] = timeout (string (ov, p + 1)); + } + + xs = true; + } +} diff --git a/clean/types-parsers.hxx b/clean/types-parsers.hxx new file mode 100644 index 0000000..46fd483 --- /dev/null +++ b/clean/types-parsers.hxx @@ -0,0 +1,28 @@ +// file : clean/types-parsers.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +// CLI parsers, included into the generated source files. +// + +#ifndef CLEAN_TYPES_PARSERS_HXX +#define CLEAN_TYPES_PARSERS_HXX + +#include <clean/options-types.hxx> + +namespace cli +{ + class scanner; + + template <typename T> + struct parser; + + template <> + struct parser<brep::toolchain_timeouts> + { + static void + parse (brep::toolchain_timeouts&, bool&, scanner&); + }; +} + +#endif // CLEAN_TYPES_PARSERS_HXX |