aboutsummaryrefslogtreecommitdiff
path: root/clean
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2017-05-22 23:31:10 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2017-05-24 16:12:02 +0300
commit94b04d166c1041028571222b9931121b0f7dfded (patch)
tree33d79d968cfd4a7e0977cb9686ae9ff4abfd52c4 /clean
parent28de800a7a38ed781b8f04f8f380d65b2d79ec14 (diff)
Implement brep-clean
Diffstat (limited to 'clean')
-rw-r--r--clean/.gitignore2
-rw-r--r--clean/buildfile30
-rw-r--r--clean/clean.cli164
-rw-r--r--clean/clean.cxx287
-rw-r--r--clean/options-types.hxx18
-rw-r--r--clean/types-parsers.cxx60
-rw-r--r--clean/types-parsers.hxx28
7 files changed, 589 insertions, 0 deletions
diff --git a/clean/.gitignore b/clean/.gitignore
new file mode 100644
index 0000000..8e11bbd
--- /dev/null
+++ b/clean/.gitignore
@@ -0,0 +1,2 @@
+*-options.?xx
+brep-clean
diff --git a/clean/buildfile b/clean/buildfile
new file mode 100644
index 0000000..b4feada
--- /dev/null
+++ b/clean/buildfile
@@ -0,0 +1,30 @@
+# file : clean/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+import libs = libodb%lib{odb}
+import libs += libodb-pgsql%lib{odb-pgsql}
+import libs += libbutl%lib{butl}
+import libs += libbbot%lib{bbot}
+
+include ../libbrep/
+
+exe{brep-clean}: {hxx ixx cxx}{* -clean-options} \
+ {hxx ixx cxx}{clean-options} \
+ ../libbrep/lib{brep} $libs
+
+# Generated options parser.
+#
+if $cli.configured
+{
+ cli.cxx{clean-options}: cli{clean}
+
+ cli.options += -I $src_root --include-with-brackets --include-prefix clean \
+--guard-prefix CLEAN --generate-specifier --page-usage print_ --ansi-color \
+--cxx-prologue "#include <clean/types-parsers.hxx>" \
+--long-usage
+
+ # Include generated cli files into the distribution.
+ #
+ cli.cxx{clean-options}: dist = true
+}
diff --git a/clean/clean.cli b/clean/clean.cli
new file mode 100644
index 0000000..3f4e2ea
--- /dev/null
+++ b/clean/clean.cli
@@ -0,0 +1,164 @@
+// file : clean/clean.cli
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+include <vector>;
+include <string>;
+include <cstdint>; // uint16_t
+
+include <clean/options-types.hxx>;
+
+"\section=1"
+"\name=brep-clean"
+"\summary=clean brep build database"
+
+{
+ "<options> <buildtab>",
+
+ "\h|SYNOPSIS|
+
+ \cb{brep-clean --help}\n
+ \cb{brep-clean --version}\n
+ \c{\b{brep-clean} [<options>] <buildtab>}
+
+ \h|DESCRIPTION|
+
+ \cb{brep-clean} deletes expired package builds from the brep \cb{build}
+ database. The build is considered expired if the package version is not
+ in the \cb{package} database, or the configuration is not listed in the
+ <buildtab> file, or the timestamp is older than the one specified for
+ this build's toolchain (see \cb{--stale-timeout}).
+
+ Note that \cb{brep-clean} expects the \cb{build} and \cb{package} database
+ schemas to have already been created using \l{brep-migrate(1)}."
+}
+
+class options
+{
+ "\h|OPTIONS|"
+
+ brep::toolchain_timeouts --stale-timeout
+ {
+ "[<name>=]<days>",
+ "Number of days to wait before considering builds for the named toolchain
+ as stale. Specify zero for <days> to make builds for a toolchain never
+ expire. Omit <name> (including \cb{=}) to specify the default timeout.
+ It will apply to all the toolchains that don't have a timeout specified
+ explicitly. If unspecified, the default timeout is zero (never expire)."
+ }
+
+ std::string --build-db-user
+ {
+ "<user>",
+ "Build database user name. If not specified, then operating system (login)
+ name is used."
+ }
+
+ std::string --build-db-password
+ {
+ "<pass>",
+ "Build database password. If not specified, then login without password is
+ expected to work."
+ }
+
+ std::string --build-db-name = "brep_build"
+ {
+ "<name>",
+ "Build database name. If not specified, then \cb{brep_build} is used by
+ default."
+ }
+
+ std::string --build-db-host
+ {
+ "<host>",
+ "Build database host name, address, or socket. If not specified, then
+ connect to \cb{localhost} using the operating system-default mechanism
+ (Unix-domain socket, etc)."
+ }
+
+ std::uint16_t --build-db-port = 0
+ {
+ "<port>",
+ "Build database port number. If not specified, the default port is used."
+ }
+
+ std::string --package-db-user
+ {
+ "<user>",
+ "Package database user name. If not specified, then operating system
+ (login) name is used."
+ }
+
+ std::string --package-db-password
+ {
+ "<pass>",
+ "Package database password. If not specified, then login without password
+ is expected to work."
+ }
+
+ std::string --package-db-name = "brep_package"
+ {
+ "<name>",
+ "Package database name. If not specified, then \cb{brep_package} is used by
+ default."
+ }
+
+ std::string --package-db-host
+ {
+ "<host>",
+ "Package database host name, address, or socket. If not specified, then
+ connect to \cb{localhost} using the operating system-default mechanism
+ (Unix-domain socket, etc)."
+ }
+
+ std::uint16_t --package-db-port = 0
+ {
+ "<port>",
+ "Package database port number. If not specified, the default port is used."
+ }
+
+ std::string --pager // String to allow empty value.
+ {
+ "<path>",
+ "The pager program to be used to show long text. Commonly used pager
+ programs are \cb{less} and \cb{more}. You can also specify additional
+ options that should be passed to the pager program with
+ \cb{--pager-option}. If an empty string is specified as the pager
+ program, then no pager will be used. If the pager program is not
+ explicitly specified, then \cb{brep-clean} will try to use \cb{less}.
+ If it is not available, then no pager will be used."
+ }
+
+ std::vector<std::string> --pager-option
+ {
+ "<opt>",
+ "Additional option to be passed to the pager program. See \cb{--pager}
+ for more information on the pager program. Repeat this option to
+ specify multiple pager options."
+ }
+
+ bool --help {"Print usage information and exit."}
+ bool --version {"Print version and exit."}
+};
+
+"\h|EXIT STATUS|
+
+\dl|
+
+\li|\cb{0}
+
+Success.|
+
+\li|\cb{1}
+
+Fatal error.|
+
+\li|\cb{2}
+
+An instance of \cb{brep-clean} or \l{brep-migrate(1)} is already running. Try
+again.|
+
+\li|\cb{3}
+
+Recoverable database error. Try again.||
+"
diff --git a/clean/clean.cxx b/clean/clean.cxx
new file mode 100644
index 0000000..fbb4d23
--- /dev/null
+++ b/clean/clean.cxx
@@ -0,0 +1,287 @@
+// file : clean/clean.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <set>
+#include <iostream>
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+#include <odb/schema-catalog.hxx>
+
+#include <odb/pgsql/database.hxx>
+
+#include <libbutl/pager.hxx>
+
+#include <libbbot/build-config.hxx>
+
+#include <libbrep/build.hxx>
+#include <libbrep/build-odb.hxx>
+#include <libbrep/package.hxx>
+#include <libbrep/package-odb.hxx>
+#include <libbrep/database-lock.hxx>
+
+#include <clean/clean-options.hxx>
+
+using namespace std;
+using namespace bbot;
+using namespace brep;
+using namespace odb::core;
+
+// Operation failed, diagnostics has already been issued.
+//
+struct failed {};
+
+static const char* help_info (
+ " info: run 'brep-clean --help' for more information");
+
+int
+main (int argc, char* argv[])
+try
+{
+ cli::argv_scanner scan (argc, argv, true);
+ options ops (scan);
+
+ // Version.
+ //
+ if (ops.version ())
+ {
+ cout << "brep-clean " << BREP_VERSION_ID << endl
+ << "libbrep " << LIBBREP_VERSION_ID << endl
+ << "libbbot " << LIBBBOT_VERSION_ID << endl
+ << "libbpkg " << LIBBPKG_VERSION_ID << endl
+ << "libbutl " << LIBBUTL_VERSION_ID << endl
+ << "Copyright (c) 2014-2017 Code Synthesis Ltd" << endl
+ << "This is free software released under the MIT license." << endl;
+
+ return 0;
+ }
+
+ // Help.
+ //
+ if (ops.help ())
+ {
+ butl::pager p ("brep-clean help",
+ false,
+ ops.pager_specified () ? &ops.pager () : nullptr,
+ &ops.pager_option ());
+
+ print_usage (p.stream ());
+
+ // If the pager failed, assume it has issued some diagnostics.
+ //
+ return p.wait () ? 0 : 1;
+ }
+
+ const toolchain_timeouts& timeouts (ops.stale_timeout ());
+
+ auto i (timeouts.find (string ()));
+ timestamp default_timeout (i != timeouts.end ()
+ ? i->second
+ : timestamp_nonexistent);
+
+ // Load configurations names.
+ //
+ if (!scan.more ())
+ {
+ cerr << "error: configuration file expected" << endl
+ << help_info << endl;
+ return 1;
+ }
+
+ set<string> configs;
+ for (auto& c: parse_buildtab (path (scan.next ())))
+ configs.emplace (move (c.name));
+
+ if (scan.more ())
+ {
+ cerr << "error: unexpected argument encountered" << endl
+ << help_info << endl;
+ return 1;
+ }
+
+ odb::pgsql::database build_db (
+ ops.build_db_user (),
+ ops.build_db_password (),
+ ops.build_db_name (),
+ ops.build_db_host (),
+ ops.build_db_port (),
+ "options='-c default_transaction_isolation=serializable'");
+
+ odb::pgsql::database package_db (
+ ops.package_db_user (),
+ ops.package_db_password (),
+ ops.package_db_name (),
+ ops.package_db_host (),
+ ops.package_db_port (),
+ "options='-c default_transaction_isolation=serializable'");
+
+ // Prevent several brep-clean/migrate instances from updating build database
+ // simultaneously.
+ //
+ database_lock l (build_db);
+
+ // Check that the build and package database schemas match the current ones.
+ //
+ const string bs ("build");
+ if (schema_catalog::current_version (build_db, bs) !=
+ build_db.schema_version (bs))
+ {
+ cerr << "error: build database schema differs from the current one"
+ << endl << " info: use brep-migrate to migrate the database" << endl;
+ return 1;
+ }
+
+ const string ps ("package");
+ if (schema_catalog::current_version (package_db, ps) !=
+ package_db.schema_version (ps))
+ {
+ cerr << "error: package database schema differs from the current one"
+ << endl << " info: use brep-migrate to migrate the database" << endl;
+ return 1;
+ }
+
+ // Prepare the build prepared query.
+ //
+ // Query package builds in chunks in order not to hold locks for too long.
+ // Sort the result by package version to minimize number of queries to the
+ // package database.
+ //
+ using bld_query = query<build>;
+ using prep_bld_query = prepared_query<build>;
+
+ size_t offset (0);
+ bld_query bq ("ORDER BY" + bld_query::id.package.name +
+ order_by_version_desc (bld_query::id.package.version, false) +
+ "OFFSET" + bld_query::_ref (offset) + "LIMIT 100");
+
+ connection_ptr bld_conn (build_db.connection ());
+
+ prep_bld_query bld_prep_query (
+ bld_conn->prepare_query<build> ("build-query", bq));
+
+ // Prepare the package version query.
+ //
+ // Query package versions every time the new package name is encountered
+ // during iterating over the package builds. Such a query will be made once
+ // per package name due to the builds query sorting criteria (see above).
+ //
+ using pkg_query = query<package_version>;
+ using prep_pkg_query = prepared_query<package_version>;
+
+ string package_name;
+ set<version> package_versions;
+
+ pkg_query pq (pkg_query::package::id.name == pkg_query::_ref (package_name));
+
+ connection_ptr pkg_conn (package_db.connection ());
+
+ prep_pkg_query pkg_prep_query (
+ pkg_conn->prepare_query<package_version> ("package-version-query", pq));
+
+ while (true)
+ {
+ // Start the build database transaction.
+ //
+ transaction bt (bld_conn->begin ());
+
+ // Query builds.
+ //
+ auto builds (bld_prep_query.execute ());
+
+ if (!builds.empty ())
+ {
+ // Start the package database transaction.
+ //
+ transaction pt (pkg_conn->begin (), false);
+
+ for (const auto& b: builds)
+ {
+ auto i (timeouts.find (b.toolchain_name));
+
+ timestamp et (i != timeouts.end ()
+ ? i->second
+ : default_timeout);
+
+ bool cleanup (
+ // Check that the build is not stale.
+ //
+ b.timestamp <= et ||
+
+ // Check that the build configuration is still present.
+ //
+ // Note that we unable to detect configuration changes and rely on
+ // periodic rebuilds to take care of that.
+ //
+ configs.find (b.configuration) == configs.end ());
+
+ // Check that the build package still exists.
+ //
+ if (!cleanup)
+ {
+ if (package_name != b.package_name)
+ {
+ // Switch to the package database transaction.
+ //
+ transaction::current (pt);
+
+ package_name = b.package_name;
+ package_versions.clear ();
+
+ for (auto& v: pkg_prep_query.execute ())
+ package_versions.emplace (move (v.version));
+
+ // Switch back to the build database transaction.
+ //
+ transaction::current (bt);
+ }
+
+ cleanup = package_versions.find (b.package_version) ==
+ package_versions.end ();
+ }
+
+ if (cleanup)
+ build_db.erase (b);
+ else
+ ++offset;
+ }
+
+ // Commit the package database transaction.
+ //
+ pt.commit ();
+ }
+
+ bt.commit ();
+
+ if (builds.empty ())
+ break;
+ }
+
+ return 0;
+}
+catch (const database_locked&)
+{
+ cerr << "brep-clean or brep-migrate is running" << endl;
+ return 2;
+}
+catch (const recoverable& e)
+{
+ cerr << "recoverable database error: " << e << endl;
+ return 3;
+}
+catch (const cli::exception& e)
+{
+ cerr << "error: " << e << endl << help_info << endl;
+ return 1;
+}
+catch (const failed&)
+{
+ return 1; // Diagnostics has already been issued.
+}
+// Fully qualified to avoid ambiguity with odb exception.
+//
+catch (const std::exception& e)
+{
+ cerr << "error: " << e << endl;
+ return 1;
+}
diff --git a/clean/options-types.hxx b/clean/options-types.hxx
new file mode 100644
index 0000000..183c0df
--- /dev/null
+++ b/clean/options-types.hxx
@@ -0,0 +1,18 @@
+// file : clean/options-types.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef CLEAN_OPTIONS_TYPES_HXX
+#define CLEAN_OPTIONS_TYPES_HXX
+
+#include <map>
+
+#include <libbrep/types.hxx>
+#include <libbrep/utility.hxx>
+
+namespace brep
+{
+ struct toolchain_timeouts: std::map<string, timestamp> {};
+}
+
+#endif // CLEAN_OPTIONS_TYPES_HXX
diff --git a/clean/types-parsers.cxx b/clean/types-parsers.cxx
new file mode 100644
index 0000000..26b9a02
--- /dev/null
+++ b/clean/types-parsers.cxx
@@ -0,0 +1,60 @@
+// file : clean/types-parsers.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <chrono>
+#include <string> // strtoull()
+
+#include <clean/types-parsers.hxx>
+
+#include <clean/options-types.hxx>
+#include <clean/clean-options.hxx> // cli namespace
+
+using namespace std;
+using namespace brep;
+
+namespace cli
+{
+ void parser<toolchain_timeouts>::
+ parse (toolchain_timeouts& x, bool& xs, scanner& s)
+ {
+ const char* o (s.next ());
+
+ if (!s.more ())
+ throw missing_value (o);
+
+ string ov (s.next ());
+ size_t p (ov.find ('='));
+
+ timestamp now (timestamp::clock::now ());
+
+ // Convert timeout duration into the time point.
+ //
+ auto timeout = [o, &ov, &now] (const string& tm) -> timestamp
+ {
+ char* e (nullptr);
+ uint64_t t (strtoull (tm.c_str (), &e, 10));
+
+ if (*e != '\0' || tm.empty ())
+ throw invalid_value (o, ov);
+
+ if (t == 0)
+ return timestamp_nonexistent;
+
+ return now - chrono::duration<uint64_t, ratio<86400>> (t);
+ };
+
+ if (p == string::npos)
+ x[string ()] = timeout (ov); // Default timeout.
+ else
+ {
+ string k (ov, 0, p);
+ if (k.empty ())
+ throw invalid_value (o, ov);
+
+ x[k] = timeout (string (ov, p + 1));
+ }
+
+ xs = true;
+ }
+}
diff --git a/clean/types-parsers.hxx b/clean/types-parsers.hxx
new file mode 100644
index 0000000..46fd483
--- /dev/null
+++ b/clean/types-parsers.hxx
@@ -0,0 +1,28 @@
+// file : clean/types-parsers.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+// CLI parsers, included into the generated source files.
+//
+
+#ifndef CLEAN_TYPES_PARSERS_HXX
+#define CLEAN_TYPES_PARSERS_HXX
+
+#include <clean/options-types.hxx>
+
+namespace cli
+{
+ class scanner;
+
+ template <typename T>
+ struct parser;
+
+ template <>
+ struct parser<brep::toolchain_timeouts>
+ {
+ static void
+ parse (brep::toolchain_timeouts&, bool&, scanner&);
+ };
+}
+
+#endif // CLEAN_TYPES_PARSERS_HXX