aboutsummaryrefslogtreecommitdiff
path: root/clean/clean.cxx
diff options
context:
space:
mode:
Diffstat (limited to 'clean/clean.cxx')
-rw-r--r--clean/clean.cxx573
1 files changed, 405 insertions, 168 deletions
diff --git a/clean/clean.cxx b/clean/clean.cxx
index 102b165..deba1db 100644
--- a/clean/clean.cxx
+++ b/clean/clean.cxx
@@ -2,7 +2,9 @@
// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
// license : MIT; see accompanying LICENSE file
+#include <map>
#include <set>
+#include <chrono>
#include <iostream>
#include <odb/database.hxx>
@@ -17,6 +19,8 @@
#include <libbrep/build.hxx>
#include <libbrep/build-odb.hxx>
+#include <libbrep/package.hxx>
+#include <libbrep/package-odb.hxx>
#include <libbrep/build-package.hxx>
#include <libbrep/build-package-odb.hxx>
#include <libbrep/database-lock.hxx>
@@ -25,226 +29,459 @@
using namespace std;
using namespace bbot;
-using namespace brep;
using namespace odb::core;
-// Operation failed, diagnostics has already been issued.
-//
-struct failed {};
+namespace brep
+{
+ // Operation failed, diagnostics has already been issued.
+ //
+ struct failed {};
-static const char* help_info (
- " info: run 'brep-clean --help' for more information");
+ static const char* help_info (
+ " info: run 'brep-clean --help' for more information");
-int
-main (int argc, char* argv[])
-try
-{
- cli::argv_scanner scan (argc, argv, true);
- options ops (scan);
+ static int
+ clean_builds (const options&, cli::argv_scanner&, odb::pgsql::database&);
- // Version.
- //
- if (ops.version ())
+ static int
+ clean_tenants (const options&, cli::argv_scanner&, odb::pgsql::database&);
+
+ static int
+ main (int argc, char* argv[])
+ try
{
- cout << "brep-clean " << BREP_VERSION_ID << endl
- << "libbrep " << LIBBREP_VERSION_ID << endl
- << "libbbot " << LIBBBOT_VERSION_ID << endl
- << "libbpkg " << LIBBPKG_VERSION_ID << endl
- << "libbutl " << LIBBUTL_VERSION_ID << endl
- << "Copyright (c) 2014-2018 Code Synthesis Ltd" << endl
- << "This is free software released under the MIT license." << endl;
+ cli::argv_scanner scan (argc, argv, true);
+ options ops (scan);
- return 0;
- }
+ // Version.
+ //
+ if (ops.version ())
+ {
+ cout << "brep-clean " << BREP_VERSION_ID << endl
+ << "libbrep " << LIBBREP_VERSION_ID << endl
+ << "libbbot " << LIBBBOT_VERSION_ID << endl
+ << "libbpkg " << LIBBPKG_VERSION_ID << endl
+ << "libbutl " << LIBBUTL_VERSION_ID << endl
+ << "Copyright (c) 2014-2018 Code Synthesis Ltd" << endl
+ << "This is free software released under the MIT license." << endl;
+
+ return 0;
+ }
- // Help.
- //
- if (ops.help ())
- {
- butl::pager p ("brep-clean help",
- false,
- ops.pager_specified () ? &ops.pager () : nullptr,
- &ops.pager_option ());
+ // Help.
+ //
+ if (ops.help ())
+ {
+ butl::pager p ("brep-clean help",
+ false,
+ ops.pager_specified () ? &ops.pager () : nullptr,
+ &ops.pager_option ());
+
+ print_usage (p.stream ());
- print_usage (p.stream ());
+ // If the pager failed, assume it has issued some diagnostics.
+ //
+ return p.wait () ? 0 : 1;
+ }
- // If the pager failed, assume it has issued some diagnostics.
+ // Detect the mode.
//
- return p.wait () ? 0 : 1;
- }
+ if (!scan.more ())
+ {
+ cerr << "error: 'builds' or 'tenants' is expected" << endl
+ << help_info << endl;
+ return 1;
+ }
- const toolchain_timeouts& timeouts (ops.stale_timeout ());
+ const string mode (scan.next ());
- auto i (timeouts.find (string ()));
- timestamp default_timeout (i != timeouts.end ()
- ? i->second
- : timestamp_nonexistent);
+ if (mode != "builds" && mode != "tenants")
+ throw cli::unknown_argument (mode);
- // Load configurations names.
- //
- if (!scan.more ())
- {
- cerr << "error: configuration file expected" << endl
- << help_info << endl;
- return 1;
- }
+ const string db_schema (mode == "builds" ? "build" : "package");
- set<string> configs;
- for (auto& c: parse_buildtab (path (scan.next ())))
- configs.emplace (move (c.name));
+ const string& db_name (!ops.db_name ().empty ()
+ ? ops.db_name ()
+ : "brep_" + db_schema);
- if (scan.more ())
- {
- cerr << "error: unexpected argument encountered" << endl
- << help_info << endl;
- return 1;
- }
+ odb::pgsql::database db (
+ ops.db_user (),
+ ops.db_password (),
+ db_name,
+ ops.db_host (),
+ ops.db_port (),
+ "options='-c default_transaction_isolation=serializable'");
- odb::pgsql::database build_db (
- ops.db_user (),
- ops.db_password (),
- ops.db_name (),
- ops.db_host (),
- ops.db_port (),
- "options='-c default_transaction_isolation=serializable'");
+ // Prevent several brep-clean/migrate instances from updating build
+ // database simultaneously.
+ //
+ database_lock l (db);
- // Prevent several brep-clean/migrate instances from updating build database
- // simultaneously.
- //
- database_lock l (build_db);
+ // Check that the database schema matches the current one.
+ //
+ if (schema_catalog::current_version (db, db_schema) !=
+ db.schema_version (db_schema))
+ {
+ cerr << "error: " << db_schema << " database schema differs from the "
+ << "current one" << endl
+ << " info: use brep-migrate to migrate the database" << endl;
+ return 1;
+ }
- // Check that the build database schema matches the current one.
+ return mode == "builds"
+ ? clean_builds (ops, scan, db)
+ : clean_tenants (ops, scan, db);
+ }
+ catch (const database_locked&)
+ {
+ cerr << "brep-clean or brep-migrate is running" << endl;
+ return 2;
+ }
+ catch (const recoverable& e)
+ {
+ cerr << "recoverable database error: " << e << endl;
+ return 3;
+ }
+ catch (const cli::exception& e)
+ {
+ cerr << "error: " << e << endl << help_info << endl;
+ return 1;
+ }
+ catch (const failed&)
+ {
+ return 1; // Diagnostics has already been issued.
+ }
+ // Fully qualified to avoid ambiguity with odb exception.
//
- const string bs ("build");
- if (schema_catalog::current_version (build_db, bs) !=
- build_db.schema_version (bs))
+ catch (const std::exception& e)
{
- cerr << "error: build database schema differs from the current one"
- << endl << " info: use brep-migrate to migrate the database" << endl;
+ cerr << "error: " << e << endl;
return 1;
}
- // Prepare the build prepared query.
+ // Convert timeout duration into the time point. Return
+ // timestamp_nonexistent (never expire) for zero argument. Return nullopt if
+ // the argument is invalid.
//
- // Query package builds in chunks in order not to hold locks for too long.
- // Sort the result by package version to minimize number of queries to the
- // package database.
- //
- using bld_query = query<build>;
- using prep_bld_query = prepared_query<build>;
+ static optional<timestamp>
+ timeout (const string& tm)
+ {
+ char* e (nullptr);
+ uint64_t t (strtoull (tm.c_str (), &e, 10));
- size_t offset (0);
- bld_query bq ("ORDER BY" +
- bld_query::id.package.tenant + "," +
- bld_query::id.package.name +
- order_by_version_desc (bld_query::id.package.version, false) +
- "OFFSET" + bld_query::_ref (offset) + "LIMIT 100");
+ if (*e != '\0' || tm.empty ())
+ return nullopt;
- connection_ptr conn (build_db.connection ());
+ if (t == 0)
+ return timestamp_nonexistent;
- prep_bld_query bld_prep_query (
- conn->prepare_query<build> ("build-query", bq));
+ return system_clock::now () - chrono::hours (t);
+ }
- // Prepare the package version query.
- //
- // Query buildable packages every time the new package name is encountered
- // during iterating over the package builds. Such a query will be made once
- // per package name due to the builds query sorting criteria (see above).
- //
- using pkg_query = query<buildable_package>;
- using prep_pkg_query = prepared_query<buildable_package>;
+ static int
+ clean_builds (const options& ops,
+ cli::argv_scanner& scan,
+ odb::pgsql::database& db)
+ {
+ // Load configurations names.
+ //
+ if (!scan.more ())
+ {
+ cerr << "error: configuration file expected" << endl
+ << help_info << endl;
+ return 1;
+ }
- package_name pkg_name;
- set<version> package_versions;
+ path cp;
- pkg_query pq (
- pkg_query::build_package::id.name == pkg_query::_ref (pkg_name));
+ try
+ {
+ cp = path (scan.next ());
+ }
+ catch (const invalid_path& e)
+ {
+ cerr << "error: configuration file expected instead of '" << e.path
+ << "'" << endl
+ << help_info << endl;
+ return 1;
+ }
- prep_pkg_query pkg_prep_query (
- conn->prepare_query<buildable_package> ("package-query", pq));
+ set<string> configs;
- for (bool ne (true); ne; )
- {
- transaction t (conn->begin ());
+ try
+ {
+ for (auto& c: parse_buildtab (cp))
+ configs.emplace (move (c.name));
+ }
+ catch (const io_error& e)
+ {
+ cerr << "error: unable to read '" << cp << "': " << e << endl;
+ return 1;
+ }
- // Query builds.
+ // Parse timestamps.
//
- auto builds (bld_prep_query.execute ());
+ map<string, timestamp> timeouts; // Toolchain timeouts.
+ timestamp default_timeout; // timestamp_nonexistent
- if ((ne = !builds.empty ()))
+ while (scan.more ())
{
- for (const auto& b: builds)
+ string a (scan.next ());
+
+ string tc;
+ optional<timestamp> to;
+
+ size_t p (a.find ('='));
+
+ if (p == string::npos)
+ to = timeout (a);
+ else if (p > 0) // Note: toolchain name can't be empty.
{
- auto i (timeouts.find (b.toolchain_name));
+ tc = string (a, 0, p);
+ to = timeout (string (a, p + 1));
+ }
- timestamp et (i != timeouts.end ()
- ? i->second
- : default_timeout);
+ // Note that the default timeout can't be zero.
+ //
+ if (!to || (*to == timestamp_nonexistent && tc.empty ()))
+ {
+ cerr << "error: timeout expected instead of '" << a << "'" << endl
+ << help_info << endl;
+ return 1;
+ }
- bool cleanup (
- // Check that the build is not stale.
- //
- b.timestamp <= et ||
+ if (tc.empty ())
+ default_timeout = *to;
- // Check that the build configuration is still present.
- //
- // Note that we unable to detect configuration changes and rely on
- // periodic rebuilds to take care of that.
- //
- configs.find (b.configuration) == configs.end ());
+ timeouts[move (tc)] = move (*to);
+ }
- // Check that the build package still exists.
- //
- if (!cleanup)
+ // Prepare the build prepared query.
+ //
+ // Query package builds in chunks in order not to hold locks for too long.
+ // Sort the result by package version to minimize number of queries to the
+ // package database.
+ //
+ using bld_query = query<build>;
+ using prep_bld_query = prepared_query<build>;
+
+ size_t offset (0);
+ bld_query bq ("ORDER BY" +
+ bld_query::id.package.tenant + "," +
+ bld_query::id.package.name +
+ order_by_version_desc (bld_query::id.package.version,
+ false) +
+ "OFFSET" + bld_query::_ref (offset) + "LIMIT 100");
+
+ connection_ptr conn (db.connection ());
+
+ prep_bld_query bld_prep_query (
+ conn->prepare_query<build> ("build-query", bq));
+
+ // Prepare the package version query.
+ //
+ // Query buildable packages every time the new tenant or package name is
+ // encountered during iterating over the package builds. Such a query will
+ // be made once per tenant package name due to the builds query sorting
+ // criteria (see above).
+ //
+ using pkg_query = query<buildable_package>;
+ using prep_pkg_query = prepared_query<buildable_package>;
+
+ string tnt;
+ package_name pkg_name;
+ set<version> package_versions;
+
+ pkg_query pq (
+ pkg_query::build_package::id.tenant == pkg_query::_ref (tnt) &&
+ pkg_query::build_package::id.name == pkg_query::_ref (pkg_name));
+
+ prep_pkg_query pkg_prep_query (
+ conn->prepare_query<buildable_package> ("package-query", pq));
+
+ for (bool ne (true); ne; )
+ {
+ transaction t (conn->begin ());
+
+ // Query builds.
+ //
+ auto builds (bld_prep_query.execute ());
+
+ if ((ne = !builds.empty ()))
+ {
+ for (const auto& b: builds)
{
- if (pkg_name != b.package_name)
- {
- pkg_name = b.package_name;
- package_versions.clear ();
+ auto i (timeouts.find (b.toolchain_name));
- for (auto& p: pkg_prep_query.execute ())
- package_versions.emplace (move (p.version));
+ timestamp et (i != timeouts.end ()
+ ? i->second
+ : default_timeout);
+
+ bool cleanup (
+ // Check that the build is not stale.
+ //
+ b.timestamp <= et ||
+
+ // Check that the build configuration is still present.
+ //
+ // Note that we unable to detect configuration changes and rely on
+ // periodic rebuilds to take care of that.
+ //
+ configs.find (b.configuration) == configs.end ());
+
+ // Check that the build package still exists.
+ //
+ if (!cleanup)
+ {
+ if (tnt != b.tenant || pkg_name != b.package_name)
+ {
+ tnt = b.tenant;
+ pkg_name = b.package_name;
+ package_versions.clear ();
+
+ for (auto& p: pkg_prep_query.execute ())
+ package_versions.emplace (move (p.version));
+ }
+
+ cleanup = package_versions.find (b.package_version) ==
+ package_versions.end ();
}
- cleanup = package_versions.find (b.package_version) ==
- package_versions.end ();
+ if (cleanup)
+ db.erase (b);
+ else
+ ++offset;
}
-
- if (cleanup)
- build_db.erase (b);
- else
- ++offset;
}
+
+ t.commit ();
}
- t.commit ();
+ return 0;
}
- return 0;
-}
-catch (const database_locked&)
-{
- cerr << "brep-clean or brep-migrate is running" << endl;
- return 2;
-}
-catch (const recoverable& e)
-{
- cerr << "recoverable database error: " << e << endl;
- return 3;
-}
-catch (const cli::exception& e)
-{
- cerr << "error: " << e << endl << help_info << endl;
- return 1;
-}
-catch (const failed&)
-{
- return 1; // Diagnostics has already been issued.
+ static int
+ clean_tenants (const options& ops,
+ cli::argv_scanner& scan,
+ odb::pgsql::database& db)
+ {
+ if (!scan.more ())
+ {
+ cerr << "error: timeout expected" << endl
+ << help_info << endl;
+ return 1;
+ }
+
+ string a (scan.next ());
+ optional<timestamp> to (timeout (a));
+
+ // Note that the timeout can't be zero.
+ //
+ if (!to || *to == timestamp_nonexistent)
+ {
+ cerr << "error: timeout expected instead of '" << a << "'" << endl
+ << help_info << endl;
+ return 1;
+ }
+
+ if (scan.more ())
+ {
+ cerr << "error: unexpected argument encountered" << endl
+ << help_info << endl;
+ return 1;
+ }
+
+ uint64_t ns (
+ chrono::duration_cast<chrono::nanoseconds> (
+ to->time_since_epoch ()).count ());
+
+ // Query tenants in chunks in order not to hold locks for too long.
+ //
+ connection_ptr conn (db.connection ());
+
+ // Archive (rather then delete) old tenants, if requested.
+ //
+ if (ops.archive ())
+ {
+ using query = query<tenant>;
+ using pquery = prepared_query<tenant>;
+
+ query q ((query::creation_timestamp < ns && !query::archived) +
+ "LIMIT 100");
+
+ pquery pq (conn->prepare_query<tenant> ("tenant-query", q));
+
+ for (bool ne (true); ne; )
+ {
+ transaction t (conn->begin ());
+
+ auto tenants (pq.execute ());
+ if ((ne = !tenants.empty ()))
+ {
+ for (auto& t: tenants)
+ {
+ t.archived = true;
+ db.update (t);
+ }
+ }
+
+ t.commit ();
+ }
+
+ return 0;
+ }
+
+ // Delete old tenants.
+ //
+ // Note that we don't delete dangling builds for the deleted packages.
+ // Doing so would require to operate on two databases, complicating the
+ // code and the utility interface. Note that dangling builds are never
+ // considered in the web interface and are always deleted with the
+ // 'brep-clean builds' command.
+ //
+ using query = query<tenant_id>;
+ using pquery = prepared_query<tenant_id>;
+
+ query q ((query::creation_timestamp < ns) + "LIMIT 100");
+ pquery pq (conn->prepare_query<tenant_id> ("tenant-id-query", q));
+
+ for (bool ne (true); ne; )
+ {
+ transaction t (conn->begin ());
+
+ auto tenant_ids (pq.execute ());
+ if ((ne = !tenant_ids.empty ()))
+ {
+ // Cache tenant ids and erase packages, repositories, and tenants at
+ // once.
+ //
+ strings tids;
+ tids.reserve (tenant_ids.size ());
+
+ for (auto& tid: tenant_ids)
+ tids.push_back (move (tid.value));
+
+ using odb::query;
+
+ db.erase_query<package> (
+ query<package>::id.tenant.in_range (tids.begin (), tids.end ()));
+
+ db.erase_query<repository> (
+ query<repository>::id.tenant.in_range (tids.begin (), tids.end ()));
+
+ db.erase_query<tenant> (
+ query<tenant>::id.in_range (tids.begin (), tids.end ()));
+ }
+
+ t.commit ();
+ }
+
+ return 0;
+ }
}
-// Fully qualified to avoid ambiguity with odb exception.
-//
-catch (const std::exception& e)
+
+int
+main (int argc, char* argv[])
{
- cerr << "error: " << e << endl;
- return 1;
+ return brep::main (argc, argv);
}