diff options
author | Boris Kolpackov <boris@codesynthesis.com> | 2016-01-18 07:35:12 +0200 |
---|---|---|
committer | Boris Kolpackov <boris@codesynthesis.com> | 2016-01-23 17:47:47 +0200 |
commit | 1dc38cf49b6c7a8b661a9cc675ded94c8ab33c36 (patch) | |
tree | 5a216148adb9d842a5a15c032a671182faa9ba06 | |
parent | fe6182a8c89675f92e72c881d707e21cdf56f376 (diff) |
Implement brep-migrate utility
-rw-r--r-- | INSTALL | 26 | ||||
-rw-r--r-- | INSTALL-DEV | 8 | ||||
-rw-r--r-- | brep/.gitignore | 2 | ||||
-rw-r--r-- | brep/buildfile | 4 | ||||
-rw-r--r-- | brep/database-lock | 43 | ||||
-rw-r--r-- | brep/database-lock.cxx | 44 | ||||
-rw-r--r-- | brep/mod-package-search.cxx | 16 | ||||
-rwxr-xr-x | brep/odb.sh | 18 | ||||
-rw-r--r-- | brep/package | 6 | ||||
-rw-r--r-- | brep/package-extra.sql | 20 | ||||
-rw-r--r-- | brep/package.xml | 406 | ||||
-rw-r--r-- | buildfile | 2 | ||||
-rw-r--r-- | etc/buildfile | 2 | ||||
-rw-r--r-- | etc/systemd/brep-load.service (renamed from etc/systemd/brep-loader.service) | 2 | ||||
-rw-r--r-- | etc/systemd/brep-load.timer (renamed from etc/systemd/brep-loader.timer) | 2 | ||||
-rw-r--r-- | load/.gitignore (renamed from loader/.gitignore) | 2 | ||||
-rw-r--r-- | load/buildfile (renamed from loader/buildfile) | 10 | ||||
-rw-r--r-- | load/load.cxx (renamed from loader/loader.cxx) | 253 | ||||
-rw-r--r-- | load/options.cli (renamed from loader/options.cli) | 32 | ||||
-rw-r--r-- | migrate/.gitignore | 3 | ||||
-rw-r--r-- | migrate/buildfile | 18 | ||||
-rw-r--r-- | migrate/migrate.cxx | 317 | ||||
-rw-r--r-- | migrate/options.cli | 90 | ||||
-rw-r--r-- | tests/buildfile | 2 | ||||
-rw-r--r-- | tests/load/1/basics/packages (renamed from tests/loader/1/basics/packages) | 0 | ||||
-rw-r--r-- | tests/load/1/basics/repositories (renamed from tests/loader/1/basics/repositories) | 0 | ||||
-rw-r--r-- | tests/load/1/math/packages (renamed from tests/loader/1/math/packages) | 0 | ||||
-rw-r--r-- | tests/load/1/math/repositories (renamed from tests/loader/1/math/repositories) | 0 | ||||
-rw-r--r-- | tests/load/1/misc/packages (renamed from tests/loader/1/misc/packages) | 0 | ||||
-rw-r--r-- | tests/load/1/misc/repositories (renamed from tests/loader/1/misc/repositories) | 0 | ||||
-rw-r--r-- | tests/load/1/stable/packages (renamed from tests/loader/1/stable/packages) | 0 | ||||
-rw-r--r-- | tests/load/1/stable/repositories (renamed from tests/loader/1/stable/repositories) | 0 | ||||
-rw-r--r-- | tests/load/1/staging/packages (renamed from tests/loader/1/staging/packages) | 0 | ||||
-rw-r--r-- | tests/load/1/staging/repositories (renamed from tests/loader/1/staging/repositories) | 0 | ||||
-rw-r--r-- | tests/load/1/testing/packages (renamed from tests/loader/1/testing/packages) | 0 | ||||
-rw-r--r-- | tests/load/1/testing/repositories (renamed from tests/loader/1/testing/repositories) | 0 | ||||
-rw-r--r-- | tests/load/buildfile (renamed from tests/loader/buildfile) | 4 | ||||
-rw-r--r-- | tests/load/driver.cxx (renamed from tests/loader/driver.cxx) | 2 | ||||
-rw-r--r-- | tests/load/r.conf (renamed from tests/loader/r.conf) | 0 | ||||
-rw-r--r-- | web/apache/service.cxx | 2 |
40 files changed, 1153 insertions, 183 deletions
@@ -122,10 +122,10 @@ To troubleshoot, see PostgreSQL logs. 5. Create Database Schema and Load Repositories $ mkdir config -$ edit config/brep-loader.conf # Loader configuration, see brep-loader(1). +$ edit config/brep-load.conf # Loader configuration, see brep-load(1). -$ psql --quiet -f install/share/brep/package.sql -$ install/bin/brep-loader config/brep-loader.conf +$ install/bin/brep-migrate +$ install/bin/brep-load config/brep-load.conf To verify: @@ -226,21 +226,21 @@ since we want the loader to run even when we are not logged in: $ sudo loginctl enable-linger brep $ mkdir -p .config/systemd/user -$ cp install/share/brep/etc/systemd/brep-loader.* .config/systemd/user/ +$ cp install/share/brep/etc/systemd/brep-load.* .config/systemd/user/ Start the service to make sure there are no issues: -$ systemctl --user start brep-loader.service +$ systemctl --user start brep-load.service $ journalctl Start the timer and monitor it to make sure it fires: -$ systemctl --user start brep-loader.timer +$ systemctl --user start brep-load.timer $ journalctl -f If everything looks good, enable the timer to be started at boot time: -$ systemctl --user enable brep-loader.timer +$ systemctl --user enable brep-load.timer 8. Upgrade Procedure @@ -262,8 +262,8 @@ $ bpkg -d brep build brep Stop and disable loader: -$ systemctl --user disable --now brep-loader.timer -$ systemctl --user stop brep-loader.service +$ systemctl --user disable --now brep-load.timer +$ systemctl --user stop brep-load.service Stop apache: @@ -280,16 +280,16 @@ $ diff -u install/share/brep/etc/brep-module.conf config/brep-module.conf Update database schema: -$ psql --quiet -f install/share/brep/package.sql +$ install/bin/brep-migrate Start and enable loader: -$ systemctl --user start brep-loader.service -$ systemctl --user status brep-loader.service +$ systemctl --user start brep-load.service +$ systemctl --user status brep-load.service If everything looks good, enable periodic execution: -$ systemctl --user enable --now brep-loader.timer +$ systemctl --user enable --now brep-load.timer Start apache: diff --git a/INSTALL-DEV b/INSTALL-DEV index b81d7fb..6e23664 100644 --- a/INSTALL-DEV +++ b/INSTALL-DEV @@ -48,8 +48,8 @@ $ sudo tail -f /var/log/postgresql/*.log All the commands are executed from brep project root. -$ psql --quiet -d brep -f brep/package.sql -$ loader/brep-loader tests/loader/r.conf # Or some other loader config. +$ migrate/brep-migrate +$ load/brep-load tests/load/r.conf # Or some other loader config. To verify: @@ -115,7 +115,7 @@ $ sudo tail -f /var/log/apache2/error.log To do a "complete reload" (i.e., recreate database schema, load the repository data, and reload the Apache2 plugin), execute the following from brep/: -psql --quiet -d brep -f brep/package.sql -loader/brep-loader tests/loader/r.conf +migrate/brep-migrate --recreate +load/brep-load tests/load/r.conf sudo /etc/init.d/apache2 restart sudo systemctl restart apache2 diff --git a/brep/.gitignore b/brep/.gitignore index 2c23e23..852a40d 100644 --- a/brep/.gitignore +++ b/brep/.gitignore @@ -2,4 +2,4 @@ options options.?xx package-odb* package.sql - +package-extra diff --git a/brep/buildfile b/brep/buildfile index 9826504..1aa4d08 100644 --- a/brep/buildfile +++ b/brep/buildfile @@ -10,7 +10,7 @@ define sql: file sql{*}: extension = sql sql{*}: install = data -./: lib{brep} mod{brep} sql{package} +./: lib{brep} mod{brep} sql{package package-extra} # lib{brep} # @@ -20,8 +20,10 @@ import libs += libbpkg%lib{bpkg} lib{brep}: \ {hxx cxx}{ package } \ +{file }{ package.xml } \ {hxx ixx cxx}{ package-odb } \ {hxx cxx}{ package-traits } \ +{hxx cxx}{ database-lock } \ {hxx }{ types } \ {hxx }{ utility } \ {hxx }{ version } \ diff --git a/brep/database-lock b/brep/database-lock new file mode 100644 index 0000000..72036a1 --- /dev/null +++ b/brep/database-lock @@ -0,0 +1,43 @@ +// file : brep/database-lock -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BREP_DATABASE_LOCK +#define BREP_DATABASE_LOCK + +#include <memory> // unique_ptr +#include <exception> + +#include <odb/pgsql/forward.hxx> // database, transaction +#include <odb/pgsql/connection.hxx> + +namespace brep +{ + struct database_locked: std::exception + { + virtual char const* + what () const throw () {return "database locked";} + }; + + // Try to "lock" the PostgreSQL database in the constructor and release the + // lock in the destructor. Throw database_locked if the database is already + // locked by someone else. May also throw odb::pgsql::database_exception. + // + // This mechanism is used by the brep loader and schema migration tool to + // make sure they don't step on each others toes. + // + // Note: movable but not copyable. + // + class database_lock + { + public: + explicit + database_lock (odb::pgsql::database&); + + private: + odb::pgsql::connection_ptr connection_; + std::unique_ptr<odb::pgsql::transaction> transaction_; + }; +} + +#endif // BREP_DATABASE_LOCK diff --git a/brep/database-lock.cxx b/brep/database-lock.cxx new file mode 100644 index 0000000..3b8ae21 --- /dev/null +++ b/brep/database-lock.cxx @@ -0,0 +1,44 @@ +// file : brep/database-lock.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <brep/database-lock> + +#include <odb/pgsql/database.hxx> +#include <odb/pgsql/exceptions.hxx> +#include <odb/pgsql/transaction.hxx> + +namespace brep +{ + using namespace odb::pgsql; + + database_lock:: + database_lock (database& db) + { + // Before locking the table make sure it exists. + // + { + transaction t (db.begin ()); + db.execute ("CREATE TABLE IF NOT EXISTS database_mutex ()"); + t.commit (); + } + + connection_ = db.connection (); + + // Don't make current. Will be rolled back in destructor. + // + transaction_.reset (new transaction (connection_->begin (), false)); + + try + { + connection_->execute ("LOCK TABLE database_mutex NOWAIT"); + } + catch (const database_exception& e) + { + if (e.sqlstate () == "55P03") // The table is already locked. + throw database_locked (); + + throw; + } + } +} diff --git a/brep/mod-package-search.cxx b/brep/mod-package-search.cxx index 4326435..d649ff4 100644 --- a/brep/mod-package-search.cxx +++ b/brep/mod-package-search.cxx @@ -9,6 +9,7 @@ #include <odb/session.hxx> #include <odb/database.hxx> #include <odb/transaction.hxx> +#include <odb/schema-catalog.hxx> #include <web/xhtml> #include <web/module> @@ -16,6 +17,7 @@ #include <brep/types> #include <brep/utility> +#include <brep/version> #include <brep/page> #include <brep/options> @@ -38,6 +40,20 @@ init (scanner& s) options_->root (dir_path ("/")); db_ = shared_database (*options_); + + // Check that the database schema matches the current one. It's enough to + // perform the check in just a single module implementation (and we don't + // do in the dispatcher because it doesn't use the database). + // + // Note that the failure can be reported by each web server worker process. + // While it could be tempting to move the check to the + // repository_root::version() function, it would be wrong. The function can + // be called by a different process (usually the web server root one) not + // having the proper permissions to access the database. + // + if (schema_catalog::current_version (*db_) != db_->schema_version ()) + fail << "database schema differs from the current one (module " + << BREP_VERSION_STR << ")"; } template <typename T> diff --git a/brep/odb.sh b/brep/odb.sh index 489d423..d68b28f 100755 --- a/brep/odb.sh +++ b/brep/odb.sh @@ -1,11 +1,13 @@ #! /usr/bin/env bash odb -d pgsql --std c++11 --generate-query --generate-schema \ - --odb-epilogue '#include <brep/wrapper-traits>' \ - --hxx-prologue '#include <brep/wrapper-traits>' \ - --hxx-prologue "#include <brep/package-traits>" \ - --sql-epilogue-file package-extra.sql \ - -I .. -I ../../libbpkg -I ../../libbutl \ - --hxx-suffix "" --include-with-brackets \ - --include-prefix brep --guard-prefix BREP \ - package + --schema-format sql --schema-format embedded \ + --odb-epilogue '#include <brep/wrapper-traits>' \ + --hxx-prologue '#include <brep/wrapper-traits>' \ + --hxx-prologue '#include <brep/package-traits>' \ + -I .. -I ../../libbpkg -I ../../libbutl \ + --hxx-suffix "" --include-with-brackets \ + --include-prefix brep --guard-prefix BREP \ + package + +xxd -i <package-extra.sql >package-extra diff --git a/brep/package b/brep/package index 92a2320..2700e8a 100644 --- a/brep/package +++ b/brep/package @@ -19,6 +19,12 @@ #include <brep/types> #include <brep/utility> +// Used by the data migration entries. +// +#define LIBBREP_SCHEMA_VERSION_BASE 1 + +#pragma db model version(LIBBREP_SCHEMA_VERSION_BASE, 1, open) + // The uint16_t value range is not fully covered by SMALLINT PostgreSQL type // to which uint16_t is mapped by default. // diff --git a/brep/package-extra.sql b/brep/package-extra.sql index dc37d1f..9a847e3 100644 --- a/brep/package-extra.sql +++ b/brep/package-extra.sql @@ -1,12 +1,22 @@ -DROP TYPE IF EXISTS weighted_text CASCADE; -CREATE TYPE weighted_text AS (a TEXT, b TEXT, c TEXT, d TEXT); - +-- This file should be parsable by the brep-migrate utility. To decrease the +-- parser complexity, the following restrictions are placed: +-- +-- * comments must start with -- at the beginning of the line (ignoring +-- leading spaces) +-- * only CREATE and DROP statements for FUNCTION and TYPE +-- * function bodies must be defined using $$-quoted strings +-- * strings other then function bodies must be quoted with ' or " +-- * statements must end with ";\n" +-- DROP FUNCTION IF EXISTS to_tsvector(IN document weighted_text); DROP FUNCTION IF EXISTS search_packages(IN query tsquery, INOUT name TEXT); DROP FUNCTION IF EXISTS search_latest_packages(IN query tsquery); DROP FUNCTION IF EXISTS latest_package(INOUT name TEXT); DROP FUNCTION IF EXISTS latest_packages(); +DROP TYPE IF EXISTS weighted_text CASCADE; +CREATE TYPE weighted_text AS (a TEXT, b TEXT, c TEXT, d TEXT); + -- Return the latest versions of internal packages as a set of package rows. -- CREATE FUNCTION @@ -72,9 +82,9 @@ $$ LANGUAGE SQL STABLE; -- Search for packages matching the search query and having the specified name. -- Return a set of rows containing the package id and search rank. If query -- is NULL, then match all packages and return 0 rank for all rows. --- +-- CREATE FUNCTION -search_packages(IN query tsquery, +search_packages(IN query tsquery, INOUT name TEXT, OUT version_epoch INTEGER, OUT version_canonical_upstream TEXT, diff --git a/brep/package.xml b/brep/package.xml new file mode 100644 index 0000000..9c88f25 --- /dev/null +++ b/brep/package.xml @@ -0,0 +1,406 @@ +<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" version="1"> + <model version="1"> + <table name="repository" kind="object"> + <column name="name" type="TEXT" null="false"/> + <column name="location" type="TEXT" null="false"/> + <column name="display_name" type="TEXT" null="false"/> + <column name="priority" type="INTEGER" null="false"/> + <column name="url" type="TEXT" null="true"/> + <column name="email" type="TEXT" null="true"/> + <column name="summary" type="TEXT" null="true"/> + <column name="description" type="TEXT" null="true"/> + <column name="local_path" type="TEXT" null="false"/> + <column name="packages_timestamp" type="BIGINT" null="false"/> + <column name="repositories_timestamp" type="BIGINT" null="false"/> + <column name="internal" type="BOOLEAN" null="false"/> + <primary-key> + <column name="name"/> + </primary-key> + </table> + <table name="repository_complements" kind="container"> + <column name="repository" type="TEXT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="complement" type="TEXT" null="false"/> + <foreign-key name="repository_fk" on-delete="CASCADE"> + <column name="repository"/> + <references table="repository"> + <column name="name"/> + </references> + </foreign-key> + <index name="repository_complements_repository_i"> + <column name="repository"/> + </index> + <index name="repository_complements_index_i"> + <column name="index"/> + </index> + <foreign-key name="complement_fk" deferrable="DEFERRED"> + <column name="complement"/> + <references table="repository"> + <column name="name"/> + </references> + </foreign-key> + </table> + <table name="repository_prerequisites" kind="container"> + <column name="repository" type="TEXT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="prerequisite" type="TEXT" null="false"/> + <foreign-key name="repository_fk" on-delete="CASCADE"> + <column name="repository"/> + <references table="repository"> + <column name="name"/> + </references> + </foreign-key> + <index name="repository_prerequisites_repository_i"> + <column name="repository"/> + </index> + <index name="repository_prerequisites_index_i"> + <column name="index"/> + </index> + <foreign-key name="prerequisite_fk" deferrable="DEFERRED"> + <column name="prerequisite"/> + <references table="repository"> + <column name="name"/> + </references> + </foreign-key> + </table> + <table name="package" kind="object"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="version_upstream" type="TEXT" null="false"/> + <column name="version_release" type="TEXT" null="true"/> + <column name="priority" type="INTEGER" null="false"/> + <column name="priority_comment" type="TEXT" null="false"/> + <column name="summary" type="TEXT" null="false"/> + <column name="description" type="TEXT" null="true"/> + <column name="changes" type="TEXT" null="false"/> + <column name="url" type="TEXT" null="false"/> + <column name="url_comment" type="TEXT" null="false"/> + <column name="package_url" type="TEXT" null="true"/> + <column name="package_url_comment" type="TEXT" null="true"/> + <column name="email" type="TEXT" null="false"/> + <column name="email_comment" type="TEXT" null="false"/> + <column name="package_email" type="TEXT" null="true"/> + <column name="package_email_comment" type="TEXT" null="true"/> + <column name="internal_repository" type="TEXT" null="true"/> + <column name="location" type="TEXT" null="true"/> + <column name="search_index" type="tsvector" null="true"/> + <primary-key> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </primary-key> + <foreign-key name="internal_repository_fk" deferrable="DEFERRED"> + <column name="internal_repository"/> + <references table="repository"> + <column name="name"/> + </references> + </foreign-key> + <index name="package_search_index_i" method="GIN"> + <column name="search_index"/> + </index> + </table> + <table name="package_license_alternatives" kind="container"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="comment" type="TEXT" null="false"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_license_alternatives_object_id_i"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <index name="package_license_alternatives_index_i"> + <column name="index"/> + </index> + </table> + <table name="package_licenses" kind="container"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="alternative_index" type="BIGINT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="license" type="TEXT" null="false"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_licenses_object_id_i"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + </table> + <table name="package_tags" kind="container"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="tag" type="TEXT" null="false"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_tags_object_id_i"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <index name="package_tags_index_i"> + <column name="index"/> + </index> + </table> + <table name="package_dependencies" kind="container"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="conditional" type="BOOLEAN" null="false"/> + <column name="comment" type="TEXT" null="false"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_dependencies_object_id_i"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <index name="package_dependencies_index_i"> + <column name="index"/> + </index> + </table> + <table name="package_dependency_alternatives" kind="container"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="dependency_index" type="BIGINT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="dep_name" type="TEXT" null="false"/> + <column name="dep_version_epoch" type="INTEGER" null="false"/> + <column name="dep_version_canonical_upstream" type="TEXT" null="false"/> + <column name="dep_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="dep_version_revision" type="INTEGER" null="false"/> + <column name="dep_min_version_epoch" type="INTEGER" null="true"/> + <column name="dep_min_version_canonical_upstream" type="TEXT" null="true"/> + <column name="dep_min_version_canonical_release" type="TEXT" null="true"/> + <column name="dep_min_version_revision" type="INTEGER" null="true"/> + <column name="dep_min_version_upstream" type="TEXT" null="true"/> + <column name="dep_min_version_release" type="TEXT" null="true"/> + <column name="dep_max_version_epoch" type="INTEGER" null="true"/> + <column name="dep_max_version_canonical_upstream" type="TEXT" null="true"/> + <column name="dep_max_version_canonical_release" type="TEXT" null="true"/> + <column name="dep_max_version_revision" type="INTEGER" null="true"/> + <column name="dep_max_version_upstream" type="TEXT" null="true"/> + <column name="dep_max_version_release" type="TEXT" null="true"/> + <column name="dep_min_open" type="BOOLEAN" null="true"/> + <column name="dep_max_open" type="BOOLEAN" null="true"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_dependency_alternatives_object_id_i"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <foreign-key name="dep_package_fk" deferrable="DEFERRED"> + <column name="dep_name"/> + <column name="dep_version_epoch"/> + <column name="dep_version_canonical_upstream"/> + <column name="dep_version_canonical_release"/> + <column name="dep_version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + </table> + <table name="package_requirements" kind="container"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="conditional" type="BOOLEAN" null="false"/> + <column name="comment" type="TEXT" null="false"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_requirements_object_id_i"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <index name="package_requirements_index_i"> + <column name="index"/> + </index> + </table> + <table name="package_requirement_alternatives" kind="container"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="requirement_index" type="BIGINT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="id" type="TEXT" null="false"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_requirement_alternatives_object_id_i"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + </table> + <table name="package_other_repositories" kind="container"> + <column name="name" type="TEXT" null="false"/> + <column name="version_epoch" type="INTEGER" null="false"/> + <column name="version_canonical_upstream" type="TEXT" null="false"/> + <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="version_revision" type="INTEGER" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="repository" type="TEXT" null="false"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + <references table="package"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </references> + </foreign-key> + <index name="package_other_repositories_object_id_i"> + <column name="name"/> + <column name="version_epoch"/> + <column name="version_canonical_upstream"/> + <column name="version_canonical_release"/> + <column name="version_revision"/> + </index> + <index name="package_other_repositories_index_i"> + <column name="index"/> + </index> + <foreign-key name="repository_fk" deferrable="DEFERRED"> + <column name="repository"/> + <references table="repository"> + <column name="name"/> + </references> + </foreign-key> + </table> + </model> +</changelog> @@ -2,7 +2,7 @@ # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -d = brep/ etc/ loader/ tests/ www/ +d = brep/ etc/ load/ migrate/ tests/ www/ ./: $d doc{INSTALL INSTALL-DEV LICENSE version} file{manifest} include $d diff --git a/etc/buildfile b/etc/buildfile index 9ebcf27..4910076 100644 --- a/etc/buildfile +++ b/etc/buildfile @@ -3,7 +3,7 @@ # license : MIT; see accompanying LICENSE file ./: file{brep-module.conf brep-apache2.conf} \ -systemd/file{brep-loader.service brep-loader.timer} +systemd/file{brep-load.service brep-load.timer} install = data/etc systemd/: install = data/etc/systemd diff --git a/etc/systemd/brep-loader.service b/etc/systemd/brep-load.service index dec7944..34a7c9a 100644 --- a/etc/systemd/brep-loader.service +++ b/etc/systemd/brep-load.service @@ -3,7 +3,7 @@ Description=brep repository loader service [Service] Type=oneshot -ExecStart=/home/brep/install/bin/brep-loader /home/brep/config/brep-loader.conf +ExecStart=/home/brep/install/bin/brep-load /home/brep/config/brep-load.conf [Install] WantedBy=default.target diff --git a/etc/systemd/brep-loader.timer b/etc/systemd/brep-load.timer index 22ff22b..713cb31 100644 --- a/etc/systemd/brep-loader.timer +++ b/etc/systemd/brep-load.timer @@ -4,7 +4,7 @@ RefuseManualStart=no RefuseManualStop=no [Timer] -Unit=brep-loader.service +Unit=brep-load.service # Don't keep track of the timer across reboots. # diff --git a/loader/.gitignore b/load/.gitignore index 820b183..2f464c5 100644 --- a/loader/.gitignore +++ b/load/.gitignore @@ -1,3 +1,3 @@ options options.?xx -brep-loader +brep-load diff --git a/loader/buildfile b/load/buildfile index 477cebd..c3a324f 100644 --- a/loader/buildfile +++ b/load/buildfile @@ -1,4 +1,4 @@ -# file : loader/buildfile +# file : load/buildfile # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file @@ -9,12 +9,12 @@ import libs += libodb%lib{odb} include ../brep/ -exe{brep-loader}: \ -{ cxx}{ loader } \ +exe{brep-load}: \ +{ cxx}{ load } \ {hxx ixx cxx}{ options } \ ../brep/lib{brep} $libs -cli.options += -I $src_root --include-with-brackets --include-prefix loader \ ---guard-prefix LOADER +cli.options += -I $src_root --include-with-brackets --include-prefix load \ +--guard-prefix LOAD {hxx ixx cxx}{options}: cli{options} diff --git a/loader/loader.cxx b/load/load.cxx index b1b7706..9e6ee5d 100644 --- a/loader/loader.cxx +++ b/load/load.cxx @@ -1,9 +1,10 @@ -// file : loader/loader.cxx -*- C++ -*- +// file : load/load.cxx -*- C++ -*- // copyright : Copyright (c) 2014-2016 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file #include <sstream> #include <fstream> +#include <ostream> #include <iostream> #include <stdexcept> // runtime_error, invalid_argument #include <algorithm> // find(), find_if() @@ -11,11 +12,9 @@ #include <odb/session.hxx> #include <odb/database.hxx> #include <odb/transaction.hxx> +#include <odb/schema-catalog.hxx> #include <odb/pgsql/database.hxx> -#include <odb/pgsql/exceptions.hxx> -#include <odb/pgsql/connection.hxx> -#include <odb/pgsql/transaction.hxx> #include <butl/filesystem> @@ -27,8 +26,9 @@ #include <brep/package> #include <brep/package-odb> +#include <brep/database-lock> -#include <loader/options> +#include <load/options> using namespace std; using namespace odb::core; @@ -36,12 +36,10 @@ using namespace butl; using namespace bpkg; using namespace brep; -namespace pgsql = odb::pgsql; - static void usage (ostream& os) { - os << "Usage: brep-loader [options] <file>" << endl + os << "Usage: brep-load [options] <file>" << endl << "File lists internal repositories." << endl << "Options:" << endl; @@ -700,152 +698,141 @@ detect_dependency_cycle (const package_id& id, package_ids& chain, database& db) int main (int argc, char* argv[]) +try { - try + cli::argv_scanner scan (argc, argv, true); + options ops (scan); + + // Version. + // + if (ops.version ()) { - cli::argv_scanner scan (argc, argv, true); - options ops (scan); + cout << "brep-load " << BREP_VERSION_STR << endl + << "libbrep " << LIBBREP_VERSION_STR << endl + << "libbpkg " << LIBBPKG_VERSION_STR << endl + << "libbutl " << LIBBUTL_VERSION_STR << endl + << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl + << "MIT; see accompanying LICENSE file" << endl; + + return 0; + } - // Version. - // - if (ops.version ()) - { - cout << "brep-loader " << BREP_VERSION_STR << endl - << "libbrep " << LIBBREP_VERSION_STR << endl - << "libbpkg " << LIBBPKG_VERSION_STR << endl - << "libbutl " << LIBBUTL_VERSION_STR << endl - << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl - << "MIT; see accompanying LICENSE file" << endl; - - return 0; - } + // Help. + // + if (ops.help ()) + { + usage (cout); + return 0; + } - // Help. - // - if (ops.help ()) - { - usage (cout); - return 0; - } + if (argc < 2) + { + cerr << "<file> argument not provided" << endl; + usage (cerr); + return 1; + } - if (argc < 2) - { - cerr << "<file> argument not provided" << endl; - usage (cerr); - return 1; - } + if (argc > 2) + { + cerr << "unexpected argument encountered" << endl; + usage (cerr); + return 1; + } - if (argc > 2) - { - cerr << "unexpected argument encountered" << endl; - usage (cerr); - return 1; - } + odb::pgsql::database db (ops.db_user (), + ops.db_password (), + ops.db_name (), + ops.db_host (), + ops.db_port ()); - pgsql::database db (ops.db_user (), - ops.db_password (), - ops.db_name (), - ops.db_host (), - ops.db_port ()); + // Prevent several brep-load/migrate instances from updating DB + // simultaneously. + // + database_lock l (db); - // Prevent several loader instances from updating DB simultaneously. - // - { - transaction t (db.begin ()); - db.execute ("CREATE TABLE IF NOT EXISTS loader_mutex ()"); - t.commit (); - } + transaction t (db.begin ()); - pgsql::connection_ptr synch_c (db.connection ()); + // Check that the database schema matches the current one. + // + if (schema_catalog::current_version (db) != db.schema_version ()) + { + cerr << "database schema differs from the current one" << endl; + return 1; + } + + // Load the description of all the internal repositories from the + // configuration file. + // + internal_repositories irs (load_repositories (path (argv[1]))); - // Don't make current. + if (changed (irs, db)) + { + // Rebuild repositories persistent state from scratch. // - pgsql::transaction synch_t (synch_c->begin (), false); + db.erase_query<package> (); + db.erase_query<repository> (); - try - { - synch_c->execute ("LOCK TABLE loader_mutex NOWAIT"); - } - catch (const pgsql::database_exception& e) + // On the first pass over the internal repositories we load their + // packages. + // + uint16_t priority (1); + for (const auto& ir: irs) { - if (e.sqlstate () == "55P03") - return 2; // Other loader instance acquired the mutex. + shared_ptr<repository> r ( + make_shared<repository> (ir.location, + move (ir.display_name), + move (ir.local_path), + priority++)); - throw; + load_packages (r, db); } - // Load the description of all the internal repositories from the - // configuration file. + // On the second pass over the internal repositories we load their + // (not yet loaded) manifest values, complement, and prerequisite + // repositories. // - internal_repositories irs (load_repositories (path (argv[1]))); - - transaction t (db.begin ()); - - if (changed (irs, db)) + for (const auto& ir: irs) { - // Rebuild repositories persistent state from scratch. - // - db.erase_query<package> (); - db.erase_query<repository> (); - - // On the first pass over the internal repositories we load their - // packages. - // - uint16_t priority (1); - for (const auto& ir: irs) - { - shared_ptr<repository> r ( - make_shared<repository> (ir.location, - move (ir.display_name), - move (ir.local_path), - priority++)); - - load_packages (r, db); - } - - // On the second pass over the internal repositories we load their - // (not yet loaded) manifest values, complement, and prerequisite - // repositories. - // - for (const auto& ir: irs) - { - shared_ptr<repository> r ( - db.load<repository> (ir.location.canonical_name ())); + shared_ptr<repository> r ( + db.load<repository> (ir.location.canonical_name ())); - load_repositories (r, db); - } - - session s; - using query = query<package>; + load_repositories (r, db); + } - // Resolve internal packages dependencies. - // - for (auto& p: - db.query<package> (query::internal_repository.is_not_null ())) - resolve_dependencies (p, db); + session s; + using query = query<package>; - // Ensure there is no package dependency cycles. - // - package_ids chain; - for (const auto& p: - db.query<package> (query::internal_repository.is_not_null ())) - detect_dependency_cycle (p.id, chain, db); - } + // Resolve internal packages dependencies. + // + for (auto& p: + db.query<package> (query::internal_repository.is_not_null ())) + resolve_dependencies (p, db); - t.commit (); - synch_t.commit (); // Release the mutex. - } - catch (const cli::exception& e) - { - cerr << e << endl; - usage (cerr); - return 1; - } - // Fully qualified to avoid ambiguity with odb exception. - // - catch (const std::exception& e) - { - cerr << e.what () << endl; - return 1; + // Ensure there is no package dependency cycles. + // + package_ids chain; + for (const auto& p: + db.query<package> (query::internal_repository.is_not_null ())) + detect_dependency_cycle (p.id, chain, db); } + + t.commit (); +} +catch (const database_locked&) +{ + cerr << "brep-load or brep-migrate instance is running" << endl; + return 2; +} +catch (const cli::exception& e) +{ + cerr << e << endl; + usage (cerr); + return 1; +} +// Fully qualified to avoid ambiguity with odb exception. +// +catch (const std::exception& e) +{ + cerr << e.what () << endl; + return 1; } diff --git a/loader/options.cli b/load/options.cli index 0899156..df91606 100644 --- a/loader/options.cli +++ b/load/options.cli @@ -1,14 +1,37 @@ -// file : loader/options.cli +// file : load/options.cli // copyright : Copyright (c) 2014-2016 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file include <string>; include <cstdint>; // uint16_t +"\section=1" +"\name=brep-load" +"\summary=load repositories into database" + +{ + "<options> <file>", + + "\h|SYNOPSIS| + + \cb{brep-load --help}\n + \cb{brep-load --version}\n + \c{\b{brep-load} [<options>] <file>} + + \h|DESCRIPTION| + + \cb{brep-load} reads the list of repositories from the specified + configuration <file>, fetches their manifest files, and loads the repository + and package information into the database, suitable for consumption by the + \cb{brep} web module. + + Note that \cb{brep-load} expects the database schema to have already been + created using \l{brep-migrate(1)}." +} + class options { - bool --help {"Print usage information and exit."} - bool --version {"Print version and exit."} + "\h|OPTIONS|" std::string --db-user|-u { @@ -43,4 +66,7 @@ class options "<port>", "Database port number. If not specified, the default port is used." } + + bool --help {"Print usage information and exit."} + bool --version {"Print version and exit."} }; diff --git a/migrate/.gitignore b/migrate/.gitignore new file mode 100644 index 0000000..580958d --- /dev/null +++ b/migrate/.gitignore @@ -0,0 +1,3 @@ +options +options.?xx +brep-migrate diff --git a/migrate/buildfile b/migrate/buildfile new file mode 100644 index 0000000..c42de5a --- /dev/null +++ b/migrate/buildfile @@ -0,0 +1,18 @@ +# file : migrate/buildfile +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libodb%lib{odb} + +include ../brep/ + +exe{brep-migrate}: \ +{ cxx}{ migrate } \ +{hxx ixx cxx}{ options } \ +../brep/lib{brep} $libs + +cli.options += -I $src_root --include-with-brackets --include-prefix migrate \ +--guard-prefix MIGRATE + +{hxx ixx cxx}{options}: cli{options} diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx new file mode 100644 index 0000000..aa71b67 --- /dev/null +++ b/migrate/migrate.cxx @@ -0,0 +1,317 @@ +// file : migrate/migrate.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <strings.h> // strcasecmp() + +#include <string> +#include <cassert> +#include <ostream> +#include <sstream> +#include <iostream> +#include <stdexcept> // runtime_error, invalid_argument + +#include <odb/database.hxx> +#include <odb/transaction.hxx> +#include <odb/schema-catalog.hxx> + +#include <odb/pgsql/database.hxx> + +#include <brep/types> +#include <brep/utility> +#include <brep/version> + +#include <brep/database-lock> + +#include <migrate/options> + +using namespace std; +using namespace odb::core; +using namespace brep; + +// Helper class that encapsulates both the ODB-generated schema and the +// extra that comes from a .sql file (via xxd). +// +class schema +{ +public: + explicit + schema (const char* extra); + + void + create (database&) const; + + void + drop (database&) const; + +private: + strings drop_statements_; + strings create_statements_; +}; + +schema:: +schema (const char* s) +{ + // Remove comments, saving the cleaned SQL code into statements. + // + string statements; + for (istringstream i (s); i; ) + { + // Skip leading spaces (including consequtive newlines). In case we + // hit eof, keep c set to '\n'. + // + char c; + static const string spaces (" \t\n\r"); + for (c = '\n'; i.get (c) && spaces.find (c) != string::npos; c = '\n') + ; + + // First non-space character (or '\n' for eof). See if this is a comment. + // + bool skip (c == '\n' || (c == '-' && i.peek () == '-')); + + // Read until newline (and don't forget the character we already have). + // + do + { + if (!skip) + statements.push_back (c); + + } while (c != '\n' && i.get (c)); + } + + istringstream i (move (statements)); + + // Build CREATE and DROP statement lists. + // + while (i) + { + string op; + if (i >> op) // Get the first word. + { + string statement (op); + + auto read_until = [&i, &statement](const char stop[2]) -> bool + { + for (char prev ('\0'), c; i.get (c); prev = c) + { + statement.push_back (c); + + if (stop[0] == prev && stop[1] == c) + return true; + } + + return false; + }; + + if (strcasecmp (op.c_str (), "CREATE") == 0) + { + string kw; + i >> kw; + statement += " " + kw; + + if (strcasecmp (kw.c_str (), "FUNCTION") == 0) + { + if (!read_until ("$$") || !read_until ("$$")) + throw invalid_argument ( + "function body must be defined using $$-quoted strings"); + } + else if (strcasecmp (kw.c_str (), "TYPE") == 0) + { + // Fall through. + } + else + throw invalid_argument ("unexpected CREATE statement"); + + if (!read_until (";\n")) + throw invalid_argument ( + "expected ';\\n' at the end of CREATE statement"); + + assert (!statement.empty ()); + create_statements_.emplace_back (move (statement)); + } + else if (strcasecmp (op.c_str (), "DROP") == 0) + { + if (!read_until (";\n")) + throw invalid_argument ( + "expected ';\\n' at the end of DROP statement"); + + assert (!statement.empty ()); + drop_statements_.emplace_back (move (statement)); + } + else + throw invalid_argument ( + "unexpected statement starting with '" + op + "'"); + } + } +} + +void schema:: +drop (database& db) const +{ + for (const auto& s: drop_statements_) + // If the statement execution fails, the corresponding source file line + // number is not reported. The line number could be usefull for the utility + // implementer only. The errors seen by the end-user should not be + // statement-specific. + // + db.execute (s); + + schema_catalog::drop_schema (db); +} + +void schema:: +create (database& db) const +{ + drop (db); + + schema_catalog::create_schema (db); + + for (const auto& s: create_statements_) + db.execute (s); +} + +// Utility functions +// +static void +usage (ostream& os) +{ + os << "Usage: brep-migrate [options]" << endl + << "Options:" << endl; + + options::print_usage (os); +} + +// main() function +// +int +main (int argc, char* argv[]) +try +{ + cli::argv_scanner scan (argc, argv, true); + options ops (scan); + + // Version. + // + if (ops.version ()) + { + cout << "brep-migrate " << BREP_VERSION_STR << endl + << "libbrep " << LIBBREP_VERSION_STR << endl + << "libbpkg " << LIBBPKG_VERSION_STR << endl + << "libbutl " << LIBBUTL_VERSION_STR << endl + << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl + << "MIT; see accompanying LICENSE file" << endl; + + return 0; + } + + // Help. + // + if (ops.help ()) + { + usage (cout); + return 0; + } + + if (argc > 1) + { + cerr << "unexpected argument encountered" << endl; + usage (cerr); + return 1; + } + + if (ops.recreate () && ops.drop ()) + { + cerr << "inconsistent options specified" << endl; + usage (cerr); + return 1; + } + + odb::pgsql::database db (ops.db_user (), + ops.db_password (), + ops.db_name (), + ops.db_host (), + ops.db_port ()); + + // Prevent several brep-migrate/load instances from updating DB + // simultaneously. + // + database_lock l (db); + + // Need to obtain schema version out of the transaction. If the + // schema_version table does not exist, the SQL query fails, which makes the + // transaction useless as all consequitive queries in that transaction will + // be ignored by PostgreSQL. + // + schema_version schema_version (db.schema_version ()); + + // It is impossible to operate with the database which is out of the + // [base_version, current_version] range due to the lack of the knowlege + // required not just for migration, but for the database wiping as well. + // + if (schema_version > 0) + { + if (schema_version < schema_catalog::base_version (db)) + throw runtime_error ("database schema is too old"); + + if (schema_version > schema_catalog::current_version (db)) + throw runtime_error ("database schema is too new"); + } + + bool drop (ops.drop ()); + bool create (ops.recreate () || (schema_version == 0 && !drop)); + assert (!create || !drop); + + // The database schema recreation requires dropping it initially, which is + // impossible before the database is migrated to the current schema version. + // Let the user decide if they want to migrate or just drop the entire + // database (followed with the database creation for the --recreate option). + // + if ((create || drop) && schema_version != 0 && + schema_version != schema_catalog::current_version (db)) + throw runtime_error ("database schema requires migration"); + + transaction t (db.begin ()); + + if (create || drop) + { + static const char extras[] = { +#include <brep/package-extra> + , '\0'}; + + schema s (extras); + + if (create) + s.create (db); + else if (drop) + s.drop (db); + } + else + { + // Register the data migration functions. + // + // static const data_migration_entry<2, LIBBREP_SCHEMA_VERSION_BASE> + // migrate_v2_entry (&migrate_v2); + // + schema_catalog::migrate (db); + } + + t.commit (); +} +catch (const database_locked&) +{ + cerr << "brep-migrate or brep-load instance is running" << endl; + return 2; +} +catch (const cli::exception& e) +{ + cerr << e << endl; + usage (cerr); + return 1; +} +// Fully qualified to avoid ambiguity with odb exception. +// +catch (const std::exception& e) +{ + cerr << e.what () << endl; + return 1; +} diff --git a/migrate/options.cli b/migrate/options.cli new file mode 100644 index 0000000..e36155c --- /dev/null +++ b/migrate/options.cli @@ -0,0 +1,90 @@ +// file : migrate/options.cli +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +include <string>; +include <cstdint>; // uint16_t + +"\section=1" +"\name=brep-migrate" +"\summary=create/drop/migrate brep database" + +{ + "<options>", + + "\h|SYNOPSIS| + + \cb{brep-migrate --help}\n + \cb{brep-migrate --version}\n + \c{\b{brep-migrate} [<options>]} + + \h|DESCRIPTION| + + In its default mode \cb{brep-migrate} creates the database schema if it + doesn't already exist. Otherwise, it migrates the existing schema and data + to the current version, if needed. + + If the \cb{--recreate} option is specified, then \cb{brep-migrate} instead + recreates the database schema. That is, it drops all the existing tables + (and their data) and then creates them from scratch. + + If the \cb{--drop} option is specified, then \cb{brep-migrate} drops all the + existing tables (and their data). + + The \cb{--recreate} and \cb{--drop} options are mutually exclusive. When + specified, they will cause \cb{brep-migrate} to fail if the database schema + requires migration. In this case you can either migrate the database first + or drop the entire database using, for example, \cb{psql(1)}." +} + +class options +{ + "\h|OPTIONS|" + + bool --recreate + { + "Recreate the database schema (all the existing data will be lost)." + } + + bool --drop + { + "Drop the database schema (all the existing data will be lost)." + } + + std::string --db-user|-u + { + "<user>", + "Database user name. If not specified, then operating system (login) + name is used." + } + + std::string --db-password + { + "<pass>", + "Database password. If not specified, then login without password is + expected to work." + } + + std::string --db-name|-n = "brep" + { + "<name>", + "Database name. If not specified, then '\cb{brep}' is used by default." + } + + std::string --db-host|-h + { + "<host>", + "Database host name, address, or socket. If not specified, then connect + to \cb{localhost} using the operating system-default mechanism + (Unix-domain socket, etc)." + } + + std::uint16_t --db-port|-p = 0 + { + "<port>", + "Database port number. If not specified, the default port is used." + } + + bool --help {"Print usage information and exit."} + bool --version {"Print version and exit."} +}; diff --git a/tests/buildfile b/tests/buildfile index ca46f59..2bdea00 100644 --- a/tests/buildfile +++ b/tests/buildfile @@ -2,6 +2,6 @@ # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -d = loader/ web/ +d = load/ web/ ./: $d include $d diff --git a/tests/loader/1/basics/packages b/tests/load/1/basics/packages index 86c20c1..86c20c1 100644 --- a/tests/loader/1/basics/packages +++ b/tests/load/1/basics/packages diff --git a/tests/loader/1/basics/repositories b/tests/load/1/basics/repositories index 57a1c7a..57a1c7a 100644 --- a/tests/loader/1/basics/repositories +++ b/tests/load/1/basics/repositories diff --git a/tests/loader/1/math/packages b/tests/load/1/math/packages index 7b81c5e..7b81c5e 100644 --- a/tests/loader/1/math/packages +++ b/tests/load/1/math/packages diff --git a/tests/loader/1/math/repositories b/tests/load/1/math/repositories index 20aa30d..20aa30d 100644 --- a/tests/loader/1/math/repositories +++ b/tests/load/1/math/repositories diff --git a/tests/loader/1/misc/packages b/tests/load/1/misc/packages index fec3780..fec3780 100644 --- a/tests/loader/1/misc/packages +++ b/tests/load/1/misc/packages diff --git a/tests/loader/1/misc/repositories b/tests/load/1/misc/repositories index 1a41290..1a41290 100644 --- a/tests/loader/1/misc/repositories +++ b/tests/load/1/misc/repositories diff --git a/tests/loader/1/stable/packages b/tests/load/1/stable/packages index afa168a..afa168a 100644 --- a/tests/loader/1/stable/packages +++ b/tests/load/1/stable/packages diff --git a/tests/loader/1/stable/repositories b/tests/load/1/stable/repositories index b692ebe..b692ebe 100644 --- a/tests/loader/1/stable/repositories +++ b/tests/load/1/stable/repositories diff --git a/tests/loader/1/staging/packages b/tests/load/1/staging/packages index e7b22b0..e7b22b0 100644 --- a/tests/loader/1/staging/packages +++ b/tests/load/1/staging/packages diff --git a/tests/loader/1/staging/repositories b/tests/load/1/staging/repositories index d72a3f8..d72a3f8 100644 --- a/tests/loader/1/staging/repositories +++ b/tests/load/1/staging/repositories diff --git a/tests/loader/1/testing/packages b/tests/load/1/testing/packages index bdebece..bdebece 100644 --- a/tests/loader/1/testing/packages +++ b/tests/load/1/testing/packages diff --git a/tests/loader/1/testing/repositories b/tests/load/1/testing/repositories index a218d5c..a218d5c 100644 --- a/tests/loader/1/testing/repositories +++ b/tests/load/1/testing/repositories diff --git a/tests/loader/buildfile b/tests/load/buildfile index d8ab9ac..e68fd1f 100644 --- a/tests/loader/buildfile +++ b/tests/load/buildfile @@ -1,4 +1,4 @@ -# file : tests/loader/buildfile +# file : tests/load/buildfile # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file @@ -18,4 +18,4 @@ exe{driver}: test = false # precondition: PostgreSQL server running port 8432 with brep schema created. # test: -# ./driver ../../loader/brep-loader --db-host localhost --db-port 8432 ./r.conf +# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./r.conf diff --git a/tests/loader/driver.cxx b/tests/load/driver.cxx index 67093d9..7a70ff5 100644 --- a/tests/loader/driver.cxx +++ b/tests/load/driver.cxx @@ -1,4 +1,4 @@ -// file : tests/loader/driver.cxx -*- C++ -*- +// file : tests/load/driver.cxx -*- C++ -*- // copyright : Copyright (c) 2014-2016 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file diff --git a/tests/loader/r.conf b/tests/load/r.conf index 8da4b77..8da4b77 100644 --- a/tests/loader/r.conf +++ b/tests/load/r.conf diff --git a/web/apache/service.cxx b/web/apache/service.cxx index c4fe462..96ff855 100644 --- a/web/apache/service.cxx +++ b/web/apache/service.cxx @@ -125,7 +125,7 @@ namespace web // Terminate the root apache process. Indeed we can only try to // terminate the process, and most likely will fail in a production - // environment where the apache root process usually runs under root + // environment where the apache root process usually runs under root, // and worker processes run under some other user. This is why the // implementation should consider the possibility of not being // initialized at the time of HTTP request processing. In such a case |