aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2016-01-18 07:35:12 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2016-01-23 17:47:47 +0200
commit1dc38cf49b6c7a8b661a9cc675ded94c8ab33c36 (patch)
tree5a216148adb9d842a5a15c032a671182faa9ba06
parentfe6182a8c89675f92e72c881d707e21cdf56f376 (diff)
Implement brep-migrate utility
-rw-r--r--INSTALL26
-rw-r--r--INSTALL-DEV8
-rw-r--r--brep/.gitignore2
-rw-r--r--brep/buildfile4
-rw-r--r--brep/database-lock43
-rw-r--r--brep/database-lock.cxx44
-rw-r--r--brep/mod-package-search.cxx16
-rwxr-xr-xbrep/odb.sh18
-rw-r--r--brep/package6
-rw-r--r--brep/package-extra.sql20
-rw-r--r--brep/package.xml406
-rw-r--r--buildfile2
-rw-r--r--etc/buildfile2
-rw-r--r--etc/systemd/brep-load.service (renamed from etc/systemd/brep-loader.service)2
-rw-r--r--etc/systemd/brep-load.timer (renamed from etc/systemd/brep-loader.timer)2
-rw-r--r--load/.gitignore (renamed from loader/.gitignore)2
-rw-r--r--load/buildfile (renamed from loader/buildfile)10
-rw-r--r--load/load.cxx (renamed from loader/loader.cxx)253
-rw-r--r--load/options.cli (renamed from loader/options.cli)32
-rw-r--r--migrate/.gitignore3
-rw-r--r--migrate/buildfile18
-rw-r--r--migrate/migrate.cxx317
-rw-r--r--migrate/options.cli90
-rw-r--r--tests/buildfile2
-rw-r--r--tests/load/1/basics/packages (renamed from tests/loader/1/basics/packages)0
-rw-r--r--tests/load/1/basics/repositories (renamed from tests/loader/1/basics/repositories)0
-rw-r--r--tests/load/1/math/packages (renamed from tests/loader/1/math/packages)0
-rw-r--r--tests/load/1/math/repositories (renamed from tests/loader/1/math/repositories)0
-rw-r--r--tests/load/1/misc/packages (renamed from tests/loader/1/misc/packages)0
-rw-r--r--tests/load/1/misc/repositories (renamed from tests/loader/1/misc/repositories)0
-rw-r--r--tests/load/1/stable/packages (renamed from tests/loader/1/stable/packages)0
-rw-r--r--tests/load/1/stable/repositories (renamed from tests/loader/1/stable/repositories)0
-rw-r--r--tests/load/1/staging/packages (renamed from tests/loader/1/staging/packages)0
-rw-r--r--tests/load/1/staging/repositories (renamed from tests/loader/1/staging/repositories)0
-rw-r--r--tests/load/1/testing/packages (renamed from tests/loader/1/testing/packages)0
-rw-r--r--tests/load/1/testing/repositories (renamed from tests/loader/1/testing/repositories)0
-rw-r--r--tests/load/buildfile (renamed from tests/loader/buildfile)4
-rw-r--r--tests/load/driver.cxx (renamed from tests/loader/driver.cxx)2
-rw-r--r--tests/load/r.conf (renamed from tests/loader/r.conf)0
-rw-r--r--web/apache/service.cxx2
40 files changed, 1153 insertions, 183 deletions
diff --git a/INSTALL b/INSTALL
index 1944228..e37c806 100644
--- a/INSTALL
+++ b/INSTALL
@@ -122,10 +122,10 @@ To troubleshoot, see PostgreSQL logs.
5. Create Database Schema and Load Repositories
$ mkdir config
-$ edit config/brep-loader.conf # Loader configuration, see brep-loader(1).
+$ edit config/brep-load.conf # Loader configuration, see brep-load(1).
-$ psql --quiet -f install/share/brep/package.sql
-$ install/bin/brep-loader config/brep-loader.conf
+$ install/bin/brep-migrate
+$ install/bin/brep-load config/brep-load.conf
To verify:
@@ -226,21 +226,21 @@ since we want the loader to run even when we are not logged in:
$ sudo loginctl enable-linger brep
$ mkdir -p .config/systemd/user
-$ cp install/share/brep/etc/systemd/brep-loader.* .config/systemd/user/
+$ cp install/share/brep/etc/systemd/brep-load.* .config/systemd/user/
Start the service to make sure there are no issues:
-$ systemctl --user start brep-loader.service
+$ systemctl --user start brep-load.service
$ journalctl
Start the timer and monitor it to make sure it fires:
-$ systemctl --user start brep-loader.timer
+$ systemctl --user start brep-load.timer
$ journalctl -f
If everything looks good, enable the timer to be started at boot time:
-$ systemctl --user enable brep-loader.timer
+$ systemctl --user enable brep-load.timer
8. Upgrade Procedure
@@ -262,8 +262,8 @@ $ bpkg -d brep build brep
Stop and disable loader:
-$ systemctl --user disable --now brep-loader.timer
-$ systemctl --user stop brep-loader.service
+$ systemctl --user disable --now brep-load.timer
+$ systemctl --user stop brep-load.service
Stop apache:
@@ -280,16 +280,16 @@ $ diff -u install/share/brep/etc/brep-module.conf config/brep-module.conf
Update database schema:
-$ psql --quiet -f install/share/brep/package.sql
+$ install/bin/brep-migrate
Start and enable loader:
-$ systemctl --user start brep-loader.service
-$ systemctl --user status brep-loader.service
+$ systemctl --user start brep-load.service
+$ systemctl --user status brep-load.service
If everything looks good, enable periodic execution:
-$ systemctl --user enable --now brep-loader.timer
+$ systemctl --user enable --now brep-load.timer
Start apache:
diff --git a/INSTALL-DEV b/INSTALL-DEV
index b81d7fb..6e23664 100644
--- a/INSTALL-DEV
+++ b/INSTALL-DEV
@@ -48,8 +48,8 @@ $ sudo tail -f /var/log/postgresql/*.log
All the commands are executed from brep project root.
-$ psql --quiet -d brep -f brep/package.sql
-$ loader/brep-loader tests/loader/r.conf # Or some other loader config.
+$ migrate/brep-migrate
+$ load/brep-load tests/load/r.conf # Or some other loader config.
To verify:
@@ -115,7 +115,7 @@ $ sudo tail -f /var/log/apache2/error.log
To do a "complete reload" (i.e., recreate database schema, load the repository
data, and reload the Apache2 plugin), execute the following from brep/:
-psql --quiet -d brep -f brep/package.sql
-loader/brep-loader tests/loader/r.conf
+migrate/brep-migrate --recreate
+load/brep-load tests/load/r.conf
sudo /etc/init.d/apache2 restart
sudo systemctl restart apache2
diff --git a/brep/.gitignore b/brep/.gitignore
index 2c23e23..852a40d 100644
--- a/brep/.gitignore
+++ b/brep/.gitignore
@@ -2,4 +2,4 @@ options
options.?xx
package-odb*
package.sql
-
+package-extra
diff --git a/brep/buildfile b/brep/buildfile
index 9826504..1aa4d08 100644
--- a/brep/buildfile
+++ b/brep/buildfile
@@ -10,7 +10,7 @@ define sql: file
sql{*}: extension = sql
sql{*}: install = data
-./: lib{brep} mod{brep} sql{package}
+./: lib{brep} mod{brep} sql{package package-extra}
# lib{brep}
#
@@ -20,8 +20,10 @@ import libs += libbpkg%lib{bpkg}
lib{brep}: \
{hxx cxx}{ package } \
+{file }{ package.xml } \
{hxx ixx cxx}{ package-odb } \
{hxx cxx}{ package-traits } \
+{hxx cxx}{ database-lock } \
{hxx }{ types } \
{hxx }{ utility } \
{hxx }{ version } \
diff --git a/brep/database-lock b/brep/database-lock
new file mode 100644
index 0000000..72036a1
--- /dev/null
+++ b/brep/database-lock
@@ -0,0 +1,43 @@
+// file : brep/database-lock -*- C++ -*-
+// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef BREP_DATABASE_LOCK
+#define BREP_DATABASE_LOCK
+
+#include <memory> // unique_ptr
+#include <exception>
+
+#include <odb/pgsql/forward.hxx> // database, transaction
+#include <odb/pgsql/connection.hxx>
+
+namespace brep
+{
+ struct database_locked: std::exception
+ {
+ virtual char const*
+ what () const throw () {return "database locked";}
+ };
+
+ // Try to "lock" the PostgreSQL database in the constructor and release the
+ // lock in the destructor. Throw database_locked if the database is already
+ // locked by someone else. May also throw odb::pgsql::database_exception.
+ //
+ // This mechanism is used by the brep loader and schema migration tool to
+ // make sure they don't step on each others toes.
+ //
+ // Note: movable but not copyable.
+ //
+ class database_lock
+ {
+ public:
+ explicit
+ database_lock (odb::pgsql::database&);
+
+ private:
+ odb::pgsql::connection_ptr connection_;
+ std::unique_ptr<odb::pgsql::transaction> transaction_;
+ };
+}
+
+#endif // BREP_DATABASE_LOCK
diff --git a/brep/database-lock.cxx b/brep/database-lock.cxx
new file mode 100644
index 0000000..3b8ae21
--- /dev/null
+++ b/brep/database-lock.cxx
@@ -0,0 +1,44 @@
+// file : brep/database-lock.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <brep/database-lock>
+
+#include <odb/pgsql/database.hxx>
+#include <odb/pgsql/exceptions.hxx>
+#include <odb/pgsql/transaction.hxx>
+
+namespace brep
+{
+ using namespace odb::pgsql;
+
+ database_lock::
+ database_lock (database& db)
+ {
+ // Before locking the table make sure it exists.
+ //
+ {
+ transaction t (db.begin ());
+ db.execute ("CREATE TABLE IF NOT EXISTS database_mutex ()");
+ t.commit ();
+ }
+
+ connection_ = db.connection ();
+
+ // Don't make current. Will be rolled back in destructor.
+ //
+ transaction_.reset (new transaction (connection_->begin (), false));
+
+ try
+ {
+ connection_->execute ("LOCK TABLE database_mutex NOWAIT");
+ }
+ catch (const database_exception& e)
+ {
+ if (e.sqlstate () == "55P03") // The table is already locked.
+ throw database_locked ();
+
+ throw;
+ }
+ }
+}
diff --git a/brep/mod-package-search.cxx b/brep/mod-package-search.cxx
index 4326435..d649ff4 100644
--- a/brep/mod-package-search.cxx
+++ b/brep/mod-package-search.cxx
@@ -9,6 +9,7 @@
#include <odb/session.hxx>
#include <odb/database.hxx>
#include <odb/transaction.hxx>
+#include <odb/schema-catalog.hxx>
#include <web/xhtml>
#include <web/module>
@@ -16,6 +17,7 @@
#include <brep/types>
#include <brep/utility>
+#include <brep/version>
#include <brep/page>
#include <brep/options>
@@ -38,6 +40,20 @@ init (scanner& s)
options_->root (dir_path ("/"));
db_ = shared_database (*options_);
+
+ // Check that the database schema matches the current one. It's enough to
+ // perform the check in just a single module implementation (and we don't
+ // do in the dispatcher because it doesn't use the database).
+ //
+ // Note that the failure can be reported by each web server worker process.
+ // While it could be tempting to move the check to the
+ // repository_root::version() function, it would be wrong. The function can
+ // be called by a different process (usually the web server root one) not
+ // having the proper permissions to access the database.
+ //
+ if (schema_catalog::current_version (*db_) != db_->schema_version ())
+ fail << "database schema differs from the current one (module "
+ << BREP_VERSION_STR << ")";
}
template <typename T>
diff --git a/brep/odb.sh b/brep/odb.sh
index 489d423..d68b28f 100755
--- a/brep/odb.sh
+++ b/brep/odb.sh
@@ -1,11 +1,13 @@
#! /usr/bin/env bash
odb -d pgsql --std c++11 --generate-query --generate-schema \
- --odb-epilogue '#include <brep/wrapper-traits>' \
- --hxx-prologue '#include <brep/wrapper-traits>' \
- --hxx-prologue "#include <brep/package-traits>" \
- --sql-epilogue-file package-extra.sql \
- -I .. -I ../../libbpkg -I ../../libbutl \
- --hxx-suffix "" --include-with-brackets \
- --include-prefix brep --guard-prefix BREP \
- package
+ --schema-format sql --schema-format embedded \
+ --odb-epilogue '#include <brep/wrapper-traits>' \
+ --hxx-prologue '#include <brep/wrapper-traits>' \
+ --hxx-prologue '#include <brep/package-traits>' \
+ -I .. -I ../../libbpkg -I ../../libbutl \
+ --hxx-suffix "" --include-with-brackets \
+ --include-prefix brep --guard-prefix BREP \
+ package
+
+xxd -i <package-extra.sql >package-extra
diff --git a/brep/package b/brep/package
index 92a2320..2700e8a 100644
--- a/brep/package
+++ b/brep/package
@@ -19,6 +19,12 @@
#include <brep/types>
#include <brep/utility>
+// Used by the data migration entries.
+//
+#define LIBBREP_SCHEMA_VERSION_BASE 1
+
+#pragma db model version(LIBBREP_SCHEMA_VERSION_BASE, 1, open)
+
// The uint16_t value range is not fully covered by SMALLINT PostgreSQL type
// to which uint16_t is mapped by default.
//
diff --git a/brep/package-extra.sql b/brep/package-extra.sql
index dc37d1f..9a847e3 100644
--- a/brep/package-extra.sql
+++ b/brep/package-extra.sql
@@ -1,12 +1,22 @@
-DROP TYPE IF EXISTS weighted_text CASCADE;
-CREATE TYPE weighted_text AS (a TEXT, b TEXT, c TEXT, d TEXT);
-
+-- This file should be parsable by the brep-migrate utility. To decrease the
+-- parser complexity, the following restrictions are placed:
+--
+-- * comments must start with -- at the beginning of the line (ignoring
+-- leading spaces)
+-- * only CREATE and DROP statements for FUNCTION and TYPE
+-- * function bodies must be defined using $$-quoted strings
+-- * strings other then function bodies must be quoted with ' or "
+-- * statements must end with ";\n"
+--
DROP FUNCTION IF EXISTS to_tsvector(IN document weighted_text);
DROP FUNCTION IF EXISTS search_packages(IN query tsquery, INOUT name TEXT);
DROP FUNCTION IF EXISTS search_latest_packages(IN query tsquery);
DROP FUNCTION IF EXISTS latest_package(INOUT name TEXT);
DROP FUNCTION IF EXISTS latest_packages();
+DROP TYPE IF EXISTS weighted_text CASCADE;
+CREATE TYPE weighted_text AS (a TEXT, b TEXT, c TEXT, d TEXT);
+
-- Return the latest versions of internal packages as a set of package rows.
--
CREATE FUNCTION
@@ -72,9 +82,9 @@ $$ LANGUAGE SQL STABLE;
-- Search for packages matching the search query and having the specified name.
-- Return a set of rows containing the package id and search rank. If query
-- is NULL, then match all packages and return 0 rank for all rows.
---
+--
CREATE FUNCTION
-search_packages(IN query tsquery,
+search_packages(IN query tsquery,
INOUT name TEXT,
OUT version_epoch INTEGER,
OUT version_canonical_upstream TEXT,
diff --git a/brep/package.xml b/brep/package.xml
new file mode 100644
index 0000000..9c88f25
--- /dev/null
+++ b/brep/package.xml
@@ -0,0 +1,406 @@
+<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" version="1">
+ <model version="1">
+ <table name="repository" kind="object">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="location" type="TEXT" null="false"/>
+ <column name="display_name" type="TEXT" null="false"/>
+ <column name="priority" type="INTEGER" null="false"/>
+ <column name="url" type="TEXT" null="true"/>
+ <column name="email" type="TEXT" null="true"/>
+ <column name="summary" type="TEXT" null="true"/>
+ <column name="description" type="TEXT" null="true"/>
+ <column name="local_path" type="TEXT" null="false"/>
+ <column name="packages_timestamp" type="BIGINT" null="false"/>
+ <column name="repositories_timestamp" type="BIGINT" null="false"/>
+ <column name="internal" type="BOOLEAN" null="false"/>
+ <primary-key>
+ <column name="name"/>
+ </primary-key>
+ </table>
+ <table name="repository_complements" kind="container">
+ <column name="repository" type="TEXT" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="complement" type="TEXT" null="false"/>
+ <foreign-key name="repository_fk" on-delete="CASCADE">
+ <column name="repository"/>
+ <references table="repository">
+ <column name="name"/>
+ </references>
+ </foreign-key>
+ <index name="repository_complements_repository_i">
+ <column name="repository"/>
+ </index>
+ <index name="repository_complements_index_i">
+ <column name="index"/>
+ </index>
+ <foreign-key name="complement_fk" deferrable="DEFERRED">
+ <column name="complement"/>
+ <references table="repository">
+ <column name="name"/>
+ </references>
+ </foreign-key>
+ </table>
+ <table name="repository_prerequisites" kind="container">
+ <column name="repository" type="TEXT" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="prerequisite" type="TEXT" null="false"/>
+ <foreign-key name="repository_fk" on-delete="CASCADE">
+ <column name="repository"/>
+ <references table="repository">
+ <column name="name"/>
+ </references>
+ </foreign-key>
+ <index name="repository_prerequisites_repository_i">
+ <column name="repository"/>
+ </index>
+ <index name="repository_prerequisites_index_i">
+ <column name="index"/>
+ </index>
+ <foreign-key name="prerequisite_fk" deferrable="DEFERRED">
+ <column name="prerequisite"/>
+ <references table="repository">
+ <column name="name"/>
+ </references>
+ </foreign-key>
+ </table>
+ <table name="package" kind="object">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="version_upstream" type="TEXT" null="false"/>
+ <column name="version_release" type="TEXT" null="true"/>
+ <column name="priority" type="INTEGER" null="false"/>
+ <column name="priority_comment" type="TEXT" null="false"/>
+ <column name="summary" type="TEXT" null="false"/>
+ <column name="description" type="TEXT" null="true"/>
+ <column name="changes" type="TEXT" null="false"/>
+ <column name="url" type="TEXT" null="false"/>
+ <column name="url_comment" type="TEXT" null="false"/>
+ <column name="package_url" type="TEXT" null="true"/>
+ <column name="package_url_comment" type="TEXT" null="true"/>
+ <column name="email" type="TEXT" null="false"/>
+ <column name="email_comment" type="TEXT" null="false"/>
+ <column name="package_email" type="TEXT" null="true"/>
+ <column name="package_email_comment" type="TEXT" null="true"/>
+ <column name="internal_repository" type="TEXT" null="true"/>
+ <column name="location" type="TEXT" null="true"/>
+ <column name="search_index" type="tsvector" null="true"/>
+ <primary-key>
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </primary-key>
+ <foreign-key name="internal_repository_fk" deferrable="DEFERRED">
+ <column name="internal_repository"/>
+ <references table="repository">
+ <column name="name"/>
+ </references>
+ </foreign-key>
+ <index name="package_search_index_i" method="GIN">
+ <column name="search_index"/>
+ </index>
+ </table>
+ <table name="package_license_alternatives" kind="container">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="comment" type="TEXT" null="false"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_license_alternatives_object_id_i">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ <index name="package_license_alternatives_index_i">
+ <column name="index"/>
+ </index>
+ </table>
+ <table name="package_licenses" kind="container">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="alternative_index" type="BIGINT" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="license" type="TEXT" null="false"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_licenses_object_id_i">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ </table>
+ <table name="package_tags" kind="container">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="tag" type="TEXT" null="false"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_tags_object_id_i">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ <index name="package_tags_index_i">
+ <column name="index"/>
+ </index>
+ </table>
+ <table name="package_dependencies" kind="container">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="conditional" type="BOOLEAN" null="false"/>
+ <column name="comment" type="TEXT" null="false"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_dependencies_object_id_i">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ <index name="package_dependencies_index_i">
+ <column name="index"/>
+ </index>
+ </table>
+ <table name="package_dependency_alternatives" kind="container">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="dependency_index" type="BIGINT" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="dep_name" type="TEXT" null="false"/>
+ <column name="dep_version_epoch" type="INTEGER" null="false"/>
+ <column name="dep_version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="dep_version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="dep_version_revision" type="INTEGER" null="false"/>
+ <column name="dep_min_version_epoch" type="INTEGER" null="true"/>
+ <column name="dep_min_version_canonical_upstream" type="TEXT" null="true"/>
+ <column name="dep_min_version_canonical_release" type="TEXT" null="true"/>
+ <column name="dep_min_version_revision" type="INTEGER" null="true"/>
+ <column name="dep_min_version_upstream" type="TEXT" null="true"/>
+ <column name="dep_min_version_release" type="TEXT" null="true"/>
+ <column name="dep_max_version_epoch" type="INTEGER" null="true"/>
+ <column name="dep_max_version_canonical_upstream" type="TEXT" null="true"/>
+ <column name="dep_max_version_canonical_release" type="TEXT" null="true"/>
+ <column name="dep_max_version_revision" type="INTEGER" null="true"/>
+ <column name="dep_max_version_upstream" type="TEXT" null="true"/>
+ <column name="dep_max_version_release" type="TEXT" null="true"/>
+ <column name="dep_min_open" type="BOOLEAN" null="true"/>
+ <column name="dep_max_open" type="BOOLEAN" null="true"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_dependency_alternatives_object_id_i">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ <foreign-key name="dep_package_fk" deferrable="DEFERRED">
+ <column name="dep_name"/>
+ <column name="dep_version_epoch"/>
+ <column name="dep_version_canonical_upstream"/>
+ <column name="dep_version_canonical_release"/>
+ <column name="dep_version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ </table>
+ <table name="package_requirements" kind="container">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="conditional" type="BOOLEAN" null="false"/>
+ <column name="comment" type="TEXT" null="false"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_requirements_object_id_i">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ <index name="package_requirements_index_i">
+ <column name="index"/>
+ </index>
+ </table>
+ <table name="package_requirement_alternatives" kind="container">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="requirement_index" type="BIGINT" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="id" type="TEXT" null="false"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_requirement_alternatives_object_id_i">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ </table>
+ <table name="package_other_repositories" kind="container">
+ <column name="name" type="TEXT" null="false"/>
+ <column name="version_epoch" type="INTEGER" null="false"/>
+ <column name="version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="version_revision" type="INTEGER" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="repository" type="TEXT" null="false"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ <references table="package">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </references>
+ </foreign-key>
+ <index name="package_other_repositories_object_id_i">
+ <column name="name"/>
+ <column name="version_epoch"/>
+ <column name="version_canonical_upstream"/>
+ <column name="version_canonical_release"/>
+ <column name="version_revision"/>
+ </index>
+ <index name="package_other_repositories_index_i">
+ <column name="index"/>
+ </index>
+ <foreign-key name="repository_fk" deferrable="DEFERRED">
+ <column name="repository"/>
+ <references table="repository">
+ <column name="name"/>
+ </references>
+ </foreign-key>
+ </table>
+ </model>
+</changelog>
diff --git a/buildfile b/buildfile
index 88a8c0b..60318fe 100644
--- a/buildfile
+++ b/buildfile
@@ -2,7 +2,7 @@
# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-d = brep/ etc/ loader/ tests/ www/
+d = brep/ etc/ load/ migrate/ tests/ www/
./: $d doc{INSTALL INSTALL-DEV LICENSE version} file{manifest}
include $d
diff --git a/etc/buildfile b/etc/buildfile
index 9ebcf27..4910076 100644
--- a/etc/buildfile
+++ b/etc/buildfile
@@ -3,7 +3,7 @@
# license : MIT; see accompanying LICENSE file
./: file{brep-module.conf brep-apache2.conf} \
-systemd/file{brep-loader.service brep-loader.timer}
+systemd/file{brep-load.service brep-load.timer}
install = data/etc
systemd/: install = data/etc/systemd
diff --git a/etc/systemd/brep-loader.service b/etc/systemd/brep-load.service
index dec7944..34a7c9a 100644
--- a/etc/systemd/brep-loader.service
+++ b/etc/systemd/brep-load.service
@@ -3,7 +3,7 @@ Description=brep repository loader service
[Service]
Type=oneshot
-ExecStart=/home/brep/install/bin/brep-loader /home/brep/config/brep-loader.conf
+ExecStart=/home/brep/install/bin/brep-load /home/brep/config/brep-load.conf
[Install]
WantedBy=default.target
diff --git a/etc/systemd/brep-loader.timer b/etc/systemd/brep-load.timer
index 22ff22b..713cb31 100644
--- a/etc/systemd/brep-loader.timer
+++ b/etc/systemd/brep-load.timer
@@ -4,7 +4,7 @@ RefuseManualStart=no
RefuseManualStop=no
[Timer]
-Unit=brep-loader.service
+Unit=brep-load.service
# Don't keep track of the timer across reboots.
#
diff --git a/loader/.gitignore b/load/.gitignore
index 820b183..2f464c5 100644
--- a/loader/.gitignore
+++ b/load/.gitignore
@@ -1,3 +1,3 @@
options
options.?xx
-brep-loader
+brep-load
diff --git a/loader/buildfile b/load/buildfile
index 477cebd..c3a324f 100644
--- a/loader/buildfile
+++ b/load/buildfile
@@ -1,4 +1,4 @@
-# file : loader/buildfile
+# file : load/buildfile
# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
@@ -9,12 +9,12 @@ import libs += libodb%lib{odb}
include ../brep/
-exe{brep-loader}: \
-{ cxx}{ loader } \
+exe{brep-load}: \
+{ cxx}{ load } \
{hxx ixx cxx}{ options } \
../brep/lib{brep} $libs
-cli.options += -I $src_root --include-with-brackets --include-prefix loader \
---guard-prefix LOADER
+cli.options += -I $src_root --include-with-brackets --include-prefix load \
+--guard-prefix LOAD
{hxx ixx cxx}{options}: cli{options}
diff --git a/loader/loader.cxx b/load/load.cxx
index b1b7706..9e6ee5d 100644
--- a/loader/loader.cxx
+++ b/load/load.cxx
@@ -1,9 +1,10 @@
-// file : loader/loader.cxx -*- C++ -*-
+// file : load/load.cxx -*- C++ -*-
// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
// license : MIT; see accompanying LICENSE file
#include <sstream>
#include <fstream>
+#include <ostream>
#include <iostream>
#include <stdexcept> // runtime_error, invalid_argument
#include <algorithm> // find(), find_if()
@@ -11,11 +12,9 @@
#include <odb/session.hxx>
#include <odb/database.hxx>
#include <odb/transaction.hxx>
+#include <odb/schema-catalog.hxx>
#include <odb/pgsql/database.hxx>
-#include <odb/pgsql/exceptions.hxx>
-#include <odb/pgsql/connection.hxx>
-#include <odb/pgsql/transaction.hxx>
#include <butl/filesystem>
@@ -27,8 +26,9 @@
#include <brep/package>
#include <brep/package-odb>
+#include <brep/database-lock>
-#include <loader/options>
+#include <load/options>
using namespace std;
using namespace odb::core;
@@ -36,12 +36,10 @@ using namespace butl;
using namespace bpkg;
using namespace brep;
-namespace pgsql = odb::pgsql;
-
static void
usage (ostream& os)
{
- os << "Usage: brep-loader [options] <file>" << endl
+ os << "Usage: brep-load [options] <file>" << endl
<< "File lists internal repositories." << endl
<< "Options:" << endl;
@@ -700,152 +698,141 @@ detect_dependency_cycle (const package_id& id, package_ids& chain, database& db)
int
main (int argc, char* argv[])
+try
{
- try
+ cli::argv_scanner scan (argc, argv, true);
+ options ops (scan);
+
+ // Version.
+ //
+ if (ops.version ())
{
- cli::argv_scanner scan (argc, argv, true);
- options ops (scan);
+ cout << "brep-load " << BREP_VERSION_STR << endl
+ << "libbrep " << LIBBREP_VERSION_STR << endl
+ << "libbpkg " << LIBBPKG_VERSION_STR << endl
+ << "libbutl " << LIBBUTL_VERSION_STR << endl
+ << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl
+ << "MIT; see accompanying LICENSE file" << endl;
+
+ return 0;
+ }
- // Version.
- //
- if (ops.version ())
- {
- cout << "brep-loader " << BREP_VERSION_STR << endl
- << "libbrep " << LIBBREP_VERSION_STR << endl
- << "libbpkg " << LIBBPKG_VERSION_STR << endl
- << "libbutl " << LIBBUTL_VERSION_STR << endl
- << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl
- << "MIT; see accompanying LICENSE file" << endl;
-
- return 0;
- }
+ // Help.
+ //
+ if (ops.help ())
+ {
+ usage (cout);
+ return 0;
+ }
- // Help.
- //
- if (ops.help ())
- {
- usage (cout);
- return 0;
- }
+ if (argc < 2)
+ {
+ cerr << "<file> argument not provided" << endl;
+ usage (cerr);
+ return 1;
+ }
- if (argc < 2)
- {
- cerr << "<file> argument not provided" << endl;
- usage (cerr);
- return 1;
- }
+ if (argc > 2)
+ {
+ cerr << "unexpected argument encountered" << endl;
+ usage (cerr);
+ return 1;
+ }
- if (argc > 2)
- {
- cerr << "unexpected argument encountered" << endl;
- usage (cerr);
- return 1;
- }
+ odb::pgsql::database db (ops.db_user (),
+ ops.db_password (),
+ ops.db_name (),
+ ops.db_host (),
+ ops.db_port ());
- pgsql::database db (ops.db_user (),
- ops.db_password (),
- ops.db_name (),
- ops.db_host (),
- ops.db_port ());
+ // Prevent several brep-load/migrate instances from updating DB
+ // simultaneously.
+ //
+ database_lock l (db);
- // Prevent several loader instances from updating DB simultaneously.
- //
- {
- transaction t (db.begin ());
- db.execute ("CREATE TABLE IF NOT EXISTS loader_mutex ()");
- t.commit ();
- }
+ transaction t (db.begin ());
- pgsql::connection_ptr synch_c (db.connection ());
+ // Check that the database schema matches the current one.
+ //
+ if (schema_catalog::current_version (db) != db.schema_version ())
+ {
+ cerr << "database schema differs from the current one" << endl;
+ return 1;
+ }
+
+ // Load the description of all the internal repositories from the
+ // configuration file.
+ //
+ internal_repositories irs (load_repositories (path (argv[1])));
- // Don't make current.
+ if (changed (irs, db))
+ {
+ // Rebuild repositories persistent state from scratch.
//
- pgsql::transaction synch_t (synch_c->begin (), false);
+ db.erase_query<package> ();
+ db.erase_query<repository> ();
- try
- {
- synch_c->execute ("LOCK TABLE loader_mutex NOWAIT");
- }
- catch (const pgsql::database_exception& e)
+ // On the first pass over the internal repositories we load their
+ // packages.
+ //
+ uint16_t priority (1);
+ for (const auto& ir: irs)
{
- if (e.sqlstate () == "55P03")
- return 2; // Other loader instance acquired the mutex.
+ shared_ptr<repository> r (
+ make_shared<repository> (ir.location,
+ move (ir.display_name),
+ move (ir.local_path),
+ priority++));
- throw;
+ load_packages (r, db);
}
- // Load the description of all the internal repositories from the
- // configuration file.
+ // On the second pass over the internal repositories we load their
+ // (not yet loaded) manifest values, complement, and prerequisite
+ // repositories.
//
- internal_repositories irs (load_repositories (path (argv[1])));
-
- transaction t (db.begin ());
-
- if (changed (irs, db))
+ for (const auto& ir: irs)
{
- // Rebuild repositories persistent state from scratch.
- //
- db.erase_query<package> ();
- db.erase_query<repository> ();
-
- // On the first pass over the internal repositories we load their
- // packages.
- //
- uint16_t priority (1);
- for (const auto& ir: irs)
- {
- shared_ptr<repository> r (
- make_shared<repository> (ir.location,
- move (ir.display_name),
- move (ir.local_path),
- priority++));
-
- load_packages (r, db);
- }
-
- // On the second pass over the internal repositories we load their
- // (not yet loaded) manifest values, complement, and prerequisite
- // repositories.
- //
- for (const auto& ir: irs)
- {
- shared_ptr<repository> r (
- db.load<repository> (ir.location.canonical_name ()));
+ shared_ptr<repository> r (
+ db.load<repository> (ir.location.canonical_name ()));
- load_repositories (r, db);
- }
-
- session s;
- using query = query<package>;
+ load_repositories (r, db);
+ }
- // Resolve internal packages dependencies.
- //
- for (auto& p:
- db.query<package> (query::internal_repository.is_not_null ()))
- resolve_dependencies (p, db);
+ session s;
+ using query = query<package>;
- // Ensure there is no package dependency cycles.
- //
- package_ids chain;
- for (const auto& p:
- db.query<package> (query::internal_repository.is_not_null ()))
- detect_dependency_cycle (p.id, chain, db);
- }
+ // Resolve internal packages dependencies.
+ //
+ for (auto& p:
+ db.query<package> (query::internal_repository.is_not_null ()))
+ resolve_dependencies (p, db);
- t.commit ();
- synch_t.commit (); // Release the mutex.
- }
- catch (const cli::exception& e)
- {
- cerr << e << endl;
- usage (cerr);
- return 1;
- }
- // Fully qualified to avoid ambiguity with odb exception.
- //
- catch (const std::exception& e)
- {
- cerr << e.what () << endl;
- return 1;
+ // Ensure there is no package dependency cycles.
+ //
+ package_ids chain;
+ for (const auto& p:
+ db.query<package> (query::internal_repository.is_not_null ()))
+ detect_dependency_cycle (p.id, chain, db);
}
+
+ t.commit ();
+}
+catch (const database_locked&)
+{
+ cerr << "brep-load or brep-migrate instance is running" << endl;
+ return 2;
+}
+catch (const cli::exception& e)
+{
+ cerr << e << endl;
+ usage (cerr);
+ return 1;
+}
+// Fully qualified to avoid ambiguity with odb exception.
+//
+catch (const std::exception& e)
+{
+ cerr << e.what () << endl;
+ return 1;
}
diff --git a/loader/options.cli b/load/options.cli
index 0899156..df91606 100644
--- a/loader/options.cli
+++ b/load/options.cli
@@ -1,14 +1,37 @@
-// file : loader/options.cli
+// file : load/options.cli
// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
// license : MIT; see accompanying LICENSE file
include <string>;
include <cstdint>; // uint16_t
+"\section=1"
+"\name=brep-load"
+"\summary=load repositories into database"
+
+{
+ "<options> <file>",
+
+ "\h|SYNOPSIS|
+
+ \cb{brep-load --help}\n
+ \cb{brep-load --version}\n
+ \c{\b{brep-load} [<options>] <file>}
+
+ \h|DESCRIPTION|
+
+ \cb{brep-load} reads the list of repositories from the specified
+ configuration <file>, fetches their manifest files, and loads the repository
+ and package information into the database, suitable for consumption by the
+ \cb{brep} web module.
+
+ Note that \cb{brep-load} expects the database schema to have already been
+ created using \l{brep-migrate(1)}."
+}
+
class options
{
- bool --help {"Print usage information and exit."}
- bool --version {"Print version and exit."}
+ "\h|OPTIONS|"
std::string --db-user|-u
{
@@ -43,4 +66,7 @@ class options
"<port>",
"Database port number. If not specified, the default port is used."
}
+
+ bool --help {"Print usage information and exit."}
+ bool --version {"Print version and exit."}
};
diff --git a/migrate/.gitignore b/migrate/.gitignore
new file mode 100644
index 0000000..580958d
--- /dev/null
+++ b/migrate/.gitignore
@@ -0,0 +1,3 @@
+options
+options.?xx
+brep-migrate
diff --git a/migrate/buildfile b/migrate/buildfile
new file mode 100644
index 0000000..c42de5a
--- /dev/null
+++ b/migrate/buildfile
@@ -0,0 +1,18 @@
+# file : migrate/buildfile
+# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+import libs += libodb-pgsql%lib{odb-pgsql}
+import libs += libodb%lib{odb}
+
+include ../brep/
+
+exe{brep-migrate}: \
+{ cxx}{ migrate } \
+{hxx ixx cxx}{ options } \
+../brep/lib{brep} $libs
+
+cli.options += -I $src_root --include-with-brackets --include-prefix migrate \
+--guard-prefix MIGRATE
+
+{hxx ixx cxx}{options}: cli{options}
diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx
new file mode 100644
index 0000000..aa71b67
--- /dev/null
+++ b/migrate/migrate.cxx
@@ -0,0 +1,317 @@
+// file : migrate/migrate.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <strings.h> // strcasecmp()
+
+#include <string>
+#include <cassert>
+#include <ostream>
+#include <sstream>
+#include <iostream>
+#include <stdexcept> // runtime_error, invalid_argument
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+#include <odb/schema-catalog.hxx>
+
+#include <odb/pgsql/database.hxx>
+
+#include <brep/types>
+#include <brep/utility>
+#include <brep/version>
+
+#include <brep/database-lock>
+
+#include <migrate/options>
+
+using namespace std;
+using namespace odb::core;
+using namespace brep;
+
+// Helper class that encapsulates both the ODB-generated schema and the
+// extra that comes from a .sql file (via xxd).
+//
+class schema
+{
+public:
+ explicit
+ schema (const char* extra);
+
+ void
+ create (database&) const;
+
+ void
+ drop (database&) const;
+
+private:
+ strings drop_statements_;
+ strings create_statements_;
+};
+
+schema::
+schema (const char* s)
+{
+ // Remove comments, saving the cleaned SQL code into statements.
+ //
+ string statements;
+ for (istringstream i (s); i; )
+ {
+ // Skip leading spaces (including consequtive newlines). In case we
+ // hit eof, keep c set to '\n'.
+ //
+ char c;
+ static const string spaces (" \t\n\r");
+ for (c = '\n'; i.get (c) && spaces.find (c) != string::npos; c = '\n')
+ ;
+
+ // First non-space character (or '\n' for eof). See if this is a comment.
+ //
+ bool skip (c == '\n' || (c == '-' && i.peek () == '-'));
+
+ // Read until newline (and don't forget the character we already have).
+ //
+ do
+ {
+ if (!skip)
+ statements.push_back (c);
+
+ } while (c != '\n' && i.get (c));
+ }
+
+ istringstream i (move (statements));
+
+ // Build CREATE and DROP statement lists.
+ //
+ while (i)
+ {
+ string op;
+ if (i >> op) // Get the first word.
+ {
+ string statement (op);
+
+ auto read_until = [&i, &statement](const char stop[2]) -> bool
+ {
+ for (char prev ('\0'), c; i.get (c); prev = c)
+ {
+ statement.push_back (c);
+
+ if (stop[0] == prev && stop[1] == c)
+ return true;
+ }
+
+ return false;
+ };
+
+ if (strcasecmp (op.c_str (), "CREATE") == 0)
+ {
+ string kw;
+ i >> kw;
+ statement += " " + kw;
+
+ if (strcasecmp (kw.c_str (), "FUNCTION") == 0)
+ {
+ if (!read_until ("$$") || !read_until ("$$"))
+ throw invalid_argument (
+ "function body must be defined using $$-quoted strings");
+ }
+ else if (strcasecmp (kw.c_str (), "TYPE") == 0)
+ {
+ // Fall through.
+ }
+ else
+ throw invalid_argument ("unexpected CREATE statement");
+
+ if (!read_until (";\n"))
+ throw invalid_argument (
+ "expected ';\\n' at the end of CREATE statement");
+
+ assert (!statement.empty ());
+ create_statements_.emplace_back (move (statement));
+ }
+ else if (strcasecmp (op.c_str (), "DROP") == 0)
+ {
+ if (!read_until (";\n"))
+ throw invalid_argument (
+ "expected ';\\n' at the end of DROP statement");
+
+ assert (!statement.empty ());
+ drop_statements_.emplace_back (move (statement));
+ }
+ else
+ throw invalid_argument (
+ "unexpected statement starting with '" + op + "'");
+ }
+ }
+}
+
+void schema::
+drop (database& db) const
+{
+ for (const auto& s: drop_statements_)
+ // If the statement execution fails, the corresponding source file line
+ // number is not reported. The line number could be usefull for the utility
+ // implementer only. The errors seen by the end-user should not be
+ // statement-specific.
+ //
+ db.execute (s);
+
+ schema_catalog::drop_schema (db);
+}
+
+void schema::
+create (database& db) const
+{
+ drop (db);
+
+ schema_catalog::create_schema (db);
+
+ for (const auto& s: create_statements_)
+ db.execute (s);
+}
+
+// Utility functions
+//
+static void
+usage (ostream& os)
+{
+ os << "Usage: brep-migrate [options]" << endl
+ << "Options:" << endl;
+
+ options::print_usage (os);
+}
+
+// main() function
+//
+int
+main (int argc, char* argv[])
+try
+{
+ cli::argv_scanner scan (argc, argv, true);
+ options ops (scan);
+
+ // Version.
+ //
+ if (ops.version ())
+ {
+ cout << "brep-migrate " << BREP_VERSION_STR << endl
+ << "libbrep " << LIBBREP_VERSION_STR << endl
+ << "libbpkg " << LIBBPKG_VERSION_STR << endl
+ << "libbutl " << LIBBUTL_VERSION_STR << endl
+ << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl
+ << "MIT; see accompanying LICENSE file" << endl;
+
+ return 0;
+ }
+
+ // Help.
+ //
+ if (ops.help ())
+ {
+ usage (cout);
+ return 0;
+ }
+
+ if (argc > 1)
+ {
+ cerr << "unexpected argument encountered" << endl;
+ usage (cerr);
+ return 1;
+ }
+
+ if (ops.recreate () && ops.drop ())
+ {
+ cerr << "inconsistent options specified" << endl;
+ usage (cerr);
+ return 1;
+ }
+
+ odb::pgsql::database db (ops.db_user (),
+ ops.db_password (),
+ ops.db_name (),
+ ops.db_host (),
+ ops.db_port ());
+
+ // Prevent several brep-migrate/load instances from updating DB
+ // simultaneously.
+ //
+ database_lock l (db);
+
+ // Need to obtain schema version out of the transaction. If the
+ // schema_version table does not exist, the SQL query fails, which makes the
+ // transaction useless as all consequitive queries in that transaction will
+ // be ignored by PostgreSQL.
+ //
+ schema_version schema_version (db.schema_version ());
+
+ // It is impossible to operate with the database which is out of the
+ // [base_version, current_version] range due to the lack of the knowlege
+ // required not just for migration, but for the database wiping as well.
+ //
+ if (schema_version > 0)
+ {
+ if (schema_version < schema_catalog::base_version (db))
+ throw runtime_error ("database schema is too old");
+
+ if (schema_version > schema_catalog::current_version (db))
+ throw runtime_error ("database schema is too new");
+ }
+
+ bool drop (ops.drop ());
+ bool create (ops.recreate () || (schema_version == 0 && !drop));
+ assert (!create || !drop);
+
+ // The database schema recreation requires dropping it initially, which is
+ // impossible before the database is migrated to the current schema version.
+ // Let the user decide if they want to migrate or just drop the entire
+ // database (followed with the database creation for the --recreate option).
+ //
+ if ((create || drop) && schema_version != 0 &&
+ schema_version != schema_catalog::current_version (db))
+ throw runtime_error ("database schema requires migration");
+
+ transaction t (db.begin ());
+
+ if (create || drop)
+ {
+ static const char extras[] = {
+#include <brep/package-extra>
+ , '\0'};
+
+ schema s (extras);
+
+ if (create)
+ s.create (db);
+ else if (drop)
+ s.drop (db);
+ }
+ else
+ {
+ // Register the data migration functions.
+ //
+ // static const data_migration_entry<2, LIBBREP_SCHEMA_VERSION_BASE>
+ // migrate_v2_entry (&migrate_v2);
+ //
+ schema_catalog::migrate (db);
+ }
+
+ t.commit ();
+}
+catch (const database_locked&)
+{
+ cerr << "brep-migrate or brep-load instance is running" << endl;
+ return 2;
+}
+catch (const cli::exception& e)
+{
+ cerr << e << endl;
+ usage (cerr);
+ return 1;
+}
+// Fully qualified to avoid ambiguity with odb exception.
+//
+catch (const std::exception& e)
+{
+ cerr << e.what () << endl;
+ return 1;
+}
diff --git a/migrate/options.cli b/migrate/options.cli
new file mode 100644
index 0000000..e36155c
--- /dev/null
+++ b/migrate/options.cli
@@ -0,0 +1,90 @@
+// file : migrate/options.cli
+// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+include <string>;
+include <cstdint>; // uint16_t
+
+"\section=1"
+"\name=brep-migrate"
+"\summary=create/drop/migrate brep database"
+
+{
+ "<options>",
+
+ "\h|SYNOPSIS|
+
+ \cb{brep-migrate --help}\n
+ \cb{brep-migrate --version}\n
+ \c{\b{brep-migrate} [<options>]}
+
+ \h|DESCRIPTION|
+
+ In its default mode \cb{brep-migrate} creates the database schema if it
+ doesn't already exist. Otherwise, it migrates the existing schema and data
+ to the current version, if needed.
+
+ If the \cb{--recreate} option is specified, then \cb{brep-migrate} instead
+ recreates the database schema. That is, it drops all the existing tables
+ (and their data) and then creates them from scratch.
+
+ If the \cb{--drop} option is specified, then \cb{brep-migrate} drops all the
+ existing tables (and their data).
+
+ The \cb{--recreate} and \cb{--drop} options are mutually exclusive. When
+ specified, they will cause \cb{brep-migrate} to fail if the database schema
+ requires migration. In this case you can either migrate the database first
+ or drop the entire database using, for example, \cb{psql(1)}."
+}
+
+class options
+{
+ "\h|OPTIONS|"
+
+ bool --recreate
+ {
+ "Recreate the database schema (all the existing data will be lost)."
+ }
+
+ bool --drop
+ {
+ "Drop the database schema (all the existing data will be lost)."
+ }
+
+ std::string --db-user|-u
+ {
+ "<user>",
+ "Database user name. If not specified, then operating system (login)
+ name is used."
+ }
+
+ std::string --db-password
+ {
+ "<pass>",
+ "Database password. If not specified, then login without password is
+ expected to work."
+ }
+
+ std::string --db-name|-n = "brep"
+ {
+ "<name>",
+ "Database name. If not specified, then '\cb{brep}' is used by default."
+ }
+
+ std::string --db-host|-h
+ {
+ "<host>",
+ "Database host name, address, or socket. If not specified, then connect
+ to \cb{localhost} using the operating system-default mechanism
+ (Unix-domain socket, etc)."
+ }
+
+ std::uint16_t --db-port|-p = 0
+ {
+ "<port>",
+ "Database port number. If not specified, the default port is used."
+ }
+
+ bool --help {"Print usage information and exit."}
+ bool --version {"Print version and exit."}
+};
diff --git a/tests/buildfile b/tests/buildfile
index ca46f59..2bdea00 100644
--- a/tests/buildfile
+++ b/tests/buildfile
@@ -2,6 +2,6 @@
# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-d = loader/ web/
+d = load/ web/
./: $d
include $d
diff --git a/tests/loader/1/basics/packages b/tests/load/1/basics/packages
index 86c20c1..86c20c1 100644
--- a/tests/loader/1/basics/packages
+++ b/tests/load/1/basics/packages
diff --git a/tests/loader/1/basics/repositories b/tests/load/1/basics/repositories
index 57a1c7a..57a1c7a 100644
--- a/tests/loader/1/basics/repositories
+++ b/tests/load/1/basics/repositories
diff --git a/tests/loader/1/math/packages b/tests/load/1/math/packages
index 7b81c5e..7b81c5e 100644
--- a/tests/loader/1/math/packages
+++ b/tests/load/1/math/packages
diff --git a/tests/loader/1/math/repositories b/tests/load/1/math/repositories
index 20aa30d..20aa30d 100644
--- a/tests/loader/1/math/repositories
+++ b/tests/load/1/math/repositories
diff --git a/tests/loader/1/misc/packages b/tests/load/1/misc/packages
index fec3780..fec3780 100644
--- a/tests/loader/1/misc/packages
+++ b/tests/load/1/misc/packages
diff --git a/tests/loader/1/misc/repositories b/tests/load/1/misc/repositories
index 1a41290..1a41290 100644
--- a/tests/loader/1/misc/repositories
+++ b/tests/load/1/misc/repositories
diff --git a/tests/loader/1/stable/packages b/tests/load/1/stable/packages
index afa168a..afa168a 100644
--- a/tests/loader/1/stable/packages
+++ b/tests/load/1/stable/packages
diff --git a/tests/loader/1/stable/repositories b/tests/load/1/stable/repositories
index b692ebe..b692ebe 100644
--- a/tests/loader/1/stable/repositories
+++ b/tests/load/1/stable/repositories
diff --git a/tests/loader/1/staging/packages b/tests/load/1/staging/packages
index e7b22b0..e7b22b0 100644
--- a/tests/loader/1/staging/packages
+++ b/tests/load/1/staging/packages
diff --git a/tests/loader/1/staging/repositories b/tests/load/1/staging/repositories
index d72a3f8..d72a3f8 100644
--- a/tests/loader/1/staging/repositories
+++ b/tests/load/1/staging/repositories
diff --git a/tests/loader/1/testing/packages b/tests/load/1/testing/packages
index bdebece..bdebece 100644
--- a/tests/loader/1/testing/packages
+++ b/tests/load/1/testing/packages
diff --git a/tests/loader/1/testing/repositories b/tests/load/1/testing/repositories
index a218d5c..a218d5c 100644
--- a/tests/loader/1/testing/repositories
+++ b/tests/load/1/testing/repositories
diff --git a/tests/loader/buildfile b/tests/load/buildfile
index d8ab9ac..e68fd1f 100644
--- a/tests/loader/buildfile
+++ b/tests/load/buildfile
@@ -1,4 +1,4 @@
-# file : tests/loader/buildfile
+# file : tests/load/buildfile
# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
@@ -18,4 +18,4 @@ exe{driver}: test = false
# precondition: PostgreSQL server running port 8432 with brep schema created.
# test:
-# ./driver ../../loader/brep-loader --db-host localhost --db-port 8432 ./r.conf
+# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./r.conf
diff --git a/tests/loader/driver.cxx b/tests/load/driver.cxx
index 67093d9..7a70ff5 100644
--- a/tests/loader/driver.cxx
+++ b/tests/load/driver.cxx
@@ -1,4 +1,4 @@
-// file : tests/loader/driver.cxx -*- C++ -*-
+// file : tests/load/driver.cxx -*- C++ -*-
// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
// license : MIT; see accompanying LICENSE file
diff --git a/tests/loader/r.conf b/tests/load/r.conf
index 8da4b77..8da4b77 100644
--- a/tests/loader/r.conf
+++ b/tests/load/r.conf
diff --git a/web/apache/service.cxx b/web/apache/service.cxx
index c4fe462..96ff855 100644
--- a/web/apache/service.cxx
+++ b/web/apache/service.cxx
@@ -125,7 +125,7 @@ namespace web
// Terminate the root apache process. Indeed we can only try to
// terminate the process, and most likely will fail in a production
- // environment where the apache root process usually runs under root
+ // environment where the apache root process usually runs under root,
// and worker processes run under some other user. This is why the
// implementation should consider the possibility of not being
// initialized at the time of HTTP request processing. In such a case