From 1dc38cf49b6c7a8b661a9cc675ded94c8ab33c36 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Mon, 18 Jan 2016 07:35:12 +0200 Subject: Implement brep-migrate utility --- INSTALL | 26 +- INSTALL-DEV | 8 +- brep/.gitignore | 2 +- brep/buildfile | 4 +- brep/database-lock | 43 ++ brep/database-lock.cxx | 44 ++ brep/mod-package-search.cxx | 16 + brep/odb.sh | 18 +- brep/package | 6 + brep/package-extra.sql | 20 +- brep/package.xml | 406 +++++++++++++++++ buildfile | 2 +- etc/buildfile | 2 +- etc/systemd/brep-load.service | 9 + etc/systemd/brep-load.timer | 23 + etc/systemd/brep-loader.service | 9 - etc/systemd/brep-loader.timer | 23 - load/.gitignore | 3 + load/buildfile | 20 + load/load.cxx | 838 +++++++++++++++++++++++++++++++++++ load/options.cli | 72 +++ loader/.gitignore | 3 - loader/buildfile | 20 - loader/loader.cxx | 851 ------------------------------------ loader/options.cli | 46 -- migrate/.gitignore | 3 + migrate/buildfile | 18 + migrate/migrate.cxx | 317 ++++++++++++++ migrate/options.cli | 90 ++++ tests/buildfile | 2 +- tests/load/1/basics/packages | 8 + tests/load/1/basics/repositories | 4 + tests/load/1/math/packages | 71 +++ tests/load/1/math/repositories | 13 + tests/load/1/misc/packages | 47 ++ tests/load/1/misc/repositories | 13 + tests/load/1/stable/packages | 54 +++ tests/load/1/stable/repositories | 14 + tests/load/1/staging/packages | 25 ++ tests/load/1/staging/repositories | 4 + tests/load/1/testing/packages | 9 + tests/load/1/testing/repositories | 9 + tests/load/buildfile | 21 + tests/load/driver.cxx | 756 ++++++++++++++++++++++++++++++++ tests/load/r.conf | 2 + tests/loader/1/basics/packages | 8 - tests/loader/1/basics/repositories | 4 - tests/loader/1/math/packages | 71 --- tests/loader/1/math/repositories | 13 - tests/loader/1/misc/packages | 47 -- tests/loader/1/misc/repositories | 13 - tests/loader/1/stable/packages | 54 --- tests/loader/1/stable/repositories | 14 - tests/loader/1/staging/packages | 25 -- tests/loader/1/staging/repositories | 4 - tests/loader/1/testing/packages | 9 - tests/loader/1/testing/repositories | 9 - tests/loader/buildfile | 21 - tests/loader/driver.cxx | 756 -------------------------------- tests/loader/r.conf | 2 - web/apache/service.cxx | 2 +- 61 files changed, 3008 insertions(+), 2038 deletions(-) create mode 100644 brep/database-lock create mode 100644 brep/database-lock.cxx create mode 100644 brep/package.xml create mode 100644 etc/systemd/brep-load.service create mode 100644 etc/systemd/brep-load.timer delete mode 100644 etc/systemd/brep-loader.service delete mode 100644 etc/systemd/brep-loader.timer create mode 100644 load/.gitignore create mode 100644 load/buildfile create mode 100644 load/load.cxx create mode 100644 load/options.cli delete mode 100644 loader/.gitignore delete mode 100644 loader/buildfile delete mode 100644 loader/loader.cxx delete mode 100644 loader/options.cli create mode 100644 migrate/.gitignore create mode 100644 migrate/buildfile create mode 100644 migrate/migrate.cxx create mode 100644 migrate/options.cli create mode 100644 tests/load/1/basics/packages create mode 100644 tests/load/1/basics/repositories create mode 100644 tests/load/1/math/packages create mode 100644 tests/load/1/math/repositories create mode 100644 tests/load/1/misc/packages create mode 100644 tests/load/1/misc/repositories create mode 100644 tests/load/1/stable/packages create mode 100644 tests/load/1/stable/repositories create mode 100644 tests/load/1/staging/packages create mode 100644 tests/load/1/staging/repositories create mode 100644 tests/load/1/testing/packages create mode 100644 tests/load/1/testing/repositories create mode 100644 tests/load/buildfile create mode 100644 tests/load/driver.cxx create mode 100644 tests/load/r.conf delete mode 100644 tests/loader/1/basics/packages delete mode 100644 tests/loader/1/basics/repositories delete mode 100644 tests/loader/1/math/packages delete mode 100644 tests/loader/1/math/repositories delete mode 100644 tests/loader/1/misc/packages delete mode 100644 tests/loader/1/misc/repositories delete mode 100644 tests/loader/1/stable/packages delete mode 100644 tests/loader/1/stable/repositories delete mode 100644 tests/loader/1/staging/packages delete mode 100644 tests/loader/1/staging/repositories delete mode 100644 tests/loader/1/testing/packages delete mode 100644 tests/loader/1/testing/repositories delete mode 100644 tests/loader/buildfile delete mode 100644 tests/loader/driver.cxx delete mode 100644 tests/loader/r.conf diff --git a/INSTALL b/INSTALL index 1944228..e37c806 100644 --- a/INSTALL +++ b/INSTALL @@ -122,10 +122,10 @@ To troubleshoot, see PostgreSQL logs. 5. Create Database Schema and Load Repositories $ mkdir config -$ edit config/brep-loader.conf # Loader configuration, see brep-loader(1). +$ edit config/brep-load.conf # Loader configuration, see brep-load(1). -$ psql --quiet -f install/share/brep/package.sql -$ install/bin/brep-loader config/brep-loader.conf +$ install/bin/brep-migrate +$ install/bin/brep-load config/brep-load.conf To verify: @@ -226,21 +226,21 @@ since we want the loader to run even when we are not logged in: $ sudo loginctl enable-linger brep $ mkdir -p .config/systemd/user -$ cp install/share/brep/etc/systemd/brep-loader.* .config/systemd/user/ +$ cp install/share/brep/etc/systemd/brep-load.* .config/systemd/user/ Start the service to make sure there are no issues: -$ systemctl --user start brep-loader.service +$ systemctl --user start brep-load.service $ journalctl Start the timer and monitor it to make sure it fires: -$ systemctl --user start brep-loader.timer +$ systemctl --user start brep-load.timer $ journalctl -f If everything looks good, enable the timer to be started at boot time: -$ systemctl --user enable brep-loader.timer +$ systemctl --user enable brep-load.timer 8. Upgrade Procedure @@ -262,8 +262,8 @@ $ bpkg -d brep build brep Stop and disable loader: -$ systemctl --user disable --now brep-loader.timer -$ systemctl --user stop brep-loader.service +$ systemctl --user disable --now brep-load.timer +$ systemctl --user stop brep-load.service Stop apache: @@ -280,16 +280,16 @@ $ diff -u install/share/brep/etc/brep-module.conf config/brep-module.conf Update database schema: -$ psql --quiet -f install/share/brep/package.sql +$ install/bin/brep-migrate Start and enable loader: -$ systemctl --user start brep-loader.service -$ systemctl --user status brep-loader.service +$ systemctl --user start brep-load.service +$ systemctl --user status brep-load.service If everything looks good, enable periodic execution: -$ systemctl --user enable --now brep-loader.timer +$ systemctl --user enable --now brep-load.timer Start apache: diff --git a/INSTALL-DEV b/INSTALL-DEV index b81d7fb..6e23664 100644 --- a/INSTALL-DEV +++ b/INSTALL-DEV @@ -48,8 +48,8 @@ $ sudo tail -f /var/log/postgresql/*.log All the commands are executed from brep project root. -$ psql --quiet -d brep -f brep/package.sql -$ loader/brep-loader tests/loader/r.conf # Or some other loader config. +$ migrate/brep-migrate +$ load/brep-load tests/load/r.conf # Or some other loader config. To verify: @@ -115,7 +115,7 @@ $ sudo tail -f /var/log/apache2/error.log To do a "complete reload" (i.e., recreate database schema, load the repository data, and reload the Apache2 plugin), execute the following from brep/: -psql --quiet -d brep -f brep/package.sql -loader/brep-loader tests/loader/r.conf +migrate/brep-migrate --recreate +load/brep-load tests/load/r.conf sudo /etc/init.d/apache2 restart sudo systemctl restart apache2 diff --git a/brep/.gitignore b/brep/.gitignore index 2c23e23..852a40d 100644 --- a/brep/.gitignore +++ b/brep/.gitignore @@ -2,4 +2,4 @@ options options.?xx package-odb* package.sql - +package-extra diff --git a/brep/buildfile b/brep/buildfile index 9826504..1aa4d08 100644 --- a/brep/buildfile +++ b/brep/buildfile @@ -10,7 +10,7 @@ define sql: file sql{*}: extension = sql sql{*}: install = data -./: lib{brep} mod{brep} sql{package} +./: lib{brep} mod{brep} sql{package package-extra} # lib{brep} # @@ -20,8 +20,10 @@ import libs += libbpkg%lib{bpkg} lib{brep}: \ {hxx cxx}{ package } \ +{file }{ package.xml } \ {hxx ixx cxx}{ package-odb } \ {hxx cxx}{ package-traits } \ +{hxx cxx}{ database-lock } \ {hxx }{ types } \ {hxx }{ utility } \ {hxx }{ version } \ diff --git a/brep/database-lock b/brep/database-lock new file mode 100644 index 0000000..72036a1 --- /dev/null +++ b/brep/database-lock @@ -0,0 +1,43 @@ +// file : brep/database-lock -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BREP_DATABASE_LOCK +#define BREP_DATABASE_LOCK + +#include // unique_ptr +#include + +#include // database, transaction +#include + +namespace brep +{ + struct database_locked: std::exception + { + virtual char const* + what () const throw () {return "database locked";} + }; + + // Try to "lock" the PostgreSQL database in the constructor and release the + // lock in the destructor. Throw database_locked if the database is already + // locked by someone else. May also throw odb::pgsql::database_exception. + // + // This mechanism is used by the brep loader and schema migration tool to + // make sure they don't step on each others toes. + // + // Note: movable but not copyable. + // + class database_lock + { + public: + explicit + database_lock (odb::pgsql::database&); + + private: + odb::pgsql::connection_ptr connection_; + std::unique_ptr transaction_; + }; +} + +#endif // BREP_DATABASE_LOCK diff --git a/brep/database-lock.cxx b/brep/database-lock.cxx new file mode 100644 index 0000000..3b8ae21 --- /dev/null +++ b/brep/database-lock.cxx @@ -0,0 +1,44 @@ +// file : brep/database-lock.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include +#include + +namespace brep +{ + using namespace odb::pgsql; + + database_lock:: + database_lock (database& db) + { + // Before locking the table make sure it exists. + // + { + transaction t (db.begin ()); + db.execute ("CREATE TABLE IF NOT EXISTS database_mutex ()"); + t.commit (); + } + + connection_ = db.connection (); + + // Don't make current. Will be rolled back in destructor. + // + transaction_.reset (new transaction (connection_->begin (), false)); + + try + { + connection_->execute ("LOCK TABLE database_mutex NOWAIT"); + } + catch (const database_exception& e) + { + if (e.sqlstate () == "55P03") // The table is already locked. + throw database_locked (); + + throw; + } + } +} diff --git a/brep/mod-package-search.cxx b/brep/mod-package-search.cxx index 4326435..d649ff4 100644 --- a/brep/mod-package-search.cxx +++ b/brep/mod-package-search.cxx @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -16,6 +17,7 @@ #include #include +#include #include #include @@ -38,6 +40,20 @@ init (scanner& s) options_->root (dir_path ("/")); db_ = shared_database (*options_); + + // Check that the database schema matches the current one. It's enough to + // perform the check in just a single module implementation (and we don't + // do in the dispatcher because it doesn't use the database). + // + // Note that the failure can be reported by each web server worker process. + // While it could be tempting to move the check to the + // repository_root::version() function, it would be wrong. The function can + // be called by a different process (usually the web server root one) not + // having the proper permissions to access the database. + // + if (schema_catalog::current_version (*db_) != db_->schema_version ()) + fail << "database schema differs from the current one (module " + << BREP_VERSION_STR << ")"; } template diff --git a/brep/odb.sh b/brep/odb.sh index 489d423..d68b28f 100755 --- a/brep/odb.sh +++ b/brep/odb.sh @@ -1,11 +1,13 @@ #! /usr/bin/env bash odb -d pgsql --std c++11 --generate-query --generate-schema \ - --odb-epilogue '#include ' \ - --hxx-prologue '#include ' \ - --hxx-prologue "#include " \ - --sql-epilogue-file package-extra.sql \ - -I .. -I ../../libbpkg -I ../../libbutl \ - --hxx-suffix "" --include-with-brackets \ - --include-prefix brep --guard-prefix BREP \ - package + --schema-format sql --schema-format embedded \ + --odb-epilogue '#include ' \ + --hxx-prologue '#include ' \ + --hxx-prologue '#include ' \ + -I .. -I ../../libbpkg -I ../../libbutl \ + --hxx-suffix "" --include-with-brackets \ + --include-prefix brep --guard-prefix BREP \ + package + +xxd -i package-extra diff --git a/brep/package b/brep/package index 92a2320..2700e8a 100644 --- a/brep/package +++ b/brep/package @@ -19,6 +19,12 @@ #include #include +// Used by the data migration entries. +// +#define LIBBREP_SCHEMA_VERSION_BASE 1 + +#pragma db model version(LIBBREP_SCHEMA_VERSION_BASE, 1, open) + // The uint16_t value range is not fully covered by SMALLINT PostgreSQL type // to which uint16_t is mapped by default. // diff --git a/brep/package-extra.sql b/brep/package-extra.sql index dc37d1f..9a847e3 100644 --- a/brep/package-extra.sql +++ b/brep/package-extra.sql @@ -1,12 +1,22 @@ -DROP TYPE IF EXISTS weighted_text CASCADE; -CREATE TYPE weighted_text AS (a TEXT, b TEXT, c TEXT, d TEXT); - +-- This file should be parsable by the brep-migrate utility. To decrease the +-- parser complexity, the following restrictions are placed: +-- +-- * comments must start with -- at the beginning of the line (ignoring +-- leading spaces) +-- * only CREATE and DROP statements for FUNCTION and TYPE +-- * function bodies must be defined using $$-quoted strings +-- * strings other then function bodies must be quoted with ' or " +-- * statements must end with ";\n" +-- DROP FUNCTION IF EXISTS to_tsvector(IN document weighted_text); DROP FUNCTION IF EXISTS search_packages(IN query tsquery, INOUT name TEXT); DROP FUNCTION IF EXISTS search_latest_packages(IN query tsquery); DROP FUNCTION IF EXISTS latest_package(INOUT name TEXT); DROP FUNCTION IF EXISTS latest_packages(); +DROP TYPE IF EXISTS weighted_text CASCADE; +CREATE TYPE weighted_text AS (a TEXT, b TEXT, c TEXT, d TEXT); + -- Return the latest versions of internal packages as a set of package rows. -- CREATE FUNCTION @@ -72,9 +82,9 @@ $$ LANGUAGE SQL STABLE; -- Search for packages matching the search query and having the specified name. -- Return a set of rows containing the package id and search rank. If query -- is NULL, then match all packages and return 0 rank for all rows. --- +-- CREATE FUNCTION -search_packages(IN query tsquery, +search_packages(IN query tsquery, INOUT name TEXT, OUT version_epoch INTEGER, OUT version_canonical_upstream TEXT, diff --git a/brep/package.xml b/brep/package.xml new file mode 100644 index 0000000..9c88f25 --- /dev/null +++ b/brep/package.xml @@ -0,0 +1,406 @@ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + diff --git a/buildfile b/buildfile index 88a8c0b..60318fe 100644 --- a/buildfile +++ b/buildfile @@ -2,7 +2,7 @@ # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -d = brep/ etc/ loader/ tests/ www/ +d = brep/ etc/ load/ migrate/ tests/ www/ ./: $d doc{INSTALL INSTALL-DEV LICENSE version} file{manifest} include $d diff --git a/etc/buildfile b/etc/buildfile index 9ebcf27..4910076 100644 --- a/etc/buildfile +++ b/etc/buildfile @@ -3,7 +3,7 @@ # license : MIT; see accompanying LICENSE file ./: file{brep-module.conf brep-apache2.conf} \ -systemd/file{brep-loader.service brep-loader.timer} +systemd/file{brep-load.service brep-load.timer} install = data/etc systemd/: install = data/etc/systemd diff --git a/etc/systemd/brep-load.service b/etc/systemd/brep-load.service new file mode 100644 index 0000000..34a7c9a --- /dev/null +++ b/etc/systemd/brep-load.service @@ -0,0 +1,9 @@ +[Unit] +Description=brep repository loader service + +[Service] +Type=oneshot +ExecStart=/home/brep/install/bin/brep-load /home/brep/config/brep-load.conf + +[Install] +WantedBy=default.target diff --git a/etc/systemd/brep-load.timer b/etc/systemd/brep-load.timer new file mode 100644 index 0000000..713cb31 --- /dev/null +++ b/etc/systemd/brep-load.timer @@ -0,0 +1,23 @@ +[Unit] +Description=brep repository loader timer +RefuseManualStart=no +RefuseManualStop=no + +[Timer] +Unit=brep-load.service + +# Don't keep track of the timer across reboots. +# +Persistent=false + +# Wait 20 seconds until the first run. +# +OnBootSec=20 + +# Then wait 5 minutes until the next run. +# +OnUnitInactiveSec=5m + + +[Install] +WantedBy=timers.target diff --git a/etc/systemd/brep-loader.service b/etc/systemd/brep-loader.service deleted file mode 100644 index dec7944..0000000 --- a/etc/systemd/brep-loader.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=brep repository loader service - -[Service] -Type=oneshot -ExecStart=/home/brep/install/bin/brep-loader /home/brep/config/brep-loader.conf - -[Install] -WantedBy=default.target diff --git a/etc/systemd/brep-loader.timer b/etc/systemd/brep-loader.timer deleted file mode 100644 index 22ff22b..0000000 --- a/etc/systemd/brep-loader.timer +++ /dev/null @@ -1,23 +0,0 @@ -[Unit] -Description=brep repository loader timer -RefuseManualStart=no -RefuseManualStop=no - -[Timer] -Unit=brep-loader.service - -# Don't keep track of the timer across reboots. -# -Persistent=false - -# Wait 20 seconds until the first run. -# -OnBootSec=20 - -# Then wait 5 minutes until the next run. -# -OnUnitInactiveSec=5m - - -[Install] -WantedBy=timers.target diff --git a/load/.gitignore b/load/.gitignore new file mode 100644 index 0000000..2f464c5 --- /dev/null +++ b/load/.gitignore @@ -0,0 +1,3 @@ +options +options.?xx +brep-load diff --git a/load/buildfile b/load/buildfile new file mode 100644 index 0000000..c3a324f --- /dev/null +++ b/load/buildfile @@ -0,0 +1,20 @@ +# file : load/buildfile +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +import libs += libbpkg%lib{bpkg} +import libs += libbutl%lib{butl} +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libodb%lib{odb} + +include ../brep/ + +exe{brep-load}: \ +{ cxx}{ load } \ +{hxx ixx cxx}{ options } \ +../brep/lib{brep} $libs + +cli.options += -I $src_root --include-with-brackets --include-prefix load \ +--guard-prefix LOAD + +{hxx ixx cxx}{options}: cli{options} diff --git a/load/load.cxx b/load/load.cxx new file mode 100644 index 0000000..9e6ee5d --- /dev/null +++ b/load/load.cxx @@ -0,0 +1,838 @@ +// file : load/load.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include +#include +#include +#include +#include // runtime_error, invalid_argument +#include // find(), find_if() + +#include +#include +#include +#include + +#include + +#include + +#include // manifest_parsing + +#include +#include +#include + +#include +#include +#include + +#include + +using namespace std; +using namespace odb::core; +using namespace butl; +using namespace bpkg; +using namespace brep; + +static void +usage (ostream& os) +{ + os << "Usage: brep-load [options] " << endl + << "File lists internal repositories." << endl + << "Options:" << endl; + + options::print_usage (os); +} + +static inline bool +space (char c) noexcept +{ + return c == ' ' || c == '\t'; +} + +struct internal_repository +{ + repository_location location; + string display_name; + dir_path local_path; + + path + packages_path () const {return local_path / path ("packages");} + + path + repositories_path () const {return local_path / path ("repositories");} +}; + +using internal_repositories = vector; + +static internal_repositories +load_repositories (path p) +{ + internal_repositories repos; + + if (p.relative ()) + p.complete (); + + ifstream ifs (p.string ()); + if (!ifs.is_open ()) + throw ifstream::failure (p.string () + ": unable to open"); + + ifs.exceptions (ifstream::badbit); + + try + { + string s; + for (uint64_t l (1); getline (ifs, s); ++l) + { + auto b (s.cbegin ()); + auto i (b); + auto e (s.cend ()); + + // Skip until first non-space (true) or space (false). + // + auto skip = [&i, &e](bool s = true) -> decltype (i) + { + for (; i != e && space (*i) == s; ++i) + ; + return i; + }; + + skip (); // Skip leading spaces. + + if (i == e || *i == '#') // Empty line or comment. + continue; + + // From now on pb will track the begining of the next part + // while i -- the end. + // + auto pb (i); // Location begin. + skip (false); // Find end of location. + + auto bad_line ([&p, l, &pb, &b](const string& d) { + ostringstream os; + os << p << ':' << l << ':' << pb - b + 1 << ": error: " << d; + throw runtime_error (os.str ()); + }); + + repository_location location; + + try + { + location = repository_location (string (pb, i)); + } + catch (const invalid_argument& e) + { + bad_line (e.what ()); + } + + if (location.local ()) + bad_line ("local repository location"); + + for (const auto& r: repos) + if (r.location.canonical_name () == location.canonical_name ()) + bad_line ("duplicate canonical name"); + + pb = skip (); // Find begin of display name. + + if (pb == e) + bad_line ("no display name found"); + + skip (false); // Find end of display name. + + string name (pb, i); + pb = skip (); // Find begin of filesystem path. + + if (pb == e) // For now filesystem path is mandatory. + bad_line ("no filesystem path found"); + + skip (false); // Find end of filesystem path (no spaces allowed). + + internal_repository r { + move (location), + move (name), + dir_path (string (pb, i))}; + + // If the internal repository local path is relative, then + // calculate its absolute local path. Such path is considered to be + // relative to configuration file directory path so result is + // independent from whichever directory is current for the loader + // process. + // + if (r.local_path.relative ()) + r.local_path = p.directory () / r.local_path; + + try + { + r.local_path.normalize (); + } + catch (const invalid_path&) + { + bad_line ("can't normalize local path"); + } + + if (!file_exists (r.packages_path ())) + bad_line ("'packages' file does not exist"); + + if (!file_exists (r.repositories_path ())) + bad_line ("'repositories' file does not exist"); + + repos.emplace_back (move (r)); + + // Check that there is no non-whitespace junk at the end. + // + if (skip () != e) + bad_line ("junk after filesystem path"); + } + } + catch (const ifstream::failure&) + { + throw ifstream::failure (p.string () + ": io failure"); + } + + return repos; +} + +// Check if repositories persistent state is outdated. If any repository +// differes from its persistent state or there is a persistent repository +// which is not listed in configuration file then the whole persistent +// state will be recreated. Will consider optimization later when the +// package model, including search related objects, settles down. +// +static bool +changed (const internal_repositories& repos, database& db) +{ + strings names; + for (auto& r: repos) + { + shared_ptr pr ( + db.find (r.location.canonical_name ())); + + if (pr == nullptr || r.location.string () != pr->location.string () || + r.display_name != pr->display_name || r.local_path != pr->local_path || + file_mtime (r.packages_path ()) != pr->packages_timestamp || + file_mtime (r.repositories_path ()) != pr->repositories_timestamp || + !pr->internal) + return true; + + names.emplace_back (r.location.canonical_name ()); + } + + using query = query; + + // Check if there is an internal repository not being listed in the + // configuration file. + // + return + !db.query ( + query::internal && !query::name.in_range (names.begin (), names.end ())). + empty (); +} + +static timestamp +manifest_stream (const path& p, ifstream& f) +{ + f.open (p.string ()); + if (!f.is_open ()) + throw ifstream::failure (p.string () + ": unable to open"); + + f.exceptions (ifstream::badbit | ifstream::failbit); + return file_mtime (p); +} + +// Load the repository packages from the 'packages' file and persist the +// repository. Should be called once per repository. +// +static void +load_packages (const shared_ptr& rp, database& db) +{ + // packages_timestamp other than timestamp_nonexistent signals the + // repository packages are already loaded. + // + assert (rp->packages_timestamp == timestamp_nonexistent); + + // Only locally accessible repositories allowed until package manager API is + // ready. + // + assert (!rp->local_path.empty ()); + + package_manifests pkm; + + { + ifstream ifs; + path p (rp->local_path / path ("packages")); + rp->packages_timestamp = manifest_stream (p, ifs); + + manifest_parser mp (ifs, p.string ()); + pkm = package_manifests (mp); + } + + for (auto& pm: pkm) + { + shared_ptr p (db.find (package_id (pm.name, pm.version))); + + if (p == nullptr) + { + if (rp->internal) + { + // Create internal package object. + // + optional dsc; + if (pm.description) + { + if (pm.description->file) + { + // @@ Pull description from the file when package manager API + // is ready. + } + else + dsc = move (*pm.description); + } + + string chn; + for (auto& c: pm.changes) + { + if (c.file) + { + // @@ Pull change notes from the file when package manager + // API is ready. + } + else + { + if (chn.empty ()) + chn = move (c); + else + chn += "\n" + c; + } + } + + dependencies ds; + + for (auto& pda: pm.dependencies) + { + ds.emplace_back (pda.conditional, move (pda.comment)); + + for (auto& pd: pda) + // Proper version will be assigned during dependency resolution + // procedure. Here we rely on the fact the foreign key constraint + // check is deferred until the current transaction commit. + // + ds.back ().push_back ({ + lazy_shared_ptr ( + db, package_id (move (pd.name), version ())), + move (pd.constraint)}); + } + + p = make_shared ( + move (pm.name), + move (pm.version), + pm.priority ? move (*pm.priority) : priority (), + move (pm.summary), + move (pm.license_alternatives), + move (pm.tags), + move (dsc), + move (chn), + move (pm.url), + move (pm.package_url), + move (pm.email), + move (pm.package_email), + move (ds), + move (pm.requirements), + move (pm.location), + rp); + } + else + // Create external package object. + // + p = make_shared (move (pm.name), move (pm.version), rp); + + db.persist (p); + } + else + { + // @@ Need to ensure that the same packages coming from different + // repositories are equal. Probably will invent hashsum at some point + // for this purpose. + // + + // As soon as internal repositories get loaded first, the internal + // package can duplicate an internal package only. + // + assert (!rp->internal || p->internal ()); + + p->other_repositories.push_back (rp); + db.update (p); + } + } + + db.persist (rp); // Save the repository state. +} + +// Load the repository manifest values, prerequsite repositories, and their +// complements state from the 'repositories' file. Update the repository +// persistent state to save changed members. Should be called once per +// persisted internal repository. +// +static void +load_repositories (const shared_ptr& rp, database& db) +{ + // repositories_timestamp other than timestamp_nonexistent signals that + // repository prerequisites are already loaded. + // + assert (rp->repositories_timestamp == timestamp_nonexistent); + + // Only locally accessible repositories allowed until package manager API is + // ready. + // + assert (!rp->local_path.empty ()); + + // Repository is already persisted by the load_packages() function call. + // + assert (db.find (rp->name) != nullptr); + + repository_manifests rpm; + + { + ifstream ifs; + path p (rp->local_path / path ("repositories")); + rp->repositories_timestamp = manifest_stream (p, ifs); + + manifest_parser mp (ifs, p.string ()); + rpm = repository_manifests (mp); + } + + for (auto& rm: rpm) + { + if (rm.effective_role () == repository_role::prerequisite && !rp->internal) + continue; // Ignore the external repository prerequisite entry. + + if (rm.effective_role () == repository_role::base) + { + assert (rp->location.remote () && !rp->url); + + // Update the base repository with manifest values. + // + rp->url = rm.effective_url (rp->location); + + // @@ Should we throw if url is not available for external repository ? + // Can, basically, repository be available on the web but have no web + // interface associated ? + // + // Yes, there can be no web interface. So we should just not form + // links to packages from such repos. + // + if (rp->url) + { + // Normalize web interface url adding trailing '/' if not present. + // + auto& u (*rp->url); + assert (!u.empty ()); + if (u.back () != '/') + u += '/'; + } + + if (rp->internal) + { + rp->email = move (rm.email); + rp->summary = move (rm.summary); + rp->description = move (rm.description); + } + + continue; + } + + // Load prerequisite or complement repository. + // + assert (!rm.location.empty ()); + + repository_location rl; + + auto bad_location ( + [&rp, &rm]() + { + ostringstream o; + o << "invalid location '" << rm.location.string () + << "' of the prerequisite repository for internal " + "repository '" << rp->location.string () << "'"; + + throw runtime_error (o.str ()); + }); + + try + { + // Absolute path location make no sense for the web interface. + // + if (rm.location.absolute ()) + bad_location (); + + // Convert the relative repository location to remote one, leave remote + // location unchanged. + // + rl = repository_location (rm.location.string (), rp->location); + } + catch (const invalid_argument&) + { + bad_location (); + } + + const auto& cn (rl.canonical_name ()); + + // Add repository to prerequisites or complements member of the dependent + // repository. + // + auto& rs (rm.effective_role () == repository_role::prerequisite + ? rp->prerequisites + : rp->complements); + + rs.emplace_back (db, cn); + + shared_ptr pr (db.find (cn)); + + if (pr != nullptr) + // The prerequisite repository is already loaded. + // + continue; + + pr = make_shared (move (rl)); + + // If the prerequsite repository location is a relative path, then + // calculate its absolute local path. + // + if (rm.location.relative ()) + { + dir_path& lp (pr->local_path); + lp = rp->local_path / rm.location.path (); + + try + { + lp.normalize (); + } + catch (const invalid_path&) + { + ostringstream o; + o << "can't normalize local path'" << lp.string () + << "' of the prerequisite repository for internal " + "repository '" << rp->location.string () << "'"; + + throw runtime_error (o.str ()); + } + } + + load_packages (pr, db); + load_repositories (pr, db); + } + + db.update (rp); +} + +// Check if the package is available from the specified repository, +// its prerequisite repositories, or one of their complements, +// recursively. +// +static bool +find (const lazy_shared_ptr& r, + const package& p, + bool prereq = true) +{ + assert (r != nullptr); + + const auto& o (p.other_repositories); + if (r == p.internal_repository || find (o.begin (), o.end (), r) != o.end ()) + return true; + + auto rp (r.load ()); + for (const auto& cr: rp->complements) + { + if (find (lazy_shared_ptr (cr), p, false)) + return true; + } + + if (prereq) + { + for (auto pr: rp->prerequisites) + { + if (find (lazy_shared_ptr (pr), p, false)) + return true; + } + } + + return false; +} + +// Resolve package dependencies. Ensure that the best matching dependency +// belongs to the package repositories, their immediate prerequisite +// repositories, or their complements, recursively. Should be called once per +// internal package. +// +static void +resolve_dependencies (package& p, database& db) +{ + // Resolve dependencies for internal packages only. + // + assert (p.internal ()); + + if (p.dependencies.empty ()) + return; + + for (auto& da: p.dependencies) + { + for (auto& d: da) + { + // Dependency should not be resolved yet. + // + assert (d.package.object_id ().version.empty ()); + + using query = query; + query q (query::id.name == d.name ()); + const auto& vm (query::id.version); + + if (d.constraint) + { + auto c (*d.constraint); + + if (c.min_version) + { + if (c.min_open) + q = q && vm > *c.min_version; + else + q = q && vm >= *c.min_version; + } + + if (c.max_version) + { + if (c.max_open) + q = q && vm < *c.max_version; + else + q = q && vm <= *c.max_version; + } + } + + for (const auto& pp: db.query (q + order_by_version_desc (vm))) + { + if (find (p.internal_repository, pp)) + { + d.package.reset (db, pp.id); + break; + } + } + + if (d.package.object_id ().version.empty ()) + { + ostringstream o; + o << "can't resolve dependency " << d << " of the package " + << p.id.name << " " << p.version.string () + << " (" << p.internal_repository.load ()->name << ")"; + + // Practically it is enough to resolve at least one dependency + // alternative to build a package. Meanwhile here we consider an error + // specifying in the manifest file an alternative which can't be + // resolved. + // + throw runtime_error (o.str ()); + } + } + } + + db.update (p); // Update the package state. +} + +using package_ids = vector; + +// Ensure the package dependency chain do not contain the package id. Throw +// runtime_error otherwise. Continue the chain with the package id and call +// itself recursively for each prerequisite of the package. Should be called +// once per internal package. +// +// @@ This should probably be eventually moved to bpkg. +// +static void +detect_dependency_cycle (const package_id& id, package_ids& chain, database& db) +{ + // Package of one version depending on the same package of another version + // is something obscure. So the comparison is made up to a package name. + // + auto pr ([&id](const package_id& i) -> bool {return i.name == id.name;}); + auto i (find_if (chain.begin (), chain.end (), pr)); + + if (i != chain.end ()) + { + ostringstream o; + o << "package dependency cycle: "; + + auto prn ( + [&o, &db](const package_id& id) + { + shared_ptr p (db.load (id)); + assert (p->internal () || !p->other_repositories.empty ()); + + shared_ptr r ( + p->internal () + ? p->internal_repository.load () + : p->other_repositories[0].load ()); + + o << id.name << " " << p->version.string () << " (" << r->name << ")"; + }); + + for (; i != chain.end (); ++i) + { + prn (*i); + o << " -> "; + } + + prn (id); + throw runtime_error (o.str ()); + } + + chain.push_back (id); + + shared_ptr p (db.load (id)); + for (const auto& da: p->dependencies) + { + for (const auto& d: da) + detect_dependency_cycle (d.package.object_id (), chain, db); + } + + chain.pop_back (); +} + +int +main (int argc, char* argv[]) +try +{ + cli::argv_scanner scan (argc, argv, true); + options ops (scan); + + // Version. + // + if (ops.version ()) + { + cout << "brep-load " << BREP_VERSION_STR << endl + << "libbrep " << LIBBREP_VERSION_STR << endl + << "libbpkg " << LIBBPKG_VERSION_STR << endl + << "libbutl " << LIBBUTL_VERSION_STR << endl + << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl + << "MIT; see accompanying LICENSE file" << endl; + + return 0; + } + + // Help. + // + if (ops.help ()) + { + usage (cout); + return 0; + } + + if (argc < 2) + { + cerr << " argument not provided" << endl; + usage (cerr); + return 1; + } + + if (argc > 2) + { + cerr << "unexpected argument encountered" << endl; + usage (cerr); + return 1; + } + + odb::pgsql::database db (ops.db_user (), + ops.db_password (), + ops.db_name (), + ops.db_host (), + ops.db_port ()); + + // Prevent several brep-load/migrate instances from updating DB + // simultaneously. + // + database_lock l (db); + + transaction t (db.begin ()); + + // Check that the database schema matches the current one. + // + if (schema_catalog::current_version (db) != db.schema_version ()) + { + cerr << "database schema differs from the current one" << endl; + return 1; + } + + // Load the description of all the internal repositories from the + // configuration file. + // + internal_repositories irs (load_repositories (path (argv[1]))); + + if (changed (irs, db)) + { + // Rebuild repositories persistent state from scratch. + // + db.erase_query (); + db.erase_query (); + + // On the first pass over the internal repositories we load their + // packages. + // + uint16_t priority (1); + for (const auto& ir: irs) + { + shared_ptr r ( + make_shared (ir.location, + move (ir.display_name), + move (ir.local_path), + priority++)); + + load_packages (r, db); + } + + // On the second pass over the internal repositories we load their + // (not yet loaded) manifest values, complement, and prerequisite + // repositories. + // + for (const auto& ir: irs) + { + shared_ptr r ( + db.load (ir.location.canonical_name ())); + + load_repositories (r, db); + } + + session s; + using query = query; + + // Resolve internal packages dependencies. + // + for (auto& p: + db.query (query::internal_repository.is_not_null ())) + resolve_dependencies (p, db); + + // Ensure there is no package dependency cycles. + // + package_ids chain; + for (const auto& p: + db.query (query::internal_repository.is_not_null ())) + detect_dependency_cycle (p.id, chain, db); + } + + t.commit (); +} +catch (const database_locked&) +{ + cerr << "brep-load or brep-migrate instance is running" << endl; + return 2; +} +catch (const cli::exception& e) +{ + cerr << e << endl; + usage (cerr); + return 1; +} +// Fully qualified to avoid ambiguity with odb exception. +// +catch (const std::exception& e) +{ + cerr << e.what () << endl; + return 1; +} diff --git a/load/options.cli b/load/options.cli new file mode 100644 index 0000000..df91606 --- /dev/null +++ b/load/options.cli @@ -0,0 +1,72 @@ +// file : load/options.cli +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +include ; +include ; // uint16_t + +"\section=1" +"\name=brep-load" +"\summary=load repositories into database" + +{ + " ", + + "\h|SYNOPSIS| + + \cb{brep-load --help}\n + \cb{brep-load --version}\n + \c{\b{brep-load} [] } + + \h|DESCRIPTION| + + \cb{brep-load} reads the list of repositories from the specified + configuration , fetches their manifest files, and loads the repository + and package information into the database, suitable for consumption by the + \cb{brep} web module. + + Note that \cb{brep-load} expects the database schema to have already been + created using \l{brep-migrate(1)}." +} + +class options +{ + "\h|OPTIONS|" + + std::string --db-user|-u + { + "", + "Database user name. If not specified, then operating system (login) + name is used." + } + + std::string --db-password + { + "", + "Database password. If not specified, then login without password is + expected to work." + } + + std::string --db-name|-n = "brep" + { + "", + "Database name. If not specified, then '\cb{brep}' is used by default." + } + + std::string --db-host|-h + { + "", + "Database host name, address, or socket. If not specified, then connect + to \cb{localhost} using the operating system-default mechanism + (Unix-domain socket, etc)." + } + + std::uint16_t --db-port|-p = 0 + { + "", + "Database port number. If not specified, the default port is used." + } + + bool --help {"Print usage information and exit."} + bool --version {"Print version and exit."} +}; diff --git a/loader/.gitignore b/loader/.gitignore deleted file mode 100644 index 820b183..0000000 --- a/loader/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -options -options.?xx -brep-loader diff --git a/loader/buildfile b/loader/buildfile deleted file mode 100644 index 477cebd..0000000 --- a/loader/buildfile +++ /dev/null @@ -1,20 +0,0 @@ -# file : loader/buildfile -# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -# license : MIT; see accompanying LICENSE file - -import libs += libbpkg%lib{bpkg} -import libs += libbutl%lib{butl} -import libs += libodb-pgsql%lib{odb-pgsql} -import libs += libodb%lib{odb} - -include ../brep/ - -exe{brep-loader}: \ -{ cxx}{ loader } \ -{hxx ixx cxx}{ options } \ -../brep/lib{brep} $libs - -cli.options += -I $src_root --include-with-brackets --include-prefix loader \ ---guard-prefix LOADER - -{hxx ixx cxx}{options}: cli{options} diff --git a/loader/loader.cxx b/loader/loader.cxx deleted file mode 100644 index b1b7706..0000000 --- a/loader/loader.cxx +++ /dev/null @@ -1,851 +0,0 @@ -// file : loader/loader.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include -#include -#include -#include // runtime_error, invalid_argument -#include // find(), find_if() - -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include // manifest_parsing - -#include -#include -#include - -#include -#include - -#include - -using namespace std; -using namespace odb::core; -using namespace butl; -using namespace bpkg; -using namespace brep; - -namespace pgsql = odb::pgsql; - -static void -usage (ostream& os) -{ - os << "Usage: brep-loader [options] " << endl - << "File lists internal repositories." << endl - << "Options:" << endl; - - options::print_usage (os); -} - -static inline bool -space (char c) noexcept -{ - return c == ' ' || c == '\t'; -} - -struct internal_repository -{ - repository_location location; - string display_name; - dir_path local_path; - - path - packages_path () const {return local_path / path ("packages");} - - path - repositories_path () const {return local_path / path ("repositories");} -}; - -using internal_repositories = vector; - -static internal_repositories -load_repositories (path p) -{ - internal_repositories repos; - - if (p.relative ()) - p.complete (); - - ifstream ifs (p.string ()); - if (!ifs.is_open ()) - throw ifstream::failure (p.string () + ": unable to open"); - - ifs.exceptions (ifstream::badbit); - - try - { - string s; - for (uint64_t l (1); getline (ifs, s); ++l) - { - auto b (s.cbegin ()); - auto i (b); - auto e (s.cend ()); - - // Skip until first non-space (true) or space (false). - // - auto skip = [&i, &e](bool s = true) -> decltype (i) - { - for (; i != e && space (*i) == s; ++i) - ; - return i; - }; - - skip (); // Skip leading spaces. - - if (i == e || *i == '#') // Empty line or comment. - continue; - - // From now on pb will track the begining of the next part - // while i -- the end. - // - auto pb (i); // Location begin. - skip (false); // Find end of location. - - auto bad_line ([&p, l, &pb, &b](const string& d) { - ostringstream os; - os << p << ':' << l << ':' << pb - b + 1 << ": error: " << d; - throw runtime_error (os.str ()); - }); - - repository_location location; - - try - { - location = repository_location (string (pb, i)); - } - catch (const invalid_argument& e) - { - bad_line (e.what ()); - } - - if (location.local ()) - bad_line ("local repository location"); - - for (const auto& r: repos) - if (r.location.canonical_name () == location.canonical_name ()) - bad_line ("duplicate canonical name"); - - pb = skip (); // Find begin of display name. - - if (pb == e) - bad_line ("no display name found"); - - skip (false); // Find end of display name. - - string name (pb, i); - pb = skip (); // Find begin of filesystem path. - - if (pb == e) // For now filesystem path is mandatory. - bad_line ("no filesystem path found"); - - skip (false); // Find end of filesystem path (no spaces allowed). - - internal_repository r { - move (location), - move (name), - dir_path (string (pb, i))}; - - // If the internal repository local path is relative, then - // calculate its absolute local path. Such path is considered to be - // relative to configuration file directory path so result is - // independent from whichever directory is current for the loader - // process. - // - if (r.local_path.relative ()) - r.local_path = p.directory () / r.local_path; - - try - { - r.local_path.normalize (); - } - catch (const invalid_path&) - { - bad_line ("can't normalize local path"); - } - - if (!file_exists (r.packages_path ())) - bad_line ("'packages' file does not exist"); - - if (!file_exists (r.repositories_path ())) - bad_line ("'repositories' file does not exist"); - - repos.emplace_back (move (r)); - - // Check that there is no non-whitespace junk at the end. - // - if (skip () != e) - bad_line ("junk after filesystem path"); - } - } - catch (const ifstream::failure&) - { - throw ifstream::failure (p.string () + ": io failure"); - } - - return repos; -} - -// Check if repositories persistent state is outdated. If any repository -// differes from its persistent state or there is a persistent repository -// which is not listed in configuration file then the whole persistent -// state will be recreated. Will consider optimization later when the -// package model, including search related objects, settles down. -// -static bool -changed (const internal_repositories& repos, database& db) -{ - strings names; - for (auto& r: repos) - { - shared_ptr pr ( - db.find (r.location.canonical_name ())); - - if (pr == nullptr || r.location.string () != pr->location.string () || - r.display_name != pr->display_name || r.local_path != pr->local_path || - file_mtime (r.packages_path ()) != pr->packages_timestamp || - file_mtime (r.repositories_path ()) != pr->repositories_timestamp || - !pr->internal) - return true; - - names.emplace_back (r.location.canonical_name ()); - } - - using query = query; - - // Check if there is an internal repository not being listed in the - // configuration file. - // - return - !db.query ( - query::internal && !query::name.in_range (names.begin (), names.end ())). - empty (); -} - -static timestamp -manifest_stream (const path& p, ifstream& f) -{ - f.open (p.string ()); - if (!f.is_open ()) - throw ifstream::failure (p.string () + ": unable to open"); - - f.exceptions (ifstream::badbit | ifstream::failbit); - return file_mtime (p); -} - -// Load the repository packages from the 'packages' file and persist the -// repository. Should be called once per repository. -// -static void -load_packages (const shared_ptr& rp, database& db) -{ - // packages_timestamp other than timestamp_nonexistent signals the - // repository packages are already loaded. - // - assert (rp->packages_timestamp == timestamp_nonexistent); - - // Only locally accessible repositories allowed until package manager API is - // ready. - // - assert (!rp->local_path.empty ()); - - package_manifests pkm; - - { - ifstream ifs; - path p (rp->local_path / path ("packages")); - rp->packages_timestamp = manifest_stream (p, ifs); - - manifest_parser mp (ifs, p.string ()); - pkm = package_manifests (mp); - } - - for (auto& pm: pkm) - { - shared_ptr p (db.find (package_id (pm.name, pm.version))); - - if (p == nullptr) - { - if (rp->internal) - { - // Create internal package object. - // - optional dsc; - if (pm.description) - { - if (pm.description->file) - { - // @@ Pull description from the file when package manager API - // is ready. - } - else - dsc = move (*pm.description); - } - - string chn; - for (auto& c: pm.changes) - { - if (c.file) - { - // @@ Pull change notes from the file when package manager - // API is ready. - } - else - { - if (chn.empty ()) - chn = move (c); - else - chn += "\n" + c; - } - } - - dependencies ds; - - for (auto& pda: pm.dependencies) - { - ds.emplace_back (pda.conditional, move (pda.comment)); - - for (auto& pd: pda) - // Proper version will be assigned during dependency resolution - // procedure. Here we rely on the fact the foreign key constraint - // check is deferred until the current transaction commit. - // - ds.back ().push_back ({ - lazy_shared_ptr ( - db, package_id (move (pd.name), version ())), - move (pd.constraint)}); - } - - p = make_shared ( - move (pm.name), - move (pm.version), - pm.priority ? move (*pm.priority) : priority (), - move (pm.summary), - move (pm.license_alternatives), - move (pm.tags), - move (dsc), - move (chn), - move (pm.url), - move (pm.package_url), - move (pm.email), - move (pm.package_email), - move (ds), - move (pm.requirements), - move (pm.location), - rp); - } - else - // Create external package object. - // - p = make_shared (move (pm.name), move (pm.version), rp); - - db.persist (p); - } - else - { - // @@ Need to ensure that the same packages coming from different - // repositories are equal. Probably will invent hashsum at some point - // for this purpose. - // - - // As soon as internal repositories get loaded first, the internal - // package can duplicate an internal package only. - // - assert (!rp->internal || p->internal ()); - - p->other_repositories.push_back (rp); - db.update (p); - } - } - - db.persist (rp); // Save the repository state. -} - -// Load the repository manifest values, prerequsite repositories, and their -// complements state from the 'repositories' file. Update the repository -// persistent state to save changed members. Should be called once per -// persisted internal repository. -// -static void -load_repositories (const shared_ptr& rp, database& db) -{ - // repositories_timestamp other than timestamp_nonexistent signals that - // repository prerequisites are already loaded. - // - assert (rp->repositories_timestamp == timestamp_nonexistent); - - // Only locally accessible repositories allowed until package manager API is - // ready. - // - assert (!rp->local_path.empty ()); - - // Repository is already persisted by the load_packages() function call. - // - assert (db.find (rp->name) != nullptr); - - repository_manifests rpm; - - { - ifstream ifs; - path p (rp->local_path / path ("repositories")); - rp->repositories_timestamp = manifest_stream (p, ifs); - - manifest_parser mp (ifs, p.string ()); - rpm = repository_manifests (mp); - } - - for (auto& rm: rpm) - { - if (rm.effective_role () == repository_role::prerequisite && !rp->internal) - continue; // Ignore the external repository prerequisite entry. - - if (rm.effective_role () == repository_role::base) - { - assert (rp->location.remote () && !rp->url); - - // Update the base repository with manifest values. - // - rp->url = rm.effective_url (rp->location); - - // @@ Should we throw if url is not available for external repository ? - // Can, basically, repository be available on the web but have no web - // interface associated ? - // - // Yes, there can be no web interface. So we should just not form - // links to packages from such repos. - // - if (rp->url) - { - // Normalize web interface url adding trailing '/' if not present. - // - auto& u (*rp->url); - assert (!u.empty ()); - if (u.back () != '/') - u += '/'; - } - - if (rp->internal) - { - rp->email = move (rm.email); - rp->summary = move (rm.summary); - rp->description = move (rm.description); - } - - continue; - } - - // Load prerequisite or complement repository. - // - assert (!rm.location.empty ()); - - repository_location rl; - - auto bad_location ( - [&rp, &rm]() - { - ostringstream o; - o << "invalid location '" << rm.location.string () - << "' of the prerequisite repository for internal " - "repository '" << rp->location.string () << "'"; - - throw runtime_error (o.str ()); - }); - - try - { - // Absolute path location make no sense for the web interface. - // - if (rm.location.absolute ()) - bad_location (); - - // Convert the relative repository location to remote one, leave remote - // location unchanged. - // - rl = repository_location (rm.location.string (), rp->location); - } - catch (const invalid_argument&) - { - bad_location (); - } - - const auto& cn (rl.canonical_name ()); - - // Add repository to prerequisites or complements member of the dependent - // repository. - // - auto& rs (rm.effective_role () == repository_role::prerequisite - ? rp->prerequisites - : rp->complements); - - rs.emplace_back (db, cn); - - shared_ptr pr (db.find (cn)); - - if (pr != nullptr) - // The prerequisite repository is already loaded. - // - continue; - - pr = make_shared (move (rl)); - - // If the prerequsite repository location is a relative path, then - // calculate its absolute local path. - // - if (rm.location.relative ()) - { - dir_path& lp (pr->local_path); - lp = rp->local_path / rm.location.path (); - - try - { - lp.normalize (); - } - catch (const invalid_path&) - { - ostringstream o; - o << "can't normalize local path'" << lp.string () - << "' of the prerequisite repository for internal " - "repository '" << rp->location.string () << "'"; - - throw runtime_error (o.str ()); - } - } - - load_packages (pr, db); - load_repositories (pr, db); - } - - db.update (rp); -} - -// Check if the package is available from the specified repository, -// its prerequisite repositories, or one of their complements, -// recursively. -// -static bool -find (const lazy_shared_ptr& r, - const package& p, - bool prereq = true) -{ - assert (r != nullptr); - - const auto& o (p.other_repositories); - if (r == p.internal_repository || find (o.begin (), o.end (), r) != o.end ()) - return true; - - auto rp (r.load ()); - for (const auto& cr: rp->complements) - { - if (find (lazy_shared_ptr (cr), p, false)) - return true; - } - - if (prereq) - { - for (auto pr: rp->prerequisites) - { - if (find (lazy_shared_ptr (pr), p, false)) - return true; - } - } - - return false; -} - -// Resolve package dependencies. Ensure that the best matching dependency -// belongs to the package repositories, their immediate prerequisite -// repositories, or their complements, recursively. Should be called once per -// internal package. -// -static void -resolve_dependencies (package& p, database& db) -{ - // Resolve dependencies for internal packages only. - // - assert (p.internal ()); - - if (p.dependencies.empty ()) - return; - - for (auto& da: p.dependencies) - { - for (auto& d: da) - { - // Dependency should not be resolved yet. - // - assert (d.package.object_id ().version.empty ()); - - using query = query; - query q (query::id.name == d.name ()); - const auto& vm (query::id.version); - - if (d.constraint) - { - auto c (*d.constraint); - - if (c.min_version) - { - if (c.min_open) - q = q && vm > *c.min_version; - else - q = q && vm >= *c.min_version; - } - - if (c.max_version) - { - if (c.max_open) - q = q && vm < *c.max_version; - else - q = q && vm <= *c.max_version; - } - } - - for (const auto& pp: db.query (q + order_by_version_desc (vm))) - { - if (find (p.internal_repository, pp)) - { - d.package.reset (db, pp.id); - break; - } - } - - if (d.package.object_id ().version.empty ()) - { - ostringstream o; - o << "can't resolve dependency " << d << " of the package " - << p.id.name << " " << p.version.string () - << " (" << p.internal_repository.load ()->name << ")"; - - // Practically it is enough to resolve at least one dependency - // alternative to build a package. Meanwhile here we consider an error - // specifying in the manifest file an alternative which can't be - // resolved. - // - throw runtime_error (o.str ()); - } - } - } - - db.update (p); // Update the package state. -} - -using package_ids = vector; - -// Ensure the package dependency chain do not contain the package id. Throw -// runtime_error otherwise. Continue the chain with the package id and call -// itself recursively for each prerequisite of the package. Should be called -// once per internal package. -// -// @@ This should probably be eventually moved to bpkg. -// -static void -detect_dependency_cycle (const package_id& id, package_ids& chain, database& db) -{ - // Package of one version depending on the same package of another version - // is something obscure. So the comparison is made up to a package name. - // - auto pr ([&id](const package_id& i) -> bool {return i.name == id.name;}); - auto i (find_if (chain.begin (), chain.end (), pr)); - - if (i != chain.end ()) - { - ostringstream o; - o << "package dependency cycle: "; - - auto prn ( - [&o, &db](const package_id& id) - { - shared_ptr p (db.load (id)); - assert (p->internal () || !p->other_repositories.empty ()); - - shared_ptr r ( - p->internal () - ? p->internal_repository.load () - : p->other_repositories[0].load ()); - - o << id.name << " " << p->version.string () << " (" << r->name << ")"; - }); - - for (; i != chain.end (); ++i) - { - prn (*i); - o << " -> "; - } - - prn (id); - throw runtime_error (o.str ()); - } - - chain.push_back (id); - - shared_ptr p (db.load (id)); - for (const auto& da: p->dependencies) - { - for (const auto& d: da) - detect_dependency_cycle (d.package.object_id (), chain, db); - } - - chain.pop_back (); -} - -int -main (int argc, char* argv[]) -{ - try - { - cli::argv_scanner scan (argc, argv, true); - options ops (scan); - - // Version. - // - if (ops.version ()) - { - cout << "brep-loader " << BREP_VERSION_STR << endl - << "libbrep " << LIBBREP_VERSION_STR << endl - << "libbpkg " << LIBBPKG_VERSION_STR << endl - << "libbutl " << LIBBUTL_VERSION_STR << endl - << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl - << "MIT; see accompanying LICENSE file" << endl; - - return 0; - } - - // Help. - // - if (ops.help ()) - { - usage (cout); - return 0; - } - - if (argc < 2) - { - cerr << " argument not provided" << endl; - usage (cerr); - return 1; - } - - if (argc > 2) - { - cerr << "unexpected argument encountered" << endl; - usage (cerr); - return 1; - } - - pgsql::database db (ops.db_user (), - ops.db_password (), - ops.db_name (), - ops.db_host (), - ops.db_port ()); - - // Prevent several loader instances from updating DB simultaneously. - // - { - transaction t (db.begin ()); - db.execute ("CREATE TABLE IF NOT EXISTS loader_mutex ()"); - t.commit (); - } - - pgsql::connection_ptr synch_c (db.connection ()); - - // Don't make current. - // - pgsql::transaction synch_t (synch_c->begin (), false); - - try - { - synch_c->execute ("LOCK TABLE loader_mutex NOWAIT"); - } - catch (const pgsql::database_exception& e) - { - if (e.sqlstate () == "55P03") - return 2; // Other loader instance acquired the mutex. - - throw; - } - - // Load the description of all the internal repositories from the - // configuration file. - // - internal_repositories irs (load_repositories (path (argv[1]))); - - transaction t (db.begin ()); - - if (changed (irs, db)) - { - // Rebuild repositories persistent state from scratch. - // - db.erase_query (); - db.erase_query (); - - // On the first pass over the internal repositories we load their - // packages. - // - uint16_t priority (1); - for (const auto& ir: irs) - { - shared_ptr r ( - make_shared (ir.location, - move (ir.display_name), - move (ir.local_path), - priority++)); - - load_packages (r, db); - } - - // On the second pass over the internal repositories we load their - // (not yet loaded) manifest values, complement, and prerequisite - // repositories. - // - for (const auto& ir: irs) - { - shared_ptr r ( - db.load (ir.location.canonical_name ())); - - load_repositories (r, db); - } - - session s; - using query = query; - - // Resolve internal packages dependencies. - // - for (auto& p: - db.query (query::internal_repository.is_not_null ())) - resolve_dependencies (p, db); - - // Ensure there is no package dependency cycles. - // - package_ids chain; - for (const auto& p: - db.query (query::internal_repository.is_not_null ())) - detect_dependency_cycle (p.id, chain, db); - } - - t.commit (); - synch_t.commit (); // Release the mutex. - } - catch (const cli::exception& e) - { - cerr << e << endl; - usage (cerr); - return 1; - } - // Fully qualified to avoid ambiguity with odb exception. - // - catch (const std::exception& e) - { - cerr << e.what () << endl; - return 1; - } -} diff --git a/loader/options.cli b/loader/options.cli deleted file mode 100644 index 0899156..0000000 --- a/loader/options.cli +++ /dev/null @@ -1,46 +0,0 @@ -// file : loader/options.cli -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -include ; -include ; // uint16_t - -class options -{ - bool --help {"Print usage information and exit."} - bool --version {"Print version and exit."} - - std::string --db-user|-u - { - "", - "Database user name. If not specified, then operating system (login) - name is used." - } - - std::string --db-password - { - "", - "Database password. If not specified, then login without password is - expected to work." - } - - std::string --db-name|-n = "brep" - { - "", - "Database name. If not specified, then '\cb{brep}' is used by default." - } - - std::string --db-host|-h - { - "", - "Database host name, address, or socket. If not specified, then connect - to \cb{localhost} using the operating system-default mechanism - (Unix-domain socket, etc)." - } - - std::uint16_t --db-port|-p = 0 - { - "", - "Database port number. If not specified, the default port is used." - } -}; diff --git a/migrate/.gitignore b/migrate/.gitignore new file mode 100644 index 0000000..580958d --- /dev/null +++ b/migrate/.gitignore @@ -0,0 +1,3 @@ +options +options.?xx +brep-migrate diff --git a/migrate/buildfile b/migrate/buildfile new file mode 100644 index 0000000..c42de5a --- /dev/null +++ b/migrate/buildfile @@ -0,0 +1,18 @@ +# file : migrate/buildfile +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libodb%lib{odb} + +include ../brep/ + +exe{brep-migrate}: \ +{ cxx}{ migrate } \ +{hxx ixx cxx}{ options } \ +../brep/lib{brep} $libs + +cli.options += -I $src_root --include-with-brackets --include-prefix migrate \ +--guard-prefix MIGRATE + +{hxx ixx cxx}{options}: cli{options} diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx new file mode 100644 index 0000000..aa71b67 --- /dev/null +++ b/migrate/migrate.cxx @@ -0,0 +1,317 @@ +// file : migrate/migrate.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include // strcasecmp() + +#include +#include +#include +#include +#include +#include // runtime_error, invalid_argument + +#include +#include +#include + +#include + +#include +#include +#include + +#include + +#include + +using namespace std; +using namespace odb::core; +using namespace brep; + +// Helper class that encapsulates both the ODB-generated schema and the +// extra that comes from a .sql file (via xxd). +// +class schema +{ +public: + explicit + schema (const char* extra); + + void + create (database&) const; + + void + drop (database&) const; + +private: + strings drop_statements_; + strings create_statements_; +}; + +schema:: +schema (const char* s) +{ + // Remove comments, saving the cleaned SQL code into statements. + // + string statements; + for (istringstream i (s); i; ) + { + // Skip leading spaces (including consequtive newlines). In case we + // hit eof, keep c set to '\n'. + // + char c; + static const string spaces (" \t\n\r"); + for (c = '\n'; i.get (c) && spaces.find (c) != string::npos; c = '\n') + ; + + // First non-space character (or '\n' for eof). See if this is a comment. + // + bool skip (c == '\n' || (c == '-' && i.peek () == '-')); + + // Read until newline (and don't forget the character we already have). + // + do + { + if (!skip) + statements.push_back (c); + + } while (c != '\n' && i.get (c)); + } + + istringstream i (move (statements)); + + // Build CREATE and DROP statement lists. + // + while (i) + { + string op; + if (i >> op) // Get the first word. + { + string statement (op); + + auto read_until = [&i, &statement](const char stop[2]) -> bool + { + for (char prev ('\0'), c; i.get (c); prev = c) + { + statement.push_back (c); + + if (stop[0] == prev && stop[1] == c) + return true; + } + + return false; + }; + + if (strcasecmp (op.c_str (), "CREATE") == 0) + { + string kw; + i >> kw; + statement += " " + kw; + + if (strcasecmp (kw.c_str (), "FUNCTION") == 0) + { + if (!read_until ("$$") || !read_until ("$$")) + throw invalid_argument ( + "function body must be defined using $$-quoted strings"); + } + else if (strcasecmp (kw.c_str (), "TYPE") == 0) + { + // Fall through. + } + else + throw invalid_argument ("unexpected CREATE statement"); + + if (!read_until (";\n")) + throw invalid_argument ( + "expected ';\\n' at the end of CREATE statement"); + + assert (!statement.empty ()); + create_statements_.emplace_back (move (statement)); + } + else if (strcasecmp (op.c_str (), "DROP") == 0) + { + if (!read_until (";\n")) + throw invalid_argument ( + "expected ';\\n' at the end of DROP statement"); + + assert (!statement.empty ()); + drop_statements_.emplace_back (move (statement)); + } + else + throw invalid_argument ( + "unexpected statement starting with '" + op + "'"); + } + } +} + +void schema:: +drop (database& db) const +{ + for (const auto& s: drop_statements_) + // If the statement execution fails, the corresponding source file line + // number is not reported. The line number could be usefull for the utility + // implementer only. The errors seen by the end-user should not be + // statement-specific. + // + db.execute (s); + + schema_catalog::drop_schema (db); +} + +void schema:: +create (database& db) const +{ + drop (db); + + schema_catalog::create_schema (db); + + for (const auto& s: create_statements_) + db.execute (s); +} + +// Utility functions +// +static void +usage (ostream& os) +{ + os << "Usage: brep-migrate [options]" << endl + << "Options:" << endl; + + options::print_usage (os); +} + +// main() function +// +int +main (int argc, char* argv[]) +try +{ + cli::argv_scanner scan (argc, argv, true); + options ops (scan); + + // Version. + // + if (ops.version ()) + { + cout << "brep-migrate " << BREP_VERSION_STR << endl + << "libbrep " << LIBBREP_VERSION_STR << endl + << "libbpkg " << LIBBPKG_VERSION_STR << endl + << "libbutl " << LIBBUTL_VERSION_STR << endl + << "Copyright (c) 2014-2016 Code Synthesis Ltd" << endl + << "MIT; see accompanying LICENSE file" << endl; + + return 0; + } + + // Help. + // + if (ops.help ()) + { + usage (cout); + return 0; + } + + if (argc > 1) + { + cerr << "unexpected argument encountered" << endl; + usage (cerr); + return 1; + } + + if (ops.recreate () && ops.drop ()) + { + cerr << "inconsistent options specified" << endl; + usage (cerr); + return 1; + } + + odb::pgsql::database db (ops.db_user (), + ops.db_password (), + ops.db_name (), + ops.db_host (), + ops.db_port ()); + + // Prevent several brep-migrate/load instances from updating DB + // simultaneously. + // + database_lock l (db); + + // Need to obtain schema version out of the transaction. If the + // schema_version table does not exist, the SQL query fails, which makes the + // transaction useless as all consequitive queries in that transaction will + // be ignored by PostgreSQL. + // + schema_version schema_version (db.schema_version ()); + + // It is impossible to operate with the database which is out of the + // [base_version, current_version] range due to the lack of the knowlege + // required not just for migration, but for the database wiping as well. + // + if (schema_version > 0) + { + if (schema_version < schema_catalog::base_version (db)) + throw runtime_error ("database schema is too old"); + + if (schema_version > schema_catalog::current_version (db)) + throw runtime_error ("database schema is too new"); + } + + bool drop (ops.drop ()); + bool create (ops.recreate () || (schema_version == 0 && !drop)); + assert (!create || !drop); + + // The database schema recreation requires dropping it initially, which is + // impossible before the database is migrated to the current schema version. + // Let the user decide if they want to migrate or just drop the entire + // database (followed with the database creation for the --recreate option). + // + if ((create || drop) && schema_version != 0 && + schema_version != schema_catalog::current_version (db)) + throw runtime_error ("database schema requires migration"); + + transaction t (db.begin ()); + + if (create || drop) + { + static const char extras[] = { +#include + , '\0'}; + + schema s (extras); + + if (create) + s.create (db); + else if (drop) + s.drop (db); + } + else + { + // Register the data migration functions. + // + // static const data_migration_entry<2, LIBBREP_SCHEMA_VERSION_BASE> + // migrate_v2_entry (&migrate_v2); + // + schema_catalog::migrate (db); + } + + t.commit (); +} +catch (const database_locked&) +{ + cerr << "brep-migrate or brep-load instance is running" << endl; + return 2; +} +catch (const cli::exception& e) +{ + cerr << e << endl; + usage (cerr); + return 1; +} +// Fully qualified to avoid ambiguity with odb exception. +// +catch (const std::exception& e) +{ + cerr << e.what () << endl; + return 1; +} diff --git a/migrate/options.cli b/migrate/options.cli new file mode 100644 index 0000000..e36155c --- /dev/null +++ b/migrate/options.cli @@ -0,0 +1,90 @@ +// file : migrate/options.cli +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +include ; +include ; // uint16_t + +"\section=1" +"\name=brep-migrate" +"\summary=create/drop/migrate brep database" + +{ + "", + + "\h|SYNOPSIS| + + \cb{brep-migrate --help}\n + \cb{brep-migrate --version}\n + \c{\b{brep-migrate} []} + + \h|DESCRIPTION| + + In its default mode \cb{brep-migrate} creates the database schema if it + doesn't already exist. Otherwise, it migrates the existing schema and data + to the current version, if needed. + + If the \cb{--recreate} option is specified, then \cb{brep-migrate} instead + recreates the database schema. That is, it drops all the existing tables + (and their data) and then creates them from scratch. + + If the \cb{--drop} option is specified, then \cb{brep-migrate} drops all the + existing tables (and their data). + + The \cb{--recreate} and \cb{--drop} options are mutually exclusive. When + specified, they will cause \cb{brep-migrate} to fail if the database schema + requires migration. In this case you can either migrate the database first + or drop the entire database using, for example, \cb{psql(1)}." +} + +class options +{ + "\h|OPTIONS|" + + bool --recreate + { + "Recreate the database schema (all the existing data will be lost)." + } + + bool --drop + { + "Drop the database schema (all the existing data will be lost)." + } + + std::string --db-user|-u + { + "", + "Database user name. If not specified, then operating system (login) + name is used." + } + + std::string --db-password + { + "", + "Database password. If not specified, then login without password is + expected to work." + } + + std::string --db-name|-n = "brep" + { + "", + "Database name. If not specified, then '\cb{brep}' is used by default." + } + + std::string --db-host|-h + { + "", + "Database host name, address, or socket. If not specified, then connect + to \cb{localhost} using the operating system-default mechanism + (Unix-domain socket, etc)." + } + + std::uint16_t --db-port|-p = 0 + { + "", + "Database port number. If not specified, the default port is used." + } + + bool --help {"Print usage information and exit."} + bool --version {"Print version and exit."} +}; diff --git a/tests/buildfile b/tests/buildfile index ca46f59..2bdea00 100644 --- a/tests/buildfile +++ b/tests/buildfile @@ -2,6 +2,6 @@ # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -d = loader/ web/ +d = load/ web/ ./: $d include $d diff --git a/tests/load/1/basics/packages b/tests/load/1/basics/packages new file mode 100644 index 0000000..86c20c1 --- /dev/null +++ b/tests/load/1/basics/packages @@ -0,0 +1,8 @@ +: 1 +name: libexpat +version: 5.1 +summary: The Expat Library +license: MIT +url: http://www.example.com/expat/ +email: expat-users@example.com +location: libexpat-5.1.tar.gz diff --git a/tests/load/1/basics/repositories b/tests/load/1/basics/repositories new file mode 100644 index 0000000..57a1c7a --- /dev/null +++ b/tests/load/1/basics/repositories @@ -0,0 +1,4 @@ +: 1 +# Local repository manifest (this repository). +# +url: http://basics.org diff --git a/tests/load/1/math/packages b/tests/load/1/math/packages new file mode 100644 index 0000000..7b81c5e --- /dev/null +++ b/tests/load/1/math/packages @@ -0,0 +1,71 @@ +: 1 +name: libstudxml +version: 1.0.0+1 +summary: Modern C++ XML API +license: MIT +tags: c++, xml, parser, serializer, pull, streaming, modern +description-file: README +changes-file: NEWS +url: http://www.codesynthesis.com/projects/libstudxml/ +email: studxml-users@codesynthesis.com; Public mailing list, posts by\ + non-members are allowed but moderated. +package-email: boris@codesynthesis.com; Direct email to the author. +depends: libexpat >= 2.0.0 +depends: libgenx +location: libstudxml-1.0.0+1.tar.gz +: +name: libexp +version: 1~1.2 +summary: The exponent +description: The exponent math function. +license: MIT +tags: c++, exponent +url: http://www.exp.com +email: users@exp.com +depends: libmisc +location: libexp-1~1.2.tar.gz +: +name: libfoo +version: 1.2.4+1 +summary: The Foo Math Library +description:\ +A modern C++ library with easy to use linear algebra and lot of optimization +tools. + +There are over 100 functions in total with an extensive test suite. The API is +similar to MATLAB. + +Useful for conversion of research code into production environments. +\ +license: LGPLv2, MIT; If using with GNU TLS. +license: BSD; If using with OpenSSL. +priority: high; Critical bug fixes, performance improvement. +tags: c++, foo, math +url: http://www.example.com/foo/; Project home page. +email: foo-users@example.com; Public mailing list. Read FAQ before posting. +package-url: http://www.example.com/foo/pack; Package details. +package-email: pack@example.com; Current packager. +depends: libmisc < 1.1 | libmisc > 2.3.0; Crashes with 1.1.0-2.3.0. +depends: libexp >= 1.0 +depends: ? libstudxml | libexpat; The newer the better. +requires: linux | windows | macosx; Symbian support is coming. +requires: c++11 +requires: ? ; libc++ standard library if using Clang on Mac OS X. +requires: ? vc++ >= 12.0; Only if using VC++ on Windows. +location: libfoo-1.2.4+1.tar.gz +changes:\ +1.2.4+1 + * applied patch for critical bug-219 + * regenerated documentation + +1.2.4 + * test suite extended significantly +\ +: +name: libfoo +version: 1.0 +summary: The Foo Lib +license: MIT +url: http://www.example.com/foo/ +email: foo-users@example.com +location: libfoo-1.0.tar.gz diff --git a/tests/load/1/math/repositories b/tests/load/1/math/repositories new file mode 100644 index 0000000..20aa30d --- /dev/null +++ b/tests/load/1/math/repositories @@ -0,0 +1,13 @@ +: 1 +# Foreign repository manifest. +# +location: ../misc +: +# Local repository manifest (this repository). +# +email: repoman@cppget.org +summary: Math C++ package repository +description: \ +This is the awesome C++ package repository full of remarkable algorithms and +APIs. +\ diff --git a/tests/load/1/misc/packages b/tests/load/1/misc/packages new file mode 100644 index 0000000..fec3780 --- /dev/null +++ b/tests/load/1/misc/packages @@ -0,0 +1,47 @@ +: 1 +name: libbar +version: 2.3.5 +priority: security; Very important to install. +summary: The Bar library +description: very very good library. +license: GPLv2 +tags: c++, bar +url: http://www.example.com/bar/ +email: bar-users@example.com +depends: libfoo +depends: libmath >= 2.0.0 +requires: linux | windows | macosx +changes: some changes +location: libbar-2.3.5.tar.gz +: +name: libfoo +version: 1.0 +summary: Foo Library +license: MIT +url: http://www.example.com/foo/ +email: foo-users@example.com +location: libfoo-1.0.tar.gz +: +name: libfoo +version: 0.1 +summary: Foo +license: MIT +url: http://www.example.com/foo/ +email: foo-users@example.com +location: libfoo-0.1.tar.gz +: +name: libfoo +version: 1.2.4+1 +summary: Foo Library +license: MIT +url: http://www.example.com/foo/ +email: foo-users@example.com +location: libfoo-1.2.4+1.tar.gz +: +name: libfoo +version: 1.2.4+2 +summary: Foo Library +license: MIT +url: http://www.example.com/foo/ +email: foo-users@example.com +location: libfoo-1.2.4+2.tar.gz diff --git a/tests/load/1/misc/repositories b/tests/load/1/misc/repositories new file mode 100644 index 0000000..1a41290 --- /dev/null +++ b/tests/load/1/misc/repositories @@ -0,0 +1,13 @@ +: 1 +# Foreign repository manifest. +# +location: ../basics +: +# Adjacent repository manifest. +# +location: ../testing +role: complement +: +# Local repository manifest (this repository). +# +url: http://misc.cppget.org/ diff --git a/tests/load/1/stable/packages b/tests/load/1/stable/packages new file mode 100644 index 0000000..afa168a --- /dev/null +++ b/tests/load/1/stable/packages @@ -0,0 +1,54 @@ +: 1 +name: libfoo +version: 1.2.3+4 +summary: The Foo library +license: MIT +tags: c++, foo +url: http://www.example.com/foo/ +email: foo-users@example.com +depends: libmisc >= 2.0.0 +location: libfoo-1.2.3+4.tar.gz +: +name: libfoo +version: 1.2.2 +summary: The Foo library +license: MIT +tags: c++, foo +url: http://www.example.com/foo/ +email: foo-users@example.com +depends: libbar <= 2.4.0 +depends: libexp == 1~1.2 +location: libfoo-1.2.2.tar.gz +: +name: libfoo +version: 1.2.2-alpha.1 +summary: The Foo library +license: MIT +tags: c++, foo +url: http://www.example.com/foo/ +email: foo-users@example.com +depends: libmisc [0.1 2.0-) | libmisc [2.0 5.0] +depends: libgenx (0.2 3.0) +depends: libexpat < 5.2 | libexpat (1 5.1] +location: libfoo-1.2.2-alpha.1.tar.gz +: +name: libfoo +version: 1.2.4 +summary: The Foo Library +description: Very good foo library. +license: MIT; Permissive free software license. +tags: c++, foo +url: http://www.example.com/foo/ +email: foo-users@example.com +depends: libmisc >= 2.0.0 +changes: some changes 1 +changes: some changes 2 +location: libfoo-1.2.4.tar.gz +: +name: libfoo +version: 1.0 +summary: The Foo Library +license: MIT +url: http://www.example.com/foo/ +email: foo-users@example.com +location: libfoo-1.0.tar.gz diff --git a/tests/load/1/stable/repositories b/tests/load/1/stable/repositories new file mode 100644 index 0000000..b692ebe --- /dev/null +++ b/tests/load/1/stable/repositories @@ -0,0 +1,14 @@ +: 1 +# Foreign repository manifest. +# +location: ../misc +: +# Adjacent repository manifest. +# +location: ../math +: +# Local repository manifest (this repository). +# +email: repoman@cppget.org +summary: General C++ package stable repository +description: This is the awesome C++ package repository full of exciting stuff. diff --git a/tests/load/1/staging/packages b/tests/load/1/staging/packages new file mode 100644 index 0000000..e7b22b0 --- /dev/null +++ b/tests/load/1/staging/packages @@ -0,0 +1,25 @@ +: 1 +name: libexpat +version: 5.1 +summary: The Expat Library +license: MIT +url: http://www.example.com/expat/ +email: expat-users@example.com +location: libexpat-5.1.tar.gz +: +name: libgenx +version: 1.0 +summary: The Genx Library +license: MIT +url: http://www.example.com/genx/ +email: genx-users@example.com +location: libgenx-1.0.tar.gz +: +name: libmisc +version: 1.0 +summary: The Misc Library +license: MIT +url: http://www.example.com/misc/ +email: misc-users@example.com +depends: libexpat >= 5.0 +location: libmisc-1.0.tar.gz diff --git a/tests/load/1/staging/repositories b/tests/load/1/staging/repositories new file mode 100644 index 0000000..d72a3f8 --- /dev/null +++ b/tests/load/1/staging/repositories @@ -0,0 +1,4 @@ +: 1 +# Local repository manifest (this repository). +# +url: ../../.. diff --git a/tests/load/1/testing/packages b/tests/load/1/testing/packages new file mode 100644 index 0000000..bdebece --- /dev/null +++ b/tests/load/1/testing/packages @@ -0,0 +1,9 @@ +: 1 +name: libmisc +version: 2.4.0 +summary: The Misc Library +license: MIT +url: http://www.example.com/misc/ +email: misc-users@example.com +depends: libexpat >= 5.0 +location: libmisc-2.4.0.tar.gz diff --git a/tests/load/1/testing/repositories b/tests/load/1/testing/repositories new file mode 100644 index 0000000..a218d5c --- /dev/null +++ b/tests/load/1/testing/repositories @@ -0,0 +1,9 @@ +: 1 +# Adjacent repository manifest. +# +location: ../staging +role: complement +: +# Local repository manifest (this repository). +# +url: http://test.cppget.org/hello diff --git a/tests/load/buildfile b/tests/load/buildfile new file mode 100644 index 0000000..e68fd1f --- /dev/null +++ b/tests/load/buildfile @@ -0,0 +1,21 @@ +# file : tests/load/buildfile +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +import libs += libbpkg%lib{bpkg} +import libs += libbutl%lib{butl} +import libs += libodb-pgsql%lib{odb-pgsql} +import libs += libodb%lib{odb} + +include ../../brep/ + +exe{driver}: cxx{driver} ../../brep/lib{brep} $libs + +# Disable for now until build2 test module supports running custom +# commands, pre/post steps. +# +exe{driver}: test = false + +# precondition: PostgreSQL server running port 8432 with brep schema created. +# test: +# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./r.conf diff --git a/tests/load/driver.cxx b/tests/load/driver.cxx new file mode 100644 index 0000000..7a70ff5 --- /dev/null +++ b/tests/load/driver.cxx @@ -0,0 +1,756 @@ +// file : tests/load/driver.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include +#include +#include // sort(), find() + +#include +#include + +#include + +#include +#include + +#include +#include + +#include +#include + +using namespace std; +using namespace odb::core; +using namespace butl; +using namespace brep; + +// @@ Rather add this to optional in libbutl. See: +// +// http://en.cppreference.com/w/cpp/experimental/optional/operator_cmp + +template +static inline auto +operator== (const optional& a, const optional& b) -> decltype (*a == *b) +{ + return !a == !b && (!a || *a == *b); +} + +// @@ Add it to libbrep rather? +// +static inline bool +operator== (const dependency_constraint& a, const dependency_constraint& b) +{ + return a.min_version == b.min_version && a.max_version == b.max_version && + a.min_open == b.min_open && a.max_open == b.max_open; +} + +static inline bool +operator== (const dependency& a, const dependency& b) +{ + return a.name () == b.name () && a.constraint == b.constraint; +} + +static bool +check_location (shared_ptr& p) +{ + if (p->internal ()) + return p->location && *p->location == + path (p->id.name + "-" + p->version.string () + ".tar.gz"); + else + return !p->location; +} + +static bool +check_external (const package& p) +{ + return p.summary.empty () && p.tags.empty () && !p.description && + p.url.empty () && !p.package_url && p.email.empty () && !p.package_email && + !p.internal () && p.other_repositories.size () > 0 && + p.priority == priority () && p.changes.empty () && + p.license_alternatives.empty () && p.dependencies.empty () && + p.requirements.empty (); +} + +int +main (int argc, char* argv[]) +{ + if (argc != 7) + { + cerr << "usage: " << argv[0] + << " --db-host --db-port " + << " " << endl; + + return 1; + } + + try + { + path cp (argv[6]); + + // Make configuration file path absolute to use it's directory as base for + // internal repositories relative local paths. + // + if (cp.relative ()) + cp.complete (); + + // Update packages file timestamp to enforce loader to update + // persistent state. + // + path p (cp.directory () / path ("1/stable/packages")); + char const* args[] = {"touch", p.string ().c_str (), nullptr}; + assert (process (args).wait ()); + + timestamp srt (file_mtime (p)); + + // Run the loader. + // + char const** ld_args (const_cast (argv + 1)); + assert (process (ld_args).wait ()); + + // Check persistent objects validity. + // + odb::pgsql::database db ("", "", "brep", argv[3], stoul (argv[5])); + + { + session s; + transaction t (db.begin ()); + + assert (db.query ().size () == 5); + assert (db.query ().size () == 15); + + shared_ptr sr (db.load ("cppget.org/stable")); + shared_ptr mr (db.load ("cppget.org/math")); + shared_ptr cr (db.load ("cppget.org/misc")); + shared_ptr tr (db.load ("cppget.org/testing")); + shared_ptr gr (db.load ("cppget.org/staging")); + + // Verify 'stable' repository. + // + assert (sr->location.canonical_name () == "cppget.org/stable"); + assert (sr->location.string () == + "http://pkg.cppget.org/1/stable"); + assert (sr->display_name == "stable"); + assert (sr->priority == 1); + assert (!sr->url); + assert (sr->email && *sr->email == "repoman@cppget.org"); + assert (sr->summary && + *sr->summary == "General C++ package stable repository"); + assert (sr->description && *sr->description == + "This is the awesome C++ package repository full of exciting " + "stuff."); + + dir_path srp (cp.directory () / dir_path ("1/stable")); + assert (sr->local_path == srp.normalize ()); + + assert (sr->packages_timestamp == srt); + assert (sr->repositories_timestamp == + file_mtime (dir_path (sr->local_path) / path ("repositories"))); + assert (sr->internal); + + shared_ptr fpv1 ( + db.load (package_id ("libfoo", version ("1.0")))); + assert (check_location (fpv1)); + + shared_ptr fpv2 ( + db.load (package_id ("libfoo", version ("1.2.2")))); + assert (check_location (fpv2)); + + shared_ptr fpv2a ( + db.load (package_id ("libfoo", version ("1.2.2-alpha.1")))); + assert (check_location (fpv2a)); + + shared_ptr fpv3 ( + db.load (package_id ("libfoo", version ("1.2.3+4")))); + assert (check_location (fpv3)); + + shared_ptr fpv4 ( + db.load (package_id ("libfoo", version ("1.2.4")))); + assert (check_location (fpv4)); + + assert (sr->complements.empty ()); + assert (sr->prerequisites.size () == 2); + assert (sr->prerequisites[0].load () == cr); + assert (sr->prerequisites[1].load () == mr); + + // Verify libfoo package versions. + // + // libfoo-1.0 + // + assert (fpv1->summary == "The Foo Library"); + assert (fpv1->tags.empty ()); + assert (!fpv1->description); + assert (fpv1->url == "http://www.example.com/foo/"); + assert (!fpv1->package_url); + assert (fpv1->email == "foo-users@example.com"); + assert (!fpv1->package_email); + + assert (fpv1->internal_repository.load () == sr); + assert (fpv1->other_repositories.size () == 2); + assert (fpv1->other_repositories[0].load () == mr); + assert (fpv1->other_repositories[1].load () == cr); + + assert (fpv1->priority == priority::low); + assert (fpv1->changes.empty ()); + + assert (fpv1->license_alternatives.size () == 1); + assert (fpv1->license_alternatives[0].size () == 1); + assert (fpv1->license_alternatives[0][0] == "MIT"); + + assert (fpv1->dependencies.empty ()); + assert (fpv1->requirements.empty ()); + + // libfoo-1.2.2 + // + assert (fpv2->summary == "The Foo library"); + assert (fpv2->tags == strings ({"c++", "foo"})); + assert (!fpv2->description); + assert (fpv2->url == "http://www.example.com/foo/"); + assert (!fpv2->package_url); + assert (fpv2->email == "foo-users@example.com"); + assert (!fpv2->package_email); + + assert (fpv2->internal_repository.load () == sr); + assert (fpv2->other_repositories.empty ()); + assert (fpv2->priority == priority::low); + assert (fpv2->changes.empty ()); + + assert (fpv2->license_alternatives.size () == 1); + assert (fpv2->license_alternatives[0].size () == 1); + assert (fpv2->license_alternatives[0][0] == "MIT"); + + assert (fpv2->dependencies.size () == 2); + assert (fpv2->dependencies[0].size () == 1); + assert (fpv2->dependencies[1].size () == 1); + + auto dep ( + [&db](const char* n, + const optional& c) -> dependency + { + return {lazy_shared_ptr (db, package_id (n, version ())), c}; + }); + + assert (fpv2->dependencies[0][0] == + dep ( + "libbar", + optional ( + dependency_constraint ( + nullopt, true, version ("2.4.0"), false)))); + + assert (fpv2->dependencies[1][0] == + dep ( + "libexp", + optional ( + dependency_constraint ( + version ("1~1.2"), false, version ("1~1.2"), false)))); + + // libfoo-1.2.2-alpha.1 + // + assert (fpv2a->summary == "The Foo library"); + assert (fpv2a->tags == strings ({"c++", "foo"})); + assert (!fpv2a->description); + assert (fpv2a->url == "http://www.example.com/foo/"); + assert (!fpv2a->package_url); + assert (fpv2a->email == "foo-users@example.com"); + assert (!fpv2a->package_email); + + assert (fpv2a->internal_repository.load () == sr); + assert (fpv2a->other_repositories.empty ()); + assert (fpv2a->priority == priority::low); + assert (fpv2a->changes.empty ()); + + assert (fpv2a->license_alternatives.size () == 1); + assert (fpv2a->license_alternatives[0].size () == 1); + assert (fpv2a->license_alternatives[0][0] == "MIT"); + + assert (fpv2a->dependencies.size () == 3); + assert (fpv2a->dependencies[0].size () == 2); + assert (fpv2a->dependencies[1].size () == 1); + assert (fpv2a->dependencies[2].size () == 2); + + assert (fpv2a->dependencies[0][0] == + dep ( + "libmisc", + optional ( + dependency_constraint ( + version ("0.1"), false, version ("2.0.0-"), true)))); + + assert (fpv2a->dependencies[0][1] == + dep ( + "libmisc", + optional ( + dependency_constraint ( + version ("2.0"), false, version ("5.0"), false)))); + + assert (fpv2a->dependencies[1][0] == + dep ( + "libgenx", + optional ( + dependency_constraint ( + version ("0.2"), true, version ("3.0"), true)))); + + assert (fpv2a->dependencies[2][0] == + dep ( + "libexpat", + optional ( + dependency_constraint ( + nullopt, true, version ("5.2"), true)))); + + assert (fpv2a->dependencies[2][1] == + dep ( + "libexpat", + optional ( + dependency_constraint ( + version ("1"), true, version ("5.1"), false)))); + + assert (fpv2a->requirements.empty ()); + + // libfoo-1.2.3-4 + // + assert (fpv3->summary == "The Foo library"); + assert (fpv3->tags == strings ({"c++", "foo"})); + assert (!fpv3->description); + assert (fpv3->url == "http://www.example.com/foo/"); + assert (!fpv3->package_url); + assert (fpv3->email == "foo-users@example.com"); + assert (!fpv3->package_email); + + assert (fpv3->internal_repository.load () == sr); + assert (fpv3->other_repositories.empty ()); + assert (fpv3->priority == priority::low); + + assert (fpv3->changes.empty ()); + + assert (fpv3->license_alternatives.size () == 1); + assert (fpv3->license_alternatives[0].size () == 1); + assert (fpv3->license_alternatives[0][0] == "MIT"); + + assert (fpv3->dependencies.size () == 1); + assert (fpv3->dependencies[0].size () == 1); + assert (fpv3->dependencies[0][0] == + dep ( + "libmisc", + optional ( + dependency_constraint ( + version ("2.0.0"), false, nullopt, true)))); + + // libfoo-1.2.4 + // + assert (fpv4->summary == "The Foo Library"); + assert (fpv4->tags == strings ({"c++", "foo"})); + assert (*fpv4->description == "Very good foo library."); + assert (fpv4->url == "http://www.example.com/foo/"); + assert (!fpv4->package_url); + assert (fpv4->email == "foo-users@example.com"); + assert (!fpv4->package_email); + + assert (fpv4->internal_repository.load () == sr); + assert (fpv4->other_repositories.empty ()); + assert (fpv4->priority == priority::low); + assert (fpv4->changes == "some changes 1\nsome changes 2"); + + assert (fpv4->license_alternatives.size () == 1); + assert (fpv4->license_alternatives[0].comment == + "Permissive free software license."); + assert (fpv4->license_alternatives[0].size () == 1); + assert (fpv4->license_alternatives[0][0] == "MIT"); + + assert (fpv4->dependencies.size () == 1); + assert (fpv4->dependencies[0].size () == 1); + assert (fpv4->dependencies[0][0] == + dep ( + "libmisc", + optional ( + dependency_constraint ( + version ("2.0.0"), false, nullopt, true)))); + + // Verify 'math' repository. + // + assert (mr->location.canonical_name () == "cppget.org/math"); + assert (mr->location.string () == + "http://pkg.cppget.org/1/math"); + assert (mr->display_name == "math"); + assert (mr->priority == 2); + assert (!mr->url); + assert (mr->email && *mr->email == "repoman@cppget.org"); + assert (mr->summary && *mr->summary == "Math C++ package repository"); + assert (mr->description && *mr->description == + "This is the awesome C++ package repository full of remarkable " + "algorithms and\nAPIs."); + + dir_path mrp (cp.directory () / dir_path ("1/math")); + assert (mr->local_path == mrp.normalize ()); + + assert (mr->packages_timestamp == + file_mtime (dir_path (mr->local_path) / path ("packages"))); + assert (mr->repositories_timestamp == + file_mtime (dir_path (mr->local_path) / path ("repositories"))); + assert (mr->internal); + + shared_ptr epv ( + db.load (package_id ("libexp", version ("1~1.2")))); + assert (check_location (epv)); + + shared_ptr fpv5 ( + db.load (package_id ("libfoo", version ("1.2.4+1")))); + assert (check_location (fpv5)); + + shared_ptr xpv ( + db.load (package_id ("libstudxml", version ("1.0.0+1")))); + assert (check_location (xpv)); + + assert (mr->complements.empty ()); + assert (mr->prerequisites.size () == 1); + assert (mr->prerequisites[0].load () == cr); + + // Verify libstudxml package version. + // + assert (xpv->summary == "Modern C++ XML API"); + assert (xpv->tags == strings ({"c++", "xml", "parser", "serializer", + "pull", "streaming", "modern"})); + assert (!xpv->description); + assert (xpv->url == "http://www.codesynthesis.com/projects/libstudxml/"); + assert (!xpv->package_url); + assert (xpv->email == + email ("studxml-users@codesynthesis.com", + "Public mailing list, posts by non-members " + "are allowed but moderated.")); + assert (xpv->package_email && + *xpv->package_email == email ("boris@codesynthesis.com", + "Direct email to the author.")); + + assert (xpv->internal_repository.load () == mr); + assert (xpv->other_repositories.empty ()); + assert (xpv->priority == priority::low); + assert (xpv->changes.empty ()); + + assert (xpv->license_alternatives.size () == 1); + assert (xpv->license_alternatives[0].size () == 1); + assert (xpv->license_alternatives[0][0] == "MIT"); + + assert (xpv->dependencies.size () == 2); + assert (xpv->dependencies[0].size () == 1); + assert (xpv->dependencies[0][0] == + dep ( + "libexpat", + optional ( + dependency_constraint ( + version ("2.0.0"), false, nullopt, true)))); + + assert (xpv->dependencies[1].size () == 1); + assert (xpv->dependencies[1][0] == dep ("libgenx", nullopt)); + + assert (xpv->requirements.empty ()); + + // Verify libfoo package versions. + // + // libfoo-1.2.4-1 + // + assert (fpv5->summary == "The Foo Math Library"); + assert (fpv5->tags == strings ({"c++", "foo", "math"})); + assert (*fpv5->description == + "A modern C++ library with easy to use linear algebra and lot of " + "optimization\ntools.\n\nThere are over 100 functions in total " + "with an extensive test suite. The API is\nsimilar to MATLAB." + "\n\nUseful for conversion of research code into production " + "environments."); + assert (fpv5->url == "http://www.example.com/foo/"); + assert (fpv5->package_url && + *fpv5->package_url == "http://www.example.com/foo/pack"); + assert (fpv5->email == "foo-users@example.com"); + assert (fpv5->package_email && + *fpv5->package_email == "pack@example.com"); + + assert (fpv5->internal_repository.load () == mr); + assert (fpv5->other_repositories.size () == 1); + assert (fpv5->other_repositories[0].load () == cr); + + assert (fpv5->priority == priority::high); + assert (fpv5->priority.comment == + "Critical bug fixes, performance improvement."); + + const char ch[] = R"DLM(1.2.4+1 + * applied patch for critical bug-219 + * regenerated documentation + +1.2.4 + * test suite extended significantly)DLM"; + + assert (fpv5->changes == ch); + + assert (fpv5->license_alternatives.size () == 2); + assert (fpv5->license_alternatives[0].comment == + "If using with GNU TLS."); + assert (fpv5->license_alternatives[0].size () == 2); + assert (fpv5->license_alternatives[0][0] == "LGPLv2"); + assert (fpv5->license_alternatives[0][1] == "MIT"); + assert (fpv5->license_alternatives[1].comment == + "If using with OpenSSL."); + assert (fpv5->license_alternatives[1].size () == 1); + assert (fpv5->license_alternatives[1][0] == "BSD"); + + assert (fpv5->dependencies.size () == 3); + assert (fpv5->dependencies[0].size () == 2); + assert (fpv5->dependencies[0].comment == + "Crashes with 1.1.0-2.3.0."); + + assert (fpv5->dependencies[0][0] == + dep ( + "libmisc", + optional ( + dependency_constraint ( + nullopt, true, version ("1.1"), true)))); + + assert (fpv5->dependencies[0][1] == + dep ( + "libmisc", + optional ( + dependency_constraint ( + version ("2.3.0"), true, nullopt, true)))); + + assert (fpv5->dependencies[1].size () == 1); + assert (fpv5->dependencies[1].comment.empty ()); + + assert (fpv5->dependencies[1][0] == + dep ("libexp", + optional ( + dependency_constraint ( + version ("1.0"), false, nullopt, true)))); + + assert (fpv5->dependencies[2].size () == 2); + assert (fpv5->dependencies[2].comment == "The newer the better."); + + assert (fpv5->dependencies[2][0] == dep ("libstudxml", nullopt)); + assert (fpv5->dependencies[2][1] == dep ("libexpat", nullopt)); + + requirements& fpvr5 (fpv5->requirements); + assert (fpvr5.size () == 4); + + assert (fpvr5[0] == strings ({"linux", "windows", "macosx"})); + assert (!fpvr5[0].conditional); + assert (fpvr5[0].comment == "Symbian support is coming."); + + assert (fpvr5[1] == strings ({"c++11"})); + assert (!fpvr5[1].conditional); + assert (fpvr5[1].comment.empty ()); + + assert (fpvr5[2].empty ()); + assert (fpvr5[2].conditional); + assert (fpvr5[2].comment == + "libc++ standard library if using Clang on Mac OS X."); + + assert (fpvr5[3] == strings ({"vc++ >= 12.0"})); + assert (fpvr5[3].conditional); + assert (fpvr5[3].comment == "Only if using VC++ on Windows."); + + // Verify libexp package version. + // + // libexp-1+1.2 + // + assert (epv->summary == "The exponent"); + assert (epv->tags == strings ({"c++", "exponent"})); + assert (epv->description && *epv->description == + "The exponent math function."); + assert (epv->url == "http://www.exp.com"); + assert (!epv->package_url); + assert (epv->email == email ("users@exp.com")); + assert (!epv->package_email); + + assert (epv->internal_repository.load () == mr); + assert (epv->other_repositories.empty ()); + assert (epv->priority == priority (priority::low)); + assert (epv->changes.empty ()); + + assert (epv->license_alternatives.size () == 1); + assert (epv->license_alternatives[0].size () == 1); + assert (epv->license_alternatives[0][0] == "MIT"); + + assert (epv->dependencies.size () == 1); + assert (epv->dependencies[0].size () == 1); + assert (epv->dependencies[0][0] == dep ("libmisc", nullopt)); + + assert (epv->requirements.empty ()); + + // Verify 'misc' repository. + // + assert (cr->location.canonical_name () == "cppget.org/misc"); + assert (cr->location.string () == + "http://pkg.cppget.org/1/misc"); + assert (cr->display_name.empty ()); + assert (cr->priority == 0); + assert (cr->url && *cr->url == "http://misc.cppget.org/"); + assert (!cr->email); + assert (!cr->summary); + assert (!cr->description); + + dir_path crp (cp.directory () / dir_path ("1/misc")); + assert (cr->local_path == crp.normalize ()); + + assert (cr->packages_timestamp == + file_mtime (dir_path (cr->local_path) / path ("packages"))); + assert (cr->repositories_timestamp == + file_mtime (dir_path (cr->local_path) / path ("repositories"))); + assert (!cr->internal); + + shared_ptr bpv ( + db.load (package_id ("libbar", version ("2.3.5")))); + assert (check_location (bpv)); + + shared_ptr fpv0 ( + db.load (package_id ("libfoo", version ("0.1")))); + assert (check_location (fpv0)); + + shared_ptr fpv6 ( + db.load (package_id ("libfoo", version ("1.2.4+2")))); + assert (check_location (fpv6)); + + assert (cr->prerequisites.empty ()); + assert (cr->complements.size () == 1); + assert (cr->complements[0].load () == tr); + + // Verify libbar package version. + // + // libbar-2.3.5 + // + assert (check_external (*bpv)); + assert (bpv->other_repositories.size () == 1); + assert (bpv->other_repositories[0].load () == cr); + + // Verify libfoo package versions. + // + // libfoo-0.1 + // + assert (check_external (*fpv0)); + assert (fpv0->other_repositories.size () == 1); + assert (fpv0->other_repositories[0].load () == cr); + + // libfoo-1.2.4-2 + // + assert (check_external (*fpv6)); + assert (fpv6->other_repositories.size () == 1); + assert (fpv6->other_repositories[0].load () == cr); + + // Verify 'testing' repository. + // + assert (tr->location.canonical_name () == "cppget.org/testing"); + assert (tr->location.string () == + "http://pkg.cppget.org/1/testing"); + assert (tr->display_name.empty ()); + assert (tr->priority == 0); + assert (tr->url && *tr->url == "http://test.cppget.org/hello/"); + assert (!tr->email); + assert (!tr->summary); + assert (!tr->description); + + dir_path trp (cp.directory () / dir_path ("1/testing")); + assert (tr->local_path == trp.normalize ()); + + assert (tr->packages_timestamp == + file_mtime (dir_path (tr->local_path) / path ("packages"))); + assert (tr->repositories_timestamp == + file_mtime (dir_path (tr->local_path) / path ("repositories"))); + assert (!tr->internal); + + shared_ptr mpv0 ( + db.load (package_id ("libmisc", version ("2.4.0")))); + assert (check_location (mpv0)); + + assert (tr->prerequisites.empty ()); + assert (tr->complements.size () == 1); + assert (tr->complements[0].load () == gr); + + // Verify libmisc package version. + // + // libmisc-2.4.0 + // + assert (check_external (*mpv0)); + assert (mpv0->other_repositories.size () == 1); + assert (mpv0->other_repositories[0].load () == tr); + + // Verify 'staging' repository. + // + assert (gr->location.canonical_name () == "cppget.org/staging"); + assert (gr->location.string () == + "http://pkg.cppget.org/1/staging"); + assert (gr->display_name.empty ()); + assert (gr->priority == 0); + assert (gr->url && *gr->url == "http://cppget.org/"); + assert (!gr->email); + assert (!gr->summary); + assert (!gr->description); + + dir_path grp (cp.directory () / dir_path ("1/staging")); + assert (gr->local_path == grp.normalize ()); + + assert (gr->packages_timestamp == + file_mtime (dir_path (gr->local_path) / path ("packages"))); + assert (gr->repositories_timestamp == + file_mtime (dir_path (gr->local_path) / path ("repositories"))); + assert (!gr->internal); + + shared_ptr tpv ( + db.load (package_id ("libexpat", version ("5.1")))); + assert (check_location (tpv)); + + shared_ptr gpv ( + db.load (package_id ("libgenx", version ("1.0")))); + assert (check_location (gpv)); + + shared_ptr mpv1 ( + db.load (package_id ("libmisc", version ("1.0")))); + assert (check_location (mpv1)); + + assert (gr->prerequisites.empty ()); + assert (gr->complements.empty ()); + + // Verify libexpat package version. + // + // libexpat-5.1 + // + assert (check_external (*tpv)); + assert (tpv->other_repositories.size () == 1); + assert (tpv->other_repositories[0].load () == gr); + + // Verify libgenx package version. + // + // libgenx-1.0 + // + assert (check_external (*gpv)); + assert (gpv->other_repositories.size () == 1); + assert (gpv->other_repositories[0].load () == gr); + + // Verify libmisc package version. + // + // libmisc-1.0 + // + assert (check_external (*mpv1)); + assert (mpv1->other_repositories.size () == 1); + assert (mpv1->other_repositories[0].load () == gr); + + // Change package summary, update the object persistent state, rerun + // loader and ensure the model were not rebuilt. + // + bpv->summary = "test"; + db.update (bpv); + + t.commit (); + } + + assert (process (ld_args).wait ()); + transaction t (db.begin ()); + + shared_ptr bpv ( + db.load (package_id ("libbar", version ("2.3.5")))); + + assert (bpv->summary == "test"); + + t.commit (); + } + // Fully qualified to avoid ambiguity with odb exception. + // + catch (const std::exception& e) + { + cerr << e.what () << endl; + return 1; + } +} diff --git a/tests/load/r.conf b/tests/load/r.conf new file mode 100644 index 0000000..8da4b77 --- /dev/null +++ b/tests/load/r.conf @@ -0,0 +1,2 @@ +http://pkg.cppget.org/1/stable stable 1/stable +http://pkg.cppget.org/1/math math 1/math diff --git a/tests/loader/1/basics/packages b/tests/loader/1/basics/packages deleted file mode 100644 index 86c20c1..0000000 --- a/tests/loader/1/basics/packages +++ /dev/null @@ -1,8 +0,0 @@ -: 1 -name: libexpat -version: 5.1 -summary: The Expat Library -license: MIT -url: http://www.example.com/expat/ -email: expat-users@example.com -location: libexpat-5.1.tar.gz diff --git a/tests/loader/1/basics/repositories b/tests/loader/1/basics/repositories deleted file mode 100644 index 57a1c7a..0000000 --- a/tests/loader/1/basics/repositories +++ /dev/null @@ -1,4 +0,0 @@ -: 1 -# Local repository manifest (this repository). -# -url: http://basics.org diff --git a/tests/loader/1/math/packages b/tests/loader/1/math/packages deleted file mode 100644 index 7b81c5e..0000000 --- a/tests/loader/1/math/packages +++ /dev/null @@ -1,71 +0,0 @@ -: 1 -name: libstudxml -version: 1.0.0+1 -summary: Modern C++ XML API -license: MIT -tags: c++, xml, parser, serializer, pull, streaming, modern -description-file: README -changes-file: NEWS -url: http://www.codesynthesis.com/projects/libstudxml/ -email: studxml-users@codesynthesis.com; Public mailing list, posts by\ - non-members are allowed but moderated. -package-email: boris@codesynthesis.com; Direct email to the author. -depends: libexpat >= 2.0.0 -depends: libgenx -location: libstudxml-1.0.0+1.tar.gz -: -name: libexp -version: 1~1.2 -summary: The exponent -description: The exponent math function. -license: MIT -tags: c++, exponent -url: http://www.exp.com -email: users@exp.com -depends: libmisc -location: libexp-1~1.2.tar.gz -: -name: libfoo -version: 1.2.4+1 -summary: The Foo Math Library -description:\ -A modern C++ library with easy to use linear algebra and lot of optimization -tools. - -There are over 100 functions in total with an extensive test suite. The API is -similar to MATLAB. - -Useful for conversion of research code into production environments. -\ -license: LGPLv2, MIT; If using with GNU TLS. -license: BSD; If using with OpenSSL. -priority: high; Critical bug fixes, performance improvement. -tags: c++, foo, math -url: http://www.example.com/foo/; Project home page. -email: foo-users@example.com; Public mailing list. Read FAQ before posting. -package-url: http://www.example.com/foo/pack; Package details. -package-email: pack@example.com; Current packager. -depends: libmisc < 1.1 | libmisc > 2.3.0; Crashes with 1.1.0-2.3.0. -depends: libexp >= 1.0 -depends: ? libstudxml | libexpat; The newer the better. -requires: linux | windows | macosx; Symbian support is coming. -requires: c++11 -requires: ? ; libc++ standard library if using Clang on Mac OS X. -requires: ? vc++ >= 12.0; Only if using VC++ on Windows. -location: libfoo-1.2.4+1.tar.gz -changes:\ -1.2.4+1 - * applied patch for critical bug-219 - * regenerated documentation - -1.2.4 - * test suite extended significantly -\ -: -name: libfoo -version: 1.0 -summary: The Foo Lib -license: MIT -url: http://www.example.com/foo/ -email: foo-users@example.com -location: libfoo-1.0.tar.gz diff --git a/tests/loader/1/math/repositories b/tests/loader/1/math/repositories deleted file mode 100644 index 20aa30d..0000000 --- a/tests/loader/1/math/repositories +++ /dev/null @@ -1,13 +0,0 @@ -: 1 -# Foreign repository manifest. -# -location: ../misc -: -# Local repository manifest (this repository). -# -email: repoman@cppget.org -summary: Math C++ package repository -description: \ -This is the awesome C++ package repository full of remarkable algorithms and -APIs. -\ diff --git a/tests/loader/1/misc/packages b/tests/loader/1/misc/packages deleted file mode 100644 index fec3780..0000000 --- a/tests/loader/1/misc/packages +++ /dev/null @@ -1,47 +0,0 @@ -: 1 -name: libbar -version: 2.3.5 -priority: security; Very important to install. -summary: The Bar library -description: very very good library. -license: GPLv2 -tags: c++, bar -url: http://www.example.com/bar/ -email: bar-users@example.com -depends: libfoo -depends: libmath >= 2.0.0 -requires: linux | windows | macosx -changes: some changes -location: libbar-2.3.5.tar.gz -: -name: libfoo -version: 1.0 -summary: Foo Library -license: MIT -url: http://www.example.com/foo/ -email: foo-users@example.com -location: libfoo-1.0.tar.gz -: -name: libfoo -version: 0.1 -summary: Foo -license: MIT -url: http://www.example.com/foo/ -email: foo-users@example.com -location: libfoo-0.1.tar.gz -: -name: libfoo -version: 1.2.4+1 -summary: Foo Library -license: MIT -url: http://www.example.com/foo/ -email: foo-users@example.com -location: libfoo-1.2.4+1.tar.gz -: -name: libfoo -version: 1.2.4+2 -summary: Foo Library -license: MIT -url: http://www.example.com/foo/ -email: foo-users@example.com -location: libfoo-1.2.4+2.tar.gz diff --git a/tests/loader/1/misc/repositories b/tests/loader/1/misc/repositories deleted file mode 100644 index 1a41290..0000000 --- a/tests/loader/1/misc/repositories +++ /dev/null @@ -1,13 +0,0 @@ -: 1 -# Foreign repository manifest. -# -location: ../basics -: -# Adjacent repository manifest. -# -location: ../testing -role: complement -: -# Local repository manifest (this repository). -# -url: http://misc.cppget.org/ diff --git a/tests/loader/1/stable/packages b/tests/loader/1/stable/packages deleted file mode 100644 index afa168a..0000000 --- a/tests/loader/1/stable/packages +++ /dev/null @@ -1,54 +0,0 @@ -: 1 -name: libfoo -version: 1.2.3+4 -summary: The Foo library -license: MIT -tags: c++, foo -url: http://www.example.com/foo/ -email: foo-users@example.com -depends: libmisc >= 2.0.0 -location: libfoo-1.2.3+4.tar.gz -: -name: libfoo -version: 1.2.2 -summary: The Foo library -license: MIT -tags: c++, foo -url: http://www.example.com/foo/ -email: foo-users@example.com -depends: libbar <= 2.4.0 -depends: libexp == 1~1.2 -location: libfoo-1.2.2.tar.gz -: -name: libfoo -version: 1.2.2-alpha.1 -summary: The Foo library -license: MIT -tags: c++, foo -url: http://www.example.com/foo/ -email: foo-users@example.com -depends: libmisc [0.1 2.0-) | libmisc [2.0 5.0] -depends: libgenx (0.2 3.0) -depends: libexpat < 5.2 | libexpat (1 5.1] -location: libfoo-1.2.2-alpha.1.tar.gz -: -name: libfoo -version: 1.2.4 -summary: The Foo Library -description: Very good foo library. -license: MIT; Permissive free software license. -tags: c++, foo -url: http://www.example.com/foo/ -email: foo-users@example.com -depends: libmisc >= 2.0.0 -changes: some changes 1 -changes: some changes 2 -location: libfoo-1.2.4.tar.gz -: -name: libfoo -version: 1.0 -summary: The Foo Library -license: MIT -url: http://www.example.com/foo/ -email: foo-users@example.com -location: libfoo-1.0.tar.gz diff --git a/tests/loader/1/stable/repositories b/tests/loader/1/stable/repositories deleted file mode 100644 index b692ebe..0000000 --- a/tests/loader/1/stable/repositories +++ /dev/null @@ -1,14 +0,0 @@ -: 1 -# Foreign repository manifest. -# -location: ../misc -: -# Adjacent repository manifest. -# -location: ../math -: -# Local repository manifest (this repository). -# -email: repoman@cppget.org -summary: General C++ package stable repository -description: This is the awesome C++ package repository full of exciting stuff. diff --git a/tests/loader/1/staging/packages b/tests/loader/1/staging/packages deleted file mode 100644 index e7b22b0..0000000 --- a/tests/loader/1/staging/packages +++ /dev/null @@ -1,25 +0,0 @@ -: 1 -name: libexpat -version: 5.1 -summary: The Expat Library -license: MIT -url: http://www.example.com/expat/ -email: expat-users@example.com -location: libexpat-5.1.tar.gz -: -name: libgenx -version: 1.0 -summary: The Genx Library -license: MIT -url: http://www.example.com/genx/ -email: genx-users@example.com -location: libgenx-1.0.tar.gz -: -name: libmisc -version: 1.0 -summary: The Misc Library -license: MIT -url: http://www.example.com/misc/ -email: misc-users@example.com -depends: libexpat >= 5.0 -location: libmisc-1.0.tar.gz diff --git a/tests/loader/1/staging/repositories b/tests/loader/1/staging/repositories deleted file mode 100644 index d72a3f8..0000000 --- a/tests/loader/1/staging/repositories +++ /dev/null @@ -1,4 +0,0 @@ -: 1 -# Local repository manifest (this repository). -# -url: ../../.. diff --git a/tests/loader/1/testing/packages b/tests/loader/1/testing/packages deleted file mode 100644 index bdebece..0000000 --- a/tests/loader/1/testing/packages +++ /dev/null @@ -1,9 +0,0 @@ -: 1 -name: libmisc -version: 2.4.0 -summary: The Misc Library -license: MIT -url: http://www.example.com/misc/ -email: misc-users@example.com -depends: libexpat >= 5.0 -location: libmisc-2.4.0.tar.gz diff --git a/tests/loader/1/testing/repositories b/tests/loader/1/testing/repositories deleted file mode 100644 index a218d5c..0000000 --- a/tests/loader/1/testing/repositories +++ /dev/null @@ -1,9 +0,0 @@ -: 1 -# Adjacent repository manifest. -# -location: ../staging -role: complement -: -# Local repository manifest (this repository). -# -url: http://test.cppget.org/hello diff --git a/tests/loader/buildfile b/tests/loader/buildfile deleted file mode 100644 index d8ab9ac..0000000 --- a/tests/loader/buildfile +++ /dev/null @@ -1,21 +0,0 @@ -# file : tests/loader/buildfile -# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -# license : MIT; see accompanying LICENSE file - -import libs += libbpkg%lib{bpkg} -import libs += libbutl%lib{butl} -import libs += libodb-pgsql%lib{odb-pgsql} -import libs += libodb%lib{odb} - -include ../../brep/ - -exe{driver}: cxx{driver} ../../brep/lib{brep} $libs - -# Disable for now until build2 test module supports running custom -# commands, pre/post steps. -# -exe{driver}: test = false - -# precondition: PostgreSQL server running port 8432 with brep schema created. -# test: -# ./driver ../../loader/brep-loader --db-host localhost --db-port 8432 ./r.conf diff --git a/tests/loader/driver.cxx b/tests/loader/driver.cxx deleted file mode 100644 index 67093d9..0000000 --- a/tests/loader/driver.cxx +++ /dev/null @@ -1,756 +0,0 @@ -// file : tests/loader/driver.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include -#include -#include // sort(), find() - -#include -#include - -#include - -#include -#include - -#include -#include - -#include -#include - -using namespace std; -using namespace odb::core; -using namespace butl; -using namespace brep; - -// @@ Rather add this to optional in libbutl. See: -// -// http://en.cppreference.com/w/cpp/experimental/optional/operator_cmp - -template -static inline auto -operator== (const optional& a, const optional& b) -> decltype (*a == *b) -{ - return !a == !b && (!a || *a == *b); -} - -// @@ Add it to libbrep rather? -// -static inline bool -operator== (const dependency_constraint& a, const dependency_constraint& b) -{ - return a.min_version == b.min_version && a.max_version == b.max_version && - a.min_open == b.min_open && a.max_open == b.max_open; -} - -static inline bool -operator== (const dependency& a, const dependency& b) -{ - return a.name () == b.name () && a.constraint == b.constraint; -} - -static bool -check_location (shared_ptr& p) -{ - if (p->internal ()) - return p->location && *p->location == - path (p->id.name + "-" + p->version.string () + ".tar.gz"); - else - return !p->location; -} - -static bool -check_external (const package& p) -{ - return p.summary.empty () && p.tags.empty () && !p.description && - p.url.empty () && !p.package_url && p.email.empty () && !p.package_email && - !p.internal () && p.other_repositories.size () > 0 && - p.priority == priority () && p.changes.empty () && - p.license_alternatives.empty () && p.dependencies.empty () && - p.requirements.empty (); -} - -int -main (int argc, char* argv[]) -{ - if (argc != 7) - { - cerr << "usage: " << argv[0] - << " --db-host --db-port " - << " " << endl; - - return 1; - } - - try - { - path cp (argv[6]); - - // Make configuration file path absolute to use it's directory as base for - // internal repositories relative local paths. - // - if (cp.relative ()) - cp.complete (); - - // Update packages file timestamp to enforce loader to update - // persistent state. - // - path p (cp.directory () / path ("1/stable/packages")); - char const* args[] = {"touch", p.string ().c_str (), nullptr}; - assert (process (args).wait ()); - - timestamp srt (file_mtime (p)); - - // Run the loader. - // - char const** ld_args (const_cast (argv + 1)); - assert (process (ld_args).wait ()); - - // Check persistent objects validity. - // - odb::pgsql::database db ("", "", "brep", argv[3], stoul (argv[5])); - - { - session s; - transaction t (db.begin ()); - - assert (db.query ().size () == 5); - assert (db.query ().size () == 15); - - shared_ptr sr (db.load ("cppget.org/stable")); - shared_ptr mr (db.load ("cppget.org/math")); - shared_ptr cr (db.load ("cppget.org/misc")); - shared_ptr tr (db.load ("cppget.org/testing")); - shared_ptr gr (db.load ("cppget.org/staging")); - - // Verify 'stable' repository. - // - assert (sr->location.canonical_name () == "cppget.org/stable"); - assert (sr->location.string () == - "http://pkg.cppget.org/1/stable"); - assert (sr->display_name == "stable"); - assert (sr->priority == 1); - assert (!sr->url); - assert (sr->email && *sr->email == "repoman@cppget.org"); - assert (sr->summary && - *sr->summary == "General C++ package stable repository"); - assert (sr->description && *sr->description == - "This is the awesome C++ package repository full of exciting " - "stuff."); - - dir_path srp (cp.directory () / dir_path ("1/stable")); - assert (sr->local_path == srp.normalize ()); - - assert (sr->packages_timestamp == srt); - assert (sr->repositories_timestamp == - file_mtime (dir_path (sr->local_path) / path ("repositories"))); - assert (sr->internal); - - shared_ptr fpv1 ( - db.load (package_id ("libfoo", version ("1.0")))); - assert (check_location (fpv1)); - - shared_ptr fpv2 ( - db.load (package_id ("libfoo", version ("1.2.2")))); - assert (check_location (fpv2)); - - shared_ptr fpv2a ( - db.load (package_id ("libfoo", version ("1.2.2-alpha.1")))); - assert (check_location (fpv2a)); - - shared_ptr fpv3 ( - db.load (package_id ("libfoo", version ("1.2.3+4")))); - assert (check_location (fpv3)); - - shared_ptr fpv4 ( - db.load (package_id ("libfoo", version ("1.2.4")))); - assert (check_location (fpv4)); - - assert (sr->complements.empty ()); - assert (sr->prerequisites.size () == 2); - assert (sr->prerequisites[0].load () == cr); - assert (sr->prerequisites[1].load () == mr); - - // Verify libfoo package versions. - // - // libfoo-1.0 - // - assert (fpv1->summary == "The Foo Library"); - assert (fpv1->tags.empty ()); - assert (!fpv1->description); - assert (fpv1->url == "http://www.example.com/foo/"); - assert (!fpv1->package_url); - assert (fpv1->email == "foo-users@example.com"); - assert (!fpv1->package_email); - - assert (fpv1->internal_repository.load () == sr); - assert (fpv1->other_repositories.size () == 2); - assert (fpv1->other_repositories[0].load () == mr); - assert (fpv1->other_repositories[1].load () == cr); - - assert (fpv1->priority == priority::low); - assert (fpv1->changes.empty ()); - - assert (fpv1->license_alternatives.size () == 1); - assert (fpv1->license_alternatives[0].size () == 1); - assert (fpv1->license_alternatives[0][0] == "MIT"); - - assert (fpv1->dependencies.empty ()); - assert (fpv1->requirements.empty ()); - - // libfoo-1.2.2 - // - assert (fpv2->summary == "The Foo library"); - assert (fpv2->tags == strings ({"c++", "foo"})); - assert (!fpv2->description); - assert (fpv2->url == "http://www.example.com/foo/"); - assert (!fpv2->package_url); - assert (fpv2->email == "foo-users@example.com"); - assert (!fpv2->package_email); - - assert (fpv2->internal_repository.load () == sr); - assert (fpv2->other_repositories.empty ()); - assert (fpv2->priority == priority::low); - assert (fpv2->changes.empty ()); - - assert (fpv2->license_alternatives.size () == 1); - assert (fpv2->license_alternatives[0].size () == 1); - assert (fpv2->license_alternatives[0][0] == "MIT"); - - assert (fpv2->dependencies.size () == 2); - assert (fpv2->dependencies[0].size () == 1); - assert (fpv2->dependencies[1].size () == 1); - - auto dep ( - [&db](const char* n, - const optional& c) -> dependency - { - return {lazy_shared_ptr (db, package_id (n, version ())), c}; - }); - - assert (fpv2->dependencies[0][0] == - dep ( - "libbar", - optional ( - dependency_constraint ( - nullopt, true, version ("2.4.0"), false)))); - - assert (fpv2->dependencies[1][0] == - dep ( - "libexp", - optional ( - dependency_constraint ( - version ("1~1.2"), false, version ("1~1.2"), false)))); - - // libfoo-1.2.2-alpha.1 - // - assert (fpv2a->summary == "The Foo library"); - assert (fpv2a->tags == strings ({"c++", "foo"})); - assert (!fpv2a->description); - assert (fpv2a->url == "http://www.example.com/foo/"); - assert (!fpv2a->package_url); - assert (fpv2a->email == "foo-users@example.com"); - assert (!fpv2a->package_email); - - assert (fpv2a->internal_repository.load () == sr); - assert (fpv2a->other_repositories.empty ()); - assert (fpv2a->priority == priority::low); - assert (fpv2a->changes.empty ()); - - assert (fpv2a->license_alternatives.size () == 1); - assert (fpv2a->license_alternatives[0].size () == 1); - assert (fpv2a->license_alternatives[0][0] == "MIT"); - - assert (fpv2a->dependencies.size () == 3); - assert (fpv2a->dependencies[0].size () == 2); - assert (fpv2a->dependencies[1].size () == 1); - assert (fpv2a->dependencies[2].size () == 2); - - assert (fpv2a->dependencies[0][0] == - dep ( - "libmisc", - optional ( - dependency_constraint ( - version ("0.1"), false, version ("2.0.0-"), true)))); - - assert (fpv2a->dependencies[0][1] == - dep ( - "libmisc", - optional ( - dependency_constraint ( - version ("2.0"), false, version ("5.0"), false)))); - - assert (fpv2a->dependencies[1][0] == - dep ( - "libgenx", - optional ( - dependency_constraint ( - version ("0.2"), true, version ("3.0"), true)))); - - assert (fpv2a->dependencies[2][0] == - dep ( - "libexpat", - optional ( - dependency_constraint ( - nullopt, true, version ("5.2"), true)))); - - assert (fpv2a->dependencies[2][1] == - dep ( - "libexpat", - optional ( - dependency_constraint ( - version ("1"), true, version ("5.1"), false)))); - - assert (fpv2a->requirements.empty ()); - - // libfoo-1.2.3-4 - // - assert (fpv3->summary == "The Foo library"); - assert (fpv3->tags == strings ({"c++", "foo"})); - assert (!fpv3->description); - assert (fpv3->url == "http://www.example.com/foo/"); - assert (!fpv3->package_url); - assert (fpv3->email == "foo-users@example.com"); - assert (!fpv3->package_email); - - assert (fpv3->internal_repository.load () == sr); - assert (fpv3->other_repositories.empty ()); - assert (fpv3->priority == priority::low); - - assert (fpv3->changes.empty ()); - - assert (fpv3->license_alternatives.size () == 1); - assert (fpv3->license_alternatives[0].size () == 1); - assert (fpv3->license_alternatives[0][0] == "MIT"); - - assert (fpv3->dependencies.size () == 1); - assert (fpv3->dependencies[0].size () == 1); - assert (fpv3->dependencies[0][0] == - dep ( - "libmisc", - optional ( - dependency_constraint ( - version ("2.0.0"), false, nullopt, true)))); - - // libfoo-1.2.4 - // - assert (fpv4->summary == "The Foo Library"); - assert (fpv4->tags == strings ({"c++", "foo"})); - assert (*fpv4->description == "Very good foo library."); - assert (fpv4->url == "http://www.example.com/foo/"); - assert (!fpv4->package_url); - assert (fpv4->email == "foo-users@example.com"); - assert (!fpv4->package_email); - - assert (fpv4->internal_repository.load () == sr); - assert (fpv4->other_repositories.empty ()); - assert (fpv4->priority == priority::low); - assert (fpv4->changes == "some changes 1\nsome changes 2"); - - assert (fpv4->license_alternatives.size () == 1); - assert (fpv4->license_alternatives[0].comment == - "Permissive free software license."); - assert (fpv4->license_alternatives[0].size () == 1); - assert (fpv4->license_alternatives[0][0] == "MIT"); - - assert (fpv4->dependencies.size () == 1); - assert (fpv4->dependencies[0].size () == 1); - assert (fpv4->dependencies[0][0] == - dep ( - "libmisc", - optional ( - dependency_constraint ( - version ("2.0.0"), false, nullopt, true)))); - - // Verify 'math' repository. - // - assert (mr->location.canonical_name () == "cppget.org/math"); - assert (mr->location.string () == - "http://pkg.cppget.org/1/math"); - assert (mr->display_name == "math"); - assert (mr->priority == 2); - assert (!mr->url); - assert (mr->email && *mr->email == "repoman@cppget.org"); - assert (mr->summary && *mr->summary == "Math C++ package repository"); - assert (mr->description && *mr->description == - "This is the awesome C++ package repository full of remarkable " - "algorithms and\nAPIs."); - - dir_path mrp (cp.directory () / dir_path ("1/math")); - assert (mr->local_path == mrp.normalize ()); - - assert (mr->packages_timestamp == - file_mtime (dir_path (mr->local_path) / path ("packages"))); - assert (mr->repositories_timestamp == - file_mtime (dir_path (mr->local_path) / path ("repositories"))); - assert (mr->internal); - - shared_ptr epv ( - db.load (package_id ("libexp", version ("1~1.2")))); - assert (check_location (epv)); - - shared_ptr fpv5 ( - db.load (package_id ("libfoo", version ("1.2.4+1")))); - assert (check_location (fpv5)); - - shared_ptr xpv ( - db.load (package_id ("libstudxml", version ("1.0.0+1")))); - assert (check_location (xpv)); - - assert (mr->complements.empty ()); - assert (mr->prerequisites.size () == 1); - assert (mr->prerequisites[0].load () == cr); - - // Verify libstudxml package version. - // - assert (xpv->summary == "Modern C++ XML API"); - assert (xpv->tags == strings ({"c++", "xml", "parser", "serializer", - "pull", "streaming", "modern"})); - assert (!xpv->description); - assert (xpv->url == "http://www.codesynthesis.com/projects/libstudxml/"); - assert (!xpv->package_url); - assert (xpv->email == - email ("studxml-users@codesynthesis.com", - "Public mailing list, posts by non-members " - "are allowed but moderated.")); - assert (xpv->package_email && - *xpv->package_email == email ("boris@codesynthesis.com", - "Direct email to the author.")); - - assert (xpv->internal_repository.load () == mr); - assert (xpv->other_repositories.empty ()); - assert (xpv->priority == priority::low); - assert (xpv->changes.empty ()); - - assert (xpv->license_alternatives.size () == 1); - assert (xpv->license_alternatives[0].size () == 1); - assert (xpv->license_alternatives[0][0] == "MIT"); - - assert (xpv->dependencies.size () == 2); - assert (xpv->dependencies[0].size () == 1); - assert (xpv->dependencies[0][0] == - dep ( - "libexpat", - optional ( - dependency_constraint ( - version ("2.0.0"), false, nullopt, true)))); - - assert (xpv->dependencies[1].size () == 1); - assert (xpv->dependencies[1][0] == dep ("libgenx", nullopt)); - - assert (xpv->requirements.empty ()); - - // Verify libfoo package versions. - // - // libfoo-1.2.4-1 - // - assert (fpv5->summary == "The Foo Math Library"); - assert (fpv5->tags == strings ({"c++", "foo", "math"})); - assert (*fpv5->description == - "A modern C++ library with easy to use linear algebra and lot of " - "optimization\ntools.\n\nThere are over 100 functions in total " - "with an extensive test suite. The API is\nsimilar to MATLAB." - "\n\nUseful for conversion of research code into production " - "environments."); - assert (fpv5->url == "http://www.example.com/foo/"); - assert (fpv5->package_url && - *fpv5->package_url == "http://www.example.com/foo/pack"); - assert (fpv5->email == "foo-users@example.com"); - assert (fpv5->package_email && - *fpv5->package_email == "pack@example.com"); - - assert (fpv5->internal_repository.load () == mr); - assert (fpv5->other_repositories.size () == 1); - assert (fpv5->other_repositories[0].load () == cr); - - assert (fpv5->priority == priority::high); - assert (fpv5->priority.comment == - "Critical bug fixes, performance improvement."); - - const char ch[] = R"DLM(1.2.4+1 - * applied patch for critical bug-219 - * regenerated documentation - -1.2.4 - * test suite extended significantly)DLM"; - - assert (fpv5->changes == ch); - - assert (fpv5->license_alternatives.size () == 2); - assert (fpv5->license_alternatives[0].comment == - "If using with GNU TLS."); - assert (fpv5->license_alternatives[0].size () == 2); - assert (fpv5->license_alternatives[0][0] == "LGPLv2"); - assert (fpv5->license_alternatives[0][1] == "MIT"); - assert (fpv5->license_alternatives[1].comment == - "If using with OpenSSL."); - assert (fpv5->license_alternatives[1].size () == 1); - assert (fpv5->license_alternatives[1][0] == "BSD"); - - assert (fpv5->dependencies.size () == 3); - assert (fpv5->dependencies[0].size () == 2); - assert (fpv5->dependencies[0].comment == - "Crashes with 1.1.0-2.3.0."); - - assert (fpv5->dependencies[0][0] == - dep ( - "libmisc", - optional ( - dependency_constraint ( - nullopt, true, version ("1.1"), true)))); - - assert (fpv5->dependencies[0][1] == - dep ( - "libmisc", - optional ( - dependency_constraint ( - version ("2.3.0"), true, nullopt, true)))); - - assert (fpv5->dependencies[1].size () == 1); - assert (fpv5->dependencies[1].comment.empty ()); - - assert (fpv5->dependencies[1][0] == - dep ("libexp", - optional ( - dependency_constraint ( - version ("1.0"), false, nullopt, true)))); - - assert (fpv5->dependencies[2].size () == 2); - assert (fpv5->dependencies[2].comment == "The newer the better."); - - assert (fpv5->dependencies[2][0] == dep ("libstudxml", nullopt)); - assert (fpv5->dependencies[2][1] == dep ("libexpat", nullopt)); - - requirements& fpvr5 (fpv5->requirements); - assert (fpvr5.size () == 4); - - assert (fpvr5[0] == strings ({"linux", "windows", "macosx"})); - assert (!fpvr5[0].conditional); - assert (fpvr5[0].comment == "Symbian support is coming."); - - assert (fpvr5[1] == strings ({"c++11"})); - assert (!fpvr5[1].conditional); - assert (fpvr5[1].comment.empty ()); - - assert (fpvr5[2].empty ()); - assert (fpvr5[2].conditional); - assert (fpvr5[2].comment == - "libc++ standard library if using Clang on Mac OS X."); - - assert (fpvr5[3] == strings ({"vc++ >= 12.0"})); - assert (fpvr5[3].conditional); - assert (fpvr5[3].comment == "Only if using VC++ on Windows."); - - // Verify libexp package version. - // - // libexp-1+1.2 - // - assert (epv->summary == "The exponent"); - assert (epv->tags == strings ({"c++", "exponent"})); - assert (epv->description && *epv->description == - "The exponent math function."); - assert (epv->url == "http://www.exp.com"); - assert (!epv->package_url); - assert (epv->email == email ("users@exp.com")); - assert (!epv->package_email); - - assert (epv->internal_repository.load () == mr); - assert (epv->other_repositories.empty ()); - assert (epv->priority == priority (priority::low)); - assert (epv->changes.empty ()); - - assert (epv->license_alternatives.size () == 1); - assert (epv->license_alternatives[0].size () == 1); - assert (epv->license_alternatives[0][0] == "MIT"); - - assert (epv->dependencies.size () == 1); - assert (epv->dependencies[0].size () == 1); - assert (epv->dependencies[0][0] == dep ("libmisc", nullopt)); - - assert (epv->requirements.empty ()); - - // Verify 'misc' repository. - // - assert (cr->location.canonical_name () == "cppget.org/misc"); - assert (cr->location.string () == - "http://pkg.cppget.org/1/misc"); - assert (cr->display_name.empty ()); - assert (cr->priority == 0); - assert (cr->url && *cr->url == "http://misc.cppget.org/"); - assert (!cr->email); - assert (!cr->summary); - assert (!cr->description); - - dir_path crp (cp.directory () / dir_path ("1/misc")); - assert (cr->local_path == crp.normalize ()); - - assert (cr->packages_timestamp == - file_mtime (dir_path (cr->local_path) / path ("packages"))); - assert (cr->repositories_timestamp == - file_mtime (dir_path (cr->local_path) / path ("repositories"))); - assert (!cr->internal); - - shared_ptr bpv ( - db.load (package_id ("libbar", version ("2.3.5")))); - assert (check_location (bpv)); - - shared_ptr fpv0 ( - db.load (package_id ("libfoo", version ("0.1")))); - assert (check_location (fpv0)); - - shared_ptr fpv6 ( - db.load (package_id ("libfoo", version ("1.2.4+2")))); - assert (check_location (fpv6)); - - assert (cr->prerequisites.empty ()); - assert (cr->complements.size () == 1); - assert (cr->complements[0].load () == tr); - - // Verify libbar package version. - // - // libbar-2.3.5 - // - assert (check_external (*bpv)); - assert (bpv->other_repositories.size () == 1); - assert (bpv->other_repositories[0].load () == cr); - - // Verify libfoo package versions. - // - // libfoo-0.1 - // - assert (check_external (*fpv0)); - assert (fpv0->other_repositories.size () == 1); - assert (fpv0->other_repositories[0].load () == cr); - - // libfoo-1.2.4-2 - // - assert (check_external (*fpv6)); - assert (fpv6->other_repositories.size () == 1); - assert (fpv6->other_repositories[0].load () == cr); - - // Verify 'testing' repository. - // - assert (tr->location.canonical_name () == "cppget.org/testing"); - assert (tr->location.string () == - "http://pkg.cppget.org/1/testing"); - assert (tr->display_name.empty ()); - assert (tr->priority == 0); - assert (tr->url && *tr->url == "http://test.cppget.org/hello/"); - assert (!tr->email); - assert (!tr->summary); - assert (!tr->description); - - dir_path trp (cp.directory () / dir_path ("1/testing")); - assert (tr->local_path == trp.normalize ()); - - assert (tr->packages_timestamp == - file_mtime (dir_path (tr->local_path) / path ("packages"))); - assert (tr->repositories_timestamp == - file_mtime (dir_path (tr->local_path) / path ("repositories"))); - assert (!tr->internal); - - shared_ptr mpv0 ( - db.load (package_id ("libmisc", version ("2.4.0")))); - assert (check_location (mpv0)); - - assert (tr->prerequisites.empty ()); - assert (tr->complements.size () == 1); - assert (tr->complements[0].load () == gr); - - // Verify libmisc package version. - // - // libmisc-2.4.0 - // - assert (check_external (*mpv0)); - assert (mpv0->other_repositories.size () == 1); - assert (mpv0->other_repositories[0].load () == tr); - - // Verify 'staging' repository. - // - assert (gr->location.canonical_name () == "cppget.org/staging"); - assert (gr->location.string () == - "http://pkg.cppget.org/1/staging"); - assert (gr->display_name.empty ()); - assert (gr->priority == 0); - assert (gr->url && *gr->url == "http://cppget.org/"); - assert (!gr->email); - assert (!gr->summary); - assert (!gr->description); - - dir_path grp (cp.directory () / dir_path ("1/staging")); - assert (gr->local_path == grp.normalize ()); - - assert (gr->packages_timestamp == - file_mtime (dir_path (gr->local_path) / path ("packages"))); - assert (gr->repositories_timestamp == - file_mtime (dir_path (gr->local_path) / path ("repositories"))); - assert (!gr->internal); - - shared_ptr tpv ( - db.load (package_id ("libexpat", version ("5.1")))); - assert (check_location (tpv)); - - shared_ptr gpv ( - db.load (package_id ("libgenx", version ("1.0")))); - assert (check_location (gpv)); - - shared_ptr mpv1 ( - db.load (package_id ("libmisc", version ("1.0")))); - assert (check_location (mpv1)); - - assert (gr->prerequisites.empty ()); - assert (gr->complements.empty ()); - - // Verify libexpat package version. - // - // libexpat-5.1 - // - assert (check_external (*tpv)); - assert (tpv->other_repositories.size () == 1); - assert (tpv->other_repositories[0].load () == gr); - - // Verify libgenx package version. - // - // libgenx-1.0 - // - assert (check_external (*gpv)); - assert (gpv->other_repositories.size () == 1); - assert (gpv->other_repositories[0].load () == gr); - - // Verify libmisc package version. - // - // libmisc-1.0 - // - assert (check_external (*mpv1)); - assert (mpv1->other_repositories.size () == 1); - assert (mpv1->other_repositories[0].load () == gr); - - // Change package summary, update the object persistent state, rerun - // loader and ensure the model were not rebuilt. - // - bpv->summary = "test"; - db.update (bpv); - - t.commit (); - } - - assert (process (ld_args).wait ()); - transaction t (db.begin ()); - - shared_ptr bpv ( - db.load (package_id ("libbar", version ("2.3.5")))); - - assert (bpv->summary == "test"); - - t.commit (); - } - // Fully qualified to avoid ambiguity with odb exception. - // - catch (const std::exception& e) - { - cerr << e.what () << endl; - return 1; - } -} diff --git a/tests/loader/r.conf b/tests/loader/r.conf deleted file mode 100644 index 8da4b77..0000000 --- a/tests/loader/r.conf +++ /dev/null @@ -1,2 +0,0 @@ -http://pkg.cppget.org/1/stable stable 1/stable -http://pkg.cppget.org/1/math math 1/math diff --git a/web/apache/service.cxx b/web/apache/service.cxx index c4fe462..96ff855 100644 --- a/web/apache/service.cxx +++ b/web/apache/service.cxx @@ -125,7 +125,7 @@ namespace web // Terminate the root apache process. Indeed we can only try to // terminate the process, and most likely will fail in a production - // environment where the apache root process usually runs under root + // environment where the apache root process usually runs under root, // and worker processes run under some other user. This is why the // implementation should consider the possibility of not being // initialized at the time of HTTP request processing. In such a case -- cgit v1.1