aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--INSTALL55
-rw-r--r--INSTALL-DEV49
-rw-r--r--brep/.gitignore5
-rw-r--r--brep/build155
-rw-r--r--brep/build.cxx45
-rw-r--r--brep/build.xml64
-rw-r--r--brep/buildfile8
-rw-r--r--brep/common341
-rw-r--r--brep/common.cxx10
-rwxr-xr-xbrep/odb.sh19
-rw-r--r--brep/package338
-rw-r--r--brep/package.cxx2
-rw-r--r--brep/package.xml2
-rw-r--r--brep/utility2
-rw-r--r--brep/version5
-rw-r--r--etc/brep-apache2.conf12
-rw-r--r--etc/brep-module.conf81
-rw-r--r--load/load.cli9
-rw-r--r--load/load.cxx18
-rw-r--r--manifest1
-rw-r--r--migrate/migrate.cli10
-rw-r--r--migrate/migrate.cxx48
-rw-r--r--mod/build-config23
-rw-r--r--mod/build-config.cxx31
-rw-r--r--mod/buildfile7
-rw-r--r--mod/database15
-rw-r--r--mod/database-module24
-rw-r--r--mod/database-module.cxx52
-rw-r--r--mod/database.cxx59
-rw-r--r--mod/mod-build-log45
-rw-r--r--mod/mod-build-log.cxx220
-rw-r--r--mod/mod-build-result42
-rw-r--r--mod/mod-build-result.cxx303
-rw-r--r--mod/mod-build-task42
-rw-r--r--mod/mod-build-task.cxx389
-rw-r--r--mod/mod-package-details.cxx14
-rw-r--r--mod/mod-package-search.cxx22
-rw-r--r--mod/mod-package-version-details.cxx6
-rw-r--r--mod/mod-repository-details.cxx6
-rw-r--r--mod/mod-repository-root6
-rw-r--r--mod/mod-repository-root.cxx91
-rw-r--r--mod/options-types1
-rw-r--r--mod/options.cli192
-rw-r--r--mod/types-parsers15
-rw-r--r--mod/types-parsers.cxx19
-rw-r--r--tests/load/driver.cxx2
-rw-r--r--web/apache/request.cxx8
47 files changed, 2388 insertions, 525 deletions
diff --git a/INSTALL b/INSTALL
index b738683..d5c5560 100644
--- a/INSTALL
+++ b/INSTALL
@@ -91,37 +91,46 @@ bpkg install brep
$ cd .. # Back to brep home.
-4. Create PostgreSQL User and Database
+4. Create PostgreSQL User and Databases
$ sudo sudo -u postgres psql # Note: double sudo is not a mistake.
-CREATE DATABASE brep TEMPLATE template0 ENCODING 'UTF8'
+CREATE DATABASE brep_package TEMPLATE template0 ENCODING 'UTF8'
+LC_COLLATE 'en_US.UTF8' LC_CTYPE 'en_US.UTF8';
+CREATE DATABASE brep_build TEMPLATE template0 ENCODING 'UTF8'
LC_COLLATE 'en_US.UTF8' LC_CTYPE 'en_US.UTF8';
CREATE USER brep;
-GRANT ALL PRIVILEGES ON DATABASE brep TO brep;
+GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO brep;
CREATE USER "www-data" INHERIT IN ROLE brep;
Exit psql (^D), then make sure the logins work:
-$ psql
+$ psql -d brep_package
+^D
+$ psql -d brep_build
^D
-$ sudo sudo -u www-data psql -d brep
+$ sudo sudo -u www-data psql -d brep_package
+^D
+$ sudo sudo -u www-data psql -d brep_build
^D
To troubleshoot, see PostgreSQL logs.
-5. Create Database Schema and Load Repositories
+5. Create Database Schemes and Load Repositories
$ mkdir config
$ edit config/loadtab # Loader configuration, see brep-load(1).
-$ install/bin/brep-migrate
+$ install/bin/brep-migrate package
$ install/bin/brep-load config/loadtab
+$ install/bin/brep-migrate build
+
To verify:
-$ psql -c 'SELECT name, summary FROM repository'
+$ psql -d brep_package -c 'SELECT name, summary FROM repository'
+$ psql -d brep_build -c 'SELECT package_name FROM build' # Empty row set.
6. Setup Apache2 Module
@@ -143,6 +152,18 @@ can also find this fragment in install/share/brep/etc/brep-apache2.conf):
LoadModule brep_module /home/brep/install/libexec/brep/mod_brep.so
</IfModule>
+ # Repository email. This email is used for the From: header in emails
+ # send by brep (for example, build failure notifications).
+ #
+ brep-email admin@example.org
+
+ # Repository host. It specifies the scheme and the host address (but
+ # not the root path; see brep-root below) that will be used whenever
+ # brep needs to construct an absolute URL to one of its locations (for
+ # example, a link to a build log that is being send via email).
+ #
+ brep-host https://example.org
+
# Repository root. This is the part of the URL between the host name
# and the start of the repository. For example, root value /pkg means
# the repository URL is http://example.org/pkg/. Specify / to use the
@@ -192,13 +213,15 @@ can also find this fragment in install/share/brep/etc/brep-apache2.conf):
# Require all granted
#</Directory>
-The output content type of the brep module is application/xhtml+xml and if you
-would like to make sure it gets compressed (along with linked CSS), also add
-the following lines:
+The output content types of the brep module are application/xhtml+xml,
+text/manifest and text/plain. If you would like to make sure they get
+compressed (along with linked CSS), also add the following lines:
# Compress brep output (xhtml+xml) and CSS.
#
AddOutputFilterByType DEFLATE application/xhtml+xml
+ AddOutputFilterByType DEFLATE text/manifest
+ AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css
Restart Apache2:
@@ -338,14 +361,16 @@ Review brep-module.conf changes that may need to be merged:
$ diff -u install/share/brep/etc/brep-module.conf config/brep-module.conf
-Migrate database schema:
+Migrate database schemes:
-$ install/bin/brep-migrate
+$ install/bin/brep-migrate package
+$ install/bin/brep-migrate build
-Note that if instead you need to recreate the whole database (e.g., migration
+Note that if instead you need to recreate the whole databases (e.g., migration
is not possible), then one way to do it would be:
-$ psql -c 'DROP OWNED BY brep'
+$ psql -d brep_package -c 'DROP OWNED BY brep'
+$ psql -d brep_build -c 'DROP OWNED BY brep'
If using systemd, then start and enable the loader:
diff --git a/INSTALL-DEV b/INSTALL-DEV
index 8ad9ae7..8f5ba6f 100644
--- a/INSTALL-DEV
+++ b/INSTALL-DEV
@@ -24,39 +24,50 @@ setfacl -m g:www-data:rx ~/ ~/projects
group, not user. However, most installations use the same name for both.]
-1. Create PostgreSQL User and Database
+1. Create PostgreSQL User and Databases
$ sudo sudo -u postgres psql # Note: double sudo is not a mistake.
-CREATE DATABASE brep TEMPLATE template0 ENCODING 'UTF8'
+CREATE DATABASE brep_package TEMPLATE template0 ENCODING 'UTF8'
+LC_COLLATE 'en_US.UTF8' LC_CTYPE 'en_US.UTF8';
+CREATE DATABASE brep_build TEMPLATE template0 ENCODING 'UTF8'
LC_COLLATE 'en_US.UTF8' LC_CTYPE 'en_US.UTF8';
CREATE USER <user>;
-GRANT ALL PRIVILEGES ON DATABASE brep TO <user>;
+GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO <user>;
CREATE USER "www-data" INHERIT IN ROLE <user>;
Exit psql (^D), then make sure the logins work:
-$ psql -d brep
-$ sudo sudo -u www-data psql -d brep
+$ psql -d brep_package
+^D
+$ psql -d brep_build
+^D
+$ sudo sudo -u www-data psql -d brep_package
+^D
+$ sudo sudo -u www-data psql -d brep_build
+^D
To troubleshoot, see PostgreSQL logs, for example:
$ sudo tail -f /var/log/postgresql/*.log
-2. Create Database Schema and Load the Repository
+2. Create Database Schemes and Load the Repository
All the commands are executed from brep project root.
-$ migrate/brep-migrate
+$ migrate/brep-migrate package
# Or use some other loader config.
#
$ load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab
+$ migrate/brep-migrate build
+
To verify:
-$ psql -d brep -c 'SELECT name, summary FROM repository'
+$ psql -d brep_package -c 'SELECT name, summary FROM repository'
+$ psql -d brep_build -c 'SELECT package_name FROM build' # Empty row set.
3. Setup Apache2 Module
@@ -73,6 +84,18 @@ replacing <BREP-OUT-ROOT> and <BREP-SRC-ROOT> with the actual absolute paths
LoadModule brep_module <BREP-OUT-ROOT>/mod/mod_brep.so
</IfModule>
+ # Repository email. This email is used for the From: header in emails
+ # send by brep (for example, build failure notifications).
+ #
+ brep-email admin@example.org
+
+ # Repository host. It specifies the scheme and the host address (but
+ # not the root path; see brep-root below) that will be used whenever
+ # brep needs to construct an absolute URL to one of its locations (for
+ # example, a link to a build log that is being send via email).
+ #
+ brep-host https://example.org
+
# Repository root. Use / for web server root. And don't forget to also
# update the Location and Alias directives below.
#
@@ -117,15 +140,17 @@ $ sudo tail -f /var/log/apache2/error.log
4. Reloading During Development
-To do a "complete reload" (i.e., recreate database schema, load the repository
+To do a "complete reload" (i.e., recreate database schemes, load the repository
data, and reload the Apache2 plugin), execute the following from brep/:
-migrate/brep-migrate --recreate
+migrate/brep-migrate --recreate package
+migrate/brep-migrate --recreate build
load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab
sudo /etc/init.d/apache2 restart
sudo systemctl restart apache2
-Note that if instead you need to recreate the whole database (e.g., migration
+Note that if instead you need to recreate the whole databases (e.g., migration
is not possible), then one way to do it would be:
-$ psql -d brep -c 'DROP OWNED BY <user>'
+$ psql -d brep_package -c 'DROP OWNED BY <user>'
+$ psql -d brep_build -c 'DROP OWNED BY <user>'
diff --git a/brep/.gitignore b/brep/.gitignore
index 687b168..9852519 100644
--- a/brep/.gitignore
+++ b/brep/.gitignore
@@ -1,3 +1,8 @@
+common-odb*
+
package-odb*
package.sql
package-extra
+
+build-odb*
+build.sql
diff --git a/brep/build b/brep/build
new file mode 100644
index 0000000..3d969db
--- /dev/null
+++ b/brep/build
@@ -0,0 +1,155 @@
+// file : brep/build -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef BREP_BUILD
+#define BREP_BUILD
+
+#include <chrono>
+
+#include <odb/core.hxx>
+#include <odb/section.hxx>
+
+#include <bbot/manifest>
+
+#include <brep/types>
+#include <brep/utility>
+
+#include <brep/common> // Must be included last (see assert).
+
+// Used by the data migration entries.
+//
+#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 1
+
+#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 1, open)
+
+// We have to keep these mappings at the global scope instead of inside
+// the brep namespace because they need to be also effective in the
+// bbot namespace from which we "borrow" types (and some of them use the mapped
+// types).
+//
+#pragma db map type(bbot::result_status) as(std::string) \
+ to(to_string (?)) \
+ from(bbot::to_result_status (?))
+
+namespace brep
+{
+ #pragma db value
+ struct build_id
+ {
+ package_id package;
+ string configuration;
+
+ build_id () = default;
+ build_id (package_id p, string c)
+ : package (move (p)), configuration (move (c)) {}
+ };
+
+ inline bool
+ operator< (const build_id& x, const build_id& y)
+ {
+ return
+ x.package < y.package ? true :
+ y.package < x.package ? false :
+ x.configuration < y.configuration;
+ }
+
+ // build_state
+ //
+ enum class build_state: std::uint8_t
+ {
+ untested,
+ testing,
+ tested
+ };
+
+ string
+ to_string (build_state);
+
+ build_state
+ to_build_state (const string&); // May throw invalid_argument.
+
+ inline ostream&
+ operator<< (ostream& os, build_state s) {return os << to_string (s);}
+
+ #pragma db map type(build_state) as(string) \
+ to(to_string (?)) \
+ from(brep::to_build_state (?))
+
+ // result_status
+ //
+ using bbot::result_status;
+
+ using optional_result_status = optional<result_status>;
+
+ #pragma db map type(optional_result_status) as(optional_string) \
+ to((?) ? bbot::to_string (*(?)) : brep::optional_string ()) \
+ from((?) \
+ ? bbot::to_result_status (*(?)) \
+ : brep::optional_result_status ())
+
+ // operation_results
+ //
+ using bbot::operation_result;
+ #pragma db value(operation_result) definition
+
+ using bbot::operation_results;
+
+ #pragma db object pointer(shared_ptr) session
+ class build
+ {
+ public:
+ using timestamp_type = brep::timestamp;
+
+ // Create the build object with the testing state, non-existent status and
+ // the timestamp set to now.
+ //
+ build (string name, version, string configuration);
+
+ build_id id;
+
+ string& package_name; // Tracks id.package.name.
+ upstream_version package_version; // Original of id.package.version.
+ string& configuration; // Tracks id.configuration.
+
+ build_state state;
+
+ // Time of the last state change (the creation time initially).
+ //
+ timestamp_type timestamp;
+
+ // Present only if the state is 'tested'.
+ //
+ optional<result_status> status;
+
+ // Note that the logs are stored as std::string/TEXT which is Ok since
+ // they are UTF-8 and our database is UTF-8.
+ //
+ #pragma db section(results_section)
+ operation_results results;
+
+ #pragma db load(lazy) update(always)
+ odb::section results_section;
+
+ // Database mapping.
+ //
+ #pragma db member(id) id column("")
+
+ #pragma db member(package_name) transient
+ #pragma db member(package_version) \
+ set(this.package_version.init (this.id.package.version, (?)))
+ #pragma db member(configuration) transient
+
+ #pragma db member(results) id_column("") value_column("")
+
+ build (const build&) = delete;
+ build& operator= (const build&) = delete;
+
+ private:
+ friend class odb::access;
+ build ()
+ : package_name (id.package.name), configuration (id.configuration) {}
+ };
+}
+
+#endif // BREP_BUILD
diff --git a/brep/build.cxx b/brep/build.cxx
new file mode 100644
index 0000000..c93c062
--- /dev/null
+++ b/brep/build.cxx
@@ -0,0 +1,45 @@
+// file : brep/build.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <brep/build>
+
+namespace brep
+{
+ // build_state
+ //
+ string
+ to_string (build_state s)
+ {
+ switch (s)
+ {
+ case build_state::untested: return "untested";
+ case build_state::testing: return "testing";
+ case build_state::tested: return "tested";
+ }
+
+ return string (); // Should never reach.
+ }
+
+ build_state
+ to_build_state (const string& s)
+ {
+ if (s == "untested") return build_state::untested;
+ else if (s == "testing") return build_state::testing;
+ else if (s == "tested") return build_state::tested;
+ else throw invalid_argument ("invalid build state '" + s + "'");
+ }
+
+ // build
+ //
+ build::
+ build (string pnm, version pvr, string cfg)
+ : id (package_id (move (pnm), pvr), move (cfg)),
+ package_name (id.package.name),
+ package_version (move (pvr)),
+ configuration (id.configuration),
+ state (build_state::testing),
+ timestamp (timestamp_type::clock::now ())
+ {
+ }
+}
diff --git a/brep/build.xml b/brep/build.xml
new file mode 100644
index 0000000..5f75928
--- /dev/null
+++ b/brep/build.xml
@@ -0,0 +1,64 @@
+<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1">
+ <model version="1">
+ <table name="build" kind="object">
+ <column name="package_name" type="TEXT" null="false"/>
+ <column name="package_version_epoch" type="INTEGER" null="false"/>
+ <column name="package_version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="package_version_revision" type="INTEGER" null="false"/>
+ <column name="configuration" type="TEXT" null="false"/>
+ <column name="package_version_upstream" type="TEXT" null="false"/>
+ <column name="package_version_release" type="TEXT" null="true"/>
+ <column name="state" type="TEXT" null="false"/>
+ <column name="timestamp" type="BIGINT" null="false"/>
+ <column name="status" type="TEXT" null="true"/>
+ <primary-key>
+ <column name="package_name"/>
+ <column name="package_version_epoch"/>
+ <column name="package_version_canonical_upstream"/>
+ <column name="package_version_canonical_release"/>
+ <column name="package_version_revision"/>
+ <column name="configuration"/>
+ </primary-key>
+ </table>
+ <table name="build_results" kind="container">
+ <column name="package_name" type="TEXT" null="false"/>
+ <column name="package_version_epoch" type="INTEGER" null="false"/>
+ <column name="package_version_canonical_upstream" type="TEXT" null="false"/>
+ <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE &quot;C&quot;"/>
+ <column name="package_version_revision" type="INTEGER" null="false"/>
+ <column name="configuration" type="TEXT" null="false"/>
+ <column name="index" type="BIGINT" null="false"/>
+ <column name="operation" type="TEXT" null="false"/>
+ <column name="status" type="TEXT" null="false"/>
+ <column name="log" type="TEXT" null="false"/>
+ <foreign-key name="object_id_fk" on-delete="CASCADE">
+ <column name="package_name"/>
+ <column name="package_version_epoch"/>
+ <column name="package_version_canonical_upstream"/>
+ <column name="package_version_canonical_release"/>
+ <column name="package_version_revision"/>
+ <column name="configuration"/>
+ <references table="build">
+ <column name="package_name"/>
+ <column name="package_version_epoch"/>
+ <column name="package_version_canonical_upstream"/>
+ <column name="package_version_canonical_release"/>
+ <column name="package_version_revision"/>
+ <column name="configuration"/>
+ </references>
+ </foreign-key>
+ <index name="build_results_object_id_i">
+ <column name="package_name"/>
+ <column name="package_version_epoch"/>
+ <column name="package_version_canonical_upstream"/>
+ <column name="package_version_canonical_release"/>
+ <column name="package_version_revision"/>
+ <column name="configuration"/>
+ </index>
+ <index name="build_results_index_i">
+ <column name="index"/>
+ </index>
+ </table>
+ </model>
+</changelog>
diff --git a/brep/buildfile b/brep/buildfile
index a7debc1..aeb1848 100644
--- a/brep/buildfile
+++ b/brep/buildfile
@@ -10,8 +10,14 @@ import int_libs = libodb%lib{odb}
import int_libs += libodb-pgsql%lib{odb-pgsql}
import int_libs += libbutl%lib{butl}
import int_libs += libbpkg%lib{bpkg}
+import int_libs += libbbot%lib{bbot}
lib{brep}: \
+{hxx cxx}{ build } \
+{file }{ build.xml } \
+{hxx ixx cxx}{ build-odb } \
+{hxx cxx}{ common } \
+{hxx ixx cxx}{ common-odb } \
{hxx cxx}{ package } \
{file }{ package.xml } \
{hxx ixx cxx}{ package-odb } \
@@ -23,7 +29,7 @@ lib{brep}: \
{hxx }{ version } \
{hxx }{ wrapper-traits } \
$int_libs \
-sql{package package-extra}
+sql{build package package-extra}
# For pre-releases use the complete version to make sure they cannot be used
# in place of another pre-release or the final version.
diff --git a/brep/common b/brep/common
new file mode 100644
index 0000000..52f225c
--- /dev/null
+++ b/brep/common
@@ -0,0 +1,341 @@
+// file : brep/common -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef BREP_COMMON
+#define BREP_COMMON
+
+#include <ratio>
+#include <chrono>
+#include <type_traits> // static_assert
+
+#include <brep/types>
+#include <brep/utility>
+
+// The uint16_t value range is not fully covered by SMALLINT PostgreSQL type
+// to which uint16_t is mapped by default.
+//
+#pragma db value(uint16_t) type("INTEGER")
+
+namespace brep
+{
+ // Use an image type to map bpkg::version to the database since there
+ // is no way to modify individual components directly.
+ //
+ #pragma db value
+ struct _version
+ {
+ uint16_t epoch;
+ string canonical_upstream;
+ string canonical_release;
+ uint16_t revision;
+ string upstream;
+ optional<string> release;
+ };
+}
+
+#include <bpkg/manifest>
+
+namespace brep
+{
+ using optional_version = optional<bpkg::version>;
+ using _optional_version = optional<_version>;
+}
+
+// Prevent assert() macro expansion in get/set expressions. This should
+// appear after all #include directives since the assert() macro is
+// redefined in each <assert.h> inclusion.
+//
+#ifdef ODB_COMPILER
+# undef assert
+# define assert assert
+void assert (int);
+#endif
+
+// We have to keep these mappings at the global scope instead of inside
+// the brep namespace because they need to be also effective in the
+// bpkg namespace from which we "borrow" types (and some of them use version).
+//
+#pragma db map type(bpkg::version) as(brep::_version) \
+ to(brep::_version{(?).epoch, \
+ (?).canonical_upstream, \
+ (?).canonical_release, \
+ (?).revision, \
+ (?).upstream, \
+ (?).release}) \
+ from(bpkg::version ((?).epoch, \
+ std::move ((?).upstream), \
+ std::move ((?).release), \
+ (?).revision))
+
+#pragma db map type(brep::optional_version) as(brep::_optional_version) \
+ to((?) \
+ ? brep::_version{(?)->epoch, \
+ (?)->canonical_upstream, \
+ (?)->canonical_release, \
+ (?)->revision, \
+ (?)->upstream, \
+ (?)->release} \
+ : brep::_optional_version ()) \
+ from((?) \
+ ? bpkg::version ((?)->epoch, \
+ std::move ((?)->upstream), \
+ std::move ((?)->release), \
+ (?)->revision) \
+ : brep::optional_version ())
+
+namespace brep
+{
+ // path
+ //
+ #pragma db map type(path) as(string) to((?).string ()) from(brep::path (?))
+
+ using optional_path = optional<path>;
+ using optional_string = optional<string>;
+
+ #pragma db map type(optional_path) as(brep::optional_string) \
+ to((?) ? (?)->string () : brep::optional_string ()) \
+ from((?) ? brep::path (*(?)) : brep::optional_path ())
+
+ #pragma db map type(dir_path) as(string) \
+ to((?).string ()) from(brep::dir_path (?))
+
+ // Ensure that timestamp can be represented in nonoseconds without loss of
+ // accuracy, so the following ODB mapping is adequate.
+ //
+ static_assert(
+ std::ratio_greater_equal<timestamp::period,
+ std::chrono::nanoseconds::period>::value,
+ "The following timestamp ODB mapping is invalid");
+
+ // As it pointed out in butl/timestamp we will overflow in year 2262, but
+ // by that time some larger basic type will be available for mapping.
+ //
+ #pragma db map type(timestamp) as(uint64_t) \
+ to(std::chrono::duration_cast<std::chrono::nanoseconds> ( \
+ (?).time_since_epoch ()).count ()) \
+ from(brep::timestamp ( \
+ std::chrono::duration_cast<brep::timestamp::duration> ( \
+ std::chrono::nanoseconds (?))))
+
+ // version
+ //
+ using bpkg::version;
+
+ #pragma db value
+ struct canonical_version
+ {
+ uint16_t epoch;
+ string canonical_upstream;
+ string canonical_release;
+ uint16_t revision;
+
+ bool
+ empty () const noexcept
+ {
+ // Note that an empty canonical_upstream doesn't denote an empty
+ // canonical_version. Remeber, that canonical_upstream doesn't include
+ // rightmost digit-only zero components? So non-empty version("0") has
+ // an empty canonical_upstream.
+ //
+ return epoch == 0 && canonical_upstream.empty () &&
+ canonical_release.empty () && revision == 0;
+ }
+
+ // Change collation to ensure the proper comparison of the "absent" release
+ // with a specified one.
+ //
+ // The default collation for UTF8-encoded TEXT columns in PostgreSQL is
+ // UCA-compliant. This makes the statement 'a' < '~' to be false, which
+ // in turn makes the statement 2.1.alpha < 2.1 to be false as well.
+ //
+ // Unicode Collation Algorithm (UCA): http://unicode.org/reports/tr10/
+ //
+ #pragma db member(canonical_release) options("COLLATE \"C\"")
+ };
+
+ #pragma db value transient
+ struct upstream_version: version
+ {
+ #pragma db member(upstream_) virtual(string) \
+ get(this.upstream) \
+ set(this = brep::version (0, std::move (?), std::string (), 0))
+
+ #pragma db member(release_) virtual(optional_string) \
+ get(this.release) \
+ set(this = brep::version ( \
+ 0, std::move (this.upstream), std::move (?), 0))
+
+ upstream_version () = default;
+ upstream_version (version v): version (move (v)) {}
+ upstream_version&
+ operator= (version v) {version& b (*this); b = v; return *this;}
+
+ void
+ init (const canonical_version& cv, const upstream_version& uv)
+ {
+ *this = version (cv.epoch, uv.upstream, uv.release, cv.revision);
+ assert (cv.canonical_upstream == canonical_upstream &&
+ cv.canonical_release == canonical_release);
+ }
+ };
+
+ // Wildcard version. Satisfies any dependency constraint and is represented
+ // as 0+0 (which is also the "stub version"; since a real version is always
+ // greater than the stub version, we reuse it to signify a special case).
+ //
+ extern const version wildcard_version;
+
+ #pragma db value
+ struct package_id
+ {
+ string name;
+ canonical_version version;
+
+ package_id () = default;
+ package_id (string n, const brep::version& v)
+ : name (move (n)),
+ version {
+ v.epoch, v.canonical_upstream, v.canonical_release, v.revision}
+ {
+ }
+ };
+
+ // Version comparison operators.
+ //
+ // They allow comparing objects that have epoch, canonical_upstream,
+ // canonical_release, and revision data members. The idea is that this
+ // works for both query members of types version and canonical_version
+ // as well as for comparing canonical_version to version.
+ //
+ template <typename T1, typename T2>
+ inline auto
+ compare_version_eq (const T1& x, const T2& y, bool revision)
+ -> decltype (x.epoch == y.epoch)
+ {
+ // Since we don't quite know what T1 and T2 are (and where the resulting
+ // expression will run), let's not push our luck with something like
+ // (!revision || x.revision == y.revision).
+ //
+ auto r (x.epoch == y.epoch &&
+ x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release == y.canonical_release);
+
+ return revision
+ ? r && x.revision == y.revision
+ : r;
+ }
+
+ template <typename T1, typename T2>
+ inline auto
+ compare_version_ne (const T1& x, const T2& y, bool revision)
+ -> decltype (x.epoch == y.epoch)
+ {
+ auto r (x.epoch != y.epoch ||
+ x.canonical_upstream != y.canonical_upstream ||
+ x.canonical_release != y.canonical_release);
+
+ return revision
+ ? r || x.revision != y.revision
+ : r;
+ }
+
+ template <typename T1, typename T2>
+ inline auto
+ compare_version_lt (const T1& x, const T2& y, bool revision)
+ -> decltype (x.epoch == y.epoch)
+ {
+ auto r (
+ x.epoch < y.epoch ||
+ (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream) ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release < y.canonical_release));
+
+ return revision
+ ? r ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release == y.canonical_release && x.revision < y.revision)
+ : r;
+ }
+
+ template <typename T1, typename T2>
+ inline auto
+ compare_version_le (const T1& x, const T2& y, bool revision)
+ -> decltype (x.epoch == y.epoch)
+ {
+ auto r (
+ x.epoch < y.epoch ||
+ (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream));
+
+ return revision
+ ? r ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release < y.canonical_release) ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release == y.canonical_release && x.revision <= y.revision)
+ : r ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release <= y.canonical_release);
+ }
+
+ template <typename T1, typename T2>
+ inline auto
+ compare_version_gt (const T1& x, const T2& y, bool revision)
+ -> decltype (x.epoch == y.epoch)
+ {
+ auto r (
+ x.epoch > y.epoch ||
+ (x.epoch == y.epoch && x.canonical_upstream > y.canonical_upstream) ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release > y.canonical_release));
+
+ return revision
+ ? r ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release == y.canonical_release && x.revision > y.revision)
+ : r;
+ }
+
+ template <typename T1, typename T2>
+ inline auto
+ compare_version_ge (const T1& x, const T2& y, bool revision)
+ -> decltype (x.epoch == y.epoch)
+ {
+ auto r (
+ x.epoch > y.epoch ||
+ (x.epoch == y.epoch && x.canonical_upstream > y.canonical_upstream));
+
+ return revision
+ ? r ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release > y.canonical_release) ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release == y.canonical_release && x.revision >= y.revision)
+ : r ||
+ (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
+ x.canonical_release >= y.canonical_release);
+ }
+
+ template <typename T>
+ inline auto
+ order_by_version_desc (const T& x) -> //decltype ("ORDER BY" + x.epoch)
+ decltype (x.epoch == 0)
+ {
+ return "ORDER BY"
+ + x.epoch + "DESC,"
+ + x.canonical_upstream + "DESC,"
+ + x.canonical_release + "DESC,"
+ + x.revision + "DESC";
+ }
+
+ inline bool
+ operator< (const package_id& x, const package_id& y)
+ {
+ if (int r = x.name.compare (y.name))
+ return r < 0;
+
+ return compare_version_lt (x.version, y.version, true);
+ }
+}
+
+#endif // BREP_COMMON
diff --git a/brep/common.cxx b/brep/common.cxx
new file mode 100644
index 0000000..4847977
--- /dev/null
+++ b/brep/common.cxx
@@ -0,0 +1,10 @@
+// file : brep/common.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <brep/common>
+
+namespace brep
+{
+ const version wildcard_version (0, "0", nullopt, 0);
+}
diff --git a/brep/odb.sh b/brep/odb.sh
index 3f8cef2..5e16f45 100755
--- a/brep/odb.sh
+++ b/brep/odb.sh
@@ -9,10 +9,17 @@ lib="\
-I$HOME/work/odb/libodb-default \
-I$HOME/work/odb/libodb"
-$odb $lib -d pgsql --std c++11 --generate-query --generate-schema \
- --schema-format sql --schema-format embedded \
+$odb $lib -d pgsql --std c++11 --generate-query \
--odb-epilogue '#include <brep/wrapper-traits>' \
--hxx-prologue '#include <brep/wrapper-traits>' \
+ -I .. -I ../../libbpkg -I ../../libbutl \
+ --hxx-suffix "" --include-with-brackets \
+ --include-prefix brep --guard-prefix BREP \
+ common
+
+$odb $lib -d pgsql --std c++11 --generate-query --generate-schema \
+ --schema-format sql --schema-format embedded --schema-name package \
+ --generate-prepared --odb-epilogue '#include <brep/wrapper-traits>' \
--hxx-prologue '#include <brep/package-traits>' \
-I .. -I ../../libbpkg -I ../../libbutl \
--hxx-suffix "" --include-with-brackets \
@@ -20,3 +27,11 @@ $odb $lib -d pgsql --std c++11 --generate-query --generate-schema \
package
xxd -i <package-extra.sql >package-extra
+
+$odb $lib -d pgsql --std c++11 --generate-query --generate-schema \
+ --schema-format sql --schema-format embedded --schema-name build \
+ --generate-prepared --odb-epilogue '#include <brep/wrapper-traits>' \
+ -I .. -I ../../libbbot -I ../../libbpkg -I ../../libbutl \
+ --hxx-suffix "" --include-with-brackets \
+ --include-prefix brep --guard-prefix BREP \
+ build
diff --git a/brep/package b/brep/package
index 13429d1..d50aa9f 100644
--- a/brep/package
+++ b/brep/package
@@ -6,94 +6,21 @@
#define BREP_PACKAGE
#include <map>
-#include <ratio>
#include <chrono>
-#include <type_traits> // static_assert
#include <odb/core.hxx>
-#include <odb/forward.hxx> // database
#include <odb/nested-container.hxx>
#include <brep/types>
#include <brep/utility>
-// Used by the data migration entries.
-//
-#define LIBBREP_SCHEMA_VERSION_BASE 3
-
-#pragma db model version(LIBBREP_SCHEMA_VERSION_BASE, 3, closed)
+#include <brep/common> // Must be included last (see assert).
-// The uint16_t value range is not fully covered by SMALLINT PostgreSQL type
-// to which uint16_t is mapped by default.
+// Used by the data migration entries.
//
-#pragma db value(uint16_t) type("INTEGER")
-
-namespace brep
-{
- // Use an image type to map bpkg::version to the database since there
- // is no way to modify individual components directly.
- //
- #pragma db value
- struct _version
- {
- uint16_t epoch;
- string canonical_upstream;
- string canonical_release;
- uint16_t revision;
- string upstream;
- optional<string> release;
- };
-}
-
-#include <bpkg/manifest>
-
-namespace brep
-{
- using optional_version = optional<bpkg::version>;
- using _optional_version = optional<_version>;
-}
+#define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 3
-// Prevent assert() macro expansion in get/set expressions. This should
-// appear after all #include directives since the assert() macro is
-// redefined in each <assert.h> inclusion.
-//
-#ifdef ODB_COMPILER
-# undef assert
-# define assert assert
-void assert (int);
-#endif
-
-// We have to keep these mappings at the global scope instead of inside
-// the brep namespace because they need to be also effective in the
-// bpkg namespace from which we "borrow" types (and some of them use version).
-//
-#pragma db map type(bpkg::version) as(brep::_version) \
- to(brep::_version{(?).epoch, \
- (?).canonical_upstream, \
- (?).canonical_release, \
- (?).revision, \
- (?).upstream, \
- (?).release}) \
- from(bpkg::version ((?).epoch, \
- std::move ((?).upstream), \
- std::move ((?).release), \
- (?).revision))
-
-#pragma db map type(brep::optional_version) as(brep::_optional_version) \
- to((?) \
- ? brep::_version{(?)->epoch, \
- (?)->canonical_upstream, \
- (?)->canonical_release, \
- (?)->revision, \
- (?)->upstream, \
- (?)->release} \
- : brep::_optional_version ()) \
- from((?) \
- ? bpkg::version ((?)->epoch, \
- std::move ((?)->upstream), \
- std::move ((?)->release), \
- (?)->revision) \
- : brep::optional_version ())
+#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 3, closed)
namespace brep
{
@@ -110,106 +37,6 @@ namespace brep
class repository;
class package;
- // path
- //
- #pragma db map type(path) as(string) to((?).string ()) from(brep::path (?))
-
- using optional_path = optional<path>;
- using optional_string = optional<string>;
-
- #pragma db map type(optional_path) as(brep::optional_string) \
- to((?) ? (?)->string () : brep::optional_string ()) \
- from((?) ? brep::path (*(?)) : brep::optional_path ())
-
- #pragma db map type(dir_path) as(string) \
- to((?).string ()) from(brep::dir_path (?))
-
- // Ensure that timestamp can be represented in nonoseconds without loss of
- // accuracy, so the following ODB mapping is adequate.
- //
- static_assert(
- std::ratio_greater_equal<timestamp::period,
- std::chrono::nanoseconds::period>::value,
- "The following timestamp ODB mapping is invalid");
-
- // As it pointed out in butl/timestamp we will overflow in year 2262, but
- // by that time some larger basic type will be available for mapping.
- //
- #pragma db map type(timestamp) as(uint64_t) \
- to(std::chrono::duration_cast<std::chrono::nanoseconds> ( \
- (?).time_since_epoch ()).count ()) \
- from(brep::timestamp ( \
- std::chrono::duration_cast<brep::timestamp::duration> ( \
- std::chrono::nanoseconds (?))))
-
- // version
- //
- using bpkg::version;
-
- #pragma db value
- struct canonical_version
- {
- uint16_t epoch;
- string canonical_upstream;
- string canonical_release;
- uint16_t revision;
-
- bool
- empty () const noexcept
- {
- // Note that an empty canonical_upstream doesn't denote an empty
- // canonical_version. Remeber, that canonical_upstream doesn't include
- // rightmost digit-only zero components? So non-empty version("0") has
- // an empty canonical_upstream.
- //
- return epoch == 0 && canonical_upstream.empty () &&
- canonical_release.empty () && revision == 0;
- }
-
- // Change collation to ensure the proper comparison of the "absent" release
- // with a specified one.
- //
- // The default collation for UTF8-encoded TEXT columns in PostgreSQL is
- // UCA-compliant. This makes the statement 'a' < '~' to be false, which
- // in turn makes the statement 2.1.alpha < 2.1 to be false as well.
- //
- // Unicode Collation Algorithm (UCA): http://unicode.org/reports/tr10/
- //
- #pragma db member(canonical_release) options("COLLATE \"C\"")
- };
-
- #pragma db value transient
- struct upstream_version: version
- {
- #pragma db member(upstream_) virtual(string) \
- get(this.upstream) \
- set(this = brep::version (0, std::move (?), std::string (), 0))
-
- #pragma db member(release_) virtual(optional_string) \
- get(this.release) \
- set(this = brep::version ( \
- 0, std::move (this.upstream), std::move (?), 0))
-
- upstream_version () = default;
- upstream_version (version v): version (move (v)) {}
- upstream_version&
- operator= (version v) {version& b (*this); b = v; return *this;}
-
- void
- init (const canonical_version& cv, const upstream_version& uv)
- {
- *this = version (cv.epoch, uv.upstream, uv.release, cv.revision);
- assert (cv.canonical_upstream == canonical_upstream &&
- cv.canonical_release == canonical_release);
- }
- };
-
- // Wildcard version. Satisfies any dependency constraint and is represented
- // as 0+0 (which is also the "stub version"; since a real version is always
- // greater than the stub version, we reuse it to signify a special case).
- //
- extern const version wildcard_version;
-
// priority
//
using bpkg::priority;
@@ -244,21 +71,6 @@ namespace brep
#pragma db value(dependency_constraint) definition
- #pragma db value
- struct package_id
- {
- string name;
- canonical_version version;
-
- package_id () = default;
- package_id (string n, const brep::version& v)
- : name (move (n)),
- version {
- v.epoch, v.canonical_upstream, v.canonical_release, v.revision}
- {
- }
- };
-
// Notes:
//
// 1. Will the package be always resolvable? What if it is in
@@ -670,141 +482,17 @@ namespace brep
package_id id;
};
- // Version comparison operators.
- //
- // They allow comparing objects that have epoch, canonical_upstream,
- // canonical_release, and revision data members. The idea is that this
- // works for both query members of types version and canonical_version
- // as well as for comparing canonical_version to version.
- //
- template <typename T1, typename T2>
- inline auto
- compare_version_eq (const T1& x, const T2& y, bool revision)
- -> decltype (x.epoch == y.epoch)
- {
- // Since we don't quite know what T1 and T2 are (and where the resulting
- // expression will run), let's not push our luck with something like
- // (!revision || x.revision == y.revision).
- //
- auto r (x.epoch == y.epoch &&
- x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release == y.canonical_release);
-
- return revision
- ? r && x.revision == y.revision
- : r;
- }
-
- template <typename T1, typename T2>
- inline auto
- compare_version_ne (const T1& x, const T2& y, bool revision)
- -> decltype (x.epoch == y.epoch)
- {
- auto r (x.epoch != y.epoch ||
- x.canonical_upstream != y.canonical_upstream ||
- x.canonical_release != y.canonical_release);
-
- return revision
- ? r || x.revision != y.revision
- : r;
- }
-
- template <typename T1, typename T2>
- inline auto
- compare_version_lt (const T1& x, const T2& y, bool revision)
- -> decltype (x.epoch == y.epoch)
- {
- auto r (
- x.epoch < y.epoch ||
- (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream) ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release < y.canonical_release));
-
- return revision
- ? r ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release == y.canonical_release && x.revision < y.revision)
- : r;
- }
-
- template <typename T1, typename T2>
- inline auto
- compare_version_le (const T1& x, const T2& y, bool revision)
- -> decltype (x.epoch == y.epoch)
- {
- auto r (
- x.epoch < y.epoch ||
- (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream));
-
- return revision
- ? r ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release < y.canonical_release) ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release == y.canonical_release && x.revision <= y.revision)
- : r ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release <= y.canonical_release);
- }
-
- template <typename T1, typename T2>
- inline auto
- compare_version_gt (const T1& x, const T2& y, bool revision)
- -> decltype (x.epoch == y.epoch)
- {
- auto r (
- x.epoch > y.epoch ||
- (x.epoch == y.epoch && x.canonical_upstream > y.canonical_upstream) ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release > y.canonical_release));
-
- return revision
- ? r ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release == y.canonical_release && x.revision > y.revision)
- : r;
- }
-
- template <typename T1, typename T2>
- inline auto
- compare_version_ge (const T1& x, const T2& y, bool revision)
- -> decltype (x.epoch == y.epoch)
+ #pragma db view object(package)
+ struct package_version
{
- auto r (
- x.epoch > y.epoch ||
- (x.epoch == y.epoch && x.canonical_upstream > y.canonical_upstream));
-
- return revision
- ? r ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release > y.canonical_release) ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release == y.canonical_release && x.revision >= y.revision)
- : r ||
- (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream &&
- x.canonical_release >= y.canonical_release);
- }
-
- template <typename T>
- inline auto
- order_by_version_desc (const T& x) -> //decltype ("ORDER BY" + x.epoch)
- decltype (x.epoch == 0)
- {
- return "ORDER BY"
- + x.epoch + "DESC,"
- + x.canonical_upstream + "DESC,"
- + x.canonical_release + "DESC,"
- + x.revision + "DESC";
- }
-
- inline bool
- operator< (const package_id& x, const package_id& y)
- {
- if (int r = x.name.compare (y.name))
- return r < 0;
+ package_id id;
+ upstream_version version;
- return compare_version_lt (x.version, y.version, true);
- }
+ // Database mapping.
+ //
+ #pragma db member(id) column("")
+ #pragma db member(version) set(this.version.init (this.id.version, (?)))
+ };
}
#endif // BREP_PACKAGE
diff --git a/brep/package.cxx b/brep/package.cxx
index eacfabe..eaf1029 100644
--- a/brep/package.cxx
+++ b/brep/package.cxx
@@ -13,8 +13,6 @@ using namespace odb::core;
namespace brep
{
- const version wildcard_version (0, "0", nullopt, 0);
-
// dependency
//
string dependency::
diff --git a/brep/package.xml b/brep/package.xml
index 69f1a6a..39494ce 100644
--- a/brep/package.xml
+++ b/brep/package.xml
@@ -1,4 +1,4 @@
-<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" version="1">
+<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1">
<model version="3">
<table name="repository" kind="object">
<column name="name" type="TEXT" null="false"/>
diff --git a/brep/utility b/brep/utility
index 631af7f..b359fc6 100644
--- a/brep/utility
+++ b/brep/utility
@@ -27,6 +27,6 @@ namespace brep
// <butl/utility>
//
using butl::reverse_iterate;
-};
+}
#endif // BREP_UTILITY
diff --git a/brep/version b/brep/version
index f648b5e..e1db981 100644
--- a/brep/version
+++ b/brep/version
@@ -6,6 +6,7 @@
#include <butl/version> // LIBBUTL_VERSION
#include <bpkg/version> // LIBBPKG_VERSION
+#include <bbot/version> // LIBBBOT_VERSION
// Version format is AABBCCDD where
//
@@ -44,4 +45,8 @@
# error incompatible libbpkg version
#endif
+#if LIBBBOT_VERSION != 49901
+# error incompatible libbbot version
+#endif
+
#endif // BREP_VERSION
diff --git a/etc/brep-apache2.conf b/etc/brep-apache2.conf
index b7a481e..5114623 100644
--- a/etc/brep-apache2.conf
+++ b/etc/brep-apache2.conf
@@ -8,6 +8,18 @@
LoadModule brep_module /home/brep/install/libexec/brep/mod_brep.so
</IfModule>
+ # Repository email. This email is used for the From: header in emails
+ # send by brep (for example, build failure notifications).
+ #
+ brep-email admin@example.org
+
+ # Repository host. It specifies the scheme and the host address (but
+ # not the root path; see brep-root below) that will be used whenever
+ # brep needs to construct an absolute URL to one of its locations (for
+ # example, a link to a build log that is being send via email).
+ #
+ brep-host https://example.org
+
# Repository root. This is the part of the URL between the host name
# and the start of the repository. For example, root value /pkg means
# the repository URL is http://example.org/pkg/. Specify / to use the
diff --git a/etc/brep-module.conf b/etc/brep-module.conf
index 760d4ce..1d9d56d 100644
--- a/etc/brep-module.conf
+++ b/etc/brep-module.conf
@@ -26,7 +26,7 @@ menu About=?about
# Number of pages in navigation (pager).
#
-# search-pages 5
+# search-pages 5
# Number of package description characters to display in brief pages.
@@ -36,31 +36,82 @@ menu About=?about
# Number of package changes characters to display in brief pages.
#
-# package-changes 5000
+# package-changes 5000
-# Database connection configuration. By default, brep will try to connect to
-# the local instance of PostgreSQL with the operating system-default mechanism
-# (Unix-domain socket, etc) and use operating system (login) user name and the
-# database called 'brep'. See brep(1) for details.
+# The package database connection configuration. By default, brep will try to
+# connect to the local instance of PostgreSQL with the operating system-default
+# mechanism (Unix-domain socket, etc) and use operating system (login) user
+# name and the database called 'brep_package'. See brep(1) for details.
#
-# db-user
-# db-password
-# db-name brep
-# db-host
-# db-port
+# package-db-user
+# package-db-password
+# package-db-name brep_package
+# package-db-host
+# package-db-port
-# The maximum number of concurrent database connections per web server
+# The maximum number of concurrent package database connections per web server
# process. If 0, then no limitation is applied.
#
-# db-max-connections 5
+# package-db-max-connections 5
-# The maximum number of times to retry database transactions in the
+# The maximum number of times to retry package database transactions in the
# face of recoverable failures (deadlock, loss of connection, etc).
#
-# db-retry 10
+# package-db-retry 10
+
+
+# Build configuration file. If not specified (default), then the package
+# building functionality will be disabled. If specified, then the build
+# database must be configured (see next). Note: must be an absolute path.
+#
+# build-config
+
+# The maximum size of the build task request manifest accepted. Note that the
+# HTTP POST request body is cached to retry database transactions in the face
+# of recoverable failures (deadlock, loss of connection, etc). Default is
+# 100K.
+#
+# build-task-request-max-size 102400
+
+
+# Time to wait before considering the expected task result lost. Must be
+# specified in seconds. Default is 3 hours.
+#
+# build-result-timeout 10800
+
+
+# The maximum size of the build result manifest accepted. Note that the HTTP
+# POST request body is cached to retry database transactions in the face of
+# recoverable failures (deadlock, loss of connection, etc). Default is 10M
+#
+# build-result-request-max-size 10485760
+
+
+# The build database connection configuration. By default, brep will try to
+# connect to the local instance of PostgreSQL with the operating system-default
+# mechanism (Unix-domain socket, etc) and use operating system (login) user
+# name and the database called 'brep_build'. See brep(1) for details.
+#
+# build-db-user
+# build-db-password
+# build-db-name brep_build
+# build-db-host
+# build-db-port
+
+
+# The maximum number of concurrent build database connections per web server
+# process. If 0, then no limitation is applied.
+#
+# build-db-max-connections 5
+
+
+# The maximum number of times to retry build database transactions in the
+# face of recoverable failures (deadlock, loss of connection, etc).
+#
+# build-db-retry 10
# Trace verbosity. Disabled by default.
diff --git a/load/load.cli b/load/load.cli
index 7850d86..fe05ad2 100644
--- a/load/load.cli
+++ b/load/load.cli
@@ -28,8 +28,8 @@ include <brep/types>;
repository and package information into the database, suitable for
consumption by the \cb{brep} web module.
- Note that \cb{brep-load} expects the database schema to have already been
- created using \l{brep-migrate(1)}.
+ Note that \cb{brep-load} expects the database \cb{package} schema to have
+ already been created using \l{brep-migrate(1)}.
Also note that \cb{brep-load} requires \l{bpkg(1)} to fetch repository
information. See \cb{--bpkg} for more information on the package manager
@@ -54,10 +54,11 @@ class options
expected to work."
}
- std::string --db-name|-n = "brep"
+ std::string --db-name|-n = "brep_package"
{
"<name>",
- "Database name. If not specified, then '\cb{brep}' is used by default."
+ "Database name. If not specified, then \cb{brep_package} is used by
+ default."
}
std::string --db-host|-h
diff --git a/load/load.cxx b/load/load.cxx
index 8856e38..e611df2 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -50,12 +50,6 @@ struct failed {};
static const char* help_info (
" info: run 'brep-load --help' for more information");
-static inline bool
-space (char c) noexcept
-{
- return c == ' ' || c == '\t';
-}
-
struct internal_repository
{
repository_location location;
@@ -297,7 +291,7 @@ repository_info (const options& lo, const string& rl, const cstrings& options)
{
cerr << "error: unable to execute " << args[0] << ": " << e << endl;
- if (e.child ())
+ if (e.child)
exit (1);
throw failed ();
@@ -965,6 +959,7 @@ try
{
cout << "brep-load " << BREP_VERSION_STR << endl
<< "libbrep " << LIBBREP_VERSION_STR << endl
+ << "libbbot " << LIBBBOT_VERSION_STR << endl
<< "libbpkg " << LIBBPKG_VERSION_STR << endl
<< "libbutl " << LIBBUTL_VERSION_STR << endl
<< "Copyright (c) 2014-2017 Code Synthesis Ltd" << endl
@@ -1018,12 +1013,13 @@ try
transaction t (db.begin ());
- // Check that the database schema matches the current one.
+ // Check that the database 'package' schema matches the current one.
//
- if (schema_catalog::current_version (db) != db.schema_version ())
+ const string ds ("package");
+ if (schema_catalog::current_version (db, ds) != db.schema_version (ds))
{
- cerr << "error: database schema differs from the current one" << endl
- << " info: use brep-migrate to migrate the database" << endl;
+ cerr << "error: database 'package' schema differs from the current one"
+ << endl << " info: use brep-migrate to migrate the database" << endl;
return 1;
}
diff --git a/manifest b/manifest
index 72ee0c7..31292d6 100644
--- a/manifest
+++ b/manifest
@@ -21,3 +21,4 @@ depends: libodb >= 2.5.0-a10
depends: libodb-pgsql >= 2.5.0-a10
depends: libbutl == 0.5.0-a1
depends: libbpkg == 0.5.0-a1
+depends: libbbot == 0.5.0-a1
diff --git a/migrate/migrate.cli b/migrate/migrate.cli
index 42a1c2e..54edd70 100644
--- a/migrate/migrate.cli
+++ b/migrate/migrate.cli
@@ -17,13 +17,14 @@ include <cstdint>; // uint16_t
\cb{brep-migrate --help}\n
\cb{brep-migrate --version}\n
- \c{\b{brep-migrate} [<options>]}
+ \c{\b{brep-migrate} [<options>] <schema>}
\h|DESCRIPTION|
In its default mode \cb{brep-migrate} creates the database schema if it
doesn't already exist. Otherwise, it migrates the existing schema and data
- to the current version, if needed.
+ to the current version, if needed. The valid schema names are \cb{package}
+ and \cb{build}.
If the \cb{--recreate} option is specified, then \cb{brep-migrate} instead
recreates the database schema. That is, it drops all the existing tables
@@ -66,10 +67,11 @@ class options
expected to work."
}
- std::string --db-name|-n = "brep"
+ std::string --db-name|-n
{
"<name>",
- "Database name. If not specified, then '\cb{brep}' is used by default."
+ "Database name. If not specified, then it is implicitly derived by
+ prefixing the schema name with \cb{brep_}."
}
std::string --db-host|-h
diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx
index 98bdfbd..c700efe 100644
--- a/migrate/migrate.cxx
+++ b/migrate/migrate.cxx
@@ -41,7 +41,7 @@ class schema
{
public:
explicit
- schema (const char* extra);
+ schema (const char* extra, string name);
void
create (database&) const;
@@ -50,12 +50,14 @@ public:
drop (database&) const;
private:
+ string name_;
strings drop_statements_;
strings create_statements_;
};
schema::
-schema (const char* s)
+schema (const char* s, string name)
+ : name_ (move (name))
{
// Remove comments, saving the cleaned SQL code into statements.
//
@@ -176,7 +178,7 @@ drop (database& db) const
//
db.execute (s);
- schema_catalog::drop_schema (db);
+ schema_catalog::drop_schema (db, name_);
}
void schema::
@@ -184,7 +186,7 @@ create (database& db) const
{
drop (db);
- schema_catalog::create_schema (db);
+ schema_catalog::create_schema (db, name_);
for (const auto& s: create_statements_)
db.execute (s);
@@ -205,6 +207,7 @@ try
{
cout << "brep-migrate " << BREP_VERSION_STR << endl
<< "libbrep " << LIBBREP_VERSION_STR << endl
+ << "libbbot " << LIBBBOT_VERSION_STR << endl
<< "libbpkg " << LIBBPKG_VERSION_STR << endl
<< "libbutl " << LIBBUTL_VERSION_STR << endl
<< "Copyright (c) 2014-2017 Code Synthesis Ltd" << endl
@@ -229,7 +232,19 @@ try
return p.wait () ? 0 : 1;
}
- if (argc > 1)
+ if (!scan.more ())
+ {
+ cerr << "error: no database schema specified" << endl
+ << help_info << endl;
+ return 1;
+ }
+
+ const string db_schema (scan.next ());
+
+ if (db_schema != "package" && db_schema != "build")
+ throw cli::unknown_argument (db_schema);
+
+ if (scan.more ())
{
cerr << "error: unexpected argument encountered" << endl
<< help_info << endl;
@@ -246,7 +261,9 @@ try
odb::pgsql::database db (
ops.db_user (),
ops.db_password (),
- ops.db_name (),
+ !ops.db_name ().empty ()
+ ? ops.db_name ()
+ : "brep_" + db_schema,
ops.db_host (),
ops.db_port (),
"options='-c default_transaction_isolation=serializable'");
@@ -261,7 +278,7 @@ try
// transaction useless as all consequitive queries in that transaction will
// be ignored by PostgreSQL.
//
- schema_version schema_version (db.schema_version ());
+ schema_version schema_version (db.schema_version (db_schema));
// It is impossible to operate with the database which is out of the
// [base_version, current_version] range due to the lack of the knowlege
@@ -269,13 +286,13 @@ try
//
if (schema_version > 0)
{
- if (schema_version < schema_catalog::base_version (db))
+ if (schema_version < schema_catalog::base_version (db, db_schema))
{
cerr << "error: database schema is too old" << endl;
throw failed ();
}
- if (schema_version > schema_catalog::current_version (db))
+ if (schema_version > schema_catalog::current_version (db, db_schema))
{
cerr << "error: database schema is too new" << endl;
throw failed ();
@@ -292,7 +309,7 @@ try
// database (followed with the database creation for the --recreate option).
//
if ((create || drop) && schema_version != 0 &&
- schema_version != schema_catalog::current_version (db))
+ schema_version != schema_catalog::current_version (db, db_schema))
{
cerr << "error: database schema requires migration" << endl
<< " info: either migrate the database first or drop the entire "
@@ -304,11 +321,14 @@ try
if (create || drop)
{
- static const char extras[] = {
+ static const char package_extras[] = {
#include <brep/package-extra>
, '\0'};
- schema s (extras);
+ schema s (db_schema == "package"
+ ? package_extras
+ : "",
+ db_schema);
if (create)
s.create (db);
@@ -319,10 +339,10 @@ try
{
// Register the data migration functions.
//
- // static const data_migration_entry<2, LIBBREP_SCHEMA_VERSION_BASE>
+ // static const data_migration_entry<2, LIBBREP_XXX_SCHEMA_VERSION_BASE>
// migrate_v2_entry (&migrate_v2);
//
- schema_catalog::migrate (db);
+ schema_catalog::migrate (db, 0, db_schema);
}
t.commit ();
diff --git a/mod/build-config b/mod/build-config
new file mode 100644
index 0000000..a5713d7
--- /dev/null
+++ b/mod/build-config
@@ -0,0 +1,23 @@
+// file : mod/build-config -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_BUILD_CONFIG
+#define MOD_BUILD_CONFIG
+
+#include <bbot/build-config>
+
+#include <brep/types>
+#include <brep/utility>
+
+namespace brep
+{
+ // Return pointer to the shared build configurations instance, creating one
+ // on the first call. Throw tab_parsing on parsing error, io_error on the
+ // underlying OS error. Is not thread-safe.
+ //
+ shared_ptr<const bbot::build_configs>
+ shared_build_config (const path&);
+}
+
+#endif // MOD_BUILD_CONFIG
diff --git a/mod/build-config.cxx b/mod/build-config.cxx
new file mode 100644
index 0000000..11be1b9
--- /dev/null
+++ b/mod/build-config.cxx
@@ -0,0 +1,31 @@
+// file : mod/build-config.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/build-config>
+
+#include <map>
+
+namespace brep
+{
+ using namespace bbot;
+
+ shared_ptr<const build_configs>
+ shared_build_config (const path& p)
+ {
+ static std::map<path, weak_ptr<build_configs>> configs;
+
+ auto i (configs.find (p));
+ if (i != configs.end ())
+ {
+ if (shared_ptr<build_configs> c = i->second.lock ())
+ return c;
+ }
+
+ shared_ptr<build_configs> c (
+ make_shared<build_configs> (parse_buildtab (p)));
+
+ configs[p] = c;
+ return c;
+ }
+}
diff --git a/mod/buildfile b/mod/buildfile
index 228c149..2926a1b 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -14,13 +14,18 @@ import libs += libodb%lib{odb}
import libs += libodb-pgsql%lib{odb-pgsql}
import libs += libbutl%lib{butl}
import libs += libbpkg%lib{bpkg}
+import libs += libbbot%lib{bbot}
include ../brep/
mod{brep}: \
+ {hxx cxx}{ build-config } \
{hxx cxx}{ database } \
{hxx cxx}{ database-module } \
{hxx cxx}{ diagnostics } \
+ {hxx cxx}{ mod-build-log } \
+ {hxx cxx}{ mod-build-result } \
+ {hxx cxx}{ mod-build-task } \
{hxx cxx}{ mod-package-details } \
{hxx cxx}{ mod-package-search } \
{hxx cxx}{ mod-package-version-details } \
@@ -57,7 +62,7 @@ if $cli.configured
# parameters uniformly with a single catch block.
#
cli.options += --std c++11 -I $src_root --include-with-brackets \
---include-prefix mod --guard-prefix MOD \
+--include-prefix mod --guard-prefix MOD --generate-specifier \
--cxx-prologue "#include <mod/types-parsers>" \
--cli-namespace brep::cli --generate-file-scanner --suppress-usage \
--generate-modifier --generate-description --option-prefix ""
diff --git a/mod/database b/mod/database
index 8e9fdd1..9a83752 100644
--- a/mod/database
+++ b/mod/database
@@ -10,17 +10,18 @@
#include <brep/types>
#include <brep/utility>
-#include <mod/options>
-
namespace brep
{
- // Returns pointer to the shared database instance, creating one on the
- // first call. On subsequent calls ensures passed host and port equals
- // to ones of the existing database instance throwing runtime_error
- // otherwise. Is not thread-safe.
+ // Return pointer to the shared database instance, creating one on the first
+ // call. Throw odb::exception on failure. Is not thread-safe.
//
shared_ptr<odb::core::database>
- shared_database (const options::db&);
+ shared_database (string user,
+ string password,
+ string name,
+ string host,
+ uint16_t port,
+ size_t max_connections);
}
#endif // MOD_DATABASE
diff --git a/mod/database-module b/mod/database-module
index 034324b..3799e7b 100644
--- a/mod/database-module
+++ b/mod/database-module
@@ -10,6 +10,8 @@
#include <brep/types>
#include <brep/utility>
+#include <bbot/build-config>
+
#include <mod/module>
#include <mod/options>
@@ -36,15 +38,31 @@ namespace brep
//
using module::init;
+ // Initialize the package database instance. Throw odb::exception on
+ // failure.
+ //
+ void
+ init (const options::package_db&, size_t retry);
+
+ // Initialize the build database instance and parse build configuration
+ // file. Throw odb::exception on database failure, tab_parsing on parsing
+ // error, system_error on the underlying OS error.
+ //
void
- init (const options::db&);
+ init (const options::build&, const options::build_db&, size_t retry);
virtual bool
handle (request&, response&) = 0;
protected:
- size_t retry_;
- shared_ptr<odb::core::database> db_;
+ size_t retry_ = 0; // Max of all retries.
+
+ shared_ptr<odb::core::database> package_db_;
+
+ // These are NULL if not building.
+ //
+ shared_ptr<odb::core::database> build_db_;
+ shared_ptr<const bbot::build_configs> build_conf_;
private:
virtual bool
diff --git a/mod/database-module.cxx b/mod/database-module.cxx
index 6fd5025..e7a6883 100644
--- a/mod/database-module.cxx
+++ b/mod/database-module.cxx
@@ -4,13 +4,23 @@
#include <mod/database-module>
+#include <errno.h> // EIO
+
+#include <sstream>
+
#include <odb/exceptions.hxx>
+#include <butl/utility> // throw_generic_error()
+
#include <mod/options>
#include <mod/database>
+#include <mod/build-config>
namespace brep
{
+ using namespace std;
+ using namespace butl;
+
// While currently the user-defined copy constructor is not required (we
// don't need to deep copy nullptr's), it is a good idea to keep the
// placeholder ready for less trivial cases.
@@ -19,15 +29,49 @@ namespace brep
database_module (const database_module& r)
: module (r),
retry_ (r.retry_),
- db_ (r.initialized_ ? r.db_ : nullptr)
+ package_db_ (r.initialized_ ? r.package_db_ : nullptr),
+ build_db_ (r.initialized_ ? r.build_db_ : nullptr),
+ build_conf_ (r.initialized_ ? r.build_conf_ : nullptr)
+ {
+ }
+
+ void database_module::
+ init (const options::package_db& o, size_t retry)
{
+ package_db_ = shared_database (o.package_db_user (),
+ o.package_db_password (),
+ o.package_db_name (),
+ o.package_db_host (),
+ o.package_db_port (),
+ o.package_db_max_connections ());
+
+ retry_ = retry_ < retry ? retry : retry_;
}
void database_module::
- init (const options::db& o)
+ init (const options::build& bo, const options::build_db& dbo, size_t retry)
{
- retry_ = o.db_retry ();
- db_ = shared_database (o);
+ try
+ {
+ build_conf_ = shared_build_config (bo.build_config ());
+ }
+ catch (const io_error& e)
+ {
+ ostringstream os;
+ os << "unable to read build configuration '" << bo.build_config ()
+ << "': " << e;
+
+ throw_generic_error (EIO, os.str ().c_str ());
+ }
+
+ build_db_ = shared_database (dbo.build_db_user (),
+ dbo.build_db_password (),
+ dbo.build_db_name (),
+ dbo.build_db_host (),
+ dbo.build_db_port (),
+ dbo.build_db_max_connections ());
+
+ retry_ = retry_ < retry ? retry : retry_;
}
bool database_module::
diff --git a/mod/database.cxx b/mod/database.cxx
index f8fa1e5..22a0563 100644
--- a/mod/database.cxx
+++ b/mod/database.cxx
@@ -11,30 +11,43 @@
namespace brep
{
- namespace options
+ struct db_key
{
- bool
- operator< (const db& x, const db& y)
- {
- int r;
- if ((r = x.db_user ().compare (y.db_user ())) != 0 ||
- (r = x.db_password ().compare (y.db_password ())) != 0 ||
- (r = x.db_name ().compare (y.db_name ())) != 0 ||
- (r = x.db_host ().compare (y.db_host ())))
- return r < 0;
-
- return x.db_port () < y.db_port ();
- }
+ string user;
+ string password;
+ string name;
+ string host;
+ uint16_t port;
+ };
+
+ static bool
+ operator< (const db_key& x, const db_key& y)
+ {
+ int r;
+ if ((r = x.user.compare (y.user)) != 0 ||
+ (r = x.password.compare (y.password)) != 0 ||
+ (r = x.name.compare (y.name)) != 0 ||
+ (r = x.host.compare (y.host)))
+ return r < 0;
+
+ return x.port < y.port;
}
using namespace odb;
shared_ptr<database>
- shared_database (const options::db& o)
+ shared_database (string user,
+ string password,
+ string name,
+ string host,
+ uint16_t port,
+ size_t max_connections)
{
- static std::map<options::db, weak_ptr<database>> databases;
+ static std::map<db_key, weak_ptr<database>> databases;
+
+ db_key k ({move (user), move (password), move (name), host, port});
- auto i (databases.find (o));
+ auto i (databases.find (k));
if (i != databases.end ())
{
if (shared_ptr<database> d = i->second.lock ())
@@ -42,19 +55,19 @@ namespace brep
}
unique_ptr<pgsql::connection_factory>
- f (new pgsql::connection_pool_factory (o.db_max_connections ()));
+ f (new pgsql::connection_pool_factory (max_connections));
shared_ptr<database> d (
make_shared<pgsql::database> (
- o.db_user (),
- o.db_password (),
- o.db_name (),
- o.db_host (),
- o.db_port (),
+ k.user,
+ k.password,
+ k.name,
+ k.host,
+ k.port,
"options='-c default_transaction_isolation=serializable'",
move (f)));
- databases[o] = d;
+ databases[move (k)] = d;
return d;
}
}
diff --git a/mod/mod-build-log b/mod/mod-build-log
new file mode 100644
index 0000000..8395546
--- /dev/null
+++ b/mod/mod-build-log
@@ -0,0 +1,45 @@
+// file : mod/mod-build-log -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_MOD_BUILD_LOG
+#define MOD_MOD_BUILD_LOG
+
+#include <brep/types>
+#include <brep/utility>
+
+#include <mod/options>
+#include <mod/database-module>
+
+namespace brep
+{
+ class build_log: public database_module
+ {
+ public:
+ build_log () = default;
+
+ // Create a shallow copy (handling instance) if initialized and a deep
+ // copy (context exemplar) otherwise.
+ //
+ explicit
+ build_log (const build_log&);
+
+ virtual bool
+ handle (request&, response&);
+
+ virtual const cli::options&
+ cli_options () const
+ {
+ return options::build_log::description ();
+ }
+
+ private:
+ virtual void
+ init (cli::scanner&);
+
+ private:
+ shared_ptr<options::build_log> options_;
+ };
+}
+
+#endif // MOD_MOD_BUILD_LOG
diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx
new file mode 100644
index 0000000..5342e3e
--- /dev/null
+++ b/mod/mod-build-log.cxx
@@ -0,0 +1,220 @@
+// file : mod/mod-build-log.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/mod-build-log>
+
+#include <algorithm> // find_if()
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
+#include <web/module>
+
+#include <brep/build>
+#include <brep/build-odb>
+#include <brep/package>
+#include <brep/package-odb>
+
+#include <mod/options>
+
+using namespace std;
+using namespace bbot;
+using namespace brep::cli;
+using namespace odb::core;
+
+// While currently the user-defined copy constructor is not required (we don't
+// need to deep copy nullptr's), it is a good idea to keep the placeholder
+// ready for less trivial cases.
+//
+brep::build_log::
+build_log (const build_log& r)
+ : database_module (r),
+ options_ (r.initialized_ ? r.options_ : nullptr)
+{
+}
+
+void brep::build_log::
+init (scanner& s)
+{
+ MODULE_DIAG;
+
+ options_ = make_shared<options::build_log> (
+ s, unknown_mode::fail, unknown_mode::fail);
+
+ database_module::init (static_cast<options::package_db> (*options_),
+ options_->package_db_retry ());
+
+ if (options_->build_config_specified ())
+ database_module::init (static_cast<options::build> (*options_),
+ static_cast<options::build_db> (*options_),
+ options_->build_db_retry ());
+
+ if (options_->root ().empty ())
+ options_->root (dir_path ("/"));
+}
+
+bool brep::build_log::
+handle (request& rq, response& rs)
+{
+ using brep::version; // Not to confuse with module::version.
+
+ MODULE_DIAG;
+
+ if (build_db_ == nullptr)
+ throw invalid_request (501, "not implemented");
+
+ // Parse the HTTP request URL path (without the root directory) to obtain
+ // the build package name/version, the configuration name and the optional
+ // operation name. If the operation is not specified then print logs for all
+ // the operations.
+ //
+ // Note that the URL path must be in the following form:
+ //
+ // <package-name>/<package-version>/log/<config-name>[/<operation>]
+ //
+ // Also note that the presence of the first 3 components is guaranteed by
+ // the repository_root module.
+ //
+ build_id id;
+ string op;
+
+ path lpath (rq.path ().leaf (options_->root ()));
+
+ try
+ {
+ auto i (lpath.begin ());
+
+ assert (i != lpath.end ());
+ string name (*i++);
+
+ if (name.empty ())
+ throw invalid_argument ("empty package name");
+
+ assert (i != lpath.end ());
+
+ version version;
+
+ // Intercept exception handling to add the parsing error attribution.
+ //
+ try
+ {
+ version = brep::version (*i++);
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_argument (
+ string ("invalid package version: ") + e.what ());
+ }
+
+ assert (i != lpath.end () && *i == "log");
+
+ if (++i == lpath.end ())
+ throw invalid_argument ("no configuration name");
+
+ id = build_id (package_id (move (name), version), *i++);
+
+ if (id.configuration.empty ())
+ throw invalid_argument ("empty configuration name");
+
+ if (i != lpath.end ())
+ op = *i++;
+
+ if (i != lpath.end ())
+ throw invalid_argument ("unexpected path component");
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ // Make sure no parameters passed.
+ //
+ try
+ {
+ name_value_scanner s (rq.parameters ());
+ params::build_log (s, unknown_mode::fail, unknown_mode::fail);
+ }
+ catch (const cli::exception& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ // If the package build configuration expired (no such configuration,
+ // package, etc), then we log this case with the trace severity and respond
+ // with the 404 HTTP code (not found but may be available in the future).
+ // The thinking is that this may be or may not be a problem with the
+ // controller's setup (expires too fast or the link from some ancient email
+ // is opened).
+ //
+ auto config_expired = [&trace, &lpath, this] (const string& d)
+ {
+ l2 ([&]{trace << "package build configuration for " << lpath
+ << " expired: " << d;});
+
+ throw invalid_request (404, "package build configuration expired: " + d);
+ };
+
+ // Make sure the build configuration still exists.
+ //
+ auto i (
+ find_if (
+ build_conf_->begin (), build_conf_->end (),
+ [&id] (const build_config& c) {return c.name == id.configuration;}));
+
+ if (i == build_conf_->end ())
+ config_expired ("no configuration");
+
+ // Make sure the package still exists.
+ //
+ {
+ transaction t (package_db_->begin ());
+ shared_ptr<package> p (package_db_->find<package> (id.package));
+ t.commit ();
+
+ if (p == nullptr)
+ config_expired ("no package");
+ }
+
+ // Load the package build configuration (if present).
+ //
+ shared_ptr<build> b;
+ {
+ transaction t (build_db_->begin ());
+ b = build_db_->find<build> (id);
+
+ if (b == nullptr)
+ config_expired ("no package configuration");
+ else if (b->state != build_state::tested)
+ config_expired ("state is " + to_string (b->state));
+ else
+ build_db_->load (*b, b->results_section);
+
+ t.commit ();
+ }
+
+ // We have all the data so don't buffer the response content.
+ //
+ ostream& os (rs.content (200, "text/plain;charset=utf-8", false));
+
+ if (op.empty ())
+ {
+ for (const auto& r: b->results)
+ os << r.log;
+ }
+ else
+ {
+ const operation_results& r (b->results);
+
+ auto i (
+ find_if (r.begin (), r.end (),
+ [&op] (const operation_result& v) {return v.operation == op;}));
+
+ if (i == r.end ())
+ config_expired ("no operation");
+
+ os << i->log;
+ }
+
+ return true;
+}
diff --git a/mod/mod-build-result b/mod/mod-build-result
new file mode 100644
index 0000000..8afa697
--- /dev/null
+++ b/mod/mod-build-result
@@ -0,0 +1,42 @@
+// file : mod/mod-build-result -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_MOD_BUILD_RESULT
+#define MOD_MOD_BUILD_RESULT
+
+#include <brep/types>
+#include <brep/utility>
+
+#include <mod/options>
+#include <mod/database-module>
+
+namespace brep
+{
+ class build_result: public database_module
+ {
+ public:
+ build_result () = default;
+
+ // Create a shallow copy (handling instance) if initialized and a deep
+ // copy (context exemplar) otherwise.
+ //
+ explicit
+ build_result (const build_result&);
+
+ virtual bool
+ handle (request&, response&);
+
+ virtual const cli::options&
+ cli_options () const {return options::build_result::description ();}
+
+ private:
+ virtual void
+ init (cli::scanner&);
+
+ private:
+ shared_ptr<options::build_result> options_;
+ };
+}
+
+#endif // MOD_MOD_BUILD_RESULT
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
new file mode 100644
index 0000000..fb6edb5
--- /dev/null
+++ b/mod/mod-build-result.cxx
@@ -0,0 +1,303 @@
+// file : mod/mod-build-result.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/mod-build-result>
+
+#include <algorithm> // find_if()
+
+#include <butl/sendmail>
+#include <butl/process-io>
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+#include <bbot/manifest>
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
+#include <web/module>
+
+#include <brep/build>
+#include <brep/build-odb>
+#include <brep/package>
+#include <brep/package-odb>
+
+#include <mod/options>
+
+using namespace std;
+using namespace butl;
+using namespace bbot;
+using namespace brep::cli;
+using namespace odb::core;
+
+// While currently the user-defined copy constructor is not required (we don't
+// need to deep copy nullptr's), it is a good idea to keep the placeholder
+// ready for less trivial cases.
+//
+brep::build_result::
+build_result (const build_result& r)
+ : database_module (r),
+ options_ (r.initialized_ ? r.options_ : nullptr)
+{
+}
+
+void brep::build_result::
+init (scanner& s)
+{
+ MODULE_DIAG;
+
+ options_ = make_shared<options::build_result> (
+ s, unknown_mode::fail, unknown_mode::fail);
+
+ database_module::init (static_cast<options::package_db> (*options_),
+ options_->package_db_retry ());
+
+ if (options_->build_config_specified ())
+ database_module::init (static_cast<options::build> (*options_),
+ static_cast<options::build_db> (*options_),
+ options_->build_db_retry ());
+
+ if (options_->root ().empty ())
+ options_->root (dir_path ("/"));
+}
+
+bool brep::build_result::
+handle (request& rq, response&)
+{
+ using brep::version; // Not to confuse with module::version.
+
+ MODULE_DIAG;
+
+ if (build_db_ == nullptr)
+ throw invalid_request (501, "not implemented");
+
+ // Make sure no parameters passed.
+ //
+ try
+ {
+ name_value_scanner s (rq.parameters ());
+ params::build_result (s, unknown_mode::fail, unknown_mode::fail);
+ }
+ catch (const cli::exception& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ result_request_manifest rqm;
+
+ try
+ {
+ size_t limit (options_->build_result_request_max_size ());
+ manifest_parser p (rq.content (limit, limit), "result_request_manifest");
+ rqm = result_request_manifest (p);
+ }
+ catch (const manifest_parsing& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ // Parse the task response session to obtain the build configuration name,
+ // and to make sure the session matches the result manifest's package name
+ // and version.
+ //
+ build_id id;
+
+ try
+ {
+ const string& s (rqm.session);
+
+ size_t p (s.find ('/')); // End of package name.
+
+ if (p == 0)
+ throw invalid_argument ("empty package name");
+
+ if (p == string::npos)
+ throw invalid_argument ("no package version");
+
+ string& name (rqm.result.name);
+ if (name.compare (0, name.size (), s, 0, p) != 0)
+ throw invalid_argument ("package name mismatch");
+
+ size_t b (p + 1); // Start of version.
+ p = s.find ('/', b); // End of version.
+
+ if (p == string::npos)
+ throw invalid_argument ("no configuration name");
+
+ version version;
+
+ // Intercept exception handling to add the parsing error attribution.
+ //
+ try
+ {
+ version = brep::version (string (s, b, p - b));
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_argument (
+ string ("invalid package version: ") + e.what ());
+ }
+
+ if (version != rqm.result.version)
+ throw invalid_argument ("package version mismatch");
+
+ id = build_id (package_id (move (name), version), string (s, p + 1));
+
+ if (id.configuration.empty ())
+ throw invalid_argument ("empty configuration name");
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_request (400, string ("invalid session: ") + e.what ());
+ }
+
+ // If the session expired (no such configuration, package, etc), then we log
+ // this case with the warning severity and respond with the 200 HTTP code as
+ // if the session is valid. The thinking is that this is a problem with the
+ // controller's setup (expires too fast), not with the agent's.
+ //
+ auto warn_expired = [&rqm, &warn] (const string& d)
+ {
+ warn << "session '" << rqm.session << "' expired: " << d;
+ };
+
+ // Make sure the build configuration still exists.
+ //
+ auto i (
+ find_if (
+ build_conf_->begin (), build_conf_->end (),
+ [&id] (const build_config& c) {return c.name == id.configuration;}));
+
+ if (i == build_conf_->end ())
+ {
+ warn_expired ("no build configuration");
+ return true;
+ }
+
+ // Load the built package (if present).
+ //
+ shared_ptr<package> p;
+ {
+ transaction t (package_db_->begin ());
+ p = package_db_->find<package> (id.package);
+ t.commit ();
+ }
+
+ if (p == nullptr)
+ {
+ warn_expired ("no package");
+ return true;
+ }
+
+ // Load and update the package build configuration (if present).
+ //
+ shared_ptr<build> b;
+ {
+ transaction t (build_db_->begin ());
+ b = build_db_->find<build> (id);
+
+ if (b == nullptr)
+ warn_expired ("no package configuration");
+ else if (b->state != build_state::testing)
+ {
+ warn_expired ("package configuration state is " + to_string (b->state));
+ b = nullptr;
+ }
+ else
+ {
+ b->state = build_state::tested;
+ b->timestamp = timestamp::clock::now ();
+
+ assert (!b->status);
+ b->status = rqm.result.status;
+
+ // Need to manually load the lazy-load section prior to changing its
+ // members and updating the object's persistent state.
+ //
+ // @@ TODO: ODB now allows marking a section as loaded so can optimize
+ // this.
+ //
+ build_db_->load (*b, b->results_section);
+ b->results = move (rqm.result.results);
+
+ build_db_->update (b);
+ }
+
+ t.commit ();
+ }
+
+ if (b == nullptr)
+ return true; // Warning is already logged.
+
+ // Send email to the package owner.
+ //
+ try
+ {
+ string subj (b->package_name + '/' + b->package_version.string () + ' ' +
+ b->configuration + " build: " + to_string (*b->status));
+
+ // If the package email address is not specified, then it is assumed to be
+ // the same as the project email address.
+ //
+ const string& to (p->package_email
+ ? *p->package_email
+ : p->email);
+
+ auto print_args = [&trace, this] (const char* args[], size_t n)
+ {
+ l2 ([&]{trace << process_args {args, n};});
+ };
+
+ // Redirect the diagnostics to webserver error log.
+ //
+ // Note: if using this somewhere else, then need to factor out all this
+ // exit status handling code.
+ //
+ sendmail sm (print_args,
+ 2,
+ options_->email (),
+ subj,
+ {to});
+
+ if (b->results.empty ())
+ sm.out << "No operations results available." << endl;
+ else
+ {
+ for (const auto& r: b->results)
+ sm.out << r.operation << ": " << r.status << ", "
+ << options_->host () << options_->root ().representation ()
+ << b->package_name << '/' << b->package_version << "/log/"
+ << b->configuration << '/' << r.operation << endl;
+ }
+
+ sm.out.close ();
+
+ if (!sm.wait ())
+ {
+ diag_record dr (error);
+ dr << "sendmail ";
+
+ assert (sm.exit);
+ const process_exit& e (*sm.exit);
+
+ if (e.normal ())
+ dr << "exited with code " << static_cast<uint16_t> (e.code ());
+ else
+ {
+ dr << "terminated abnormally: " << e.description ();
+
+ if (e.core ())
+ dr << " (core dumped)";
+ }
+ }
+ }
+ // Handle process_error and io_error (both derive from system_error).
+ //
+ catch (const system_error& e)
+ {
+ error << "sendmail error: " << e;
+ }
+
+ return true;
+}
diff --git a/mod/mod-build-task b/mod/mod-build-task
new file mode 100644
index 0000000..051357e
--- /dev/null
+++ b/mod/mod-build-task
@@ -0,0 +1,42 @@
+// file : mod/mod-build-task -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_MOD_BUILD_TASK
+#define MOD_MOD_BUILD_TASK
+
+#include <brep/types>
+#include <brep/utility>
+
+#include <mod/options>
+#include <mod/database-module>
+
+namespace brep
+{
+ class build_task: public database_module
+ {
+ public:
+ build_task () = default;
+
+ // Create a shallow copy (handling instance) if initialized and a deep
+ // copy (context exemplar) otherwise.
+ //
+ explicit
+ build_task (const build_task&);
+
+ virtual bool
+ handle (request&, response&);
+
+ virtual const cli::options&
+ cli_options () const {return options::build_task::description ();}
+
+ private:
+ virtual void
+ init (cli::scanner&);
+
+ private:
+ shared_ptr<options::build_task> options_;
+ };
+}
+
+#endif // MOD_MOD_BUILD_TASK
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
new file mode 100644
index 0000000..1cb5386
--- /dev/null
+++ b/mod/mod-build-task.cxx
@@ -0,0 +1,389 @@
+// file : mod/mod-build-task.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/mod-build-task>
+
+#include <map>
+#include <chrono>
+
+#include <butl/utility> // compare_c_string
+#include <butl/filesystem> // path_match()
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+#include <bbot/manifest>
+#include <bbot/build-config>
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
+#include <web/module>
+
+#include <brep/build>
+#include <brep/build-odb>
+#include <brep/package>
+#include <brep/package-odb>
+
+#include <mod/options>
+
+using namespace std;
+using namespace butl;
+using namespace bbot;
+using namespace brep::cli;
+using namespace odb::core;
+
+// While currently the user-defined copy constructor is not required (we don't
+// need to deep copy nullptr's), it is a good idea to keep the placeholder
+// ready for less trivial cases.
+//
+brep::build_task::
+build_task (const build_task& r)
+ : database_module (r),
+ options_ (r.initialized_ ? r.options_ : nullptr)
+{
+}
+
+void brep::build_task::
+init (scanner& s)
+{
+ MODULE_DIAG;
+
+ options_ = make_shared<options::build_task> (
+ s, unknown_mode::fail, unknown_mode::fail);
+
+ database_module::init (static_cast<options::package_db> (*options_),
+ options_->package_db_retry ());
+
+ if (options_->build_config_specified ())
+ database_module::init (static_cast<options::build> (*options_),
+ static_cast<options::build_db> (*options_),
+ options_->build_db_retry ());
+
+ if (options_->root ().empty ())
+ options_->root (dir_path ("/"));
+}
+
+bool brep::build_task::
+handle (request& rq, response& rs)
+{
+ MODULE_DIAG;
+
+ if (build_db_ == nullptr)
+ throw invalid_request (501, "not implemented");
+
+ // Make sure no parameters passed.
+ //
+ try
+ {
+ name_value_scanner s (rq.parameters ());
+ params::build_task (s, unknown_mode::fail, unknown_mode::fail);
+ }
+ catch (const cli::exception& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ task_request_manifest tqm;
+
+ try
+ {
+ size_t limit (options_->build_task_request_max_size ());
+ manifest_parser p (rq.content (limit, limit), "task_request_manifest");
+ tqm = task_request_manifest (p);
+ }
+ catch (const manifest_parsing& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ task_response_manifest tsm;
+
+ // Map build configurations to machines that are capable of building them.
+ // The first matching machine is selected for each configuration. Also
+ // create the configuration name list for use in database queries.
+ //
+ struct config_machine
+ {
+ const build_config* config;
+ const machine_header_manifest* machine;
+ };
+
+ using config_machines = map<const char*, config_machine, compare_c_string>;
+
+ cstrings cfg_names;
+ config_machines cfg_machines;
+
+ for (const auto& c: *build_conf_)
+ {
+ for (auto& m: tqm.machines)
+ {
+ if (path_match (c.machine_pattern, m.name) &&
+ cfg_machines.insert (
+ make_pair (c.name.c_str (), config_machine ({&c, &m}))).second)
+ cfg_names.push_back (c.name.c_str ());
+ }
+ }
+
+ // Go through packages until we find one that has no build configuration
+ // present in the database, or has the untested one, or in the testing state
+ // but expired, or the one, which build failed abnormally and expired. If
+ // such a package configuration is found then put it into the testing state,
+ // set the current timestamp and respond with the task for building this
+ // package configuration.
+ //
+ if (!cfg_machines.empty ())
+ {
+ // Calculate the expiration time for package configurations being in the
+ // testing state or those, which build failed abnormally.
+ //
+ timestamp expiration (timestamp::clock::now () -
+ chrono::seconds (options_->build_result_timeout ()));
+
+ uint64_t expiration_ns (
+ std::chrono::duration_cast<std::chrono::nanoseconds> (
+ expiration.time_since_epoch ()).count ());
+
+ // Prepare the package version prepared query.
+ //
+ // Note that the number of packages can be large and so, in order not to
+ // hold locks for too long, we will restrict the number of packages being
+ // queried in a single transaction. To achieve this we will iterate through
+ // packages using the OFFSET/LIMIT pair and sort the query result.
+ //
+ // Note that this approach can result in missing some packages or
+ // iterating multiple times over some of them. However there is nothing
+ // harmful in that: updates are infrequent and missed packages will be
+ // picked up on the next request.
+ //
+ using pkg_query = query<package_version>;
+ using prep_pkg_query = prepared_query<package_version>;
+
+ size_t offset (0); // See the package version query.
+
+ // Skip external and stub packages.
+ //
+ pkg_query pq ((pkg_query::internal_repository.is_not_null () &&
+ compare_version_ne (pkg_query::id.version,
+ wildcard_version,
+ true)) +
+ "ORDER BY" +
+ pkg_query::id.name + "," +
+ pkg_query::id.version.epoch + "," +
+ pkg_query::id.version.canonical_upstream + "," +
+ pkg_query::id.version.canonical_release + "," +
+ pkg_query::id.version.revision +
+ "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50");
+
+ connection_ptr pkg_conn (package_db_->connection ());
+
+ prep_pkg_query pkg_prep_query (
+ pkg_conn->prepare_query<package_version> (
+ "mod-build-task-package-version-query", pq));
+
+ // Prepare the build prepared query.
+ //
+ // Note that we can not query the database for configurations that a
+ // package was not built with, as the database contains only those package
+ // configurations that have already been acted upon (initially empty).
+ //
+ // This is why we query the database for package configurations that
+ // should not be built (in the tested state with the build terminated
+ // normally or not expired, or in the testing state and not expired).
+ // Having such a list we will select the first build configuration that is
+ // not in the list (if available) for the response.
+ //
+ using bld_query = query<build>;
+ using prep_bld_query = prepared_query<build>;
+
+ package_id id; // See the build query.
+
+ const auto& qv (bld_query::id.package.version);
+
+ bld_query bq (
+ bld_query::id.package.name == bld_query::_ref (id.name) &&
+
+ qv.epoch == bld_query::_ref (id.version.epoch) &&
+ qv.canonical_upstream ==
+ bld_query::_ref (id.version.canonical_upstream) &&
+ qv.canonical_release == bld_query::_ref (id.version.canonical_release) &&
+ qv.revision == bld_query::_ref (id.version.revision) &&
+
+ bld_query::id.configuration.in_range (cfg_names.begin (),
+ cfg_names.end ()) &&
+
+ ((bld_query::state == "tested" &&
+ ((bld_query::status != "abort" && bld_query::status != "abnormal") ||
+ bld_query::timestamp > expiration_ns)) ||
+
+ (bld_query::state == "testing" &&
+ bld_query::timestamp > expiration_ns)));
+
+ connection_ptr bld_conn (build_db_->connection ());
+
+ prep_bld_query bld_prep_query (
+ bld_conn->prepare_query<build> (
+ "mod-build-task-package-build-query", bq));
+
+ while (tsm.session.empty ())
+ {
+ // Start the package database transaction.
+ //
+ transaction pt (pkg_conn->begin ());
+
+ // Query package versions.
+ //
+ auto package_versions (pkg_prep_query.execute ());
+
+ // Bail out if there is nothing left.
+ //
+ if (package_versions.empty ())
+ {
+ pt.commit ();
+ break;
+ }
+
+ offset += package_versions.size ();
+
+ // Start the build database transaction.
+ //
+ {
+ transaction bt (bld_conn->begin (), false);
+ transaction::current (bt);
+
+ // Iterate over packages until we find one that needs building.
+ //
+ for (auto& pv: package_versions)
+ {
+ id = move (pv.id);
+
+ // Iterate through the package configurations and erase those that
+ // don't need building from the build configuration map. All those
+ // configurations that remained can be built. We will take the first
+ // one, if present.
+ //
+ config_machines configs (cfg_machines); // Make a copy for this pkg.
+
+ for (const auto& pc: bld_prep_query.execute ())
+ {
+ auto i (configs.find (pc.id.configuration.c_str ()));
+
+ // Outdated configurations are already excluded with the database
+ // query.
+ //
+ assert (i != configs.end ());
+ configs.erase (i);
+ }
+
+ if (!configs.empty ())
+ {
+ config_machine& cm (configs.begin ()->second);
+ const build_config& cfg (*cm.config);
+
+ build_id bid (move (id), cfg.name);
+ shared_ptr<build> b (build_db_->find<build> (bid));
+
+ // If build configuration doesn't exist then create the new one
+ // and persist. Otherwise put it into the testing state, refresh
+ // the timestamp and update.
+ //
+ if (b == nullptr)
+ {
+ b = make_shared<build> (move (bid.package.name),
+ move (pv.version),
+ move (bid.configuration));
+
+ build_db_->persist (b);
+ }
+ else
+ {
+ // If the package configuration is in the tested state, then we
+ // need to cleanup the status and results prior to the update.
+ // Otherwise the status is already absent and there are no
+ // results.
+ //
+ // Load the section to make sure results are updated for the
+ // tested state, otherwise assert there are no results.
+ //
+ build_db_->load (*b, b->results_section);
+
+ if (b->state == build_state::tested)
+ {
+ assert (b->status);
+ b->status = nullopt;
+
+ b->results.clear ();
+ }
+ else
+ {
+ assert (!b->status && b->results.empty ());
+ }
+
+ b->state = build_state::testing;
+ b->timestamp = timestamp::clock::now ();
+
+ build_db_->update (b);
+ }
+
+ // Finally, prepare the task response manifest.
+ //
+ tsm.session = b->package_name + '/' +
+ b->package_version.string () + '/' + b->configuration;
+
+ // @@ We don't support challenge at the moment, so leave it absent.
+ //
+
+ tsm.result_url = options_->host () + options_->root ().string () +
+ "?build-result";
+
+ // Switch to the package database transaction to load the package.
+ //
+ transaction::current (pt);
+
+ shared_ptr<package> p (package_db_->load<package> (b->id.package));
+ shared_ptr<repository> r (p->internal_repository.load ());
+
+ // Switch back to the build database transaction.
+ //
+ transaction::current (bt);
+
+ strings fp;
+ if (r->certificate)
+ fp.emplace_back (move (r->certificate->fingerprint));
+
+ tsm.task = task_manifest (
+ move (b->package_name),
+ move (b->package_version),
+ move (r->location),
+ move (fp),
+ move (cm.machine->name),
+ cfg.target,
+ cfg.vars);
+ }
+
+ // If task response manifest is filled, then can bail out from the
+ // package loop, commit transactions and respond.
+ //
+ if (!tsm.session.empty ())
+ break;
+ }
+
+ bt.commit (); // Commit the build database transaction.
+ }
+
+ transaction::current (pt); // Switch to the package database transaction.
+ pt.commit ();
+ }
+ }
+
+ // @@ Probably it would be a good idea to also send some cache control
+ // headers to avoid caching by HTTP proxies. That would require extension
+ // of the web::response interface.
+ //
+
+ manifest_serializer s (rs.content (200, "text/manifest;charset=utf-8"),
+ "task_response_manifest");
+ tsm.serialize (s);
+
+ return true;
+}
diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx
index 95bf087..fd3fd6d 100644
--- a/mod/mod-package-details.cxx
+++ b/mod/mod-package-details.cxx
@@ -42,7 +42,7 @@ init (scanner& s)
options_ = make_shared<options::package_details> (
s, unknown_mode::fail, unknown_mode::fail);
- database_module::init (*options_);
+ database_module::init (*options_, options_->package_db_retry ());
if (options_->root ().empty ())
options_->root (dir_path ("/"));
@@ -148,17 +148,17 @@ handle (request& rq, response& rs)
<< ~DIV;
session sn;
- transaction t (db_->begin ());
+ transaction t (package_db_->begin ());
shared_ptr<package> pkg;
{
latest_package lp;
- if (!db_->query_one<latest_package> (
+ if (!package_db_->query_one<latest_package> (
query<latest_package>(
"(" + query<latest_package>::_val (name) + ")"), lp))
throw invalid_request (404, "Package '" + name + "' not found");
- pkg = db_->load<package> (lp.id);
+ pkg = package_db_->load<package> (lp.id);
}
const auto& licenses (pkg->license_alternatives);
@@ -187,7 +187,7 @@ handle (request& rq, response& rs)
}
auto pkg_count (
- db_->query_value<package_count> (
+ package_db_->query_value<package_count> (
search_params<package_count> (name, squery)));
s << FORM_SEARCH (squery)
@@ -197,7 +197,7 @@ handle (request& rq, response& rs)
//
s << DIV;
for (const auto& pr:
- db_->query<package_search_rank> (
+ package_db_->query<package_search_rank> (
search_params<package_search_rank> (name, squery) +
"ORDER BY rank DESC, version_epoch DESC, "
"version_canonical_upstream DESC, version_canonical_release DESC, "
@@ -205,7 +205,7 @@ handle (request& rq, response& rs)
"OFFSET" + to_string (page * res_page) +
"LIMIT" + to_string (res_page)))
{
- shared_ptr<package> p (db_->load<package> (pr.id));
+ shared_ptr<package> p (package_db_->load<package> (pr.id));
s << TABLE(CLASS="proplist version")
<< TBODY
diff --git a/mod/mod-package-search.cxx b/mod/mod-package-search.cxx
index 02b10d6..4988d42 100644
--- a/mod/mod-package-search.cxx
+++ b/mod/mod-package-search.cxx
@@ -44,14 +44,14 @@ init (scanner& s)
options_ = make_shared<options::package_search> (
s, unknown_mode::fail, unknown_mode::fail);
- database_module::init (*options_);
+ database_module::init (*options_, options_->package_db_retry ());
if (options_->root ().empty ())
options_->root (dir_path ("/"));
- // Check that the database schema matches the current one. It's enough to
- // perform the check in just a single module implementation (and we don't
- // do in the dispatcher because it doesn't use the database).
+ // Check that the database 'package' schema matches the current one. It's
+ // enough to perform the check in just a single module implementation (and we
+ // don't do in the dispatcher because it doesn't use the database).
//
// Note that the failure can be reported by each web server worker process.
// While it could be tempting to move the check to the
@@ -59,8 +59,10 @@ init (scanner& s)
// be called by a different process (usually the web server root one) not
// having the proper permissions to access the database.
//
- if (schema_catalog::current_version (*db_) != db_->schema_version ())
- fail << "database schema differs from the current one (module "
+ const string ds ("package");
+ if (schema_catalog::current_version (*package_db_, ds) !=
+ package_db_->schema_version (ds))
+ fail << "database 'package' schema differs from the current one (module "
<< BREP_VERSION_STR << ")";
}
@@ -133,10 +135,10 @@ handle (request& rq, response& rs)
<< DIV(ID="content");
session sn;
- transaction t (db_->begin ());
+ transaction t (package_db_->begin ());
auto pkg_count (
- db_->query_value<latest_package_count> (
+ package_db_->query_value<latest_package_count> (
search_param<latest_package_count> (squery)));
s << FORM_SEARCH (squery)
@@ -146,13 +148,13 @@ handle (request& rq, response& rs)
//
s << DIV;
for (const auto& pr:
- db_->query<latest_package_search_rank> (
+ package_db_->query<latest_package_search_rank> (
search_param<latest_package_search_rank> (squery) +
"ORDER BY rank DESC, name" +
"OFFSET" + to_string (page * res_page) +
"LIMIT" + to_string (res_page)))
{
- shared_ptr<package> p (db_->load<package> (pr.id));
+ shared_ptr<package> p (package_db_->load<package> (pr.id));
s << TABLE(CLASS="proplist package")
<< TBODY
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index 89acf89..b0e6ead 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -43,7 +43,7 @@ init (scanner& s)
options_ = make_shared<options::package_version_details> (
s, unknown_mode::fail, unknown_mode::fail);
- database_module::init (*options_);
+ database_module::init (*options_, options_->package_db_retry ());
if (options_->root ().empty ())
options_->root (dir_path ("/"));
@@ -130,11 +130,11 @@ handle (request& rq, response& rs)
shared_ptr<package> pkg;
session sn;
- transaction t (db_->begin ());
+ transaction t (package_db_->begin ());
try
{
- pkg = db_->load<package> (package_id (name, ver));
+ pkg = package_db_->load<package> (package_id (name, ver));
// If the requested package turned up to be an "external" one just
// respond that no "internal" package is present.
diff --git a/mod/mod-repository-details.cxx b/mod/mod-repository-details.cxx
index ab87cb2..3a73955 100644
--- a/mod/mod-repository-details.cxx
+++ b/mod/mod-repository-details.cxx
@@ -49,7 +49,7 @@ init (scanner& s)
options_ = make_shared<options::repository_details> (
s, unknown_mode::fail, unknown_mode::fail);
- database_module::init (*options_);
+ database_module::init (*options_, options_->package_db_retry ());
if (options_->root ().empty ())
options_->root (dir_path ("/"));
@@ -90,12 +90,12 @@ handle (request& rq, response& rs)
<< DIV_HEADER (root, options_->logo (), options_->menu ())
<< DIV(ID="content");
- transaction t (db_->begin ());
+ transaction t (package_db_->begin ());
using query = query<repository>;
for (const auto& r:
- db_->query<repository> (
+ package_db_->query<repository> (
query::internal + "ORDER BY" + query::priority))
{
//@@ Feels like a lot of trouble (e.g., id_attribute()) for very
diff --git a/mod/mod-repository-root b/mod/mod-repository-root
index 57db93a..6a40e10 100644
--- a/mod/mod-repository-root
+++ b/mod/mod-repository-root
@@ -17,6 +17,9 @@ namespace brep
class package_details;
class package_version_details;
class repository_details;
+ class build_task;
+ class build_result;
+ class build_log;
class repository_root: public module
{
@@ -55,6 +58,9 @@ namespace brep
shared_ptr<package_details> package_details_;
shared_ptr<package_version_details> package_version_details_;
shared_ptr<repository_details> repository_details_;
+ shared_ptr<build_task> build_task_;
+ shared_ptr<build_result> build_result_;
+ shared_ptr<build_log> build_log_;
shared_ptr<options::repository_root> options_;
// Sub-module the request is dispatched to. Initially is NULL. It is set
diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx
index 1bc10cb..e156308 100644
--- a/mod/mod-repository-root.cxx
+++ b/mod/mod-repository-root.cxx
@@ -12,6 +12,9 @@
#include <mod/module>
#include <mod/options>
+#include <mod/mod-build-log>
+#include <mod/mod-build-task>
+#include <mod/mod-build-result>
#include <mod/mod-package-search>
#include <mod/mod-package-details>
#include <mod/mod-repository-details>
@@ -55,7 +58,10 @@ namespace brep
: package_search_ (make_shared<package_search> ()),
package_details_ (make_shared<package_details> ()),
package_version_details_ (make_shared<package_version_details> ()),
- repository_details_ (make_shared<repository_details> ())
+ repository_details_ (make_shared<repository_details> ()),
+ build_task_ (make_shared<build_task> ()),
+ build_result_ (make_shared<build_result> ()),
+ build_log_ (make_shared<build_log> ())
{
}
@@ -83,6 +89,18 @@ namespace brep
r.initialized_
? r.repository_details_
: make_shared<repository_details> (*r.repository_details_)),
+ build_task_ (
+ r.initialized_
+ ? r.build_task_
+ : make_shared<build_task> (*r.build_task_)),
+ build_result_ (
+ r.initialized_
+ ? r.build_result_
+ : make_shared<build_result> (*r.build_result_)),
+ build_log_ (
+ r.initialized_
+ ? r.build_log_
+ : make_shared<build_log> (*r.build_log_)),
options_ (
r.initialized_
? r.options_
@@ -101,6 +119,9 @@ namespace brep
append (r, package_details_->options ());
append (r, package_version_details_->options ());
append (r, repository_details_->options ());
+ append (r, build_task_->options ());
+ append (r, build_result_->options ());
+ append (r, build_log_->options ());
return r;
}
@@ -109,17 +130,38 @@ namespace brep
void repository_root::
init (const name_values& v)
{
- auto sub_init ([this, &v](module& m)
+ auto sub_init = [this, &v] (module& m, const char* name)
+ {
+ // Initialize sub-module. Intercept exception handling to add sub-module
+ // attribution.
+ //
+ try
{
m.init (filter (v, m.options ()), *log_);
- });
+ }
+ catch (const std::exception& e)
+ {
+ // Any exception thrown by this function terminates the web server. All
+ // exception types inherited from std::exception are handled by the web
+ // server as std::exception. The only sensible way to handle them is to
+ // log the error prior terminating. By that reason it is valid to
+ // reduce all these types to a single one.
+ //
+ ostringstream os;
+ os << name << ": " << e;
+ throw runtime_error (os.str ());
+ }
+ };
// Initialize sub-modules.
//
- sub_init (*package_search_);
- sub_init (*package_details_);
- sub_init (*package_version_details_);
- sub_init (*repository_details_);
+ sub_init (*package_search_, "package_search");
+ sub_init (*package_details_, "package_details");
+ sub_init (*package_version_details_, "package_version_details");
+ sub_init (*repository_details_, "repository_details");
+ sub_init (*build_task_, "build_task");
+ sub_init (*build_result_, "build_result");
+ sub_init (*build_log_, "build_log");
// Parse own configuration options.
//
@@ -155,8 +197,7 @@ namespace brep
// Delegate the request handling to the selected sub-module. Intercept
// exception handling to add sub-module attribution.
//
- auto handle =
- [&rs, this] (request& rq, const char* name) -> bool
+ auto handle = [&rs, this] (request& rq, const char* name) -> bool
{
try
{
@@ -194,15 +235,24 @@ namespace brep
//
if (lpath.empty ())
{
- // Dispatch request handling to the repository_details or the
- // package_search module depending on the function name passed as a
- // first HTTP request parameter. The parameter should have no value
- // specified. Example: cppget.org/?about
+ // Dispatch request handling to the repository_details, the build_task,
+ // the build_result or the package_search module depending on the
+ // function name passed as a first HTTP request parameter. The parameter
+ // should have no value specified. Example: cppget.org/?about
//
const name_values& params (rq.parameters ());
if (!params.empty () && !params.front ().value)
{
- if (params.front ().name == "about")
+ // Cleanup not to confuse the selected module with the unknown
+ // parameter.
+ //
+ name_values p (params);
+ p.erase (p.begin ());
+
+ request_proxy rp (rq, p);
+ const string& fn (params.front ().name);
+
+ if (fn == "about")
{
if (handler_ == nullptr)
handler_.reset (new repository_details (*repository_details_));
@@ -236,8 +286,9 @@ namespace brep
}
else
{
- // Dispatch request handling to the package_details or the
- // package_version_details module depending on the HTTP request URL path.
+ // Dispatch request handling to the package_details, the
+ // package_version_details or the build_log module depending on the HTTP
+ // request URL path.
//
auto i (lpath.begin ());
assert (i != lpath.end ());
@@ -279,6 +330,13 @@ namespace brep
return handle (rq, "package_version_details");
}
+ else if (*i == "log")
+ {
+ if (handler_ == nullptr)
+ handler_.reset (new build_log (*build_log_));
+
+ return handle (rq, "build_log");
+ }
}
}
@@ -295,6 +353,7 @@ namespace brep
info << "module " << BREP_VERSION_STR
<< ", libbrep " << LIBBREP_VERSION_STR
+ << ", libbbot " << LIBBBOT_VERSION_STR
<< ", libbpkg " << LIBBPKG_VERSION_STR
<< ", libbutl " << LIBBUTL_VERSION_STR;
}
diff --git a/mod/options-types b/mod/options-types
index 6c2e42f..d9eaf9e 100644
--- a/mod/options-types
+++ b/mod/options-types
@@ -26,7 +26,6 @@ namespace brep
page_menu () = default;
page_menu (string b, string l): label (move (b)), link (move (l)) {}
};
-
}
#endif // MOD_OPTIONS_TYPES
diff --git a/mod/options.cli b/mod/options.cli
index fb86fcd..4dbcd89 100644
--- a/mod/options.cli
+++ b/mod/options.cli
@@ -18,13 +18,30 @@ namespace brep
//
class module
{
+ string email
+ {
+ "<email>",
+ "Repository email. This email is used for the \cb{From:} header in
+ emails send by \cb{brep} (for example, build failure notifications)."
+ }
+
+ string host
+ {
+ "<host>",
+ "Repository host. It specifies the scheme and the host address (but
+ not the root path; see \cb{root} below) that will be used whenever
+ \cb{brep} needs to construct an absolute URL to one of its locations
+ (for example, a link to a build log that is being send via email)."
+ }
+
dir_path root = "/"
{
"<path>"
"Repository root. That is, this is the part of the URL between the
host name and the start of the repository. For example, root value
- '\cb{/pkg}' means the repository URL is http://example.org/pkg/.
- Specify '\cb{/}' to use the web server root (http://example.org/)."
+ '\cb{/pkg}' means the repository URL is \cb{http://example.org/pkg/}.
+ Specify '\cb{/}' to use the web server root
+ (\cb{http://example.org/})."
}
uint16_t verbosity = 0
@@ -35,56 +52,124 @@ namespace brep
}
};
- class db
+ class package_db
{
- string db-user
+ string package-db-user
{
"<user>",
- "Database user name. If not specified, then operating system (login)
- name is used."
+ "Package database user name. If not specified, then operating system
+ (login) name is used."
}
- string db-password
+ string package-db-password
{
"<pass>",
- "Database password. If not specified, then login without password is
- expected to work."
+ "Package database password. If not specified, then login without
+ password is expected to work."
}
- string db-name = "brep"
+ string package-db-name = "brep_package"
{
"<name>",
- "Database name. If not specified, then '\cb{brep}' is used by
- default."
+ "Package database name. If not specified, then \cb{brep_package} is
+ used by default."
}
- string db-host
+ string package-db-host
{
"<host>",
- "Database host name, address, or socket. If not specified, then
+ "Package database host name, address, or socket. If not specified, then
connect to \cb{localhost} using the operating system-default
mechanism (Unix-domain socket, etc)."
}
- uint16_t db-port = 0
+ uint16_t package-db-port = 0
{
"<port>",
- "Database port number. If not specified, the default port is used."
+ "Package database port number. If not specified, the default port is
+ used."
}
- size_t db-max-connections = 5
+ size_t package-db-max-connections = 5
{
"<num>",
- "The maximum number of concurrent database connections per web server
- process. If 0, then no limitation is applied. The default is 5."
+ "The maximum number of concurrent package database connections per web
+ server process. If 0, then no limitation is applied. The default is
+ 5."
}
- size_t db-retry = 10
+ size_t package-db-retry = 10
{
"<num>",
- "The maximum number of times to retry database transactions in the
- face of recoverable failures (deadlock, loss of connection, etc). The
- default is 10."
+ "The maximum number of times to retry package database transactions in
+ the face of recoverable failures (deadlock, loss of connection, etc).
+ The default is 10."
+ }
+ };
+
+ class build
+ {
+ path build-config
+ {
+ "<buildtab>",
+ "Build configuration file. If not specified, then the package building
+ functionality will be disabled. If specified, then the build database
+ must be configured (see \cb{build-db-*})."
+ }
+ };
+
+ class build_db
+ {
+ string build-db-user
+ {
+ "<user>",
+ "Build database user name. If not specified, then operating system
+ (login) name is used."
+ }
+
+ string build-db-password
+ {
+ "<pass>",
+ "Build database password. If not specified, then login without
+ password is expected to work."
+ }
+
+ string build-db-name = "brep_build"
+ {
+ "<name>",
+ "Build database name. If not specified, then \cb{brep_build} is used
+ by default."
+ }
+
+ string build-db-host
+ {
+ "<host>",
+ "Build database host name, address, or socket. If not specified, then
+ connect to \cb{localhost} using the operating system-default
+ mechanism (Unix-domain socket, etc)."
+ }
+
+ uint16_t build-db-port = 0
+ {
+ "<port>",
+ "Build database port number. If not specified, the default port is
+ used."
+ }
+
+ size_t build-db-max-connections = 5
+ {
+ "<num>",
+ "The maximum number of concurrent build database connections per web
+ server process. If 0, then no limitation is applied. The default is
+ 5."
+ }
+
+ size_t build-db-retry = 10
+ {
+ "<num>",
+ "The maximum number of times to retry build database transactions in
+ the face of recoverable failures (deadlock, loss of connection, etc).
+ The default is 10."
}
};
@@ -141,7 +226,7 @@ namespace brep
// Module options.
//
- class package_search: search, db, page, module
+ class package_search: search, package_db, page, module
{
string search-title = "Packages"
{
@@ -151,21 +236,56 @@ namespace brep
}
};
- class package_details: package, search, db, page, module
+ class package_details: package, search, package_db, page, module
{
};
- class package_version_details: package, db, page, module
+ class package_version_details: package, package_db, page, module
{
};
- class repository_details: db, page, module
+ class repository_details: package_db, page, module
{
};
class repository_root: module
{
};
+
+ class build_task: build, package_db, build_db, module
+ {
+ size_t build-task-request-max-size = 102400
+ {
+ "<bytes>",
+ "The maximum size of the build task request manifest accepted. Note
+ that the HTTP POST request body is cached to retry database
+ transactions in the face of recoverable failures (deadlock, loss of
+ connection, etc). The default is 100K."
+ }
+
+ size_t build-result-timeout = 10800
+ {
+ "<minutes>",
+ "Time to wait before considering the expected task result lost. Must be
+ specified in seconds. The default is 3 hours."
+ }
+ };
+
+ class build_result: build, package_db, build_db, module
+ {
+ size_t build-result-request-max-size = 10240000
+ {
+ "<bytes>",
+ "The maximum size of the build result manifest accepted. Note that the
+ HTTP POST request body is cached to retry database transactions in the
+ face of recoverable failures (deadlock, loss of connection, etc). The
+ default is 10M."
+ }
+ };
+
+ class build_log: build, package_db, build_db, module
+ {
+ };
}
// Web module HTTP request parameters.
@@ -213,5 +333,23 @@ namespace brep
// No parameters so far.
//
};
+
+ class build_task
+ {
+ // No parameters so far.
+ //
+ };
+
+ class build_result
+ {
+ // No parameters so far.
+ //
+ };
+
+ class build_log
+ {
+ // No parameters so far.
+ //
+ };
}
}
diff --git a/mod/types-parsers b/mod/types-parsers
index 1082292..eb206e4 100644
--- a/mod/types-parsers
+++ b/mod/types-parsers
@@ -25,31 +25,38 @@ namespace brep
struct parser;
template <>
+ struct parser<path>
+ {
+ static void
+ parse (path&, bool&, scanner&);
+ };
+
+ template <>
struct parser<dir_path>
{
static void
- parse (dir_path&, scanner&);
+ parse (dir_path&, bool&, scanner&);
};
template <>
struct parser<page_form>
{
static void
- parse (page_form&, scanner&);
+ parse (page_form&, bool&, scanner&);
};
template <>
struct parser<page_menu>
{
static void
- parse (page_menu&, scanner&);
+ parse (page_menu&, bool&, scanner&);
};
template <>
struct parser<web::xhtml::fragment>
{
static void
- parse (web::xhtml::fragment&, scanner&);
+ parse (web::xhtml::fragment&, bool&, scanner&);
};
}
}
diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx
index 3c51da9..fb293a3 100644
--- a/mod/types-parsers.cxx
+++ b/mod/types-parsers.cxx
@@ -36,17 +36,26 @@ namespace brep
}
}
+ void parser<path>::
+ parse (path& x, bool& xs, scanner& s)
+ {
+ xs = true;
+ parse_path (x, s);
+ }
+
void parser<dir_path>::
- parse (dir_path& x, scanner& s)
+ parse (dir_path& x, bool& xs, scanner& s)
{
+ xs = true;
parse_path (x, s);
}
// Parse page_form.
//
void parser<page_form>::
- parse (page_form& x, scanner& s)
+ parse (page_form& x, bool& xs, scanner& s)
{
+ xs = true;
const char* o (s.next ());
if (!s.more ())
@@ -64,8 +73,9 @@ namespace brep
// Parse page_menu.
//
void parser<page_menu>::
- parse (page_menu& x, scanner& s)
+ parse (page_menu& x, bool& xs, scanner& s)
{
+ xs = true;
const char* o (s.next ());
if (!s.more ())
@@ -92,8 +102,9 @@ namespace brep
// Parse web::xhtml::fragment.
//
void parser<fragment>::
- parse (fragment& x, scanner& s)
+ parse (fragment& x, bool& xs, scanner& s)
{
+ xs = true;
const char* o (s.next ());
if (!s.more ())
diff --git a/tests/load/driver.cxx b/tests/load/driver.cxx
index a0ab10f..f2c97b5 100644
--- a/tests/load/driver.cxx
+++ b/tests/load/driver.cxx
@@ -63,7 +63,7 @@ main (int argc, char* argv[])
string user;
string password;
- string name ("brep");
+ string name ("brep_package");
string host;
unsigned int port (0);
int i (2);
diff --git a/web/apache/request.cxx b/web/apache/request.cxx
index f69fedc..b4fb080 100644
--- a/web/apache/request.cxx
+++ b/web/apache/request.cxx
@@ -29,10 +29,12 @@
#include <algorithm> // min()
#include <butl/optional>
+#include <butl/timestamp>
#include <web/mime-url-encoding>
using namespace std;
+using namespace butl;
namespace web
{
@@ -519,10 +521,8 @@ namespace web
if (max_age)
{
- chrono::system_clock::time_point tp (
- chrono::system_clock::now () + *max_age);
-
- time_t t (chrono::system_clock::to_time_t (tp));
+ timestamp tp (timestamp::clock::now () + *max_age);
+ time_t t (timestamp::clock::to_time_t (tp));
// Assume global locale is not changed and still "C".
//