aboutsummaryrefslogtreecommitdiff
path: root/mod
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2017-04-04 20:53:00 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2017-04-19 22:16:46 +0300
commitdbbc19b77dcf6ea828aabd64d7aa8cab9635aaf5 (patch)
treec0b9b449b7064dff3613628022224e6c18148c3e /mod
parentefb9c3e0e6b612d5bfadc7a2b984c14b5439335c (diff)
Implement build task, result and log requests handling
Diffstat (limited to 'mod')
-rw-r--r--mod/build-config23
-rw-r--r--mod/build-config.cxx31
-rw-r--r--mod/buildfile7
-rw-r--r--mod/database15
-rw-r--r--mod/database-module24
-rw-r--r--mod/database-module.cxx52
-rw-r--r--mod/database.cxx59
-rw-r--r--mod/mod-build-log45
-rw-r--r--mod/mod-build-log.cxx220
-rw-r--r--mod/mod-build-result42
-rw-r--r--mod/mod-build-result.cxx303
-rw-r--r--mod/mod-build-task42
-rw-r--r--mod/mod-build-task.cxx389
-rw-r--r--mod/mod-package-details.cxx14
-rw-r--r--mod/mod-package-search.cxx22
-rw-r--r--mod/mod-package-version-details.cxx6
-rw-r--r--mod/mod-repository-details.cxx6
-rw-r--r--mod/mod-repository-root6
-rw-r--r--mod/mod-repository-root.cxx91
-rw-r--r--mod/options-types1
-rw-r--r--mod/options.cli192
-rw-r--r--mod/types-parsers15
-rw-r--r--mod/types-parsers.cxx19
23 files changed, 1511 insertions, 113 deletions
diff --git a/mod/build-config b/mod/build-config
new file mode 100644
index 0000000..a5713d7
--- /dev/null
+++ b/mod/build-config
@@ -0,0 +1,23 @@
+// file : mod/build-config -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_BUILD_CONFIG
+#define MOD_BUILD_CONFIG
+
+#include <bbot/build-config>
+
+#include <brep/types>
+#include <brep/utility>
+
+namespace brep
+{
+ // Return pointer to the shared build configurations instance, creating one
+ // on the first call. Throw tab_parsing on parsing error, io_error on the
+ // underlying OS error. Is not thread-safe.
+ //
+ shared_ptr<const bbot::build_configs>
+ shared_build_config (const path&);
+}
+
+#endif // MOD_BUILD_CONFIG
diff --git a/mod/build-config.cxx b/mod/build-config.cxx
new file mode 100644
index 0000000..11be1b9
--- /dev/null
+++ b/mod/build-config.cxx
@@ -0,0 +1,31 @@
+// file : mod/build-config.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/build-config>
+
+#include <map>
+
+namespace brep
+{
+ using namespace bbot;
+
+ shared_ptr<const build_configs>
+ shared_build_config (const path& p)
+ {
+ static std::map<path, weak_ptr<build_configs>> configs;
+
+ auto i (configs.find (p));
+ if (i != configs.end ())
+ {
+ if (shared_ptr<build_configs> c = i->second.lock ())
+ return c;
+ }
+
+ shared_ptr<build_configs> c (
+ make_shared<build_configs> (parse_buildtab (p)));
+
+ configs[p] = c;
+ return c;
+ }
+}
diff --git a/mod/buildfile b/mod/buildfile
index 228c149..2926a1b 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -14,13 +14,18 @@ import libs += libodb%lib{odb}
import libs += libodb-pgsql%lib{odb-pgsql}
import libs += libbutl%lib{butl}
import libs += libbpkg%lib{bpkg}
+import libs += libbbot%lib{bbot}
include ../brep/
mod{brep}: \
+ {hxx cxx}{ build-config } \
{hxx cxx}{ database } \
{hxx cxx}{ database-module } \
{hxx cxx}{ diagnostics } \
+ {hxx cxx}{ mod-build-log } \
+ {hxx cxx}{ mod-build-result } \
+ {hxx cxx}{ mod-build-task } \
{hxx cxx}{ mod-package-details } \
{hxx cxx}{ mod-package-search } \
{hxx cxx}{ mod-package-version-details } \
@@ -57,7 +62,7 @@ if $cli.configured
# parameters uniformly with a single catch block.
#
cli.options += --std c++11 -I $src_root --include-with-brackets \
---include-prefix mod --guard-prefix MOD \
+--include-prefix mod --guard-prefix MOD --generate-specifier \
--cxx-prologue "#include <mod/types-parsers>" \
--cli-namespace brep::cli --generate-file-scanner --suppress-usage \
--generate-modifier --generate-description --option-prefix ""
diff --git a/mod/database b/mod/database
index 8e9fdd1..9a83752 100644
--- a/mod/database
+++ b/mod/database
@@ -10,17 +10,18 @@
#include <brep/types>
#include <brep/utility>
-#include <mod/options>
-
namespace brep
{
- // Returns pointer to the shared database instance, creating one on the
- // first call. On subsequent calls ensures passed host and port equals
- // to ones of the existing database instance throwing runtime_error
- // otherwise. Is not thread-safe.
+ // Return pointer to the shared database instance, creating one on the first
+ // call. Throw odb::exception on failure. Is not thread-safe.
//
shared_ptr<odb::core::database>
- shared_database (const options::db&);
+ shared_database (string user,
+ string password,
+ string name,
+ string host,
+ uint16_t port,
+ size_t max_connections);
}
#endif // MOD_DATABASE
diff --git a/mod/database-module b/mod/database-module
index 034324b..3799e7b 100644
--- a/mod/database-module
+++ b/mod/database-module
@@ -10,6 +10,8 @@
#include <brep/types>
#include <brep/utility>
+#include <bbot/build-config>
+
#include <mod/module>
#include <mod/options>
@@ -36,15 +38,31 @@ namespace brep
//
using module::init;
+ // Initialize the package database instance. Throw odb::exception on
+ // failure.
+ //
+ void
+ init (const options::package_db&, size_t retry);
+
+ // Initialize the build database instance and parse build configuration
+ // file. Throw odb::exception on database failure, tab_parsing on parsing
+ // error, system_error on the underlying OS error.
+ //
void
- init (const options::db&);
+ init (const options::build&, const options::build_db&, size_t retry);
virtual bool
handle (request&, response&) = 0;
protected:
- size_t retry_;
- shared_ptr<odb::core::database> db_;
+ size_t retry_ = 0; // Max of all retries.
+
+ shared_ptr<odb::core::database> package_db_;
+
+ // These are NULL if not building.
+ //
+ shared_ptr<odb::core::database> build_db_;
+ shared_ptr<const bbot::build_configs> build_conf_;
private:
virtual bool
diff --git a/mod/database-module.cxx b/mod/database-module.cxx
index 6fd5025..e7a6883 100644
--- a/mod/database-module.cxx
+++ b/mod/database-module.cxx
@@ -4,13 +4,23 @@
#include <mod/database-module>
+#include <errno.h> // EIO
+
+#include <sstream>
+
#include <odb/exceptions.hxx>
+#include <butl/utility> // throw_generic_error()
+
#include <mod/options>
#include <mod/database>
+#include <mod/build-config>
namespace brep
{
+ using namespace std;
+ using namespace butl;
+
// While currently the user-defined copy constructor is not required (we
// don't need to deep copy nullptr's), it is a good idea to keep the
// placeholder ready for less trivial cases.
@@ -19,15 +29,49 @@ namespace brep
database_module (const database_module& r)
: module (r),
retry_ (r.retry_),
- db_ (r.initialized_ ? r.db_ : nullptr)
+ package_db_ (r.initialized_ ? r.package_db_ : nullptr),
+ build_db_ (r.initialized_ ? r.build_db_ : nullptr),
+ build_conf_ (r.initialized_ ? r.build_conf_ : nullptr)
+ {
+ }
+
+ void database_module::
+ init (const options::package_db& o, size_t retry)
{
+ package_db_ = shared_database (o.package_db_user (),
+ o.package_db_password (),
+ o.package_db_name (),
+ o.package_db_host (),
+ o.package_db_port (),
+ o.package_db_max_connections ());
+
+ retry_ = retry_ < retry ? retry : retry_;
}
void database_module::
- init (const options::db& o)
+ init (const options::build& bo, const options::build_db& dbo, size_t retry)
{
- retry_ = o.db_retry ();
- db_ = shared_database (o);
+ try
+ {
+ build_conf_ = shared_build_config (bo.build_config ());
+ }
+ catch (const io_error& e)
+ {
+ ostringstream os;
+ os << "unable to read build configuration '" << bo.build_config ()
+ << "': " << e;
+
+ throw_generic_error (EIO, os.str ().c_str ());
+ }
+
+ build_db_ = shared_database (dbo.build_db_user (),
+ dbo.build_db_password (),
+ dbo.build_db_name (),
+ dbo.build_db_host (),
+ dbo.build_db_port (),
+ dbo.build_db_max_connections ());
+
+ retry_ = retry_ < retry ? retry : retry_;
}
bool database_module::
diff --git a/mod/database.cxx b/mod/database.cxx
index f8fa1e5..22a0563 100644
--- a/mod/database.cxx
+++ b/mod/database.cxx
@@ -11,30 +11,43 @@
namespace brep
{
- namespace options
+ struct db_key
{
- bool
- operator< (const db& x, const db& y)
- {
- int r;
- if ((r = x.db_user ().compare (y.db_user ())) != 0 ||
- (r = x.db_password ().compare (y.db_password ())) != 0 ||
- (r = x.db_name ().compare (y.db_name ())) != 0 ||
- (r = x.db_host ().compare (y.db_host ())))
- return r < 0;
-
- return x.db_port () < y.db_port ();
- }
+ string user;
+ string password;
+ string name;
+ string host;
+ uint16_t port;
+ };
+
+ static bool
+ operator< (const db_key& x, const db_key& y)
+ {
+ int r;
+ if ((r = x.user.compare (y.user)) != 0 ||
+ (r = x.password.compare (y.password)) != 0 ||
+ (r = x.name.compare (y.name)) != 0 ||
+ (r = x.host.compare (y.host)))
+ return r < 0;
+
+ return x.port < y.port;
}
using namespace odb;
shared_ptr<database>
- shared_database (const options::db& o)
+ shared_database (string user,
+ string password,
+ string name,
+ string host,
+ uint16_t port,
+ size_t max_connections)
{
- static std::map<options::db, weak_ptr<database>> databases;
+ static std::map<db_key, weak_ptr<database>> databases;
+
+ db_key k ({move (user), move (password), move (name), host, port});
- auto i (databases.find (o));
+ auto i (databases.find (k));
if (i != databases.end ())
{
if (shared_ptr<database> d = i->second.lock ())
@@ -42,19 +55,19 @@ namespace brep
}
unique_ptr<pgsql::connection_factory>
- f (new pgsql::connection_pool_factory (o.db_max_connections ()));
+ f (new pgsql::connection_pool_factory (max_connections));
shared_ptr<database> d (
make_shared<pgsql::database> (
- o.db_user (),
- o.db_password (),
- o.db_name (),
- o.db_host (),
- o.db_port (),
+ k.user,
+ k.password,
+ k.name,
+ k.host,
+ k.port,
"options='-c default_transaction_isolation=serializable'",
move (f)));
- databases[o] = d;
+ databases[move (k)] = d;
return d;
}
}
diff --git a/mod/mod-build-log b/mod/mod-build-log
new file mode 100644
index 0000000..8395546
--- /dev/null
+++ b/mod/mod-build-log
@@ -0,0 +1,45 @@
+// file : mod/mod-build-log -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_MOD_BUILD_LOG
+#define MOD_MOD_BUILD_LOG
+
+#include <brep/types>
+#include <brep/utility>
+
+#include <mod/options>
+#include <mod/database-module>
+
+namespace brep
+{
+ class build_log: public database_module
+ {
+ public:
+ build_log () = default;
+
+ // Create a shallow copy (handling instance) if initialized and a deep
+ // copy (context exemplar) otherwise.
+ //
+ explicit
+ build_log (const build_log&);
+
+ virtual bool
+ handle (request&, response&);
+
+ virtual const cli::options&
+ cli_options () const
+ {
+ return options::build_log::description ();
+ }
+
+ private:
+ virtual void
+ init (cli::scanner&);
+
+ private:
+ shared_ptr<options::build_log> options_;
+ };
+}
+
+#endif // MOD_MOD_BUILD_LOG
diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx
new file mode 100644
index 0000000..5342e3e
--- /dev/null
+++ b/mod/mod-build-log.cxx
@@ -0,0 +1,220 @@
+// file : mod/mod-build-log.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/mod-build-log>
+
+#include <algorithm> // find_if()
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
+#include <web/module>
+
+#include <brep/build>
+#include <brep/build-odb>
+#include <brep/package>
+#include <brep/package-odb>
+
+#include <mod/options>
+
+using namespace std;
+using namespace bbot;
+using namespace brep::cli;
+using namespace odb::core;
+
+// While currently the user-defined copy constructor is not required (we don't
+// need to deep copy nullptr's), it is a good idea to keep the placeholder
+// ready for less trivial cases.
+//
+brep::build_log::
+build_log (const build_log& r)
+ : database_module (r),
+ options_ (r.initialized_ ? r.options_ : nullptr)
+{
+}
+
+void brep::build_log::
+init (scanner& s)
+{
+ MODULE_DIAG;
+
+ options_ = make_shared<options::build_log> (
+ s, unknown_mode::fail, unknown_mode::fail);
+
+ database_module::init (static_cast<options::package_db> (*options_),
+ options_->package_db_retry ());
+
+ if (options_->build_config_specified ())
+ database_module::init (static_cast<options::build> (*options_),
+ static_cast<options::build_db> (*options_),
+ options_->build_db_retry ());
+
+ if (options_->root ().empty ())
+ options_->root (dir_path ("/"));
+}
+
+bool brep::build_log::
+handle (request& rq, response& rs)
+{
+ using brep::version; // Not to confuse with module::version.
+
+ MODULE_DIAG;
+
+ if (build_db_ == nullptr)
+ throw invalid_request (501, "not implemented");
+
+ // Parse the HTTP request URL path (without the root directory) to obtain
+ // the build package name/version, the configuration name and the optional
+ // operation name. If the operation is not specified then print logs for all
+ // the operations.
+ //
+ // Note that the URL path must be in the following form:
+ //
+ // <package-name>/<package-version>/log/<config-name>[/<operation>]
+ //
+ // Also note that the presence of the first 3 components is guaranteed by
+ // the repository_root module.
+ //
+ build_id id;
+ string op;
+
+ path lpath (rq.path ().leaf (options_->root ()));
+
+ try
+ {
+ auto i (lpath.begin ());
+
+ assert (i != lpath.end ());
+ string name (*i++);
+
+ if (name.empty ())
+ throw invalid_argument ("empty package name");
+
+ assert (i != lpath.end ());
+
+ version version;
+
+ // Intercept exception handling to add the parsing error attribution.
+ //
+ try
+ {
+ version = brep::version (*i++);
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_argument (
+ string ("invalid package version: ") + e.what ());
+ }
+
+ assert (i != lpath.end () && *i == "log");
+
+ if (++i == lpath.end ())
+ throw invalid_argument ("no configuration name");
+
+ id = build_id (package_id (move (name), version), *i++);
+
+ if (id.configuration.empty ())
+ throw invalid_argument ("empty configuration name");
+
+ if (i != lpath.end ())
+ op = *i++;
+
+ if (i != lpath.end ())
+ throw invalid_argument ("unexpected path component");
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ // Make sure no parameters passed.
+ //
+ try
+ {
+ name_value_scanner s (rq.parameters ());
+ params::build_log (s, unknown_mode::fail, unknown_mode::fail);
+ }
+ catch (const cli::exception& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ // If the package build configuration expired (no such configuration,
+ // package, etc), then we log this case with the trace severity and respond
+ // with the 404 HTTP code (not found but may be available in the future).
+ // The thinking is that this may be or may not be a problem with the
+ // controller's setup (expires too fast or the link from some ancient email
+ // is opened).
+ //
+ auto config_expired = [&trace, &lpath, this] (const string& d)
+ {
+ l2 ([&]{trace << "package build configuration for " << lpath
+ << " expired: " << d;});
+
+ throw invalid_request (404, "package build configuration expired: " + d);
+ };
+
+ // Make sure the build configuration still exists.
+ //
+ auto i (
+ find_if (
+ build_conf_->begin (), build_conf_->end (),
+ [&id] (const build_config& c) {return c.name == id.configuration;}));
+
+ if (i == build_conf_->end ())
+ config_expired ("no configuration");
+
+ // Make sure the package still exists.
+ //
+ {
+ transaction t (package_db_->begin ());
+ shared_ptr<package> p (package_db_->find<package> (id.package));
+ t.commit ();
+
+ if (p == nullptr)
+ config_expired ("no package");
+ }
+
+ // Load the package build configuration (if present).
+ //
+ shared_ptr<build> b;
+ {
+ transaction t (build_db_->begin ());
+ b = build_db_->find<build> (id);
+
+ if (b == nullptr)
+ config_expired ("no package configuration");
+ else if (b->state != build_state::tested)
+ config_expired ("state is " + to_string (b->state));
+ else
+ build_db_->load (*b, b->results_section);
+
+ t.commit ();
+ }
+
+ // We have all the data so don't buffer the response content.
+ //
+ ostream& os (rs.content (200, "text/plain;charset=utf-8", false));
+
+ if (op.empty ())
+ {
+ for (const auto& r: b->results)
+ os << r.log;
+ }
+ else
+ {
+ const operation_results& r (b->results);
+
+ auto i (
+ find_if (r.begin (), r.end (),
+ [&op] (const operation_result& v) {return v.operation == op;}));
+
+ if (i == r.end ())
+ config_expired ("no operation");
+
+ os << i->log;
+ }
+
+ return true;
+}
diff --git a/mod/mod-build-result b/mod/mod-build-result
new file mode 100644
index 0000000..8afa697
--- /dev/null
+++ b/mod/mod-build-result
@@ -0,0 +1,42 @@
+// file : mod/mod-build-result -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_MOD_BUILD_RESULT
+#define MOD_MOD_BUILD_RESULT
+
+#include <brep/types>
+#include <brep/utility>
+
+#include <mod/options>
+#include <mod/database-module>
+
+namespace brep
+{
+ class build_result: public database_module
+ {
+ public:
+ build_result () = default;
+
+ // Create a shallow copy (handling instance) if initialized and a deep
+ // copy (context exemplar) otherwise.
+ //
+ explicit
+ build_result (const build_result&);
+
+ virtual bool
+ handle (request&, response&);
+
+ virtual const cli::options&
+ cli_options () const {return options::build_result::description ();}
+
+ private:
+ virtual void
+ init (cli::scanner&);
+
+ private:
+ shared_ptr<options::build_result> options_;
+ };
+}
+
+#endif // MOD_MOD_BUILD_RESULT
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
new file mode 100644
index 0000000..fb6edb5
--- /dev/null
+++ b/mod/mod-build-result.cxx
@@ -0,0 +1,303 @@
+// file : mod/mod-build-result.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/mod-build-result>
+
+#include <algorithm> // find_if()
+
+#include <butl/sendmail>
+#include <butl/process-io>
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+#include <bbot/manifest>
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
+#include <web/module>
+
+#include <brep/build>
+#include <brep/build-odb>
+#include <brep/package>
+#include <brep/package-odb>
+
+#include <mod/options>
+
+using namespace std;
+using namespace butl;
+using namespace bbot;
+using namespace brep::cli;
+using namespace odb::core;
+
+// While currently the user-defined copy constructor is not required (we don't
+// need to deep copy nullptr's), it is a good idea to keep the placeholder
+// ready for less trivial cases.
+//
+brep::build_result::
+build_result (const build_result& r)
+ : database_module (r),
+ options_ (r.initialized_ ? r.options_ : nullptr)
+{
+}
+
+void brep::build_result::
+init (scanner& s)
+{
+ MODULE_DIAG;
+
+ options_ = make_shared<options::build_result> (
+ s, unknown_mode::fail, unknown_mode::fail);
+
+ database_module::init (static_cast<options::package_db> (*options_),
+ options_->package_db_retry ());
+
+ if (options_->build_config_specified ())
+ database_module::init (static_cast<options::build> (*options_),
+ static_cast<options::build_db> (*options_),
+ options_->build_db_retry ());
+
+ if (options_->root ().empty ())
+ options_->root (dir_path ("/"));
+}
+
+bool brep::build_result::
+handle (request& rq, response&)
+{
+ using brep::version; // Not to confuse with module::version.
+
+ MODULE_DIAG;
+
+ if (build_db_ == nullptr)
+ throw invalid_request (501, "not implemented");
+
+ // Make sure no parameters passed.
+ //
+ try
+ {
+ name_value_scanner s (rq.parameters ());
+ params::build_result (s, unknown_mode::fail, unknown_mode::fail);
+ }
+ catch (const cli::exception& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ result_request_manifest rqm;
+
+ try
+ {
+ size_t limit (options_->build_result_request_max_size ());
+ manifest_parser p (rq.content (limit, limit), "result_request_manifest");
+ rqm = result_request_manifest (p);
+ }
+ catch (const manifest_parsing& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ // Parse the task response session to obtain the build configuration name,
+ // and to make sure the session matches the result manifest's package name
+ // and version.
+ //
+ build_id id;
+
+ try
+ {
+ const string& s (rqm.session);
+
+ size_t p (s.find ('/')); // End of package name.
+
+ if (p == 0)
+ throw invalid_argument ("empty package name");
+
+ if (p == string::npos)
+ throw invalid_argument ("no package version");
+
+ string& name (rqm.result.name);
+ if (name.compare (0, name.size (), s, 0, p) != 0)
+ throw invalid_argument ("package name mismatch");
+
+ size_t b (p + 1); // Start of version.
+ p = s.find ('/', b); // End of version.
+
+ if (p == string::npos)
+ throw invalid_argument ("no configuration name");
+
+ version version;
+
+ // Intercept exception handling to add the parsing error attribution.
+ //
+ try
+ {
+ version = brep::version (string (s, b, p - b));
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_argument (
+ string ("invalid package version: ") + e.what ());
+ }
+
+ if (version != rqm.result.version)
+ throw invalid_argument ("package version mismatch");
+
+ id = build_id (package_id (move (name), version), string (s, p + 1));
+
+ if (id.configuration.empty ())
+ throw invalid_argument ("empty configuration name");
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_request (400, string ("invalid session: ") + e.what ());
+ }
+
+ // If the session expired (no such configuration, package, etc), then we log
+ // this case with the warning severity and respond with the 200 HTTP code as
+ // if the session is valid. The thinking is that this is a problem with the
+ // controller's setup (expires too fast), not with the agent's.
+ //
+ auto warn_expired = [&rqm, &warn] (const string& d)
+ {
+ warn << "session '" << rqm.session << "' expired: " << d;
+ };
+
+ // Make sure the build configuration still exists.
+ //
+ auto i (
+ find_if (
+ build_conf_->begin (), build_conf_->end (),
+ [&id] (const build_config& c) {return c.name == id.configuration;}));
+
+ if (i == build_conf_->end ())
+ {
+ warn_expired ("no build configuration");
+ return true;
+ }
+
+ // Load the built package (if present).
+ //
+ shared_ptr<package> p;
+ {
+ transaction t (package_db_->begin ());
+ p = package_db_->find<package> (id.package);
+ t.commit ();
+ }
+
+ if (p == nullptr)
+ {
+ warn_expired ("no package");
+ return true;
+ }
+
+ // Load and update the package build configuration (if present).
+ //
+ shared_ptr<build> b;
+ {
+ transaction t (build_db_->begin ());
+ b = build_db_->find<build> (id);
+
+ if (b == nullptr)
+ warn_expired ("no package configuration");
+ else if (b->state != build_state::testing)
+ {
+ warn_expired ("package configuration state is " + to_string (b->state));
+ b = nullptr;
+ }
+ else
+ {
+ b->state = build_state::tested;
+ b->timestamp = timestamp::clock::now ();
+
+ assert (!b->status);
+ b->status = rqm.result.status;
+
+ // Need to manually load the lazy-load section prior to changing its
+ // members and updating the object's persistent state.
+ //
+ // @@ TODO: ODB now allows marking a section as loaded so can optimize
+ // this.
+ //
+ build_db_->load (*b, b->results_section);
+ b->results = move (rqm.result.results);
+
+ build_db_->update (b);
+ }
+
+ t.commit ();
+ }
+
+ if (b == nullptr)
+ return true; // Warning is already logged.
+
+ // Send email to the package owner.
+ //
+ try
+ {
+ string subj (b->package_name + '/' + b->package_version.string () + ' ' +
+ b->configuration + " build: " + to_string (*b->status));
+
+ // If the package email address is not specified, then it is assumed to be
+ // the same as the project email address.
+ //
+ const string& to (p->package_email
+ ? *p->package_email
+ : p->email);
+
+ auto print_args = [&trace, this] (const char* args[], size_t n)
+ {
+ l2 ([&]{trace << process_args {args, n};});
+ };
+
+ // Redirect the diagnostics to webserver error log.
+ //
+ // Note: if using this somewhere else, then need to factor out all this
+ // exit status handling code.
+ //
+ sendmail sm (print_args,
+ 2,
+ options_->email (),
+ subj,
+ {to});
+
+ if (b->results.empty ())
+ sm.out << "No operations results available." << endl;
+ else
+ {
+ for (const auto& r: b->results)
+ sm.out << r.operation << ": " << r.status << ", "
+ << options_->host () << options_->root ().representation ()
+ << b->package_name << '/' << b->package_version << "/log/"
+ << b->configuration << '/' << r.operation << endl;
+ }
+
+ sm.out.close ();
+
+ if (!sm.wait ())
+ {
+ diag_record dr (error);
+ dr << "sendmail ";
+
+ assert (sm.exit);
+ const process_exit& e (*sm.exit);
+
+ if (e.normal ())
+ dr << "exited with code " << static_cast<uint16_t> (e.code ());
+ else
+ {
+ dr << "terminated abnormally: " << e.description ();
+
+ if (e.core ())
+ dr << " (core dumped)";
+ }
+ }
+ }
+ // Handle process_error and io_error (both derive from system_error).
+ //
+ catch (const system_error& e)
+ {
+ error << "sendmail error: " << e;
+ }
+
+ return true;
+}
diff --git a/mod/mod-build-task b/mod/mod-build-task
new file mode 100644
index 0000000..051357e
--- /dev/null
+++ b/mod/mod-build-task
@@ -0,0 +1,42 @@
+// file : mod/mod-build-task -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef MOD_MOD_BUILD_TASK
+#define MOD_MOD_BUILD_TASK
+
+#include <brep/types>
+#include <brep/utility>
+
+#include <mod/options>
+#include <mod/database-module>
+
+namespace brep
+{
+ class build_task: public database_module
+ {
+ public:
+ build_task () = default;
+
+ // Create a shallow copy (handling instance) if initialized and a deep
+ // copy (context exemplar) otherwise.
+ //
+ explicit
+ build_task (const build_task&);
+
+ virtual bool
+ handle (request&, response&);
+
+ virtual const cli::options&
+ cli_options () const {return options::build_task::description ();}
+
+ private:
+ virtual void
+ init (cli::scanner&);
+
+ private:
+ shared_ptr<options::build_task> options_;
+ };
+}
+
+#endif // MOD_MOD_BUILD_TASK
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
new file mode 100644
index 0000000..1cb5386
--- /dev/null
+++ b/mod/mod-build-task.cxx
@@ -0,0 +1,389 @@
+// file : mod/mod-build-task.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/mod-build-task>
+
+#include <map>
+#include <chrono>
+
+#include <butl/utility> // compare_c_string
+#include <butl/filesystem> // path_match()
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+#include <bbot/manifest>
+#include <bbot/build-config>
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
+#include <web/module>
+
+#include <brep/build>
+#include <brep/build-odb>
+#include <brep/package>
+#include <brep/package-odb>
+
+#include <mod/options>
+
+using namespace std;
+using namespace butl;
+using namespace bbot;
+using namespace brep::cli;
+using namespace odb::core;
+
+// While currently the user-defined copy constructor is not required (we don't
+// need to deep copy nullptr's), it is a good idea to keep the placeholder
+// ready for less trivial cases.
+//
+brep::build_task::
+build_task (const build_task& r)
+ : database_module (r),
+ options_ (r.initialized_ ? r.options_ : nullptr)
+{
+}
+
+void brep::build_task::
+init (scanner& s)
+{
+ MODULE_DIAG;
+
+ options_ = make_shared<options::build_task> (
+ s, unknown_mode::fail, unknown_mode::fail);
+
+ database_module::init (static_cast<options::package_db> (*options_),
+ options_->package_db_retry ());
+
+ if (options_->build_config_specified ())
+ database_module::init (static_cast<options::build> (*options_),
+ static_cast<options::build_db> (*options_),
+ options_->build_db_retry ());
+
+ if (options_->root ().empty ())
+ options_->root (dir_path ("/"));
+}
+
+bool brep::build_task::
+handle (request& rq, response& rs)
+{
+ MODULE_DIAG;
+
+ if (build_db_ == nullptr)
+ throw invalid_request (501, "not implemented");
+
+ // Make sure no parameters passed.
+ //
+ try
+ {
+ name_value_scanner s (rq.parameters ());
+ params::build_task (s, unknown_mode::fail, unknown_mode::fail);
+ }
+ catch (const cli::exception& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ task_request_manifest tqm;
+
+ try
+ {
+ size_t limit (options_->build_task_request_max_size ());
+ manifest_parser p (rq.content (limit, limit), "task_request_manifest");
+ tqm = task_request_manifest (p);
+ }
+ catch (const manifest_parsing& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ task_response_manifest tsm;
+
+ // Map build configurations to machines that are capable of building them.
+ // The first matching machine is selected for each configuration. Also
+ // create the configuration name list for use in database queries.
+ //
+ struct config_machine
+ {
+ const build_config* config;
+ const machine_header_manifest* machine;
+ };
+
+ using config_machines = map<const char*, config_machine, compare_c_string>;
+
+ cstrings cfg_names;
+ config_machines cfg_machines;
+
+ for (const auto& c: *build_conf_)
+ {
+ for (auto& m: tqm.machines)
+ {
+ if (path_match (c.machine_pattern, m.name) &&
+ cfg_machines.insert (
+ make_pair (c.name.c_str (), config_machine ({&c, &m}))).second)
+ cfg_names.push_back (c.name.c_str ());
+ }
+ }
+
+ // Go through packages until we find one that has no build configuration
+ // present in the database, or has the untested one, or in the testing state
+ // but expired, or the one, which build failed abnormally and expired. If
+ // such a package configuration is found then put it into the testing state,
+ // set the current timestamp and respond with the task for building this
+ // package configuration.
+ //
+ if (!cfg_machines.empty ())
+ {
+ // Calculate the expiration time for package configurations being in the
+ // testing state or those, which build failed abnormally.
+ //
+ timestamp expiration (timestamp::clock::now () -
+ chrono::seconds (options_->build_result_timeout ()));
+
+ uint64_t expiration_ns (
+ std::chrono::duration_cast<std::chrono::nanoseconds> (
+ expiration.time_since_epoch ()).count ());
+
+ // Prepare the package version prepared query.
+ //
+ // Note that the number of packages can be large and so, in order not to
+ // hold locks for too long, we will restrict the number of packages being
+ // queried in a single transaction. To achieve this we will iterate through
+ // packages using the OFFSET/LIMIT pair and sort the query result.
+ //
+ // Note that this approach can result in missing some packages or
+ // iterating multiple times over some of them. However there is nothing
+ // harmful in that: updates are infrequent and missed packages will be
+ // picked up on the next request.
+ //
+ using pkg_query = query<package_version>;
+ using prep_pkg_query = prepared_query<package_version>;
+
+ size_t offset (0); // See the package version query.
+
+ // Skip external and stub packages.
+ //
+ pkg_query pq ((pkg_query::internal_repository.is_not_null () &&
+ compare_version_ne (pkg_query::id.version,
+ wildcard_version,
+ true)) +
+ "ORDER BY" +
+ pkg_query::id.name + "," +
+ pkg_query::id.version.epoch + "," +
+ pkg_query::id.version.canonical_upstream + "," +
+ pkg_query::id.version.canonical_release + "," +
+ pkg_query::id.version.revision +
+ "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50");
+
+ connection_ptr pkg_conn (package_db_->connection ());
+
+ prep_pkg_query pkg_prep_query (
+ pkg_conn->prepare_query<package_version> (
+ "mod-build-task-package-version-query", pq));
+
+ // Prepare the build prepared query.
+ //
+ // Note that we can not query the database for configurations that a
+ // package was not built with, as the database contains only those package
+ // configurations that have already been acted upon (initially empty).
+ //
+ // This is why we query the database for package configurations that
+ // should not be built (in the tested state with the build terminated
+ // normally or not expired, or in the testing state and not expired).
+ // Having such a list we will select the first build configuration that is
+ // not in the list (if available) for the response.
+ //
+ using bld_query = query<build>;
+ using prep_bld_query = prepared_query<build>;
+
+ package_id id; // See the build query.
+
+ const auto& qv (bld_query::id.package.version);
+
+ bld_query bq (
+ bld_query::id.package.name == bld_query::_ref (id.name) &&
+
+ qv.epoch == bld_query::_ref (id.version.epoch) &&
+ qv.canonical_upstream ==
+ bld_query::_ref (id.version.canonical_upstream) &&
+ qv.canonical_release == bld_query::_ref (id.version.canonical_release) &&
+ qv.revision == bld_query::_ref (id.version.revision) &&
+
+ bld_query::id.configuration.in_range (cfg_names.begin (),
+ cfg_names.end ()) &&
+
+ ((bld_query::state == "tested" &&
+ ((bld_query::status != "abort" && bld_query::status != "abnormal") ||
+ bld_query::timestamp > expiration_ns)) ||
+
+ (bld_query::state == "testing" &&
+ bld_query::timestamp > expiration_ns)));
+
+ connection_ptr bld_conn (build_db_->connection ());
+
+ prep_bld_query bld_prep_query (
+ bld_conn->prepare_query<build> (
+ "mod-build-task-package-build-query", bq));
+
+ while (tsm.session.empty ())
+ {
+ // Start the package database transaction.
+ //
+ transaction pt (pkg_conn->begin ());
+
+ // Query package versions.
+ //
+ auto package_versions (pkg_prep_query.execute ());
+
+ // Bail out if there is nothing left.
+ //
+ if (package_versions.empty ())
+ {
+ pt.commit ();
+ break;
+ }
+
+ offset += package_versions.size ();
+
+ // Start the build database transaction.
+ //
+ {
+ transaction bt (bld_conn->begin (), false);
+ transaction::current (bt);
+
+ // Iterate over packages until we find one that needs building.
+ //
+ for (auto& pv: package_versions)
+ {
+ id = move (pv.id);
+
+ // Iterate through the package configurations and erase those that
+ // don't need building from the build configuration map. All those
+ // configurations that remained can be built. We will take the first
+ // one, if present.
+ //
+ config_machines configs (cfg_machines); // Make a copy for this pkg.
+
+ for (const auto& pc: bld_prep_query.execute ())
+ {
+ auto i (configs.find (pc.id.configuration.c_str ()));
+
+ // Outdated configurations are already excluded with the database
+ // query.
+ //
+ assert (i != configs.end ());
+ configs.erase (i);
+ }
+
+ if (!configs.empty ())
+ {
+ config_machine& cm (configs.begin ()->second);
+ const build_config& cfg (*cm.config);
+
+ build_id bid (move (id), cfg.name);
+ shared_ptr<build> b (build_db_->find<build> (bid));
+
+ // If build configuration doesn't exist then create the new one
+ // and persist. Otherwise put it into the testing state, refresh
+ // the timestamp and update.
+ //
+ if (b == nullptr)
+ {
+ b = make_shared<build> (move (bid.package.name),
+ move (pv.version),
+ move (bid.configuration));
+
+ build_db_->persist (b);
+ }
+ else
+ {
+ // If the package configuration is in the tested state, then we
+ // need to cleanup the status and results prior to the update.
+ // Otherwise the status is already absent and there are no
+ // results.
+ //
+ // Load the section to make sure results are updated for the
+ // tested state, otherwise assert there are no results.
+ //
+ build_db_->load (*b, b->results_section);
+
+ if (b->state == build_state::tested)
+ {
+ assert (b->status);
+ b->status = nullopt;
+
+ b->results.clear ();
+ }
+ else
+ {
+ assert (!b->status && b->results.empty ());
+ }
+
+ b->state = build_state::testing;
+ b->timestamp = timestamp::clock::now ();
+
+ build_db_->update (b);
+ }
+
+ // Finally, prepare the task response manifest.
+ //
+ tsm.session = b->package_name + '/' +
+ b->package_version.string () + '/' + b->configuration;
+
+ // @@ We don't support challenge at the moment, so leave it absent.
+ //
+
+ tsm.result_url = options_->host () + options_->root ().string () +
+ "?build-result";
+
+ // Switch to the package database transaction to load the package.
+ //
+ transaction::current (pt);
+
+ shared_ptr<package> p (package_db_->load<package> (b->id.package));
+ shared_ptr<repository> r (p->internal_repository.load ());
+
+ // Switch back to the build database transaction.
+ //
+ transaction::current (bt);
+
+ strings fp;
+ if (r->certificate)
+ fp.emplace_back (move (r->certificate->fingerprint));
+
+ tsm.task = task_manifest (
+ move (b->package_name),
+ move (b->package_version),
+ move (r->location),
+ move (fp),
+ move (cm.machine->name),
+ cfg.target,
+ cfg.vars);
+ }
+
+ // If task response manifest is filled, then can bail out from the
+ // package loop, commit transactions and respond.
+ //
+ if (!tsm.session.empty ())
+ break;
+ }
+
+ bt.commit (); // Commit the build database transaction.
+ }
+
+ transaction::current (pt); // Switch to the package database transaction.
+ pt.commit ();
+ }
+ }
+
+ // @@ Probably it would be a good idea to also send some cache control
+ // headers to avoid caching by HTTP proxies. That would require extension
+ // of the web::response interface.
+ //
+
+ manifest_serializer s (rs.content (200, "text/manifest;charset=utf-8"),
+ "task_response_manifest");
+ tsm.serialize (s);
+
+ return true;
+}
diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx
index 95bf087..fd3fd6d 100644
--- a/mod/mod-package-details.cxx
+++ b/mod/mod-package-details.cxx
@@ -42,7 +42,7 @@ init (scanner& s)
options_ = make_shared<options::package_details> (
s, unknown_mode::fail, unknown_mode::fail);
- database_module::init (*options_);
+ database_module::init (*options_, options_->package_db_retry ());
if (options_->root ().empty ())
options_->root (dir_path ("/"));
@@ -148,17 +148,17 @@ handle (request& rq, response& rs)
<< ~DIV;
session sn;
- transaction t (db_->begin ());
+ transaction t (package_db_->begin ());
shared_ptr<package> pkg;
{
latest_package lp;
- if (!db_->query_one<latest_package> (
+ if (!package_db_->query_one<latest_package> (
query<latest_package>(
"(" + query<latest_package>::_val (name) + ")"), lp))
throw invalid_request (404, "Package '" + name + "' not found");
- pkg = db_->load<package> (lp.id);
+ pkg = package_db_->load<package> (lp.id);
}
const auto& licenses (pkg->license_alternatives);
@@ -187,7 +187,7 @@ handle (request& rq, response& rs)
}
auto pkg_count (
- db_->query_value<package_count> (
+ package_db_->query_value<package_count> (
search_params<package_count> (name, squery)));
s << FORM_SEARCH (squery)
@@ -197,7 +197,7 @@ handle (request& rq, response& rs)
//
s << DIV;
for (const auto& pr:
- db_->query<package_search_rank> (
+ package_db_->query<package_search_rank> (
search_params<package_search_rank> (name, squery) +
"ORDER BY rank DESC, version_epoch DESC, "
"version_canonical_upstream DESC, version_canonical_release DESC, "
@@ -205,7 +205,7 @@ handle (request& rq, response& rs)
"OFFSET" + to_string (page * res_page) +
"LIMIT" + to_string (res_page)))
{
- shared_ptr<package> p (db_->load<package> (pr.id));
+ shared_ptr<package> p (package_db_->load<package> (pr.id));
s << TABLE(CLASS="proplist version")
<< TBODY
diff --git a/mod/mod-package-search.cxx b/mod/mod-package-search.cxx
index 02b10d6..4988d42 100644
--- a/mod/mod-package-search.cxx
+++ b/mod/mod-package-search.cxx
@@ -44,14 +44,14 @@ init (scanner& s)
options_ = make_shared<options::package_search> (
s, unknown_mode::fail, unknown_mode::fail);
- database_module::init (*options_);
+ database_module::init (*options_, options_->package_db_retry ());
if (options_->root ().empty ())
options_->root (dir_path ("/"));
- // Check that the database schema matches the current one. It's enough to
- // perform the check in just a single module implementation (and we don't
- // do in the dispatcher because it doesn't use the database).
+ // Check that the database 'package' schema matches the current one. It's
+ // enough to perform the check in just a single module implementation (and we
+ // don't do in the dispatcher because it doesn't use the database).
//
// Note that the failure can be reported by each web server worker process.
// While it could be tempting to move the check to the
@@ -59,8 +59,10 @@ init (scanner& s)
// be called by a different process (usually the web server root one) not
// having the proper permissions to access the database.
//
- if (schema_catalog::current_version (*db_) != db_->schema_version ())
- fail << "database schema differs from the current one (module "
+ const string ds ("package");
+ if (schema_catalog::current_version (*package_db_, ds) !=
+ package_db_->schema_version (ds))
+ fail << "database 'package' schema differs from the current one (module "
<< BREP_VERSION_STR << ")";
}
@@ -133,10 +135,10 @@ handle (request& rq, response& rs)
<< DIV(ID="content");
session sn;
- transaction t (db_->begin ());
+ transaction t (package_db_->begin ());
auto pkg_count (
- db_->query_value<latest_package_count> (
+ package_db_->query_value<latest_package_count> (
search_param<latest_package_count> (squery)));
s << FORM_SEARCH (squery)
@@ -146,13 +148,13 @@ handle (request& rq, response& rs)
//
s << DIV;
for (const auto& pr:
- db_->query<latest_package_search_rank> (
+ package_db_->query<latest_package_search_rank> (
search_param<latest_package_search_rank> (squery) +
"ORDER BY rank DESC, name" +
"OFFSET" + to_string (page * res_page) +
"LIMIT" + to_string (res_page)))
{
- shared_ptr<package> p (db_->load<package> (pr.id));
+ shared_ptr<package> p (package_db_->load<package> (pr.id));
s << TABLE(CLASS="proplist package")
<< TBODY
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index 89acf89..b0e6ead 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -43,7 +43,7 @@ init (scanner& s)
options_ = make_shared<options::package_version_details> (
s, unknown_mode::fail, unknown_mode::fail);
- database_module::init (*options_);
+ database_module::init (*options_, options_->package_db_retry ());
if (options_->root ().empty ())
options_->root (dir_path ("/"));
@@ -130,11 +130,11 @@ handle (request& rq, response& rs)
shared_ptr<package> pkg;
session sn;
- transaction t (db_->begin ());
+ transaction t (package_db_->begin ());
try
{
- pkg = db_->load<package> (package_id (name, ver));
+ pkg = package_db_->load<package> (package_id (name, ver));
// If the requested package turned up to be an "external" one just
// respond that no "internal" package is present.
diff --git a/mod/mod-repository-details.cxx b/mod/mod-repository-details.cxx
index ab87cb2..3a73955 100644
--- a/mod/mod-repository-details.cxx
+++ b/mod/mod-repository-details.cxx
@@ -49,7 +49,7 @@ init (scanner& s)
options_ = make_shared<options::repository_details> (
s, unknown_mode::fail, unknown_mode::fail);
- database_module::init (*options_);
+ database_module::init (*options_, options_->package_db_retry ());
if (options_->root ().empty ())
options_->root (dir_path ("/"));
@@ -90,12 +90,12 @@ handle (request& rq, response& rs)
<< DIV_HEADER (root, options_->logo (), options_->menu ())
<< DIV(ID="content");
- transaction t (db_->begin ());
+ transaction t (package_db_->begin ());
using query = query<repository>;
for (const auto& r:
- db_->query<repository> (
+ package_db_->query<repository> (
query::internal + "ORDER BY" + query::priority))
{
//@@ Feels like a lot of trouble (e.g., id_attribute()) for very
diff --git a/mod/mod-repository-root b/mod/mod-repository-root
index 57db93a..6a40e10 100644
--- a/mod/mod-repository-root
+++ b/mod/mod-repository-root
@@ -17,6 +17,9 @@ namespace brep
class package_details;
class package_version_details;
class repository_details;
+ class build_task;
+ class build_result;
+ class build_log;
class repository_root: public module
{
@@ -55,6 +58,9 @@ namespace brep
shared_ptr<package_details> package_details_;
shared_ptr<package_version_details> package_version_details_;
shared_ptr<repository_details> repository_details_;
+ shared_ptr<build_task> build_task_;
+ shared_ptr<build_result> build_result_;
+ shared_ptr<build_log> build_log_;
shared_ptr<options::repository_root> options_;
// Sub-module the request is dispatched to. Initially is NULL. It is set
diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx
index 1bc10cb..e156308 100644
--- a/mod/mod-repository-root.cxx
+++ b/mod/mod-repository-root.cxx
@@ -12,6 +12,9 @@
#include <mod/module>
#include <mod/options>
+#include <mod/mod-build-log>
+#include <mod/mod-build-task>
+#include <mod/mod-build-result>
#include <mod/mod-package-search>
#include <mod/mod-package-details>
#include <mod/mod-repository-details>
@@ -55,7 +58,10 @@ namespace brep
: package_search_ (make_shared<package_search> ()),
package_details_ (make_shared<package_details> ()),
package_version_details_ (make_shared<package_version_details> ()),
- repository_details_ (make_shared<repository_details> ())
+ repository_details_ (make_shared<repository_details> ()),
+ build_task_ (make_shared<build_task> ()),
+ build_result_ (make_shared<build_result> ()),
+ build_log_ (make_shared<build_log> ())
{
}
@@ -83,6 +89,18 @@ namespace brep
r.initialized_
? r.repository_details_
: make_shared<repository_details> (*r.repository_details_)),
+ build_task_ (
+ r.initialized_
+ ? r.build_task_
+ : make_shared<build_task> (*r.build_task_)),
+ build_result_ (
+ r.initialized_
+ ? r.build_result_
+ : make_shared<build_result> (*r.build_result_)),
+ build_log_ (
+ r.initialized_
+ ? r.build_log_
+ : make_shared<build_log> (*r.build_log_)),
options_ (
r.initialized_
? r.options_
@@ -101,6 +119,9 @@ namespace brep
append (r, package_details_->options ());
append (r, package_version_details_->options ());
append (r, repository_details_->options ());
+ append (r, build_task_->options ());
+ append (r, build_result_->options ());
+ append (r, build_log_->options ());
return r;
}
@@ -109,17 +130,38 @@ namespace brep
void repository_root::
init (const name_values& v)
{
- auto sub_init ([this, &v](module& m)
+ auto sub_init = [this, &v] (module& m, const char* name)
+ {
+ // Initialize sub-module. Intercept exception handling to add sub-module
+ // attribution.
+ //
+ try
{
m.init (filter (v, m.options ()), *log_);
- });
+ }
+ catch (const std::exception& e)
+ {
+ // Any exception thrown by this function terminates the web server. All
+ // exception types inherited from std::exception are handled by the web
+ // server as std::exception. The only sensible way to handle them is to
+ // log the error prior terminating. By that reason it is valid to
+ // reduce all these types to a single one.
+ //
+ ostringstream os;
+ os << name << ": " << e;
+ throw runtime_error (os.str ());
+ }
+ };
// Initialize sub-modules.
//
- sub_init (*package_search_);
- sub_init (*package_details_);
- sub_init (*package_version_details_);
- sub_init (*repository_details_);
+ sub_init (*package_search_, "package_search");
+ sub_init (*package_details_, "package_details");
+ sub_init (*package_version_details_, "package_version_details");
+ sub_init (*repository_details_, "repository_details");
+ sub_init (*build_task_, "build_task");
+ sub_init (*build_result_, "build_result");
+ sub_init (*build_log_, "build_log");
// Parse own configuration options.
//
@@ -155,8 +197,7 @@ namespace brep
// Delegate the request handling to the selected sub-module. Intercept
// exception handling to add sub-module attribution.
//
- auto handle =
- [&rs, this] (request& rq, const char* name) -> bool
+ auto handle = [&rs, this] (request& rq, const char* name) -> bool
{
try
{
@@ -194,15 +235,24 @@ namespace brep
//
if (lpath.empty ())
{
- // Dispatch request handling to the repository_details or the
- // package_search module depending on the function name passed as a
- // first HTTP request parameter. The parameter should have no value
- // specified. Example: cppget.org/?about
+ // Dispatch request handling to the repository_details, the build_task,
+ // the build_result or the package_search module depending on the
+ // function name passed as a first HTTP request parameter. The parameter
+ // should have no value specified. Example: cppget.org/?about
//
const name_values& params (rq.parameters ());
if (!params.empty () && !params.front ().value)
{
- if (params.front ().name == "about")
+ // Cleanup not to confuse the selected module with the unknown
+ // parameter.
+ //
+ name_values p (params);
+ p.erase (p.begin ());
+
+ request_proxy rp (rq, p);
+ const string& fn (params.front ().name);
+
+ if (fn == "about")
{
if (handler_ == nullptr)
handler_.reset (new repository_details (*repository_details_));
@@ -236,8 +286,9 @@ namespace brep
}
else
{
- // Dispatch request handling to the package_details or the
- // package_version_details module depending on the HTTP request URL path.
+ // Dispatch request handling to the package_details, the
+ // package_version_details or the build_log module depending on the HTTP
+ // request URL path.
//
auto i (lpath.begin ());
assert (i != lpath.end ());
@@ -279,6 +330,13 @@ namespace brep
return handle (rq, "package_version_details");
}
+ else if (*i == "log")
+ {
+ if (handler_ == nullptr)
+ handler_.reset (new build_log (*build_log_));
+
+ return handle (rq, "build_log");
+ }
}
}
@@ -295,6 +353,7 @@ namespace brep
info << "module " << BREP_VERSION_STR
<< ", libbrep " << LIBBREP_VERSION_STR
+ << ", libbbot " << LIBBBOT_VERSION_STR
<< ", libbpkg " << LIBBPKG_VERSION_STR
<< ", libbutl " << LIBBUTL_VERSION_STR;
}
diff --git a/mod/options-types b/mod/options-types
index 6c2e42f..d9eaf9e 100644
--- a/mod/options-types
+++ b/mod/options-types
@@ -26,7 +26,6 @@ namespace brep
page_menu () = default;
page_menu (string b, string l): label (move (b)), link (move (l)) {}
};
-
}
#endif // MOD_OPTIONS_TYPES
diff --git a/mod/options.cli b/mod/options.cli
index fb86fcd..4dbcd89 100644
--- a/mod/options.cli
+++ b/mod/options.cli
@@ -18,13 +18,30 @@ namespace brep
//
class module
{
+ string email
+ {
+ "<email>",
+ "Repository email. This email is used for the \cb{From:} header in
+ emails send by \cb{brep} (for example, build failure notifications)."
+ }
+
+ string host
+ {
+ "<host>",
+ "Repository host. It specifies the scheme and the host address (but
+ not the root path; see \cb{root} below) that will be used whenever
+ \cb{brep} needs to construct an absolute URL to one of its locations
+ (for example, a link to a build log that is being send via email)."
+ }
+
dir_path root = "/"
{
"<path>"
"Repository root. That is, this is the part of the URL between the
host name and the start of the repository. For example, root value
- '\cb{/pkg}' means the repository URL is http://example.org/pkg/.
- Specify '\cb{/}' to use the web server root (http://example.org/)."
+ '\cb{/pkg}' means the repository URL is \cb{http://example.org/pkg/}.
+ Specify '\cb{/}' to use the web server root
+ (\cb{http://example.org/})."
}
uint16_t verbosity = 0
@@ -35,56 +52,124 @@ namespace brep
}
};
- class db
+ class package_db
{
- string db-user
+ string package-db-user
{
"<user>",
- "Database user name. If not specified, then operating system (login)
- name is used."
+ "Package database user name. If not specified, then operating system
+ (login) name is used."
}
- string db-password
+ string package-db-password
{
"<pass>",
- "Database password. If not specified, then login without password is
- expected to work."
+ "Package database password. If not specified, then login without
+ password is expected to work."
}
- string db-name = "brep"
+ string package-db-name = "brep_package"
{
"<name>",
- "Database name. If not specified, then '\cb{brep}' is used by
- default."
+ "Package database name. If not specified, then \cb{brep_package} is
+ used by default."
}
- string db-host
+ string package-db-host
{
"<host>",
- "Database host name, address, or socket. If not specified, then
+ "Package database host name, address, or socket. If not specified, then
connect to \cb{localhost} using the operating system-default
mechanism (Unix-domain socket, etc)."
}
- uint16_t db-port = 0
+ uint16_t package-db-port = 0
{
"<port>",
- "Database port number. If not specified, the default port is used."
+ "Package database port number. If not specified, the default port is
+ used."
}
- size_t db-max-connections = 5
+ size_t package-db-max-connections = 5
{
"<num>",
- "The maximum number of concurrent database connections per web server
- process. If 0, then no limitation is applied. The default is 5."
+ "The maximum number of concurrent package database connections per web
+ server process. If 0, then no limitation is applied. The default is
+ 5."
}
- size_t db-retry = 10
+ size_t package-db-retry = 10
{
"<num>",
- "The maximum number of times to retry database transactions in the
- face of recoverable failures (deadlock, loss of connection, etc). The
- default is 10."
+ "The maximum number of times to retry package database transactions in
+ the face of recoverable failures (deadlock, loss of connection, etc).
+ The default is 10."
+ }
+ };
+
+ class build
+ {
+ path build-config
+ {
+ "<buildtab>",
+ "Build configuration file. If not specified, then the package building
+ functionality will be disabled. If specified, then the build database
+ must be configured (see \cb{build-db-*})."
+ }
+ };
+
+ class build_db
+ {
+ string build-db-user
+ {
+ "<user>",
+ "Build database user name. If not specified, then operating system
+ (login) name is used."
+ }
+
+ string build-db-password
+ {
+ "<pass>",
+ "Build database password. If not specified, then login without
+ password is expected to work."
+ }
+
+ string build-db-name = "brep_build"
+ {
+ "<name>",
+ "Build database name. If not specified, then \cb{brep_build} is used
+ by default."
+ }
+
+ string build-db-host
+ {
+ "<host>",
+ "Build database host name, address, or socket. If not specified, then
+ connect to \cb{localhost} using the operating system-default
+ mechanism (Unix-domain socket, etc)."
+ }
+
+ uint16_t build-db-port = 0
+ {
+ "<port>",
+ "Build database port number. If not specified, the default port is
+ used."
+ }
+
+ size_t build-db-max-connections = 5
+ {
+ "<num>",
+ "The maximum number of concurrent build database connections per web
+ server process. If 0, then no limitation is applied. The default is
+ 5."
+ }
+
+ size_t build-db-retry = 10
+ {
+ "<num>",
+ "The maximum number of times to retry build database transactions in
+ the face of recoverable failures (deadlock, loss of connection, etc).
+ The default is 10."
}
};
@@ -141,7 +226,7 @@ namespace brep
// Module options.
//
- class package_search: search, db, page, module
+ class package_search: search, package_db, page, module
{
string search-title = "Packages"
{
@@ -151,21 +236,56 @@ namespace brep
}
};
- class package_details: package, search, db, page, module
+ class package_details: package, search, package_db, page, module
{
};
- class package_version_details: package, db, page, module
+ class package_version_details: package, package_db, page, module
{
};
- class repository_details: db, page, module
+ class repository_details: package_db, page, module
{
};
class repository_root: module
{
};
+
+ class build_task: build, package_db, build_db, module
+ {
+ size_t build-task-request-max-size = 102400
+ {
+ "<bytes>",
+ "The maximum size of the build task request manifest accepted. Note
+ that the HTTP POST request body is cached to retry database
+ transactions in the face of recoverable failures (deadlock, loss of
+ connection, etc). The default is 100K."
+ }
+
+ size_t build-result-timeout = 10800
+ {
+ "<minutes>",
+ "Time to wait before considering the expected task result lost. Must be
+ specified in seconds. The default is 3 hours."
+ }
+ };
+
+ class build_result: build, package_db, build_db, module
+ {
+ size_t build-result-request-max-size = 10240000
+ {
+ "<bytes>",
+ "The maximum size of the build result manifest accepted. Note that the
+ HTTP POST request body is cached to retry database transactions in the
+ face of recoverable failures (deadlock, loss of connection, etc). The
+ default is 10M."
+ }
+ };
+
+ class build_log: build, package_db, build_db, module
+ {
+ };
}
// Web module HTTP request parameters.
@@ -213,5 +333,23 @@ namespace brep
// No parameters so far.
//
};
+
+ class build_task
+ {
+ // No parameters so far.
+ //
+ };
+
+ class build_result
+ {
+ // No parameters so far.
+ //
+ };
+
+ class build_log
+ {
+ // No parameters so far.
+ //
+ };
}
}
diff --git a/mod/types-parsers b/mod/types-parsers
index 1082292..eb206e4 100644
--- a/mod/types-parsers
+++ b/mod/types-parsers
@@ -25,31 +25,38 @@ namespace brep
struct parser;
template <>
+ struct parser<path>
+ {
+ static void
+ parse (path&, bool&, scanner&);
+ };
+
+ template <>
struct parser<dir_path>
{
static void
- parse (dir_path&, scanner&);
+ parse (dir_path&, bool&, scanner&);
};
template <>
struct parser<page_form>
{
static void
- parse (page_form&, scanner&);
+ parse (page_form&, bool&, scanner&);
};
template <>
struct parser<page_menu>
{
static void
- parse (page_menu&, scanner&);
+ parse (page_menu&, bool&, scanner&);
};
template <>
struct parser<web::xhtml::fragment>
{
static void
- parse (web::xhtml::fragment&, scanner&);
+ parse (web::xhtml::fragment&, bool&, scanner&);
};
}
}
diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx
index 3c51da9..fb293a3 100644
--- a/mod/types-parsers.cxx
+++ b/mod/types-parsers.cxx
@@ -36,17 +36,26 @@ namespace brep
}
}
+ void parser<path>::
+ parse (path& x, bool& xs, scanner& s)
+ {
+ xs = true;
+ parse_path (x, s);
+ }
+
void parser<dir_path>::
- parse (dir_path& x, scanner& s)
+ parse (dir_path& x, bool& xs, scanner& s)
{
+ xs = true;
parse_path (x, s);
}
// Parse page_form.
//
void parser<page_form>::
- parse (page_form& x, scanner& s)
+ parse (page_form& x, bool& xs, scanner& s)
{
+ xs = true;
const char* o (s.next ());
if (!s.more ())
@@ -64,8 +73,9 @@ namespace brep
// Parse page_menu.
//
void parser<page_menu>::
- parse (page_menu& x, scanner& s)
+ parse (page_menu& x, bool& xs, scanner& s)
{
+ xs = true;
const char* o (s.next ());
if (!s.more ())
@@ -92,8 +102,9 @@ namespace brep
// Parse web::xhtml::fragment.
//
void parser<fragment>::
- parse (fragment& x, scanner& s)
+ parse (fragment& x, bool& xs, scanner& s)
{
+ xs = true;
const char* o (s.next ());
if (!s.more ())