aboutsummaryrefslogtreecommitdiff
path: root/mod/mod-build-task.cxx
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2017-04-04 20:53:00 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2017-04-19 22:16:46 +0300
commitdbbc19b77dcf6ea828aabd64d7aa8cab9635aaf5 (patch)
treec0b9b449b7064dff3613628022224e6c18148c3e /mod/mod-build-task.cxx
parentefb9c3e0e6b612d5bfadc7a2b984c14b5439335c (diff)
Implement build task, result and log requests handling
Diffstat (limited to 'mod/mod-build-task.cxx')
-rw-r--r--mod/mod-build-task.cxx389
1 files changed, 389 insertions, 0 deletions
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
new file mode 100644
index 0000000..1cb5386
--- /dev/null
+++ b/mod/mod-build-task.cxx
@@ -0,0 +1,389 @@
+// file : mod/mod-build-task.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <mod/mod-build-task>
+
+#include <map>
+#include <chrono>
+
+#include <butl/utility> // compare_c_string
+#include <butl/filesystem> // path_match()
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+#include <bbot/manifest>
+#include <bbot/build-config>
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
+#include <web/module>
+
+#include <brep/build>
+#include <brep/build-odb>
+#include <brep/package>
+#include <brep/package-odb>
+
+#include <mod/options>
+
+using namespace std;
+using namespace butl;
+using namespace bbot;
+using namespace brep::cli;
+using namespace odb::core;
+
+// While currently the user-defined copy constructor is not required (we don't
+// need to deep copy nullptr's), it is a good idea to keep the placeholder
+// ready for less trivial cases.
+//
+brep::build_task::
+build_task (const build_task& r)
+ : database_module (r),
+ options_ (r.initialized_ ? r.options_ : nullptr)
+{
+}
+
+void brep::build_task::
+init (scanner& s)
+{
+ MODULE_DIAG;
+
+ options_ = make_shared<options::build_task> (
+ s, unknown_mode::fail, unknown_mode::fail);
+
+ database_module::init (static_cast<options::package_db> (*options_),
+ options_->package_db_retry ());
+
+ if (options_->build_config_specified ())
+ database_module::init (static_cast<options::build> (*options_),
+ static_cast<options::build_db> (*options_),
+ options_->build_db_retry ());
+
+ if (options_->root ().empty ())
+ options_->root (dir_path ("/"));
+}
+
+bool brep::build_task::
+handle (request& rq, response& rs)
+{
+ MODULE_DIAG;
+
+ if (build_db_ == nullptr)
+ throw invalid_request (501, "not implemented");
+
+ // Make sure no parameters passed.
+ //
+ try
+ {
+ name_value_scanner s (rq.parameters ());
+ params::build_task (s, unknown_mode::fail, unknown_mode::fail);
+ }
+ catch (const cli::exception& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ task_request_manifest tqm;
+
+ try
+ {
+ size_t limit (options_->build_task_request_max_size ());
+ manifest_parser p (rq.content (limit, limit), "task_request_manifest");
+ tqm = task_request_manifest (p);
+ }
+ catch (const manifest_parsing& e)
+ {
+ throw invalid_request (400, e.what ());
+ }
+
+ task_response_manifest tsm;
+
+ // Map build configurations to machines that are capable of building them.
+ // The first matching machine is selected for each configuration. Also
+ // create the configuration name list for use in database queries.
+ //
+ struct config_machine
+ {
+ const build_config* config;
+ const machine_header_manifest* machine;
+ };
+
+ using config_machines = map<const char*, config_machine, compare_c_string>;
+
+ cstrings cfg_names;
+ config_machines cfg_machines;
+
+ for (const auto& c: *build_conf_)
+ {
+ for (auto& m: tqm.machines)
+ {
+ if (path_match (c.machine_pattern, m.name) &&
+ cfg_machines.insert (
+ make_pair (c.name.c_str (), config_machine ({&c, &m}))).second)
+ cfg_names.push_back (c.name.c_str ());
+ }
+ }
+
+ // Go through packages until we find one that has no build configuration
+ // present in the database, or has the untested one, or in the testing state
+ // but expired, or the one, which build failed abnormally and expired. If
+ // such a package configuration is found then put it into the testing state,
+ // set the current timestamp and respond with the task for building this
+ // package configuration.
+ //
+ if (!cfg_machines.empty ())
+ {
+ // Calculate the expiration time for package configurations being in the
+ // testing state or those, which build failed abnormally.
+ //
+ timestamp expiration (timestamp::clock::now () -
+ chrono::seconds (options_->build_result_timeout ()));
+
+ uint64_t expiration_ns (
+ std::chrono::duration_cast<std::chrono::nanoseconds> (
+ expiration.time_since_epoch ()).count ());
+
+ // Prepare the package version prepared query.
+ //
+ // Note that the number of packages can be large and so, in order not to
+ // hold locks for too long, we will restrict the number of packages being
+ // queried in a single transaction. To achieve this we will iterate through
+ // packages using the OFFSET/LIMIT pair and sort the query result.
+ //
+ // Note that this approach can result in missing some packages or
+ // iterating multiple times over some of them. However there is nothing
+ // harmful in that: updates are infrequent and missed packages will be
+ // picked up on the next request.
+ //
+ using pkg_query = query<package_version>;
+ using prep_pkg_query = prepared_query<package_version>;
+
+ size_t offset (0); // See the package version query.
+
+ // Skip external and stub packages.
+ //
+ pkg_query pq ((pkg_query::internal_repository.is_not_null () &&
+ compare_version_ne (pkg_query::id.version,
+ wildcard_version,
+ true)) +
+ "ORDER BY" +
+ pkg_query::id.name + "," +
+ pkg_query::id.version.epoch + "," +
+ pkg_query::id.version.canonical_upstream + "," +
+ pkg_query::id.version.canonical_release + "," +
+ pkg_query::id.version.revision +
+ "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50");
+
+ connection_ptr pkg_conn (package_db_->connection ());
+
+ prep_pkg_query pkg_prep_query (
+ pkg_conn->prepare_query<package_version> (
+ "mod-build-task-package-version-query", pq));
+
+ // Prepare the build prepared query.
+ //
+ // Note that we can not query the database for configurations that a
+ // package was not built with, as the database contains only those package
+ // configurations that have already been acted upon (initially empty).
+ //
+ // This is why we query the database for package configurations that
+ // should not be built (in the tested state with the build terminated
+ // normally or not expired, or in the testing state and not expired).
+ // Having such a list we will select the first build configuration that is
+ // not in the list (if available) for the response.
+ //
+ using bld_query = query<build>;
+ using prep_bld_query = prepared_query<build>;
+
+ package_id id; // See the build query.
+
+ const auto& qv (bld_query::id.package.version);
+
+ bld_query bq (
+ bld_query::id.package.name == bld_query::_ref (id.name) &&
+
+ qv.epoch == bld_query::_ref (id.version.epoch) &&
+ qv.canonical_upstream ==
+ bld_query::_ref (id.version.canonical_upstream) &&
+ qv.canonical_release == bld_query::_ref (id.version.canonical_release) &&
+ qv.revision == bld_query::_ref (id.version.revision) &&
+
+ bld_query::id.configuration.in_range (cfg_names.begin (),
+ cfg_names.end ()) &&
+
+ ((bld_query::state == "tested" &&
+ ((bld_query::status != "abort" && bld_query::status != "abnormal") ||
+ bld_query::timestamp > expiration_ns)) ||
+
+ (bld_query::state == "testing" &&
+ bld_query::timestamp > expiration_ns)));
+
+ connection_ptr bld_conn (build_db_->connection ());
+
+ prep_bld_query bld_prep_query (
+ bld_conn->prepare_query<build> (
+ "mod-build-task-package-build-query", bq));
+
+ while (tsm.session.empty ())
+ {
+ // Start the package database transaction.
+ //
+ transaction pt (pkg_conn->begin ());
+
+ // Query package versions.
+ //
+ auto package_versions (pkg_prep_query.execute ());
+
+ // Bail out if there is nothing left.
+ //
+ if (package_versions.empty ())
+ {
+ pt.commit ();
+ break;
+ }
+
+ offset += package_versions.size ();
+
+ // Start the build database transaction.
+ //
+ {
+ transaction bt (bld_conn->begin (), false);
+ transaction::current (bt);
+
+ // Iterate over packages until we find one that needs building.
+ //
+ for (auto& pv: package_versions)
+ {
+ id = move (pv.id);
+
+ // Iterate through the package configurations and erase those that
+ // don't need building from the build configuration map. All those
+ // configurations that remained can be built. We will take the first
+ // one, if present.
+ //
+ config_machines configs (cfg_machines); // Make a copy for this pkg.
+
+ for (const auto& pc: bld_prep_query.execute ())
+ {
+ auto i (configs.find (pc.id.configuration.c_str ()));
+
+ // Outdated configurations are already excluded with the database
+ // query.
+ //
+ assert (i != configs.end ());
+ configs.erase (i);
+ }
+
+ if (!configs.empty ())
+ {
+ config_machine& cm (configs.begin ()->second);
+ const build_config& cfg (*cm.config);
+
+ build_id bid (move (id), cfg.name);
+ shared_ptr<build> b (build_db_->find<build> (bid));
+
+ // If build configuration doesn't exist then create the new one
+ // and persist. Otherwise put it into the testing state, refresh
+ // the timestamp and update.
+ //
+ if (b == nullptr)
+ {
+ b = make_shared<build> (move (bid.package.name),
+ move (pv.version),
+ move (bid.configuration));
+
+ build_db_->persist (b);
+ }
+ else
+ {
+ // If the package configuration is in the tested state, then we
+ // need to cleanup the status and results prior to the update.
+ // Otherwise the status is already absent and there are no
+ // results.
+ //
+ // Load the section to make sure results are updated for the
+ // tested state, otherwise assert there are no results.
+ //
+ build_db_->load (*b, b->results_section);
+
+ if (b->state == build_state::tested)
+ {
+ assert (b->status);
+ b->status = nullopt;
+
+ b->results.clear ();
+ }
+ else
+ {
+ assert (!b->status && b->results.empty ());
+ }
+
+ b->state = build_state::testing;
+ b->timestamp = timestamp::clock::now ();
+
+ build_db_->update (b);
+ }
+
+ // Finally, prepare the task response manifest.
+ //
+ tsm.session = b->package_name + '/' +
+ b->package_version.string () + '/' + b->configuration;
+
+ // @@ We don't support challenge at the moment, so leave it absent.
+ //
+
+ tsm.result_url = options_->host () + options_->root ().string () +
+ "?build-result";
+
+ // Switch to the package database transaction to load the package.
+ //
+ transaction::current (pt);
+
+ shared_ptr<package> p (package_db_->load<package> (b->id.package));
+ shared_ptr<repository> r (p->internal_repository.load ());
+
+ // Switch back to the build database transaction.
+ //
+ transaction::current (bt);
+
+ strings fp;
+ if (r->certificate)
+ fp.emplace_back (move (r->certificate->fingerprint));
+
+ tsm.task = task_manifest (
+ move (b->package_name),
+ move (b->package_version),
+ move (r->location),
+ move (fp),
+ move (cm.machine->name),
+ cfg.target,
+ cfg.vars);
+ }
+
+ // If task response manifest is filled, then can bail out from the
+ // package loop, commit transactions and respond.
+ //
+ if (!tsm.session.empty ())
+ break;
+ }
+
+ bt.commit (); // Commit the build database transaction.
+ }
+
+ transaction::current (pt); // Switch to the package database transaction.
+ pt.commit ();
+ }
+ }
+
+ // @@ Probably it would be a good idea to also send some cache control
+ // headers to avoid caching by HTTP proxies. That would require extension
+ // of the web::response interface.
+ //
+
+ manifest_serializer s (rs.content (200, "text/manifest;charset=utf-8"),
+ "task_response_manifest");
+ tsm.serialize (s);
+
+ return true;
+}