aboutsummaryrefslogtreecommitdiff
path: root/load/load.cxx
diff options
context:
space:
mode:
Diffstat (limited to 'load/load.cxx')
-rw-r--r--load/load.cxx358
1 files changed, 261 insertions, 97 deletions
diff --git a/load/load.cxx b/load/load.cxx
index 31230a7..14b8374 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -5,10 +5,9 @@
#include <cerrno>
#include <chrono>
-#include <thread> // this_thread::sleep_for()
-#include <cstring> // strncmp()
+#include <thread> // this_thread::sleep_for()
+#include <cstring> // strncmp()
#include <iostream>
-#include <algorithm> // find(), find_if()
#include <odb/session.hxx>
#include <odb/database.hxx>
@@ -18,13 +17,13 @@
#include <odb/pgsql/database.hxx>
-#include <libbutl/pager.mxx>
-#include <libbutl/sha256.mxx>
-#include <libbutl/process.mxx>
-#include <libbutl/fdstream.mxx>
-#include <libbutl/filesystem.mxx>
-#include <libbutl/tab-parser.mxx>
-#include <libbutl/manifest-parser.mxx>
+#include <libbutl/pager.hxx>
+#include <libbutl/sha256.hxx>
+#include <libbutl/process.hxx>
+#include <libbutl/fdstream.hxx>
+#include <libbutl/filesystem.hxx>
+#include <libbutl/tab-parser.hxx>
+#include <libbutl/manifest-parser.hxx>
#include <libbpkg/manifest.hxx>
@@ -262,7 +261,7 @@ load_repositories (path p)
bad_line ("invalid buildable option value");
}
else
- bad_line ("invalid option '" + nv + "'");
+ bad_line ("invalid option '" + nv + '\'');
}
// For now cache option is mandatory.
@@ -369,7 +368,8 @@ load_packages (const shared_ptr<repository>& rp,
const repository_location& cl,
database& db,
bool ignore_unknown,
- const manifest_name_values& overrides)
+ const manifest_name_values& overrides,
+ const string& overrides_name)
{
// packages_timestamp other than timestamp_nonexistent signals the
// repository packages are already loaded.
@@ -405,8 +405,8 @@ load_packages (const shared_ptr<repository>& rp,
mp,
move (nv),
ignore_unknown,
- false /* complete_depends */,
- package_manifest_flags::forbid_incomplete_dependencies);
+ false /* complete_values */,
+ package_manifest_flags::forbid_incomplete_values);
}
else
pms = pkg_package_manifests (mp, ignore_unknown);
@@ -418,6 +418,8 @@ load_packages (const shared_ptr<repository>& rp,
}
using brep::dependency;
+ using brep::dependency_alternative;
+ using brep::dependency_alternatives;
for (package_manifest& pm: pms)
{
@@ -433,92 +435,144 @@ load_packages (const shared_ptr<repository>& rp,
{
if (rp->internal)
{
+ if (!overrides.empty ())
try
{
- pm.override (overrides, "" /* name */);
+ pm.override (overrides, overrides_name);
}
- catch (const manifest_parsing&)
+ catch (const manifest_parsing& e)
{
- // Overrides are already validated (see below).
- //
- assert (false);
+ cerr << "error: unable to override " << p << " manifest: " << e
+ << endl;
+
+ throw failed ();
}
// Create internal package object.
//
- optional<string> dsc;
- optional<text_type> dst;
-
- if (pm.description)
+ // Return nullopt if the text is in a file (can happen if the
+ // repository is of a type other than pkg) or if the type is not
+ // recognized (can only happen in the "ignore unknown" mode).
+ //
+ auto to_typed_text = [&cl, ignore_unknown] (typed_text_file&& v)
{
+ optional<typed_text> r;
+
// The description value should not be of the file type if the
// package manifest comes from the pkg repository.
//
- assert (!pm.description->file || cl.type () != repository_type::pkg);
+ assert (!v.file || cl.type () != repository_type::pkg);
- if (!pm.description->file)
+ if (!v.file)
{
- dst = pm.effective_description_type (ignore_unknown);
+ // Cannot throw since the manifest parser has already verified the
+ // effective type in the same "ignore unknown" mode.
+ //
+ optional<text_type> t (v.effective_type (ignore_unknown));
// If the description type is unknown (which may be the case for
// some "transitional" period and only if --ignore-unknown is
// specified) we just silently drop the description.
//
- assert (dst || ignore_unknown);
+ assert (t || ignore_unknown);
- if (dst)
- dsc = move (pm.description->text);
+ if (t)
+ r = typed_text {move (v.text), *t};
}
- }
- string chn;
+ return r;
+ };
+
+ // Convert descriptions.
+ //
+ optional<typed_text> ds (
+ pm.description
+ ? to_typed_text (move (*pm.description))
+ : optional<typed_text> ());
+
+ optional<typed_text> pds (
+ pm.package_description
+ ? to_typed_text (move (*pm.package_description))
+ : optional<typed_text> ());
+
+ // Merge changes into a single typed text object.
+ //
+ // If the text type is not recognized for any changes entry or some
+ // entry refers to a file, then assume that no changes are specified.
+ //
+ optional<typed_text> chn;
+
for (auto& c: pm.changes)
{
- // The changes value should not be of the file type if the package
- // manifest comes from the pkg repository.
- //
- assert (!c.file || cl.type () != repository_type::pkg);
+ optional<typed_text> tc (to_typed_text (move (c)));
- if (!c.file)
+ if (!tc)
{
- if (chn.empty ())
- chn = move (c.text);
- else
- {
- if (chn.back () != '\n')
- chn += '\n'; // Always have a blank line as a separator.
+ chn = nullopt;
+ break;
+ }
- chn += "\n" + c.text;
- }
+ if (!chn)
+ {
+ chn = move (*tc);
+ }
+ else
+ {
+ // Should have failed while parsing the manifest otherwise.
+ //
+ assert (tc->type == chn->type);
+
+ string& v (chn->text);
+
+ assert (!v.empty ()); // Changes manifest value cannot be empty.
+
+ if (v.back () != '\n')
+ v += '\n'; // Always have a blank line as a separator.
+
+ v += '\n';
+ v += tc->text;
}
}
- dependencies ds;
+ dependencies tds;
- for (auto& pda: pm.dependencies)
+ for (auto& das: pm.dependencies)
{
- // Ignore special build2 and bpkg dependencies. We may not have
- // packages for them and also showing them for every package is
- // probably not very helpful.
- //
- if (pda.buildtime && !pda.empty ())
+ dependency_alternatives tdas (das.buildtime, move (das.comment));
+
+ for (auto& da: das)
{
- const package_name& n (pda.front ().name);
- if (n == "build2" || n == "bpkg")
- continue;
- }
+ dependency_alternative tda (move (da.enable),
+ move (da.reflect),
+ move (da.prefer),
+ move (da.accept),
+ move (da.require));
- ds.emplace_back (pda.conditional, pda.buildtime, move (pda.comment));
+ for (auto& d: da)
+ {
+ package_name& n (d.name);
- for (auto& pd: pda)
- {
- // The package member will be assigned during dependency
- // resolution procedure.
- //
- ds.back ().push_back (dependency {move (pd.name),
- move (pd.constraint),
- nullptr /* package */});
+ // Ignore special build2 and bpkg dependencies. We may not have
+ // packages for them and also showing them for every package is
+ // probably not very helpful.
+ //
+ if (das.buildtime && (n == "build2" || n == "bpkg"))
+ continue;
+
+ // The package member will be assigned during dependency
+ // resolution procedure.
+ //
+ tda.push_back (dependency {move (n),
+ move (d.constraint),
+ nullptr /* package */});
+ }
+
+ if (!tda.empty ())
+ tdas.push_back (move (tda));
}
+
+ if (!tdas.empty ())
+ tds.push_back (move (tdas));
}
small_vector<brep::test_dependency, 1> ts;
@@ -528,7 +582,12 @@ load_packages (const shared_ptr<repository>& rp,
ts.reserve (pm.tests.size ());
for (bpkg::test_dependency& td: pm.tests)
- ts.emplace_back (move (td.name), td.type, move (td.constraint));
+ ts.emplace_back (move (td.name),
+ td.type,
+ td.buildtime,
+ move (td.constraint),
+ move (td.enable),
+ move (td.reflect));
}
// Cache before the package name is moved.
@@ -545,8 +604,8 @@ load_packages (const shared_ptr<repository>& rp,
move (pm.license_alternatives),
move (pm.topics),
move (pm.keywords),
- move (dsc),
- move (dst),
+ move (ds),
+ move (pds),
move (chn),
move (pm.url),
move (pm.doc_url),
@@ -557,11 +616,13 @@ load_packages (const shared_ptr<repository>& rp,
move (pm.build_email),
move (pm.build_warning_email),
move (pm.build_error_email),
- move (ds),
+ move (tds),
move (pm.requirements),
move (ts),
move (pm.builds),
move (pm.build_constraints),
+ move (pm.build_auxiliaries),
+ move (pm.build_configs),
move (pm.location),
move (pm.fragment),
move (pm.sha256sum),
@@ -574,6 +635,8 @@ load_packages (const shared_ptr<repository>& rp,
move (pm.version),
move (pm.builds),
move (pm.build_constraints),
+ move (pm.build_auxiliaries),
+ move (pm.build_configs),
rp);
db.persist (p);
@@ -664,6 +727,9 @@ load_repositories (const options& lo,
manifest_parser mp (ifs, p.string ());
rpm = pkg_repository_manifests (mp, ignore_unknown);
+
+ if (rpm.empty ())
+ rpm.emplace_back (repository_manifest ()); // Add the base repository.
}
catch (const io_error& e)
{
@@ -956,7 +1022,8 @@ load_repositories (const options& lo,
!pr->cache_location.empty () ? pr->cache_location : cl,
db,
ignore_unknown,
- manifest_name_values () /* overrides */);
+ manifest_name_values () /* overrides */,
+ "" /* overrides_name */);
load_repositories (lo,
pr,
@@ -1004,18 +1071,26 @@ find (const lazy_shared_ptr<repository>& r,
return false;
}
-// Resolve package run-time dependencies and external tests. Make sure that
-// the best matching dependency belongs to the package repositories, their
+// Resolve package regular dependencies and external tests. Make sure that the
+// best matching dependency belongs to the package repositories, their
// complements, recursively, or their immediate prerequisite repositories
-// (only for run-time dependencies). Set the buildable flag to false for the
-// resolved external tests packages. Fail if unable to resolve a dependency,
-// unless ignore_unresolved is true in which case leave this dependency
-// NULL. Should be called once per internal package.
+// (only for regular dependencies). Set the buildable flag to false for the
+// resolved external tests packages. Fail if unable to resolve a regular
+// dependency, unless ignore_unresolved is true in which case leave this
+// dependency NULL. Fail if unable to resolve an external test, unless
+// ignore_unresolved or ignore_unresolved_tests is true in which case leave
+// this dependency NULL, if ignore_unresolved_tests is false, and remove the
+// respective tests manifest entry otherwise. Should be called once per
+// internal package.
//
static void
-resolve_dependencies (package& p, database& db, bool ignore_unresolved)
+resolve_dependencies (package& p,
+ database& db,
+ bool ignore_unresolved,
+ bool ignore_unresolved_tests)
{
using brep::dependency;
+ using brep::dependency_alternative;
using brep::dependency_alternatives;
// Resolve dependencies for internal packages only.
@@ -1114,34 +1189,50 @@ resolve_dependencies (package& p, database& db, bool ignore_unresolved)
return false;
};
- auto bail = [&p] (const dependency& d, const char* what)
+ auto bail = [&p] (const dependency& d, const string& what)
{
- cerr << "error: can't resolve " << what << " " << d << " for the package "
- << p.name << " " << p.version << endl
+ cerr << "error: can't resolve " << what << ' ' << d << " for the package "
+ << p.name << ' ' << p.version << endl
<< " info: repository " << p.internal_repository.load ()->location
<< " appears to be broken" << endl;
throw failed ();
};
- for (dependency_alternatives& da: p.dependencies)
+ for (dependency_alternatives& das: p.dependencies)
{
- for (dependency& d: da)
+ // Practically it is enough to resolve at least one dependency alternative
+ // to build a package. Meanwhile here we consider an error specifying in
+ // the manifest file an alternative which can't be resolved, unless
+ // unresolved dependencies are allowed.
+ //
+ for (dependency_alternative& da: das)
{
- // Practically it is enough to resolve at least one dependency
- // alternative to build a package. Meanwhile here we consider an error
- // specifying in the manifest file an alternative which can't be
- // resolved, unless unresolved dependencies are allowed.
- //
- if (!resolve (d, false /* test */) && !ignore_unresolved)
- bail (d, "dependency");
+ for (dependency& d: da)
+ {
+ if (!resolve (d, false /* test */) && !ignore_unresolved)
+ bail (d, "dependency");
+ }
}
}
- for (brep::test_dependency& td: p.tests)
+ for (auto i (p.tests.begin ()); i != p.tests.end (); )
{
- if (!resolve (td, true /* test */) && !ignore_unresolved)
- bail (td, td.name.string ().c_str ());
+ brep::test_dependency& td (*i);
+
+ if (!resolve (td, true /* test */))
+ {
+ if (!ignore_unresolved && !ignore_unresolved_tests)
+ bail (td, to_string (td.type));
+
+ if (ignore_unresolved_tests)
+ {
+ i = p.tests.erase (i);
+ continue;
+ }
+ }
+
+ ++i;
}
db.update (p); // Update the package state.
@@ -1198,10 +1289,13 @@ detect_dependency_cycle (const package_id& id,
chain.push_back (id);
shared_ptr<package> p (db.load<package> (id));
- for (const auto& da: p->dependencies)
+ for (const auto& das: p->dependencies)
{
- for (const auto& d: da)
- detect_dependency_cycle (d.package.object_id (), chain, db);
+ for (const auto& da: das)
+ {
+ for (const auto& d: da)
+ detect_dependency_cycle (d.package.object_id (), chain, db);
+ }
}
chain.pop_back ();
@@ -1421,8 +1515,46 @@ try
throw failed ();
}
+ // Verify the --service-* options.
+ //
+ if (ops.service_id_specified ())
+ {
+ if (!ops.tenant_specified ())
+ {
+ cerr << "error: --service-id requires --tenant" << endl;
+ throw failed ();
+ }
+
+ if (ops.service_type ().empty ())
+ {
+ cerr << "error: --service-id requires --service-type"
+ << endl;
+ throw failed ();
+ }
+ }
+ else
+ {
+ if (ops.service_type_specified ())
+ {
+ cerr << "error: --service-type requires --service-id"
+ << endl;
+ throw failed ();
+ }
+
+ if (ops.service_data_specified ())
+ {
+ cerr << "error: --service-data requires --service-id"
+ << endl;
+ throw failed ();
+ }
+ }
+
// Parse and validate overrides, if specified.
//
+ // Note that here we make sure that the overrides manifest is valid.
+ // Applying overrides to a specific package manifest may still fail (see
+ // package_manifest::validate_overrides() for details).
+ //
manifest_name_values overrides;
if (ops.overrides_file_specified ())
@@ -1474,6 +1606,11 @@ try
throw failed ();
}
+ // Note: the interactive tenant implies private.
+ //
+ if (ops.interactive_specified ())
+ ops.private_ (true);
+
// Load the description of all the internal repositories from the
// configuration file.
//
@@ -1511,7 +1648,30 @@ try
// Persist the tenant.
//
- db.persist (tenant (tnt));
+ // Note that if the tenant service is specified and some tenant with the
+ // same service id and type is already persisted, then we will end up with
+ // the `object already persistent` error and terminate with the exit code
+ // 1 (fatal error). We could potentially dedicate a special exit code for
+ // such a case, so that the caller may recognize it and behave accordingly
+ // (CI request handler can treat it as a client error rather than an
+ // internal error, etc). However, let's first see if it ever becomes a
+ // problem.
+ //
+ optional<tenant_service> service;
+
+ if (ops.service_id_specified ())
+ service = tenant_service (ops.service_id (),
+ ops.service_type (),
+ (ops.service_data_specified ()
+ ? ops.service_data ()
+ : optional<string> ()));
+
+ db.persist (tenant (tnt,
+ ops.private_ (),
+ (ops.interactive_specified ()
+ ? ops.interactive ()
+ : optional<string> ()),
+ move (service)));
// On the first pass over the internal repositories we load their
// certificate information and packages.
@@ -1540,7 +1700,8 @@ try
r->cache_location,
db,
ops.ignore_unknown (),
- overrides);
+ overrides,
+ ops.overrides_file ().string ());
}
// On the second pass over the internal repositories we load their
@@ -1572,7 +1733,10 @@ try
db.query<package> (
query::id.tenant == tnt &&
query::internal_repository.canonical_name.is_not_null ()))
- resolve_dependencies (p, db, ops.shallow ());
+ resolve_dependencies (p,
+ db,
+ ops.shallow (),
+ ops.ignore_unresolved_tests ());
if (!ops.shallow ())
{