diff options
Diffstat (limited to 'load/load.cxx')
-rw-r--r-- | load/load.cxx | 358 |
1 files changed, 261 insertions, 97 deletions
diff --git a/load/load.cxx b/load/load.cxx index 31230a7..14b8374 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -5,10 +5,9 @@ #include <cerrno> #include <chrono> -#include <thread> // this_thread::sleep_for() -#include <cstring> // strncmp() +#include <thread> // this_thread::sleep_for() +#include <cstring> // strncmp() #include <iostream> -#include <algorithm> // find(), find_if() #include <odb/session.hxx> #include <odb/database.hxx> @@ -18,13 +17,13 @@ #include <odb/pgsql/database.hxx> -#include <libbutl/pager.mxx> -#include <libbutl/sha256.mxx> -#include <libbutl/process.mxx> -#include <libbutl/fdstream.mxx> -#include <libbutl/filesystem.mxx> -#include <libbutl/tab-parser.mxx> -#include <libbutl/manifest-parser.mxx> +#include <libbutl/pager.hxx> +#include <libbutl/sha256.hxx> +#include <libbutl/process.hxx> +#include <libbutl/fdstream.hxx> +#include <libbutl/filesystem.hxx> +#include <libbutl/tab-parser.hxx> +#include <libbutl/manifest-parser.hxx> #include <libbpkg/manifest.hxx> @@ -262,7 +261,7 @@ load_repositories (path p) bad_line ("invalid buildable option value"); } else - bad_line ("invalid option '" + nv + "'"); + bad_line ("invalid option '" + nv + '\''); } // For now cache option is mandatory. @@ -369,7 +368,8 @@ load_packages (const shared_ptr<repository>& rp, const repository_location& cl, database& db, bool ignore_unknown, - const manifest_name_values& overrides) + const manifest_name_values& overrides, + const string& overrides_name) { // packages_timestamp other than timestamp_nonexistent signals the // repository packages are already loaded. @@ -405,8 +405,8 @@ load_packages (const shared_ptr<repository>& rp, mp, move (nv), ignore_unknown, - false /* complete_depends */, - package_manifest_flags::forbid_incomplete_dependencies); + false /* complete_values */, + package_manifest_flags::forbid_incomplete_values); } else pms = pkg_package_manifests (mp, ignore_unknown); @@ -418,6 +418,8 @@ load_packages (const shared_ptr<repository>& rp, } using brep::dependency; + using brep::dependency_alternative; + using brep::dependency_alternatives; for (package_manifest& pm: pms) { @@ -433,92 +435,144 @@ load_packages (const shared_ptr<repository>& rp, { if (rp->internal) { + if (!overrides.empty ()) try { - pm.override (overrides, "" /* name */); + pm.override (overrides, overrides_name); } - catch (const manifest_parsing&) + catch (const manifest_parsing& e) { - // Overrides are already validated (see below). - // - assert (false); + cerr << "error: unable to override " << p << " manifest: " << e + << endl; + + throw failed (); } // Create internal package object. // - optional<string> dsc; - optional<text_type> dst; - - if (pm.description) + // Return nullopt if the text is in a file (can happen if the + // repository is of a type other than pkg) or if the type is not + // recognized (can only happen in the "ignore unknown" mode). + // + auto to_typed_text = [&cl, ignore_unknown] (typed_text_file&& v) { + optional<typed_text> r; + // The description value should not be of the file type if the // package manifest comes from the pkg repository. // - assert (!pm.description->file || cl.type () != repository_type::pkg); + assert (!v.file || cl.type () != repository_type::pkg); - if (!pm.description->file) + if (!v.file) { - dst = pm.effective_description_type (ignore_unknown); + // Cannot throw since the manifest parser has already verified the + // effective type in the same "ignore unknown" mode. + // + optional<text_type> t (v.effective_type (ignore_unknown)); // If the description type is unknown (which may be the case for // some "transitional" period and only if --ignore-unknown is // specified) we just silently drop the description. // - assert (dst || ignore_unknown); + assert (t || ignore_unknown); - if (dst) - dsc = move (pm.description->text); + if (t) + r = typed_text {move (v.text), *t}; } - } - string chn; + return r; + }; + + // Convert descriptions. + // + optional<typed_text> ds ( + pm.description + ? to_typed_text (move (*pm.description)) + : optional<typed_text> ()); + + optional<typed_text> pds ( + pm.package_description + ? to_typed_text (move (*pm.package_description)) + : optional<typed_text> ()); + + // Merge changes into a single typed text object. + // + // If the text type is not recognized for any changes entry or some + // entry refers to a file, then assume that no changes are specified. + // + optional<typed_text> chn; + for (auto& c: pm.changes) { - // The changes value should not be of the file type if the package - // manifest comes from the pkg repository. - // - assert (!c.file || cl.type () != repository_type::pkg); + optional<typed_text> tc (to_typed_text (move (c))); - if (!c.file) + if (!tc) { - if (chn.empty ()) - chn = move (c.text); - else - { - if (chn.back () != '\n') - chn += '\n'; // Always have a blank line as a separator. + chn = nullopt; + break; + } - chn += "\n" + c.text; - } + if (!chn) + { + chn = move (*tc); + } + else + { + // Should have failed while parsing the manifest otherwise. + // + assert (tc->type == chn->type); + + string& v (chn->text); + + assert (!v.empty ()); // Changes manifest value cannot be empty. + + if (v.back () != '\n') + v += '\n'; // Always have a blank line as a separator. + + v += '\n'; + v += tc->text; } } - dependencies ds; + dependencies tds; - for (auto& pda: pm.dependencies) + for (auto& das: pm.dependencies) { - // Ignore special build2 and bpkg dependencies. We may not have - // packages for them and also showing them for every package is - // probably not very helpful. - // - if (pda.buildtime && !pda.empty ()) + dependency_alternatives tdas (das.buildtime, move (das.comment)); + + for (auto& da: das) { - const package_name& n (pda.front ().name); - if (n == "build2" || n == "bpkg") - continue; - } + dependency_alternative tda (move (da.enable), + move (da.reflect), + move (da.prefer), + move (da.accept), + move (da.require)); - ds.emplace_back (pda.conditional, pda.buildtime, move (pda.comment)); + for (auto& d: da) + { + package_name& n (d.name); - for (auto& pd: pda) - { - // The package member will be assigned during dependency - // resolution procedure. - // - ds.back ().push_back (dependency {move (pd.name), - move (pd.constraint), - nullptr /* package */}); + // Ignore special build2 and bpkg dependencies. We may not have + // packages for them and also showing them for every package is + // probably not very helpful. + // + if (das.buildtime && (n == "build2" || n == "bpkg")) + continue; + + // The package member will be assigned during dependency + // resolution procedure. + // + tda.push_back (dependency {move (n), + move (d.constraint), + nullptr /* package */}); + } + + if (!tda.empty ()) + tdas.push_back (move (tda)); } + + if (!tdas.empty ()) + tds.push_back (move (tdas)); } small_vector<brep::test_dependency, 1> ts; @@ -528,7 +582,12 @@ load_packages (const shared_ptr<repository>& rp, ts.reserve (pm.tests.size ()); for (bpkg::test_dependency& td: pm.tests) - ts.emplace_back (move (td.name), td.type, move (td.constraint)); + ts.emplace_back (move (td.name), + td.type, + td.buildtime, + move (td.constraint), + move (td.enable), + move (td.reflect)); } // Cache before the package name is moved. @@ -545,8 +604,8 @@ load_packages (const shared_ptr<repository>& rp, move (pm.license_alternatives), move (pm.topics), move (pm.keywords), - move (dsc), - move (dst), + move (ds), + move (pds), move (chn), move (pm.url), move (pm.doc_url), @@ -557,11 +616,13 @@ load_packages (const shared_ptr<repository>& rp, move (pm.build_email), move (pm.build_warning_email), move (pm.build_error_email), - move (ds), + move (tds), move (pm.requirements), move (ts), move (pm.builds), move (pm.build_constraints), + move (pm.build_auxiliaries), + move (pm.build_configs), move (pm.location), move (pm.fragment), move (pm.sha256sum), @@ -574,6 +635,8 @@ load_packages (const shared_ptr<repository>& rp, move (pm.version), move (pm.builds), move (pm.build_constraints), + move (pm.build_auxiliaries), + move (pm.build_configs), rp); db.persist (p); @@ -664,6 +727,9 @@ load_repositories (const options& lo, manifest_parser mp (ifs, p.string ()); rpm = pkg_repository_manifests (mp, ignore_unknown); + + if (rpm.empty ()) + rpm.emplace_back (repository_manifest ()); // Add the base repository. } catch (const io_error& e) { @@ -956,7 +1022,8 @@ load_repositories (const options& lo, !pr->cache_location.empty () ? pr->cache_location : cl, db, ignore_unknown, - manifest_name_values () /* overrides */); + manifest_name_values () /* overrides */, + "" /* overrides_name */); load_repositories (lo, pr, @@ -1004,18 +1071,26 @@ find (const lazy_shared_ptr<repository>& r, return false; } -// Resolve package run-time dependencies and external tests. Make sure that -// the best matching dependency belongs to the package repositories, their +// Resolve package regular dependencies and external tests. Make sure that the +// best matching dependency belongs to the package repositories, their // complements, recursively, or their immediate prerequisite repositories -// (only for run-time dependencies). Set the buildable flag to false for the -// resolved external tests packages. Fail if unable to resolve a dependency, -// unless ignore_unresolved is true in which case leave this dependency -// NULL. Should be called once per internal package. +// (only for regular dependencies). Set the buildable flag to false for the +// resolved external tests packages. Fail if unable to resolve a regular +// dependency, unless ignore_unresolved is true in which case leave this +// dependency NULL. Fail if unable to resolve an external test, unless +// ignore_unresolved or ignore_unresolved_tests is true in which case leave +// this dependency NULL, if ignore_unresolved_tests is false, and remove the +// respective tests manifest entry otherwise. Should be called once per +// internal package. // static void -resolve_dependencies (package& p, database& db, bool ignore_unresolved) +resolve_dependencies (package& p, + database& db, + bool ignore_unresolved, + bool ignore_unresolved_tests) { using brep::dependency; + using brep::dependency_alternative; using brep::dependency_alternatives; // Resolve dependencies for internal packages only. @@ -1114,34 +1189,50 @@ resolve_dependencies (package& p, database& db, bool ignore_unresolved) return false; }; - auto bail = [&p] (const dependency& d, const char* what) + auto bail = [&p] (const dependency& d, const string& what) { - cerr << "error: can't resolve " << what << " " << d << " for the package " - << p.name << " " << p.version << endl + cerr << "error: can't resolve " << what << ' ' << d << " for the package " + << p.name << ' ' << p.version << endl << " info: repository " << p.internal_repository.load ()->location << " appears to be broken" << endl; throw failed (); }; - for (dependency_alternatives& da: p.dependencies) + for (dependency_alternatives& das: p.dependencies) { - for (dependency& d: da) + // Practically it is enough to resolve at least one dependency alternative + // to build a package. Meanwhile here we consider an error specifying in + // the manifest file an alternative which can't be resolved, unless + // unresolved dependencies are allowed. + // + for (dependency_alternative& da: das) { - // Practically it is enough to resolve at least one dependency - // alternative to build a package. Meanwhile here we consider an error - // specifying in the manifest file an alternative which can't be - // resolved, unless unresolved dependencies are allowed. - // - if (!resolve (d, false /* test */) && !ignore_unresolved) - bail (d, "dependency"); + for (dependency& d: da) + { + if (!resolve (d, false /* test */) && !ignore_unresolved) + bail (d, "dependency"); + } } } - for (brep::test_dependency& td: p.tests) + for (auto i (p.tests.begin ()); i != p.tests.end (); ) { - if (!resolve (td, true /* test */) && !ignore_unresolved) - bail (td, td.name.string ().c_str ()); + brep::test_dependency& td (*i); + + if (!resolve (td, true /* test */)) + { + if (!ignore_unresolved && !ignore_unresolved_tests) + bail (td, to_string (td.type)); + + if (ignore_unresolved_tests) + { + i = p.tests.erase (i); + continue; + } + } + + ++i; } db.update (p); // Update the package state. @@ -1198,10 +1289,13 @@ detect_dependency_cycle (const package_id& id, chain.push_back (id); shared_ptr<package> p (db.load<package> (id)); - for (const auto& da: p->dependencies) + for (const auto& das: p->dependencies) { - for (const auto& d: da) - detect_dependency_cycle (d.package.object_id (), chain, db); + for (const auto& da: das) + { + for (const auto& d: da) + detect_dependency_cycle (d.package.object_id (), chain, db); + } } chain.pop_back (); @@ -1421,8 +1515,46 @@ try throw failed (); } + // Verify the --service-* options. + // + if (ops.service_id_specified ()) + { + if (!ops.tenant_specified ()) + { + cerr << "error: --service-id requires --tenant" << endl; + throw failed (); + } + + if (ops.service_type ().empty ()) + { + cerr << "error: --service-id requires --service-type" + << endl; + throw failed (); + } + } + else + { + if (ops.service_type_specified ()) + { + cerr << "error: --service-type requires --service-id" + << endl; + throw failed (); + } + + if (ops.service_data_specified ()) + { + cerr << "error: --service-data requires --service-id" + << endl; + throw failed (); + } + } + // Parse and validate overrides, if specified. // + // Note that here we make sure that the overrides manifest is valid. + // Applying overrides to a specific package manifest may still fail (see + // package_manifest::validate_overrides() for details). + // manifest_name_values overrides; if (ops.overrides_file_specified ()) @@ -1474,6 +1606,11 @@ try throw failed (); } + // Note: the interactive tenant implies private. + // + if (ops.interactive_specified ()) + ops.private_ (true); + // Load the description of all the internal repositories from the // configuration file. // @@ -1511,7 +1648,30 @@ try // Persist the tenant. // - db.persist (tenant (tnt)); + // Note that if the tenant service is specified and some tenant with the + // same service id and type is already persisted, then we will end up with + // the `object already persistent` error and terminate with the exit code + // 1 (fatal error). We could potentially dedicate a special exit code for + // such a case, so that the caller may recognize it and behave accordingly + // (CI request handler can treat it as a client error rather than an + // internal error, etc). However, let's first see if it ever becomes a + // problem. + // + optional<tenant_service> service; + + if (ops.service_id_specified ()) + service = tenant_service (ops.service_id (), + ops.service_type (), + (ops.service_data_specified () + ? ops.service_data () + : optional<string> ())); + + db.persist (tenant (tnt, + ops.private_ (), + (ops.interactive_specified () + ? ops.interactive () + : optional<string> ()), + move (service))); // On the first pass over the internal repositories we load their // certificate information and packages. @@ -1540,7 +1700,8 @@ try r->cache_location, db, ops.ignore_unknown (), - overrides); + overrides, + ops.overrides_file ().string ()); } // On the second pass over the internal repositories we load their @@ -1572,7 +1733,10 @@ try db.query<package> ( query::id.tenant == tnt && query::internal_repository.canonical_name.is_not_null ())) - resolve_dependencies (p, db, ops.shallow ()); + resolve_dependencies (p, + db, + ops.shallow (), + ops.ignore_unresolved_tests ()); if (!ops.shallow ()) { |