From 1374560d558f4cab72f66b3d851e2a052f59998d Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Thu, 1 Mar 2018 18:33:16 +0300 Subject: Update pkg-build --- bpkg/manifest-utility.cxx | 5 +- bpkg/manifest-utility.hxx | 12 +++ bpkg/package.cxx | 29 +++-- bpkg/package.hxx | 44 ++++---- bpkg/pkg-build.cxx | 198 +++++++++++++++++++++++++++++----- tests/common/satisfy/t4d/repositories | 1 + tests/pkg-build.test | 92 +++++++++++++++- tests/pkg-drop.test | 6 +- 8 files changed, 324 insertions(+), 63 deletions(-) diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx index 8377bab..7b039c9 100644 --- a/bpkg/manifest-utility.cxx +++ b/bpkg/manifest-utility.cxx @@ -146,13 +146,12 @@ namespace bpkg bool repository_name (const string& s) { - size_t n (s.size ()); size_t p (s.find (':')); - // If it has no scheme or starts with the URL scheme (followed by ://) then + // If it has no scheme or starts with the URL scheme (followed by :/) then // this is not a canonical name. // - if (p == string::npos || (p + 2 < n && s[p + 1] == '/' && s[p + 2] == '/')) + if (p == string::npos || s[p + 1] == '/') return false; // This is a canonical name if the scheme is convertible to the repository diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx index a0966d4..5cc1c24 100644 --- a/bpkg/manifest-utility.hxx +++ b/bpkg/manifest-utility.hxx @@ -31,9 +31,21 @@ namespace bpkg string parse_package_name (const char*); + inline string + parse_package_name (const string& s) + { + return parse_package_name (s.c_str ()); + } + version parse_package_version (const char*); + inline version + parse_package_version (const string& s) + { + return parse_package_version (s.c_str ()); + } + // If the passed location is a relative local path, then assume this is a // relative path to the repository directory and complete it based on the // current working directory. Diagnose invalid locations and throw failed. diff --git a/bpkg/package.cxx b/bpkg/package.cxx index e9db6d6..2e5bffe 100644 --- a/bpkg/package.cxx +++ b/bpkg/package.cxx @@ -41,15 +41,19 @@ namespace bpkg static shared_ptr find (const shared_ptr& r, const shared_ptr& ap, - repositories& chain) + repositories& chain, + bool prereq) { + // Prerequisites are not searched through recursively. + // + assert (!prereq || chain.empty ()); + auto pr = [&r] (const shared_ptr& i) -> bool {return i == r;}; auto i (find_if (chain.begin (), chain.end (), pr)); if (i != chain.end ()) return nullptr; - bool prereq (chain.empty ()); // Check prerequisites in top-level only. chain.emplace_back (r); unique_ptr deleter ( @@ -81,7 +85,7 @@ namespace bpkg // Should we consider prerequisites of our complements as our // prerequisites? I'd say not. // - if (shared_ptr r = find (cr.load (), ap, chain)) + if (shared_ptr r = find (cr.load (), ap, chain, false)) return r; } @@ -89,7 +93,7 @@ namespace bpkg { for (const lazy_weak_ptr& pr: ps) { - if (shared_ptr r = find (pr.load (), ap, chain)) + if (shared_ptr r = find (pr.load (), ap, chain, false)) return r; } } @@ -100,20 +104,23 @@ namespace bpkg static inline shared_ptr find (const shared_ptr& r, - const shared_ptr& ap) + const shared_ptr& ap, + bool prereq) { repositories chain; - return find (r, ap, chain); + return find (r, ap, chain, prereq); } vector> - filter (const shared_ptr& r, result&& apr) + filter (const shared_ptr& r, + result&& apr, + bool prereq) { vector> aps; for (shared_ptr ap: pointer_result (apr)) { - if (find (r, ap) != nullptr) + if (find (r, ap, prereq) != nullptr) aps.push_back (move (ap)); } @@ -121,13 +128,15 @@ namespace bpkg } pair, shared_ptr> - filter_one (const shared_ptr& r, result&& apr) + filter_one (const shared_ptr& r, + result&& apr, + bool prereq) { using result = pair, shared_ptr>; for (shared_ptr ap: pointer_result (apr)) { - if (shared_ptr pr = find (r, ap)) + if (shared_ptr pr = find (r, ap, prereq)) return result (move (ap), move (pr)); } diff --git a/bpkg/package.hxx b/bpkg/package.hxx index e864b59..35b9332 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -473,16 +473,20 @@ namespace bpkg operator size_t () const {return result;} }; - // Only return packages that are in the specified repository or its - // complements, recursively. While you could maybe come up with a - // (barely comprehensible) view/query to achieve this, doing it on - // the "client side" is definitely more straightforward. + // Only return packages that are in the specified repository, its + // complements or prerequisites (if prereq is true), recursively. While you + // could maybe come up with a (barely comprehensible) view/query to achieve + // this, doing it on the "client side" is definitely more straightforward. // vector> - filter (const shared_ptr&, odb::result&&); + filter (const shared_ptr&, + odb::result&&, + bool prereq = true); pair, shared_ptr> - filter_one (const shared_ptr&, odb::result&&); + filter_one (const shared_ptr&, + odb::result&&, + bool prereq = true); // package_state // @@ -791,24 +795,22 @@ namespace bpkg // Return a list of packages available from this repository. // - #pragma db view object(repository) \ - table("available_package_locations" = "pl" inner: \ - "pl.repository = " + repository::name) \ - object(available_package inner: \ - "pl.name = " + available_package::id.name + "AND" + \ - "pl.version_epoch = " + \ - available_package::id.version.epoch + "AND" + \ - "pl.version_canonical_upstream = " + \ - available_package::id.version.canonical_upstream + "AND" + \ - "pl.version_canonical_release = " + \ - available_package::id.version.canonical_release + "AND" + \ - "pl.version_revision = " + \ - available_package::id.version.revision) + #pragma db view object(repository) \ + table("available_package_locations" = "pl" inner: \ + "pl.repository = " + repository::name) \ + object(available_package = package inner: \ + "pl.name = " + package::id.name + "AND" + \ + "pl.version_epoch = " + package::id.version.epoch + "AND" + \ + "pl.version_canonical_upstream = " + \ + package::id.version.canonical_upstream + "AND" + \ + "pl.version_canonical_release = " + \ + package::id.version.canonical_release + "AND" + \ + "pl.version_revision = " + package::id.version.revision) struct repository_package { - shared_ptr object; + shared_ptr package; // Must match the alias (see above). - operator const shared_ptr () const {return object;} + operator const shared_ptr () const {return package;} }; // Version comparison operators. diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index fc6ff26..439aee7 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -9,7 +9,7 @@ #include #include // strlen() #include // cout -#include // find() +#include // find(), find_if() #include #include @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,8 @@ namespace bpkg find_available (database& db, const string& name, const shared_ptr& r, - const optional& c) + const optional& c, + bool prereq = true) { using query = query; @@ -124,7 +126,7 @@ namespace bpkg // Filter the result based on the repository to which each version // belongs. // - return filter_one (r, db.query (q)); + return filter_one (r, db.query (q), prereq); } // Create a transient (or fake, if you prefer) available_package object @@ -1071,35 +1073,188 @@ namespace bpkg fail << "package name argument expected" << info << "run 'bpkg help pkg-build' for more information"; + // Check if the argument has the @ form. Return the + // delimiter position if that's the case. Otherwise return string::npos. + // + // We consider '@' to be such a delimiter if it comes before ':' (e.g., a + // URL which could contain its own '@'). + // + auto location = [] (const string& arg) -> size_t + { + size_t p (arg.find_first_of ("@:")); + return p != string::npos && arg[p] == '@' ? p : string::npos; + }; + + // Collect repository locations from @ arguments, + // suppressing duplicates. + // + // Note that the last repository location overrides the previous ones with + // the same canonical name. + // + strings args; + vector locations; + + while (a.more ()) + { + string arg (a.next ()); + size_t p (location (arg)); + + if (p != string::npos) + { + repository_location l ( + parse_location (string (arg, p + 1), nullopt /* type */)); + + auto pr = [&l] (const repository_location& i) -> bool + { + return i.canonical_name () == l.canonical_name (); + }; + + auto i (find_if (locations.begin (), locations.end (), pr)); + + if (i != locations.end ()) + *i = move (l); + else + locations.push_back (move (l)); + } + + args.push_back (move (arg)); + } + + database db (open (c, trace)); // Also populates the system repository. + + // Note that the session spans all our transactions. The idea here is + // that selected_package objects in the build_packages list below will + // be cached in this session. When subsequent transactions modify any + // of these objects, they will modify the cached instance, which means + // our list will always "see" their updated state. + // + // Also note that rep_fetch() must be called in session. + // + session s; + + if (!locations.empty ()) + rep_fetch (o, c, db, locations); + + // Expand @ arguments. + // + strings eargs; + { + transaction t (db.begin ()); + + for (string& arg: args) + { + size_t p (location (arg)); + + if (p == string::npos) + { + eargs.push_back (move (arg)); + continue; + } + + repository_location l ( + parse_location (string (arg, p + 1), nullopt /* type */)); + + shared_ptr r (db.load (l.canonical_name ())); + + // If no packages are specified explicitly (the argument starts with + // '@') then we select latest versions of all the packages from this + // repository. Otherwise, we search for the specified packages and + // versions (if specified) or latest versions (if unspecified) in the + // repository and its complements (recursively), failing if any of + // them are not found. + // + if (p == 0) // No packages are specified explicitly. + { + // Collect the latest package version. + // + map pvs; + + using query = query; + + for (const auto& rp: db.query ( + (query::repository::name == r->name) + + order_by_version_desc (query::package::id.version))) + { + const shared_ptr& p (rp); + pvs.insert (make_pair (p->id.name, p->version)); + } + + // Populate the argument list with the latest package versions. + // + for (const auto& pv: pvs) + eargs.push_back (pv.first + '/' + pv.second.string ()); + } + else // Packages with optional versions in the coma-separated list. + { + string ps (arg, 0, p); + for (size_t b (0); b != string::npos;) + { + // Extract the package. + // + p = ps.find (',', b); + + string s (ps, b, p != string::npos ? p - b : p); + string n (parse_package_name (s)); + version v (parse_package_version (s)); + + optional c ( + !v.empty () ? optional (v) : nullopt); + + // Check if the package is present in the repository and its + // complements, recursively. + // + shared_ptr ap ( + find_available (db, n, r, c, false /* prereq */).first); + + if (ap == nullptr) + { + diag_record dr (fail); + dr << "package " << s << " is not found in " << r->name; + + if (!r->complements.empty ()) + dr << " nor its complements"; + } + + // Add the package/version to the argument list. + // + eargs.push_back (ap->id.name + '/' + ap->version.string ()); + + b = p != string::npos ? p + 1 : p; + } + } + } + + t.commit (); + } + map package_arg; // Check if the package is a duplicate. Return true if it is but harmless. // - auto check_dup = [&package_arg] (const string& n, const char* arg) -> bool + auto check_dup = [&package_arg] (const string& n, const string& a) -> bool { - auto r (package_arg.emplace (n, arg)); + auto r (package_arg.emplace (n, a)); - if (!r.second && r.first->second != arg) + if (!r.second && r.first->second != a) fail << "duplicate package " << n << info << "first mentioned as " << r.first->second << - info << "second mentioned as " << arg; + info << "second mentioned as " << a; return !r.second; }; // Pre-scan the arguments and sort them out into optional and mandatory. // - strings args; - while (a.more ()) + strings pargs; + for (const string& arg: eargs) { - const char* arg (a.next ()); - const char* s (arg); + const char* s (arg.c_str ()); bool opt (s[0] == '?'); if (opt) ++s; else - args.push_back (s); + pargs.emplace_back (s); if (parse_package_scheme (s) == package_scheme::sys) { @@ -1113,7 +1268,10 @@ namespace bpkg v = wildcard_version; const system_package* sp (system_repository.find (n)); - if (sp == nullptr) // Will deal with all the duplicates later. + + // Will deal with all the duplicates later. + // + if (sp == nullptr || !sp->authoritative) system_repository.insert (n, v, true); } else if (opt) @@ -1121,22 +1279,12 @@ namespace bpkg info << "package is ignored"; } - if (args.empty ()) + if (pargs.empty ()) { warn << "nothing to build"; return 0; } - database db (open (c, trace)); - - // Note that the session spans all our transactions. The idea here is - // that selected_package objects in the build_packages list below will - // be cached in this session. When subsequent transactions modify any - // of these objects, they will modify the cached instance, which means - // our list will always "see" their updated state. - // - session s; - // Assemble the list of packages we will need to build. // build_packages pkgs; @@ -1156,7 +1304,7 @@ namespace bpkg // diagnostics if the name/version guess doesn't pan out. // bool diag (false); - for (auto i (args.cbegin ()); i != args.cend (); ) + for (auto i (pargs.cbegin ()); i != pargs.cend (); ) { const char* package (i->c_str ()); diff --git a/tests/common/satisfy/t4d/repositories b/tests/common/satisfy/t4d/repositories index f0e1983..6277925 100644 --- a/tests/common/satisfy/t4d/repositories +++ b/tests/common/satisfy/t4d/repositories @@ -1,3 +1,4 @@ : 1 location: ../t4c +role: complement : diff --git a/tests/pkg-build.test b/tests/pkg-build.test index 0c9180a..2b54d12 100644 --- a/tests/pkg-build.test +++ b/tests/pkg-build.test @@ -48,7 +48,7 @@ # | |-- libfoo-1.0.0.tar.gz # | `-- repositories # | -# |-- t4d -> t4c (prerequisite) +# |-- t4d -> t4c (complement) # | |-- libbiz-1.0.0.tar.gz -> libfox, libfoo, libbaz # | |-- libfox-1.0.0.tar.gz # | `-- repositories @@ -1305,6 +1305,96 @@ rep_fetch += -d cfg --auth all --trust-yes 2>! $pkg_purge libfoo 2>'purged libfoo/1.0.0' } +: repository-location +: +{ + test.arguments += --yes --auth all --trust-yes + + : all-packages + : + { + $clone_root_cfg; + + $* "@$rep/t4d" 2>>~%EOE%; + %.+ + %info: .+libfox-1.0.0.+ is up to date% + %info: .+libbiz-1.0.0.+ is up to date% + updated libfox/1.0.0 + updated libbiz/1.0.0 + EOE + + $pkg_disfigure libbiz 2>'disfigured libbiz/1.0.0'; + $pkg_purge libbiz 2>'purged libbiz/1.0.0'; + + $pkg_disfigure libfox 2>'disfigured libfox/1.0.0'; + $pkg_purge libfox 2>'purged libfox/1.0.0'; + + $pkg_disfigure libbaz 2>'disfigured libbaz/1.1.0'; + $pkg_purge libbaz 2>'purged libbaz/1.1.0'; + + $pkg_disfigure libbar 2>'disfigured libbar/1.1.0'; + $pkg_purge libbar 2>'purged libbar/1.1.0'; + + $pkg_disfigure libfoo 2>'disfigured libfoo/1.1.0'; + $pkg_purge libfoo 2>'purged libfoo/1.1.0' + } + + : multiple-packages + : + { + $clone_root_cfg; + + $* "libfox,libbiz/1.0.0@$rep/t4d" 2>>~%EOE%; + %.+ + %info: .+libfox-1.0.0.+ is up to date% + %info: .+libbiz-1.0.0.+ is up to date% + updated libfox/1.0.0 + updated libbiz/1.0.0 + EOE + + $pkg_disfigure libbiz 2>'disfigured libbiz/1.0.0'; + $pkg_purge libbiz 2>'purged libbiz/1.0.0'; + + $pkg_disfigure libfox 2>'disfigured libfox/1.0.0'; + $pkg_purge libfox 2>'purged libfox/1.0.0'; + + $pkg_disfigure libbaz 2>'disfigured libbaz/1.1.0'; + $pkg_purge libbaz 2>'purged libbaz/1.1.0'; + + $pkg_disfigure libbar 2>'disfigured libbar/1.1.0'; + $pkg_purge libbar 2>'purged libbar/1.1.0'; + + $pkg_disfigure libfoo 2>'disfigured libfoo/1.1.0'; + $pkg_purge libfoo 2>'purged libfoo/1.1.0' + } + + : package-in-complement + : + { + $clone_root_cfg; + + $* "libfoo@$rep/t4d" 2>>~%EOE%; + %.+ + %info: .+libfoo-1.0.0.+ is up to date% + updated libfoo/1.0.0 + EOE + + $pkg_disfigure libfoo 2>'disfigured libfoo/1.0.0'; + $pkg_purge libfoo 2>'purged libfoo/1.0.0' + } + + : non-existent-package + : + { + $clone_root_cfg; + + $* "libbar@$rep/t4d" 2>>~%EOE% != 0 + %.+ + error: package libbar is not found in bpkg:build2.org/pkg-build/t4d nor its complements + EOE + } +} + : git-repos : if ($git_supported != true) diff --git a/tests/pkg-drop.test b/tests/pkg-drop.test index 07c1f93..9e63470 100644 --- a/tests/pkg-drop.test +++ b/tests/pkg-drop.test @@ -17,7 +17,7 @@ # | |-- libbaz-1.1.0.tar.gz -> libfoo, libbar # | |-- libfoo-1.0.0.tar.gz # | `-- repositories -# `-- t4d -> t4c (prerequisite) +# `-- t4d -> t4c (complement) # |-- libbiz-1.0.0.tar.gz -> libfox, libfoo, libbaz # |-- libfox-1.0.0.tar.gz # `-- repositories @@ -369,9 +369,9 @@ $* libfoo/1.0.0 2>>~%EOE% != 0 EOE -$pkg_status libfox/1.0.0 >'available' - -$pkg_status libfoo/1.1.0 >'unknown' + -$pkg_status libfoo/1.1.0 >'available' -$pkg_status libbar/1.1.0 >'unknown' - -$pkg_status libbaz/1.1.0 >'unknown' + -$pkg_status libbaz/1.1.0 >'available' -$pkg_status libbiz/1.0.0 >'available' } -- cgit v1.1