From a7328a438ed26dc0d74238a575c118ca82892b5e Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Sat, 18 Jun 2016 21:28:19 +0200 Subject: Rename cfg-{add,fetch} to rep-{add,fetch} (aliases stay same) The rationale: we now need the 'add' command for certificates (i.e., add a trusted certificate to the configuration). In the old naming scheme we would have to call it cfg-add which is already taken. In the new scheme we will call it crt-add. --- NEWS | 5 +- bpkg/auth | 2 +- bpkg/bpkg.cli | 34 +++--- bpkg/bpkg.cxx | 28 ++--- bpkg/buildfile | 8 +- bpkg/cfg-add | 19 ---- bpkg/cfg-add.cli | 32 ------ bpkg/cfg-add.cxx | 66 ----------- bpkg/cfg-fetch | 19 ---- bpkg/cfg-fetch.cli | 32 ------ bpkg/cfg-fetch.cxx | 328 ----------------------------------------------------- bpkg/package | 2 +- bpkg/pkg-build.cli | 4 +- bpkg/pkg-build.cxx | 4 +- bpkg/pkg-fetch.cli | 2 +- bpkg/pkg-fetch.cxx | 4 +- bpkg/rep-add | 19 ++++ bpkg/rep-add.cli | 32 ++++++ bpkg/rep-add.cxx | 66 +++++++++++ bpkg/rep-fetch | 19 ++++ bpkg/rep-fetch.cli | 32 ++++++ bpkg/rep-fetch.cxx | 328 +++++++++++++++++++++++++++++++++++++++++++++++++++++ doc/buildfile | 4 +- doc/cli.sh | 6 +- tests/test.sh | 240 +++++++++++++++++++-------------------- 25 files changed, 668 insertions(+), 667 deletions(-) delete mode 100644 bpkg/cfg-add delete mode 100644 bpkg/cfg-add.cli delete mode 100644 bpkg/cfg-add.cxx delete mode 100644 bpkg/cfg-fetch delete mode 100644 bpkg/cfg-fetch.cli delete mode 100644 bpkg/cfg-fetch.cxx create mode 100644 bpkg/rep-add create mode 100644 bpkg/rep-add.cli create mode 100644 bpkg/rep-add.cxx create mode 100644 bpkg/rep-fetch create mode 100644 bpkg/rep-fetch.cli create mode 100644 bpkg/rep-fetch.cxx diff --git a/NEWS b/NEWS index b561411..64b9672 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,8 @@ Version 0.4.0 + * Rename the cfg-add command to rep-add (the add alias stays the same) and + cfg-fetch to rep-fetch (the fetch alias stays the same). + * The pkg-build command now offers to automatically update dependent packages that were reconfigured. @@ -12,7 +15,7 @@ Version 0.4.0 the command's man page for details. * Add support for repository authentication. The rep-create command can now - sign the repository, and cfg-fetch and rep-info commands can authenticate + sign the repository, and rep-fetch and rep-info commands can authenticate the repository being fetched. Version 0.3.0 diff --git a/bpkg/auth b/bpkg/auth index 54071b7..3d1ff65 100644 --- a/bpkg/auth +++ b/bpkg/auth @@ -28,7 +28,7 @@ namespace bpkg // // Note that one drawback of doing this as part of an existing transaction // is that if things go south and the transaction gets aborted, then all the - // user's confirmations will be lost. For example, cfg-fetch could fail + // user's confirmations will be lost. For example, rep-fetch could fail // because it was unable to fetch some prerequisite repositories. // shared_ptr diff --git a/bpkg/bpkg.cli b/bpkg/bpkg.cli index 107ef0a..13299b3 100644 --- a/bpkg/bpkg.cli +++ b/bpkg/bpkg.cli @@ -155,16 +155,26 @@ namespace bpkg "\l{bpkg-cfg-create(1)} \- create configuration" } - bool cfg-add|add + bool rep-info { - "", - "\l{bpkg-cfg-add(1)} \- add repository to configuration", - "" + "\l{bpkg-rep-info(1)} \- print repository information" } - bool cfg-fetch|fetch + bool rep-add|add { - "\l{bpkg-cfg-fetch(1)} \- fetch list of available packages" + "\l{bpkg-rep-add(1)} \- add repository to configuration" + } + + bool rep-fetch|fetch + { + "\l{bpkg-rep-fetch(1)} \- fetch list of available packages" + } + + bool rep-create + { + "[]", + "\l{bpkg-rep-create(1)} \- create repository", + "" } bool pkg-status|status @@ -238,18 +248,6 @@ namespace bpkg { "\l{bpkg-pkg-purge(1)} \- purge package" } - - bool rep-info - { - "\l{bpkg-rep-info(1)} \- print repository information" - } - - bool rep-create - { - "[]", - "\l{bpkg-rep-create(1)} \- create repository", - "" - } }; // Make sure these don't conflict with command names above. diff --git a/bpkg/bpkg.cxx b/bpkg/bpkg.cxx index c94dde9..561e32e 100644 --- a/bpkg/bpkg.cxx +++ b/bpkg/bpkg.cxx @@ -16,6 +16,8 @@ // #include +#include + #include #include #include @@ -31,12 +33,10 @@ #include #include -#include -#include -#include - -#include +#include #include +#include +#include using namespace std; using namespace bpkg; @@ -205,6 +205,12 @@ try break; \ } + // cfg-* commands + // +#define CFG_COMMAND(CMD) COMMAND_IMPL(cfg_, "cfg-", CMD) + + CFG_COMMAND (create); + // pkg-* commands // #define PKG_COMMAND(CMD) COMMAND_IMPL(pkg_, "pkg-", CMD) @@ -224,20 +230,14 @@ try PKG_COMMAND (update); PKG_COMMAND (verify); - // cfg-* commands - // -#define CFG_COMMAND(CMD) COMMAND_IMPL(cfg_, "cfg-", CMD) - - CFG_COMMAND (add); - CFG_COMMAND (create); - CFG_COMMAND (fetch); - // rep-* commands // #define REP_COMMAND(CMD) COMMAND_IMPL(rep_, "rep-", CMD) - REP_COMMAND (info); + REP_COMMAND (add); REP_COMMAND (create); + REP_COMMAND (fetch); + REP_COMMAND (info); assert (false); fail << "unhandled command"; diff --git a/bpkg/buildfile b/bpkg/buildfile index 8436c7c..5067780 100644 --- a/bpkg/buildfile +++ b/bpkg/buildfile @@ -13,9 +13,7 @@ exe{bpkg}: \ {hxx cxx}{ auth } \ {hxx }{ bpkg-version } \ { cxx}{ bpkg } {hxx ixx cxx}{ bpkg-options } \ -{hxx cxx}{ cfg-add } {hxx ixx cxx}{ cfg-add-options } \ {hxx cxx}{ cfg-create } {hxx ixx cxx}{ cfg-create-options } \ -{hxx cxx}{ cfg-fetch } {hxx ixx cxx}{ cfg-fetch-options } \ {hxx cxx}{ checksum } \ {hxx ixx cxx}{ common-options } \ {hxx ixx cxx}{ configuration-options } \ @@ -43,7 +41,9 @@ exe{bpkg}: \ {hxx cxx}{ pkg-unpack } {hxx ixx cxx}{ pkg-unpack-options } \ {hxx }{ pkg-update } {hxx ixx cxx}{ pkg-update-options } \ {hxx cxx}{ pkg-verify } {hxx ixx cxx}{ pkg-verify-options } \ +{hxx cxx}{ rep-add } {hxx ixx cxx}{ rep-add-options } \ {hxx cxx}{ rep-create } {hxx ixx cxx}{ rep-create-options } \ +{hxx cxx}{ rep-fetch } {hxx ixx cxx}{ rep-fetch-options } \ {hxx cxx}{ rep-info } {hxx ixx cxx}{ rep-info-options } \ {hxx cxx}{ satisfaction } \ {hxx }{ types } \ @@ -98,8 +98,8 @@ if! $cli.loaded # rep-* command. # -{hxx ixx cxx}{cfg-add-options}: cli{cfg-add} -{hxx ixx cxx}{cfg-fetch-options}: cli{cfg-fetch} +{hxx ixx cxx}{rep-add-options}: cli{rep-add} +{hxx ixx cxx}{rep-fetch-options}: cli{rep-fetch} {hxx ixx cxx}{rep-info-options}: cli{rep-info} {hxx ixx cxx}{rep-create-options}: cli{rep-create} diff --git a/bpkg/cfg-add b/bpkg/cfg-add deleted file mode 100644 index eec8859..0000000 --- a/bpkg/cfg-add +++ /dev/null @@ -1,19 +0,0 @@ -// file : bpkg/cfg-add -*- C++ -*- -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef BPKG_CFG_ADD -#define BPKG_CFG_ADD - -#include -#include - -#include - -namespace bpkg -{ - int - cfg_add (const cfg_add_options&, cli::scanner& args); -} - -#endif // BPKG_CFG_ADD diff --git a/bpkg/cfg-add.cli b/bpkg/cfg-add.cli deleted file mode 100644 index 8bc17c3..0000000 --- a/bpkg/cfg-add.cli +++ /dev/null @@ -1,32 +0,0 @@ -// file : bpkg/cfg-add.cli -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -include ; - -"\section=1" -"\name=bpkg-cfg-add" -"\summary=add repository to configuration" - -namespace bpkg -{ - { - " ", - - "\h|SYNOPSIS| - - \c{\b{bpkg cfg-add}|\b{add} [] } - - \h|DESCRIPTION| - - The \cb{cfg-add} command adds the specified package repository to the - configuration. Note that it doesn't fetch the list of available packages - for the newly added repository. For that, use the \l{bpkg-cfg-fetch(1)} - command, normally, after adding all the repositories you wish to use." - } - - class cfg_add_options: configuration_options - { - "\h|CFG-ADD OPTIONS|" - }; -} diff --git a/bpkg/cfg-add.cxx b/bpkg/cfg-add.cxx deleted file mode 100644 index 5386fff..0000000 --- a/bpkg/cfg-add.cxx +++ /dev/null @@ -1,66 +0,0 @@ -// file : bpkg/cfg-add.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include - -#include -#include -#include -#include -#include - -using namespace std; -using namespace butl; - -namespace bpkg -{ - int - cfg_add (const cfg_add_options& o, cli::scanner& args) - { - tracer trace ("cfg_add"); - - dir_path c (o.directory ()); - l4 ([&]{trace << "configuration: " << c;}); - - if (!args.more ()) - fail << "repository location argument expected" << - info << "run 'bpkg help cfg-add' for more information"; - - repository_location rl (parse_location (args.next ())); - const string& rn (rl.canonical_name ()); - - // Create the new repository and add is as a complement to the root. - // - database db (open (c, trace)); - transaction t (db.begin ()); - session s; // Repository dependencies can have cycles. - - // It is possible that this repository is already in the database. - // For example, it might be a prerequisite of one of the already - // added repository. - // - shared_ptr r (db.find (rl.canonical_name ())); - - if (r == nullptr) - { - r.reset (new repository (rl)); - db.persist (r); - } - - shared_ptr root (db.load ("")); - - if (!root->complements.insert (lazy_shared_ptr (db, r)).second) - { - fail << rn << " is already a repository of this configuration"; - } - - db.update (root); - t.commit (); - - if (verb) - text << "added repository " << rn; - - return 0; - } -} diff --git a/bpkg/cfg-fetch b/bpkg/cfg-fetch deleted file mode 100644 index 57a4095..0000000 --- a/bpkg/cfg-fetch +++ /dev/null @@ -1,19 +0,0 @@ -// file : bpkg/cfg-fetch -*- C++ -*- -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef BPKG_CFG_FETCH -#define BPKG_CFG_FETCH - -#include -#include - -#include - -namespace bpkg -{ - int - cfg_fetch (const cfg_fetch_options&, cli::scanner& args); -} - -#endif // BPKG_CFG_FETCH diff --git a/bpkg/cfg-fetch.cli b/bpkg/cfg-fetch.cli deleted file mode 100644 index 9cc75fd..0000000 --- a/bpkg/cfg-fetch.cli +++ /dev/null @@ -1,32 +0,0 @@ -// file : bpkg/cfg-fetch.cli -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -include ; - -"\section=1" -"\name=bpkg-cfg-fetch" -"\summary=fetch list of available packages" - -namespace bpkg -{ - { - "", - - "\h|SYNOPSIS| - - \c{\b{bpkg cfg-fetch}|\b{fetch} []} - - \h|DESCRIPTION| - - The \cb{cfg-fetch} command fetches the list of available packages for all - the repositories that were previously added with the \l{bpkg-cfg-add(1)} - command as well as all their complement and prerequisite repositories, - recursively." - } - - class cfg_fetch_options: configuration_options - { - "\h|CFG-FETCH OPTIONS|" - }; -} diff --git a/bpkg/cfg-fetch.cxx b/bpkg/cfg-fetch.cxx deleted file mode 100644 index dfbcc98..0000000 --- a/bpkg/cfg-fetch.cxx +++ /dev/null @@ -1,328 +0,0 @@ -// file : bpkg/cfg-fetch.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include - -#include - -#include - -#include -#include -#include -#include -#include -#include - -using namespace std; -using namespace butl; - -namespace bpkg -{ - static void - cfg_fetch (const configuration_options& co, - transaction& t, - const shared_ptr& r, - const shared_ptr& root, - const string& reason) - { - tracer trace ("cfg_fetch(rep)"); - - database& db (t.database ()); - tracer_guard tg (db, trace); - - const repository_location& rl (r->location); - l4 ([&]{trace << r->name << " " << rl;}); - assert (rl.absolute () || rl.remote ()); - - // The fetch_*() functions below will be quiet at level 1, which - // can be quite confusing if the download hangs. - // - if (verb) - { - diag_record dr (text); - - dr << "fetching " << r->name; - - const auto& ua (root->complements); - - if (ua.find (lazy_shared_ptr (db, r)) == ua.end ()) - { - assert (!reason.empty ()); - dr << " (" << reason << ")"; - } - } - - r->fetched = true; // Mark as being fetched. - - // Load the 'repositories' file and use it to populate the prerequisite - // and complement repository sets. - // - pair rmc ( - fetch_repositories (co, rl, true)); - - repository_manifests& rms (rmc.first); - - bool a (co.auth () != auth::none && - (co.auth () == auth::all || rl.remote ())); - - shared_ptr cert; - - if (a) - { - cert = authenticate_certificate ( - co, &co.directory (), rms.back ().certificate, rl); - - a = !cert->dummy (); - } - - // Load the 'packages' file. - // - pair pmc ( - fetch_packages (co, rl, true)); - - package_manifests& pms (pmc.first); - - if (rmc.second != pms.sha256sum) - fail << "repositories manifest file checksum mismatch for " - << rl.canonical_name () << - info << "try again"; - - if (a) - { - signature_manifest sm (fetch_signature (co, rl, true)); - - if (sm.sha256sum != pmc.second) - fail << "packages manifest file checksum mismatch for " - << rl.canonical_name () << - info << "try again"; - - assert (cert != nullptr); - authenticate_repository (co, &co.directory (), nullopt, *cert, sm, rl); - } - - for (repository_manifest& rm: rms) - { - repository_role rr (rm.effective_role ()); - - if (rr == repository_role::base) - continue; // Entry for this repository. - - // If the location is relative, complete it using this repository - // as a base. - // - if (rm.location.relative ()) - { - try - { - rm.location = repository_location (rm.location, rl); - } - catch (const invalid_argument& e) - { - fail << "invalid relative repository location '" << rm.location - << "': " << e.what () << - info << "base repository location is " << rl; - } - } - - // We might already have this repository in the database. - // - shared_ptr pr ( - db.find ( - rm.location.canonical_name ())); - - if (pr == nullptr) - { - pr = make_shared (move (rm.location)); - db.persist (pr); // Enter into session, important if recursive. - } - - // Load the prerequisite repository unless it has already been - // (or is already being) fetched. - // - if (!pr->fetched) - { - string reason; - switch (rr) - { - case repository_role::complement: reason = "complements "; break; - case repository_role::prerequisite: reason = "prerequisite of "; break; - case repository_role::base: assert (false); - } - reason += r->name; - - cfg_fetch (co, t, pr, root, reason); - } - - // @@ What if we have duplicated? Ideally, we would like to check - // this once and as early as possible. The original idea was to - // do it during manifest parsing and serialization. But at that - // stage we have no way of completing relative locations (which - // is required to calculate canonical names). Current thinking is - // that we should have something like rep-verify (similar to - // pkg-verify) that performs (potentially expensive) repository - // verifications, including making sure prerequisites can be - // satisfied from the listed repositories, etc. Perhaps we can - // also re-use some of that functionality here. I.e., instead of - // calling the "naked" fetch_repositories() above, we will call - // a function from rep-verify that will perform extra verifications. - // - // @@ Also check for self-prerequisite. - // - switch (rr) - { - case repository_role::complement: - { - l4 ([&]{trace << pr->name << " complement of " << r->name;}); - r->complements.insert (lazy_shared_ptr (db, pr)); - break; - } - case repository_role::prerequisite: - { - l4 ([&]{trace << pr->name << " prerequisite of " << r->name;}); - r->prerequisites.insert (lazy_weak_ptr (db, pr)); - break; - } - case repository_role::base: - assert (false); - } - } - - // "Suspend" session while persisting packages to reduce memory - // consumption. - // - session& s (session::current ()); - session::reset_current (); - - for (package_manifest& pm: pms) - { - // We might already have this package in the database. - // - bool persist (false); - - shared_ptr p ( - db.find ( - available_package_id (pm.name, pm.version))); - - if (p == nullptr) - { - p = make_shared (move (pm)); - persist = true; - } - else - { - // Make sure this is the same package. - // - assert (p->sha256sum && !p->locations.empty ()); // Can't be transient. - - if (*pm.sha256sum != *p->sha256sum) - { - // All the previous repositories that contain this package have the - // same checksum (since they passed this test), so we can pick any - // to show to the user. - // - const string& r1 (rl.canonical_name ()); - const string& r2 (p->locations[0].repository.object_id ()); - - fail << "checksum mismatch for " << pm.name << " " << pm.version << - info << r1 << " has " << *pm.sha256sum << - info << r2 << " has " << *p->sha256sum << - info << "consider reporting this to the repository maintainers"; - } - } - - // This repository shouldn't already be in the location set since - // that would mean it has already been loaded and we shouldn't be - // here. - // - p->locations.push_back ( - package_location {lazy_shared_ptr (db, r), - move (*pm.location)}); - - if (persist) - db.persist (p); - else - db.update (p); - } - - session::current (s); // "Resume". - - // Save the changes to the repository object. - // - db.update (r); - } - - int - cfg_fetch (const cfg_fetch_options& o, cli::scanner&) - { - tracer trace ("cfg_fetch"); - - dir_path c (o.directory ()); - l4 ([&]{trace << "configuration: " << c;}); - - database db (open (c, trace)); - transaction t (db.begin ()); - session s; // Repository dependencies can have cycles. - - shared_ptr root (db.load ("")); - const auto& ua (root->complements); // User-added repositories. - - if (ua.empty ()) - fail << "configuration " << c << " has no repositories" << - info << "use 'bpkg cfg-add' to add a repository"; - - // Clean repositories and available packages. At the end only - // repositories that were explicitly added by the user and the - // special root repository should remain. - // - db.erase_query (); - - for (shared_ptr r: pointer_result (db.query ())) - { - if (r == root) - { - l5 ([&]{trace << "skipping root";}); - } - else if (ua.find (lazy_shared_ptr (db, r)) != ua.end ()) - { - l4 ([&]{trace << "cleaning " << r->name;}); - - r->complements.clear (); - r->prerequisites.clear (); - r->fetched = false; - db.update (r); - } - else - { - l4 ([&]{trace << "erasing " << r->name;}); - db.erase (r); - } - } - - // Now recursively fetch prerequisite/complement repositories and - // their packages. - // - for (const lazy_shared_ptr& lp: ua) - { - shared_ptr r (lp.load ()); - - if (!r->fetched) // Can already be loaded as a prerequisite/complement. - cfg_fetch (o, t, r, root, ""); // No reason (user-added). - } - - size_t rcount, pcount; - if (verb) - { - rcount = db.query_value (); - pcount = db.query_value (); - } - - t.commit (); - - if (verb) - text << pcount << " package(s) in " << rcount << " repository(s)"; - - return 0; - } -} diff --git a/bpkg/package b/bpkg/package index 2bfc575..42cf2af 100644 --- a/bpkg/package +++ b/bpkg/package @@ -435,7 +435,7 @@ namespace bpkg // Repository from which this package came. Note that it is not // a pointer to the repository object because it could be wiped - // out (e.g., as a result of cfg-fetch). We call such packages + // out (e.g., as a result of rep-fetch). We call such packages // "orphans". While we can get a list of orphan's prerequisites // (by loading its manifest), we wouldn't know which repository // to use as a base to resolve them. As a result, an orphan that diff --git a/bpkg/pkg-build.cli b/bpkg/pkg-build.cli index e0ec9b1..222519b 100644 --- a/bpkg/pkg-build.cli +++ b/bpkg/pkg-build.cli @@ -25,8 +25,8 @@ namespace bpkg Each package can be specified as just the name () with optional package version () in which case the package will be automatically - fetched from one of the repositories. See the \l{bpkg-cfg-add(1)} and - \l{bpkg-cfg-fetch(1)} commands for more information on package + fetched from one of the repositories. See the \l{bpkg-rep-add(1)} and + \l{bpkg-rep-fetch(1)} commands for more information on package repositories. If is not specified, then the latest available version will be built. To downgrade, the desired version must be specified explicitly. diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index efb708b..21822f7 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -1042,10 +1042,10 @@ namespace bpkg // if (db.query_value () == 0) dr << info << "configuration " << c << " has no repositories" - << info << "use 'bpkg cfg-add' to add a repository"; + << info << "use 'bpkg rep-add' to add a repository"; else if (db.query_value () == 0) dr << info << "configuration " << c << " has no available packages" - << info << "use 'bpkg cfg-fetch' to fetch available packages " + << info << "use 'bpkg rep-fetch' to fetch available packages " << "list"; } diff --git a/bpkg/pkg-fetch.cli b/bpkg/pkg-fetch.cli index 562daab..cefb74e 100644 --- a/bpkg/pkg-fetch.cli +++ b/bpkg/pkg-fetch.cli @@ -20,7 +20,7 @@ namespace bpkg \h|DESCRIPTION| The \cb{pkg-fetch} command fetches the archive for the specified package - name and version from one of the repositories (\l{bpkg-cfg-add(1)}). If + name and version from one of the repositories (\l{bpkg-rep-add(1)}). If the \cb{--replace|-r} option is specified, then \cb{pkg-fetch} will replace the archive of a package that is already in the \cb{fetched} or \cb{unpacked} state (\l{bpkg-pkg-status(1)}). Otherwise, \cb{pkg-fetch} diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx index 8aba4be..0534e65 100644 --- a/bpkg/pkg-fetch.cxx +++ b/bpkg/pkg-fetch.cxx @@ -178,11 +178,11 @@ namespace bpkg if (db.query_value () == 0) fail << "configuration " << c << " has no repositories" << - info << "use 'bpkg cfg-add' to add a repository"; + info << "use 'bpkg rep-add' to add a repository"; if (db.query_value () == 0) fail << "configuration " << c << " has no available packages" << - info << "use 'bpkg cfg-fetch' to fetch available packages list"; + info << "use 'bpkg rep-fetch' to fetch available packages list"; // Note that here we compare including the revision (unlike, say in // pkg-status). Which means one cannot just specify 1.0.0 and get 1.0.0+1 diff --git a/bpkg/rep-add b/bpkg/rep-add new file mode 100644 index 0000000..7dcb9b0 --- /dev/null +++ b/bpkg/rep-add @@ -0,0 +1,19 @@ +// file : bpkg/rep-add -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BPKG_REP_ADD +#define BPKG_REP_ADD + +#include +#include + +#include + +namespace bpkg +{ + int + rep_add (const rep_add_options&, cli::scanner& args); +} + +#endif // BPKG_REP_ADD diff --git a/bpkg/rep-add.cli b/bpkg/rep-add.cli new file mode 100644 index 0000000..c1130c0 --- /dev/null +++ b/bpkg/rep-add.cli @@ -0,0 +1,32 @@ +// file : bpkg/rep-add.cli +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +include ; + +"\section=1" +"\name=bpkg-rep-add" +"\summary=add repository to configuration" + +namespace bpkg +{ + { + " ", + + "\h|SYNOPSIS| + + \c{\b{bpkg rep-add}|\b{add} [] } + + \h|DESCRIPTION| + + The \cb{rep-add} command adds the specified package repository to the + configuration. Note that it doesn't fetch the list of available packages + for the newly added repository. For that, use the \l{bpkg-rep-fetch(1)} + command, normally, after adding all the repositories you wish to use." + } + + class rep_add_options: configuration_options + { + "\h|REP-ADD OPTIONS|" + }; +} diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx new file mode 100644 index 0000000..da5bd4b --- /dev/null +++ b/bpkg/rep-add.cxx @@ -0,0 +1,66 @@ +// file : bpkg/rep-add.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include +#include +#include +#include + +using namespace std; +using namespace butl; + +namespace bpkg +{ + int + rep_add (const rep_add_options& o, cli::scanner& args) + { + tracer trace ("rep_add"); + + dir_path c (o.directory ()); + l4 ([&]{trace << "configuration: " << c;}); + + if (!args.more ()) + fail << "repository location argument expected" << + info << "run 'bpkg help rep-add' for more information"; + + repository_location rl (parse_location (args.next ())); + const string& rn (rl.canonical_name ()); + + // Create the new repository and add is as a complement to the root. + // + database db (open (c, trace)); + transaction t (db.begin ()); + session s; // Repository dependencies can have cycles. + + // It is possible that this repository is already in the database. + // For example, it might be a prerequisite of one of the already + // added repository. + // + shared_ptr r (db.find (rl.canonical_name ())); + + if (r == nullptr) + { + r.reset (new repository (rl)); + db.persist (r); + } + + shared_ptr root (db.load ("")); + + if (!root->complements.insert (lazy_shared_ptr (db, r)).second) + { + fail << rn << " is already a repository of this configuration"; + } + + db.update (root); + t.commit (); + + if (verb) + text << "added repository " << rn; + + return 0; + } +} diff --git a/bpkg/rep-fetch b/bpkg/rep-fetch new file mode 100644 index 0000000..2adf59a --- /dev/null +++ b/bpkg/rep-fetch @@ -0,0 +1,19 @@ +// file : bpkg/rep-fetch -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BPKG_REP_FETCH +#define BPKG_REP_FETCH + +#include +#include + +#include + +namespace bpkg +{ + int + rep_fetch (const rep_fetch_options&, cli::scanner& args); +} + +#endif // BPKG_REP_FETCH diff --git a/bpkg/rep-fetch.cli b/bpkg/rep-fetch.cli new file mode 100644 index 0000000..cada453 --- /dev/null +++ b/bpkg/rep-fetch.cli @@ -0,0 +1,32 @@ +// file : bpkg/rep-fetch.cli +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +include ; + +"\section=1" +"\name=bpkg-rep-fetch" +"\summary=fetch list of available packages" + +namespace bpkg +{ + { + "", + + "\h|SYNOPSIS| + + \c{\b{bpkg rep-fetch}|\b{fetch} []} + + \h|DESCRIPTION| + + The \cb{rep-fetch} command fetches the list of available packages for all + the repositories that were previously added with the \l{bpkg-rep-add(1)} + command as well as all their complement and prerequisite repositories, + recursively." + } + + class rep_fetch_options: configuration_options + { + "\h|REP-FETCH OPTIONS|" + }; +} diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx new file mode 100644 index 0000000..fb22986 --- /dev/null +++ b/bpkg/rep-fetch.cxx @@ -0,0 +1,328 @@ +// file : bpkg/rep-fetch.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace butl; + +namespace bpkg +{ + static void + rep_fetch (const configuration_options& co, + transaction& t, + const shared_ptr& r, + const shared_ptr& root, + const string& reason) + { + tracer trace ("rep_fetch(rep)"); + + database& db (t.database ()); + tracer_guard tg (db, trace); + + const repository_location& rl (r->location); + l4 ([&]{trace << r->name << " " << rl;}); + assert (rl.absolute () || rl.remote ()); + + // The fetch_*() functions below will be quiet at level 1, which + // can be quite confusing if the download hangs. + // + if (verb) + { + diag_record dr (text); + + dr << "fetching " << r->name; + + const auto& ua (root->complements); + + if (ua.find (lazy_shared_ptr (db, r)) == ua.end ()) + { + assert (!reason.empty ()); + dr << " (" << reason << ")"; + } + } + + r->fetched = true; // Mark as being fetched. + + // Load the 'repositories' file and use it to populate the prerequisite + // and complement repository sets. + // + pair rmc ( + fetch_repositories (co, rl, true)); + + repository_manifests& rms (rmc.first); + + bool a (co.auth () != auth::none && + (co.auth () == auth::all || rl.remote ())); + + shared_ptr cert; + + if (a) + { + cert = authenticate_certificate ( + co, &co.directory (), rms.back ().certificate, rl); + + a = !cert->dummy (); + } + + // Load the 'packages' file. + // + pair pmc ( + fetch_packages (co, rl, true)); + + package_manifests& pms (pmc.first); + + if (rmc.second != pms.sha256sum) + fail << "repositories manifest file checksum mismatch for " + << rl.canonical_name () << + info << "try again"; + + if (a) + { + signature_manifest sm (fetch_signature (co, rl, true)); + + if (sm.sha256sum != pmc.second) + fail << "packages manifest file checksum mismatch for " + << rl.canonical_name () << + info << "try again"; + + assert (cert != nullptr); + authenticate_repository (co, &co.directory (), nullopt, *cert, sm, rl); + } + + for (repository_manifest& rm: rms) + { + repository_role rr (rm.effective_role ()); + + if (rr == repository_role::base) + continue; // Entry for this repository. + + // If the location is relative, complete it using this repository + // as a base. + // + if (rm.location.relative ()) + { + try + { + rm.location = repository_location (rm.location, rl); + } + catch (const invalid_argument& e) + { + fail << "invalid relative repository location '" << rm.location + << "': " << e.what () << + info << "base repository location is " << rl; + } + } + + // We might already have this repository in the database. + // + shared_ptr pr ( + db.find ( + rm.location.canonical_name ())); + + if (pr == nullptr) + { + pr = make_shared (move (rm.location)); + db.persist (pr); // Enter into session, important if recursive. + } + + // Load the prerequisite repository unless it has already been + // (or is already being) fetched. + // + if (!pr->fetched) + { + string reason; + switch (rr) + { + case repository_role::complement: reason = "complements "; break; + case repository_role::prerequisite: reason = "prerequisite of "; break; + case repository_role::base: assert (false); + } + reason += r->name; + + rep_fetch (co, t, pr, root, reason); + } + + // @@ What if we have duplicated? Ideally, we would like to check + // this once and as early as possible. The original idea was to + // do it during manifest parsing and serialization. But at that + // stage we have no way of completing relative locations (which + // is required to calculate canonical names). Current thinking is + // that we should have something like rep-verify (similar to + // pkg-verify) that performs (potentially expensive) repository + // verifications, including making sure prerequisites can be + // satisfied from the listed repositories, etc. Perhaps we can + // also re-use some of that functionality here. I.e., instead of + // calling the "naked" fetch_repositories() above, we will call + // a function from rep-verify that will perform extra verifications. + // + // @@ Also check for self-prerequisite. + // + switch (rr) + { + case repository_role::complement: + { + l4 ([&]{trace << pr->name << " complement of " << r->name;}); + r->complements.insert (lazy_shared_ptr (db, pr)); + break; + } + case repository_role::prerequisite: + { + l4 ([&]{trace << pr->name << " prerequisite of " << r->name;}); + r->prerequisites.insert (lazy_weak_ptr (db, pr)); + break; + } + case repository_role::base: + assert (false); + } + } + + // "Suspend" session while persisting packages to reduce memory + // consumption. + // + session& s (session::current ()); + session::reset_current (); + + for (package_manifest& pm: pms) + { + // We might already have this package in the database. + // + bool persist (false); + + shared_ptr p ( + db.find ( + available_package_id (pm.name, pm.version))); + + if (p == nullptr) + { + p = make_shared (move (pm)); + persist = true; + } + else + { + // Make sure this is the same package. + // + assert (p->sha256sum && !p->locations.empty ()); // Can't be transient. + + if (*pm.sha256sum != *p->sha256sum) + { + // All the previous repositories that contain this package have the + // same checksum (since they passed this test), so we can pick any + // to show to the user. + // + const string& r1 (rl.canonical_name ()); + const string& r2 (p->locations[0].repository.object_id ()); + + fail << "checksum mismatch for " << pm.name << " " << pm.version << + info << r1 << " has " << *pm.sha256sum << + info << r2 << " has " << *p->sha256sum << + info << "consider reporting this to the repository maintainers"; + } + } + + // This repository shouldn't already be in the location set since + // that would mean it has already been loaded and we shouldn't be + // here. + // + p->locations.push_back ( + package_location {lazy_shared_ptr (db, r), + move (*pm.location)}); + + if (persist) + db.persist (p); + else + db.update (p); + } + + session::current (s); // "Resume". + + // Save the changes to the repository object. + // + db.update (r); + } + + int + rep_fetch (const rep_fetch_options& o, cli::scanner&) + { + tracer trace ("rep_fetch"); + + dir_path c (o.directory ()); + l4 ([&]{trace << "configuration: " << c;}); + + database db (open (c, trace)); + transaction t (db.begin ()); + session s; // Repository dependencies can have cycles. + + shared_ptr root (db.load ("")); + const auto& ua (root->complements); // User-added repositories. + + if (ua.empty ()) + fail << "configuration " << c << " has no repositories" << + info << "use 'bpkg rep-add' to add a repository"; + + // Clean repositories and available packages. At the end only + // repositories that were explicitly added by the user and the + // special root repository should remain. + // + db.erase_query (); + + for (shared_ptr r: pointer_result (db.query ())) + { + if (r == root) + { + l5 ([&]{trace << "skipping root";}); + } + else if (ua.find (lazy_shared_ptr (db, r)) != ua.end ()) + { + l4 ([&]{trace << "cleaning " << r->name;}); + + r->complements.clear (); + r->prerequisites.clear (); + r->fetched = false; + db.update (r); + } + else + { + l4 ([&]{trace << "erasing " << r->name;}); + db.erase (r); + } + } + + // Now recursively fetch prerequisite/complement repositories and + // their packages. + // + for (const lazy_shared_ptr& lp: ua) + { + shared_ptr r (lp.load ()); + + if (!r->fetched) // Can already be loaded as a prerequisite/complement. + rep_fetch (o, t, r, root, ""); // No reason (user-added). + } + + size_t rcount, pcount; + if (verb) + { + rcount = db.query_value (); + pcount = db.query_value (); + } + + t.commit (); + + if (verb) + text << pcount << " package(s) in " << rcount << " repository(s)"; + + return 0; + } +} diff --git a/doc/buildfile b/doc/buildfile index 189eb83..ad5b9b9 100644 --- a/doc/buildfile +++ b/doc/buildfile @@ -3,9 +3,7 @@ # license : MIT; see accompanying LICENSE file cmds = \ -bpkg-cfg-add \ bpkg-cfg-create \ -bpkg-cfg-fetch \ bpkg-help \ bpkg-pkg-build \ bpkg-pkg-clean \ @@ -21,7 +19,9 @@ bpkg-pkg-uninstall \ bpkg-pkg-unpack \ bpkg-pkg-update \ bpkg-pkg-verify \ +bpkg-rep-add \ bpkg-rep-create \ +bpkg-rep-fetch \ bpkg-rep-info define css: file diff --git a/doc/cli.sh b/doc/cli.sh index 07f56e4..a5c5fe0 100755 --- a/doc/cli.sh +++ b/doc/cli.sh @@ -54,9 +54,9 @@ o="--output-prefix bpkg- --class-doc bpkg::common_options=short" compile "common" $o --output-suffix "-options" --class-doc bpkg::common_options=long compile "bpkg" $o --output-prefix "" --suppress-undocumented --class-doc bpkg::commands=short --class-doc bpkg::topics=short -pages="cfg-add cfg-create cfg-fetch help pkg-build pkg-clean pkg-configure \ -pkg-disfigure pkg-drop pkg-fetch pkg-install pkg-purge pkg-status \ -pkg-test pkg-uninstall pkg-unpack pkg-update pkg-verify rep-create rep-info" +pages="cfg-create help pkg-build pkg-clean pkg-configure pkg-disfigure \ +pkg-drop pkg-fetch pkg-install pkg-purge pkg-status pkg-test pkg-uninstall \ +pkg-unpack pkg-update pkg-verify rep-add rep-create rep-fetch rep-info" for p in $pages; do compile $p $o diff --git a/tests/test.sh b/tests/test.sh index de40ed8..b630bda 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -114,7 +114,7 @@ function test () ops="-d $cfg" fi - if [ "$cmd" = "cfg-fetch" -o \ + if [ "$cmd" = "rep-fetch" -o \ "$cmd" = "rep-info" ]; then ops="$ops --auth all" fi @@ -142,7 +142,7 @@ function fail () ops="-d $cfg" fi - if [ "$cmd" = "cfg-fetch" -o \ + if [ "$cmd" = "rep-fetch" -o \ "$cmd" = "rep-info" ]; then ops="$ops --auth all" fi @@ -328,58 +328,58 @@ stat libfoo unknown ## -## cfg-add +## rep-add ## test cfg-create --wipe -fail cfg-add # repository location expected -fail cfg-add stable # invalid location -fail cfg-add http:// # invalid location +fail rep-add # repository location expected +fail rep-add stable # invalid location +fail rep-add http:// # invalid location # relative path # -test cfg-add ./1/bar/stable -fail cfg-add ./1/../1/bar/stable # duplicate +test rep-add ./1/bar/stable +fail rep-add ./1/../1/bar/stable # duplicate # absolute path # -test cfg-add /tmp/1/foo/stable -fail cfg-add /tmp/1/../1/foo/stable # duplicate +test rep-add /tmp/1/foo/stable +fail rep-add /tmp/1/../1/foo/stable # duplicate # remote URL # -test cfg-add http://pkg.example.org/1/testing -fail cfg-add https://www.example.org/1/testing # duplicate +test rep-add http://pkg.example.org/1/testing +fail rep-add https://www.example.org/1/testing # duplicate ## -## cfg-fetch +## rep-fetch ## test cfg-create --wipe -fail cfg-fetch # no repositories +fail rep-fetch # no repositories # hello repository # test cfg-create --wipe -test cfg-add $rep/common/hello -test cfg-fetch --trust $hello_fp -test cfg-fetch +test rep-add $rep/common/hello +test rep-fetch --trust $hello_fp +test rep-fetch # bar/unstable repository # test cfg-create --wipe -test cfg-add $rep/common/bar/unstable -test cfg-fetch --trust-yes -test cfg-fetch +test rep-add $rep/common/bar/unstable +test rep-fetch --trust-yes +test rep-fetch # both # test cfg-create --wipe -test cfg-add $rep/common/hello -test cfg-add $rep/common/bar/unstable -test cfg-fetch --trust-yes -test cfg-fetch +test rep-add $rep/common/hello +test rep-add $rep/common/bar/unstable +test rep-fetch --trust-yes +test rep-fetch ## @@ -396,13 +396,13 @@ fail pkg-fetch libfoo # package version expected fail pkg-fetch libfoo/1/2/3 # invalid package version fail pkg-fetch libfoo/1.0.0 # no repositories -test cfg-add $rep/fetch/t1 +test rep-add $rep/fetch/t1 fail pkg-fetch libfoo/1.0.0 # no packages -test cfg-fetch --trust-yes +test rep-fetch --trust-yes fail pkg-fetch libfoo/2+1.0.0 # not available test cfg-create --wipe -test cfg-add $rep/fetch/t1 -test cfg-fetch --trust-yes +test rep-add $rep/fetch/t1 +test rep-fetch --trust-yes test pkg-fetch libfoo/1.0.0 stat libfoo/1.0.0 fetched fail pkg-fetch libfoo/1.0.0 @@ -425,8 +425,8 @@ test pkg-purge libfoo # hello # test cfg-create --wipe -test cfg-add $rep/common/hello -test cfg-fetch --trust $hello_fp +test rep-add $rep/common/hello +test rep-fetch --trust $hello_fp test pkg-fetch libhello/1.0.0+1 test pkg-purge libhello @@ -439,8 +439,8 @@ fail pkg-unpack -r # replace only with existing fail pkg-unpack -e # package directory expected fail pkg-unpack # package name expected -test cfg-add $rep/fetch/t1 -test cfg-fetch --trust-yes +test rep-add $rep/fetch/t1 +test rep-fetch --trust-yes # existing # @@ -482,8 +482,8 @@ test pkg-purge libfoo # hello # test cfg-create --wipe -test cfg-add $rep/common/hello -test cfg-fetch --trust $hello_fp +test rep-add $rep/common/hello +test rep-fetch --trust $hello_fp test pkg-fetch libhello/1.0.0+1 test pkg-unpack libhello test pkg-purge libhello @@ -590,8 +590,8 @@ stat libfoo unknown ## pkg-configure/pkg-disfigure ## test cfg-create --wipe -test cfg-add $rep/common/hello -test cfg-fetch --trust $hello_fp +test rep-add $rep/common/hello +test rep-fetch --trust $hello_fp fail pkg-configure # package name expected fail pkg-configure config.dist.root=/tmp # ditto @@ -689,8 +689,8 @@ fi # test rep-create pkg/1/build2.org/depend/stable test cfg-create --wipe -test cfg-add $rep/depend/stable -test cfg-fetch --trust-yes +test rep-add $rep/depend/stable +test rep-fetch --trust-yes test pkg-fetch libbar/1.0.0 test pkg-unpack libbar @@ -766,8 +766,8 @@ test rep-create pkg/1/build2.org/status/unstable test cfg-create --wipe stat libfoo/1.0.0 "unknown" stat libfoo "unknown" -test cfg-add $rep/status/stable -test cfg-fetch --trust-yes +test rep-add $rep/status/stable +test rep-fetch --trust-yes stat libfoo/1.0.0 "available" stat libfoo "available 1.0.0" test pkg-fetch libfoo/1.0.0 @@ -777,21 +777,21 @@ stat libfoo "fetched 1.0.0" # multiple versions/revisions # test cfg-create --wipe -test cfg-add $rep/status/extra -test cfg-fetch --trust-yes +test rep-add $rep/status/extra +test rep-fetch --trust-yes stat libbar "available 1.1.0+1" -test cfg-add $rep/status/stable -test cfg-fetch --trust-yes +test rep-add $rep/status/stable +test rep-fetch --trust-yes stat libbar "available 1.1.0+1 1.0.0" test cfg-create --wipe -test cfg-add $rep/status/testing -test cfg-fetch --trust-yes +test rep-add $rep/status/testing +test rep-fetch --trust-yes stat libbar "available 1.1.0 1.0.0+1 1.0.0" test cfg-create --wipe -test cfg-add $rep/status/unstable -test cfg-fetch --trust-yes +test rep-add $rep/status/unstable +test rep-fetch --trust-yes stat libbar "available 2.0.0 1.1.0 1.0.0+1 1.0.0" test pkg-fetch libbar/1.0.0+1 stat libbar "fetched 1.0.0+1; available 2.0.0 1.1.0" @@ -804,8 +804,8 @@ stat libbar "fetched 2.0.0" ## pkg-update ## test cfg-create --wipe -test cfg-add $rep/common/hello -test cfg-fetch --trust $hello_fp +test rep-add $rep/common/hello +test rep-fetch --trust $hello_fp fail pkg-update # package name expected fail pkg-update libhello # no such package @@ -838,8 +838,8 @@ test pkg-purge libhello ## pkg-clean ## test cfg-create --wipe -test cfg-add $rep/common/hello -test cfg-fetch --trust $hello_fp +test rep-add $rep/common/hello +test rep-fetch --trust $hello_fp fail pkg-clean # package name expected fail pkg-clean libhello # no such package @@ -878,8 +878,8 @@ test pkg-purge libhello # build and clean package # test cfg-create --wipe cxx -test cfg-add $rep/common/hello -test cfg-fetch --trust $hello_fp +test rep-add $rep/common/hello +test rep-fetch --trust $hello_fp test pkg-fetch libhello/1.0.0+1 test pkg-unpack libhello test pkg-configure libhello @@ -917,8 +917,8 @@ test pkg-build -p libfoo/1.1.0 libfoo/1.1.0 <<< "build libfoo 1.1.0" fail pkg-build -p libfoo/1.0.0 test pkg-purge libfoo -test cfg-add $rep/satisfy/t1 -test cfg-fetch --trust-yes +test rep-add $rep/satisfy/t1 +test rep-fetch --trust-yes test pkg-build -p libfoo <<< "build libfoo 1.0.0" test pkg-build -p libfoo/1.0.0 <<< "build libfoo 1.0.0" test pkg-build -p libfoo libfoo <<< "build libfoo 1.0.0" @@ -947,8 +947,8 @@ test cfg-create --wipe fail pkg-build pkg/1/build2.org/satisfy/libbar-1.0.0.tar.gz -test cfg-add $rep/satisfy/t2 -test cfg-fetch --trust-yes +test rep-add $rep/satisfy/t2 +test rep-fetch --trust-yes test pkg-build -p libbar <