From 7e4b2dcd0e5ddd37276879e699fd84059183f5e2 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Tue, 6 Mar 2018 23:52:11 +0300 Subject: Add support for dir repository --- bpkg/bpkg.cli | 6 +- bpkg/manifest-utility.cxx | 4 +- bpkg/pkg-build.cxx | 111 ++++++++++---- bpkg/pkg-checkout.hxx | 3 +- bpkg/pkg-fetch.cxx | 7 +- bpkg/pkg-fetch.hxx | 3 +- bpkg/pkg-unpack.cli | 26 ++-- bpkg/pkg-unpack.cxx | 226 +++++++++++++++++++++------- bpkg/pkg-unpack.hxx | 12 ++ bpkg/rep-add.cli | 57 +++++--- bpkg/rep-fetch.cxx | 365 ++++++++++++++++++++++++++++++---------------- bpkg/rep-info.cli | 4 +- doc/manual.cli | 31 ++-- tests/pkg-build.test | 18 ++- tests/pkg-checkout.test | 2 +- tests/pkg-fetch.test | 8 +- tests/pkg-status.test | 2 +- tests/pkg-unpack.test | 74 +++++++++- tests/rep-fetch.test | 33 ++++- tests/rep-info.test | 2 +- tests/rep-list.test | 2 +- tests/rep-remove.test | 2 +- 22 files changed, 719 insertions(+), 279 deletions(-) diff --git a/bpkg/bpkg.cli b/bpkg/bpkg.cli index c19138b..1e3697c 100644 --- a/bpkg/bpkg.cli +++ b/bpkg/bpkg.cli @@ -57,9 +57,9 @@ namespace bpkg archives from repositories. A \i{bpkg repository} is a collection of packages as well as prerequisite - and complement repositories. Both \i{archive} and \i{version - control}-based repository types are supported. A repository is identified - by its location which can be a local filesystem path or a URL. + and complement repositories. \i{Archive}, \i{directory} and \i{version + control}-based repositories are supported. A repository is identified by + its location which can be a local filesystem path or a URL. A typical \cb{bpkg} workflow would consist of the following steps. diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx index 9f4a012..a2b817d 100644 --- a/bpkg/manifest-utility.cxx +++ b/bpkg/manifest-utility.cxx @@ -137,7 +137,9 @@ namespace bpkg { switch (l.type ()) { - case repository_type::pkg: return dir_path (); // No state. + case repository_type::pkg: + case repository_type::dir: return dir_path (); // No state. + case repository_type::git: { return dir_path (sha256 (l.canonical_name ()).abbreviated_string (16)); diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index 49b027b..3a301c8 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -616,9 +616,8 @@ namespace bpkg if (dap == nullptr) { - diag_record dr; - dr << fail << "unknown prerequisite " << d << " of package " - << name; + diag_record dr (fail); + dr << "unknown prerequisite " << d << " of package " << name; if (!ar->location.empty ()) dr << info << "repository " << ar->location << " appears to " @@ -947,9 +946,9 @@ namespace bpkg if (!satisfies (av, c)) { - diag_record dr; + diag_record dr (fail); - dr << fail << "unable to " << (ud < 0 ? "up" : "down") << "grade " + dr << "unable to " << (ud < 0 ? "up" : "down") << "grade " << "package " << *sp << " to "; // Print both (old and new) package names in full if the system @@ -1580,8 +1579,7 @@ namespace bpkg if (!found) { - diag_record dr; - dr << fail; + diag_record dr (fail); if (!sys_advise) { @@ -1952,44 +1950,64 @@ namespace bpkg if (pl.repository.object_id () != "") // Special root? { - // Go through package repositories to decide if we should fetch or - // checkout. Preferring a local one over the remotes seems like a - // sensible thing to do. + // Go through package repositories to decide if we should fetch, + // checkout or unpack depending on the available repository basis. + // Preferring a local one over the remotes seems like a sensible + // thing to do. // - optional fetch; + optional basis; for (const package_location& l: ap->locations) { const repository_location& rl (l.repository.load ()->location); - if (!fetch || rl.local ()) // First or local? + if (!basis || rl.local ()) // First or local? { - fetch = rl.archive_based (); + basis = rl.basis (); if (rl.local ()) break; } } - assert (fetch); + assert (basis); transaction t (db.begin ()); - // Both calls commit the transaction. + // All calls commit the transaction. // - sp = *fetch - ? pkg_fetch (o, - c, - t, - ap->id.name, - p.available_version (), - true /* replace */) - : pkg_checkout (o, - c, - t, - ap->id.name, - p.available_version (), - true /* replace */); + switch (*basis) + { + case repository_basis::archive: + { + sp = pkg_fetch (o, + c, + t, + ap->id.name, + p.available_version (), + true /* replace */); + break; + } + case repository_basis::version_control: + { + sp = pkg_checkout (o, + c, + t, + ap->id.name, + p.available_version (), + true /* replace */); + break; + } + case repository_basis::directory: + { + sp = pkg_unpack (c, + t, + ap->id.name, + p.available_version (), + true /* replace */); + break; + } + } } // Directory case is handled by unpack. // @@ -2008,12 +2026,41 @@ namespace bpkg if (sp != nullptr) // Actually fetched or checked out something? { assert (sp->state == package_state::fetched || - sp->state == package_state::unpacked); // Checked out. + sp->state == package_state::unpacked); if (verb) - text << (sp->state == package_state::fetched - ? "fetched " - : "checked out ") << *sp; + { + const repository_location& rl (sp->repository); + + repository_basis basis ( + !rl.empty () + ? rl.basis () + : repository_basis::archive); // Archive path case. + + diag_record dr (text); + + switch (basis) + { + case repository_basis::archive: + { + assert (sp->state == package_state::fetched); + dr << "fetched " << *sp; + break; + } + case repository_basis::directory: + { + assert (sp->state == package_state::unpacked); + dr << "using " << *sp << " (external)"; + break; + } + case repository_basis::version_control: + { + assert (sp->state == package_state::unpacked); + dr << "checked out " << *sp; + break; + } + } + } } } diff --git a/bpkg/pkg-checkout.hxx b/bpkg/pkg-checkout.hxx index 252c8cd..b217100 100644 --- a/bpkg/pkg-checkout.hxx +++ b/bpkg/pkg-checkout.hxx @@ -18,7 +18,8 @@ namespace bpkg int pkg_checkout (const pkg_checkout_options&, cli::scanner& args); - // Check out the package from a repository and commit the transaction. + // Check out the package from a version control-based repository and commit + // the transaction. // shared_ptr pkg_checkout (const common_options&, diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx index 1dd9f2e..ff04411 100644 --- a/bpkg/pkg-fetch.cxx +++ b/bpkg/pkg-fetch.cxx @@ -297,7 +297,12 @@ namespace bpkg } if (verb) - text << "fetched " << *p; + { + if (!o.existing ()) + text << "fetched " << *p; + else + text << "using " << *p << " (external)"; + } return 0; } diff --git a/bpkg/pkg-fetch.hxx b/bpkg/pkg-fetch.hxx index d17dbdf..fe51ba6 100644 --- a/bpkg/pkg-fetch.hxx +++ b/bpkg/pkg-fetch.hxx @@ -28,7 +28,8 @@ namespace bpkg bool replace, bool purge); - // Fetch the package from a repository and commit the transaction. + // Fetch the package from an archive-based repository and commit the + // transaction. // shared_ptr pkg_fetch (const common_options&, diff --git a/bpkg/pkg-unpack.cli b/bpkg/pkg-unpack.cli index 44447c5..56682d9 100644 --- a/bpkg/pkg-unpack.cli +++ b/bpkg/pkg-unpack.cli @@ -11,25 +11,31 @@ include ; namespace bpkg { { - " ", + " ", "\h|SYNOPSIS| - \c{\b{bpkg pkg-unpack} [] ( | \b{--existing|-e} )} + \c{\b{bpkg pkg-unpack} [] ([\b{/}] | \b{--existing|-e} )} \h|DESCRIPTION| - The \cb{pkg-unpack} command unpacks the archive for the previously - fetched (\l{bpkg-pkg-fetch(1)}) package. The resulting package state is - \cb{unpacked} (\l{bpkg-pkg-status(1)}). + If only the package name is specified, then the \cb{pkg-unpack} command + unpacks the archive for the previously fetched (\l{bpkg-pkg-fetch(1)}) + package. The resulting package state is \cb{unpacked} + (\l{bpkg-pkg-status(1)}). + + If the package version is also specified, then the (source) directory + from one of the directory-based repositories (\l{bpkg-rep-add(1)}) is + used in place, without copying it into the configuration directory. Such + a package is called \i{external}. If the \cb{--existing|-e} option is used, then instead of the package name, \cb{pkg-unpack} expects a local path to an existing package directory. In this case, \cb{bpkg} will use the (source) directory in - place, without copying it into the configuration directory. Also, unless - the \cb{--purge|-p} option is specified, \cb{bpkg} will not attempt to - remove this directory when the package is later purged with the - \l{bpkg-pkg-purge(1)} command. Such a package is called \i{external}. + place, the same as for packages from directory-based repositories. Also, + unless the \cb{--purge|-p} option is specified, \cb{bpkg} will not + attempt to remove this directory when the package is later purged with + the \l{bpkg-pkg-purge(1)} command. Such a package is also \i{external}. If \cb{--existing|-e} is specified together with the \cb{--replace|-r} option, then \cb{pkg-unpack} will replace the archive and/or source @@ -67,7 +73,7 @@ namespace bpkg bool --replace|-r { "Replace the source directory if the package is already unpacked or - fetched. Can only be specified together with \cb{--existing|-e}." + fetched. Can only be specified with an external package." } }; } diff --git a/bpkg/pkg-unpack.cxx b/bpkg/pkg-unpack.cxx index edd833d..aa3390d 100644 --- a/bpkg/pkg-unpack.cxx +++ b/bpkg/pkg-unpack.cxx @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -25,43 +26,21 @@ using namespace butl; namespace bpkg { - shared_ptr - pkg_unpack (const dir_path& c, - transaction& t, - const dir_path& d, - bool replace, - bool purge) + // Check if the package already exists in this configuration and + // diagnose all the illegal cases. + // + static void + pkg_unpack_check (const dir_path& c, + transaction& t, + const string& n, + bool replace) { - tracer trace ("pkg_unpack"); + tracer trace ("pkg_update_check"); database& db (t.database ()); tracer_guard tg (db, trace); - if (!exists (d)) - fail << "package directory " << d << " does not exist"; - - // Verify the directory is a package and get its manifest. - // - package_manifest m (pkg_verify (d, true)); - l4 ([&]{trace << d << ": " << m.name << " " << m.version;}); - - // Make the package and configuration paths absolute and normalized. - // If the package is inside the configuration, use the relative path. - // This way we can move the configuration around. - // - dir_path ac (c), ad (d); - ac.complete ().normalize (); - ad.complete ().normalize (); - - if (ad.sub (ac)) - ad = ad.leaf (ac); - - // See if this package already exists in this configuration. - // - const string& n (m.name); - shared_ptr p (db.find (n)); - - if (p != nullptr) + if (shared_ptr p = db.find (n)) { bool s (p->state == package_state::fetched || p->state == package_state::unpacked); @@ -78,19 +57,47 @@ namespace bpkg if (s) // Suitable state for replace? dr << info << "use 'pkg-unpack --replace|-r' to replace"; } + } + } + static shared_ptr + pkg_unpack (dir_path c, + transaction& t, + string n, + version v, + dir_path d, + repository_location rl, + bool purge) + { + tracer trace ("pkg_unpack"); + + database& db (t.database ()); + tracer_guard tg (db, trace); + + // Make the package and configuration paths absolute and normalized. + // If the package is inside the configuration, use the relative path. + // This way we can move the configuration around. + // + c.complete ().normalize (); + d.complete ().normalize (); + + if (d.sub (c)) + d = d.leaf (c); + + shared_ptr p (db.find (n)); + + if (p != nullptr) + { // Clean up the source directory and archive of the package we are // replacing. Once this is done, there is no going back. If things // go badly, we can't simply abort the transaction. // pkg_purge_fs (c, t, p); - // Use the special root repository as the repository of this package. - // - p->version = move (m.version); + p->version = move (v); p->state = package_state::unpacked; - p->repository = repository_location (); - p->src_root = move (ad); + p->repository = move (rl); + p->src_root = move (d); p->purge_src = purge; db.update (p); @@ -98,16 +105,16 @@ namespace bpkg else { p.reset (new selected_package { - move (m.name), - move (m.version), + move (n), + move (v), package_state::unpacked, package_substate::none, - false, // hold package - false, // hold version - repository_location (), // Root repository. + false, // hold package + false, // hold version + move (rl), nullopt, // No archive false, // Don't purge archive. - move (ad), + move (d), purge, nullopt, // No output directory yet. {}}); // No prerequisites captured yet. @@ -120,6 +127,109 @@ namespace bpkg } shared_ptr + pkg_unpack (const dir_path& c, + transaction& t, + const dir_path& d, + bool replace, + bool purge) + { + tracer trace ("pkg_unpack"); + + if (!exists (d)) + fail << "package directory " << d << " does not exist"; + + // Verify the directory is a package and get its manifest. + // + package_manifest m (pkg_verify (d, true)); + l4 ([&]{trace << d << ": " << m.name << " " << m.version;}); + + // Check/diagnose an already existing package. + // + pkg_unpack_check (c, t, m.name, replace); + + // Use the special root repository as the repository of this + // package. + // + return pkg_unpack (c, + t, + move (m.name), + move (m.version), + d, + repository_location (), + purge); + } + + shared_ptr + pkg_unpack (const dir_path& c, + transaction& t, + string n, + version v, + bool replace) + { + tracer trace ("pkg_unpack"); + + database& db (t.database ()); + tracer_guard tg (db, trace); + + // Check/diagnose an already existing package. + // + pkg_unpack_check (c, t, n, replace); + + if (db.query_value () == 0) + fail << "configuration " << c << " has no repositories" << + info << "use 'bpkg rep-add' to add a repository"; + + if (db.query_value () == 0) + fail << "configuration " << c << " has no available packages" << + info << "use 'bpkg rep-fetch' to fetch available packages list"; + + // Note that here we compare including the revision (see pkg-fetch() + // implementation for more details). + // + shared_ptr ap ( + db.find (available_package_id (n, v))); + + if (ap == nullptr) + fail << "package " << n << " " << v << " is not available"; + + // Pick a directory-based repository. Preferring a local one over the + // remotes seems like a sensible thing to do. + // + const package_location* pl (nullptr); + + for (const package_location& l: ap->locations) + { + const repository_location& rl (l.repository.load ()->location); + + if (rl.directory_based () && (pl == nullptr || rl.local ())) + { + pl = &l; + + if (rl.local ()) + break; + } + } + + if (pl == nullptr) + fail << "package " << n << " " << v + << " is not available from a directory-based repository"; + + if (verb > 1) + text << "unpacking " << pl->location.leaf () << " " + << "from " << pl->repository->name; + + const repository_location& rl (pl->repository->location); + + return pkg_unpack (c, + t, + move (n), + move (v), + path_cast (rl.path () / pl->location), + rl, + false); // Purge. + } + + shared_ptr pkg_unpack (const common_options& co, const dir_path& c, transaction& t, @@ -287,9 +397,6 @@ namespace bpkg { tracer trace ("pkg_unpack"); - if (o.replace () && !o.existing ()) - fail << "--replace|-r can only be specified with --existing|-e"; - const dir_path& c (o.directory ()); l4 ([&]{trace << "configuration: " << c;}); @@ -297,6 +404,7 @@ namespace bpkg transaction t (db.begin ()); shared_ptr p; + bool external (o.existing ()); if (o.existing ()) { @@ -311,17 +419,37 @@ namespace bpkg } else { - // The package name case. + // The package name[/version] case. // if (!args.more ()) fail << "package name argument expected" << info << "run 'bpkg help pkg-unpack' for more information"; - p = pkg_unpack (o, c, t, args.next ()); + const char* arg (args.next ()); + string n (parse_package_name (arg)); + version v (parse_package_version (arg)); + + external = !v.empty (); + + if (o.replace () && !external) + fail << "--replace|-r can only be specified with external package"; + + // If the package version is not specified then we expect the package to + // already be fetched and so unpack it from the archive. Otherwise, we + // "unpack" it from the directory-based repository. + // + p = v.empty () + ? pkg_unpack (o, c, t, n) + : pkg_unpack (c, t, move (n), move (v), o.replace ()); } if (verb) - text << "unpacked " << *p; + { + if (!external) + text << "unpacked " << *p; + else + text << "using " << *p << " (external)"; + } return 0; } diff --git a/bpkg/pkg-unpack.hxx b/bpkg/pkg-unpack.hxx index 39b4e83..9fdc848 100644 --- a/bpkg/pkg-unpack.hxx +++ b/bpkg/pkg-unpack.hxx @@ -5,6 +5,8 @@ #ifndef BPKG_PKG_UNPACK_HXX #define BPKG_PKG_UNPACK_HXX +#include // version + #include #include // transaction, selected_package #include @@ -32,6 +34,16 @@ namespace bpkg const dir_path& configuration, transaction&, const string& name); + + // Unpack the package as a source directory from a directory-based + // repository and commit the transaction. + // + shared_ptr + pkg_unpack (const dir_path& configuration, + transaction&, + string name, + version, + bool replace); } #endif // BPKG_PKG_UNPACK_HXX diff --git a/bpkg/rep-add.cli b/bpkg/rep-add.cli index 72da5e5..fd76be9 100644 --- a/bpkg/rep-add.cli +++ b/bpkg/rep-add.cli @@ -30,19 +30,40 @@ namespace bpkg the newly added repository. For that, use the \l{bpkg-rep-fetch(1)} command, normally, after adding all the repositories you wish to use. - Currently two types of repositories are supported: \cb{pkg} and \cb{git}. - Normally the repository type can be automatically guessed by examining - its URL (for example, the presence of the \cb{.git} extension) or, in - case of a local repository, its content (for example, the presence of the - \cb{.git/} subdirectory). Without any identifying information the - \cb{pkg} type is assumed unless explicitly specified with the \cb{--type} - option. + Currently three types of repositories are supported: \cb{pkg}, \cb{dir}, + and \cb{git}. Normally the repository type can be automatically guessed + by examining its URL (for example, the presence of the \cb{.git} + extension) or, in case of a local repository, its content (for example, + the presence of the \cb{.git/} subdirectory). Without any identifying + information the \cb{pkg} type is assumed unless explicitly specified with + the \cb{--type} option. Note, however, that the \cb{dir} repository type + is never guessed since it is not easily distinguishable from local + \cb{pkg} and \cb{git} repositories. A \cb{pkg} repository is \i{archive}-based. That is, it contains a collection of various packages/versions as archive files. For more information on the structure of \cb{pkg} repositories refer to the \l{bpkg \cb{bpkg} manual}. + A \cb{dir} repository is \i{directory}-based. That is, it contains a + collection of various packages as directories (but only a single version + per package can be present is such a repository). The \cb{dir} repository + location can be a local directory path or a \cb{file://} URL. + + A \cb{dir} repository is expected to contain either the \cb{manifest} or + \cb{packages.manifest} file in the root directory of the repository. If + it only contains \cb{manifest}, then it is assumed to be a single-package + repository with the \cb{manifest} file being its package manifest. + Otherwise, the \cb{packages.manifest} file should list the available + packages as described in \l{bpkg#manifest-package-list-dir Package List + Manifest for \cb{dir} Repositories}. + + A \cb{dir} repository may also contain the \cb{repositories.manifest} + file in the root directory of the repository. This file can be used to + describe the repository itself as well as specify its prerequisite and + complement repositories. See \l{bpkg#manifest-repository-list Repository + List Manifest} for details on the format and semantics of this file. + A \cb{git} repository is \i{version control}-based. That is, it normally contains multiple versions of the same package (but can also contain several packages in the same repository). @@ -83,19 +104,11 @@ namespace bpkg https://example.com/foo.git#deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@ \ - A \cb{git} repository is expected to contain either the \cb{manifest} or - \cb{packages.manifest} file in the root directory of the repository. If - it only contains \cb{manifest}, then it is assumed to be a single-package - repository with the \cb{manifest} file being its package manifest. - Otherwise the \cb{packages.manifest} file should list the available - packages as described in \l{bpkg#manifest-package-list-git Package List - Manifest for \cb{git} Repositories}. - - A \cb{git} repository may also contain the \cb{repositories.manifest} - file in the root directory of the repository. This file can be used to - describe the repository itself as well as specify its prerequisite and - complement repositories. See \l{bpkg#manifest-repository-list Repository - List Manifest} for details on the format and semantics of this file. + A \cb{git} repository has the same structure and manifest files as the + \cb{dir} repository. See \l{bpkg#manifest-package-list-dir Package List + Manifest for \cb{dir} Repositories} and \l{bpkg#manifest-repository-list + Repository List Manifest} for details on the format and semantics of the + manifest files. Supported git protocols are \cb{git://}, \cb{http://}, and \cb{https://} for remote repositories and \cb{file://} for local repositories. While @@ -138,8 +151,8 @@ namespace bpkg repository_type --type { "", - "Specify the repository type with valid values being \cb{pkg} and - \cb{git}." + "Specify the repository type with valid values being \cb{pkg}, \cb{dir}, + and \cb{git}." } }; } diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index 31e7b11..2e884c4 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -127,119 +127,60 @@ namespace bpkg } } - static rep_fetch_data - rep_fetch_git (const common_options& co, - const dir_path* conf, - const repository_location& rl, - bool ignore_unknown) + // Parse the repositories manifest file if exists. Otherwise return the + // repository manifest list containing the only (trivial) base repository. + // + template + static M + parse_repository_manifests (const path& f, + bool iu, + const repository_location& rl) { - // Plan: - // - // 1. Check repos_dir//: - // - // 1.a If does not exist, git-clone into temp_dir///. - // - // 1.a Otherwise, move as temp_dir// and git-fetch. - // - // 2. Move from temp_dir// to repos_dir/// - // - // 3. Check if repos_dir///repositories.manifest exists: - // - // 3.a If exists, load. - // - // 3.b Otherwise, synthesize repository list with base repository. - // - // 4. Check if repos_dir///packages.manifest exists: - // - // 4.a If exists, load. (into "skeleton" packages list to be filled?) - // - // 4.b Otherwise, synthesize as if single 'location: ./'. - // - // 5. For each package location obtained on step 4: - // - // 5.a Load repos_dir////manifest. - // - // 5.b Run 'b info: repos_dir////' and fix-up - // package version. - // - // 6. Return repository and package manifests (certificate is NULL). - // - - if (conf != nullptr && conf->empty ()) - conf = dir_exists (bpkg_dir) ? ¤t_dir : nullptr; - - assert (conf == nullptr || !conf->empty ()); - - // Clone or fetch the repository. - // - dir_path sd (repository_state (rl)); - - auto_rmdir rm (temp_dir / sd); - dir_path& td (rm.path); - - if (exists (td)) - rm_r (td); - - // If the git repository directory already exists, then we are fetching - // an already cloned repository. Move it to the temporary directory. - // - // In this case also set the filesystem_state_changed flag since we are - // modifying the repository filesystem state. - // - // In the future we can probably do something smarter about the flag, - // keeping it unset unless the repository state directory is really - // changed. - // - dir_path rd; - bool fetch (false); - if (conf != nullptr) - { - rd = *conf / repos_dir / sd; - - if (exists (rd)) - { - mv (rd, td); - filesystem_state_changed = true; - fetch = true; - } - } + M r; + if (exists (f)) + r = parse_manifest (f, iu, rl); + else + r.emplace_back (repository_manifest ()); // Add the base repository. - dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td)); - dir_path fd (td / nm); // Full directory path. + return r; + } - // Produce repository manifest list. - // - git_repository_manifests rms; + // Parse the package directories manifest file if exists. Otherwise treat + // the current directory as a package and return the manifest list with the + // single entry referencing this package. + // + template + static M + parse_directory_manifests (const path& f, + bool iu, + const repository_location& rl) + { + M r; + if (exists (f)) + r = parse_manifest (f, iu, rl); + else { - path f (fd / repositories_file); - - if (exists (f)) - rms = parse_manifest (f, ignore_unknown, rl); - else - rms.emplace_back (repository_manifest ()); // Add the base repository. + r.push_back (package_manifest ()); + r.back ().location = current_dir; } - // Produce the "skeleton" package manifest list. - // - git_package_manifests pms; - { - path f (fd / packages_file); - - if (exists (f)) - pms = parse_manifest (f, ignore_unknown, rl); - else - { - pms.push_back (package_manifest ()); - pms.back ().location = current_dir; - } - } + return r; + } + // Parse package manifests referenced by the package directory manifests. + // + static vector + parse_package_manifests (const common_options& co, + const dir_path& repo_dir, + const string& repo_fragment, + vector&& sms, + bool iu, + const repository_location& rl) + { vector fps; - fps.reserve (pms.size ()); + fps.reserve (sms.size ()); - // Parse package manifests. - // - for (package_manifest& sm: pms) + for (package_manifest& sm: sms) { assert (sm.location); @@ -260,7 +201,7 @@ namespace bpkg package_info (dr); }; - dir_path d (fd / path_cast (*sm.location)); + dir_path d (repo_dir / path_cast (*sm.location)); path f (d / path ("manifest")); if (!exists (f)) @@ -270,7 +211,7 @@ namespace bpkg { ifdstream ifs (f); manifest_parser mp (ifs, f.string ()); - package_manifest m (pkg_package_manifest (mp, ignore_unknown)); + package_manifest m (pkg_package_manifest (mp, iu)); // Save the package manifest, preserving its location. // @@ -290,7 +231,9 @@ namespace bpkg package_info (dr); } - // Fix-up the package version. + // Fix-up the package version. Note that the package may have the + // version module enable and the directory repository may well be a git + // repository. // const char* b (name_b (co)); @@ -361,7 +304,7 @@ namespace bpkg if (pr.wait ()) { fps.emplace_back (rep_fetch_data::package {move (sm), - nm.string ()}); + repo_fragment}); continue; } @@ -387,6 +330,157 @@ namespace bpkg } } + return fps; + } + + static rep_fetch_data + rep_fetch_dir (const common_options& co, + const repository_location& rl, + bool ignore_unknown) + { + assert (rl.absolute ()); + + dir_path rd (path_cast (rl.path ())); + + dir_repository_manifests rms ( + parse_repository_manifests ( + rd / repositories_file, + ignore_unknown, + rl)); + + dir_package_manifests pms ( + parse_directory_manifests ( + rd / packages_file, + ignore_unknown, + rl)); + + vector fps ( + parse_package_manifests (co, + rd, + string () /* repo_fragment */, + move (pms), + ignore_unknown, + rl)); + + // @@ Here we will need to go through packages and check if there is the + // selected package of the same version (without regards to the + // iteration component), and having the different package manifest + // hash. If that's the case then set the manifest version iteration + // component as the selected package version iteation + 1. + // + // @@ It also seems that the manifest hash should be stored in the (new) + // member of the package_location struct. Later this value will be + // transmitted into the selected package objects by pkg_unpack(). + // + // @@ Should we later check and fail if any available package contains + // several package locations with different non-zero (for those that + // come from the directory repo) manifest hashes? + // + return rep_fetch_data {move (rms), move (fps), nullptr}; + } + + static rep_fetch_data + rep_fetch_git (const common_options& co, + const dir_path* conf, + const repository_location& rl, + bool ignore_unknown) + { + // Plan: + // + // 1. Check repos_dir//: + // + // 1.a If does not exist, git-clone into temp_dir///. + // + // 1.a Otherwise, move as temp_dir// and git-fetch. + // + // 2. Move from temp_dir// to repos_dir/// + // + // 3. Check if repos_dir///repositories.manifest exists: + // + // 3.a If exists, load. + // + // 3.b Otherwise, synthesize repository list with base repository. + // + // 4. Check if repos_dir///packages.manifest exists: + // + // 4.a If exists, load. (into "skeleton" packages list to be filled?) + // + // 4.b Otherwise, synthesize as if single 'location: ./'. + // + // 5. For each package location obtained on step 4: + // + // 5.a Load repos_dir////manifest. + // + // 5.b Run 'b info: repos_dir////' and fix-up + // package version. + // + // 6. Return repository and package manifests (certificate is NULL). + // + + if (conf != nullptr && conf->empty ()) + conf = dir_exists (bpkg_dir) ? ¤t_dir : nullptr; + + assert (conf == nullptr || !conf->empty ()); + + // Clone or fetch the repository. + // + dir_path sd (repository_state (rl)); + + auto_rmdir rm (temp_dir / sd); + dir_path& td (rm.path); + + if (exists (td)) + rm_r (td); + + // If the git repository directory already exists, then we are fetching + // an already cloned repository. Move it to the temporary directory. + // + // In this case also set the filesystem_state_changed flag since we are + // modifying the repository filesystem state. + // + // In the future we can probably do something smarter about the flag, + // keeping it unset unless the repository state directory is really + // changed. + // + dir_path rd; + bool fetch (false); + if (conf != nullptr) + { + rd = *conf / repos_dir / sd; + + if (exists (rd)) + { + mv (rd, td); + filesystem_state_changed = true; + fetch = true; + } + } + + dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td)); + dir_path fd (td / nm); // Full directory path. + + // Parse manifests. + // + git_repository_manifests rms ( + parse_repository_manifests ( + fd / repositories_file, + ignore_unknown, + rl)); + + git_package_manifests pms ( + parse_directory_manifests ( + fd / packages_file, + ignore_unknown, + rl)); + + vector fps ( + parse_package_manifests (co, + fd, + nm.string (), + move (pms), + ignore_unknown, + rl)); + // Move the state directory to its proper place. // // If there is no configuration directory then we let auto_rmdir clean it @@ -411,6 +505,7 @@ namespace bpkg switch (rl.type ()) { case repository_type::pkg: return rep_fetch_pkg (co, conf, rl, iu); + case repository_type::dir: return rep_fetch_dir (co, rl, iu); case repository_type::git: return rep_fetch_git (co, conf, rl, iu); } @@ -592,19 +687,33 @@ namespace bpkg } } - // For git repositories that have neither prerequisites nor complements - // we use the root repository as the default complement. + // For dir and git repositories that have neither prerequisites nor + // complements we use the root repository as the default complement. // // This supports the common use case where the user has a single-package - // git repository and doesn't want to bother with the - // repositories.manifest file. This way their package will still pick up - // its dependencies from the configuration, without regards from which - // repositories they came from. + // repository and doesn't want to bother with the repositories.manifest + // file. This way their package will still pick up its dependencies from + // the configuration, without regards from which repositories they came + // from. // - if (rl.type () == repository_type::git && - r->complements.empty () && - r->prerequisites.empty ()) - r->complements.insert (lazy_shared_ptr (db, string ())); + switch (rl.type ()) + { + case repository_type::git: + case repository_type::dir: + { + if (r->complements.empty () && r->prerequisites.empty ()) + r->complements.insert (lazy_shared_ptr (db, string ())); + + break; + } + case repository_type::pkg: + { + // Pkg repository is a "strict" one, that requires all the + // prerequisites and complements to be listed. + // + break; + } + } // Save the changes to the repository object. // @@ -805,20 +914,24 @@ namespace bpkg if (repository_name (a)) { - lazy_shared_ptr rp (db, a); + r = lazy_shared_ptr (db, a); - if (ua.find (rp) != ua.end ()) - r = move (rp); - else + if (ua.find (r) == ua.end ()) fail << "repository '" << a << "' does not exist in this " << "configuration"; } else - //@@ TODO: check if exists in root & same location and avoid - // calling rep_add. Get rid of quiet mode. + { + repository_location rl (parse_location (a, nullopt /* type */)); + r = lazy_shared_ptr (db, rl.canonical_name ()); + + // If the repository is not the root complement yet or has + // a different location then we add it to the configuration. // - r = lazy_shared_ptr ( - db, rep_add (t, parse_location (a, nullopt /* type */))); + auto i (ua.find (r)); + if (i == ua.end () || i->load ()->location.url () != rl.url ()) + r = lazy_shared_ptr (db, rep_add (t, rl)); + } repos.emplace_back (move (r)); } diff --git a/bpkg/rep-info.cli b/bpkg/rep-info.cli index df0eee3..80f1b8e 100644 --- a/bpkg/rep-info.cli +++ b/bpkg/rep-info.cli @@ -96,8 +96,8 @@ namespace bpkg repository_type --type { "", - "Specify the repository type with valid values being \cb{pkg} and - \cb{git}. Refer to \l{bpkg-rep-add(1)} for details." + "Specify the repository type with valid values being \cb{pkg}, + \cb{dir}, and \cb{git}. Refer to \l{bpkg-rep-add(1)} for details." } string --directory|-d // String to allow empty value. diff --git a/doc/manual.cli b/doc/manual.cli index 0143919..74c9be7 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -944,10 +944,10 @@ The SHA256 checksum of the package archive file. The \i{sum} value should be markers), be calculated in the binary mode, and use lower-case letters. -\h#manifest-package-list-git|Package List Manifest for \cb{git} Repositories| +\h#manifest-package-list-dir|Package List Manifest for \cb{dir} Repositories| The package list manifest (the \c{packages.manifest} file found in the -\cb{git} repository root directory) describes the list of packages available +\cb{dir} repository root directory) describes the list of packages available in the repository. It is a (potentially empty) sequence of manifests with the following synopsis: @@ -968,7 +968,7 @@ location: src/libfoo/ location: src/foo/ \ -\h2#manifest-package-list-git-location|\c{location}| +\h2#manifest-package-list-dir-location|\c{location}| \ location: @@ -981,13 +981,13 @@ be in the POSIX representation. \h#manifest-repository|Repository Manifest| The repository manifest (only used as part of the repository manifest list -described below) describes a \cb{pkg} or \cb{git} repository. The manifest -synopsis is presented next followed by the detailed description of each value -in subsequent sections. +described below) describes a \cb{pkg}, \cb{dir}, or \cb{git} repository. The +manifest synopsis is presented next followed by the detailed description of +each value in subsequent sections. \ [location]: -[type]: pkg|git +[type]: pkg|dir|git [role]: base|prerequisite|complement [url]: [email]: [; ] @@ -1004,10 +1004,17 @@ See also the Repository Chaining documentation for further information @@ TODO. [location]: \ -The location of the repository root. It can be an HTTP(S) URL or a filesystem -path. The location can only and must be omitted for the base repository. If -the path is relative, then it is treated as relative to the base repository -location. +The repository location. The location can only and must be omitted for the +base repository. If the location is a relative path, then it is treated as +relative to the base repository location. + +For the \cb{git} repository type the relative location does not inherit the +URL fragment from the base repository. + +For the \cb{dir} repository type the relative location may also contain the +URL fragment to make the same repository information usable in case the base +and prerequisites are also exposed as \cb{git} repositories. For the \cb{dir} +repository type such a fragment is ignored. While POSIX systems normally only support POSIX paths (that is, forward slashes only), Windows is generally able to handle both slash types. As a @@ -1022,7 +1029,7 @@ that is not possible (for example, there is a drive letter in the path).] \h2#manifest-repository-type|\c{type}| \ -[type]: pkg|git +[type]: pkg|dir|git \ The repository type. The type must be omitted for the base repository. If the diff --git a/tests/pkg-build.test b/tests/pkg-build.test index 879fc43..03ed040 100644 --- a/tests/pkg-build.test +++ b/tests/pkg-build.test @@ -1447,7 +1447,23 @@ rep_fetch += -d cfg --auth all --trust-yes 2>! } } -: git-repos +: dir-rep +: +{ + $clone_root_cfg && $rep_add $src/libfoo-1.1.0 --type dir && $rep_fetch; + + $* libfoo 2>>~%EOE%; + using libfoo/1.1.0 (external) + configured libfoo/1.1.0 + %info: .+dir\{libfoo-1.1.0.\} is up to date% + updated libfoo/1.1.0 + EOE + + $pkg_disfigure libfoo 2>'disfigured libfoo/1.1.0'; + $pkg_purge libfoo 2>'purged libfoo/1.1.0' +} + +: git-rep : if ($git_supported != true) { diff --git a/tests/pkg-checkout.test b/tests/pkg-checkout.test index ec750e6..714ad5b 100644 --- a/tests/pkg-checkout.test +++ b/tests/pkg-checkout.test @@ -21,7 +21,7 @@ $git_extract $src/git/style-basic1.tar &$out_git/state1/*** end -: git-repos +: git-rep : if ($git_supported != true) { diff --git a/tests/pkg-fetch.test b/tests/pkg-fetch.test index a63bfbb..919db7d 100644 --- a/tests/pkg-fetch.test +++ b/tests/pkg-fetch.test @@ -145,7 +145,7 @@ $* libfoo/1.0.0 2>>/EOE != 0 { $clone_cfg; - $* -e $src/t1/libfoo-1.0.0.tar.gz 2>'fetched libfoo/1.0.0'; + $* -e $src/t1/libfoo-1.0.0.tar.gz 2>'using libfoo/1.0.0 (external)'; $pkg_status libfoo/1.0.0 1>'fetched; available'; $pkg_unpack libfoo 2>'unpacked libfoo/1.0.0'; @@ -158,7 +158,7 @@ $* libfoo/1.0.0 2>>/EOE != 0 $pkg_status libfoo/1.1.0 1>'fetched; available'; $pkg_unpack libfoo 2>'unpacked libfoo/1.1.0'; - $* -e $src/t1/libfoo-1.0.0.tar.gz 2>'fetched libfoo/1.0.0'; + $* -e $src/t1/libfoo-1.0.0.tar.gz 2>'using libfoo/1.0.0 (external)'; $pkg_status libfoo/1.0.0 1>'fetched; available'; $* libfoo/1.1.0 2>>~%EOE%; @@ -167,7 +167,7 @@ $* libfoo/1.0.0 2>>/EOE != 0 EOE $pkg_status libfoo/1.1.0 1>'fetched; available'; - $* -e $src/t1/libfoo-1.0.0.tar.gz 2>'fetched libfoo/1.0.0'; + $* -e $src/t1/libfoo-1.0.0.tar.gz 2>'using libfoo/1.0.0 (external)'; $pkg_status libfoo/1.0.0 1>'fetched; available'; $pkg_purge libfoo 2>'purged libfoo/1.0.0' @@ -179,7 +179,7 @@ $* libfoo/1.0.0 2>>/EOE != 0 $clone_cfg; cp --no-cleanup $src/t1/libfoo-1.0.0.tar.gz ./; - $* -p -e libfoo-1.0.0.tar.gz 2>'fetched libfoo/1.0.0'; + $* -p -e libfoo-1.0.0.tar.gz 2>'using libfoo/1.0.0 (external)'; $pkg_purge libfoo 2>'purged libfoo/1.0.0' } diff --git a/tests/pkg-status.test b/tests/pkg-status.test index b2ecb57..8d3fc36 100644 --- a/tests/pkg-status.test +++ b/tests/pkg-status.test @@ -178,7 +178,7 @@ rep_fetch += -d cfg --auth all --trust-yes 2>! } } -: git-repos +: git-rep : if ($git_supported != true) { diff --git a/tests/pkg-unpack.test b/tests/pkg-unpack.test index e4f07d4..af07fc5 100644 --- a/tests/pkg-unpack.test +++ b/tests/pkg-unpack.test @@ -47,8 +47,8 @@ rep_fetch += -d cfg --auth all 2>! : replace-with-existing : $clone_cfg; -$* -r 2>>EOE != 0 - error: --replace|-r can only be specified with --existing|-e +$* foo -r 2>>EOE != 0 + error: --replace|-r can only be specified with external package EOE : no-dir @@ -98,7 +98,7 @@ $* 2>>EOE != 0 EOE $pkg_purge libfoo 2>'purged libfoo/1.0.0'; - $* -e $src/libfoo-1.1.0 2>'unpacked libfoo/1.1.0'; + $* -e $src/libfoo-1.1.0 2>'using libfoo/1.1.0 (external)'; $pkg_status libfoo/1.1.0 1>'unpacked; available'; $pkg_purge libfoo 2>'purged libfoo/1.1.0' @@ -115,7 +115,7 @@ $* 2>>EOE != 0 info: use 'pkg-unpack --replace|-r' to replace EOE - $* -r -e $src/libfoo-1.1.0 2>'unpacked libfoo/1.1.0'; + $* -r -e $src/libfoo-1.1.0 2>'using libfoo/1.1.0 (external)'; $pkg_status libfoo/1.1.0 1>'unpacked; available'; $pkg_purge libfoo 2>'purged libfoo/1.1.0' @@ -134,7 +134,7 @@ $* 2>>EOE != 0 info: use 'pkg-unpack --replace|-r' to replace EOE - $* -r -e $src/libfoo-1.1.0 2>'unpacked libfoo/1.1.0'; + $* -r -e $src/libfoo-1.1.0 2>'using libfoo/1.1.0 (external)'; $pkg_status libfoo/1.1.0 1>'unpacked; available'; $pkg_purge libfoo 2>'purged libfoo/1.1.0' @@ -149,7 +149,7 @@ $* 2>>EOE != 0 error: package libfoo does not exist in configuration cfg/ EOE - $* -e $src/libfoo-1.1.0 2>'unpacked libfoo/1.1.0'; + $* -e $src/libfoo-1.1.0 2>'using libfoo/1.1.0 (external)'; $* libfoo 2>>EOE != 0; error: package libfoo is unpacked @@ -177,7 +177,7 @@ $* 2>>EOE != 0 $clone_cfg; cp --no-cleanup -r $src/libfoo-1.1.0 ./; - $* -p -e libfoo-1.1.0 2>'unpacked libfoo/1.1.0'; + $* -p -e libfoo-1.1.0 2>'using libfoo/1.1.0 (external)'; $pkg_purge libfoo 2>'purged libfoo/1.1.0' } @@ -194,3 +194,63 @@ $* 2>>EOE != 0 $pkg_purge libhello 2>'purged libhello/1.0.0' } + +: dir-rep +: +{ + rep_add += --type dir + + : no-repos + : + { + $clone_root_cfg; + + $* libfoo/1.1.0 2>>/EOE != 0 + error: configuration cfg/ has no repositories + info: use 'bpkg rep-add' to add a repository + EOE + } + + : unfetched + : + { + $clone_root_cfg && $rep_add $src/libfoo-1.1.0; + + $* libfoo/1.1.0 2>>/EOE != 0 + error: configuration cfg/ has no available packages + info: use 'bpkg rep-fetch' to fetch available packages list + EOE + } + + : unavailable + : + { + $clone_root_cfg && $rep_add $src/libfoo-1.1.0 && $rep_fetch; + + $* libfoo/1.0.0 2>>EOE != 0 + error: package libfoo 1.0.0 is not available + EOE + } + + : unavailable-dir-based + : + { + $clone_root_cfg && $rep_add $rep/t1 --type pkg && $rep_fetch --trust-yes; + + $* libfoo/1.0.0 2>>EOE != 0 + error: package libfoo 1.0.0 is not available from a directory-based repository + EOE + } + + : available + : + { + $clone_root_cfg && $rep_add $src/libfoo-1.1.0 && $rep_fetch; + + $* libfoo/1.1.0 2>>EOE; + using libfoo/1.1.0 (external) + EOE + + $pkg_status libfoo 1>'unpacked 1.1.0; available sys:?' + } +} diff --git a/tests/rep-fetch.test b/tests/rep-fetch.test index ffc833c..f7be436 100644 --- a/tests/rep-fetch.test +++ b/tests/rep-fetch.test @@ -114,7 +114,7 @@ $* 2>>/EOE != 0 info: use 'bpkg rep-add' to add a repository EOE -: pkg-repos +: pkg-rep : { test.options += --auth all @@ -319,7 +319,36 @@ $* 2>>/EOE != 0 } } -: git-repos +: dir-rep +: +{ + rep_add += --type dir + + : prerequisites + : + if ($remote != true) + { + # Let's reuse local git repositories that have the same repository + # structure as for dir type. + # + rep = $canonicalize([dir_path] $out_git/state0) + + $clone_root_cfg && $rep_add $rep/libbar.git; + + $* 2>>"EOE"; + fetching dir:($rep/libbar.git) + fetching dir:($rep/style-basic.git) \(prerequisite of dir:($rep/libbar.git)\) + 3 package\(s\) in 2 repository\(s\) + EOE + + $rep_list >>"EOO" + dir:($rep/libbar.git) ($rep/libbar.git) + prerequisite dir:($rep/style-basic.git) ($rep/style-basic.git) + EOO + } +} + +: git-rep : if ($git_supported != true) { diff --git a/tests/rep-info.test b/tests/rep-info.test index 13a1148..9a70b49 100644 --- a/tests/rep-info.test +++ b/tests/rep-info.test @@ -138,7 +138,7 @@ $* --name $rep/testing >"pkg:build2.org/rep-info/testing ($rep/testing)" $* --cert-email >'info@build2.org' : email } -: git-repos +: git-rep : if ($git_supported != true) { diff --git a/tests/rep-list.test b/tests/rep-list.test index c2a2900..970379a 100644 --- a/tests/rep-list.test +++ b/tests/rep-list.test @@ -121,7 +121,7 @@ rep_fetch += -d cfg --auth all --trust-yes 2>! EOO } -: git-repos +: git-rep : if ($git_supported != true) { diff --git a/tests/rep-remove.test b/tests/rep-remove.test index 98d566f..01e91d4 100644 --- a/tests/rep-remove.test +++ b/tests/rep-remove.test @@ -183,7 +183,7 @@ pkg_status += -d cfg $pkg_status libbar >'unknown' } -: git-repos +: git-rep : if ($git_supported != true) { -- cgit v1.1