From 42ae47c3033a8c9ce70f1e6fb4c88ed70ac679fb Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Sat, 23 Dec 2017 19:05:22 +0300 Subject: Add repository type detection --- bpkg/auth.cxx | 2 +- bpkg/fetch.cxx | 86 +++++++++++++++++----------------------------- bpkg/manifest-utility.cxx | 87 +++++++++++++++++++++++++++++++++++++++++++---- bpkg/manifest-utility.hxx | 8 ++--- bpkg/package.hxx | 23 ++++++++++--- bpkg/package.xml | 8 +++-- bpkg/rep-add.cli | 12 +++++++ bpkg/rep-add.cxx | 7 +++- bpkg/rep-info.cli | 12 +++++++ bpkg/rep-info.cxx | 6 +++- bpkg/types-parsers.cxx | 21 ++++++++++++ bpkg/types-parsers.hxx | 9 +++++ tests/rep-add.test | 84 ++++++++++++++++++++++++++++++++++++++++++--- 13 files changed, 286 insertions(+), 79 deletions(-) diff --git a/bpkg/auth.cxx b/bpkg/auth.cxx index dc4aba8..da4d0b5 100644 --- a/bpkg/auth.cxx +++ b/bpkg/auth.cxx @@ -70,7 +70,7 @@ namespace bpkg if (rl.remote ()) return repository_location (p.posix_string (), rl).canonical_name (); else - return (rl.path () / p).normalize ().string (); + return (path_cast (rl.path ()) / p).normalize ().string (); } // Authenticate a dummy certificate. If trusted, it will authenticate all diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx index d6e7d76..7389c75 100644 --- a/bpkg/fetch.cxx +++ b/bpkg/fetch.cxx @@ -548,58 +548,25 @@ namespace bpkg } } - using protocol = repository_location::protocol; - - static string - to_url (protocol proto, const string& host, uint16_t port, const path& file) - { - assert (file.relative ()); - - if (*file.begin () == "..") - fail << "invalid URL path " << file; - - string url; - - switch (proto) - { - case protocol::http: url = "http://"; break; - case protocol::https: url = "https://"; break; - } - - url += host; - - if (port != 0) - url += ":" + to_string (port); - - url += "/" + file.posix_string (); - - return url; - } - static path fetch_file (const common_options& o, - protocol proto, - const string& host, - uint16_t port, - const path& f, + const repository_url& u, const dir_path& d) { - path r (d / f.leaf ()); + path r (d / u.path->leaf ()); if (exists (r)) fail << "file " << r << " already exists"; - string url (to_url (proto, host, port, f)); - auto_rm arm (r); - process pr (start (o, url, r)); + process pr (start (o, u.string (), r)); if (!pr.wait ()) { // While it is reasonable to assuming the child process issued // diagnostics, some may not mention the URL. // - fail << "unable to fetch " << url << + fail << "unable to fetch " << u << info << "re-run with -v for more information"; } @@ -610,13 +577,10 @@ namespace bpkg template static pair fetch_manifest (const common_options& o, - protocol proto, - const string& host, - uint16_t port, - const path& f, + const repository_url& u, bool ignore_unknown) { - string url (to_url (proto, host, port, f)); + string url (u.string ()); process pr (start (o, url)); try @@ -747,11 +711,13 @@ namespace bpkg { assert (rl.remote () || rl.absolute ()); - path f (rl.path () / repositories); + repository_url u (rl.url ()); + + path& f (*u.path); + f /= repositories; return rl.remote () - ? fetch_manifest ( - o, rl.proto (), rl.host (), rl.port (), f, iu) + ? fetch_manifest (o, u, iu) : fetch_manifest (&o, f, iu); } @@ -770,11 +736,13 @@ namespace bpkg { assert (rl.remote () || rl.absolute ()); - path f (rl.path () / packages); + repository_url u (rl.url ()); + + path& f (*u.path); + f /= packages; return rl.remote () - ? fetch_manifest ( - o, rl.proto (), rl.host (), rl.port (), f, iu) + ? fetch_manifest (o, u, iu) : fetch_manifest (&o, f, iu); } @@ -787,11 +755,13 @@ namespace bpkg { assert (rl.remote () || rl.absolute ()); - path f (rl.path () / signature); + repository_url u (rl.url ()); + + path& f (*u.path); + f /= signature; return rl.remote () - ? fetch_manifest ( - o, rl.proto (), rl.host (), rl.port (), f, iu).first + ? fetch_manifest (o, u, iu).first : fetch_manifest (nullptr, f, iu).first; } @@ -804,19 +774,27 @@ namespace bpkg assert (!a.empty () && a.relative ()); assert (rl.remote () || rl.absolute ()); - path f (rl.path () / a); + repository_url u (rl.url ()); + + path& f (*u.path); + f /= a; + + auto bad_loc = [&u] () {fail << "invalid archive location " << u;}; try { f.normalize (); + + if (*f.begin () == "..") // Can be the case for the remote location. + bad_loc (); } catch (const invalid_path&) { - fail << "invalid archive location " << rl << "/" << f; + bad_loc (); } return rl.remote () - ? fetch_file (o, rl.proto (), rl.host (), rl.port (), f, d) + ? fetch_file (o, u, d) : fetch_file (f, d); } } diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx index 850ce59..031d51a 100644 --- a/bpkg/manifest-utility.cxx +++ b/bpkg/manifest-utility.cxx @@ -7,6 +7,7 @@ #include using namespace std; +using namespace butl; namespace bpkg { @@ -65,19 +66,93 @@ namespace bpkg } repository_location - parse_location (const char* s) + parse_location (const char* s, optional ot) try { - repository_location rl (s, repository_location ()); + repository_url u (s); - if (rl.relative ()) // Throws if the location is empty. - rl = repository_location ( - dir_path (s).complete ().normalize ().string ()); + if (u.empty ()) + fail << "empty repository location"; - return rl; + assert (u.path); + + // Make the relative path absolute using the current directory. + // + if (u.scheme == repository_protocol::file && u.path->relative ()) + u.path->complete ().normalize (); + + // Guess the repository type to construct the repository location: + // + // 1. If type is specified as an option use that (but validate + // incompatible scheme/type e.g., git/bpkg). + // + // 2. If scheme is git then git. + // + // 3. If scheme is http(s), then check if path has the .git extension, + // then git, otherwise bpkg. + // + // 4. If local (which will normally be without the .git extension), check + // if directory contains the .git/ subdirectory then git, otherwise + // bpkg. + // + repository_type t; + + if (ot) + t = *ot; + else + { + switch (u.scheme) + { + case repository_protocol::git: + { + t = repository_type::git; + break; + } + case repository_protocol::http: + case repository_protocol::https: + { + t = u.path->extension () == "git" + ? repository_type::git + : repository_type::bpkg; + break; + } + case repository_protocol::file: + { + t = exists (path_cast (*u.path) / dir_path (".git")) + ? repository_type::git + : repository_type::bpkg; + break; + } + } + } + + try + { + // Don't move the URL since it may still be needed for diagnostics. + // + return repository_location (u, t); + } + catch (const invalid_argument& e) + { + diag_record dr; + dr << fail << "invalid " << t << " repository location '" << u << "': " + << e; + + // If the bpkg repository type was guessed, then suggest the user to + // specify the type explicitly. + // + if (!ot && t == repository_type::bpkg) + dr << info << "consider using --type to specify repository type"; + + dr << endf; + } } catch (const invalid_argument& e) { fail << "invalid repository location '" << s << "': " << e << endf; } + catch (const invalid_path& e) + { + fail << "invalid repository path '" << s << "': " << e << endf; + } } diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx index afa93f5..f4dadb4 100644 --- a/bpkg/manifest-utility.hxx +++ b/bpkg/manifest-utility.hxx @@ -34,12 +34,12 @@ namespace bpkg version parse_package_version (const char*); - // First use the passed location as is. If the result is relative, - // then assume this is a relative path to the repository directory - // and complete it based on the current working directory. + // If the passed location is a relative local path, then assume this is a + // relative path to the repository directory and complete it based on the + // current working directory. Diagnose invalid locations and throw failed. // repository_location - parse_location (const char*); + parse_location (const char*, optional); } #endif // BPKG_MANIFEST_UTILITY_HXX diff --git a/bpkg/package.hxx b/bpkg/package.hxx index b5ad0f3..00a54d6 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -19,7 +19,7 @@ #include #include -#pragma db model version(3, 3, closed) +#pragma db model version(4, 4, open) namespace bpkg { @@ -200,9 +200,24 @@ namespace bpkg // repository_location // - #pragma db map type(repository_location) as(string) \ - to((?).string ()) from(bpkg::repository_location (?)) + #pragma db value + struct _repository_location + { + string url; + repository_type type; + }; + + // Note that the type() call fails for an empty repository location. + // + #pragma db map type(repository_location) as(_repository_location) \ + to({(?).string (), \ + (?).empty () ? bpkg::repository_type::bpkg : (?).type ()}) \ + from(bpkg::repository_location ((?).url, (?).type)) + + #pragma db map type(repository_type) as(string) \ + to(to_string (?)) \ + from(bpkg::to_repository_type (?)) // repository // @@ -240,7 +255,7 @@ namespace bpkg // #pragma db member(name) id - #pragma db member(location) \ + #pragma db member(location) column("") \ set(this.location = std::move (?); \ assert (this.name == this.location.canonical_name ())) diff --git a/bpkg/package.xml b/bpkg/package.xml index 78a49d4..f77653f 100644 --- a/bpkg/package.xml +++ b/bpkg/package.xml @@ -1,8 +1,9 @@ - + - + + @@ -190,7 +191,8 @@ - + + diff --git a/bpkg/rep-add.cli b/bpkg/rep-add.cli index cbd73a4..eddea45 100644 --- a/bpkg/rep-add.cli +++ b/bpkg/rep-add.cli @@ -2,6 +2,8 @@ // copyright : Copyright (c) 2014-2017 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file +include ; + include ; "\section=1" @@ -28,5 +30,15 @@ namespace bpkg class rep_add_options: configuration_options { "\h|REP-ADD OPTIONS|" + + repository_type --type + { + "", + "Specify the repository type with valid values being \cb{bpkg} and + \cb{git}. Normally the repository type can be automatically guessed by + examining its URL (for example, the presence of the \cb{.git} + extension) or, in case of a local repository, its content (for example, + the presence of the \cb{.git} subdirectory)." + } }; } diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx index 6de0738..d18536f 100644 --- a/bpkg/rep-add.cxx +++ b/bpkg/rep-add.cxx @@ -27,7 +27,12 @@ namespace bpkg fail << "repository location argument expected" << info << "run 'bpkg help rep-add' for more information"; - repository_location rl (parse_location (args.next ())); + repository_location rl ( + parse_location (args.next (), + o.type_specified () + ? optional (o.type ()) + : nullopt)); + const string& rn (rl.canonical_name ()); // Create the new repository and add is as a complement to the root. diff --git a/bpkg/rep-info.cli b/bpkg/rep-info.cli index 8bad234..d2ce9b6 100644 --- a/bpkg/rep-info.cli +++ b/bpkg/rep-info.cli @@ -2,6 +2,8 @@ // copyright : Copyright (c) 2014-2017 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file +include ; + include ; "\section=1" @@ -91,6 +93,16 @@ namespace bpkg manifests." } + repository_type --type + { + "", + "Specify the repository type with valid values being \cb{bpkg} and + \cb{git}. Normally the repository type can be automatically guessed by + examining its URL (for example, the presence of the \cb{.git} + extension) or, in case of a local repository, its content (for example, + the presence of the \cb{.git} subdirectory)." + } + string --directory|-d // String to allow empty value. { "", diff --git a/bpkg/rep-info.cxx b/bpkg/rep-info.cxx index f6a8d2d..322ff6b 100644 --- a/bpkg/rep-info.cxx +++ b/bpkg/rep-info.cxx @@ -31,7 +31,11 @@ namespace bpkg fail << "repository location argument expected" << info << "run 'bpkg help rep-info' for more information"; - repository_location rl (parse_location (args.next ())); + repository_location rl ( + parse_location (args.next (), + o.type_specified () + ? optional (o.type ()) + : nullopt)); // Fetch everything we will need before printing anything. Ignore // unknown manifest entries unless we are dumping them. First fetch diff --git a/bpkg/types-parsers.cxx b/bpkg/types-parsers.cxx index 6da395c..41b231f 100644 --- a/bpkg/types-parsers.cxx +++ b/bpkg/types-parsers.cxx @@ -67,5 +67,26 @@ namespace bpkg else throw invalid_value (o, v); } + + void parser:: + parse (repository_type& x, bool& xs, scanner& s) + { + xs = true; + const char* o (s.next ()); + + if (!s.more ()) + throw missing_value (o); + + const string v (s.next ()); + + try + { + x = to_repository_type(v); + } + catch (const invalid_argument&) + { + throw invalid_value (o, v); + } + } } } diff --git a/bpkg/types-parsers.hxx b/bpkg/types-parsers.hxx index bddb425..4f1dfc7 100644 --- a/bpkg/types-parsers.hxx +++ b/bpkg/types-parsers.hxx @@ -8,6 +8,8 @@ #ifndef BPKG_TYPES_PARSERS_HXX #define BPKG_TYPES_PARSERS_HXX +#include + #include #include @@ -40,6 +42,13 @@ namespace bpkg static void parse (auth&, bool&, scanner&); }; + + template <> + struct parser + { + static void + parse (repository_type&, bool&, scanner&); + }; } } diff --git a/tests/rep-add.test b/tests/rep-add.test index a1b9104..28a7a20 100644 --- a/tests/rep-add.test +++ b/tests/rep-add.test @@ -7,6 +7,8 @@ : location : { + +$clone_cfg + : none : $* 2>>EOE != 0 @@ -14,17 +16,89 @@ info: run 'bpkg help rep-add' for more information EOE + : empty + : + $* '' 2>>EOE != 0 + error: empty repository location + EOE + + : unknown-type + : + $* 'repo' --type unknown 2>>EOE != 0 + error: invalid value 'unknown' for option '--type' + EOE + : no-version : - $* 'stable' 2>>EOE != 0 - error: invalid repository location 'stable': missing repository version + $* 'stable' 2>>/~%EOE% != 0 + %error: invalid bpkg repository location '.+/no-version/stable': missing repository version% + info: consider using --type to specify repository type EOE - : invalid-host + : git-no-branch : - $* 'http://' 2>>EOE != 0 - error: invalid repository location 'http://': invalid host + $* 'git://example.org/repo' 2>>EOE != 0 + error: invalid git repository location 'git://example.org/repo': missing branch/tag for git repository EOE + + : bpkg-git-scheme + : + $* 'git://example.org/repo' --type bpkg 2>>EOE != 0 + error: invalid bpkg repository location 'git://example.org/repo': unsupported scheme for bpkg repository + EOE + + : invalid-path + : + { + s="../../../../../../../../../../../../../../../../../../../../../../../" + s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s" + $* "$s" 2>>~%EOE% != 0 + %error: invalid repository path '.+/': invalid filesystem path% + EOE + } + + : type-detection + : + { + +$clone_cfg + + : git-scheme + : + $clone_cfg; + $* 'git://example.org/repo#master' 2>>EOE + added repository example.org/repo + EOE + + : http-git + : + $clone_cfg; + $* 'http://example.org/repo.git#master' 2>>EOE + added repository example.org/repo + EOE + + : http-bpkg + : + $clone_cfg; + $* 'http://example.org/1/repo' 2>>EOE + added repository example.org/repo + EOE + + : file-git + : + $clone_cfg && mkdir -p repo/.git; + + $* 'repo' 2>>~%EOE% != 0 + %error: invalid git repository location '.+repo': missing branch/tag for git repository% + EOE + + : file-bpkg + : + $clone_cfg; + + $* '1/repo' 2>>/~%EOE% + %added repository .+/repo% + EOE + } } : relative-path -- cgit v1.1