Add support for repository typed URLs (git+https://..., etc)

3 files changed, 216 insertions, 23 deletions

diff --git a/libbpkg/manifest.cxx b/libbpkg/manifest.cxx
index fff3174..a88ea41 100644
--- a/libbpkg/manifest.cxx
+++ b/libbpkg/manifest.cxx
@@ -14,6 +14,7 @@
 #include <algorithm> // find(), find_if_not(), find_first_of(), replace()
 #include <stdexcept> // invalid_argument
 
+#include <libbutl/url.mxx>
 #include <libbutl/path.mxx>
 #include <libbutl/base64.mxx>
 #include <libbutl/utility.mxx>             // casecmp(), lcase(), alpha(),
@@ -2147,18 +2148,29 @@ namespace bpkg
     return string ();
   }
 
-  repository_type
-  to_repository_type (const string& t)
+  inline static optional<repository_type>
+  parse_repository_type (const string& t)
   {
          if (t == "pkg") return repository_type::pkg;
     else if (t == "dir") return repository_type::dir;
     else if (t == "git") return repository_type::git;
-    else throw invalid_argument ("invalid repository type '" + t + "'");
+    else                 return nullopt;
+  }
+
+  repository_type
+  to_repository_type (const string& t)
+  {
+    if (optional<repository_type> r = parse_repository_type (t))
+      return *r;
+
+    throw invalid_argument ("invalid repository type '" + t + "'");
   }
 
   repository_type
   guess_type (const repository_url& url, bool local)
   {
+    assert (!url.empty ());
+
     switch (url.scheme)
     {
     case repository_protocol::git:
@@ -2187,6 +2199,49 @@ namespace bpkg
     return repository_type::pkg;
   }
 
+  // typed_repository_url
+  //
+  typed_repository_url::
+  typed_repository_url (const string& s)
+  {
+    using traits = butl::url::traits;
+
+    if (traits::find (s) == 0) // Looks like a non-rootless URL?
+    {
+      size_t p (s.find_first_of ("+:"));
+
+      assert (p != string::npos); // At least the colon is present.
+
+      if (s[p] == '+')
+      {
+        string r (s, p + 1);
+        optional<repository_type> t;
+
+        if (traits::find (r) == 0 &&                        // URL notation?
+            (t = parse_repository_type (string (s, 0, p)))) // Valid type?
+        {
+          repository_url u (r);
+
+          // Only consider the URL to be typed if it is not a relative
+          // path. And yes, we may end up with a relative path for the URL
+          // string (e.g. ftp://example.com).
+          //
+          if (!(u.scheme == repository_protocol::file && u.path->relative ()))
+          {
+            type = move (t);
+            url  = move (u);
+          }
+        }
+      }
+    }
+
+    // Parse the whole string as a repository URL if we failed to extract the
+    // type.
+    //
+    if (url.empty ())
+      url = repository_url (s); // Throws if empty.
+  }
+
   // repository_location
   //
   static string
@@ -2301,6 +2356,24 @@ namespace bpkg
   }
 
   repository_location::
+  repository_location (const std::string& s,
+                       const optional<repository_type>& t,
+                       bool local)
+  {
+    typed_repository_url tu (s);
+
+    if (t && tu.type && t != tu.type)
+      throw invalid_argument (
+        "mismatching repository types: " + to_string (*t) + " specified, " +
+        to_string (*tu.type) + " in URL scheme");
+
+    *this = repository_location (move (tu.url),
+                                 tu.type ? *tu.type :
+                                 t       ? *t       :
+                                 guess_type (tu.url, local));
+  }
+
+  repository_location::
   repository_location (repository_url u,
                        repository_type t,
                        const repository_location& b)
@@ -2552,6 +2625,35 @@ namespace bpkg
     }
   }
 
+  string repository_location::
+  string () const
+  {
+    if (empty ()    ||
+        relative () ||
+        guess_type (url_, false /* local */) == type_)
+      return url_.string ();
+
+    std::string r (to_string (type_) + '+');
+
+    // Enforce the 'file://' notation for local URLs, adding the empty
+    // authority (see manifest.hxx for details).
+    //
+    if (url_.scheme == repository_protocol::file &&
+        !url_.authority                          &&
+        !url_.fragment)
+    {
+      repository_url u (url_.scheme,
+                        repository_url::authority_type (),
+                        url_.path);
+
+      r += u.string ();
+    }
+    else
+      r += url_.string ();
+
+    return r;
+  }
+
   // git_ref_filter
   //
   git_ref_filter::
@@ -2942,6 +3044,9 @@ namespace bpkg
 
     s.next ("", "1"); // Start of manifest.
 
+    // Note that the location can be relative, so we also serialize the
+    // repository type.
+    //
     if (!location.empty ())
     {
       s.next ("location", location.string ());
diff --git a/libbpkg/manifest.hxx b/libbpkg/manifest.hxx
index a017772..20c9caf 100644
--- a/libbpkg/manifest.hxx
+++ b/libbpkg/manifest.hxx
@@ -698,7 +698,7 @@ namespace bpkg
     version_control
   };
 
-  // Guess the repository type for the URL:
+  // Guess the repository type from the URL:
   //
   // 1. If scheme is git then git.
   //
@@ -714,6 +714,38 @@ namespace bpkg
   LIBBPKG_EXPORT repository_type
   guess_type (const repository_url&, bool local);
 
+  // Repository URL that may have a repository type specified as part of its
+  // scheme in the [<type>'+']<protocol> form. For example:
+  //
+  // git+http://example.com/repo  (repository type + protocol)
+  // git://example.com/repo       (protocol only)
+  //
+  // If the substring preceding the '+' character is not a valid repository
+  // type or the part that follows doesn't conform to the repository URL
+  // notation, then the whole string is considered to be a repository URL.
+  // For example, for all of the following strings the repository URL is
+  // untyped (local) and relative:
+  //
+  // foo+http://example.com/repo  (invalid repository type)
+  // git+ftp://example.com/repo   (invalid repository protocol)
+  // git+file://example.com/repo  (invalid authority)
+  // git+c:/repo                  (not a URL notation)
+  //
+  // Note also that in quite a few manifests where we specify the location we
+  // also allow specifying the type as a separate value. While this may seem
+  // redundant (and it now is in a few cases, at least for the time being),
+  // keep in mind that for the local relative path the type cannot be
+  // specified as part of the URL (since its representation is a non-URL).
+  //
+  struct LIBBPKG_EXPORT typed_repository_url
+  {
+    repository_url url;
+    butl::optional<repository_type> type;
+
+    explicit
+    typed_repository_url (const std::string&);
+  };
+
   class LIBBPKG_EXPORT repository_location
   {
   public:
@@ -721,6 +753,23 @@ namespace bpkg
     //
     repository_location () = default;
 
+    // Create a remote or absolute repository location from a potentially
+    // typed repository URL (see above).
+    //
+    // If the type is not specified in the URL scheme then use the one passed
+    // as an argument or, if not present, guess it according to the specified
+    // local flag (see above). Throw std::invalid_argument if the argument
+    // doesn't represent a valid remote or absolute repository location or
+    // mismatching types are specified in the URL scheme and in the argument.
+    // Underlying OS errors (which may happen when guessing the type when the
+    // local flag is set) are reported by throwing std::system_error.
+    //
+    explicit
+    repository_location (
+      const std::string&,
+      const butl::optional<repository_type>& = butl::nullopt,
+      bool local = false);
+
     // Create remote, absolute or empty repository location making sure that
     // the URL matches the repository type. Throw std::invalid_argument if the
     // URL object is a relative local path.
@@ -746,7 +795,7 @@ namespace bpkg
     //
     repository_location (repository_url, repository_type);
 
-    // Create a potentially relative pkg repository location. If base is not
+    // Create a potentially relative repository location. If base is not
     // empty, use it to complete the relative location to the remote/absolute
     // one. Throw std::invalid_argument if base is not empty but the location
     // is empty, base itself is relative, or the resulting completed location
@@ -830,7 +879,7 @@ namespace bpkg
       return repository_basis::archive;
     }
 
-    // URL of an empty location is empty.
+    // Note that the URL of an empty location is empty.
     //
     const repository_url&
     url () const
@@ -906,13 +955,14 @@ namespace bpkg
       return basis () == repository_basis::version_control;
     }
 
-    // String representation of an empty location is the empty string.
+    // Return an untyped URL if the correct type can be guessed just from
+    // the URL. Otherwise, return the typed URL.
+    //
+    // String representation is empty for an empty location and is always
+    // untyped for the relative location (which is a non-URL).
     //
     std::string
-    string () const
-    {
-      return url_.string ();
-    }
+    string () const;
 
   private:
     std::string canonical_name_;
diff --git a/tests/repository-location/driver.cxx b/tests/repository-location/driver.cxx
index c2153ff..3f03a5d 100644
--- a/tests/repository-location/driver.cxx
+++ b/tests/repository-location/driver.cxx
@@ -55,6 +55,26 @@ namespace bpkg
     }
   }
 
+  inline static repository_location
+  typed_loc (const string& u, optional<repository_type> t = nullopt)
+  {
+    return repository_location (u, t);
+  }
+
+  inline static bool
+  bad_typed_loc (const string& u, optional<repository_type> t = nullopt)
+  {
+    try
+    {
+      repository_location bl (u, t);
+      return false;
+    }
+    catch (const invalid_argument&)
+    {
+      return true;
+    }
+  }
+
   inline static bool
   bad_loc (const string& l,
            const repository_location& b,
@@ -207,6 +227,16 @@ namespace bpkg
     //
     assert (bad_loc ("http://example.com/dir", repository_type::dir));
 
+    // Invalid typed repository location.
+    //
+    assert (bad_typed_loc (""));                            // Empty.
+    assert (bad_typed_loc ("abc+http://example.com/repo")); // Relative.
+
+    assert (bad_typed_loc ("git+http://example.com/repo",   // Types mismatch.
+                           repository_type::pkg));
+
+    assert (bad_typed_loc ("http://example.com/repo"));     // Invalid for pkg.
+
     // Invalid web interface URL.
     //
     assert (bad_url (".a/..", loc ("http://stable.cppget.org/1/misc")));
@@ -313,28 +343,28 @@ namespace bpkg
     {
       repository_location l (loc ("file:/git/repo#branch",
                                   repository_type::git));
-      assert (l.string () == "file:/git/repo#branch");
+      assert (l.string () == "git+file:/git/repo#branch");
       assert (l.canonical_name () == "git:/git/repo#branch");
     }
     {
       repository_location l (loc ("/git/repo#branch", repository_type::git));
-      assert (l.string () == "file:/git/repo#branch");
+      assert (l.string () == "git+file:/git/repo#branch");
       assert (l.canonical_name () == "git:/git/repo#branch");
     }
     {
       repository_location l (loc ("file://localhost/", repository_type::git));
-      assert (l.string () == "/");
+      assert (l.string () == "git+file:///");
       assert (l.canonical_name () == "git:/");
     }
     {
       repository_location l (loc ("file://localhost/#master",
                                   repository_type::git));
-      assert (l.string () == "file:/#master");
+      assert (l.string () == "git+file:/#master");
       assert (l.canonical_name () == "git:/#master");
     }
     {
       repository_location l (loc ("/home/user/repo", repository_type::dir));
-      assert (l.string () == "/home/user/repo");
+      assert (l.string () == "dir+file:///home/user/repo");
       assert (l.canonical_name () == "dir:/home/user/repo");
     }
 #else
@@ -389,30 +419,30 @@ namespace bpkg
     {
       repository_location l (loc ("file:/c:/git/repo#branch",
                                   repository_type::git));
-      assert (l.string () == "file:/c:/git/repo#branch");
+      assert (l.string () == "git+file:/c:/git/repo#branch");
       assert (l.canonical_name () == "git:c:\\git\\repo#branch");
     }
     {
       repository_location l (loc ("c:\\git\\repo#branch",
                                   repository_type::git));
-      assert (l.string () == "file:/c:/git/repo#branch");
+      assert (l.string () == "git+file:/c:/git/repo#branch");
       assert (l.canonical_name () == "git:c:\\git\\repo#branch");
     }
     {
       repository_location l (loc ("file://localhost/c:/",
                                   repository_type::git));
-      assert (l.string () == "c:");
+      assert (l.string () == "git+file:///c:");
       assert (l.canonical_name () == "git:c:");
     }
     {
       repository_location l (loc ("file://localhost/c:/#master",
                                   repository_type::git));
-      assert (l.string () == "file:/c:#master");
+      assert (l.string () == "git+file:/c:#master");
       assert (l.canonical_name () == "git:c:#master");
     }
     {
       repository_location l (loc ("c:\\user\\repo", repository_type::dir));
-      assert (l.string () == "c:\\user\\repo");
+      assert (l.string () == "dir+file:///c:/user/repo");
       assert (l.canonical_name () == "dir:c:\\user\\repo");
     }
 #endif
@@ -506,7 +536,7 @@ namespace bpkg
     {
       repository_location l (loc ("http://git.example.com#master",
                                   repository_type::git));
-      assert (l.string () == "http://git.example.com/#master");
+      assert (l.string () == "git+http://git.example.com/#master");
       assert (l.canonical_name () == "git:example.com#master");
     }
     {
@@ -514,7 +544,7 @@ namespace bpkg
       *u.path /= path ("..");
 
       repository_location l (u, repository_type::git);
-      assert (l.string () == "http://git.example.com/#master");
+      assert (l.string () == "git+http://git.example.com/#master");
       assert (l.canonical_name () == "git:example.com#master");
     }
     {
@@ -585,6 +615,14 @@ namespace bpkg
       assert (l.canonical_name () == "pkg:stable.cppget.org");
     }
     {
+      repository_location l (typed_loc ("git+http://example.com/repo"));
+      assert (l.string () == "git+http://example.com/repo");
+    }
+    {
+      repository_location l (typed_loc ("http://example.com/repo.git"));
+      assert (l.string () == "http://example.com/repo.git");
+    }
+    {
       repository_location l1 (loc ("http://stable.cppget.org/1/misc"));
       repository_location l2 (loc ("../../1/math", l1));
       assert (l2.string () == "http://stable.cppget.org/1/math");