Add repository type detection

13 files changed, 286 insertions, 79 deletions

diff --git a/bpkg/auth.cxx b/bpkg/auth.cxx
index dc4aba8..da4d0b5 100644
--- a/bpkg/auth.cxx
+++ b/bpkg/auth.cxx
@@ -70,7 +70,7 @@ namespace bpkg
     if (rl.remote ())
       return repository_location (p.posix_string (), rl).canonical_name ();
     else
-      return (rl.path () / p).normalize ().string ();
+      return (path_cast<dir_path> (rl.path ()) / p).normalize ().string ();
   }
 
   // Authenticate a dummy certificate. If trusted, it will authenticate all
diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx
index d6e7d76..7389c75 100644
--- a/bpkg/fetch.cxx
+++ b/bpkg/fetch.cxx
@@ -548,58 +548,25 @@ namespace bpkg
     }
   }
 
-  using protocol = repository_location::protocol;
-
-  static string
-  to_url (protocol proto, const string& host, uint16_t port, const path& file)
-  {
-    assert (file.relative ());
-
-    if (*file.begin () == "..")
-      fail << "invalid URL path " << file;
-
-    string url;
-
-    switch (proto)
-    {
-    case protocol::http: url = "http://"; break;
-    case protocol::https: url = "https://"; break;
-    }
-
-    url += host;
-
-    if (port != 0)
-      url += ":" + to_string (port);
-
-    url += "/" + file.posix_string ();
-
-    return url;
-  }
-
   static path
   fetch_file (const common_options& o,
-              protocol proto,
-              const string& host,
-              uint16_t port,
-              const path& f,
+              const repository_url& u,
               const dir_path& d)
   {
-    path r (d / f.leaf ());
+    path r (d / u.path->leaf ());
 
     if (exists (r))
       fail << "file " << r << " already exists";
 
-    string url (to_url (proto, host, port, f));
-
     auto_rm arm (r);
-    process pr (start (o, url, r));
+    process pr (start (o, u.string (), r));
 
     if (!pr.wait ())
     {
       // While it is reasonable to assuming the child process issued
       // diagnostics, some may not mention the URL.
       //
-      fail << "unable to fetch " << url <<
+      fail << "unable to fetch " << u <<
         info << "re-run with -v for more information";
     }
 
@@ -610,13 +577,10 @@ namespace bpkg
   template <typename M>
   static pair<M, string/*checksum*/>
   fetch_manifest (const common_options& o,
-                  protocol proto,
-                  const string& host,
-                  uint16_t port,
-                  const path& f,
+                  const repository_url& u,
                   bool ignore_unknown)
   {
-    string url (to_url (proto, host, port, f));
+    string url (u.string ());
     process pr (start (o, url));
 
     try
@@ -747,11 +711,13 @@ namespace bpkg
   {
     assert (rl.remote () || rl.absolute ());
 
-    path f (rl.path () / repositories);
+    repository_url u (rl.url ());
+
+    path& f (*u.path);
+    f /= repositories;
 
     return rl.remote ()
-      ? fetch_manifest<repository_manifests> (
-          o, rl.proto (), rl.host (), rl.port (), f, iu)
+      ? fetch_manifest<repository_manifests> (o, u, iu)
       : fetch_manifest<repository_manifests> (&o, f, iu);
   }
 
@@ -770,11 +736,13 @@ namespace bpkg
   {
     assert (rl.remote () || rl.absolute ());
 
-    path f (rl.path () / packages);
+    repository_url u (rl.url ());
+
+    path& f (*u.path);
+    f /= packages;
 
     return rl.remote ()
-      ? fetch_manifest<package_manifests> (
-          o, rl.proto (), rl.host (), rl.port (), f, iu)
+      ? fetch_manifest<package_manifests> (o, u, iu)
       : fetch_manifest<package_manifests> (&o, f, iu);
   }
 
@@ -787,11 +755,13 @@ namespace bpkg
   {
     assert (rl.remote () || rl.absolute ());
 
-    path f (rl.path () / signature);
+    repository_url u (rl.url ());
+
+    path& f (*u.path);
+    f /= signature;
 
     return rl.remote ()
-      ? fetch_manifest<signature_manifest> (
-          o, rl.proto (), rl.host (), rl.port (), f, iu).first
+      ? fetch_manifest<signature_manifest> (o, u, iu).first
       : fetch_manifest<signature_manifest> (nullptr, f, iu).first;
   }
 
@@ -804,19 +774,27 @@ namespace bpkg
     assert (!a.empty () && a.relative ());
     assert (rl.remote () || rl.absolute ());
 
-    path f (rl.path () / a);
+    repository_url u (rl.url ());
+
+    path& f (*u.path);
+    f /= a;
+
+    auto bad_loc = [&u] () {fail << "invalid archive location " << u;};
 
     try
     {
       f.normalize ();
+
+      if (*f.begin () == "..") // Can be the case for the remote location.
+        bad_loc ();
     }
     catch (const invalid_path&)
     {
-      fail << "invalid archive location " << rl << "/" << f;
+      bad_loc ();
     }
 
     return rl.remote ()
-      ? fetch_file (o, rl.proto (), rl.host (), rl.port (), f, d)
+      ? fetch_file (o, u, d)
       : fetch_file (f, d);
   }
 }
diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx
index 850ce59..031d51a 100644
--- a/bpkg/manifest-utility.cxx
+++ b/bpkg/manifest-utility.cxx
@@ -7,6 +7,7 @@
 #include <bpkg/diagnostics.hxx>
 
 using namespace std;
+using namespace butl;
 
 namespace bpkg
 {
@@ -65,19 +66,93 @@ namespace bpkg
   }
 
   repository_location
-  parse_location (const char* s)
+  parse_location (const char* s, optional<repository_type> ot)
   try
   {
-    repository_location rl (s, repository_location ());
+    repository_url u (s);
 
-    if (rl.relative ()) // Throws if the location is empty.
-      rl = repository_location (
-        dir_path (s).complete ().normalize ().string ());
+    if (u.empty ())
+      fail << "empty repository location";
 
-    return rl;
+    assert (u.path);
+
+    // Make the relative path absolute using the current directory.
+    //
+    if (u.scheme == repository_protocol::file && u.path->relative ())
+      u.path->complete ().normalize ();
+
+    // Guess the repository type to construct the repository location:
+    //
+    // 1. If type is specified as an option use that (but validate
+    //    incompatible scheme/type e.g., git/bpkg).
+    //
+    // 2. If scheme is git then git.
+    //
+    // 3. If scheme is http(s), then check if path has the .git extension,
+    //    then git, otherwise bpkg.
+    //
+    // 4. If local (which will normally be without the .git extension), check
+    //    if directory contains the .git/ subdirectory then git, otherwise
+    //    bpkg.
+    //
+    repository_type t;
+
+    if (ot)
+      t = *ot;
+    else
+    {
+      switch (u.scheme)
+      {
+      case repository_protocol::git:
+        {
+          t = repository_type::git;
+          break;
+        }
+      case repository_protocol::http:
+      case repository_protocol::https:
+        {
+          t = u.path->extension () == "git"
+            ? repository_type::git
+            : repository_type::bpkg;
+          break;
+        }
+      case repository_protocol::file:
+        {
+          t = exists (path_cast<dir_path> (*u.path) / dir_path (".git"))
+            ? repository_type::git
+            : repository_type::bpkg;
+          break;
+        }
+      }
+    }
+
+    try
+    {
+      // Don't move the URL since it may still be needed for diagnostics.
+      //
+      return repository_location (u, t);
+    }
+    catch (const invalid_argument& e)
+    {
+      diag_record dr;
+      dr << fail << "invalid " << t << " repository location '" << u << "': "
+         << e;
+
+      // If the bpkg repository type was guessed, then suggest the user to
+      // specify the type explicitly.
+      //
+      if (!ot && t == repository_type::bpkg)
+        dr << info << "consider using --type to specify repository type";
+
+      dr << endf;
+    }
   }
   catch (const invalid_argument& e)
   {
     fail << "invalid repository location '" << s << "': " << e << endf;
   }
+  catch (const invalid_path& e)
+  {
+    fail << "invalid repository path '" << s << "': " << e << endf;
+  }
 }
diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx
index afa93f5..f4dadb4 100644
--- a/bpkg/manifest-utility.hxx
+++ b/bpkg/manifest-utility.hxx
@@ -34,12 +34,12 @@ namespace bpkg
   version
   parse_package_version (const char*);
 
-  // First use the passed location as is. If the result is relative,
-  // then assume this is a relative path to the repository directory
-  // and complete it based on the current working directory.
+  // If the passed location is a relative local path, then assume this is a
+  // relative path to the repository directory and complete it based on the
+  // current working directory. Diagnose invalid locations and throw failed.
   //
   repository_location
-  parse_location (const char*);
+  parse_location (const char*, optional<repository_type>);
 }
 
 #endif // BPKG_MANIFEST_UTILITY_HXX
diff --git a/bpkg/package.hxx b/bpkg/package.hxx
index b5ad0f3..00a54d6 100644
--- a/bpkg/package.hxx
+++ b/bpkg/package.hxx
@@ -19,7 +19,7 @@
 #include <bpkg/types.hxx>
 #include <bpkg/utility.hxx>
 
-#pragma db model version(3, 3, closed)
+#pragma db model version(4, 4, open)
 
 namespace bpkg
 {
@@ -200,9 +200,24 @@ namespace bpkg
 
   // repository_location
   //
-  #pragma db map type(repository_location) as(string)     \
-    to((?).string ()) from(bpkg::repository_location (?))
 
+  #pragma db value
+  struct _repository_location
+  {
+    string url;
+    repository_type type;
+  };
+
+  // Note that the type() call fails for an empty repository location.
+  //
+  #pragma db map type(repository_location) as(_repository_location) \
+    to({(?).string (),                                              \
+        (?).empty () ? bpkg::repository_type::bpkg : (?).type ()})  \
+    from(bpkg::repository_location ((?).url, (?).type))
+
+  #pragma db map type(repository_type) as(string) \
+    to(to_string (?))                             \
+    from(bpkg::to_repository_type (?))
 
   // repository
   //
@@ -240,7 +255,7 @@ namespace bpkg
     //
     #pragma db member(name) id
 
-    #pragma db member(location)                                  \
+    #pragma db member(location) column("")                       \
       set(this.location = std::move (?);                         \
           assert (this.name == this.location.canonical_name ()))
 
diff --git a/bpkg/package.xml b/bpkg/package.xml
index 78a49d4..f77653f 100644
--- a/bpkg/package.xml
+++ b/bpkg/package.xml
@@ -1,8 +1,9 @@
 <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="sqlite" version="1">
-  <model version="3">
+  <model version="4">
     <table name="repository" kind="object">
       <column name="name" type="TEXT" null="true"/>
-      <column name="location" type="TEXT" null="true"/>
+      <column name="url" type="TEXT" null="true"/>
+      <column name="type" type="TEXT" null="true"/>
       <primary-key>
         <column name="name"/>
       </primary-key>
@@ -190,7 +191,8 @@
       <column name="substate" type="TEXT" null="true"/>
       <column name="hold_package" type="INTEGER" null="true"/>
       <column name="hold_version" type="INTEGER" null="true"/>
-      <column name="repository" type="TEXT" null="true"/>
+      <column name="repository_url" type="TEXT" null="true"/>
+      <column name="repository_type" type="TEXT" null="true"/>
       <column name="archive" type="TEXT" null="true"/>
       <column name="purge_archive" type="INTEGER" null="true"/>
       <column name="src_root" type="TEXT" null="true"/>
diff --git a/bpkg/rep-add.cli b/bpkg/rep-add.cli
index cbd73a4..eddea45 100644
--- a/bpkg/rep-add.cli
+++ b/bpkg/rep-add.cli
@@ -2,6 +2,8 @@
 // copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 // license   : MIT; see accompanying LICENSE file
 
+include <libbpkg/manifest.hxx>;
+
 include <bpkg/configuration.cli>;
 
 "\section=1"
@@ -28,5 +30,15 @@ namespace bpkg
   class rep_add_options: configuration_options
   {
     "\h|REP-ADD OPTIONS|"
+
+    repository_type --type
+    {
+      "<type>",
+      "Specify the repository type with valid values being \cb{bpkg} and
+       \cb{git}. Normally the repository type can be automatically guessed by
+       examining its URL (for example, the presence of the \cb{.git}
+       extension) or, in case of a local repository, its content (for example,
+       the presence of the \cb{.git} subdirectory)."
+    }
   };
 }
diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx
index 6de0738..d18536f 100644
--- a/bpkg/rep-add.cxx
+++ b/bpkg/rep-add.cxx
@@ -27,7 +27,12 @@ namespace bpkg
       fail << "repository location argument expected" <<
         info << "run 'bpkg help rep-add' for more information";
 
-    repository_location rl (parse_location (args.next ()));
+    repository_location rl (
+      parse_location (args.next (),
+                      o.type_specified ()
+                      ? optional<repository_type> (o.type ())
+                      : nullopt));
+
     const string& rn (rl.canonical_name ());
 
     // Create the new repository and add is as a complement to the root.
diff --git a/bpkg/rep-info.cli b/bpkg/rep-info.cli
index 8bad234..d2ce9b6 100644
--- a/bpkg/rep-info.cli
+++ b/bpkg/rep-info.cli
@@ -2,6 +2,8 @@
 // copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 // license   : MIT; see accompanying LICENSE file
 
+include <libbpkg/manifest.hxx>;
+
 include <bpkg/common.cli>;
 
 "\section=1"
@@ -91,6 +93,16 @@ namespace bpkg
        manifests."
     }
 
+    repository_type --type
+    {
+      "<type>",
+      "Specify the repository type with valid values being \cb{bpkg} and
+       \cb{git}. Normally the repository type can be automatically guessed by
+       examining its URL (for example, the presence of the \cb{.git}
+       extension) or, in case of a local repository, its content (for example,
+       the presence of the \cb{.git} subdirectory)."
+    }
+
     string --directory|-d // String to allow empty value.
     {
       "<dir>",
diff --git a/bpkg/rep-info.cxx b/bpkg/rep-info.cxx
index f6a8d2d..322ff6b 100644
--- a/bpkg/rep-info.cxx
+++ b/bpkg/rep-info.cxx
@@ -31,7 +31,11 @@ namespace bpkg
       fail << "repository location argument expected" <<
         info << "run 'bpkg help rep-info' for more information";
 
-    repository_location rl (parse_location (args.next ()));
+    repository_location rl (
+      parse_location (args.next (),
+                      o.type_specified ()
+                      ? optional<repository_type> (o.type ())
+                      : nullopt));
 
     // Fetch everything we will need before printing anything. Ignore
     // unknown manifest entries unless we are dumping them. First fetch
diff --git a/bpkg/types-parsers.cxx b/bpkg/types-parsers.cxx
index 6da395c..41b231f 100644
--- a/bpkg/types-parsers.cxx
+++ b/bpkg/types-parsers.cxx
@@ -67,5 +67,26 @@ namespace bpkg
       else
         throw invalid_value (o, v);
     }
+
+    void parser<repository_type>::
+    parse (repository_type& x, bool& xs, scanner& s)
+    {
+      xs = true;
+      const char* o (s.next ());
+
+      if (!s.more ())
+        throw missing_value (o);
+
+      const string v (s.next ());
+
+      try
+      {
+        x = to_repository_type(v);
+      }
+      catch (const invalid_argument&)
+      {
+        throw invalid_value (o, v);
+      }
+    }
   }
 }
diff --git a/bpkg/types-parsers.hxx b/bpkg/types-parsers.hxx
index bddb425..4f1dfc7 100644
--- a/bpkg/types-parsers.hxx
+++ b/bpkg/types-parsers.hxx
@@ -8,6 +8,8 @@
 #ifndef BPKG_TYPES_PARSERS_HXX
 #define BPKG_TYPES_PARSERS_HXX
 
+#include <libbpkg/manifest.hxx>
+
 #include <bpkg/types.hxx>
 #include <bpkg/options-types.hxx>
 
@@ -40,6 +42,13 @@ namespace bpkg
       static void
       parse (auth&, bool&, scanner&);
     };
+
+    template <>
+    struct parser<repository_type>
+    {
+      static void
+      parse (repository_type&, bool&, scanner&);
+    };
   }
 }
 
diff --git a/tests/rep-add.test b/tests/rep-add.test
index a1b9104..28a7a20 100644
--- a/tests/rep-add.test
+++ b/tests/rep-add.test
@@ -7,6 +7,8 @@
 : location
 :
 {
+  +$clone_cfg
+
   : none
   :
   $* 2>>EOE != 0
@@ -14,17 +16,89 @@
     info: run 'bpkg help rep-add' for more information
   EOE
 
+  : empty
+  :
+  $* '' 2>>EOE != 0
+  error: empty repository location
+  EOE
+
+  : unknown-type
+  :
+  $* 'repo' --type unknown 2>>EOE != 0
+  error: invalid value 'unknown' for option '--type'
+  EOE
+
   : no-version
   :
-  $* 'stable' 2>>EOE != 0
-  error: invalid repository location 'stable': missing repository version
+  $* 'stable' 2>>/~%EOE% != 0
+  %error: invalid bpkg repository location '.+/no-version/stable': missing repository version%
+    info: consider using --type to specify repository type
   EOE
 
-  : invalid-host
+  : git-no-branch
   :
-  $* 'http://' 2>>EOE != 0
-  error: invalid repository location 'http://': invalid host
+  $* 'git://example.org/repo' 2>>EOE != 0
+  error: invalid git repository location 'git://example.org/repo': missing branch/tag for git repository
   EOE
+
+  : bpkg-git-scheme
+  :
+  $* 'git://example.org/repo' --type bpkg 2>>EOE != 0
+  error: invalid bpkg repository location 'git://example.org/repo': unsupported scheme for bpkg repository
+  EOE
+
+  : invalid-path
+  :
+  {
+    s="../../../../../../../../../../../../../../../../../../../../../../../"
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s"
+    $* "$s" 2>>~%EOE% != 0
+    %error: invalid repository path '.+/': invalid filesystem path%
+    EOE
+  }
+
+  : type-detection
+  :
+  {
+    +$clone_cfg
+
+    : git-scheme
+    :
+    $clone_cfg;
+    $* 'git://example.org/repo#master' 2>>EOE
+      added repository example.org/repo
+      EOE
+
+    : http-git
+    :
+    $clone_cfg;
+    $* 'http://example.org/repo.git#master' 2>>EOE
+      added repository example.org/repo
+      EOE
+
+    : http-bpkg
+    :
+    $clone_cfg;
+    $* 'http://example.org/1/repo' 2>>EOE
+      added repository example.org/repo
+      EOE
+
+    : file-git
+    :
+    $clone_cfg && mkdir -p repo/.git;
+
+    $* 'repo' 2>>~%EOE% != 0
+      %error: invalid git repository location '.+repo': missing branch/tag for git repository%
+      EOE
+
+    : file-bpkg
+    :
+    $clone_cfg;
+
+    $* '1/repo' 2>>/~%EOE%
+      %added repository .+/repo%
+      EOE
+  }
 }
 
 : relative-path