Implement archive fetching, complete pkg-fetch

7 files changed, 312 insertions, 69 deletions

diff --git a/bpkg/fetch b/bpkg/fetch
index 49d8cd9..9a58cc3 100644
--- a/bpkg/fetch
+++ b/bpkg/fetch
@@ -20,6 +20,12 @@ namespace bpkg
   package_manifests fetch_packages (const dir_path&);
   package_manifests fetch_packages (const common_options&,
                                     const repository_location&);
+
+  path
+  fetch_archive (const common_options&,
+                 const repository_location&,
+                 const path& archive,
+                 const dir_path& destdir);
 }
 
 #endif // BPKG_FETCH
diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx
index 10acf1f..3ddd2e9 100644
--- a/bpkg/fetch.cxx
+++ b/bpkg/fetch.cxx
@@ -85,8 +85,12 @@ namespace bpkg
   }
 
   static process
-  start_wget (const path& prog, const strings& ops, const string& url)
+  start_wget (const path& prog,
+              const strings& ops,
+              const string& url,
+              const path& out)
   {
+    bool fo (!out.empty ()); // Output to file.
     string ua (BPKG_USER_AGENT " wget/" + to_string (wget_major) + "."
                + to_string (wget_minor));
 
@@ -95,11 +99,12 @@ namespace bpkg
       "-U", ua.c_str ()
     };
 
-    // Map verbosity level. If we are running quiet or at level 1,
-    // then run wget quiet. At level 2 and 3 run it at the default
-    // level (so we will print the command line and it will display
-    // the progress, error messages, etc). Higher than that -- run
-    // it with debug output.
+    // Map verbosity level. If we are running quiet or at level 1
+    // and the output is STDOUT, then run wget quiet. If at level
+    // 1 and the output is a file, then show the progress bar. At
+    // level 2 and 3 run it at the default level (so we will print
+    // the command line and it will display the progress, error
+    // messages, etc). Higher than that -- run it with debug output.
     //
     // In the wget world quiet means don't print anything, not even
     // error messages. There is also the -nv mode (aka "non-verbose")
@@ -108,8 +113,20 @@ namespace bpkg
     // If things go south, we suggest (in fetch_url()) below that the
     // user re-runs the command with -v to see all the gory details.
     //
-    if (verb < 2)
+    if (verb < (fo ? 1 : 2))
       args.push_back ("-q");
+    else if (fo && verb == 1)
+    {
+      // Wget 1.16 introduced the --show-progress option which in the
+      // quiet mode shows a nice and tidy progress bar (if only it also
+      // showed errors, then it would have been perfect).
+      //
+      if (wget_major > 1 || (wget_major == 1 && wget_minor >= 16))
+      {
+        args.push_back ("-q");
+        args.push_back ("--show-progress");
+      }
+    }
     else if (verb > 3)
       args.push_back ("-d");
 
@@ -119,15 +136,26 @@ namespace bpkg
     for (const string& o: ops)
       args.push_back (o.c_str ());
 
-    args.push_back ("-O");         // Output to...
-    args.push_back ("-");          // ...STDOUT.
+    // Output.
+    //
+    string o (fo ? out.leaf ().string () : "-");
+    args.push_back ("-O");
+    args.push_back (o.c_str ());
+
     args.push_back (url.c_str ());
     args.push_back (nullptr);
 
     if (verb >= 2)
       print_process (args);
 
-    return process (args.data (), 0, -1); // Failure handled by the caller.
+    // If we are fetching into a file, change the wget's directory to
+    // that of the output file. We do it this way so that we end up with
+    // just the file name (rather than the whole path) in the progress
+    // report. Process exceptions must be handled by the caller.
+    //
+    return fo
+      ? process (out.directory ().string ().c_str (), args.data ())
+      : process (args.data (), 0, -1);
   }
 
   // curl
@@ -163,8 +191,13 @@ namespace bpkg
   }
 
   static process
-  start_curl (const path& prog, const strings& ops, const string& url)
+  start_curl (const path& prog,
+              const strings& ops,
+              const string& url,
+              const path& out)
   {
+    bool fo (!out.empty ()); // Output to file.
+
     cstrings args {
       prog.string ().c_str (),
       "-f", // Fail on HTTP errors (e.g., 404).
@@ -172,16 +205,20 @@ namespace bpkg
       "-A", (BPKG_USER_AGENT " curl")
     };
 
-    // Map verbosity level. If we are running quiet or at level 1,
-    // then run curl quiet. At level 2 and 3 run it at the default
-    // level (so we will print the command line and it will display
-    // the progress). Higher than that -- run it verbose.
+    // Map verbosity level. If we are running quiet or at level 1
+    // and the output is STDOUT, then run curl quiet. If at level
+    // 1 and the output is a file, then show the progress bar. At
+    // level 2 and 3 run it at the default level (so we will print
+    // the command line and it will display its elaborate progress).
+    // Higher than that -- run it verbose.
     //
-    if (verb < 2)
+    if (verb < (fo ? 1 : 2))
     {
       args.push_back ("-s");
       args.push_back ("-S"); // But show errors.
     }
+    else if (fo && verb == 1)
+      args.push_back ("--progress-bar");
     else if (verb > 3)
       args.push_back ("-v");
 
@@ -191,13 +228,32 @@ namespace bpkg
     for (const string& o: ops)
       args.push_back (o.c_str ());
 
+    // Output. By default curl writes to STDOUT.
+    //
+    if (fo)
+    {
+      args.push_back ("-o");
+      args.push_back (out.string ().c_str ());
+    }
+
     args.push_back (url.c_str ());
     args.push_back (nullptr);
 
     if (verb >= 2)
       print_process (args);
+    else if (verb == 1 && fo)
+      //
+      // Unfortunately curl doesn't print the filename being fetched
+      // next to the progress bar. So the best we can do is print it
+      // on the previous line. Ugly, I know.
+      //
+      text << out.leaf () << ':';
 
-    return process (args.data (), 0, -1); // Failure handled by the caller.
+    // Process exceptions must be handled by the caller.
+    //
+    return fo
+      ? process (args.data ())
+      : process (args.data (), 0, -1);
   }
 
   // fetch
@@ -235,22 +291,27 @@ namespace bpkg
   }
 
   static process
-  start_fetch (const path& prog, const strings& ops, const string& url)
+  start_fetch (const path& prog,
+               const strings& ops,
+               const string& url,
+               const path& out)
   {
-    // -T|--timeout   120 seconds by default, leave it at that for now.
-    // -n|--no-mtime
-    //
+    bool fo (!out.empty ()); // Output to file.
+
     cstrings args {
       prog.string ().c_str (),
       "--user-agent", (BPKG_USER_AGENT " fetch")
     };
 
-    // Map verbosity level. If we are running quiet or at level 1,
-    // then run fetch quiet. At level 2 and 3 run it at the default
-    // level (so we will print the command line and it will display
+    if (fo)
+      args.push_back ("--no-mtime"); // Use our own mtime.
+
+    // Map verbosity level. If we are running quiet then run fetch quiet.
+    // If we are at level 1 and we are fetching into a file or we are at
+    // level 2 or 3, then run it at the default level (so it will display
     // the progress). Higher than that -- run it verbose.
     //
-    if (verb < 2)
+    if (verb < (fo ? 1 : 2))
       args.push_back ("-q");
     else if (verb > 3)
       args.push_back ("-v");
@@ -261,15 +322,26 @@ namespace bpkg
     for (const string& o: ops)
       args.push_back (o.c_str ());
 
-    args.push_back ("-o");         // Output to...
-    args.push_back ("-");          // ...STDOUT.
+    // Output.
+    //
+    string o (fo ? out.leaf ().string () : "-");
+    args.push_back ("-o");
+    args.push_back (o.c_str ());
+
     args.push_back (url.c_str ());
     args.push_back (nullptr);
 
     if (verb >= 2)
       print_process (args);
 
-    return process (args.data (), 0, -1); // Failure handled by the caller.
+    // If we are fetching into a file, change the fetch's directory to
+    // that of the output file. We do it this way so that we end up with
+    // just the file name (rather than the whole path) in the progress
+    // report. Process exceptions must be handled by the caller.
+    //
+    return fo
+      ? process (out.directory ().string ().c_str (), args.data ())
+      : process (args.data (), 0, -1);
   }
 
   // The dispatcher.
@@ -361,21 +433,25 @@ namespace bpkg
     return fetch_kind;
   }
 
+  // If out is empty, then fetch to STDOUT. In this case also don't
+  // show any progress unless we are running verbose.
+  //
   static process
-  start (const common_options& o, const string& url)
+  start (const common_options& o, const string& url, const path& out = path ())
   {
-    process (*start) (const path&, const strings&, const string&) = nullptr;
+    process (*f) (
+      const path&, const strings&, const string&, const path&) = nullptr;
 
     switch (check (o))
     {
-    case wget:  start = &start_wget;  break;
-    case curl:  start = &start_curl;  break;
-    case fetch: start = &start_fetch; break;
+    case wget:  f = &start_wget;  break;
+    case curl:  f = &start_curl;  break;
+    case fetch: f = &start_fetch; break;
     }
 
     try
     {
-      return start (fetch_path, o.fetch_option (), url);
+      return f (fetch_path, o.fetch_option (), url, out);
     }
     catch (const process_error& e)
     {
@@ -388,15 +464,9 @@ namespace bpkg
     }
   }
 
-  template <typename M>
-  static M
-  fetch_url (const common_options& o,
-             const string& host,
-             uint16_t port,
-             const path& file)
+  static string
+  to_url (const string& host, uint16_t port, const path& file)
   {
-    // Assemble the URL.
-    //
     //@@ Absolute path in URL: how is this going to work on Windows?
     //   Change to relative: watch for empty path.
     //
@@ -409,7 +479,44 @@ namespace bpkg
       url += ":" + to_string (port);
 
     url += file.posix_string ();
+    return url;
+  }
+
+  static path
+  fetch_file (const common_options& o,
+              const string& host,
+              uint16_t port,
+              const path& f,
+              const dir_path& d)
+  {
+    path r (d / f.leaf ());
+
+    if (exists (r))
+      fail << "file " << r << " already exists";
+
+    string url (to_url (host, port, f));
+    process pr (start (o, url, r));
+
+    if (!pr.wait ())
+    {
+      // While it is reasonable to assuming the child process issued
+      // diagnostics, some may not mention the URL.
+      //
+      fail << "unable to fetch " << url <<
+        info << "re-run with -v for more information";
+    }
 
+    return r;
+  }
+
+  template <typename M>
+  static M
+  fetch_manifest (const common_options& o,
+                  const string& host,
+                  uint16_t port,
+                  const path& f)
+  {
+    string url (to_url (host, port, f));
     process pr (start (o, url));
 
     try
@@ -452,9 +559,48 @@ namespace bpkg
     throw failed ();
   }
 
+  static path
+  fetch_file (const path& f, const dir_path& d)
+  {
+    if (!exists (f))
+      fail << "file " << f << " does not exist";
+
+    path r (d / f.leaf ());
+
+    if (exists (r))
+      fail << "file " << r << " already exists";
+
+    try
+    {
+      ifstream ifs (f.string (), ios::binary);
+      if (!ifs.is_open ())
+        fail << "unable to open " << f << " in read mode";
+
+      ofstream ofs (r.string (), ios::binary);
+      if (!ofs.is_open ())
+        fail << "unable to open " << r << " in write mode";
+
+      ifs.exceptions (ofstream::badbit | ofstream::failbit);
+      ofs.exceptions (ofstream::badbit | ofstream::failbit);
+
+      ofs << ifs.rdbuf();
+
+      // In case they throw.
+      //
+      ifs.close ();
+      ofs.close ();
+    }
+    catch (const iostream::failure&)
+    {
+      fail << "unable to copy " << f << " to " << r << ": io error";
+    }
+
+    return r;
+  }
+
   template <typename M>
   static M
-  fetch_file (const path& f)
+  fetch_manifest (const path& f)
   {
     if (!exists (f))
       fail << "file " << f << " does not exist";
@@ -485,7 +631,7 @@ namespace bpkg
   repository_manifests
   fetch_repositories (const dir_path& d)
   {
-    return fetch_file<repository_manifests> (d / repositories);
+    return fetch_manifest<repository_manifests> (d / repositories);
   }
 
   repository_manifests
@@ -496,8 +642,8 @@ namespace bpkg
     path f (rl.path () / repositories);
 
     return rl.remote ()
-      ? fetch_url<repository_manifests> (o, rl.host (), rl.port (), f)
-      : fetch_file<repository_manifests> (f);
+      ? fetch_manifest<repository_manifests> (o, rl.host (), rl.port (), f)
+      : fetch_manifest<repository_manifests> (f);
   }
 
   static const path packages ("packages");
@@ -505,7 +651,7 @@ namespace bpkg
   package_manifests
   fetch_packages (const dir_path& d)
   {
-    return fetch_file<package_manifests> (d / packages);
+    return fetch_manifest<package_manifests> (d / packages);
   }
 
   package_manifests
@@ -516,7 +662,23 @@ namespace bpkg
     path f (rl.path () / packages);
 
     return rl.remote ()
-      ? fetch_url<package_manifests> (o, rl.host (), rl.port (), f)
-      : fetch_file<package_manifests> (f);
+      ? fetch_manifest<package_manifests> (o, rl.host (), rl.port (), f)
+      : fetch_manifest<package_manifests> (f);
+  }
+
+  path
+  fetch_archive (const common_options& o,
+                 const repository_location& rl,
+                 const path& a,
+                 const dir_path& d)
+  {
+    assert (a.relative ());
+    assert (rl.remote () || rl.absolute ());
+
+    path f (rl.path () / a);
+
+    return rl.remote ()
+      ? fetch_file (o, rl.host (), rl.port (), f, d)
+      : fetch_file (f, d);
   }
 }
diff --git a/bpkg/package b/bpkg/package
index f387d35..2601b7b 100644
--- a/bpkg/package
+++ b/bpkg/package
@@ -146,6 +146,8 @@ namespace bpkg
   {
     #pragma db column("count(" + repository::id.name + ")")
     size_t result;
+
+    operator size_t () const {return result;}
   };
 
   // package_version_id
@@ -237,6 +239,8 @@ namespace bpkg
   {
     #pragma db column("count(" + available_package::id.data.name + ")")
     size_t result;
+
+    operator size_t () const {return result;}
   };
 
   // state
diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx
index 63dd31f..bfa6e54 100644
--- a/bpkg/pkg-fetch.cxx
+++ b/bpkg/pkg-fetch.cxx
@@ -6,6 +6,7 @@
 
 #include <bpkg/manifest>
 
+#include <bpkg/fetch>
 #include <bpkg/types>
 #include <bpkg/package>
 #include <bpkg/package-odb>
@@ -29,6 +30,8 @@ namespace bpkg
     level4 ([&]{trace << "configuration: " << c;});
 
     database db (open (c, trace));
+    transaction t (db.begin ());
+    session s;
 
     path a;
     bool purge;
@@ -52,21 +55,60 @@ namespace bpkg
         fail << "package name argument expected" <<
           info << "run 'bpkg help pkg-fetch' for more information";
 
-      string name (args.next ());
+      string n (args.next ());
 
       if (!args.more ())
         fail << "package version argument expected" <<
           info << "run 'bpkg help pkg-fetch' for more information";
 
-      string ver (args.next ());
-
-      // TODO:
+      //@@ Same code as in pkg-status. Similar problem to repo_location;
+      //   need a place for such utilities.
       //
-      // - Will need to use some kind or auto_rm/exception guard on
-      //   fetched file.
+      version v;
+      {
+        const char* s (args.next ());
+        try
+        {
+          v = version (s);
+        }
+        catch (const invalid_argument& e)
+        {
+          fail << "invalid package version '" << s << "': " << e.what ();
+        }
+      }
+
+      if (db.query_value<repository_count> () == 0)
+        fail << "configuration " << c << " has no repositories" <<
+          info << "use 'bpkg rep-add' to add a repository";
+
+      if (db.query_value<available_package_count> () == 0)
+        fail << "configuration " << c << " has no available packages" <<
+          info << "use 'bpkg rep-fetch' to fetch available packages list";
+
+      shared_ptr<available_package> p (
+        db.find<available_package> (package_version_id (n, v)));
+
+      if (p == nullptr)
+        fail << "package " << n << " " << v << " is not available";
+
+      // Pick a repository. Prefer local ones over the remote.
       //
-
-      fail << "pkg-fetch " << name << " " << ver << " not yet implemented";
+      const package_location* pl (&p->locations.front ());
+
+      for (const package_location& l: p->locations)
+      {
+        if (!l.repository.load ()->location.remote ())
+        {
+          pl = &l;
+          break;
+        }
+      }
+
+      if (verb > 1)
+        text << "fetching " << pl->location.leaf () << " "
+             << "from " << pl->repository->name ();
+
+      a = fetch_archive (o, pl->repository->location, pl->location, c);
       purge = true;
     }
 
@@ -79,10 +121,6 @@ namespace bpkg
 
     const auto& n (m.name);
 
-    // Database time.
-    //
-    transaction t (db.begin ());
-
     // See if this package already exists in this configuration.
     //
     if (shared_ptr<package> p = db.find<package> (n))
diff --git a/bpkg/pkg-status.cxx b/bpkg/pkg-status.cxx
index a5eab3e..8daf781 100644
--- a/bpkg/pkg-status.cxx
+++ b/bpkg/pkg-status.cxx
@@ -42,7 +42,7 @@ namespace bpkg
       }
       catch (const invalid_argument& e)
       {
-        fail << "invalid package version '" << s << "'";
+        fail << "invalid package version '" << s << "': " << e.what ();
       }
     }
 
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index e112bf7..641efb2 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -166,7 +166,7 @@ namespace bpkg
     const auto& ua (root->complements); // User-added repositories.
 
     if (ua.empty ())
-      fail << "configuration has no repositories" <<
+      fail << "configuration " << c << " has no repositories" <<
         info << "use 'bpkg rep-add' to add a repository";
 
     // Clean repositories and available packages. At the end only
@@ -206,8 +206,8 @@ namespace bpkg
     size_t rcount, pcount;
     if (verb)
     {
-      rcount = db.query_value<repository_count> ().result;
-      pcount = db.query_value<available_package_count> ().result;
+      rcount = db.query_value<repository_count> ();
+      pcount = db.query_value<available_package_count> ();
     }
 
     t.commit ();
diff --git a/bpkg/test.sh b/bpkg/test.sh
index d1ab6f2..63b6fb2 100755
--- a/bpkg/test.sh
+++ b/bpkg/test.sh
@@ -163,17 +163,50 @@ test cfg-create --wipe
 test rep-add http://pkg.cppget.org/1/hello
 test rep-fetch
 
-## @@
 ##
+## pkg-fetch
 ##
 
-test cfg-create --wipe config.cxx=g++-4.9 cxx config.install.root=/tmp/install
-stat unknown
+test cfg-create --wipe
+
+fail pkg-fetch -e                # archive expected
+fail pkg-fetch -e ./no-such-file # archive does not exist
+
+fail pkg-fetch                  # package name expected
+fail pkg-fetch libhello         # package version expected
+fail pkg-fetch libhello 1/2/3   # invalid package version
 
+fail pkg-fetch libhello 1.0.0   # no repositories
+test rep-add $rep
+fail pkg-fetch libhello 1.0.0   # no packages
+test rep-fetch
+fail pkg-fetch libhello 2+1.0.0 # not available
+
+# local
+#
+test cfg-create --wipe
+test rep-add $rep
+test rep-fetch
+test pkg-fetch libhello 1.0.0
+test pkg-unpack libhello
+test pkg-purge libhello
+
+# remote
+#
+test cfg-create --wipe
+test rep-add http://pkg.cppget.org/1/hello
+test rep-fetch
+#test pkg-fetch libheavy 1.0.0
+test pkg-fetch libhello 1.0.0
+test pkg-unpack libhello
+test pkg-purge libhello
+
+## @@
 ##
-## pkg-fetch
 ##
 
+test cfg-create --wipe config.cxx=g++-4.9 cxx config.install.root=/tmp/install
+stat unknown
 
 # fetch existing archive
 #