diff options
| -rw-r--r-- | bpkg/fetch | 6 | ||||
| -rw-r--r-- | bpkg/fetch.cxx | 260 | ||||
| -rw-r--r-- | bpkg/package | 4 | ||||
| -rw-r--r-- | bpkg/pkg-fetch.cxx | 62 | ||||
| -rw-r--r-- | bpkg/pkg-status.cxx | 2 | ||||
| -rw-r--r-- | bpkg/rep-fetch.cxx | 6 | ||||
| -rwxr-xr-x | bpkg/test.sh | 41 |
7 files changed, 312 insertions, 69 deletions
@@ -20,6 +20,12 @@ namespace bpkg package_manifests fetch_packages (const dir_path&); package_manifests fetch_packages (const common_options&, const repository_location&); + + path + fetch_archive (const common_options&, + const repository_location&, + const path& archive, + const dir_path& destdir); } #endif // BPKG_FETCH diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx index 10acf1f..3ddd2e9 100644 --- a/bpkg/fetch.cxx +++ b/bpkg/fetch.cxx @@ -85,8 +85,12 @@ namespace bpkg } static process - start_wget (const path& prog, const strings& ops, const string& url) + start_wget (const path& prog, + const strings& ops, + const string& url, + const path& out) { + bool fo (!out.empty ()); // Output to file. string ua (BPKG_USER_AGENT " wget/" + to_string (wget_major) + "." + to_string (wget_minor)); @@ -95,11 +99,12 @@ namespace bpkg "-U", ua.c_str () }; - // Map verbosity level. If we are running quiet or at level 1, - // then run wget quiet. At level 2 and 3 run it at the default - // level (so we will print the command line and it will display - // the progress, error messages, etc). Higher than that -- run - // it with debug output. + // Map verbosity level. If we are running quiet or at level 1 + // and the output is STDOUT, then run wget quiet. If at level + // 1 and the output is a file, then show the progress bar. At + // level 2 and 3 run it at the default level (so we will print + // the command line and it will display the progress, error + // messages, etc). Higher than that -- run it with debug output. // // In the wget world quiet means don't print anything, not even // error messages. There is also the -nv mode (aka "non-verbose") @@ -108,8 +113,20 @@ namespace bpkg // If things go south, we suggest (in fetch_url()) below that the // user re-runs the command with -v to see all the gory details. // - if (verb < 2) + if (verb < (fo ? 1 : 2)) args.push_back ("-q"); + else if (fo && verb == 1) + { + // Wget 1.16 introduced the --show-progress option which in the + // quiet mode shows a nice and tidy progress bar (if only it also + // showed errors, then it would have been perfect). + // + if (wget_major > 1 || (wget_major == 1 && wget_minor >= 16)) + { + args.push_back ("-q"); + args.push_back ("--show-progress"); + } + } else if (verb > 3) args.push_back ("-d"); @@ -119,15 +136,26 @@ namespace bpkg for (const string& o: ops) args.push_back (o.c_str ()); - args.push_back ("-O"); // Output to... - args.push_back ("-"); // ...STDOUT. + // Output. + // + string o (fo ? out.leaf ().string () : "-"); + args.push_back ("-O"); + args.push_back (o.c_str ()); + args.push_back (url.c_str ()); args.push_back (nullptr); if (verb >= 2) print_process (args); - return process (args.data (), 0, -1); // Failure handled by the caller. + // If we are fetching into a file, change the wget's directory to + // that of the output file. We do it this way so that we end up with + // just the file name (rather than the whole path) in the progress + // report. Process exceptions must be handled by the caller. + // + return fo + ? process (out.directory ().string ().c_str (), args.data ()) + : process (args.data (), 0, -1); } // curl @@ -163,8 +191,13 @@ namespace bpkg } static process - start_curl (const path& prog, const strings& ops, const string& url) + start_curl (const path& prog, + const strings& ops, + const string& url, + const path& out) { + bool fo (!out.empty ()); // Output to file. + cstrings args { prog.string ().c_str (), "-f", // Fail on HTTP errors (e.g., 404). @@ -172,16 +205,20 @@ namespace bpkg "-A", (BPKG_USER_AGENT " curl") }; - // Map verbosity level. If we are running quiet or at level 1, - // then run curl quiet. At level 2 and 3 run it at the default - // level (so we will print the command line and it will display - // the progress). Higher than that -- run it verbose. + // Map verbosity level. If we are running quiet or at level 1 + // and the output is STDOUT, then run curl quiet. If at level + // 1 and the output is a file, then show the progress bar. At + // level 2 and 3 run it at the default level (so we will print + // the command line and it will display its elaborate progress). + // Higher than that -- run it verbose. // - if (verb < 2) + if (verb < (fo ? 1 : 2)) { args.push_back ("-s"); args.push_back ("-S"); // But show errors. } + else if (fo && verb == 1) + args.push_back ("--progress-bar"); else if (verb > 3) args.push_back ("-v"); @@ -191,13 +228,32 @@ namespace bpkg for (const string& o: ops) args.push_back (o.c_str ()); + // Output. By default curl writes to STDOUT. + // + if (fo) + { + args.push_back ("-o"); + args.push_back (out.string ().c_str ()); + } + args.push_back (url.c_str ()); args.push_back (nullptr); if (verb >= 2) print_process (args); + else if (verb == 1 && fo) + // + // Unfortunately curl doesn't print the filename being fetched + // next to the progress bar. So the best we can do is print it + // on the previous line. Ugly, I know. + // + text << out.leaf () << ':'; - return process (args.data (), 0, -1); // Failure handled by the caller. + // Process exceptions must be handled by the caller. + // + return fo + ? process (args.data ()) + : process (args.data (), 0, -1); } // fetch @@ -235,22 +291,27 @@ namespace bpkg } static process - start_fetch (const path& prog, const strings& ops, const string& url) + start_fetch (const path& prog, + const strings& ops, + const string& url, + const path& out) { - // -T|--timeout 120 seconds by default, leave it at that for now. - // -n|--no-mtime - // + bool fo (!out.empty ()); // Output to file. + cstrings args { prog.string ().c_str (), "--user-agent", (BPKG_USER_AGENT " fetch") }; - // Map verbosity level. If we are running quiet or at level 1, - // then run fetch quiet. At level 2 and 3 run it at the default - // level (so we will print the command line and it will display + if (fo) + args.push_back ("--no-mtime"); // Use our own mtime. + + // Map verbosity level. If we are running quiet then run fetch quiet. + // If we are at level 1 and we are fetching into a file or we are at + // level 2 or 3, then run it at the default level (so it will display // the progress). Higher than that -- run it verbose. // - if (verb < 2) + if (verb < (fo ? 1 : 2)) args.push_back ("-q"); else if (verb > 3) args.push_back ("-v"); @@ -261,15 +322,26 @@ namespace bpkg for (const string& o: ops) args.push_back (o.c_str ()); - args.push_back ("-o"); // Output to... - args.push_back ("-"); // ...STDOUT. + // Output. + // + string o (fo ? out.leaf ().string () : "-"); + args.push_back ("-o"); + args.push_back (o.c_str ()); + args.push_back (url.c_str ()); args.push_back (nullptr); if (verb >= 2) print_process (args); - return process (args.data (), 0, -1); // Failure handled by the caller. + // If we are fetching into a file, change the fetch's directory to + // that of the output file. We do it this way so that we end up with + // just the file name (rather than the whole path) in the progress + // report. Process exceptions must be handled by the caller. + // + return fo + ? process (out.directory ().string ().c_str (), args.data ()) + : process (args.data (), 0, -1); } // The dispatcher. @@ -361,21 +433,25 @@ namespace bpkg return fetch_kind; } + // If out is empty, then fetch to STDOUT. In this case also don't + // show any progress unless we are running verbose. + // static process - start (const common_options& o, const string& url) + start (const common_options& o, const string& url, const path& out = path ()) { - process (*start) (const path&, const strings&, const string&) = nullptr; + process (*f) ( + const path&, const strings&, const string&, const path&) = nullptr; switch (check (o)) { - case wget: start = &start_wget; break; - case curl: start = &start_curl; break; - case fetch: start = &start_fetch; break; + case wget: f = &start_wget; break; + case curl: f = &start_curl; break; + case fetch: f = &start_fetch; break; } try { - return start (fetch_path, o.fetch_option (), url); + return f (fetch_path, o.fetch_option (), url, out); } catch (const process_error& e) { @@ -388,15 +464,9 @@ namespace bpkg } } - template <typename M> - static M - fetch_url (const common_options& o, - const string& host, - uint16_t port, - const path& file) + static string + to_url (const string& host, uint16_t port, const path& file) { - // Assemble the URL. - // //@@ Absolute path in URL: how is this going to work on Windows? // Change to relative: watch for empty path. // @@ -409,7 +479,44 @@ namespace bpkg url += ":" + to_string (port); url += file.posix_string (); + return url; + } + + static path + fetch_file (const common_options& o, + const string& host, + uint16_t port, + const path& f, + const dir_path& d) + { + path r (d / f.leaf ()); + + if (exists (r)) + fail << "file " << r << " already exists"; + + string url (to_url (host, port, f)); + process pr (start (o, url, r)); + + if (!pr.wait ()) + { + // While it is reasonable to assuming the child process issued + // diagnostics, some may not mention the URL. + // + fail << "unable to fetch " << url << + info << "re-run with -v for more information"; + } + return r; + } + + template <typename M> + static M + fetch_manifest (const common_options& o, + const string& host, + uint16_t port, + const path& f) + { + string url (to_url (host, port, f)); process pr (start (o, url)); try @@ -452,9 +559,48 @@ namespace bpkg throw failed (); } + static path + fetch_file (const path& f, const dir_path& d) + { + if (!exists (f)) + fail << "file " << f << " does not exist"; + + path r (d / f.leaf ()); + + if (exists (r)) + fail << "file " << r << " already exists"; + + try + { + ifstream ifs (f.string (), ios::binary); + if (!ifs.is_open ()) + fail << "unable to open " << f << " in read mode"; + + ofstream ofs (r.string (), ios::binary); + if (!ofs.is_open ()) + fail << "unable to open " << r << " in write mode"; + + ifs.exceptions (ofstream::badbit | ofstream::failbit); + ofs.exceptions (ofstream::badbit | ofstream::failbit); + + ofs << ifs.rdbuf(); + + // In case they throw. + // + ifs.close (); + ofs.close (); + } + catch (const iostream::failure&) + { + fail << "unable to copy " << f << " to " << r << ": io error"; + } + + return r; + } + template <typename M> static M - fetch_file (const path& f) + fetch_manifest (const path& f) { if (!exists (f)) fail << "file " << f << " does not exist"; @@ -485,7 +631,7 @@ namespace bpkg repository_manifests fetch_repositories (const dir_path& d) { - return fetch_file<repository_manifests> (d / repositories); + return fetch_manifest<repository_manifests> (d / repositories); } repository_manifests @@ -496,8 +642,8 @@ namespace bpkg path f (rl.path () / repositories); return rl.remote () - ? fetch_url<repository_manifests> (o, rl.host (), rl.port (), f) - : fetch_file<repository_manifests> (f); + ? fetch_manifest<repository_manifests> (o, rl.host (), rl.port (), f) + : fetch_manifest<repository_manifests> (f); } static const path packages ("packages"); @@ -505,7 +651,7 @@ namespace bpkg package_manifests fetch_packages (const dir_path& d) { - return fetch_file<package_manifests> (d / packages); + return fetch_manifest<package_manifests> (d / packages); } package_manifests @@ -516,7 +662,23 @@ namespace bpkg path f (rl.path () / packages); return rl.remote () - ? fetch_url<package_manifests> (o, rl.host (), rl.port (), f) - : fetch_file<package_manifests> (f); + ? fetch_manifest<package_manifests> (o, rl.host (), rl.port (), f) + : fetch_manifest<package_manifests> (f); + } + + path + fetch_archive (const common_options& o, + const repository_location& rl, + const path& a, + const dir_path& d) + { + assert (a.relative ()); + assert (rl.remote () || rl.absolute ()); + + path f (rl.path () / a); + + return rl.remote () + ? fetch_file (o, rl.host (), rl.port (), f, d) + : fetch_file (f, d); } } diff --git a/bpkg/package b/bpkg/package index f387d35..2601b7b 100644 --- a/bpkg/package +++ b/bpkg/package @@ -146,6 +146,8 @@ namespace bpkg { #pragma db column("count(" + repository::id.name + ")") size_t result; + + operator size_t () const {return result;} }; // package_version_id @@ -237,6 +239,8 @@ namespace bpkg { #pragma db column("count(" + available_package::id.data.name + ")") size_t result; + + operator size_t () const {return result;} }; // state diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx index 63dd31f..bfa6e54 100644 --- a/bpkg/pkg-fetch.cxx +++ b/bpkg/pkg-fetch.cxx @@ -6,6 +6,7 @@ #include <bpkg/manifest> +#include <bpkg/fetch> #include <bpkg/types> #include <bpkg/package> #include <bpkg/package-odb> @@ -29,6 +30,8 @@ namespace bpkg level4 ([&]{trace << "configuration: " << c;}); database db (open (c, trace)); + transaction t (db.begin ()); + session s; path a; bool purge; @@ -52,21 +55,60 @@ namespace bpkg fail << "package name argument expected" << info << "run 'bpkg help pkg-fetch' for more information"; - string name (args.next ()); + string n (args.next ()); if (!args.more ()) fail << "package version argument expected" << info << "run 'bpkg help pkg-fetch' for more information"; - string ver (args.next ()); - - // TODO: + //@@ Same code as in pkg-status. Similar problem to repo_location; + // need a place for such utilities. // - // - Will need to use some kind or auto_rm/exception guard on - // fetched file. + version v; + { + const char* s (args.next ()); + try + { + v = version (s); + } + catch (const invalid_argument& e) + { + fail << "invalid package version '" << s << "': " << e.what (); + } + } + + if (db.query_value<repository_count> () == 0) + fail << "configuration " << c << " has no repositories" << + info << "use 'bpkg rep-add' to add a repository"; + + if (db.query_value<available_package_count> () == 0) + fail << "configuration " << c << " has no available packages" << + info << "use 'bpkg rep-fetch' to fetch available packages list"; + + shared_ptr<available_package> p ( + db.find<available_package> (package_version_id (n, v))); + + if (p == nullptr) + fail << "package " << n << " " << v << " is not available"; + + // Pick a repository. Prefer local ones over the remote. // - - fail << "pkg-fetch " << name << " " << ver << " not yet implemented"; + const package_location* pl (&p->locations.front ()); + + for (const package_location& l: p->locations) + { + if (!l.repository.load ()->location.remote ()) + { + pl = &l; + break; + } + } + + if (verb > 1) + text << "fetching " << pl->location.leaf () << " " + << "from " << pl->repository->name (); + + a = fetch_archive (o, pl->repository->location, pl->location, c); purge = true; } @@ -79,10 +121,6 @@ namespace bpkg const auto& n (m.name); - // Database time. - // - transaction t (db.begin ()); - // See if this package already exists in this configuration. // if (shared_ptr<package> p = db.find<package> (n)) diff --git a/bpkg/pkg-status.cxx b/bpkg/pkg-status.cxx index a5eab3e..8daf781 100644 --- a/bpkg/pkg-status.cxx +++ b/bpkg/pkg-status.cxx @@ -42,7 +42,7 @@ namespace bpkg } catch (const invalid_argument& e) { - fail << "invalid package version '" << s << "'"; + fail << "invalid package version '" << s << "': " << e.what (); } } diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index e112bf7..641efb2 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -166,7 +166,7 @@ namespace bpkg const auto& ua (root->complements); // User-added repositories. if (ua.empty ()) - fail << "configuration has no repositories" << + fail << "configuration " << c << " has no repositories" << info << "use 'bpkg rep-add' to add a repository"; // Clean repositories and available packages. At the end only @@ -206,8 +206,8 @@ namespace bpkg size_t rcount, pcount; if (verb) { - rcount = db.query_value<repository_count> ().result; - pcount = db.query_value<available_package_count> ().result; + rcount = db.query_value<repository_count> (); + pcount = db.query_value<available_package_count> (); } t.commit (); diff --git a/bpkg/test.sh b/bpkg/test.sh index d1ab6f2..63b6fb2 100755 --- a/bpkg/test.sh +++ b/bpkg/test.sh @@ -163,17 +163,50 @@ test cfg-create --wipe test rep-add http://pkg.cppget.org/1/hello test rep-fetch -## @@ ## +## pkg-fetch ## -test cfg-create --wipe config.cxx=g++-4.9 cxx config.install.root=/tmp/install -stat unknown +test cfg-create --wipe + +fail pkg-fetch -e # archive expected +fail pkg-fetch -e ./no-such-file # archive does not exist + +fail pkg-fetch # package name expected +fail pkg-fetch libhello # package version expected +fail pkg-fetch libhello 1/2/3 # invalid package version +fail pkg-fetch libhello 1.0.0 # no repositories +test rep-add $rep +fail pkg-fetch libhello 1.0.0 # no packages +test rep-fetch +fail pkg-fetch libhello 2+1.0.0 # not available + +# local +# +test cfg-create --wipe +test rep-add $rep +test rep-fetch +test pkg-fetch libhello 1.0.0 +test pkg-unpack libhello +test pkg-purge libhello + +# remote +# +test cfg-create --wipe +test rep-add http://pkg.cppget.org/1/hello +test rep-fetch +#test pkg-fetch libheavy 1.0.0 +test pkg-fetch libhello 1.0.0 +test pkg-unpack libhello +test pkg-purge libhello + +## @@ ## -## pkg-fetch ## +test cfg-create --wipe config.cxx=g++-4.9 cxx config.install.root=/tmp/install +stat unknown # fetch existing archive # |