aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bdep/ci.cxx71
-rw-r--r--bdep/http-service.cxx466
-rw-r--r--bdep/http-service.hxx59
-rw-r--r--bdep/publish.cxx485
-rw-r--r--tests/ci.test198
-rw-r--r--tests/publish.test26
6 files changed, 820 insertions, 485 deletions
diff --git a/bdep/ci.cxx b/bdep/ci.cxx
index 200cffa..42758e1 100644
--- a/bdep/ci.cxx
+++ b/bdep/ci.cxx
@@ -4,21 +4,26 @@
#include <bdep/ci.hxx>
+#include <libbpkg/manifest.hxx>
+
#include <bdep/git.hxx>
#include <bdep/project.hxx>
#include <bdep/database.hxx>
#include <bdep/diagnostics.hxx>
+#include <bdep/http-service.hxx>
using namespace std;
using namespace butl;
namespace bdep
{
- // Get the project's remote repository URL corresponding to the current
+ using bpkg::repository_location;
+
+ // Get the project's remote repository location corresponding to the current
// (local) state of the repository. Fail if the working directory is not
// clean or if the local state isn't in sync with the remote.
//
- static url
+ static repository_location
git_repository_url (const cmd_ci_options& o, const dir_path& prj)
{
// This is what we need to do:
@@ -142,23 +147,41 @@ namespace bdep
// We treat the URL specified with --repository as a "base", that is, we
// still add the fragment.
//
- url r (o.repository_specified ()
+ url u (o.repository_specified ()
? o.repository ()
: git_remote_url (prj, "--repository"));
- if (r.fragment)
- fail << "remote git repository URL '" << r << "' already has fragment";
+ if (u.fragment)
+ fail << "remote git repository URL '" << u << "' already has fragment";
- // We specify both the branch and the commit to give bpkg every chance to
- // minimize the amount of history to fetch (see bpkg-repository-types(1)
- // for details).
+ // Try to construct the remote repository location out of the URL and fail
+ // if that's not possible.
//
- r.fragment = branch + '@' + commit;
+ try
+ {
+ // We specify both the branch and the commit to give bpkg every chance
+ // to minimize the amount of history to fetch (see
+ // bpkg-repository-types(1) for details).
+ //
+ repository_location r (
+ bpkg::repository_url (u.string () + '#' + branch + '@' + commit),
+ bpkg::repository_type::git);
+
+ if (!r.local ())
+ return r;
+
+ // Fall through.
+ }
+ catch (const invalid_argument&)
+ {
+ // Fall through.
+ }
- return r;
+ fail << "unable to derive bpkg repository location from git repository "
+ << "URL '" << u << "'" << endf;
}
- static url
+ static repository_location
repository_url (const cmd_ci_options& o, const dir_path& prj)
{
if (git_repository (prj))
@@ -249,7 +272,7 @@ namespace bdep
// Get the server and repository URLs.
//
const url& srv (o.server ());
- const url rep (repository_url (o, prj));
+ const repository_location rep (repository_url (o, prj));
// Print the plan and ask for confirmation.
//
@@ -284,10 +307,30 @@ namespace bdep
if (verb && o.yes ())
text << "submitting to " << srv;
- //@@ TODO call submit()
+ url u (srv);
+ u.query = "ci";
+
+ using namespace http_service;
+
+ parameters params ({{parameter::text, "repository", rep.string ()}});
+
+ for (const package& p: pkgs)
+ params.push_back ({parameter::text,
+ "package",
+ p.name.string () + '/' + p.version.string ()});
+
+ if (o.simulate_specified ())
+ params.push_back ({parameter::text, "simulate", o.simulate ()});
+
+ // Disambiguates with odb::result.
+ //
+ http_service::result r (post (o, u, params));
+
+ if (!r.reference)
+ fail << "no reference specified";
if (verb)
- text << "@@ TODO: print response";
+ text << r.message << " (" << *r.reference << ")";
}
return 0;
diff --git a/bdep/http-service.cxx b/bdep/http-service.cxx
new file mode 100644
index 0000000..faec3a7
--- /dev/null
+++ b/bdep/http-service.cxx
@@ -0,0 +1,466 @@
+// file : bdep/submit.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <bdep/http-service.hxx>
+
+#include <cstdlib> // strtoul()
+
+#include <libbutl/fdstream.mxx> // fdterm()
+
+#include <bdep/diagnostics.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace bdep
+{
+ namespace http_service
+ {
+ result
+ post (const common_options& o, const url& u, const parameters& params)
+ {
+ using parser = manifest_parser;
+ using parsing = manifest_parsing;
+ using name_value = manifest_name_value;
+
+ // The overall plan is to post the data using the curl program, read
+ // the HTTP response status and content type, read and parse the body
+ // according to the content type, and obtain the result message and
+ // optional reference in case of both the request success and failure.
+ //
+ // The successful request response (HTTP status code 200) is expected to
+ // contain the result manifest (text/manifest content type). The faulty
+ // response (HTTP status code other than 200) can either contain the
+ // result manifest or a plain text error description (text/plain content
+ // type) or some other content (for example text/html). We will print
+ // the manifest message value, if available or the first line of the
+ // plain text error description or, as a last resort, construct the
+ // message from the HTTP status code and reason phrase.
+ //
+ string message;
+ optional<uint16_t> status; // Request result manifest status value.
+ optional<string> reference;
+ vector<name_value> body;
+
+ // None of the 3XX redirect code semantics assume automatic re-posting.
+ // We will treat all such codes as failures, additionally printing the
+ // location header value to advise the user to try the other URL for the
+ // request.
+ //
+ // Note that services that move to a new URL may well be responding with
+ // the 301 (moved permanently) code.
+ //
+ optional<url> location;
+
+ // Note that it's a bad idea to issue the diagnostics while curl is
+ // running, as it will be messed up with the progress output. Thus, we
+ // throw the runtime_error exception on the HTTP response parsing error
+ // (rather than use our fail stream) and issue the diagnostics after
+ // curl finishes.
+ //
+ // Also note that we prefer the start/finish process facility for
+ // running curl over using butl::curl because in this context it is
+ // restrictive and inconvenient.
+ //
+ process pr;
+ bool io (false);
+ try
+ {
+ // Map the verbosity level.
+ //
+ cstrings v;
+ if (verb < 1)
+ {
+ v.push_back ("-s");
+ v.push_back ("-S"); // But show errors.
+ }
+ else if (verb == 1 && fdterm (2))
+ v.push_back ("--progress-bar");
+ else if (verb > 3)
+ v.push_back ("-v");
+
+ // Convert the submit arguments to curl's --form* options.
+ //
+ strings fos;
+ for (const parameter& p: params)
+ {
+ fos.emplace_back (p.type == parameter::file
+ ? "--form"
+ : "--form-string");
+
+ fos.emplace_back (p.type == parameter::file
+ ? p.name + "=@" + p.value
+ : p.name + "=" + p.value);
+ }
+
+ // Start curl program.
+ //
+ fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate.
+
+ // Note that we don't specify any default timeouts, assuming that bdep
+ // is an interactive program and the user can always interrupt the
+ // command (or pass the timeout with --curl-option).
+ //
+ pr = start (0 /* stdin */,
+ pipe /* stdout */,
+ 2 /* stderr */,
+ o.curl (),
+ v,
+ "-A", (BDEP_USER_AGENT " curl"),
+
+ o.curl_option (),
+
+ // Include the response headers in the output so we can
+ // get the status code/reason, content type, and the
+ // redirect location.
+ //
+ "--include",
+
+ fos,
+ u.string ());
+
+ pipe.out.close ();
+
+ // First we read the HTTP response status line and headers. At this
+ // stage we will read until the empty line (containing just CRLF). Not
+ // being able to reach such a line is an error, which is the reason
+ // for the exception mask choice.
+ //
+ ifdstream is (
+ move (pipe.in),
+ fdstream_mode::skip,
+ ifdstream::badbit | ifdstream::failbit | ifdstream::eofbit);
+
+ // Parse and return the HTTP status code. Return 0 if the argument is
+ // invalid.
+ //
+ auto status_code = [] (const string& s)
+ {
+ char* e (nullptr);
+ unsigned long c (strtoul (s.c_str (), &e, 10)); // Can't throw.
+ assert (e != nullptr);
+
+ return *e == '\0' && c >= 100 && c < 600
+ ? static_cast<uint16_t> (c)
+ : 0;
+ };
+
+ // Read the CRLF-terminated line from the stream stripping the
+ // trailing CRLF.
+ //
+ auto read_line = [&is] ()
+ {
+ string l;
+ getline (is, l); // Strips the trailing LF (0xA).
+
+ // Note that on POSIX CRLF is not automatically translated into
+ // LF, so we need to strip CR (0xD) manually.
+ //
+ if (!l.empty () && l.back () == '\r')
+ l.pop_back ();
+
+ return l;
+ };
+
+ auto bad_response = [] (const string& d) {throw runtime_error (d);};
+
+ // Read and parse the HTTP response status line, return the status
+ // code and the reason phrase.
+ //
+ struct http_status
+ {
+ uint16_t code;
+ string reason;
+ };
+
+ auto read_status = [&read_line, &status_code, &bad_response] ()
+ {
+ string l (read_line ());
+
+ for (;;) // Breakout loop.
+ {
+ if (l.compare (0, 5, "HTTP/") != 0)
+ break;
+
+ size_t p (l.find (' ', 5)); // The protocol end.
+ if (p == string::npos)
+ break;
+
+ p = l.find_first_not_of (' ', p + 1); // The code start.
+ if (p == string::npos)
+ break;
+
+ size_t e (l.find (' ', p + 1)); // The code end.
+ if (e == string::npos)
+ break;
+
+ uint16_t c (status_code (string (l, p, e - p)));
+ if (c == 0)
+ break;
+
+ string r;
+ p = l.find_first_not_of (' ', e + 1); // The reason start.
+ if (p != string::npos)
+ {
+ e = l.find_last_not_of (' '); // The reason end.
+ assert (e != string::npos && e >= p);
+
+ r = string (l, p, e - p + 1);
+ }
+
+ return http_status {c, move (r)};
+ }
+
+ bad_response ("invalid HTTP response status line '" + l + "'");
+
+ assert (false); // Can't be here.
+ return http_status {};
+ };
+
+ // The curl output for a successfull request looks like this:
+ //
+ // HTTP/1.1 100 Continue
+ //
+ // HTTP/1.1 200 OK
+ // Content-Length: 83
+ // Content-Type: text/manifest;charset=utf-8
+ //
+ // : 1
+ // status: 200
+ // message: submission is queued
+ // reference: 256910ca46d5
+ //
+ // curl normally sends the 'Expect: 100-continue' header for uploads,
+ // so we need to handle the interim HTTP server response with the
+ // continue (100) status code.
+ //
+ // Interestingly, Apache can respond with the continue (100) code and
+ // with the not found (404) code afterwords. Can it be configured to
+ // just respond with 404?
+ //
+ http_status rs (read_status ());
+
+ if (rs.code == 100)
+ {
+ while (!read_line ().empty ()) ; // Skips the interim response.
+ rs = read_status (); // Reads the final status code.
+ }
+
+ // Read through the response headers until the empty line is
+ // encountered and obtain the content type and/or the redirect
+ // location, if present.
+ //
+ optional<string> ctype;
+
+ // Check if the line contains the specified header and return its
+ // value if that's the case. Return nullopt otherwise.
+ //
+ // Note that we don't expect the header values that we are interested
+ // in to span over multiple lines.
+ //
+ string l;
+ auto header = [&l] (const char* name) -> optional<string>
+ {
+ size_t n (string::traits_type::length (name));
+ if (!(casecmp (name, l, n) == 0 && l[n] == ':'))
+ return nullopt;
+
+ string r;
+ size_t p (l.find_first_not_of (' ', n + 1)); // The value begin.
+ if (p != string::npos)
+ {
+ size_t e (l.find_last_not_of (' ')); // The value end.
+ assert (e != string::npos && e >= p);
+
+ r = string (l, p, e - p + 1);
+ }
+
+ return optional<string> (move (r));
+ };
+
+ while (!(l = read_line ()).empty ())
+ {
+ if (optional<string> v = header ("Content-Type"))
+ ctype = move (v);
+ else if (optional<string> v = header ("Location"))
+ {
+ if ((rs.code >= 301 && rs.code <= 303) || rs.code == 307)
+ try
+ {
+ location = url (*v);
+ location->query = nullopt; // Can possibly contain '?submit'.
+ }
+ catch (const invalid_argument&)
+ {
+ // Let's just ignore invalid locations.
+ //
+ }
+ }
+ }
+
+ assert (!eof (is)); // Would have already failed otherwise.
+
+ // Now parse the response payload if the content type is specified and
+ // is recognized (text/manifest or text/plain), skip it (with the
+ // ifdstream's close() function) otherwise.
+ //
+ // Note that eof and getline() fail conditions are not errors anymore,
+ // so we adjust the exception mask accordingly.
+ //
+ is.exceptions (ifdstream::badbit);
+
+ if (ctype)
+ {
+ if (casecmp ("text/manifest", *ctype, 13) == 0)
+ {
+ parser p (is, "manifest");
+ name_value nv (p.next ());
+
+ if (nv.empty ())
+ bad_response ("empty manifest");
+
+ const string& n (nv.name);
+ string& v (nv.value);
+
+ // The format version pair is verified by the parser.
+ //
+ assert (n.empty () && v == "1");
+
+ body.push_back (move (nv)); // Save the format version pair.
+
+ auto bad_value = [&p, &nv] (const string& d) {
+ throw parsing (p.name (), nv.value_line, nv.value_column, d);};
+
+ // Get and verify the HTTP status.
+ //
+ nv = p.next ();
+ if (n != "status")
+ bad_value ("no status specified");
+
+ uint16_t c (status_code (v));
+ if (c == 0)
+ bad_value ("invalid HTTP status '" + v + "'");
+
+ if (c != rs.code)
+ bad_value ("status " + v + " doesn't match HTTP response "
+ "code " + to_string (rs.code));
+
+ // Get the message.
+ //
+ nv = p.next ();
+ if (n != "message" || v.empty ())
+ bad_value ("no message specified");
+
+ message = move (v);
+
+ // Try to get an optional reference.
+ //
+ nv = p.next ();
+
+ if (n == "reference")
+ {
+ if (v.empty ())
+ bad_value ("empty reference specified");
+
+ reference = move (v);
+
+ nv = p.next ();
+ }
+
+ // Save the remaining name/value pairs.
+ //
+ for (; !nv.empty (); nv = p.next ())
+ body.push_back (move (nv));
+
+ status = c;
+ }
+ else if (casecmp ("text/plain", *ctype, 10) == 0)
+ getline (is, message); // Can result in the empty message.
+ }
+
+ is.close (); // Detect errors.
+
+ // The meaningful result we expect is either manifest (status code is
+ // not necessarily 200) or HTTP redirect (location is present). We
+ // unable to interpret any other cases and so report them as a bad
+ // response.
+ //
+ if (!status)
+ {
+ if (rs.code == 200)
+ bad_response ("manifest expected");
+
+ if (message.empty ())
+ {
+ message = "HTTP status code " + to_string (rs.code);
+
+ if (!rs.reason.empty ())
+ message += " (" + lcase (rs.reason) + ")";
+ }
+
+ if (!location)
+ bad_response (message);
+ }
+ }
+ catch (const io_error&)
+ {
+ // Presumably the child process failed and issued diagnostics so let
+ // finish() try to deal with that first.
+ //
+ io = true;
+ }
+ // Handle all parsing errors, including the manifest_parsing exception
+ // that inherits from the runtime_error exception.
+ //
+ // Note that the io_error class inherits from the runtime_error class,
+ // so this catch-clause must go last.
+ //
+ catch (const runtime_error& e)
+ {
+ finish (o.curl (), pr); // Throws on process failure.
+
+ // Finally we can safely issue the diagnostics (see above for
+ // details).
+ //
+ diag_record dr (fail);
+
+ url du (u);
+ du.query = nullopt; // Strip URL parameters from the diagnostics.
+
+ dr << e <<
+ info << "consider reporting this to " << du << " maintainers";
+
+ if (reference)
+ dr << info << "reference: " << *reference;
+ }
+
+ finish (o.curl (), pr, io);
+
+ assert (!message.empty ());
+
+ // Print the request failure reason and fail.
+ //
+ if (!status || *status != 200)
+ {
+ diag_record dr (fail);
+ dr << message;
+
+ if (reference)
+ dr << info << "reference: " << *reference;
+
+ if (location)
+ dr << info << "new location: " << *location;
+
+ // In case of a server error advise the user to re-try later, assuming
+ // that the issue is temporary (service overload, network connectivity
+ // loss, etc.).
+ //
+ if (status && *status >= 500 && *status < 600)
+ dr << info << "try again later";
+ }
+
+ return result {move (message), move (reference), move (body)};
+ }
+ }
+}
diff --git a/bdep/http-service.hxx b/bdep/http-service.hxx
new file mode 100644
index 0000000..b1db7e7
--- /dev/null
+++ b/bdep/http-service.hxx
@@ -0,0 +1,59 @@
+// file : bdep/http-service.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#ifndef BDEP_HTTP_SERVICE_HXX
+#define BDEP_HTTP_SERVICE_HXX
+
+#include <libbutl/manifest-parser.mxx>
+
+#include <bdep/types.hxx>
+#include <bdep/utility.hxx>
+
+#include <bdep/common-options.hxx>
+
+namespace bdep
+{
+ namespace http_service
+ {
+ // If type is file, then the value is a path to be uploaded.
+ //
+ struct parameter
+ {
+ enum {text, file} type;
+ string name;
+ string value;
+ };
+ using parameters = vector<parameter>;
+
+ struct result
+ {
+ string message;
+ optional<string> reference;
+
+ // Does not include status, message, or reference.
+ //
+ vector<butl::manifest_name_value> body;
+ };
+
+ // Submit text parameters and/or upload files to an HTTP service via the
+ // POST method. Use the multipart/form-data content type if any files are
+ // uploaded and application/x-www-form-urlencoded otherwise.
+ //
+ // On success, return the response manifest message and reference (if
+ // present, see below) and the rest of the manifest values, if any. Issue
+ // diagnostics and fail if anything goes wrong or the response manifest
+ // status value is not 200 (success).
+ //
+ // Note that the HTTP service is expected to respond with the result
+ // manifest that starts with the 'status' (HTTP status code) and 'message'
+ // (diagnostics message) values optionally followed by 'reference' and
+ // then other manifest values. If the status is not 200 and reference is
+ // present, then it is included in the diagnostics.
+ //
+ result
+ post (const common_options& o, const url&, const parameters&);
+ }
+}
+
+#endif // BDEP_HTTP_SERVICE_HXX
diff --git a/bdep/publish.cxx b/bdep/publish.cxx
index 81afcdb..de05534 100644
--- a/bdep/publish.cxx
+++ b/bdep/publish.cxx
@@ -4,9 +4,6 @@
#include <bdep/publish.hxx>
-#include <cstdlib> // strtoul()
-
-#include <libbutl/fdstream.mxx> // fdterm()
#include <libbutl/manifest-parser.mxx>
#include <libbutl/manifest-serializer.mxx>
@@ -17,6 +14,7 @@
#include <bdep/project-author.hxx>
#include <bdep/database.hxx>
#include <bdep/diagnostics.hxx>
+#include <bdep/http-service.hxx>
#include <bdep/sync.hxx>
@@ -51,460 +49,6 @@ namespace bdep
info << "use --control to specify explicitly" << endf;
}
- // Submit package archive using the curl program and parse the response
- // manifest. On success, return the submission reference (first) and message
- // (second). Issue diagnostics and fail if anything goes wrong.
- //
- static pair<string, string>
- submit (const cmd_publish_options& o,
- const path& archive,
- const string& checksum,
- const string& section,
- const project_author& author,
- const optional<url>& ctrl)
- {
- using parser = manifest_parser;
- using parsing = manifest_parsing;
- using name_value = manifest_name_value;
-
- // The overall plan is to post the archive using the curl program, read
- // the HTTP response status and content type, read and parse the body
- // according to the content type, and obtain the result reference and
- // message in case of both the submission success and failure.
- //
- // The successful submission response (HTTP status code 200) is expected
- // to contain the submission result manifest (text/manifest content type).
- // The faulty response (HTTP status code other than 200) can either
- // contain the result manifest or a plain text error description
- // (text/plain content type) or some other content (for example
- // text/html). We will print the manifest message value, if available or
- // the first line of the plain text error description or, as a last
- // resort, construct the message from the HTTP status code and reason
- // phrase.
- //
- string message;
- optional<uint16_t> status; // Submission result manifest status value.
- optional<string> reference; // Must be present on the submission success.
-
- // None of the 3XX redirect code semantics assume automatic re-posting. We
- // will treat all such codes as failures, additionally printing the
- // location header value to advise the user to try the other URL for the
- // package submission.
- //
- // Note that repositories that move to a new URL may well be responding
- // with the moved permanently (301) code.
- //
- optional<url> location;
-
- // Note that it's a bad idea to issue the diagnostics while curl is
- // running, as it will be messed up with the progress output. Thus, we
- // throw the runtime_error exception on the HTTP response parsing error
- // (rather than use our fail stream) and issue the diagnostics after curl
- // finishes.
- //
- // Also note that we prefer the start/finish process facility for running
- // curl over using butl::curl because in this context it is restrictive
- // and inconvenient.
- //
- process pr;
- bool io (false);
- try
- {
- url u (o.repository ());
- u.query = "submit";
-
- // Map the verbosity level.
- //
- cstrings v;
- if (verb < 1)
- {
- v.push_back ("-s");
- v.push_back ("-S"); // But show errors.
- }
- else if (verb == 1 && fdterm (2))
- v.push_back ("--progress-bar");
- else if (verb > 3)
- v.push_back ("-v");
-
- // Start curl program.
- //
- fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate.
-
- // Note that we don't specify any default timeouts, assuming that bdep
- // is an interactive program and the user can always interrupt the
- // command (or pass the timeout with --curl-option).
- //
- pr = start (0 /* stdin */,
- pipe /* stdout */,
- 2 /* stderr */,
- o.curl (),
- v,
- "-A", (BDEP_USER_AGENT " curl"),
-
- o.curl_option (),
-
- // Include the response headers in the output so we can get
- // the status code/reason, content type, and the redirect
- // location.
- //
- "--include",
-
- "--form", "archive=@" + archive.string (),
- "--form-string", "sha256sum=" + checksum,
- "--form-string", "section=" + section,
- "--form-string", "author-name=" + *author.name,
- "--form-string", "author-email=" + *author.email,
-
- ctrl
- ? strings ({"--form-string", "control=" + ctrl->string ()})
- : strings (),
-
- o.simulate_specified ()
- ? strings ({"--form-string", "simulate=" + o.simulate ()})
- : strings (),
-
- u.string ());
-
- pipe.out.close ();
-
- // First we read the HTTP response status line and headers. At this
- // stage we will read until the empty line (containing just CRLF). Not
- // being able to reach such a line is an error, which is the reason for
- // the exception mask choice.
- //
- ifdstream is (
- move (pipe.in),
- fdstream_mode::skip,
- ifdstream::badbit | ifdstream::failbit | ifdstream::eofbit);
-
- // Parse and return the HTTP status code. Return 0 if the argument is
- // invalid.
- //
- auto status_code = [] (const string& s)
- {
- char* e (nullptr);
- unsigned long c (strtoul (s.c_str (), &e, 10)); // Can't throw.
- assert (e != nullptr);
-
- return *e == '\0' && c >= 100 && c < 600
- ? static_cast<uint16_t> (c)
- : 0;
- };
-
- // Read the CRLF-terminated line from the stream stripping the trailing
- // CRLF.
- //
- auto read_line = [&is] ()
- {
- string l;
- getline (is, l); // Strips the trailing LF (0xA).
-
- // Note that on POSIX CRLF is not automatically translated into LF,
- // so we need to strip CR (0xD) manually.
- //
- if (!l.empty () && l.back () == '\r')
- l.pop_back ();
-
- return l;
- };
-
- auto bad_response = [] (const string& d) {throw runtime_error (d);};
-
- // Read and parse the HTTP response status line, return the status code
- // and the reason phrase.
- //
- struct http_status
- {
- uint16_t code;
- string reason;
- };
-
- auto read_status = [&read_line, &status_code, &bad_response] ()
- {
- string l (read_line ());
-
- for (;;) // Breakout loop.
- {
- if (l.compare (0, 5, "HTTP/") != 0)
- break;
-
- size_t p (l.find (' ', 5)); // Finds the protocol end.
- if (p == string::npos)
- break;
-
- p = l.find_first_not_of (' ', p + 1); // Finds the code start.
- if (p == string::npos)
- break;
-
- size_t e (l.find (' ', p + 1)); // Finds the code end.
- if (e == string::npos)
- break;
-
- uint16_t c (status_code (string (l, p, e - p)));
- if (c == 0)
- break;
-
- string r;
- p = l.find_first_not_of (' ', e + 1); // Finds the reason start.
- if (p != string::npos)
- {
- e = l.find_last_not_of (' '); // Finds the reason end.
- assert (e != string::npos && e >= p);
-
- r = string (l, p, e - p + 1);
- }
-
- return http_status {c, move (r)};
- }
-
- bad_response ("invalid HTTP response status line '" + l + "'");
-
- assert (false); // Can't be here.
- return http_status {};
- };
-
- // The curl output for a successfull submission looks like this:
- //
- // HTTP/1.1 100 Continue
- //
- // HTTP/1.1 200 OK
- // Content-Length: 83
- // Content-Type: text/manifest;charset=utf-8
- //
- // : 1
- // status: 200
- // message: submission queued
- // reference: 256910ca46d5
- //
- // curl normally sends the 'Expect: 100-continue' header for uploads,
- // so we need to handle the interim HTTP server response with the
- // continue (100) status code.
- //
- // Interestingly, Apache can respond with the continue (100) code and
- // with the not found (404) code afterwords. Can it be configured to
- // just respond with 404?
- //
- http_status rs (read_status ());
-
- if (rs.code == 100)
- {
- while (!read_line ().empty ()) ; // Skips the interim response.
- rs = read_status (); // Reads the final status code.
- }
-
- // Read through the response headers until the empty line is encountered
- // and obtain the content type and/or the redirect location, if present.
- //
- optional<string> ctype;
-
- // Check if the line contains the specified header and return its value
- // if that's the case. Return nullopt otherwise.
- //
- // Note that we don't expect the header values that we are interested in
- // to span over multiple lines.
- //
- string l;
- auto header = [&l] (const char* name) -> optional<string>
- {
- size_t n (string::traits_type::length (name));
- if (!(casecmp (name, l, n) == 0 && l[n] == ':'))
- return nullopt;
-
- string r;
- size_t p (l.find_first_not_of (' ', n + 1)); // Finds value begin.
- if (p != string::npos)
- {
- size_t e (l.find_last_not_of (' ')); // Finds value end.
- assert (e != string::npos && e >= p);
-
- r = string (l, p, e - p + 1);
- }
-
- return optional<string> (move (r));
- };
-
- while (!(l = read_line ()).empty ())
- {
- if (optional<string> v = header ("Content-Type"))
- ctype = move (v);
- else if (optional<string> v = header ("Location"))
- {
- if ((rs.code >= 301 && rs.code <= 303) || rs.code == 307)
- try
- {
- location = url (*v);
- location->query = nullopt; // Can possibly contain '?submit'.
- }
- catch (const invalid_argument&)
- {
- // Let's just ignore invalid locations.
- //
- }
- }
- }
-
- assert (!eof (is)); // Would have already failed otherwise.
-
- // Now parse the response payload if the content type is specified and
- // is recognized (text/manifest or text/plain), skip it (with the
- // ifdstream's close() function) otherwise.
- //
- // Note that eof and getline() fail conditions are not errors anymore,
- // so we adjust the exception mask accordingly.
- //
- is.exceptions (ifdstream::badbit);
-
- if (ctype)
- {
- if (casecmp ("text/manifest", *ctype, 13) == 0)
- {
- parser p (is, "manifest");
- name_value nv (p.next ());
-
- if (nv.empty ())
- bad_response ("empty manifest");
-
- const string& n (nv.name);
- string& v (nv.value);
-
- // The format version pair is verified by the parser.
- //
- assert (n.empty () && v == "1");
-
- auto bad_value = [&p, &nv] (const string& d) {
- throw parsing (p.name (), nv.value_line, nv.value_column, d);};
-
- // Get and verify the HTTP status.
- //
- nv = p.next ();
- if (n != "status")
- bad_value ("no status specified");
-
- uint16_t c (status_code (v));
- if (c == 0)
- bad_value ("invalid HTTP status '" + v + "'");
-
- if (c != rs.code)
- bad_value ("status " + v + " doesn't match HTTP response "
- "code " + to_string (rs.code));
-
- // Get the message.
- //
- nv = p.next ();
- if (n != "message" || v.empty ())
- bad_value ("no message specified");
-
- message = move (v);
-
- // Try to get an optional reference and make sure it is present if
- // the submission succeeded.
- //
- nv = p.next ();
-
- if (n == "reference")
- {
- if (v.empty ())
- bad_value ("empty reference specified");
-
- reference = move (v);
-
- nv = p.next ();
- }
- else if (c == 200)
- bad_value ("no reference specified");
-
- // Skip the remaining name/value pairs.
- //
- for (; !nv.empty (); nv = p.next ()) ;
-
- status = c;
- }
- else if (casecmp ("text/plain", *ctype, 10) == 0)
- getline (is, message); // Can result in the empty message.
- }
-
- is.close (); // Detect errors.
-
- // The meaningful result we expect is either manifest (status code is
- // not necessarily 200) or HTTP redirect (location is present). We
- // unable to interpret any other cases and so report them as a bad
- // response.
- //
- if (!status)
- {
- if (rs.code == 200)
- bad_response ("manifest expected");
-
- if (message.empty ())
- {
- message = "HTTP status code " + to_string (rs.code);
-
- if (!rs.reason.empty ())
- message += " (" + lcase (rs.reason) + ")";
- }
-
- if (!location)
- bad_response (message);
- }
- }
- catch (const io_error&)
- {
- // Presumably the child process failed and issued diagnostics so let
- // finish() try to deal with that first.
- //
- io = true;
- }
- // Handle all parsing errors, including the manifest_parsing exception that
- // inherits from the runtime_error exception.
- //
- // Note that the io_error class inherits from the runtime_error class, so
- // this catch-clause must go last.
- //
- catch (const runtime_error& e)
- {
- finish (o.curl (), pr); // Throws on process failure.
-
- // Finally we can safely issue the diagnostics (see above for details).
- //
- diag_record dr (fail);
- dr << e <<
- info << "consider reporting this to " << o.repository ()
- << " repository maintainers";
-
- if (reference)
- dr << info << "reference: " << *reference;
- }
-
- finish (o.curl (), pr, io);
-
- assert (!message.empty ());
-
- // Print the submission failure reason and fail.
- //
- if (!status || *status != 200)
- {
- diag_record dr (fail);
- dr << message;
-
- if (reference)
- dr << info << "reference: " << *reference;
-
- if (location)
- dr << info << "new repository location: " << *location;
-
- // In case of a server error advise the user to re-try later, assuming
- // that the issue is temporary (service overload, network connectivity
- // loss, etc.).
- //
- if (status && *status >= 500 && *status < 600)
- dr << info << "try again later";
- }
-
- assert (reference); // Should be present if the submission succeeded.
-
- return make_pair (move (*reference), move (message));
- }
-
static int
cmd_publish (const cmd_publish_options& o,
const dir_path& prj,
@@ -1157,11 +701,32 @@ namespace bdep
if (verb)
text << "submitting " << p.archive.leaf ();
- pair<string, string> r (
- submit (o, p.archive, p.checksum, p.section, author, ctrl));
+ url u (o.repository ());
+ u.query = "submit";
+
+ using namespace http_service;
+
+ parameters params ({{parameter::file, "archive", p.archive.string ()},
+ {parameter::text, "sha256sum", p.checksum},
+ {parameter::text, "section", p.section},
+ {parameter::text, "author-name", *author.name},
+ {parameter::text, "author-email", *author.email}});
+
+ if (ctrl)
+ params.push_back ({parameter::text, "control", ctrl->string ()});
+
+ if (o.simulate_specified ())
+ params.push_back ({parameter::text, "simulate", o.simulate ()});
+
+ // Disambiguates with odb::result.
+ //
+ http_service::result r (post (o, u, params));
+
+ if (!r.reference)
+ fail << "no reference specified";
if (verb)
- text << r.second << " (" << r.first << ")";
+ text << r.message << " (" << *r.reference << ")";
}
return 0;
diff --git a/tests/ci.test b/tests/ci.test
new file mode 100644
index 0000000..85155de
--- /dev/null
+++ b/tests/ci.test
@@ -0,0 +1,198 @@
+# file : tests/ci.test
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+.include common.test project.test
+
+# bdep-ci requirements for the minimum supported git version are higher then
+# the default 2.1.0 (see bdep/ci.cxx for details).
+#
++if! ($git_version_major > 2 || \
+ $git_version_major == 2 && $git_version_minor >= 11)
+ exit
+end
+
+# Server to use for the CI request submissions simulation.
+#
+# Note that the empty config.bdep.test.repository value is used to suppress
+# these tests (which require network access).
+#
+server = ($config.bdep.test.repository == [null] \
+ ? ($build.version.stage \
+ ? 'https://stage.build2.org' \
+ : 'https://ci.cppget.org') \
+ : "$config.bdep.test.repository")
+
++if ("$server" == '')
+ exit
+end
+
+# Create the remote repository.
+#
++mkdir --no-cleanup prj.git
++git -C prj.git init --bare 2>! >&2 &prj.git/***
+
+# Adjust the local repository and push it to the remote one.
+#
++sed -i -e 's/^(version:) .*$/\1 1.0.1/' prj/manifest
+
+g = git -C prj 2>! >&2
+
++$g remote add origin $~/prj.git
++$g add '*'
++$g commit -m 'Create'
++$g push --set-upstream origin master
+
+# Repository the CI-ed packages come from.
+#
+repository='http://example.com/prj.git'
+
+test.arguments += --yes --repository "$repository" --server "$server" \
+--simulate 'success'
+
+cxx = cc "config.cxx=$config.cxx"
+
+new += 2>!
+init += $cxx -d prj 2>! &prj/**/bootstrap/***
+
+windows = ($cxx.target.class == 'windows')
+
+: single-pkg
+:
+{
+ : single-cfg
+ :
+ {
+ $clone_root_prj;
+ $init -C @cfg &prj-cfg/***;
+
+ $* 2>>~"%EOE%"
+ submitting to $server
+ %.*
+ %CI request is queued \(.+\)%
+ EOE
+ }
+
+ : no-cfg
+ :
+ {
+ $new prj &prj/***;
+
+ $* 2>>~%EOE% != 0
+ %error: no default configuration in project .+%
+ % info: use .+%
+ EOE
+ }
+
+ : multi-cfg
+ :
+ {
+ $clone_root_prj;
+ $init -C @cfg1 &prj-cfg1/***;
+ $init -C @cfg2 &prj-cfg2/***;
+
+ $* --all 2>'error: multiple configurations specified for ci' != 0
+ }
+
+ : no-commits
+ :
+ {
+ $new prj &prj/***;
+ $init -C @cfg &prj-cfg/***;
+
+ $* 2>>~%EOE% != 0
+ error: no commits in project repository
+ % info: run .+%
+ EOE
+ }
+
+ : invalid-repo
+ :
+ {
+ test.arguments += --repository "$repository#frag"
+
+ $clone_root_prj;
+ $init -C @cfg &prj-cfg/***;
+
+ $* 2>>"EOE" != 0
+ error: remote git repository URL '$repository#frag' already has fragment
+ EOE
+ }
+}
+
+: multi-pkg
+:
+{
+ # Create the remote repository.
+ #
+ +mkdir --no-cleanup prj.git
+ +git -C prj.git init --bare 2>! >&2 &prj.git/***
+
+ # Create the local repository and push it to the remote one.
+ #
+ +$new -t empty prj &prj/***
+ +$new --package -t lib libprj -d prj
+ +$new --package -t exe prj -d prj
+ +sed -i -e 's/^(version:) .*$/\1 1.0.1/' prj/libprj/manifest
+ +sed -i -e 's/^(version:) .*$/\1 1.0.1/' prj/prj/manifest
+
+ +$g remote add origin $~/prj.git
+ +$g add '*'
+ +$g commit -m 'Create'
+ +$g push --set-upstream origin master
+
+ : both
+ :
+ {
+ $clone_prj;
+ $init -C @cfg &prj-cfg/***;
+
+ $* 2>>~"%EOE%"
+ submitting to $server
+ %.*
+ %CI request is queued \(.+\)%
+ EOE
+ }
+
+ : single
+ :
+ {
+ $clone_prj;
+ $init -C @cfg &prj-cfg/***;
+
+ # CI the single libprj package rather than the whole prj project.
+ #
+ test.arguments = $regex.apply($test.arguments, '^(prj)$', '\1/libprj');
+
+ $* 2>>~"%EOE%"
+ submitting to $server
+ %.*
+ %CI request is queued \(.+\)%
+ EOE
+ }
+
+ : prompt
+ :
+ {
+ $clone_prj;
+ $init -C @cfg &prj-cfg/***;
+
+ # Suppress the --yes option.
+ #
+ test.arguments = $regex.apply($test.arguments, '^(--yes)$', '');
+
+ $* <'y' 2>>~"%EOE%"
+ submitting:
+ to: $server
+ % in: $repository#master@.{40}%
+
+ package: libprj
+ version: 1.0.1
+
+ package: prj
+ version: 1.0.1
+ %.*
+ %CI request is queued \(.+\)%
+ EOE
+ }
+}
diff --git a/tests/publish.test b/tests/publish.test
index 4756abc..ba88e9a 100644
--- a/tests/publish.test
+++ b/tests/publish.test
@@ -14,8 +14,8 @@ end
# Repository to use for the package submissions simulation.
#
-# Note: could use empty config.bdep.test.repository value to suppress
-# these test (which require network access).
+# Note that the empty config.bdep.test.repository value is used to suppress
+# these tests (which require network access).
#
repository = ($config.bdep.test.repository == [null] \
? ($build.version.stage \
@@ -23,6 +23,10 @@ repository = ($config.bdep.test.repository == [null] \
: 'https://cppget.org') \
: "$config.bdep.test.repository")
++if ("$repository" == '')
+ exit
+end
+
test.arguments += --repository "$repository" --yes \
--author-name user --author-email user@example.com
@@ -49,7 +53,7 @@ windows = ($cxx.target.class == 'windows')
{
test.arguments += --simulate 'success'
- : basic
+ : single-cfg
:
{
$clone_root_prj;
@@ -205,7 +209,7 @@ windows = ($cxx.target.class == 'windows')
submitting prj-1.0.6.tar.gz
%.*
error: submission handling failed
- % info: consider reporting this to .+ repository maintainers%
+ % info: consider reporting this to .+ maintainers%
EOE
}
@@ -224,7 +228,7 @@ windows = ($cxx.target.class == 'windows')
submitting prj-1.0.7.tar.gz
%.*
error: HTTP status code 500 (internal server error)
- % info: consider reporting this to .+ repository maintainers%
+ % info: consider reporting this to .+ maintainers%
EOE
}
}
@@ -237,19 +241,19 @@ windows = ($cxx.target.class == 'windows')
# simulation. We specify it to enable the control branch-related
# functionality.
#
- test.arguments += --simulate 'success' --control "http://example.com/rep.git"
+ test.arguments += --simulate 'success' --control 'http://example.com/rep.git'
# Create the remote repository.
#
+mkdir --no-cleanup prj.git
- +git -C prj.git init --bar >! &prj.git/***
+ +git -C prj.git init --bare >! &prj.git/***
+$clone_prj
g = git -C prj >! 2>!
- +$g config user.name "Test Script"
- +$g config user.email "testscript@example.com"
+ +$g config user.name 'Test Script'
+ +$g config user.email 'testscript@example.com'
clone_rep = cp --no-cleanup -r ../prj.git ./ &prj.git/***
clone_prj = cp --no-cleanup -r ../prj ./ &prj/***
@@ -365,8 +369,8 @@ windows = ($cxx.target.class == 'windows')
git clone "$rep" prj2 &prj2/*** 2>&1 | \
sed -e 's/warning: (remote HEAD refers to nonexistent .*)/info: \1/' >&2 2>!;
- $g2 config user.name "Test Script";
- $g2 config user.email "testscript@example.com";
+ $g2 config user.name 'Test Script';
+ $g2 config user.email 'testscript@example.com';
$g2 checkout -b build2-control --track origin/build2-control;
$g2 commit --allow-empty -m 'Dummy1';
$g2 push;