diff options
| -rw-r--r-- | bdep/ci.cxx | 71 | ||||
| -rw-r--r-- | bdep/http-service.cxx | 466 | ||||
| -rw-r--r-- | bdep/http-service.hxx | 59 | ||||
| -rw-r--r-- | bdep/publish.cxx | 485 | ||||
| -rw-r--r-- | tests/ci.test | 198 | ||||
| -rw-r--r-- | tests/publish.test | 26 |
6 files changed, 820 insertions, 485 deletions
diff --git a/bdep/ci.cxx b/bdep/ci.cxx index 200cffa..42758e1 100644 --- a/bdep/ci.cxx +++ b/bdep/ci.cxx @@ -4,21 +4,26 @@ #include <bdep/ci.hxx> +#include <libbpkg/manifest.hxx> + #include <bdep/git.hxx> #include <bdep/project.hxx> #include <bdep/database.hxx> #include <bdep/diagnostics.hxx> +#include <bdep/http-service.hxx> using namespace std; using namespace butl; namespace bdep { - // Get the project's remote repository URL corresponding to the current + using bpkg::repository_location; + + // Get the project's remote repository location corresponding to the current // (local) state of the repository. Fail if the working directory is not // clean or if the local state isn't in sync with the remote. // - static url + static repository_location git_repository_url (const cmd_ci_options& o, const dir_path& prj) { // This is what we need to do: @@ -142,23 +147,41 @@ namespace bdep // We treat the URL specified with --repository as a "base", that is, we // still add the fragment. // - url r (o.repository_specified () + url u (o.repository_specified () ? o.repository () : git_remote_url (prj, "--repository")); - if (r.fragment) - fail << "remote git repository URL '" << r << "' already has fragment"; + if (u.fragment) + fail << "remote git repository URL '" << u << "' already has fragment"; - // We specify both the branch and the commit to give bpkg every chance to - // minimize the amount of history to fetch (see bpkg-repository-types(1) - // for details). + // Try to construct the remote repository location out of the URL and fail + // if that's not possible. // - r.fragment = branch + '@' + commit; + try + { + // We specify both the branch and the commit to give bpkg every chance + // to minimize the amount of history to fetch (see + // bpkg-repository-types(1) for details). + // + repository_location r ( + bpkg::repository_url (u.string () + '#' + branch + '@' + commit), + bpkg::repository_type::git); + + if (!r.local ()) + return r; + + // Fall through. + } + catch (const invalid_argument&) + { + // Fall through. + } - return r; + fail << "unable to derive bpkg repository location from git repository " + << "URL '" << u << "'" << endf; } - static url + static repository_location repository_url (const cmd_ci_options& o, const dir_path& prj) { if (git_repository (prj)) @@ -249,7 +272,7 @@ namespace bdep // Get the server and repository URLs. // const url& srv (o.server ()); - const url rep (repository_url (o, prj)); + const repository_location rep (repository_url (o, prj)); // Print the plan and ask for confirmation. // @@ -284,10 +307,30 @@ namespace bdep if (verb && o.yes ()) text << "submitting to " << srv; - //@@ TODO call submit() + url u (srv); + u.query = "ci"; + + using namespace http_service; + + parameters params ({{parameter::text, "repository", rep.string ()}}); + + for (const package& p: pkgs) + params.push_back ({parameter::text, + "package", + p.name.string () + '/' + p.version.string ()}); + + if (o.simulate_specified ()) + params.push_back ({parameter::text, "simulate", o.simulate ()}); + + // Disambiguates with odb::result. + // + http_service::result r (post (o, u, params)); + + if (!r.reference) + fail << "no reference specified"; if (verb) - text << "@@ TODO: print response"; + text << r.message << " (" << *r.reference << ")"; } return 0; diff --git a/bdep/http-service.cxx b/bdep/http-service.cxx new file mode 100644 index 0000000..faec3a7 --- /dev/null +++ b/bdep/http-service.cxx @@ -0,0 +1,466 @@ +// file : bdep/submit.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <bdep/http-service.hxx> + +#include <cstdlib> // strtoul() + +#include <libbutl/fdstream.mxx> // fdterm() + +#include <bdep/diagnostics.hxx> + +using namespace std; +using namespace butl; + +namespace bdep +{ + namespace http_service + { + result + post (const common_options& o, const url& u, const parameters& params) + { + using parser = manifest_parser; + using parsing = manifest_parsing; + using name_value = manifest_name_value; + + // The overall plan is to post the data using the curl program, read + // the HTTP response status and content type, read and parse the body + // according to the content type, and obtain the result message and + // optional reference in case of both the request success and failure. + // + // The successful request response (HTTP status code 200) is expected to + // contain the result manifest (text/manifest content type). The faulty + // response (HTTP status code other than 200) can either contain the + // result manifest or a plain text error description (text/plain content + // type) or some other content (for example text/html). We will print + // the manifest message value, if available or the first line of the + // plain text error description or, as a last resort, construct the + // message from the HTTP status code and reason phrase. + // + string message; + optional<uint16_t> status; // Request result manifest status value. + optional<string> reference; + vector<name_value> body; + + // None of the 3XX redirect code semantics assume automatic re-posting. + // We will treat all such codes as failures, additionally printing the + // location header value to advise the user to try the other URL for the + // request. + // + // Note that services that move to a new URL may well be responding with + // the 301 (moved permanently) code. + // + optional<url> location; + + // Note that it's a bad idea to issue the diagnostics while curl is + // running, as it will be messed up with the progress output. Thus, we + // throw the runtime_error exception on the HTTP response parsing error + // (rather than use our fail stream) and issue the diagnostics after + // curl finishes. + // + // Also note that we prefer the start/finish process facility for + // running curl over using butl::curl because in this context it is + // restrictive and inconvenient. + // + process pr; + bool io (false); + try + { + // Map the verbosity level. + // + cstrings v; + if (verb < 1) + { + v.push_back ("-s"); + v.push_back ("-S"); // But show errors. + } + else if (verb == 1 && fdterm (2)) + v.push_back ("--progress-bar"); + else if (verb > 3) + v.push_back ("-v"); + + // Convert the submit arguments to curl's --form* options. + // + strings fos; + for (const parameter& p: params) + { + fos.emplace_back (p.type == parameter::file + ? "--form" + : "--form-string"); + + fos.emplace_back (p.type == parameter::file + ? p.name + "=@" + p.value + : p.name + "=" + p.value); + } + + // Start curl program. + // + fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate. + + // Note that we don't specify any default timeouts, assuming that bdep + // is an interactive program and the user can always interrupt the + // command (or pass the timeout with --curl-option). + // + pr = start (0 /* stdin */, + pipe /* stdout */, + 2 /* stderr */, + o.curl (), + v, + "-A", (BDEP_USER_AGENT " curl"), + + o.curl_option (), + + // Include the response headers in the output so we can + // get the status code/reason, content type, and the + // redirect location. + // + "--include", + + fos, + u.string ()); + + pipe.out.close (); + + // First we read the HTTP response status line and headers. At this + // stage we will read until the empty line (containing just CRLF). Not + // being able to reach such a line is an error, which is the reason + // for the exception mask choice. + // + ifdstream is ( + move (pipe.in), + fdstream_mode::skip, + ifdstream::badbit | ifdstream::failbit | ifdstream::eofbit); + + // Parse and return the HTTP status code. Return 0 if the argument is + // invalid. + // + auto status_code = [] (const string& s) + { + char* e (nullptr); + unsigned long c (strtoul (s.c_str (), &e, 10)); // Can't throw. + assert (e != nullptr); + + return *e == '\0' && c >= 100 && c < 600 + ? static_cast<uint16_t> (c) + : 0; + }; + + // Read the CRLF-terminated line from the stream stripping the + // trailing CRLF. + // + auto read_line = [&is] () + { + string l; + getline (is, l); // Strips the trailing LF (0xA). + + // Note that on POSIX CRLF is not automatically translated into + // LF, so we need to strip CR (0xD) manually. + // + if (!l.empty () && l.back () == '\r') + l.pop_back (); + + return l; + }; + + auto bad_response = [] (const string& d) {throw runtime_error (d);}; + + // Read and parse the HTTP response status line, return the status + // code and the reason phrase. + // + struct http_status + { + uint16_t code; + string reason; + }; + + auto read_status = [&read_line, &status_code, &bad_response] () + { + string l (read_line ()); + + for (;;) // Breakout loop. + { + if (l.compare (0, 5, "HTTP/") != 0) + break; + + size_t p (l.find (' ', 5)); // The protocol end. + if (p == string::npos) + break; + + p = l.find_first_not_of (' ', p + 1); // The code start. + if (p == string::npos) + break; + + size_t e (l.find (' ', p + 1)); // The code end. + if (e == string::npos) + break; + + uint16_t c (status_code (string (l, p, e - p))); + if (c == 0) + break; + + string r; + p = l.find_first_not_of (' ', e + 1); // The reason start. + if (p != string::npos) + { + e = l.find_last_not_of (' '); // The reason end. + assert (e != string::npos && e >= p); + + r = string (l, p, e - p + 1); + } + + return http_status {c, move (r)}; + } + + bad_response ("invalid HTTP response status line '" + l + "'"); + + assert (false); // Can't be here. + return http_status {}; + }; + + // The curl output for a successfull request looks like this: + // + // HTTP/1.1 100 Continue + // + // HTTP/1.1 200 OK + // Content-Length: 83 + // Content-Type: text/manifest;charset=utf-8 + // + // : 1 + // status: 200 + // message: submission is queued + // reference: 256910ca46d5 + // + // curl normally sends the 'Expect: 100-continue' header for uploads, + // so we need to handle the interim HTTP server response with the + // continue (100) status code. + // + // Interestingly, Apache can respond with the continue (100) code and + // with the not found (404) code afterwords. Can it be configured to + // just respond with 404? + // + http_status rs (read_status ()); + + if (rs.code == 100) + { + while (!read_line ().empty ()) ; // Skips the interim response. + rs = read_status (); // Reads the final status code. + } + + // Read through the response headers until the empty line is + // encountered and obtain the content type and/or the redirect + // location, if present. + // + optional<string> ctype; + + // Check if the line contains the specified header and return its + // value if that's the case. Return nullopt otherwise. + // + // Note that we don't expect the header values that we are interested + // in to span over multiple lines. + // + string l; + auto header = [&l] (const char* name) -> optional<string> + { + size_t n (string::traits_type::length (name)); + if (!(casecmp (name, l, n) == 0 && l[n] == ':')) + return nullopt; + + string r; + size_t p (l.find_first_not_of (' ', n + 1)); // The value begin. + if (p != string::npos) + { + size_t e (l.find_last_not_of (' ')); // The value end. + assert (e != string::npos && e >= p); + + r = string (l, p, e - p + 1); + } + + return optional<string> (move (r)); + }; + + while (!(l = read_line ()).empty ()) + { + if (optional<string> v = header ("Content-Type")) + ctype = move (v); + else if (optional<string> v = header ("Location")) + { + if ((rs.code >= 301 && rs.code <= 303) || rs.code == 307) + try + { + location = url (*v); + location->query = nullopt; // Can possibly contain '?submit'. + } + catch (const invalid_argument&) + { + // Let's just ignore invalid locations. + // + } + } + } + + assert (!eof (is)); // Would have already failed otherwise. + + // Now parse the response payload if the content type is specified and + // is recognized (text/manifest or text/plain), skip it (with the + // ifdstream's close() function) otherwise. + // + // Note that eof and getline() fail conditions are not errors anymore, + // so we adjust the exception mask accordingly. + // + is.exceptions (ifdstream::badbit); + + if (ctype) + { + if (casecmp ("text/manifest", *ctype, 13) == 0) + { + parser p (is, "manifest"); + name_value nv (p.next ()); + + if (nv.empty ()) + bad_response ("empty manifest"); + + const string& n (nv.name); + string& v (nv.value); + + // The format version pair is verified by the parser. + // + assert (n.empty () && v == "1"); + + body.push_back (move (nv)); // Save the format version pair. + + auto bad_value = [&p, &nv] (const string& d) { + throw parsing (p.name (), nv.value_line, nv.value_column, d);}; + + // Get and verify the HTTP status. + // + nv = p.next (); + if (n != "status") + bad_value ("no status specified"); + + uint16_t c (status_code (v)); + if (c == 0) + bad_value ("invalid HTTP status '" + v + "'"); + + if (c != rs.code) + bad_value ("status " + v + " doesn't match HTTP response " + "code " + to_string (rs.code)); + + // Get the message. + // + nv = p.next (); + if (n != "message" || v.empty ()) + bad_value ("no message specified"); + + message = move (v); + + // Try to get an optional reference. + // + nv = p.next (); + + if (n == "reference") + { + if (v.empty ()) + bad_value ("empty reference specified"); + + reference = move (v); + + nv = p.next (); + } + + // Save the remaining name/value pairs. + // + for (; !nv.empty (); nv = p.next ()) + body.push_back (move (nv)); + + status = c; + } + else if (casecmp ("text/plain", *ctype, 10) == 0) + getline (is, message); // Can result in the empty message. + } + + is.close (); // Detect errors. + + // The meaningful result we expect is either manifest (status code is + // not necessarily 200) or HTTP redirect (location is present). We + // unable to interpret any other cases and so report them as a bad + // response. + // + if (!status) + { + if (rs.code == 200) + bad_response ("manifest expected"); + + if (message.empty ()) + { + message = "HTTP status code " + to_string (rs.code); + + if (!rs.reason.empty ()) + message += " (" + lcase (rs.reason) + ")"; + } + + if (!location) + bad_response (message); + } + } + catch (const io_error&) + { + // Presumably the child process failed and issued diagnostics so let + // finish() try to deal with that first. + // + io = true; + } + // Handle all parsing errors, including the manifest_parsing exception + // that inherits from the runtime_error exception. + // + // Note that the io_error class inherits from the runtime_error class, + // so this catch-clause must go last. + // + catch (const runtime_error& e) + { + finish (o.curl (), pr); // Throws on process failure. + + // Finally we can safely issue the diagnostics (see above for + // details). + // + diag_record dr (fail); + + url du (u); + du.query = nullopt; // Strip URL parameters from the diagnostics. + + dr << e << + info << "consider reporting this to " << du << " maintainers"; + + if (reference) + dr << info << "reference: " << *reference; + } + + finish (o.curl (), pr, io); + + assert (!message.empty ()); + + // Print the request failure reason and fail. + // + if (!status || *status != 200) + { + diag_record dr (fail); + dr << message; + + if (reference) + dr << info << "reference: " << *reference; + + if (location) + dr << info << "new location: " << *location; + + // In case of a server error advise the user to re-try later, assuming + // that the issue is temporary (service overload, network connectivity + // loss, etc.). + // + if (status && *status >= 500 && *status < 600) + dr << info << "try again later"; + } + + return result {move (message), move (reference), move (body)}; + } + } +} diff --git a/bdep/http-service.hxx b/bdep/http-service.hxx new file mode 100644 index 0000000..b1db7e7 --- /dev/null +++ b/bdep/http-service.hxx @@ -0,0 +1,59 @@ +// file : bdep/http-service.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BDEP_HTTP_SERVICE_HXX +#define BDEP_HTTP_SERVICE_HXX + +#include <libbutl/manifest-parser.mxx> + +#include <bdep/types.hxx> +#include <bdep/utility.hxx> + +#include <bdep/common-options.hxx> + +namespace bdep +{ + namespace http_service + { + // If type is file, then the value is a path to be uploaded. + // + struct parameter + { + enum {text, file} type; + string name; + string value; + }; + using parameters = vector<parameter>; + + struct result + { + string message; + optional<string> reference; + + // Does not include status, message, or reference. + // + vector<butl::manifest_name_value> body; + }; + + // Submit text parameters and/or upload files to an HTTP service via the + // POST method. Use the multipart/form-data content type if any files are + // uploaded and application/x-www-form-urlencoded otherwise. + // + // On success, return the response manifest message and reference (if + // present, see below) and the rest of the manifest values, if any. Issue + // diagnostics and fail if anything goes wrong or the response manifest + // status value is not 200 (success). + // + // Note that the HTTP service is expected to respond with the result + // manifest that starts with the 'status' (HTTP status code) and 'message' + // (diagnostics message) values optionally followed by 'reference' and + // then other manifest values. If the status is not 200 and reference is + // present, then it is included in the diagnostics. + // + result + post (const common_options& o, const url&, const parameters&); + } +} + +#endif // BDEP_HTTP_SERVICE_HXX diff --git a/bdep/publish.cxx b/bdep/publish.cxx index 81afcdb..de05534 100644 --- a/bdep/publish.cxx +++ b/bdep/publish.cxx @@ -4,9 +4,6 @@ #include <bdep/publish.hxx> -#include <cstdlib> // strtoul() - -#include <libbutl/fdstream.mxx> // fdterm() #include <libbutl/manifest-parser.mxx> #include <libbutl/manifest-serializer.mxx> @@ -17,6 +14,7 @@ #include <bdep/project-author.hxx> #include <bdep/database.hxx> #include <bdep/diagnostics.hxx> +#include <bdep/http-service.hxx> #include <bdep/sync.hxx> @@ -51,460 +49,6 @@ namespace bdep info << "use --control to specify explicitly" << endf; } - // Submit package archive using the curl program and parse the response - // manifest. On success, return the submission reference (first) and message - // (second). Issue diagnostics and fail if anything goes wrong. - // - static pair<string, string> - submit (const cmd_publish_options& o, - const path& archive, - const string& checksum, - const string& section, - const project_author& author, - const optional<url>& ctrl) - { - using parser = manifest_parser; - using parsing = manifest_parsing; - using name_value = manifest_name_value; - - // The overall plan is to post the archive using the curl program, read - // the HTTP response status and content type, read and parse the body - // according to the content type, and obtain the result reference and - // message in case of both the submission success and failure. - // - // The successful submission response (HTTP status code 200) is expected - // to contain the submission result manifest (text/manifest content type). - // The faulty response (HTTP status code other than 200) can either - // contain the result manifest or a plain text error description - // (text/plain content type) or some other content (for example - // text/html). We will print the manifest message value, if available or - // the first line of the plain text error description or, as a last - // resort, construct the message from the HTTP status code and reason - // phrase. - // - string message; - optional<uint16_t> status; // Submission result manifest status value. - optional<string> reference; // Must be present on the submission success. - - // None of the 3XX redirect code semantics assume automatic re-posting. We - // will treat all such codes as failures, additionally printing the - // location header value to advise the user to try the other URL for the - // package submission. - // - // Note that repositories that move to a new URL may well be responding - // with the moved permanently (301) code. - // - optional<url> location; - - // Note that it's a bad idea to issue the diagnostics while curl is - // running, as it will be messed up with the progress output. Thus, we - // throw the runtime_error exception on the HTTP response parsing error - // (rather than use our fail stream) and issue the diagnostics after curl - // finishes. - // - // Also note that we prefer the start/finish process facility for running - // curl over using butl::curl because in this context it is restrictive - // and inconvenient. - // - process pr; - bool io (false); - try - { - url u (o.repository ()); - u.query = "submit"; - - // Map the verbosity level. - // - cstrings v; - if (verb < 1) - { - v.push_back ("-s"); - v.push_back ("-S"); // But show errors. - } - else if (verb == 1 && fdterm (2)) - v.push_back ("--progress-bar"); - else if (verb > 3) - v.push_back ("-v"); - - // Start curl program. - // - fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate. - - // Note that we don't specify any default timeouts, assuming that bdep - // is an interactive program and the user can always interrupt the - // command (or pass the timeout with --curl-option). - // - pr = start (0 /* stdin */, - pipe /* stdout */, - 2 /* stderr */, - o.curl (), - v, - "-A", (BDEP_USER_AGENT " curl"), - - o.curl_option (), - - // Include the response headers in the output so we can get - // the status code/reason, content type, and the redirect - // location. - // - "--include", - - "--form", "archive=@" + archive.string (), - "--form-string", "sha256sum=" + checksum, - "--form-string", "section=" + section, - "--form-string", "author-name=" + *author.name, - "--form-string", "author-email=" + *author.email, - - ctrl - ? strings ({"--form-string", "control=" + ctrl->string ()}) - : strings (), - - o.simulate_specified () - ? strings ({"--form-string", "simulate=" + o.simulate ()}) - : strings (), - - u.string ()); - - pipe.out.close (); - - // First we read the HTTP response status line and headers. At this - // stage we will read until the empty line (containing just CRLF). Not - // being able to reach such a line is an error, which is the reason for - // the exception mask choice. - // - ifdstream is ( - move (pipe.in), - fdstream_mode::skip, - ifdstream::badbit | ifdstream::failbit | ifdstream::eofbit); - - // Parse and return the HTTP status code. Return 0 if the argument is - // invalid. - // - auto status_code = [] (const string& s) - { - char* e (nullptr); - unsigned long c (strtoul (s.c_str (), &e, 10)); // Can't throw. - assert (e != nullptr); - - return *e == '\0' && c >= 100 && c < 600 - ? static_cast<uint16_t> (c) - : 0; - }; - - // Read the CRLF-terminated line from the stream stripping the trailing - // CRLF. - // - auto read_line = [&is] () - { - string l; - getline (is, l); // Strips the trailing LF (0xA). - - // Note that on POSIX CRLF is not automatically translated into LF, - // so we need to strip CR (0xD) manually. - // - if (!l.empty () && l.back () == '\r') - l.pop_back (); - - return l; - }; - - auto bad_response = [] (const string& d) {throw runtime_error (d);}; - - // Read and parse the HTTP response status line, return the status code - // and the reason phrase. - // - struct http_status - { - uint16_t code; - string reason; - }; - - auto read_status = [&read_line, &status_code, &bad_response] () - { - string l (read_line ()); - - for (;;) // Breakout loop. - { - if (l.compare (0, 5, "HTTP/") != 0) - break; - - size_t p (l.find (' ', 5)); // Finds the protocol end. - if (p == string::npos) - break; - - p = l.find_first_not_of (' ', p + 1); // Finds the code start. - if (p == string::npos) - break; - - size_t e (l.find (' ', p + 1)); // Finds the code end. - if (e == string::npos) - break; - - uint16_t c (status_code (string (l, p, e - p))); - if (c == 0) - break; - - string r; - p = l.find_first_not_of (' ', e + 1); // Finds the reason start. - if (p != string::npos) - { - e = l.find_last_not_of (' '); // Finds the reason end. - assert (e != string::npos && e >= p); - - r = string (l, p, e - p + 1); - } - - return http_status {c, move (r)}; - } - - bad_response ("invalid HTTP response status line '" + l + "'"); - - assert (false); // Can't be here. - return http_status {}; - }; - - // The curl output for a successfull submission looks like this: - // - // HTTP/1.1 100 Continue - // - // HTTP/1.1 200 OK - // Content-Length: 83 - // Content-Type: text/manifest;charset=utf-8 - // - // : 1 - // status: 200 - // message: submission queued - // reference: 256910ca46d5 - // - // curl normally sends the 'Expect: 100-continue' header for uploads, - // so we need to handle the interim HTTP server response with the - // continue (100) status code. - // - // Interestingly, Apache can respond with the continue (100) code and - // with the not found (404) code afterwords. Can it be configured to - // just respond with 404? - // - http_status rs (read_status ()); - - if (rs.code == 100) - { - while (!read_line ().empty ()) ; // Skips the interim response. - rs = read_status (); // Reads the final status code. - } - - // Read through the response headers until the empty line is encountered - // and obtain the content type and/or the redirect location, if present. - // - optional<string> ctype; - - // Check if the line contains the specified header and return its value - // if that's the case. Return nullopt otherwise. - // - // Note that we don't expect the header values that we are interested in - // to span over multiple lines. - // - string l; - auto header = [&l] (const char* name) -> optional<string> - { - size_t n (string::traits_type::length (name)); - if (!(casecmp (name, l, n) == 0 && l[n] == ':')) - return nullopt; - - string r; - size_t p (l.find_first_not_of (' ', n + 1)); // Finds value begin. - if (p != string::npos) - { - size_t e (l.find_last_not_of (' ')); // Finds value end. - assert (e != string::npos && e >= p); - - r = string (l, p, e - p + 1); - } - - return optional<string> (move (r)); - }; - - while (!(l = read_line ()).empty ()) - { - if (optional<string> v = header ("Content-Type")) - ctype = move (v); - else if (optional<string> v = header ("Location")) - { - if ((rs.code >= 301 && rs.code <= 303) || rs.code == 307) - try - { - location = url (*v); - location->query = nullopt; // Can possibly contain '?submit'. - } - catch (const invalid_argument&) - { - // Let's just ignore invalid locations. - // - } - } - } - - assert (!eof (is)); // Would have already failed otherwise. - - // Now parse the response payload if the content type is specified and - // is recognized (text/manifest or text/plain), skip it (with the - // ifdstream's close() function) otherwise. - // - // Note that eof and getline() fail conditions are not errors anymore, - // so we adjust the exception mask accordingly. - // - is.exceptions (ifdstream::badbit); - - if (ctype) - { - if (casecmp ("text/manifest", *ctype, 13) == 0) - { - parser p (is, "manifest"); - name_value nv (p.next ()); - - if (nv.empty ()) - bad_response ("empty manifest"); - - const string& n (nv.name); - string& v (nv.value); - - // The format version pair is verified by the parser. - // - assert (n.empty () && v == "1"); - - auto bad_value = [&p, &nv] (const string& d) { - throw parsing (p.name (), nv.value_line, nv.value_column, d);}; - - // Get and verify the HTTP status. - // - nv = p.next (); - if (n != "status") - bad_value ("no status specified"); - - uint16_t c (status_code (v)); - if (c == 0) - bad_value ("invalid HTTP status '" + v + "'"); - - if (c != rs.code) - bad_value ("status " + v + " doesn't match HTTP response " - "code " + to_string (rs.code)); - - // Get the message. - // - nv = p.next (); - if (n != "message" || v.empty ()) - bad_value ("no message specified"); - - message = move (v); - - // Try to get an optional reference and make sure it is present if - // the submission succeeded. - // - nv = p.next (); - - if (n == "reference") - { - if (v.empty ()) - bad_value ("empty reference specified"); - - reference = move (v); - - nv = p.next (); - } - else if (c == 200) - bad_value ("no reference specified"); - - // Skip the remaining name/value pairs. - // - for (; !nv.empty (); nv = p.next ()) ; - - status = c; - } - else if (casecmp ("text/plain", *ctype, 10) == 0) - getline (is, message); // Can result in the empty message. - } - - is.close (); // Detect errors. - - // The meaningful result we expect is either manifest (status code is - // not necessarily 200) or HTTP redirect (location is present). We - // unable to interpret any other cases and so report them as a bad - // response. - // - if (!status) - { - if (rs.code == 200) - bad_response ("manifest expected"); - - if (message.empty ()) - { - message = "HTTP status code " + to_string (rs.code); - - if (!rs.reason.empty ()) - message += " (" + lcase (rs.reason) + ")"; - } - - if (!location) - bad_response (message); - } - } - catch (const io_error&) - { - // Presumably the child process failed and issued diagnostics so let - // finish() try to deal with that first. - // - io = true; - } - // Handle all parsing errors, including the manifest_parsing exception that - // inherits from the runtime_error exception. - // - // Note that the io_error class inherits from the runtime_error class, so - // this catch-clause must go last. - // - catch (const runtime_error& e) - { - finish (o.curl (), pr); // Throws on process failure. - - // Finally we can safely issue the diagnostics (see above for details). - // - diag_record dr (fail); - dr << e << - info << "consider reporting this to " << o.repository () - << " repository maintainers"; - - if (reference) - dr << info << "reference: " << *reference; - } - - finish (o.curl (), pr, io); - - assert (!message.empty ()); - - // Print the submission failure reason and fail. - // - if (!status || *status != 200) - { - diag_record dr (fail); - dr << message; - - if (reference) - dr << info << "reference: " << *reference; - - if (location) - dr << info << "new repository location: " << *location; - - // In case of a server error advise the user to re-try later, assuming - // that the issue is temporary (service overload, network connectivity - // loss, etc.). - // - if (status && *status >= 500 && *status < 600) - dr << info << "try again later"; - } - - assert (reference); // Should be present if the submission succeeded. - - return make_pair (move (*reference), move (message)); - } - static int cmd_publish (const cmd_publish_options& o, const dir_path& prj, @@ -1157,11 +701,32 @@ namespace bdep if (verb) text << "submitting " << p.archive.leaf (); - pair<string, string> r ( - submit (o, p.archive, p.checksum, p.section, author, ctrl)); + url u (o.repository ()); + u.query = "submit"; + + using namespace http_service; + + parameters params ({{parameter::file, "archive", p.archive.string ()}, + {parameter::text, "sha256sum", p.checksum}, + {parameter::text, "section", p.section}, + {parameter::text, "author-name", *author.name}, + {parameter::text, "author-email", *author.email}}); + + if (ctrl) + params.push_back ({parameter::text, "control", ctrl->string ()}); + + if (o.simulate_specified ()) + params.push_back ({parameter::text, "simulate", o.simulate ()}); + + // Disambiguates with odb::result. + // + http_service::result r (post (o, u, params)); + + if (!r.reference) + fail << "no reference specified"; if (verb) - text << r.second << " (" << r.first << ")"; + text << r.message << " (" << *r.reference << ")"; } return 0; diff --git a/tests/ci.test b/tests/ci.test new file mode 100644 index 0000000..85155de --- /dev/null +++ b/tests/ci.test @@ -0,0 +1,198 @@ +# file : tests/ci.test +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +.include common.test project.test + +# bdep-ci requirements for the minimum supported git version are higher then +# the default 2.1.0 (see bdep/ci.cxx for details). +# ++if! ($git_version_major > 2 || \ + $git_version_major == 2 && $git_version_minor >= 11) + exit +end + +# Server to use for the CI request submissions simulation. +# +# Note that the empty config.bdep.test.repository value is used to suppress +# these tests (which require network access). +# +server = ($config.bdep.test.repository == [null] \ + ? ($build.version.stage \ + ? 'https://stage.build2.org' \ + : 'https://ci.cppget.org') \ + : "$config.bdep.test.repository") + ++if ("$server" == '') + exit +end + +# Create the remote repository. +# ++mkdir --no-cleanup prj.git ++git -C prj.git init --bare 2>! >&2 &prj.git/*** + +# Adjust the local repository and push it to the remote one. +# ++sed -i -e 's/^(version:) .*$/\1 1.0.1/' prj/manifest + +g = git -C prj 2>! >&2 + ++$g remote add origin $~/prj.git ++$g add '*' ++$g commit -m 'Create' ++$g push --set-upstream origin master + +# Repository the CI-ed packages come from. +# +repository='http://example.com/prj.git' + +test.arguments += --yes --repository "$repository" --server "$server" \ +--simulate 'success' + +cxx = cc "config.cxx=$config.cxx" + +new += 2>! +init += $cxx -d prj 2>! &prj/**/bootstrap/*** + +windows = ($cxx.target.class == 'windows') + +: single-pkg +: +{ + : single-cfg + : + { + $clone_root_prj; + $init -C @cfg &prj-cfg/***; + + $* 2>>~"%EOE%" + submitting to $server + %.* + %CI request is queued \(.+\)% + EOE + } + + : no-cfg + : + { + $new prj &prj/***; + + $* 2>>~%EOE% != 0 + %error: no default configuration in project .+% + % info: use .+% + EOE + } + + : multi-cfg + : + { + $clone_root_prj; + $init -C @cfg1 &prj-cfg1/***; + $init -C @cfg2 &prj-cfg2/***; + + $* --all 2>'error: multiple configurations specified for ci' != 0 + } + + : no-commits + : + { + $new prj &prj/***; + $init -C @cfg &prj-cfg/***; + + $* 2>>~%EOE% != 0 + error: no commits in project repository + % info: run .+% + EOE + } + + : invalid-repo + : + { + test.arguments += --repository "$repository#frag" + + $clone_root_prj; + $init -C @cfg &prj-cfg/***; + + $* 2>>"EOE" != 0 + error: remote git repository URL '$repository#frag' already has fragment + EOE + } +} + +: multi-pkg +: +{ + # Create the remote repository. + # + +mkdir --no-cleanup prj.git + +git -C prj.git init --bare 2>! >&2 &prj.git/*** + + # Create the local repository and push it to the remote one. + # + +$new -t empty prj &prj/*** + +$new --package -t lib libprj -d prj + +$new --package -t exe prj -d prj + +sed -i -e 's/^(version:) .*$/\1 1.0.1/' prj/libprj/manifest + +sed -i -e 's/^(version:) .*$/\1 1.0.1/' prj/prj/manifest + + +$g remote add origin $~/prj.git + +$g add '*' + +$g commit -m 'Create' + +$g push --set-upstream origin master + + : both + : + { + $clone_prj; + $init -C @cfg &prj-cfg/***; + + $* 2>>~"%EOE%" + submitting to $server + %.* + %CI request is queued \(.+\)% + EOE + } + + : single + : + { + $clone_prj; + $init -C @cfg &prj-cfg/***; + + # CI the single libprj package rather than the whole prj project. + # + test.arguments = $regex.apply($test.arguments, '^(prj)$', '\1/libprj'); + + $* 2>>~"%EOE%" + submitting to $server + %.* + %CI request is queued \(.+\)% + EOE + } + + : prompt + : + { + $clone_prj; + $init -C @cfg &prj-cfg/***; + + # Suppress the --yes option. + # + test.arguments = $regex.apply($test.arguments, '^(--yes)$', ''); + + $* <'y' 2>>~"%EOE%" + submitting: + to: $server + % in: $repository#master@.{40}% + + package: libprj + version: 1.0.1 + + package: prj + version: 1.0.1 + %.* + %CI request is queued \(.+\)% + EOE + } +} diff --git a/tests/publish.test b/tests/publish.test index 4756abc..ba88e9a 100644 --- a/tests/publish.test +++ b/tests/publish.test @@ -14,8 +14,8 @@ end # Repository to use for the package submissions simulation. # -# Note: could use empty config.bdep.test.repository value to suppress -# these test (which require network access). +# Note that the empty config.bdep.test.repository value is used to suppress +# these tests (which require network access). # repository = ($config.bdep.test.repository == [null] \ ? ($build.version.stage \ @@ -23,6 +23,10 @@ repository = ($config.bdep.test.repository == [null] \ : 'https://cppget.org') \ : "$config.bdep.test.repository") ++if ("$repository" == '') + exit +end + test.arguments += --repository "$repository" --yes \ --author-name user --author-email user@example.com @@ -49,7 +53,7 @@ windows = ($cxx.target.class == 'windows') { test.arguments += --simulate 'success' - : basic + : single-cfg : { $clone_root_prj; @@ -205,7 +209,7 @@ windows = ($cxx.target.class == 'windows') submitting prj-1.0.6.tar.gz %.* error: submission handling failed - % info: consider reporting this to .+ repository maintainers% + % info: consider reporting this to .+ maintainers% EOE } @@ -224,7 +228,7 @@ windows = ($cxx.target.class == 'windows') submitting prj-1.0.7.tar.gz %.* error: HTTP status code 500 (internal server error) - % info: consider reporting this to .+ repository maintainers% + % info: consider reporting this to .+ maintainers% EOE } } @@ -237,19 +241,19 @@ windows = ($cxx.target.class == 'windows') # simulation. We specify it to enable the control branch-related # functionality. # - test.arguments += --simulate 'success' --control "http://example.com/rep.git" + test.arguments += --simulate 'success' --control 'http://example.com/rep.git' # Create the remote repository. # +mkdir --no-cleanup prj.git - +git -C prj.git init --bar >! &prj.git/*** + +git -C prj.git init --bare >! &prj.git/*** +$clone_prj g = git -C prj >! 2>! - +$g config user.name "Test Script" - +$g config user.email "testscript@example.com" + +$g config user.name 'Test Script' + +$g config user.email 'testscript@example.com' clone_rep = cp --no-cleanup -r ../prj.git ./ &prj.git/*** clone_prj = cp --no-cleanup -r ../prj ./ &prj/*** @@ -365,8 +369,8 @@ windows = ($cxx.target.class == 'windows') git clone "$rep" prj2 &prj2/*** 2>&1 | \ sed -e 's/warning: (remote HEAD refers to nonexistent .*)/info: \1/' >&2 2>!; - $g2 config user.name "Test Script"; - $g2 config user.email "testscript@example.com"; + $g2 config user.name 'Test Script'; + $g2 config user.email 'testscript@example.com'; $g2 checkout -b build2-control --track origin/build2-control; $g2 commit --allow-empty -m 'Dummy1'; $g2 push; |