Implement curl process

10 files changed, 573 insertions, 15 deletions

diff --git a/butl/buildfile b/butl/buildfile
index 7d33535..328afc0 100644
--- a/butl/buildfile
+++ b/butl/buildfile
@@ -6,6 +6,7 @@ lib{butl}:                                  \
    {hxx         cxx}{ base64              } \
    {hxx         cxx}{ char-scanner        } \
    {hxx            }{ const-ptr           } \
+   {hxx ixx txx cxx}{ curl                } \
    {hxx         cxx}{ diagnostics         } \
    {hxx            }{ export              } \
    {hxx ixx     cxx}{ fdstream            } \
@@ -22,6 +23,7 @@ lib{butl}:                                  \
    {hxx     txx    }{ prefix-map          } \
    {hxx ixx     cxx}{ process             } \
    {hxx            }{ process-details     } \
+   {hxx            }{ process-io          } \
    {        txx cxx}{ process-run         } \
    {hxx ixx     cxx}{ sendmail            } \
    {hxx         cxx}{ sha256              } \
diff --git a/butl/curl b/butl/curl
new file mode 100644
index 0000000..c8285ea
--- /dev/null
+++ b/butl/curl
@@ -0,0 +1,173 @@
+// file      : butl/curl -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUTL_CURL
+#define BUTL_CURL
+
+#include <string>
+#include <type_traits>
+
+#include <butl/export>
+
+#include <butl/process>
+#include <butl/fdstream>
+#include <butl/small-vector>
+
+namespace butl
+{
+  // Perform a method (GET, POST, PUT) on a URL using the curl(1) program.
+  // Throw process_error and io_error (both derive from system_error) in case
+  // of errors.
+  //
+  // The I (in) and O (out) can be of the following types/values:
+  //
+  // nullfd   Signal that no input/output is expected.
+  //
+  // path     Read input/write output from/to a file. If the special "-"
+  //          value is used, then instead input is connected to the curl::out
+  //          ofdstream member and output -- to the curl::in ifdstream member.
+  //          Note that the argument type should be path, not string (i.e.,
+  //          pass path("-")).
+  //
+  // other    Forwarded as is to process_start(). Normally either int or
+  //          auto_fd.
+  //
+  // For example:
+  //
+  //   curl (nullfd,     // No input expected for GET.
+  //         path ("-"), // Write response to curl::in.
+  //         2,
+  //         curl::get,
+  //         "http://example.org");
+  //
+  //   curl (path ("-"),         // Read request from curl::out.
+  //         path::temp_path (), // Write result to a file.
+  //         2,
+  //         curl::post,
+  //         "http://example.org");
+  //
+  //   curl (nullfd,
+  //         fdnull (), // Write result to /dev/null.
+  //         2,
+  //         curl::get,
+  //         "tftp://localhost/foo");
+  //
+  // Typical usage:
+  //
+  // try
+  // {
+  //   curl c (nullfd,                 // No input expected.
+  //           path ("-"),             // Output to curl::in.
+  //           2,                      // Diagnostics to stderr.
+  //           curl::get,              // GET method.
+  //           "https://example.org",
+  //           "-A", "foobot/1.2.3");  // Additional curl(1) options.
+  //
+  //   for (string s; getline (c.in, s); )
+  //     cout << s << endl;
+  //
+  //   c.in.close ();
+  //
+  //   if (!c.wait ())
+  //     ... // curl returned non-zero status.
+  // }
+  // catch (const std::system_error& e)
+  // {
+  //   cerr << "curl error: " << e << endl;
+  // }
+  //
+  // Notes:
+  //
+  // 1. If opened, in/out streams are in the binary mode.
+  //
+  // 2. If opened, in/out must be explicitly closed before calling wait().
+  //
+  // 3. Only binary data HTTP POST is currently supported (the --data-binary
+  //    curl option).
+  //
+  class LIBBUTL_EXPORT curl: public process
+  {
+  public:
+    enum method_type {get, put, post};
+
+    ifdstream in;
+    ofdstream out;
+
+    template <typename I,
+              typename O,
+              typename E,
+              typename... A>
+    curl (I&& in,
+          O&& out,
+          E&& err,
+          method_type,
+          const std::string& url,
+          A&&... options);
+
+    // Version with the command line callback (see process_run() for details).
+    //
+    template <typename C,
+              typename I,
+              typename O,
+              typename E,
+              typename... A>
+    curl (const C&,
+          I&& in,
+          O&& out,
+          E&& err,
+          method_type,
+          const std::string& url,
+          A&&... options);
+
+  private:
+    enum method_proto {ftp_get, ftp_put, http_get, http_post};
+    using method_proto_options = small_vector<const char*, 2>;
+
+    method_proto
+    translate (method_type, const std::string& url, method_proto_options&);
+
+  private:
+    template <typename T>
+    struct is_other
+    {
+      using type = typename std::remove_reference<
+        typename std::remove_cv<T>::type>::type;
+
+      static const bool value = !(std::is_same<type, nullfd_t>::value ||
+                                  std::is_same<type, path>::value);
+    };
+
+    struct io_data
+    {
+      fdpipe pipe;
+      method_proto_options options;
+      std::string storage;
+    };
+
+    int
+    map_in (nullfd_t, method_proto, io_data&);
+
+    int
+    map_in (const path&, method_proto, io_data&);
+
+    template <typename I>
+    typename std::enable_if<is_other<I>::value, I>::type
+    map_in (I&&, method_proto, io_data&);
+
+    int
+    map_out (nullfd_t, method_proto, io_data&);
+
+    int
+    map_out (const path&, method_proto, io_data&);
+
+    template <typename O>
+    typename std::enable_if<is_other<O>::value, O>::type
+    map_out (O&&, method_proto, io_data&);
+  };
+}
+
+#include <butl/curl.ixx>
+#include <butl/curl.txx>
+
+#endif // BUTL_CURL
diff --git a/butl/curl.cxx b/butl/curl.cxx
new file mode 100644
index 0000000..4951b52
--- /dev/null
+++ b/butl/curl.cxx
@@ -0,0 +1,166 @@
+// file      : butl/curl.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <butl/curl>
+
+#include <utility>   // move(), forward()
+#include <exception> // invalid_argument
+
+#include <butl/utility> // casecmp()
+
+using namespace std;
+
+namespace butl
+{
+  int curl::
+  map_in (nullfd_t, method_proto mp, io_data& d)
+  {
+    switch (mp)
+    {
+    case ftp_put:
+      throw invalid_argument ("no input specified for PUT method");
+    case http_post:
+      throw invalid_argument ("no input specified for POST method");
+    case ftp_get:
+    case http_get:
+      {
+        d.pipe.in.reset (fdnull ()); // /dev/null
+        return d.pipe.in.get ();
+      }
+    }
+
+    return -1;
+  }
+
+  int curl::
+  map_in (const path& f, method_proto mp, io_data& d)
+  {
+    switch (mp)
+    {
+    case ftp_put:
+    case http_post:
+      {
+        if (mp == ftp_put)
+        {
+          d.options.push_back ("--upload-file");
+          d.options.push_back (f.string ().c_str ());
+        }
+        else
+        {
+          d.storage = '@' + f.string ();
+
+          d.options.push_back ("--data-binary");
+          d.options.push_back (d.storage.c_str ());
+        }
+
+        if (f.string () == "-")
+        {
+          d.pipe = fdopen_pipe (fdopen_mode::binary);
+          out.open (move (d.pipe.out));
+        }
+        else
+          d.pipe.in.reset (fdnull ()); // /dev/null
+
+        return d.pipe.in.get ();
+      }
+    case ftp_get:
+    case http_get:
+      {
+        throw invalid_argument ("file input specified for GET method");
+      }
+    }
+
+    return -1;
+  }
+
+  int curl::
+  map_out (nullfd_t, method_proto mp, io_data& d)
+  {
+    switch (mp)
+    {
+    case ftp_get:
+    case http_get:
+      throw invalid_argument ("no output specified for GET method");
+    case ftp_put:
+    case http_post: // May or may not produce output.
+      {
+        d.pipe.out.reset (fdnull ());
+        return d.pipe.out.get (); // /dev/null
+      }
+    }
+
+    return -1;
+  }
+
+  int curl::
+  map_out (const path& f, method_proto mp, io_data& d)
+  {
+    switch (mp)
+    {
+    case ftp_get:
+    case http_get:
+    case http_post:
+      {
+        if (f.string () == "-")
+        {
+          // Note: no need for any options, curl writes to stdout by default.
+          //
+          d.pipe = fdopen_pipe (fdopen_mode::binary);
+          in.open (move (d.pipe.in));
+        }
+        else
+        {
+          d.options.push_back ("-o");
+          d.options.push_back (f.string ().c_str ());
+          d.pipe.out.reset (fdnull ()); // /dev/null
+        }
+
+        return d.pipe.out.get ();
+      }
+    case ftp_put:
+      {
+        throw invalid_argument ("file output specified for PUT method");
+      }
+    }
+
+    return -1;
+  }
+
+  curl::method_proto curl::
+  translate (method_type m, const string& u, method_proto_options& o)
+  {
+    size_t n (u.find ("://"));
+
+    if (n == string::npos)
+      throw invalid_argument ("no protocol in URL");
+
+    if (casecmp (u, "ftp", n) == 0 ||
+        casecmp (u, "tftp", n) == 0)
+    {
+      switch (m)
+      {
+      case method_type::get: return method_proto::ftp_get;
+      case method_type::put: return method_proto::ftp_put;
+      case method_type::post:
+        throw invalid_argument ("POST method with FTP protocol");
+      }
+    }
+    else if (casecmp (u, "http", n) == 0 ||
+             casecmp (u, "https", n) == 0)
+    {
+      o.push_back ("--fail");     // Fail on HTTP errors (e.g., 404).
+      o.push_back ("--location"); // Follow redirects.
+
+      switch (m)
+      {
+      case method_type::get:  return method_proto::http_get;
+      case method_type::post: return method_proto::http_post;
+      case method_type::put:
+        throw invalid_argument ("PUT method with HTTP protocol");
+      }
+    }
+
+    throw invalid_argument ("unsupported protocol");
+  }
+}
diff --git a/butl/curl.ixx b/butl/curl.ixx
new file mode 100644
index 0000000..83b388b
--- /dev/null
+++ b/butl/curl.ixx
@@ -0,0 +1,29 @@
+// file      : butl/curl.ixx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <utility> // move(), forward()
+
+namespace butl
+{
+  template <typename I,
+            typename O,
+            typename E,
+            typename... A>
+  inline curl::
+  curl (I&& in,
+        O&& out,
+        E&& err,
+        method_type m,
+        const std::string& url,
+        A&&... options)
+      : curl ([] (const char* [], std::size_t) {},
+              std::forward<I> (in),
+              std::forward<O> (out),
+              std::forward<E> (err),
+              m,
+              url,
+              std::forward<A> (options)...)
+  {
+  }
+}
diff --git a/butl/curl.txx b/butl/curl.txx
new file mode 100644
index 0000000..fe8a25f
--- /dev/null
+++ b/butl/curl.txx
@@ -0,0 +1,99 @@
+// file      : butl/curl.txx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <utility>   // move(), forward()
+#include <exception> // invalid_argument
+
+namespace butl
+{
+  template <typename I>
+  typename std::enable_if<curl::is_other<I>::value, I>::type curl::
+  map_in (I&& in, method_proto mp, io_data& d)
+  {
+    switch (mp)
+    {
+    case ftp_put:
+      {
+        d.options.push_back ("--upload-file");
+        d.options.push_back ("-");
+        break;
+      }
+    case http_post:
+      {
+        d.options.push_back ("--data-binary");
+        d.options.push_back ("@-");
+        break;
+      }
+    case ftp_get:
+    case http_get:
+      {
+        throw std::invalid_argument ("input specified for GET method");
+      }
+    }
+
+    return std::forward<I> (in);
+  }
+
+  template <typename O>
+  typename std::enable_if<curl::is_other<O>::value, O>::type curl::
+  map_out (O&& out, method_proto mp, io_data&)
+  {
+    switch (mp)
+    {
+    case ftp_get:
+    case http_get:
+    case http_post:
+      {
+        // Note: no need for any options, curl writes to stdout by default.
+        //
+        break;
+      }
+    case ftp_put:
+      {
+        throw std::invalid_argument ("output specified for PUT method");
+      }
+    }
+
+    return std::forward<O> (out);
+  }
+
+  template <typename C,
+            typename I,
+            typename O,
+            typename E,
+            typename... A>
+  curl::
+  curl (const C& cmdc,
+        I&& in,
+        O&& out,
+        E&& err,
+        method_type m,
+        const std::string& url,
+        A&&... options)
+  {
+    method_proto_options mpo;
+    method_proto mp (translate (m, url, mpo));
+
+    io_data in_data;
+    io_data out_data;
+
+    process& p (*this);
+    p = process_start (
+      cmdc,
+      map_in  (std::forward<I> (in),  mp, in_data),
+      map_out (std::forward<O> (out), mp, out_data),
+      std::forward<E> (err),
+      dir_path (),
+      "curl",
+      "-s", // Silent.
+      "-S", // But do show diagnostics.
+      mpo,
+      in_data.options,
+      out_data.options,
+      std::forward<A> (options)...,
+      url);
+
+    // Note: leaving this scope closes any open ends of the pipes in io_data.
+  }
+}
diff --git a/butl/fdstream b/butl/fdstream
index 8f49990..df6b3f1 100644
--- a/butl/fdstream
+++ b/butl/fdstream
@@ -612,6 +612,13 @@ namespace butl
   {
     auto_fd in;
     auto_fd out;
+
+    void
+    close ()
+    {
+      in.close ();
+      out.close ();
+    }
   };
 
   // Create a pipe. Throw ios::failure on the underlying OS error. By default
diff --git a/butl/process b/butl/process
index c138edc..282a994 100644
--- a/butl/process
+++ b/butl/process
@@ -9,6 +9,7 @@
 #  include <sys/types.h> // pid_t
 #endif
 
+#include <vector>
 #include <iosfwd>
 #include <cassert>
 #include <cstdint>      // uint32_t
@@ -17,7 +18,9 @@
 #include <butl/path>
 #include <butl/export>
 #include <butl/optional>
-#include <butl/fdstream> // auto_fd, fdpipe
+#include <butl/fdstream>     // auto_fd, fdpipe
+#include <butl/vector-view>
+#include <butl/small-vector>
 
 namespace butl
 {
@@ -389,9 +392,8 @@ namespace butl
   //
   // The A arguments can be anything convertible to const char* via the
   // overloaded process_arg_as() (see below). Out of the box you can use const
-  // char*, std::string, path/dir_path, and numeric types.
-  //
-  //
+  // char*, std::string, path/dir_path, (as well as [small_]vector[_view] of
+  // these), and numeric types.
   //
   template <typename I,
             typename O,
@@ -480,6 +482,8 @@ namespace butl
     return p.string ().c_str ();
   }
 
+  // char[N]
+  //
   inline const char*
   process_arg_as (const char* s, std::string&) {return s;}
 
@@ -490,6 +494,63 @@ namespace butl
   template <std::size_t N>
   inline const char*
   process_arg_as (const char (&s)[N], std::string&) {return s;}
+
+  template <typename V, typename T>
+  inline void
+  process_args_as (V& v, const T& x, std::string& storage)
+  {
+    v.push_back (process_arg_as (x, storage));
+  }
+
+  // [small_]vector[_view]<>
+  //
+  template <typename V>
+  inline void
+  process_args_as (V& v, const std::vector<std::string>& vs, std::string&)
+  {
+    for (const std::string& s: vs)
+      v.push_back (s.c_str ());
+  }
+
+  template <typename V, std::size_t N>
+  inline void
+  process_args_as (V& v, const small_vector<std::string, N>& vs, std::string&)
+  {
+    for (const std::string& s: vs)
+      v.push_back (s.c_str ());
+  }
+
+  template <typename V>
+  inline void
+  process_args_as (V& v, const vector_view<std::string>& vs, std::string&)
+  {
+    for (const std::string& s: vs)
+      v.push_back (s.c_str ());
+  }
+
+  template <typename V>
+  inline void
+  process_args_as (V& v, const std::vector<const char*>& vs, std::string&)
+  {
+    for (const char* s: vs)
+      v.push_back (s);
+  }
+
+  template <typename V, std::size_t N>
+  inline void
+  process_args_as (V& v, const small_vector<const char*, N>& vs, std::string&)
+  {
+    for (const char* s: vs)
+      v.push_back (s);
+  }
+
+  template <typename V>
+  inline void
+  process_args_as (V& v, const vector_view<const char*>& vs, std::string&)
+  {
+    for (const char* s: vs)
+      v.push_back (s);
+  }
 }
 
 #include <butl/process.ixx>
diff --git a/butl/process-io b/butl/process-io
new file mode 100644
index 0000000..42a720a
--- /dev/null
+++ b/butl/process-io
@@ -0,0 +1,21 @@
+// file      : butl/process-io -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUTL_PROCESS_IO
+#define BUTL_PROCESS_IO
+
+#include <ostream>
+
+#include <butl/process>
+
+namespace butl
+{
+  inline std::ostream&
+  operator<< (std::ostream& o, const process_path& p)
+  {
+    return o << p.recall_string ();
+  }
+}
+
+#endif // BUTL_PROCESS_IO
diff --git a/butl/process-run.txx b/butl/process-run.txx
index 8368890..4437488 100644
--- a/butl/process-run.txx
+++ b/butl/process-run.txx
@@ -55,20 +55,21 @@ namespace butl
     //
     const std::size_t args_size (sizeof... (args));
 
+    small_vector<const char*, args_size + 2> cmd;
+    cmd.push_back (pp.recall_string ());
+
     std::string storage[args_size != 0 ? args_size : 1];
-    const char* cmd[args_size + 2] = {
-      pp.recall_string (),
-      process_arg_as (args, storage[index])...,
-      nullptr};
+    const char* dummy[] = {
+      nullptr, (process_args_as (cmd, args, storage[index]), nullptr)... };
 
-    // The last argument can be NULL (used to handle zero A... pack).
-    //
-    cmdc (cmd, args_size + 2);
+    cmd.push_back (dummy[0]); // NULL (and get rid of unused warning).
+
+    cmdc (cmd.data (), cmd.size ());
 
     // @@ Do we need to make sure certain fd's are closed before calling
     //    wait()? Is this only the case with pipes? Needs thinking.
 
-    return process_start (cwd, pp, cmd, in_i, out_i, err_i);
+    return process_start (cwd, pp, cmd.data (), in_i, out_i, err_i);
   }
 
   template <typename C,
diff --git a/butl/sendmail b/butl/sendmail
index 706f110..fe86285 100644
--- a/butl/sendmail
+++ b/butl/sendmail
@@ -15,7 +15,7 @@
 
 namespace butl
 {
-  // Send email using the sendmail program.
+  // Send email using the sendmail(1) program.
   //
   // Write the body of the email to out. Note that you must explicitly close
   // it before calling wait(). Throw process_error and io_error (both derive
@@ -64,8 +64,7 @@ namespace butl
               const recipients_type& bcc = recipients_type (),
               O&&... options);
 
-    // Version with the command line callback (see process_run() for
-    // details).
+    // Version with the command line callback (see process_run() for details).
     //
     template <typename C, typename E, typename... O>
     sendmail (const C&,