Implement manifests and build_config

1 files changed, 931 insertions, 0 deletions

diff --git a/bbot/manifest.cxx b/bbot/manifest.cxx
new file mode 100644
index 0000000..6c9fe4d
--- /dev/null
+++ b/bbot/manifest.cxx
@@ -0,0 +1,931 @@
+// file      : bbot/manifest.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <bbot/manifest>
+
+#include <vector>
+#include <string>
+#include <cassert>
+#include <sstream>
+#include <cstddef>   // size_t
+#include <utility>   // move()
+#include <cstdint>   // uint64_t
+#include <algorithm> // find()
+
+#include <butl/utility>             // digit()
+#include <butl/tab-parser>
+#include <butl/manifest-parser>
+#include <butl/manifest-serializer>
+
+using namespace std;
+using namespace butl;
+using namespace bpkg;
+
+namespace bbot
+{
+  using parser = manifest_parser;
+  using parsing = manifest_parsing;
+  using serializer = manifest_serializer;
+  using serialization = manifest_serialization;
+  using name_value = manifest_name_value;
+
+  using strings = vector<string>;
+
+  static const strings machine_type_names ({"vm", "container"});
+
+  static const strings result_status_names ({
+    "success",
+    "warning",
+    "error",
+    "abort",
+    "abnormal"});
+
+  ostream&
+  operator<< (ostream& o, result_status s)
+  {
+    size_t i (static_cast<size_t> (s));
+    assert (i < result_status_names.size ());
+    o << result_status_names[i];
+    return o;
+  }
+
+  inline static bool
+  valid_sha256 (const string& s) noexcept
+  {
+    if (s.size () != 64)
+      return false;
+
+    for (const auto& c: s)
+    {
+      if ((c < 'a' || c > 'f' ) && !digit (c))
+        return false;
+    }
+
+    return true;
+  }
+
+  // machine_manifest
+  //
+  machine_manifest::
+  machine_manifest (parser& p, bool iu)
+      : machine_manifest (p, p.next (), false, iu)
+  {
+    // Make sure this is the end.
+    //
+    name_value nv (p.next ());
+    if (!nv.empty ())
+      throw parsing (p.name (), nv.name_line, nv.name_column,
+                     "single machine manifest expected");
+  }
+
+  machine_manifest::
+  machine_manifest (parser& p, name_value nv, bool iu)
+      : machine_manifest (p, nv, true, iu)
+  {
+  }
+
+  machine_manifest::
+  machine_manifest (parser& p, name_value nv, bool il, bool iu)
+  {
+    auto bad_name = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.name_line, nv.name_column, d);
+    };
+
+    auto bad_value = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.value_line, nv.value_column, d);
+    };
+
+    // Make sure this is the start and we support the version.
+    //
+    if (!nv.name.empty ())
+      bad_name ("start of machine manifest expected");
+
+    if (nv.value != "1")
+      bad_value ("unsupported format version");
+
+    for (nv = p.next (); !nv.empty (); nv = p.next ())
+    {
+      string& n (nv.name);
+      string& v (nv.value);
+
+      if (n == "id")
+      {
+        if (!id.empty ())
+          bad_name ("machine id redefinition");
+
+        if (v.empty ())
+          bad_value ("empty machine id");
+
+        id = move (v);
+      }
+      else if (n == "name")
+      {
+        if (!name.empty ())
+          bad_name ("machine name redefinition");
+
+        if (v.empty ())
+          bad_value ("empty machine name");
+
+        name = move (v);
+      }
+      else if (n == "type")
+      {
+        if (il)
+          bad_name ("machine type not allowed");
+
+        if (type)
+          bad_name ("machine type redefinition");
+
+        auto b (machine_type_names.cbegin ());
+        auto e (machine_type_names.cend ());
+        auto i (find (b, e, v));
+
+        if (i == e)
+          bad_value ("invalid machine type");
+
+        type = static_cast<machine_type> (i - b);
+      }
+      else if (n == "summary")
+      {
+        if (!summary.empty ())
+          bad_name ("machine summary redefinition");
+
+        if (v.empty ())
+          bad_value ("empty machine summary");
+
+        summary = move (v);
+      }
+      else if (!iu)
+        bad_name ("unknown name '" + n + "' in machine manifest");
+    }
+
+    // Verify all non-optional values were specified.
+    //
+    if (id.empty ())
+      bad_value ("no machine id specified");
+
+    if (name.empty ())
+      bad_value ("no machine name specified");
+
+    if (!il && !type)
+      bad_value ("no machine type specified");
+
+    if (summary.empty ())
+      bad_value ("no machine summary specified");
+  }
+
+  void machine_manifest::
+  serialize (serializer& s) const
+  {
+    // @@ Should we check that all non-optional values are specified and all
+    //    values are valid?
+    //
+    s.next ("", "1"); // Start of manifest.
+    s.next ("id", id);
+    s.next ("name", name);
+
+    if (type)
+    {
+      size_t v (static_cast<size_t> (*type));
+      assert (v < machine_type_names.size ());
+      s.next ("type", machine_type_names[v]);
+    }
+
+    s.next ("summary", summary);
+    s.next ("", ""); // End of manifest.
+  }
+
+  // task_request_manifest
+  //
+  task_request_manifest::
+  task_request_manifest (parser& p, bool iu)
+  {
+    name_value nv (p.next ());
+
+    auto bad_name = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.name_line, nv.name_column, d);
+    };
+
+    auto bad_value = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.value_line, nv.value_column, d);
+    };
+
+    // Make sure this is the start and we support the version.
+    //
+    if (!nv.name.empty ())
+      bad_name ("start of task request manifest expected");
+
+    if (nv.value != "1")
+      bad_value ("unsupported format version");
+
+    // Parse the task request manifest.
+    //
+    for (nv = p.next (); !nv.empty (); nv = p.next ())
+    {
+      string& n (nv.name);
+      string& v (nv.value);
+
+      if (n == "agent")
+      {
+        if (!agent.empty ())
+          bad_name ("task request agent redefinition");
+
+        if (v.empty ())
+          bad_value ("empty task request agent");
+
+        agent = move (v);
+      }
+      else if (n == "fingerprint")
+      {
+        if (!fingerprint.empty ())
+          bad_name ("task request fingerprint redefinition");
+
+        if (!valid_sha256 (v))
+          bad_value ("invalid task request fingerprint");
+
+        fingerprint = move (v);
+      }
+      else if (!iu)
+        bad_name ("unknown name '" + n + "' in task request manifest");
+    }
+
+    // Verify all non-optional values were specified.
+    //
+    if (agent.empty ())
+      bad_value ("no task request agent specified");
+
+    if (fingerprint.empty ())
+      bad_value ("no task request fingerprint specified");
+
+    // Parse machine manifests.
+    //
+    for (nv = p.next (); !nv.empty (); nv = p.next ())
+      machines.emplace_back (machine_manifest (p, nv, iu));
+
+    if (machines.empty ())
+      bad_value ("no task request machines specified");
+  }
+
+  void task_request_manifest::
+  serialize (serializer& s) const
+  {
+    // @@ Should we check that all non-optional values are specified and all
+    //    values are valid?
+    //
+    s.next ("", "1"); // Start of manifest.
+    s.next ("agent", agent);
+    s.next ("fingerprint", fingerprint);
+    s.next ("", "");  // End of manifest.
+
+    for (const machine_manifest& m: machines)
+    {
+      if (m.type)
+        throw serialization (s.name (), "machine type is forbidden");
+
+      m.serialize (s);
+    }
+
+    s.next ("", ""); // End of stream.
+  }
+
+  // task_manifest
+  //
+  task_manifest::
+  task_manifest (parser& p, bool iu)
+      : task_manifest (p, p.next (), iu)
+  {
+    // Make sure this is the end.
+    //
+    name_value nv (p.next ());
+    if (!nv.empty ())
+      throw parsing (p.name (), nv.name_line, nv.name_column,
+                     "single task manifest expected");
+  }
+
+  task_manifest::
+  task_manifest (parser& p, name_value nv, bool iu)
+  {
+    auto bad_name = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.name_line, nv.name_column, d);
+    };
+
+    // Offsets are used to tie an error to the specific position inside a
+    // manifest value (possibly a multiline one).
+    //
+    auto bad_value = [&p, &nv] (
+      const string& d, uint64_t column_offset = 0, uint64_t line_offset = 0)
+    {
+      throw parsing (p.name (),
+                     nv.value_line + line_offset,
+                     (line_offset == 0 ? nv.value_column : 1) + column_offset,
+                     d);
+    };
+
+    // Make sure this is the start and we support the version.
+    //
+    if (!nv.name.empty ())
+      bad_name ("start of task manifest expected");
+
+    if (nv.value != "1")
+      bad_value ("unsupported format version");
+
+    // Parse the task manifest.
+    //
+    for (nv = p.next (); !nv.empty (); nv = p.next ())
+    {
+      string& n (nv.name);
+      string& v (nv.value);
+
+      if (n == "name")
+      {
+        if (!name.empty ())
+          bad_name ("task package name redefinition");
+
+        if (v.empty ())
+          bad_value ("empty task package name");
+
+        name = move (v);
+      }
+      else if (n == "version")
+      {
+        if (!version.empty ())
+          bad_name ("task package version redefinition");
+
+        try
+        {
+          version = bpkg::version (move (v));
+        }
+        catch (const invalid_argument& e)
+        {
+          bad_value (string ("invalid task package version: ") + e.what ());
+        }
+
+        // Versions like 1.2.3- are forbidden in manifest as intended to be
+        // used for version constrains rather than actual releases.
+        //
+        if (version.release && version.release->empty ())
+          bad_value ("invalid task package version release");
+      }
+      else if (n == "repository")
+      {
+        if (!repository.empty ())
+          bad_name ("task repository redefinition");
+
+        if (v.empty ())
+          bad_value ("empty task repository");
+
+        try
+        {
+          // Call remote/absolute repository location constructor (throws
+          // invalid_argument for relative location).
+          //
+          repository = repository_location (move (v));
+        }
+        catch (const invalid_argument& e)
+        {
+          bad_value (string ("invalid task repository: ") + e.what ());
+        }
+      }
+      else if (n == "machine")
+      {
+        if (!machine.empty ())
+          bad_name ("task machine redefinition");
+
+        if (v.empty ())
+          bad_value ("empty task machine");
+
+        machine = move (v);
+      }
+      else if (n == "target")
+      {
+        if (target)
+          bad_name ("task target redefinition");
+
+        try
+        {
+          target = target_triplet (v);
+        }
+        catch (const invalid_argument& e)
+        {
+          bad_value (string ("invalid task target: ") + e.what ());
+        }
+      }
+      else if (n == "config")
+      {
+        if (!config.empty ())
+          bad_name ("task configuration redefinition");
+
+        // Note that when reporting errors we combine the manifest value
+        // position with the respective field and error positions.
+        //
+        try
+        {
+          istringstream is (v);
+          tab_parser parser (is, "");
+
+          // Here we naturally support multiline config manifest.
+          //
+          tab_fields tl;
+          while (!(tl = parser.next ()).empty ())
+          {
+            for (auto& tf: tl)
+            {
+              try
+              {
+                config.emplace_back (variable (move (tf.value)));
+              }
+              catch (const invalid_variable& e)
+              {
+                bad_value (string ("invalid task configuration: ") + e.what (),
+                           tf.column - 1 + e.pos,
+                           tl.line - 1);
+              }
+            }
+          }
+        }
+        catch (const tab_parsing& e)
+        {
+          bad_value ("invalid task configuration: " + e.description,
+                     e.column - 1,
+                     e.line - 1);
+        }
+
+        if (config.empty ())
+          bad_value ("empty task configuration");
+      }
+      else if (!iu)
+        bad_name ("unknown name '" + n + "' in task manifest");
+    }
+
+    // Verify all non-optional values were specified.
+    //
+    if (name.empty ())
+      bad_value ("no task package name specified");
+
+    if (version.empty ())
+      bad_value ("no task package version specified");
+
+    if (repository.empty ())
+      bad_value ("no task repository specified");
+
+    if (machine.empty ())
+      bad_value ("no task machine specified");
+  }
+
+  void task_manifest::
+  serialize (serializer& s) const
+  {
+    // @@ Should we check that all non-optional values are specified and all
+    //    values are valid?
+    //
+    s.next ("", "1"); // Start of manifest.
+    s.next ("name", name);
+    s.next ("version", version.string ());
+    s.next ("repository", repository.string ());
+    s.next ("machine", machine);
+
+    if (target)
+      s.next ("target", target->string ());
+
+    // Recompose config string as a space-separated variable list,
+    //
+    if (!config.empty ())
+    {
+      string v;
+      for (auto b (config.cbegin ()), i (b), e (config.cend ()); i != e; ++i)
+      {
+        if (i != b)
+          v += ' ';
+
+        v += *i;
+      }
+
+      s.next ("config", v);
+    }
+
+    s.next ("", ""); // End of manifest.
+  }
+
+  // task_response_manifest
+  //
+  task_response_manifest::
+  task_response_manifest (parser& p, bool iu)
+  {
+    name_value nv (p.next ());
+
+    auto bad_name = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.name_line, nv.name_column, d);
+    };
+
+    auto bad_value = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.value_line, nv.value_column, d);
+    };
+
+    // Make sure this is the start and we support the version.
+    //
+    if (!nv.name.empty ())
+      bad_name ("start of task response manifest expected");
+
+    if (nv.value != "1")
+      bad_value ("unsupported format version");
+
+    // Parse the task response manifest.
+    //
+    // Note that we need to distinguish an empty and absent session.
+    //
+    optional<string> sess;
+    for (nv = p.next (); !nv.empty (); nv = p.next ())
+    {
+      string& n (nv.name);
+      string& v (nv.value);
+
+      if (n == "session")
+      {
+        if (sess)
+          bad_name ("task response session redefinition");
+
+        sess = move (v);
+      }
+      else if (n == "challenge")
+      {
+        if (challenge)
+          bad_name ("task response challenge redefinition");
+
+        if (v.empty ())
+          bad_value ("empty task response challenge");
+
+        challenge = move (v);
+      }
+      else if (!iu)
+        bad_name ("unknown name '" + n + "' in task response manifest");
+    }
+
+    // Verify all non-optional values were specified, and all values are
+    // expected.
+    //
+    if (!sess)
+      bad_value ("no task response session specified");
+
+    session = move (*sess);
+
+    // If session is not empty then the challenge must present, otherwise it
+    // shouldn't.
+    //
+    if (!session.empty ())
+    {
+      if (!challenge)
+        bad_value ("no task response challenge specified");
+    }
+    else if (challenge)
+      bad_value ("unexpected task response challenge");
+
+    // If session is not empty then the task manifest must follow, otherwise it
+    // shouldn't.
+    //
+    nv = p.next ();
+
+    if (!session.empty ())
+    {
+      if (nv.empty ())
+        bad_value ("task manifest expected");
+
+      task = task_manifest (p, nv, iu);
+
+      nv = p.next ();
+    }
+
+    // Make sure this is the end.
+    //
+    if (!nv.empty ())
+      throw parsing (p.name (), nv.name_line, nv.name_column,
+                     "single task response manifest expected");
+  }
+
+  void task_response_manifest::
+  serialize (serializer& s) const
+  {
+    // @@ Should we check that all non-optional values are specified and all
+    //    values are valid?
+    //
+    s.next ("", "1"); // Start of manifest.
+    s.next ("session", session);
+
+    if (challenge)
+      s.next ("challenge", *challenge);
+
+    s.next ("", ""); // End of manifest.
+
+    if (task)
+      task->serialize (s);
+
+    s.next ("", ""); // End of stream.
+  }
+
+  // result_manifest
+  //
+  result_manifest::
+  result_manifest (parser& p, bool iu)
+      : result_manifest (p, p.next (), iu)
+  {
+    // Make sure this is the end.
+    //
+    name_value nv (p.next ());
+    if (!nv.empty ())
+      throw parsing (p.name (), nv.name_line, nv.name_column,
+                     "single result manifest expected");
+  }
+
+  result_manifest::
+  result_manifest (parser& p, name_value nv, bool iu)
+  {
+    auto bad_name = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.name_line, nv.name_column, d);
+    };
+
+    auto bad_value = [&p, &nv] (const string& d, size_t offset = 0)
+    {
+      throw parsing (p.name (), nv.value_line, nv.value_column + offset, d);
+    };
+
+    auto result_stat =
+      [&bad_value] (const string& v, const string& what) -> result_status
+    {
+      auto b (result_status_names.cbegin ());
+      auto e (result_status_names.cend ());
+      auto i (find (b, e, v));
+
+      if (i == e)
+        bad_value ("invalid " + what);
+
+      return static_cast<result_status> (i - b);
+    };
+
+    // Make sure this is the start and we support the version.
+    //
+    if (!nv.name.empty ())
+      bad_name ("start of result manifest expected");
+
+    if (nv.value != "1")
+      bad_value ("unsupported format version");
+
+    // Parse the result manifest.
+    //
+    optional<result_status> stat;
+
+    // Number of parsed *-log values. Also denotes the next expected log type.
+    //
+    size_t nlog (0);
+
+    for (nv = p.next (); !nv.empty (); nv = p.next ())
+    {
+      string& n (nv.name);
+      string& v (nv.value);
+
+      if (n == "name")
+      {
+        if (!name.empty ())
+          bad_name ("result package name redefinition");
+
+        if (v.empty ())
+          bad_value ("empty result package name");
+
+        name = move (v);
+      }
+      else if (n == "version")
+      {
+        if (!version.empty ())
+          bad_name ("result package version redefinition");
+
+        try
+        {
+          version = bpkg::version (move (v));
+        }
+        catch (const invalid_argument& e)
+        {
+          bad_value (string ("invalid result package version: ") + e.what ());
+        }
+
+        // Versions like 1.2.3- are forbidden in manifest as intended to be
+        // used for version constrains rather than actual releases.
+        //
+        if (version.release && version.release->empty ())
+          bad_value ("invalid result package version release");
+      }
+      else if (n == "status")
+      {
+        if (stat)
+          bad_name ("result status redefinition");
+
+        stat = result_stat (v, "result status");
+      }
+      else
+      {
+        size_t nn (n.size ()); // Name length.
+
+        // Note: returns false if nothing preceeds a suffix.
+        //
+        auto suffix = [&n, nn] (const char* s, size_t sn) -> bool
+        {
+          return nn > sn && n.compare (nn - sn, sn, s) == 0;
+        };
+
+        size_t sn;
+        if (suffix ("-status", sn = 7))
+        {
+          if (!stat)
+            bad_name ("result status must appear first");
+
+          if (nlog > 0) // Some logs have already been parsed.
+            bad_name (n + " after operations logs");
+
+          string op (n, 0, nn - sn);
+
+          // Make sure the operation result status is not redefined.
+          //
+          for (const auto& r: results)
+          {
+            if (r.operation == op)
+              bad_name ("result " + n + " redefinition");
+          }
+
+          // Add the operation result (log will come later).
+          //
+          results.push_back ({move (op), result_stat (v, n), string ()});
+        }
+        else if (suffix ("-log", sn = 4))
+        {
+          string op (n, 0, nn - sn);
+
+          // Check that specifically this operation log is expected.
+          //
+          if (nlog >= results.size ())
+            bad_name ("unexpected " + n);
+
+          if (results[nlog].operation != op)
+            bad_name (results[nlog].operation + "-log is expected");
+
+          // Save operation log.
+          //
+          results[nlog++].log = move (v);
+        }
+        else if (!iu)
+          bad_name ("unknown name '" + n + "' in result manifest");
+      }
+    }
+
+    // Verify all non-optional values were specified.
+    //
+    if (name.empty ())
+      bad_value ("no result package name specified");
+
+    if (version.empty ())
+      bad_value ("no result package version specified");
+
+    if (!stat)
+      bad_value ("no result status specified");
+
+    // @@ Checking that the result status is consistent with operations
+    //    statuses is a bit hairy, so let's postpone for now.
+    //
+    status = move (*stat);
+
+    // Check that we have log for every operation result status.
+    //
+    if (nlog < results.size ())
+      bad_name ("no result " + results[nlog].operation + "-log specified");
+  }
+
+  void result_manifest::
+  serialize (serializer& s) const
+  {
+    // @@ Should we check that all non-optional values are specified and all
+    //    values are valid?
+    //
+    s.next ("", "1"); // Start of manifest.
+    s.next ("name", name);
+    s.next ("version", version.string ());
+
+    size_t v (static_cast<size_t> (status));
+    assert (v < result_status_names.size ());
+    s.next ("status", result_status_names[v]);
+
+    // Serialize *-status values.
+    //
+    for (const auto& r: results)
+    {
+      size_t rs (static_cast<size_t> (r.status));
+      assert (rs < result_status_names.size ());
+
+      s.next (r.operation + "-status", result_status_names[rs]);
+    }
+
+    // Serialize *-log values.
+    //
+    for (const auto& r: results)
+      s.next (r.operation + "-log", r.log);
+
+    s.next ("", ""); // End of manifest.
+  }
+
+  // result_request_manifest
+  //
+  result_request_manifest::
+  result_request_manifest (parser& p, bool iu)
+  {
+    name_value nv (p.next ());
+
+    auto bad_name = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.name_line, nv.name_column, d);
+    };
+
+    auto bad_value = [&p, &nv] (const string& d)
+    {
+      throw parsing (p.name (), nv.value_line, nv.value_column, d);
+    };
+
+    // Make sure this is the start and we support the version.
+    //
+    if (!nv.name.empty ())
+      bad_name ("start of result request manifest expected");
+
+    if (nv.value != "1")
+      bad_value ("unsupported format version");
+
+    // Parse the result request manifest.
+    //
+    for (nv = p.next (); !nv.empty (); nv = p.next ())
+    {
+      string& n (nv.name);
+      string& v (nv.value);
+
+      if (n == "session")
+      {
+        if (!session.empty ())
+          bad_name ("result request session redefinition");
+
+        if (v.empty ())
+          bad_value ("empty result request session");
+
+        session = move (v);
+      }
+      else if (n == "challenge")
+      {
+        if (!challenge.empty ())
+          bad_name ("result request challenge redefinition");
+
+        if (v.empty ())
+          bad_value ("empty result request challenge");
+
+        challenge = move (v);
+      }
+      else if (!iu)
+        bad_name ("unknown name '" + n + "' in result request manifest");
+    }
+
+    // Verify all non-optional values were specified.
+    //
+    if (session.empty ())
+      bad_value ("no result request session specified");
+
+    if (challenge.empty ())
+      bad_value ("no result request challenge specified");
+
+    nv = p.next ();
+    if (nv.empty ())
+      bad_value ("result manifest expected");
+
+    result = result_manifest (p, nv, iu);
+
+    // Make sure this is the end.
+    //
+    nv = p.next ();
+    if (!nv.empty ())
+      throw parsing (p.name (), nv.name_line, nv.name_column,
+                     "single result request manifest expected");
+  }
+
+  void result_request_manifest::
+  serialize (serializer& s) const
+  {
+    // @@ Should we check that all non-optional values are specified and all
+    //    values are valid?
+    //
+    s.next ("", "1"); // Start of manifest.
+    s.next ("session", session);
+    s.next ("challenge", challenge);
+    s.next ("", ""); // End of manifest.
+
+    result.serialize (s);
+    s.next ("", ""); // End of stream.
+  }
+}