From ed7134d6ab78e6a608d065846eb153a0580e3443 Mon Sep 17 00:00:00 2001
From: Boris Kolpackov <boris@codesynthesis.com>
Date: Thu, 9 Sep 2021 08:43:34 +0200
Subject: Compress result manifest uploaded by worker

These files contain build logs and can be quite large. And large files
sometimes trip up upload on Windows.
---
 bbot/agent/agent.cxx   |  4 ++--
 bbot/types.hxx         | 13 +++++++++++++
 bbot/utility.hxx       |  8 ++------
 bbot/utility.txx       | 21 +++++++++++++++++++--
 bbot/worker/worker.cxx | 16 ++++++++++------
 5 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx
index 31becd3..60f7271 100644
--- a/bbot/agent/agent.cxx
+++ b/bbot/agent/agent.cxx
@@ -836,8 +836,8 @@ try
   try_mkdir_p (gd);
   try_mkdir_p (pd);
 
-  path tf (gd / "task.manifest");   // Task manifest file.
-  path rf (pd / "result.manifest"); // Result manifest file.
+  path tf (gd / "task.manifest");       // Task manifest file.
+  path rf (pd / "result.manifest.lz4"); // Result manifest file.
 
   serialize_manifest (tm, tf, "task");
 
diff --git a/bbot/types.hxx b/bbot/types.hxx
index 7e485f7..a39abe2 100644
--- a/bbot/types.hxx
+++ b/bbot/types.hxx
@@ -21,6 +21,8 @@
 
 #include <libbutl/path.mxx>
 #include <libbutl/optional.mxx>
+#include <libbutl/fdstream.mxx>
+#include <libbutl/lz4-stream.hxx>
 #include <libbutl/vector-view.mxx>
 #include <libbutl/small-vector.mxx>
 #include <libbutl/standard-version.mxx>
@@ -69,6 +71,17 @@ namespace bbot
   using butl::optional;
   using butl::nullopt;
 
+  // <libbutl/fdstream.mxx>
+  //
+  using butl::auto_fd;
+  using butl::ifdstream;
+  using butl::ofdstream;
+
+  // <libbutl/lz4-stream.hxx>
+  //
+  using olz4stream = butl::lz4::ostream;
+  using ilz4stream = butl::lz4::istream;
+
   // <libbutl/vector-view.mxx>
   //
   using butl::vector_view;
diff --git a/bbot/utility.hxx b/bbot/utility.hxx
index 4545bd3..b93f8b6 100644
--- a/bbot/utility.hxx
+++ b/bbot/utility.hxx
@@ -16,7 +16,6 @@
 #include <libbutl/process.mxx>
 #include <libbutl/process-io.mxx>
 #include <libbutl/utility.mxx>    // icasecmp(), reverse_iterate(), etc
-#include <libbutl/fdstream.mxx>
 #include <libbutl/filesystem.mxx>
 
 #include <bbot/types.hxx>
@@ -46,10 +45,6 @@ namespace bbot
   using butl::setenv;
   using butl::unsetenv;
 
-  // <libbutl/fdstream.mxx>
-  //
-  using butl::auto_fd;
-
   // <libbutl/filesystem.mxx>
   //
   using butl::auto_rmdir;
@@ -132,7 +127,8 @@ namespace bbot
 
   // Manifest parsing and serialization.
   //
-  // For parsing, if path is '-', then read from stdin.
+  // For parsing, if path is '-', then read from stdin. If path has the .lz4
+  // extension, then assume the content is compressed.
   //
   template <typename T>
   T
diff --git a/bbot/utility.txx b/bbot/utility.txx
index e485a80..3199a61 100644
--- a/bbot/utility.txx
+++ b/bbot/utility.txx
@@ -150,8 +150,25 @@ namespace bbot
       if (!file_exists (f))
         fail (fh) << what << " manifest file " << f << " does not exist";
 
-      ifdstream ifs (f);
-      return parse_manifest<T> (ifs, f.string (), what, fh, iu);
+      bool d (f.extension () == "lz4");
+
+      ifdstream ifs (f, (d
+                         ? ifdstream::badbit
+                         : ifdstream::badbit | ifdstream::failbit));
+      ilz4stream izs;
+
+      if (d)
+        izs.open (ifs, true /* end */);
+
+      return parse_manifest<T> (d
+                                ? static_cast<istream&> (izs)
+                                : static_cast<istream&> (ifs),
+                                f.string (), what, fh, iu);
+    }
+    catch (const invalid_argument& e) // Invalid compressed content.
+    {
+      fail (fh) << "invalid " << what << " manifest " << f << ": " << e
+                << endf;
     }
     catch (const system_error& e) // EACCES, etc.
     {
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index 1f5c946..f9081a2 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -547,9 +547,9 @@ run_b (step_id step,
                 verbosity, buildspec, forward<A> (a)...);
 }
 
-// Upload manifest to the specified TFTP URL with curl. Issue diagnostics and
-// throw failed on invalid manifest or process management errors and throw
-// io_error for input/output errors or non-zero curl exit.
+// Upload compressed manifest to the specified TFTP URL with curl. Issue
+// diagnostics and throw failed on invalid manifest or process management
+// errors and throw io_error for input/output errors or non-zero curl exit.
 //
 template <typename T>
 static void
@@ -564,9 +564,11 @@ upload_manifest (tracer& trace,
     // broken pipe error on the client and partial/truncated upload on the
     // server. This happens quite regularly on older Linux distributions
     // (e.g., Debian 8, Ubuntu 16.04) but also sometimes on Windows. On the
-    // other hand, uploading from a file appears to work reliably.
+    // other hand, uploading from a file appears to work reliably (we still
+    // get an odd error on Windows from time to time with larger uploads).
     //
 #if 0
+    // Note: need to add compression support if re-enable this.
     tftp_curl c (trace,
                  path ("-"),
                  nullfd,
@@ -582,10 +584,12 @@ upload_manifest (tracer& trace,
     auto_rmfile tmp;
     try
     {
-      tmp = auto_rmfile (path::temp_path (what + "-manifest"));
+      tmp = auto_rmfile (path::temp_path (what + "-manifest.lz4"));
       ofdstream ofs (tmp.path);
-      manifest_serializer s (ofs, tmp.path.string ());
+      olz4stream ozs (ofs, 5 /* 256KB */, 9, nullopt /* content_size */);
+      manifest_serializer s (ozs, tmp.path.string ());
       m.serialize (s);
+      ozs.close ();
       ofs.close ();
     }
     catch (const io_error& e) // In case not derived from system_error.
-- 
cgit v1.1