Fix result manifest request failures due to exceeding request size limit

4 files changed, 37 insertions, 10 deletions

diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx
index 982f67c..3cfc4b8 100644
--- a/bbot/agent/agent.cxx
+++ b/bbot/agent/agent.cxx
@@ -1432,7 +1432,11 @@ try
 
           try
           {
-            serialize_manifest (tq, c.out, u, "task request", false);
+            serialize_manifest (tq,
+                                c.out,
+                                u,
+                                "task request",
+                                false /* fail_hard */);
           }
           catch (const failed&) {f = true;}
 
@@ -1629,7 +1633,20 @@ try
 
         try
         {
-          serialize_manifest (rq, c.out, u, "task request");
+          // Don't break lines in the manifest values not to further increase
+          // the size of the result request manifest encoded representation.
+          // Note that this manifest can contain quite a few lines in the
+          // operation logs, potentially truncated to fit the upload limit
+          // (see worker/worker.cxx for details). Breaking these lines can
+          // increase the request size beyond this limit and so we can end up
+          // with the request failure.
+          //
+          serialize_manifest (rq,
+                              c.out,
+                              u,
+                              "result request",
+                              true /* fail_hard */,
+                              true /* long_lines */);
         }
         catch (const failed&) {f = true;}
 
diff --git a/bbot/utility.hxx b/bbot/utility.hxx
index 2bc0320..35136bd 100644
--- a/bbot/utility.hxx
+++ b/bbot/utility.hxx
@@ -147,7 +147,10 @@ namespace bbot
 
   template <typename T>
   void
-  serialize_manifest (const T&, const path&, const char* what);
+  serialize_manifest (const T&,
+                      const path&,
+                      const char* what,
+                      bool long_lines = false);
 
   template <typename T>
   void
@@ -155,7 +158,8 @@ namespace bbot
                       ostream&,
                       const string& name,
                       const char* what,
-                      bool fail_hard = true);
+                      bool fail_hard = true,
+                      bool long_lines = false);
 }
 
 #include <bbot/utility.txx>
diff --git a/bbot/utility.txx b/bbot/utility.txx
index 398c78d..d2aad10 100644
--- a/bbot/utility.txx
+++ b/bbot/utility.txx
@@ -208,7 +208,7 @@ namespace bbot
 
   template <typename T>
   void
-  serialize_manifest (const T& m, const path& f, const char* what)
+  serialize_manifest (const T& m, const path& f, const char* what, bool ll)
   {
     using namespace std;
     using namespace butl;
@@ -218,7 +218,7 @@ namespace bbot
       ofdstream ofs (f, fdopen_mode::binary);
       auto_rmfile arm (f); // Try to remove on failure ignoring errors.
 
-      serialize_manifest (m, ofs, f.string (), what, true);
+      serialize_manifest (m, ofs, f.string (), what, true, ll);
 
       ofs.close ();
       arm.cancel ();
@@ -235,13 +235,14 @@ namespace bbot
                       ostream& os,
                       const string& name,
                       const char* what,
-                      bool fh)
+                      bool fh,
+                      bool ll)
   {
     using namespace butl;
 
     try
     {
-      manifest_serializer s (os, name);
+      manifest_serializer s (os, name, ll);
       m.serialize (s);
       return;
     }
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index e4b5e5e..18546e4 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -719,6 +719,11 @@ upload_manifest (tracer& trace,
     // other hand, uploading from a file appears to work reliably (we still
     // get an odd error on Windows from time to time with larger uploads).
     //
+    // Let's not break lines in the manifest values not to further increase
+    // the size of the manifest encoded representation. Also here we don't
+    // care much about readability of the manifest since it will only be read
+    // by the bbot agent anyway.
+    //
 #if 0
     // Note: need to add compression support if re-enable this.
     tftp_curl c (trace,
@@ -729,7 +734,7 @@ upload_manifest (tracer& trace,
                  "--tftp-blksize", tftp_blksize,
                  "--max-time", tftp_put_timeout);
 
-    manifest_serializer s (c.out, url);
+    manifest_serializer s (c.out, url, true /* long_lines */);
     m.serialize (s);
     c.out.close ();
 #else
@@ -739,7 +744,7 @@ upload_manifest (tracer& trace,
       tmp = auto_rmfile (path::temp_path (what + "-manifest.lz4"));
       ofdstream ofs (tmp.path, fdopen_mode::binary);
       olz4stream ozs (ofs, 9, 5 /* 256KB */, nullopt /* content_size */);
-      manifest_serializer s (ozs, tmp.path.string ());
+      manifest_serializer s (ozs, tmp.path.string (), true /* long_lines */);
       m.serialize (s);
       ozs.close ();
       ofs.close ();