aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2022-02-11 19:44:19 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2022-02-14 10:26:49 +0300
commit422563c63b2aedaa32fa217aca92b369ac4be755 (patch)
treeec2de1c8f5fc14fb839ce0cbc69dcf4b160b7c73
parent30411915355d6946de11638daf98724ab41a9acd (diff)
Fix result manifest request failures due to exceeding request size limit
-rw-r--r--bbot/agent/agent.cxx21
-rw-r--r--bbot/utility.hxx8
-rw-r--r--bbot/utility.txx9
-rw-r--r--bbot/worker/worker.cxx9
4 files changed, 37 insertions, 10 deletions
diff --git a/bbot/agent/agent.cxx b/bbot/agent/agent.cxx
index 982f67c..3cfc4b8 100644
--- a/bbot/agent/agent.cxx
+++ b/bbot/agent/agent.cxx
@@ -1432,7 +1432,11 @@ try
try
{
- serialize_manifest (tq, c.out, u, "task request", false);
+ serialize_manifest (tq,
+ c.out,
+ u,
+ "task request",
+ false /* fail_hard */);
}
catch (const failed&) {f = true;}
@@ -1629,7 +1633,20 @@ try
try
{
- serialize_manifest (rq, c.out, u, "task request");
+ // Don't break lines in the manifest values not to further increase
+ // the size of the result request manifest encoded representation.
+ // Note that this manifest can contain quite a few lines in the
+ // operation logs, potentially truncated to fit the upload limit
+ // (see worker/worker.cxx for details). Breaking these lines can
+ // increase the request size beyond this limit and so we can end up
+ // with the request failure.
+ //
+ serialize_manifest (rq,
+ c.out,
+ u,
+ "result request",
+ true /* fail_hard */,
+ true /* long_lines */);
}
catch (const failed&) {f = true;}
diff --git a/bbot/utility.hxx b/bbot/utility.hxx
index 2bc0320..35136bd 100644
--- a/bbot/utility.hxx
+++ b/bbot/utility.hxx
@@ -147,7 +147,10 @@ namespace bbot
template <typename T>
void
- serialize_manifest (const T&, const path&, const char* what);
+ serialize_manifest (const T&,
+ const path&,
+ const char* what,
+ bool long_lines = false);
template <typename T>
void
@@ -155,7 +158,8 @@ namespace bbot
ostream&,
const string& name,
const char* what,
- bool fail_hard = true);
+ bool fail_hard = true,
+ bool long_lines = false);
}
#include <bbot/utility.txx>
diff --git a/bbot/utility.txx b/bbot/utility.txx
index 398c78d..d2aad10 100644
--- a/bbot/utility.txx
+++ b/bbot/utility.txx
@@ -208,7 +208,7 @@ namespace bbot
template <typename T>
void
- serialize_manifest (const T& m, const path& f, const char* what)
+ serialize_manifest (const T& m, const path& f, const char* what, bool ll)
{
using namespace std;
using namespace butl;
@@ -218,7 +218,7 @@ namespace bbot
ofdstream ofs (f, fdopen_mode::binary);
auto_rmfile arm (f); // Try to remove on failure ignoring errors.
- serialize_manifest (m, ofs, f.string (), what, true);
+ serialize_manifest (m, ofs, f.string (), what, true, ll);
ofs.close ();
arm.cancel ();
@@ -235,13 +235,14 @@ namespace bbot
ostream& os,
const string& name,
const char* what,
- bool fh)
+ bool fh,
+ bool ll)
{
using namespace butl;
try
{
- manifest_serializer s (os, name);
+ manifest_serializer s (os, name, ll);
m.serialize (s);
return;
}
diff --git a/bbot/worker/worker.cxx b/bbot/worker/worker.cxx
index e4b5e5e..18546e4 100644
--- a/bbot/worker/worker.cxx
+++ b/bbot/worker/worker.cxx
@@ -719,6 +719,11 @@ upload_manifest (tracer& trace,
// other hand, uploading from a file appears to work reliably (we still
// get an odd error on Windows from time to time with larger uploads).
//
+ // Let's not break lines in the manifest values not to further increase
+ // the size of the manifest encoded representation. Also here we don't
+ // care much about readability of the manifest since it will only be read
+ // by the bbot agent anyway.
+ //
#if 0
// Note: need to add compression support if re-enable this.
tftp_curl c (trace,
@@ -729,7 +734,7 @@ upload_manifest (tracer& trace,
"--tftp-blksize", tftp_blksize,
"--max-time", tftp_put_timeout);
- manifest_serializer s (c.out, url);
+ manifest_serializer s (c.out, url, true /* long_lines */);
m.serialize (s);
c.out.close ();
#else
@@ -739,7 +744,7 @@ upload_manifest (tracer& trace,
tmp = auto_rmfile (path::temp_path (what + "-manifest.lz4"));
ofdstream ofs (tmp.path, fdopen_mode::binary);
olz4stream ozs (ofs, 9, 5 /* 256KB */, nullopt /* content_size */);
- manifest_serializer s (ozs, tmp.path.string ());
+ manifest_serializer s (ozs, tmp.path.string (), true /* long_lines */);
m.serialize (s);
ozs.close ();
ofs.close ();