aboutsummaryrefslogtreecommitdiff
path: root/libbuild2/script/run.cxx
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2022-11-02 22:22:09 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2022-11-09 21:45:02 +0300
commitfca95dafaaab44d3e06e10bc9f3b45d00c82301f (patch)
tree69f59c57b0aa6100e01264a5d10e6a155ffff79e /libbuild2/script/run.cxx
parentf34dd20fb7578874925dacf21b32338af50e8c12 (diff)
Use diag_buffer in script
Diffstat (limited to 'libbuild2/script/run.cxx')
-rw-r--r--libbuild2/script/run.cxx1269
1 files changed, 844 insertions, 425 deletions
diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx
index 0d9e472..1ce5564 100644
--- a/libbuild2/script/run.cxx
+++ b/libbuild2/script/run.cxx
@@ -972,203 +972,539 @@ namespace build2
: path (c.program.recall_string ());
}
+ // Read the stream content into a string, optionally splitting the input
+ // data at whitespaces or newlines in which case return one, potentially
+ // incomplete, substring at a time (see the set builtin options for the
+ // splitting semantics). Throw io_error on the underlying OS error.
+ //
+ // On POSIX expects the stream to be non-blocking and its exception mask
+ // to have at least badbit. On Windows can also handle a blocking stream.
+ //
+ // Note that on Windows we can only turn pipe file descriptors into the
+ // non-blocking mode. Thus, we have no choice but to read from descriptors
+ // of other types synchronously there. That implies that we can
+ // potentially block indefinitely reading a file and missing a deadline on
+ // Windows. Note though, that the user can normally rewrite the command,
+ // for example, `set foo <<<file` with `cat file | set foo` to avoid this
+ // problem.
+ //
+ class stream_reader
+ {
+ public:
+ stream_reader (ifdstream&, bool whitespace, bool newline, bool exact);
+
+ // Read next substring. Return true if the substring has been read or
+ // false if it should be called again once the stream has more data to
+ // read. Also return true on eof (in which case no substring is read).
+ // The string must be empty on the first call. Throw ios::failure on the
+ // underlying OS error.
+ //
+ // Note that there could still be data to read in the stream's buffer
+ // (as opposed to file descriptor) after this function returns true and
+ // you should be careful not to block on fdselect() in this case. The
+ // recommended usage pattern is similar to that of
+ // butl::getline_non_blocking(). The only difference is that
+ // ifdstream::eof() needs to be used instead of butl::eof() since this
+ // function doesn't set failbit and only sets eofbit after the last
+ // substring is returned.
+ //
+ bool
+ next (string&);
+
+ private:
+ ifdstream& is_;
+ bool whitespace_;
+ bool newline_;
+ bool exact_;
+
+ bool empty_ = true; // Set to false after the first character is read.
+ };
+
stream_reader::
- stream_reader(auto_fd&& in,
-#ifndef _WIN32
- bool,
-#else
- bool pipe,
-#endif
- bool ws, bool nl, bool ex,
- const optional<deadline>& dl,
- const command& dc,
- const location& l)
- : whitespace_ (ws),
+ stream_reader (ifdstream& is, bool ws, bool nl, bool ex)
+ : is_ (is),
+ whitespace_ (ws),
newline_ (nl),
- exact_ (ex),
- deadline_cmd_ (dc),
- location_ (l)
+ exact_ (ex)
+ {
+ }
+
+ bool stream_reader::
+ next (string& ss)
{
#ifndef _WIN32
- if (dl)
+ assert ((is_.exceptions () & ifdstream::badbit) != 0 && !is_.blocking ());
#else
- if (dl && pipe)
+ assert ((is_.exceptions () & ifdstream::badbit) != 0);
#endif
- {
- is_.open (move (in), fdstream_mode::non_blocking);
- deadline_ = dl;
- }
- else
- is_.open (move (in));
- }
- optional<string> stream_reader::
- next ()
- {
- if (!is_.is_open ())
- return nullopt;
+ fdstreambuf& sb (*static_cast<fdstreambuf*> (is_.rdbuf ()));
- // If eos is not reached, then read and return a character. Otherwise
- // close the stream and return nullopt. If the deadline is specified and
- // is reached, then return nullopt for the successful deadline (as if
- // eof is reached) and fail otherwise.
+ // Return the number of characters available in the stream buffer's get
+ // area, which can be:
//
- // Set the empty_ flag to false after the first character is read.
+ // -1 -- EOF.
+ // 0 -- no data since blocked before encountering more data/EOF.
+ // >0 -- there is some data.
//
- // @@ PERF: reading one character at a time is not ideal.
+ // Note that on Windows if the stream is blocking, then the lambda calls
+ // underflow() instead of returning 0.
//
- auto get = [this] () -> optional<char>
+ // @@ Probably we can call underflow() only once per the next() call,
+ // emulating the 'no data' case. This will allow the caller to
+ // perform some housekeeping (reading other streams, checking for the
+ // deadline, etc). But let's keep it simple for now.
+ //
+ auto avail = [&sb] () -> streamsize
{
- char r;
+ // Note that here we reasonably assume that any failure in in_avail()
+ // will lead to badbit and thus an exception (see showmanyc()).
+ //
+ streamsize r (sb.in_avail ());
- if (deadline_) // Reading a character in the non-blocking mode.
+#ifdef _WIN32
+ if (r == 0 && sb.blocking ())
{
- fdselect_set fds {is_.fd ()};
+ if (sb.underflow () == ifdstream::traits_type::eof ())
+ return -1;
- // Only fallback to ifdselect() if there is no character immediately
- // available.
+ r = sb.in_avail ();
+
+ assert (r != 0); // We wouldn't be here otherwise.
+ }
+#endif
+
+ return r;
+ };
+
+ // Read until blocked (0), EOF (-1) or encounter the delimiter.
+ //
+ streamsize s;
+ while ((s = avail ()) > 0)
+ {
+ if (empty_)
+ empty_ = false;
+
+ const char* p (sb.gptr ());
+ size_t n (sb.egptr () - p);
+
+ // We move p and bump by the number of consumed characters.
+ //
+ auto bump = [&sb, &p] () {sb.gbump (static_cast<int> (p - sb.gptr ()));};
+
+ if (whitespace_) // The whitespace mode.
+ {
+ const char* sep (" \n\r\t");
+
+ // Skip the whitespaces.
//
- for (;;)
- {
- streamsize n (is_.readsome (&r, 1));
+ for (; n != 0 && strchr (sep, *p) != nullptr; ++p, --n) ;
- if (n == 1)
- break;
+ // If there are any non-whitespace characters in the get area, then
+ // append them to the resulting substring until a whitespace
+ // character is encountered.
+ //
+ if (n != 0)
+ {
+ // Append the non-whitespace characters.
+ //
+ for (char c; n != 0 && strchr (sep, c = *p) == nullptr; ++p, --n)
+ ss += c;
- if (is_.eof ())
+ // If a separator is encountered, then consume it, bump, and
+ // return the substring.
+ //
+ if (n != 0)
{
- is_.close ();
- return nullopt;
+ ++p; --n; // Consume the separator character.
+
+ bump ();
+ return true;
}
- const timestamp& dlt (deadline_->value);
- timestamp now (system_clock::now ());
+ // Fall through.
+ }
- if (dlt <= now || ifdselect (fds, dlt - now) == 0)
+ bump (); // Bump and continue reading.
+ }
+ else // The newline or no-split mode.
+ {
+ // Note that we don't collapse multiple consecutive newlines.
+ //
+ // Note also that we always sanitize CRs, so in the no-split mode we
+ // need to loop rather than consume the whole get area at once.
+ //
+ while (n != 0)
+ {
+ // Append the characters until the newline character or the end of
+ // the get area is encountered.
+ //
+ char c;
+ for (; n != 0 && (c = *p) != '\n'; ++p, --n)
+ ss += c;
+
+ // If the newline character is encountered, then sanitize CRs and
+ // return the substring in the newline mode and continue
+ // parsing/reading otherwise.
+ //
+ if (n != 0)
{
- is_.close ();
+ // Strip the trailing CRs that can appear while, for example,
+ // cross-testing Windows target or as a part of msvcrt junk
+ // production (see above).
+ //
+ while (!ss.empty () && ss.back () == '\r')
+ ss.pop_back ();
- if (!deadline_->success)
- fail (location_) << cmd_path (deadline_cmd_)
- << " terminated: execution timeout expired";
- else
- return nullopt;
+ assert (c == '\n');
+
+ ++p; --n; // Consume the newline character.
+
+ if (newline_)
+ {
+ bump ();
+ return true;
+ }
+
+ ss += c; // Append newline to the resulting string.
+
+ // Fall through.
}
+
+ bump (); // Bump and continue parsing/reading.
}
}
- else // Reading a character in the blocking mode.
+ }
+
+ // Here s can be:
+ //
+ // -1 -- EOF.
+ // 0 -- blocked before encountering delimiter/EOF.
+ //
+ // Note: >0 (encountered the delimiter) case is handled in-place.
+ //
+ assert (s == -1 || s == 0);
+
+ if (s == -1)
+ {
+ // Return the last substring if it is not empty or it is the trailing
+ // "blank" in the exact mode. Otherwise, set eofbit for the stream
+ // indicating that we are done.
+ //
+ if (!ss.empty () || (exact_ && !empty_))
{
- if (is_.peek () == ifdstream::traits_type::eof ())
+ // Also, strip the trailing newline character, if present, in the
+ // no-split no-exact mode.
+ //
+ if (!ss.empty () && ss.back () == '\n' && // Trailing newline.
+ !newline_ && !whitespace_ && !exact_) // No-split no-exact mode.
{
- is_.close ();
- return nullopt;
+ ss.pop_back ();
}
- is_.get (r);
+ exact_ = false; // Make sure we will set eofbit on the next call.
}
+ else
+ is_.setstate (ifdstream::eofbit);
+ }
- empty_ = false;
- return r;
- };
+ return s == -1;
+ }
- if (whitespace_) // The whitespace mode.
- {
- const char* sep (" \n\r\t");
+ // Stack-allocated linked list of information about the running pipeline
+ // processes and builtins.
+ //
+ // Note: constructed incrementally.
+ //
+ struct pipe_command
+ {
+ // Initially NULL. Set to the address of the process or builtin object
+ // when it is created. Reset back to NULL when the respective
+ // process/builtin is executed and its exit status is collected (see
+ // complete_pipe() for details).
+ //
+ // We could probably use a union here, but let's keep it simple for now
+ // (at least one is NULL).
+ //
+ process* proc = nullptr;
+ builtin* bltn = nullptr;
- // Note that we collapse multiple consecutive whitespaces.
- //
- optional<char> c;
+ const command& cmd;
+ const cstrings* args = nullptr;
+ const optional<deadline>& dl;
- // Skip the whitespaces.
- //
- while ((c = get ()) && strchr (sep, *c) != nullptr) ;
+ diag_buffer dbuf;
- // Bail out for the trailing whitespace(s) or an empty stream.
- //
- if (!c)
+ bool terminated = false; // True if this command has been terminated.
+
+ // Only for diagnostics.
+ //
+ const location& loc;
+ const path* isp = nullptr; // stdin cache.
+ const path* osp = nullptr; // stdout cache.
+ const path* esp = nullptr; // stderr cache.
+
+ pipe_command* prev; // NULL for the left-most command.
+ pipe_command* next; // Left-most command for the right-most command.
+
+ pipe_command (context& x,
+ const command& c,
+ const optional<deadline>& d,
+ const location& l,
+ pipe_command* p,
+ pipe_command* f)
+ : cmd (c), dl (d), dbuf (x), loc (l), prev (p), next (f) {}
+ };
+
+ // Wait for a process/builtin to complete until the deadline is reached
+ // and return the underlying wait function result (optional<something>).
+ //
+ template<typename P>
+ static auto
+ timed_wait (P& p, const timestamp& deadline) -> decltype(p.try_wait ())
+ {
+ timestamp now (system_clock::now ());
+ return deadline > now ? p.timed_wait (deadline - now) : p.try_wait ();
+ }
+
+ // Terminate the pipeline processes starting from the specified one and up
+ // to the leftmost one and then kill those which didn't terminate after 2
+ // seconds.
+ //
+ // After that wait for the pipeline builtins completion. Since their
+ // standard streams should no longer be written to or read from by any
+ // process, that shouldn't take long. If, however, they won't be able to
+ // complete in 2 seconds, then some of them have probably stuck while
+ // communicating with a slow filesystem device or similar, and since we
+ // currently have no way to terminate asynchronous builtins, we have no
+ // choice but to abort.
+ //
+ // Issue diagnostics and fail if something goes wrong, but still try to
+ // terminate/kill all the pipe processes.
+ //
+ static void
+ term_pipe (pipe_command* pc, tracer& trace)
+ {
+ auto prog = [] (pipe_command* c) {return cmd_path (c->cmd);};
+
+ // Terminate processes gracefully and set the terminate flag for the
+ // pipe commands.
+ //
+ diag_record dr;
+ for (pipe_command* c (pc); c != nullptr; c = c->prev)
+ {
+ if (process* p = c->proc)
+ try
+ {
+ l5 ([&]{trace (c->loc) << "terminating: " << c->cmd;});
+
+ p->term ();
+ }
+ catch (const process_error& e)
{
- // Return the trailing "blank" after the trailing whitespaces in the
- // exact mode, unless the stream is empty.
+ // If unable to terminate the process for any reason (the process is
+ // exiting on Windows, etc) then just ignore this, postponing the
+ // potential failure till the kill() call.
//
- return exact_ && !empty_ ? empty_string : optional<string> ();
+ l5 ([&]{trace (c->loc) << "unable to terminate " << prog (c)
+ << ": " << e;});
}
- // Read the word until eof or a whitespace character is encountered.
- //
- string r (1, *c);
- while ((c = get ()) && strchr (sep, *c) == nullptr)
- r += *c;
+ c->terminated = true;
+ }
+
+ // Wait a bit for the processes to terminate and kill the remaining
+ // ones.
+ //
+ timestamp dl (system_clock::now () + chrono::seconds (2));
- return optional<string> (move (r));
+ for (pipe_command* c (pc); c != nullptr; c = c->prev)
+ {
+ if (process* p = c->proc)
+ try
+ {
+ l5 ([&]{trace (c->loc) << "waiting: " << c->cmd;});
+
+ if (!timed_wait (*p, dl))
+ {
+ l5 ([&]{trace (c->loc) << "killing: " << c->cmd;});
+
+ p->kill ();
+ p->wait ();
+ }
+ }
+ catch (const process_error& e)
+ {
+ dr << fail (c->loc) << "unable to wait/kill " << prog (c) << ": "
+ << e;
+ }
}
- else // The newline or no-split mode.
+
+ // Wait a bit for the builtins to complete and abort if any remain
+ // running.
+ //
+ dl = system_clock::now () + chrono::seconds (2);
+
+ for (pipe_command* c (pc); c != nullptr; c = c->prev)
{
- // Note that we don't collapse multiple consecutive newlines.
- //
- // Note also that we always sanitize CRs, so in the no-split mode we
- // need to loop rather than read the whole text at once.
- //
- optional<string> r;
+ if (builtin* b = c->bltn)
+ try
+ {
+ l5 ([&]{trace (c->loc) << "waiting: " << c->cmd;});
- do
+ if (!timed_wait (*b, dl))
+ {
+ error (c->loc) << prog (c) << " builtin hanged, aborting";
+ terminate (false /* trace */);
+ }
+ }
+ catch (const system_error& e)
{
- string l;
- optional<char> c;
+ dr << fail (c->loc) << "unable to wait for " << prog (c) << ": "
+ << e;
+ }
+ }
+ }
- // Read the line until eof or newline character is encountered.
- //
- while ((c = get ()) && *c != '\n')
- l += *c;
+ void
+ read (auto_fd&& in,
+ bool whitespace, bool newline, bool exact,
+ const function<void (string&&)>& cf,
+ pipe_command* pipeline,
+ const optional<deadline>& dl,
+ const location& ll,
+ const char* what)
+ {
+ tracer trace ("script::stream_read");
- // Strip the trailing CRs that can appear while, for example,
- // cross-testing Windows target or as a part of msvcrt junk
- // production (see above).
- //
- while (!l.empty () && l.back () == '\r')
- l.pop_back ();
+ // Note: stays blocking on Windows if the descriptor is not of the pipe
+ // type.
+ //
+#ifndef _WIN32
+ fdstream_mode m (fdstream_mode::non_blocking);
+#else
+ fdstream_mode m (pipeline != nullptr
+ ? fdstream_mode::non_blocking
+ : fdstream_mode::blocking);
+#endif
+
+ ifdstream is (move (in), m, ifdstream::badbit);
+ stream_reader sr (is, whitespace, newline, exact);
- // Append the line.
+ fdselect_set fds;
+ for (pipe_command* c (pipeline); c != nullptr; c = c->prev)
+ {
+ diag_buffer& b (c->dbuf);
+
+ if (b.is.is_open ())
+ fds.emplace_back (b.is.fd (), &b);
+ }
+
+ fds.emplace_back (is.fd ());
+ fdselect_state& ist (fds.back ());
+
+ optional<timestamp> dlt (dl ? dl->value : optional<timestamp> ());
+
+ // If there are some left-hand side processes/builtins running, then
+ // terminate them and reset the deadline to nullopt. Otherwise, fail
+ // straigh away.
+ //
+ // Note that in the former case the further reading will be performed
+ // without timeout. This, however, is fine since all the processes and
+ // builtins are terminated and we only need to read out the buffered
+ // data.
+ //
+ auto term = [&dlt, pipeline, &trace, &ll, what] ()
+ {
+ if (pipeline != nullptr)
+ {
+ term_pipe (pipeline, trace);
+ dlt = nullopt;
+ }
+ else
+ fail (ll) << what << " terminated: execution timeout expired";
+ };
+
+ // Note that on Windows if the file descriptor is not a pipe, then
+ // ifdstream assumes the blocking mode for which ifdselect() would throw
+ // invalid_argument. Such a descriptor can, however, only appear for the
+ // first command in the pipeline and so fds will only contain the input
+ // stream's descriptor. That all means that this descriptor will be read
+ // out by a series of the stream_reader::next() calls which can only
+ // return true and thus no ifdselect() calls will ever be made.
+ //
+ string s;
+ for (size_t unread (fds.size ()); unread != 0;)
+ {
+ // Read any pending data from the input stream.
+ //
+ if (ist.fd != nullfd)
+ {
+ // Prior to reading let's check that the deadline, if specified, is
+ // not reached. This way we handle the (hypothetical) case when we
+ // are continuously fed with the data without delays and thus can
+ // never get to ifdselect() which watches for the deadline. Also
+ // this check is the only way to bail out early on Windows for a
+ // blocking file descriptor.
//
- if (!l.empty () || // Non-empty.
- c || // Empty, non-trailing.
- (exact_ && // Empty, trailing, in the exact mode for
- !empty_)) // non-empty stream.
+ if (dlt && *dlt <= system_clock::now ())
+ term ();
+
+ if (sr.next (s))
{
- if (newline_ || !r)
+ if (!is.eof ())
{
- r = move (l);
+ // Consume the substring.
+ //
+ cf (move (s));
+ s.clear ();
}
else
{
- *r += '\n';
- *r += l;
+ ist.fd = nullfd;
+ --unread;
}
+
+ continue;
}
}
- while (!newline_ && is_.is_open ());
- return r;
- }
- }
+ try
+ {
+ // Wait until the data appear in any of the streams. If a deadline
+ // is specified, then pass the timeout to fdselect().
+ //
+ if (dlt)
+ {
+ timestamp now (system_clock::now ());
- string
- stream_read (auto_fd&& in,
- bool pipe,
- const optional<deadline>& dl,
- const command& dc,
- const location& ll)
- {
- stream_reader sr (move (in),
- pipe,
- false /* whitespace */,
- false /* newline */,
- true /* exact */,
- dl,
- dc,
- ll);
-
- optional<string> s (sr.next ());
- return s ? move (*s) : empty_string;
+ if (*dlt <= now || ifdselect (fds, *dlt - now) == 0)
+ {
+ term ();
+ continue;
+ }
+ }
+ else
+ ifdselect (fds);
+
+ // Read out the pending data from the stderr streams.
+ //
+ for (fdselect_state& s: fds)
+ {
+ if (s.ready &&
+ s.data != nullptr &&
+ !static_cast<diag_buffer*> (s.data)->read ())
+ {
+ s.fd = nullfd;
+ --unread;
+ }
+ }
+ }
+ catch (const io_error& e)
+ {
+ fail (ll) << "io error reading pipeline streams: " << e;
+ }
+ }
}
// The set pseudo-builtin: set variable from the stdin input.
@@ -1179,11 +1515,12 @@ namespace build2
set_builtin (environment& env,
const strings& args,
auto_fd in,
- bool pipe,
+ pipe_command* pipeline,
const optional<deadline>& dl,
- const command& deadline_cmd,
const location& ll)
{
+ tracer trace ("script::set_builtin");
+
try
{
// Parse arguments.
@@ -1219,17 +1556,17 @@ namespace build2
fail (ll) << "set: unexpected argument '" << scan.next () << "'";
}
- stream_reader sr (move (in), pipe,
- ops.whitespace (), ops.newline (), ops.exact (),
- dl, deadline_cmd,
- ll);
-
// Parse the stream content into the variable value.
//
names ns;
- for (optional<string> s; (s = sr.next ()); )
- ns.emplace_back (move (*s));
+ read (move (in),
+ ops.whitespace (), ops.newline (), ops.exact (),
+ [&ns] (string&& s) {ns.emplace_back (move (s));},
+ pipeline,
+ dl,
+ ll,
+ "set");
env.set_variable (move (vname), move (ns), attrs, ll);
}
@@ -1258,41 +1595,6 @@ namespace build2
name);
}
- // Stack-allocated linked list of information about the running pipeline
- // processes and builtins.
- //
- struct pipe_command
- {
- // We could probably use a union here, but let's keep it simple for now
- // (one is NULL).
- //
- process* proc;
- builtin* bltn;
-
- // True if this command has been terminated.
- //
- bool terminated = false;
-
- // Only for diagnostics.
- //
- const command& cmd;
- const location& loc;
-
- pipe_command* prev; // NULL for the left-most command.
-
- pipe_command (process& p,
- const command& c,
- const location& l,
- pipe_command* v)
- : proc (&p), bltn (nullptr), cmd (c), loc (l), prev (v) {}
-
- pipe_command (builtin& b,
- const command& c,
- const location& l,
- pipe_command* v)
- : proc (nullptr), bltn (&b), cmd (c), loc (l), prev (v) {}
- };
-
static bool
run_pipe (environment& env,
command_pipe::const_iterator bc,
@@ -1303,7 +1605,6 @@ namespace build2
bool diag,
const function<command_function>& cf, bool last_cmd,
optional<deadline> dl = nullopt,
- const command* dl_cmd = nullptr, // env -t <cmd>
pipe_command* prev_cmd = nullptr)
{
tracer trace ("script::run_pipe");
@@ -1325,8 +1626,8 @@ namespace build2
try
{
cf (env, strings () /* arguments */,
- move (ifd), true /* pipe */,
- dl, dl_cmd != nullptr ? *dl_cmd : c,
+ move (ifd), prev_cmd,
+ dl,
ll);
}
catch (const io_error& e)
@@ -1648,8 +1949,7 @@ namespace build2
// Calculate the process/builtin execution deadline. Note that we should
// also consider the left-hand side processes deadlines, not to keep
// them waiting for us and allow them to terminate not later than their
- // deadlines. Thus, let's also track which command has introduced the
- // deadline, so we can report it if the deadline is missed.
+ // deadlines.
//
dl = earlier (dl, env.effective_deadline ());
@@ -1657,10 +1957,7 @@ namespace build2
{
deadline d (system_clock::now () + *c.timeout, false /* success */);
if (!dl || d < *dl)
- {
dl = d;
- dl_cmd = &c;
- }
}
// Prior to opening file descriptors for command outputs redirects
@@ -1693,11 +1990,7 @@ namespace build2
if (verb >= 2)
print_process (process_args ());
- set_builtin (env, c.arguments,
- move (ifd), !first,
- dl, dl_cmd != nullptr ? *dl_cmd : c,
- ll);
-
+ set_builtin (env, c.arguments, move (ifd), prev_cmd, dl, ll);
return true;
}
@@ -1712,10 +2005,7 @@ namespace build2
try
{
- cf (env, c.arguments,
- move (ifd), !first,
- dl, dl_cmd != nullptr ? *dl_cmd : c,
- ll);
+ cf (env, c.arguments, move (ifd), prev_cmd, dl, ll);
}
catch (const io_error& e)
{
@@ -1734,6 +2024,20 @@ namespace build2
return true;
}
+ // Propagate the pointer to the left-most command.
+ //
+ pipe_command pc (env.context,
+ c,
+ dl,
+ ll,
+ prev_cmd,
+ prev_cmd != nullptr ? prev_cmd->next : nullptr);
+
+ if (prev_cmd != nullptr)
+ prev_cmd->next = &pc;
+ else
+ pc.next = &pc; // Points to itself.
+
// Open a file for command output redirect if requested explicitly
// (file overwrite/append redirects) or for the purpose of the output
// validation (none, here_*, file comparison redirects), register the
@@ -1743,9 +2047,9 @@ namespace build2
// or null-device descriptor for merge, pass or null redirects
// respectively (not opening any file).
//
- auto open = [&env, &wdir, &ll, &std_path] (const redirect& r,
- int dfd,
- path& p) -> auto_fd
+ auto open = [&env, &wdir, &ll, &std_path, &c, &pc] (const redirect& r,
+ int dfd,
+ path& p) -> auto_fd
{
assert (dfd == 1 || dfd == 2);
const char* what (dfd == 1 ? "stdout" : "stderr");
@@ -1761,6 +2065,24 @@ namespace build2
{
case redirect_type::pass:
{
+ if (dfd == 2) // stderr?
+ {
+ // Deduce the args0 argument similar to cmd_path().
+ //
+ process::pipe ep (
+ pc.dbuf.open ((c.program.initial == nullptr
+ ? c.program.recall.string ().c_str ()
+ : c.program.recall_string ()),
+ false /* force */,
+ fdstream_mode::non_blocking));
+
+ if (ep.out != 2) // Are we buffering stderr?
+ {
+ ep.own_out = false;
+ return auto_fd (ep.out);
+ }
+ }
+
try
{
return fddup (dfd);
@@ -1896,111 +2218,313 @@ namespace build2
//
assert (ofd.out != nullfd && efd != nullfd);
- // Wait for a process/builtin to complete until the deadline is reached
- // and return the underlying wait function result (optional<something>).
- //
- auto timed_wait = [] (auto& p, const timestamp& deadline)
- {
- timestamp now (system_clock::now ());
- return deadline > now ? p.timed_wait (deadline - now) : p.try_wait ();
- };
+ pc.isp = &isp;
+ pc.osp = &osp;
+ pc.esp = &esp;
- // Terminate the pipeline processes starting from the specified one and
- // up to the leftmost one and then kill those which didn't terminate
- // after 2 seconds.
- //
- // After that wait for the pipeline builtins completion. Since their
- // standard streams should no longer be written to or read from by any
- // process, that shouldn't take long. If, however, they won't be able to
- // complete in 2 seconds, then some of them have probably stuck while
- // communicating with a slow filesystem device or similar, and since we
- // currently have no way to terminate asynchronous builtins, we have no
- // choice but to abort.
+ // Read out all the pipeline's buffered strerr streams watching for the
+ // deadline, if specified. If the deadline is reached, then terminate
+ // the whole pipeline, reset the deadline to nullopt, and continue
+ // reading. Note that the further reading will be performed without
+ // timeout. This, however, is fine since all the processes and builtins
+ // are terminated and we only need to read out the buffered data.
//
- // Issue diagnostics and fail if something goes wrong, but still try to
- // terminate/kill all the pipe processes.
+ // Also note that this is a reduced version of the above read() function.
//
- auto term_pipe = [&timed_wait, &trace] (pipe_command* pc)
+ auto read_pipe = [&pc, &ll, &trace] ()
{
- diag_record dr;
+ fdselect_set fds;
+ for (pipe_command* c (&pc); c != nullptr; c = c->prev)
+ {
+ diag_buffer& b (c->dbuf);
- auto prog = [] (pipe_command* c) {return cmd_path (c->cmd);};
+ if (b.is.is_open ())
+ fds.emplace_back (b.is.fd (), &b);
+ }
- // Terminate processes gracefully and set the terminate flag for the
- // pipe commands.
+ // Note that the current command deadline is the earliest (see above).
//
- for (pipe_command* c (pc); c != nullptr; c = c->prev)
+ optional<timestamp> dlt (pc.dl ? pc.dl->value : optional<timestamp> ());
+
+ for (size_t unread (fds.size ()); unread != 0;)
{
- if (process* p = c->proc)
try
{
- l5 ([&]{trace (c->loc) << "terminating: " << c->cmd;});
+ // If a deadline is specified, then pass the timeout to fdselect().
+ //
+ if (dlt)
+ {
+ timestamp now (system_clock::now ());
- p->term ();
+ if (*dlt <= now || ifdselect (fds, *dlt - now) == 0)
+ {
+ term_pipe (&pc, trace);
+ dlt = nullopt;
+ continue;
+ }
+ }
+ else
+ ifdselect (fds);
+
+ for (fdselect_state& s: fds)
+ {
+ if (s.ready && !static_cast<diag_buffer*> (s.data)->read ())
+ {
+ s.fd = nullfd;
+ --unread;
+ }
+ }
}
- catch (const process_error& e)
+ catch (const io_error& e)
{
- // If unable to terminate the process for any reason (the process
- // is exiting on Windows, etc) then just ignore this, postponing
- // the potential failure till the kill() call.
- //
- l5 ([&]{trace (c->loc) << "unable to terminate " << prog (c)
- << ": " << e;});
+ fail (ll) << "io error reading pipeline streams: " << e;
}
-
- c->terminated = true;
}
+ };
- // Wait a bit for the processes to terminate and kill the remaining
- // ones.
- //
- timestamp dl (system_clock::now () + chrono::seconds (2));
-
- for (pipe_command* c (pc); c != nullptr; c = c->prev)
+ // Wait for the pipeline processes and builtins to complete, watching
+ // for their deadlines if present. If a deadline is reached for any of
+ // them, then terminate the whole pipeline.
+ //
+ // Note: must be called after read_pipe().
+ //
+ auto wait_pipe = [&pc, &dl, &trace] ()
+ {
+ for (pipe_command* c (&pc); c != nullptr; c = c->prev)
{
- if (process* p = c->proc)
try
{
- l5 ([&]{trace (c->loc) << "waiting: " << c->cmd;});
-
- if (!timed_wait (*p, dl))
+ if (process* p = c->proc)
+ {
+ if (!dl)
+ p->wait ();
+ else if (!timed_wait (*p, dl->value))
+ term_pipe (c, trace);
+ }
+ else
{
- l5 ([&]{trace (c->loc) << "killing: " << c->cmd;});
+ builtin* b (c->bltn);
- p->kill ();
- p->wait ();
+ if (!dl)
+ b->wait ();
+ else if (!timed_wait (*b, dl->value))
+ term_pipe (c, trace);
}
}
catch (const process_error& e)
{
- dr << fail (c->loc) << "unable to wait/kill " << prog (c) << ": "
- << e;
+ fail (c->loc) << "unable to wait " << cmd_path (c->cmd) << ": "
+ << e;
}
}
+ };
- // Wait a bit for the builtins to complete and abort if any remain
- // running.
- //
- dl = system_clock::now () + chrono::seconds (2);
+ // Iterate over the pipeline processes and builtins left to right,
+ // printing their stderr if buffered and issuing the diagnostics if the
+ // exit code is not available (terminated abnormally or due to a
+ // deadline) or is unexpected. Throw failed at the end if the exit code
+ // for any of them is not available. Return false if exit code for any
+ // of them is unexpected (the return is used, for example, in the if-
+ // conditions).
+ //
+ // Note: must be called after wait_pipe() and only once.
+ //
+ auto complete_pipe = [&pc, &env, diag] ()
+ {
+ bool r (true);
+ bool fail (false);
+
+ pipe_command* c (pc.next); // Left-most command.
+ assert (c != nullptr); // Since the lambda must be called once.
- for (pipe_command* c (pc); c != nullptr; c = c->prev)
+ for (pc.next = nullptr; c != nullptr; c = c->next)
{
- if (builtin* b = c->bltn)
- try
+ // Collect the exit status, if present.
+ //
+ // Absent if the process/builtin misses the "unsuccessful" deadline.
+ //
+ optional<process_exit> exit;
+
+ const char* w (c->bltn != nullptr ? "builtin" : "process");
+
+ if (c->bltn != nullptr)
+ {
+ // Note that this also handles ad hoc termination (without the
+ // call to term_pipe()) by the sleep builtin.
+ //
+ if (c->terminated)
+ {
+ if (c->dl && c->dl->success)
+ exit = process_exit (0);
+ }
+ else
+ exit = process_exit (c->bltn->wait ());
+
+ c->bltn = nullptr;
+ }
+ else if (c->proc != nullptr)
{
- l5 ([&]{trace (c->loc) << "waiting: " << c->cmd;});
+ const process& pr (*c->proc);
- if (!timed_wait (*b, dl))
+#ifndef _WIN32
+ if (c->terminated &&
+ !pr.exit->normal () &&
+ pr.exit->signal () == SIGTERM)
+#else
+ if (c->terminated &&
+ !pr.exit->normal () &&
+ pr.exit->status == DBG_TERMINATE_PROCESS)
+#endif
{
- error (c->loc) << prog (c) << " builtin hanged, aborting";
- terminate (false /* trace */);
+ if (c->dl && c->dl->success)
+ exit = process_exit (0);
}
+ else
+ exit = pr.exit;
+
+ c->proc = nullptr;
}
- catch (const system_error& e)
+ else
+ assert (false); // The lambda can only be called once.
+
+ const command& cmd (c->cmd);
+ const location& ll (c->loc);
+
+ // Verify the exit status and issue the diagnostics on failure.
+ //
+ diag_record dr;
+
+ path pr (cmd_path (cmd));
+
+ // Fail if the process is terminated due to reaching the deadline.
+ //
+ if (!exit)
{
- dr << fail (c->loc) << "unable to wait for " << prog (c) << ": "
- << e;
+ dr << error (ll) << w << ' ' << pr
+ << " terminated: execution timeout expired";
+
+ if (verb == 1)
+ {
+ dr << info << "command line: ";
+ print_process (dr, *c->args);
+ }
+
+ fail = true;
}
+ else
+ {
+ // If there is no valid exit code available by whatever reason
+ // then we print the proper diagnostics, dump stderr (if cached
+ // and not too large) and fail the whole script. Otherwise if the
+ // exit code is not correct then we print diagnostics if requested
+ // and fail the pipeline.
+ //
+ bool valid (exit->normal ());
+
+ // On Windows the exit code can be out of the valid codes range
+ // being defined as uint16_t.
+ //
+#ifdef _WIN32
+ if (valid)
+ valid = exit->code () < 256;
+#endif
+
+ // In the presense of a valid exit code we print the diagnostics
+ // and return false rather than throw.
+ //
+ if (!valid)
+ fail = true;
+
+ exit_comparison cmp (cmd.exit
+ ? cmd.exit->comparison
+ : exit_comparison::eq);
+
+ uint16_t exc (cmd.exit ? cmd.exit->code : 0);
+
+ bool success (valid &&
+ (cmp == exit_comparison::eq) ==
+ (exc == exit->code ()));
+
+ if (!success)
+ r = false;
+
+ if (!valid || (!success && diag))
+ {
+ dr << error (ll) << w << ' ';
+
+ if (!exit->normal ())
+ dr << pr << ' ' << *exit;
+ else
+ {
+ uint16_t ec (exit->code ()); // Make sure printed as integer.
+
+ if (!valid)
+ dr << pr << " exit code " << ec << " out of 0-255 range";
+ else if (!success)
+ {
+ if (diag)
+ {
+ if (cmd.exit)
+ dr << pr << " exit code " << ec
+ << (cmp == exit_comparison::eq ? " != " : " == ")
+ << exc;
+ else
+ dr << pr << " exited with code " << ec;
+ }
+ }
+ else
+ assert (false);
+ }
+
+ if (verb == 1)
+ {
+ dr << info << "command line: ";
+ print_process (dr, *c->args);
+ }
+
+ if (non_empty (*c->esp, ll) && avail_on_failure (*c->esp, env))
+ dr << info << "stderr: " << *c->esp;
+
+ if (non_empty (*c->osp, ll) && avail_on_failure (*c->osp, env))
+ dr << info << "stdout: " << *c->osp;
+
+ if (non_empty (*c->isp, ll) && avail_on_failure (*c->isp, env))
+ dr << info << "stdin: " << *c->isp;
+
+ // Print cached stderr.
+ //
+ print_file (dr, *c->esp, ll);
+ }
+ }
+
+ // Now print the buffered stderr, if present, and/or flush the
+ // diagnostics, if issued.
+ //
+ if (c->dbuf.is_open ())
+ c->dbuf.close (move (dr));
+ }
+
+ // Fail if required.
+ //
+ if (fail)
+ throw failed ();
+
+ return r;
+ };
+
+ // Close all buffered pipeline stderr streams ignoring io_error
+ // exceptions.
+ //
+ auto close_pipe = [&pc] ()
+ {
+ for (pipe_command* c (&pc); c != nullptr; c = c->prev)
+ {
+ if (c->dbuf.is.is_open ())
+ try
+ {
+ c->dbuf.is.close();
+ }
+ catch (const io_error&) {}
}
};
@@ -2026,9 +2550,8 @@ namespace build2
fail (ll) << "specified working directory " << cwd
<< " does not exist";
- // Absent if the process/builtin misses the "unsuccessful" deadline.
- //
- optional<process_exit> exit;
+ cstrings args (process_args ());
+ pc.args = &args;
const builtin_info* bi (resolve ? builtins.find (program) : nullptr);
@@ -2042,7 +2565,7 @@ namespace build2
// used for the commands execution flow control.
//
if (verb >= 2 && program != "true" && program != "false")
- print_process (process_args ());
+ print_process (args);
// Some of the script builtins (cp, mkdir, etc) extend libbutl
// builtins (via callbacks) registering/moving cleanups for the
@@ -2083,18 +2606,6 @@ namespace build2
// We also extend the sleep builtin, deactivating the thread before
// going to sleep and waking up before the deadline is reached.
//
- // Let's "wrap up" the sleep-related values into the single object to
- // rely on "small function object" optimization.
- //
- struct sleep
- {
- optional<timestamp> deadline;
- bool terminated = false;
-
- sleep (const optional<timestamp>& d): deadline (d) {}
- };
- sleep slp (dl ? dl->value : optional<timestamp> ());
-
builtin_callbacks bcs {
// create
@@ -2256,16 +2767,19 @@ namespace build2
// sleep
//
- [&env, &slp] (const duration& d)
+ [&env, &pc] (const duration& d)
{
duration t (d);
- const optional<timestamp>& dl (slp.deadline);
+ const optional<timestamp>& dl (pc.dl
+ ? pc.dl->value
+ : optional<timestamp> ());
if (dl)
{
timestamp now (system_clock::now ());
- slp.terminated = now + t > *dl;
+ if (now + t > *dl)
+ pc.terminated = true;
if (*dl <= now)
return;
@@ -2290,19 +2804,19 @@ namespace build2
move (ifd), move (ofd.out), move (efd),
cwd,
bcs));
+ pc.bltn = &b;
- pipe_command pc (b, c, ll, prev_cmd);
-
- // If the deadline is specified, then make sure we don't miss it
- // waiting indefinitely in the builtin destructor on the right-hand
- // side of the pipe failure.
+ // If the right-hand part of the pipe fails, then make sure we don't
+ // wait indefinitely in the process destructor if the deadlines are
+ // specified or just because a process is blocked on stderr.
//
- auto g (make_exception_guard ([&dl, &pc, &term_pipe] ()
+ auto g (make_exception_guard ([&pc, &close_pipe, &trace] ()
{
- if (dl)
+ if (pc.bltn != nullptr)
try
{
- term_pipe (&pc);
+ close_pipe ();
+ term_pipe (&pc, trace);
}
catch (const failed&)
{
@@ -2315,26 +2829,19 @@ namespace build2
move (ofd.in),
ii, li, ci + 1, ll, diag,
cf, last_cmd,
- dl, dl_cmd,
+ dl,
&pc);
- if (!dl)
- b.wait ();
- else if (!timed_wait (b, dl->value))
- term_pipe (&pc);
-
- // Note that this also handles ad hoc termination (without the call
- // to term_pipe()) by the sleep builtin (see above).
+ // Complete the pipeline execution, if not done yet.
//
- if (pc.terminated || slp.terminated)
+ if (pc.bltn != nullptr)
{
- assert (dl);
+ read_pipe ();
+ wait_pipe ();
- if (dl->success)
- exit = process_exit (0);
+ if (!complete_pipe ())
+ success = false;
}
- else
- exit = process_exit (r);
}
catch (const system_error& e)
{
@@ -2346,8 +2853,6 @@ namespace build2
{
// Execute the process.
//
- cstrings args (process_args ());
-
// If the process path is not pre-searched then resolve the relative
// non-simple program path against the script's working directory. The
// simple one will be left for the process path search machinery. Also
@@ -2418,18 +2923,19 @@ namespace build2
ofd.out.reset ();
efd.reset ();
- pipe_command pc (pr, c, ll, prev_cmd);
+ pc.proc = &pr;
- // If the deadline is specified, then make sure we don't miss it
- // waiting indefinitely in the process destructor on the right-hand
- // part of the pipe failure.
+ // If the right-hand part of the pipe fails, then make sure we don't
+ // wait indefinitely in the process destructor (see above for
+ // details).
//
- auto g (make_exception_guard ([&dl, &pc, &term_pipe] ()
+ auto g (make_exception_guard ([&pc, &close_pipe, &trace] ()
{
- if (dl)
+ if (pc.proc != nullptr)
try
{
- term_pipe (&pc);
+ close_pipe ();
+ term_pipe (&pc, trace);
}
catch (const failed&)
{
@@ -2442,31 +2948,19 @@ namespace build2
move (ofd.in),
ii, li, ci + 1, ll, diag,
cf, last_cmd,
- dl, dl_cmd,
+ dl,
&pc);
- if (!dl)
- pr.wait ();
- else if (!timed_wait (pr, dl->value))
- term_pipe (&pc);
-
-#ifndef _WIN32
- if (pc.terminated &&
- !pr.exit->normal () &&
- pr.exit->signal () == SIGTERM)
-#else
- if (pc.terminated &&
- !pr.exit->normal () &&
- pr.exit->status == DBG_TERMINATE_PROCESS)
-#endif
+ // Complete the pipeline execution, if not done yet.
+ //
+ if (pc.proc != nullptr)
{
- assert (dl);
+ read_pipe ();
+ wait_pipe ();
- if (dl->success)
- exit = process_exit (0);
+ if (!complete_pipe ())
+ success = false;
}
- else
- exit = pr.exit;
}
catch (const process_error& e)
{
@@ -2479,98 +2973,23 @@ namespace build2
}
}
- // If the righ-hand side pipeline failed than the whole pipeline fails,
- // and no further checks are required.
- //
- if (!success)
- return false;
-
- // Fail if the process is terminated due to reaching the deadline.
- //
- if (!exit)
- fail (ll) << cmd_path (dl_cmd != nullptr ? *dl_cmd : c)
- << " terminated: execution timeout expired";
-
- path pr (cmd_path (c));
-
- // If there is no valid exit code available by whatever reason then we
- // print the proper diagnostics, dump stderr (if cached and not too
- // large) and fail the whole script. Otherwise if the exit code is not
- // correct then we print diagnostics if requested and fail the pipeline.
- //
- bool valid (exit->normal ());
-
- // On Windows the exit code can be out of the valid codes range being
- // defined as uint16_t.
- //
-#ifdef _WIN32
- if (valid)
- valid = exit->code () < 256;
-#endif
-
- exit_comparison cmp (c.exit ? c.exit->comparison : exit_comparison::eq);
- uint16_t exc (c.exit ? c.exit->code : 0);
-
- success = valid &&
- (cmp == exit_comparison::eq) == (exc == exit->code ());
-
- if (!valid || (!success && diag))
- {
- // In the presense of a valid exit code we print the diagnostics and
- // return false rather than throw.
- //
- diag_record d (valid ? error (ll) : fail (ll));
-
- if (!exit->normal ())
- d << pr << " " << *exit;
- else
- {
- uint16_t ec (exit->code ()); // Make sure is printed as integer.
-
- if (!valid)
- d << pr << " exit code " << ec << " out of 0-255 range";
- else if (!success)
- {
- if (diag)
- {
- if (c.exit)
- d << pr << " exit code " << ec
- << (cmp == exit_comparison::eq ? " != " : " == ") << exc;
- else
- d << pr << " exited with code " << ec;
- }
- }
- else
- assert (false);
- }
-
- if (non_empty (esp, ll) && avail_on_failure (esp, env))
- d << info << "stderr: " << esp;
-
- if (non_empty (osp, ll) && avail_on_failure (osp, env))
- d << info << "stdout: " << osp;
-
- if (non_empty (isp, ll) && avail_on_failure (isp, env))
- d << info << "stdin: " << isp;
-
- // Print cached stderr.
- //
- print_file (d, esp, ll);
- }
-
- // If exit code is correct then check if the standard outputs match the
- // expectations. Note that stdout is only redirected to file for the
- // last command in the pipeline.
+ // If the pipeline or the righ-hand side outputs check failed, then no
+ // further checks are required. Otherwise, check if the standard outputs
+ // match the expectations. Note that stdout can only be redirected to
+ // file for the last command in the pipeline.
//
// The thinking behind matching stderr first is that if it mismatches,
// then the program probably misbehaves (executes wrong functionality,
// etc) in which case its stdout doesn't really matter.
//
if (success)
- success =
- check_output (pr, esp, isp, err, ll, env, diag, "stderr") &&
- (out == nullptr ||
- check_output (pr, osp, isp, *out, ll, env, diag, "stdout"));
+ {
+ path pr (cmd_path (c));
+
+ success = check_output (pr, esp, isp, err, ll, env, diag, "stderr") &&
+ (out == nullptr ||
+ check_output (pr, osp, isp, *out, ll, env, diag, "stdout"));
+ }
return success;
}