diff options
Diffstat (limited to 'libbuild2/script/run.cxx')
| -rw-r--r-- | libbuild2/script/run.cxx | 2135 |
1 files changed, 1813 insertions, 322 deletions
diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx index 8c71b32..f8f98c1 100644 --- a/libbuild2/script/run.cxx +++ b/libbuild2/script/run.cxx @@ -3,23 +3,32 @@ #include <libbuild2/script/run.hxx> -#include <ios> // streamsize +#ifndef _WIN32 +# include <signal.h> // SIG* +#else +# include <libbutl/win32-utility.hxx> // DBG_TERMINATE_PROCESS +#endif + +#include <ios> // streamsize +#include <cstring> // strchr() -#include <libbutl/regex.mxx> -#include <libbutl/builtin.mxx> -#include <libbutl/fdstream.mxx> // fdopen_mode, fddup() -#include <libbutl/filesystem.mxx> // path_search() -#include <libbutl/path-pattern.mxx> +#include <libbutl/regex.hxx> +#include <libbutl/builtin.hxx> +#include <libbutl/fdstream.hxx> // fdopen_mode, fddup() +#include <libbutl/filesystem.hxx> // path_search() #include <libbuild2/filesystem.hxx> #include <libbuild2/diagnostics.hxx> #include <libbuild2/script/regex.hxx> +#include <libbuild2/script/timeout.hxx> #include <libbuild2/script/builtin-options.hxx> using namespace std; using namespace butl; +namespace cli = build2::build::cli; + namespace build2 { namespace script @@ -524,7 +533,7 @@ namespace build2 }; // Save the regex to file for troubleshooting, return the file path - // it have been saved to. + // it has been saved to. // // Note that we save the regex on line regex creation failure or if // the program output doesn't match. @@ -638,33 +647,45 @@ namespace build2 } } - // Create line regex. + // Issue regex error diagnostics and fail. // - line_regex regex; - - try + auto fail_regex = [&rl, &rd, &loc, &env, &output_info, &save_regex] + (const regex_error& e, const string& what) { - regex = line_regex (move (rls), move (pool)); - } - catch (const regex_error& e) - { - // Note that line regex creation can not fail for here-string - // redirect as it doesn't have syntax line chars. That in - // particular means that end_line and end_column are meaningful. + const auto& ls (rl.lines); + + // Note that the parser treats both empty here-string (for example + // >:~'') and empty here-document redirects as an error and so there + // should be at least one line in the list. // - assert (rd.type == redirect_type::here_doc_regex); + assert (!ls.empty ()); - diag_record d (fail (loc (rd.end_line, rd.end_column))); + diag_record d (fail (rd.type == redirect_type::here_doc_regex + ? loc (rd.end_line, rd.end_column) + : loc (ls[0].line, ls[0].column))); // Print regex_error description if meaningful. // - d << "invalid " << what << " regex redirect" << e; + d << what << " regex redirect" << e; // It would be a waste to save the regex into the file just to // remove it. // if (env.temp_dir_keep) output_info (d, save_regex (), "", " regex"); + }; + + // Create line regex. + // + line_regex regex; + + try + { + regex = line_regex (move (rls), move (pool)); + } + catch (const regex_error& e) + { + fail_regex (e, string ("invalid ") + what); } // Parse the output into the literal line string. @@ -704,6 +725,26 @@ namespace build2 while (!s.empty () && s.back () == '\r') s.pop_back (); + // Some regex implementations (e.g., libstdc++, MSVC) are unable + // to match long strings which they "signal" by running out of + // stack or otherwise crashing instead of throwing an exception. + // So we impose some sensible limit that all of them are able to + // handle for basic expressions (e.g., [ab]+; GCC's limits are the + // lowest, see bug 86164). See also another check (for the lines + // number) below. + // + // BTW, if we ever need to overcome this limitation (along with + // various hacks for the two-dimensional regex support), one way + // would be to factor libc++'s implementation (which doesn't seem + // to have any stack-related limits) and use it everywhere. + // + if (s.size () > 16384) + { + diag_record d (fail (ll)); + d << pr << " " << what << " lines too long to match with regex"; + output_info (d, op); + } + ls += line_char (move (s), regex.pool); } } @@ -712,10 +753,56 @@ namespace build2 fail (ll) << "unable to read " << op << ": " << e; } + if (ls.size () > 12288) + { + diag_record d (fail (ll)); + d << pr << " " << what << " has too many lines to match with regex"; + output_info (d, op); + } + + // Note that a here-document regex without ':' modifier can never + // match an empty output since it always contains the trailing empty + // line-char. This can be confusing, as for example while testing a + // program which can print some line or nothing with the following + // test: + // + // $* >>~%EOO% + // %( + // Hello, World! + // %)? + // EOO + // + // Note that the above line-regex contains 4 line-chars and will never + // match empty output. + // + // Thus, let's complete an empty output with an empty line-char for + // such a regex, so it may potentially match. + // + if (ls.empty () && + rd.type == redirect_type::here_doc_regex && + rd.modifiers ().find (':') == string::npos) + { + ls += line_char (string (), regex.pool); + } + // Match the output with the regex. // - if (regex_match (ls, regex)) // Doesn't throw. - return true; + // Note that we don't distinguish between the line_regex and + // char_regex match failures. While it would be convenient for the + // user if we provide additional information in the latter case (regex + // line number, etc), the implementation feels too hairy for now + // (would require to pull additional information into char_regex, + // etc). Though, we may want to implement it in the future. + // + try + { + if (regex_match (ls, regex)) + return true; + } + catch (const regex_error& e) + { + fail_regex (e, string ("unable to match ") + what); + } // Output doesn't match the regex. // @@ -723,7 +810,7 @@ namespace build2 // regex to file for troubleshooting regardless of whether we print // the diagnostics or not. We, however, register it for cleanup in the // later case (the expression may still succeed, we can be evaluating - // the if condition, etc). + // the flow control construct condition, etc). // optional<path> rp; if (env.temp_dir_keep) @@ -757,6 +844,95 @@ namespace build2 return false; } + // The export pseudo-builtin: add/remove the variables to/from the script + // commands execution environment and/or clear the previous additions/ + // removals. + // + // export [-c|--clear <name>]... [-u|--unset <name>]... [<name>=<value>]... + // + static void + export_builtin (environment& env, const strings& args, const location& ll) + { + try + { + cli::vector_scanner scan (args); + export_options ops (scan); + + // Validate a variable name. + // + auto verify_name = [&ll] (const string& name, const char* opt) + { + verify_environment_var_name (name, "export: ", ll, opt); + }; + + // Parse options (variable set/unset cleanups and unsets). + // + for (const string& v: ops.clear ()) + { + verify_name (v, "-c|--clear"); + + environment_vars::iterator i (env.exported_vars.find (v)); + + if (i != env.exported_vars.end ()) + env.exported_vars.erase (i); + } + + for (string& v: ops.unset ()) + { + verify_name (v, "-u|--unset"); + + env.exported_vars.add (move (v)); + } + + // Parse arguments (variable sets). + // + while (scan.more ()) + { + string a (scan.next ()); + verify_environment_var_assignment (a, "export: ", ll); + + env.exported_vars.add (move (a)); + } + } + catch (const cli::exception& e) + { + fail (ll) << "export: " << e; + } + } + + // The timeout pseudo-builtin: set the script timeout. See the script- + // specific set_timeout() implementations for the exact semantics. + // + // timeout [-s|--success] <timeout> + // + static void + timeout_builtin (environment& env, + const strings& args, + const location& ll) + { + try + { + // Parse arguments. + // + cli::vector_scanner scan (args); + timeout_options ops (scan); + + if (!scan.more ()) + fail (ll) << "timeout: missing timeout"; + + string a (scan.next ()); + + if (scan.more ()) + fail (ll) << "timeout: unexpected argument '" << scan.next () << "'"; + + env.set_timeout (a, ops.success (), ll); + } + catch (const cli::exception& e) + { + fail (ll) << "timeout: " << e; + } + } + // The exit pseudo-builtin: exit the script successfully, or print the // diagnostics and exit the script unsuccessfully. Always throw exit // exception. @@ -780,119 +956,728 @@ namespace build2 const string& s (*i++); if (i != e) - fail (ll) << "unexpected argument '" << *i << "'"; + fail (ll) << "exit: unexpected argument '" << *i << "'"; error (ll) << s; throw exit (false); } - // The set pseudo-builtin: set variable from the stdin input. + // Return the command program path for diagnostics. // - // set [-e|--exact] [(-n|--newline)|(-w|--whitespace)] [<attr>] <var> + static inline path + cmd_path (const command& c) + { + return c.program.initial == nullptr // Not pre-searched? + ? c.program.recall + : path (c.program.recall_string ()); + } + + // Read the stream content into a string, optionally splitting the input + // data at whitespaces or newlines in which case return one, potentially + // incomplete, substring at a time (see the set builtin options for the + // splitting semantics). Throw io_error on the underlying OS error. // - static void - set_builtin (environment& env, - const strings& args, - auto_fd in, - const location& ll) + // On POSIX expects the stream to be non-blocking and its exception mask + // to have at least badbit. On Windows can also handle a blocking stream. + // + // Note that on Windows we can only turn pipe file descriptors into the + // non-blocking mode. Thus, we have no choice but to read from descriptors + // of other types synchronously there. That implies that we can + // potentially block indefinitely reading a file and missing a deadline on + // Windows. Note though, that the user can normally rewrite the command, + // for example, `set foo <<<file` with `cat file | set foo` to avoid this + // problem. + // + class stream_reader { - try + public: + stream_reader (ifdstream&, bool whitespace, bool newline, bool exact); + + // Read next substring. Return true if the substring has been read or + // false if it should be called again once the stream has more data to + // read. Also return true on eof (in which case no substring is read). + // The string must be empty on the first call. Throw ios::failure on the + // underlying OS error. + // + // Note that there could still be data to read in the stream's buffer + // (as opposed to file descriptor) after this function returns true and + // you should be careful not to block on fdselect() in this case. The + // recommended usage pattern is similar to that of + // butl::getline_non_blocking(). The only difference is that + // ifdstream::eof() needs to be used instead of butl::eof() since this + // function doesn't set failbit and only sets eofbit after the last + // substring is returned. + // + bool + next (string&); + + private: + ifdstream& is_; + bool whitespace_; + bool newline_; + bool exact_; + + bool empty_ = true; // Set to false after the first character is read. + }; + + stream_reader:: + stream_reader (ifdstream& is, bool ws, bool nl, bool ex) + : is_ (is), + whitespace_ (ws), + newline_ (nl), + exact_ (ex) + { + } + + bool stream_reader:: + next (string& ss) + { +#ifndef _WIN32 + assert ((is_.exceptions () & ifdstream::badbit) != 0 && !is_.blocking ()); +#else + assert ((is_.exceptions () & ifdstream::badbit) != 0); +#endif + + fdstreambuf& sb (*static_cast<fdstreambuf*> (is_.rdbuf ())); + + // Return the number of characters available in the stream buffer's get + // area, which can be: + // + // -1 -- EOF. + // 0 -- no data since blocked before encountering more data/EOF. + // >0 -- there is some data. + // + // Note that on Windows if the stream is blocking, then the lambda calls + // underflow() instead of returning 0. + // + // @@ Probably we can call underflow() only once per the next() call, + // emulating the 'no data' case. This will allow the caller to + // perform some housekeeping (reading other streams, checking for the + // deadline, etc). But let's keep it simple for now. + // + auto avail = [&sb] () -> streamsize { - // Do not throw when eofbit is set (end of stream reached), and - // when failbit is set (read operation failed to extract any - // character). + // Note that here we reasonably assume that any failure in in_avail() + // will lead to badbit and thus an exception (see showmanyc()). // - ifdstream cin (move (in), ifdstream::badbit); + streamsize r (sb.in_avail ()); - // Parse arguments. +#ifdef _WIN32 + if (r == 0 && sb.blocking ()) + { + if (sb.underflow () == ifdstream::traits_type::eof ()) + return -1; + + r = sb.in_avail (); + + assert (r != 0); // We wouldn't be here otherwise. + } +#endif + + return r; + }; + + // Read until blocked (0), EOF (-1) or encounter the delimiter. + // + streamsize s; + while ((s = avail ()) > 0) + { + if (empty_) + empty_ = false; + + const char* p (sb.gptr ()); + size_t n (sb.egptr () - p); + + // We move p and bump by the number of consumed characters. // - cli::vector_scanner scan (args); - set_options ops (scan); + auto bump = [&sb, &p] () {sb.gbump (static_cast<int> (p - sb.gptr ()));}; - if (ops.whitespace () && ops.newline ()) - fail (ll) << "both -n|--newline and -w|--whitespace specified"; + if (whitespace_) // The whitespace mode. + { + const char* sep (" \n\r\t"); - if (!scan.more ()) - fail (ll) << "missing variable name"; + // Skip the whitespaces. + // + for (; n != 0 && strchr (sep, *p) != nullptr; ++p, --n) ; - string a (scan.next ()); // Either attributes or variable name. - const string* ats (!scan.more () ? nullptr : &a); - string vname (!scan.more () ? move (a) : scan.next ()); + // If there are any non-whitespace characters in the get area, then + // append them to the resulting substring until a whitespace + // character is encountered. + // + if (n != 0) + { + // Append the non-whitespace characters. + // + for (char c; n != 0 && strchr (sep, c = *p) == nullptr; ++p, --n) + ss += c; - if (scan.more ()) - fail (ll) << "unexpected argument '" << scan.next () << "'"; + // If a separator is encountered, then consume it, bump, and + // return the substring. + // + if (n != 0) + { + ++p; --n; // Consume the separator character. - if (ats != nullptr && ats->empty ()) - fail (ll) << "empty variable attributes"; + bump (); + return true; + } - if (vname.empty ()) - fail (ll) << "empty variable name"; + // Fall through. + } + + bump (); // Bump and continue reading. + } + else // The newline or no-split mode. + { + // Note that we don't collapse multiple consecutive newlines. + // + // Note also that we always sanitize CRs, so in the no-split mode we + // need to loop rather than consume the whole get area at once. + // + while (n != 0) + { + // Append the characters until the newline character or the end of + // the get area is encountered. + // + char c; + for (; n != 0 && (c = *p) != '\n'; ++p, --n) + ss += c; + + // If the newline character is encountered, then sanitize CRs and + // return the substring in the newline mode and continue + // parsing/reading otherwise. + // + if (n != 0) + { + // Strip the trailing CRs that can appear while, for example, + // cross-testing Windows target or as a part of msvcrt junk + // production (see above). + // + while (!ss.empty () && ss.back () == '\r') + ss.pop_back (); + + assert (c == '\n'); + + ++p; --n; // Consume the newline character. + + if (newline_) + { + bump (); + return true; + } + + ss += c; // Append newline to the resulting string. + + // Fall through. + } + + bump (); // Bump and continue parsing/reading. + } + } + } + + // Here s can be: + // + // -1 -- EOF. + // 0 -- blocked before encountering delimiter/EOF. + // + // Note: >0 (encountered the delimiter) case is handled in-place. + // + assert (s == -1 || s == 0); - // Read the input. + if (s == -1) + { + // Return the last substring if it is not empty or it is the trailing + // "blank" in the exact mode. Otherwise, set eofbit for the stream + // indicating that we are done. // - cin.peek (); // Sets eofbit for an empty stream. + if (!ss.empty () || (exact_ && !empty_)) + { + // Also, strip the trailing newline character, if present, in the + // no-split no-exact mode. + // + if (!ss.empty () && ss.back () == '\n' && // Trailing newline. + !newline_ && !whitespace_ && !exact_) // No-split no-exact mode. + { + ss.pop_back (); + } - names ns; - while (!cin.eof ()) + exact_ = false; // Make sure we will set eofbit on the next call. + } + else + is_.setstate (ifdstream::eofbit); + } + + return s == -1; + } + + // Stack-allocated linked list of information about the running pipeline + // processes and builtins. + // + // Note: constructed incrementally. + // + struct pipe_command + { + // Initially NULL. Set to the address of the process or builtin object + // when it is created. Reset back to NULL when the respective + // process/builtin is executed and its exit status is collected (see + // complete_pipe() for details). + // + // We could probably use a union here, but let's keep it simple for now + // (at least one is NULL). + // + process* proc = nullptr; + builtin* bltn = nullptr; + + const command& cmd; + const cstrings* args = nullptr; + const optional<deadline>& dl; + + diag_buffer dbuf; + + bool terminated = false; // True if this command has been terminated. + + // True if this command has been terminated but we failed to read out + // its stdout and/or stderr streams in the reasonable timeframe (2 + // seconds) after the termination. + // + // Note that this may happen if there is a still running child process + // of the terminated command which has inherited the parent's stdout and + // stderr file descriptors. + // + bool unread_stdout = false; + bool unread_stderr = false; + + // Only for diagnostics. + // + const location& loc; + const path* isp = nullptr; // stdin cache. + const path* osp = nullptr; // stdout cache. + const path* esp = nullptr; // stderr cache. + + pipe_command* prev; // NULL for the left-most command. + pipe_command* next; // Left-most command for the right-most command. + + pipe_command (context& x, + const command& c, + const optional<deadline>& d, + const location& l, + pipe_command* p, + pipe_command* f) + : cmd (c), dl (d), dbuf (x), loc (l), prev (p), next (f) {} + }; + + // Wait for a process/builtin to complete until the deadline is reached + // and return the underlying wait function result (optional<something>). + // + template<typename P> + static auto + timed_wait (P& p, const timestamp& deadline) -> decltype(p.try_wait ()) + { + timestamp now (system_clock::now ()); + return deadline > now ? p.timed_wait (deadline - now) : p.try_wait (); + } + + // Terminate the pipeline processes starting from the specified one and up + // to the leftmost one and then kill those which didn't terminate after 2 + // seconds. + // + // After that wait for the pipeline builtins completion. Since their + // standard streams should no longer be written to or read from by any + // process, that shouldn't take long. If, however, they won't be able to + // complete in 2 seconds, then some of them have probably stuck while + // communicating with a slow filesystem device or similar, and since we + // currently have no way to terminate asynchronous builtins, we have no + // choice but to abort. + // + // Issue diagnostics and fail if something goes wrong, but still try to + // terminate/kill all the pipe processes. + // + static void + term_pipe (pipe_command* pc, tracer& trace) + { + auto prog = [] (pipe_command* c) {return cmd_path (c->cmd);}; + + // Terminate processes gracefully and set the terminate flag for the + // pipe commands. + // + diag_record dr; + for (pipe_command* c (pc); c != nullptr; c = c->prev) + { + if (process* p = c->proc) + try + { + l5 ([&]{trace (c->loc) << "terminating: " << c->cmd;}); + + p->term (); + } + catch (const process_error& e) { - // Read next element that depends on the whitespace mode being - // enabled or not. For the later case it also make sense to strip - // the trailing CRs that can appear while, for example, - // cross-testing Windows target or as a part of msvcrt junk - // production (see above). + // If unable to terminate the process for any reason (the process is + // exiting on Windows, etc) then just ignore this, postponing the + // potential failure till the kill() call. // - string s; - if (ops.whitespace ()) - cin >> s; - else + l5 ([&]{trace (c->loc) << "unable to terminate " << prog (c) + << ": " << e;}); + } + + c->terminated = true; + } + + // Wait a bit for the processes to terminate and kill the remaining + // ones. + // + timestamp dl (system_clock::now () + chrono::seconds (2)); + + for (pipe_command* c (pc); c != nullptr; c = c->prev) + { + if (process* p = c->proc) + try + { + l5 ([&]{trace (c->loc) << "waiting: " << c->cmd;}); + + if (!timed_wait (*p, dl)) { - getline (cin, s); + l5 ([&]{trace (c->loc) << "killing: " << c->cmd;}); - while (!s.empty () && s.back () == '\r') - s.pop_back (); + p->kill (); + p->wait (); } + } + catch (const process_error& e) + { + dr << fail (c->loc) << "unable to wait/kill " << prog (c) << ": " + << e; + } + } + + // Wait a bit for the builtins to complete and abort if any remain + // running. + // + dl = system_clock::now () + chrono::seconds (2); + + for (pipe_command* c (pc); c != nullptr; c = c->prev) + { + if (builtin* b = c->bltn) + try + { + l5 ([&]{trace (c->loc) << "waiting: " << c->cmd;}); + + if (!timed_wait (*b, dl)) + { + error (c->loc) << prog (c) << " builtin hanged, aborting"; + terminate (false /* trace */); + } + } + catch (const system_error& e) + { + dr << fail (c->loc) << "unable to wait for " << prog (c) << ": " + << e; + } + } + } + + void + read (auto_fd&& in, + bool whitespace, bool newline, bool exact, + const function<void (string&&)>& cf, + pipe_command* pipeline, + const optional<deadline>& dl, + const location& ll, + const char* what) + { + tracer trace ("script::stream_read"); + + // Note: stays blocking on Windows if the descriptor is not of the pipe + // type. + // +#ifndef _WIN32 + fdstream_mode m (fdstream_mode::non_blocking); +#else + fdstream_mode m (pipeline != nullptr + ? fdstream_mode::non_blocking + : fdstream_mode::blocking); +#endif + + ifdstream is (move (in), m, ifdstream::badbit); + stream_reader sr (is, whitespace, newline, exact); + + fdselect_set fds; + for (pipe_command* c (pipeline); c != nullptr; c = c->prev) + { + diag_buffer& b (c->dbuf); + + if (b.is.is_open ()) + fds.emplace_back (b.is.fd (), c); + } + + fds.emplace_back (is.fd ()); + fdselect_state& ist (fds.back ()); + size_t unread (fds.size ()); + + optional<timestamp> dlt (dl ? dl->value : optional<timestamp> ()); + + // If there are some left-hand side processes/builtins running, then + // terminate them and, if there are unread stdout/stderr file + // descriptors, then increase the deadline by another 2 seconds and + // return true. In this case the term() should be called again upon + // reaching the timeout. Otherwise return false. If there are no + // left-hand side processes/builtins running, then fail straight away. + // + // Note that in the former case the further reading will be performed + // with the adjusted timeout. We assume that this timeout is normally + // sufficient to read out the buffered data written by the already + // terminated processes. If, however, that's not the case (see + // pipe_command for the possible reasons), then term() needs to be + // called for the second time and the reading should be interrupted + // afterwards. + // + auto term = [&dlt, pipeline, &fds, &ist, &is, &unread, + &trace, &ll, what, terminated = false] () mutable -> bool + { + // Can only be called if the deadline is specified. + // + assert (dlt); + + if (pipeline == nullptr) + fail (ll) << what << " terminated: execution timeout expired"; + + if (!terminated) + { + // Terminate the pipeline and adjust the deadline. + // - // If failbit is set then we read nothing into the string as eof is - // reached. That in particular means that the stream has trailing - // whitespaces (possibly including newlines) if the whitespace mode - // is enabled, or the trailing newline otherwise. If so then - // we append the "blank" to the variable value in the exact mode - // prior to bailing out. + // Note that if we are still reading the stream and it's a builtin + // stdout, then we need to close it before terminating the pipeline. + // Not doing so can result in blocking this builtin on the write + // operation and thus aborting the build2 process (see term_pipe() + // for details). + // + // Should we do the same for all the pipeline builtins' stderr + // streams? No we don't, since the builtin diagnostics is assumed to + // always fit the pipe buffer (see libbutl/builtin.cxx for details). + // Thus, we will leave them open to fully read out the diagnostics. // - if (cin.fail ()) + if (ist.fd != nullfd && pipeline->bltn != nullptr) { - if (ops.exact ()) + try { - if (ops.whitespace () || ops.newline ()) - ns.emplace_back (move (s)); // Reuse empty string. - else if (ns.empty ()) - ns.emplace_back ("\n"); - else - ns[0].value += '\n'; + is.close (); + } + catch (const io_error&) + { + // Not much we can do here. } - break; + ist.fd = nullfd; + --unread; } - if (ops.whitespace () || ops.newline () || ns.empty ()) - ns.emplace_back (move (s)); + term_pipe (pipeline, trace); + terminated = true; + + if (unread != 0) + dlt = system_clock::now () + chrono::seconds (2); + + return unread != 0; + } + else + { + // Set the unread_{stderr,stdout} flags to true for the commands + // whose streams are not fully read yet. + // + + // Can only be called after the first call of term() which would + // throw failed if pipeline is NULL. + // + assert (pipeline != nullptr); + + for (fdselect_state& s: fds) + { + if (s.fd != nullfd) + { + if (s.data != nullptr) // stderr. + { + pipe_command* c (static_cast<pipe_command*> (s.data)); + + c->unread_stderr = true; + + // Let's also close the stderr stream not to confuse + // diag_buffer::close() with a not fully read stream (eof is + // not reached, etc). + // + try + { + c->dbuf.is.close (); + } + catch (const io_error&) + { + // Not much we can do here. Anyway the diagnostics will be + // issued by complete_pipe(). + } + } + else // stdout. + pipeline->unread_stdout = true; + } + } + + return false; + } + }; + + // Note that on Windows if the file descriptor is not a pipe, then + // ifdstream assumes the blocking mode for which ifdselect() would throw + // invalid_argument. Such a descriptor can, however, only appear for the + // first command in the pipeline and so fds will only contain the input + // stream's descriptor. That all means that this descriptor will be read + // out by a series of the stream_reader::next() calls which can only + // return true and thus no ifdselect() calls will ever be made. + // + string s; + while (unread != 0) + { + // Read any pending data from the input stream. + // + if (ist.fd != nullfd) + { + // Prior to reading let's check that the deadline, if specified, is + // not reached. This way we handle the (hypothetical) case when we + // are continuously fed with the data without delays and thus can + // never get to ifdselect() which watches for the deadline. Also + // this check is the only way to bail out early on Windows for a + // blocking file descriptor. + // + if (dlt && *dlt <= system_clock::now ()) + { + if (!term ()) + break; + } + + if (sr.next (s)) + { + if (!is.eof ()) + { + // Consume the substring. + // + cf (move (s)); + s.clear (); + } + else + { + ist.fd = nullfd; + --unread; + } + + continue; + } + } + + try + { + // Wait until the data appear in any of the streams. If a deadline + // is specified, then pass the timeout to fdselect(). + // + if (dlt) + { + timestamp now (system_clock::now ()); + + if (*dlt <= now || ifdselect (fds, *dlt - now) == 0) + { + if (term ()) + continue; + else + break; + } + } else + ifdselect (fds); + + // Read out the pending data from the stderr streams. + // + for (fdselect_state& s: fds) { - ns[0].value += '\n'; - ns[0].value += s; + if (s.ready && + s.data != nullptr && + !static_cast<pipe_command*> (s.data)->dbuf.read ()) + { + s.fd = nullfd; + --unread; + } } } + catch (const io_error& e) + { + fail (ll) << "io error reading pipeline streams: " << e; + } + } + } + + // The set pseudo-builtin: set variable from the stdin input. + // + // set [-e|--exact] [(-n|--newline)|(-w|--whitespace)] <var> [<attr>] + // + static void + set_builtin (environment& env, + const strings& args, + auto_fd in, + pipe_command* pipeline, + const optional<deadline>& dl, + const location& ll) + { + tracer trace ("script::set_builtin"); + + try + { + // Parse arguments. + // + cli::vector_scanner scan (args); + set_options ops (scan); + + if (ops.whitespace () && ops.newline ()) + fail (ll) << "set: both -n|--newline and -w|--whitespace specified"; + + if (!scan.more ()) + fail (ll) << "set: missing variable name"; + + string vname (scan.next ()); + if (vname.empty ()) + fail (ll) << "set: empty variable name"; + + // Detect patterns analogous to parser::parse_variable_name() (so we + // diagnose `set x[string]`). + // + if (vname.find_first_of ("[*?") != string::npos) + fail (ll) << "set: expected variable name instead of " << vname; + + string attrs; + if (scan.more ()) + { + attrs = scan.next (); + + if (attrs.empty ()) + fail (ll) << "set: empty variable attributes"; + + if (scan.more ()) + fail (ll) << "set: unexpected argument '" << scan.next () << "'"; + } + + // Parse the stream content into the variable value. + // + names ns; - cin.close (); + read (move (in), + ops.whitespace (), ops.newline (), ops.exact (), + [&ns] (string&& s) {ns.emplace_back (move (s));}, + pipeline, + dl, + ll, + "set"); - env.set_variable (move (vname), - move (ns), - ats != nullptr ? *ats : empty_string, - ll); + env.set_variable (move (vname), move (ns), attrs, ll); } catch (const io_error& e) { - fail (ll) << "set: " << e; + fail (ll) << "set: unable to read from stdin: " << e; } catch (const cli::exception& e) { @@ -920,22 +1705,64 @@ namespace build2 command_pipe::const_iterator bc, command_pipe::const_iterator ec, auto_fd ifd, - size_t ci, size_t li, const location& ll, - bool diag) + const iteration_index* ii, size_t li, size_t ci, + const location& ll, + bool diag, + const function<command_function>& cf, bool last_cmd, + optional<deadline> dl = nullopt, + pipe_command* prev_cmd = nullptr) { - if (bc == ec) // End of the pipeline. + tracer trace ("script::run_pipe"); + + // At the end of the pipeline read out its stdout, if requested. + // + if (bc == ec) + { + if (cf != nullptr) + { + assert (!last_cmd); // Otherwise we wouldn't be here. + + // The pipeline can't be empty. + // + assert (ifd != nullfd && prev_cmd != nullptr); + + const command& c (prev_cmd->cmd); + + try + { + cf (env, strings () /* arguments */, + move (ifd), prev_cmd, + dl, + ll); + } + catch (const io_error& e) + { + fail (ll) << "unable to read from " << cmd_path (c) << " stdout: " + << e; + } + } + return true; + } - // The overall plan is to run the first command in the pipe, reading - // its input from the file descriptor passed (or, for the first - // command, according to stdin redirect specification) and redirecting - // its output to the right-hand part of the pipe recursively. Fail if - // the right-hand part fails. Otherwise check the process exit code, - // match stderr (and stdout for the last command in the pipe) according - // to redirect specification(s) and fail if any of the above fails. + // The overall plan is to run the first command in the pipe, reading its + // input from the file descriptor passed (or, for the first command, + // according to stdin redirect specification) and redirecting its output + // to the right-hand part of the pipe recursively. Fail if the + // right-hand part fails. Otherwise check the process exit code, match + // stderr (and stdout for the last command in the pipe) according to + // redirect specification(s) and fail if any of the above fails. + // + // If the command has a deadline, then terminate the whole pipeline when + // the deadline is reached. This way the pipeline processes get a chance + // to terminate gracefully, which in particular may require to interrupt + // their IO operations, closing their standard streams readers and + // writers. // const command& c (*bc); + const dir_path& wdir (*env.work_dir.path); + // Register the command explicit cleanups. Verify that the path being // cleaned up is a sub-path of the script working directory. Fail if // this is not the case. @@ -943,7 +1770,7 @@ namespace build2 for (const auto& cl: c.cleanups) { const path& p (cl.path); - path np (normalize (p, *env.work_dir.path, ll)); + path np (normalize (p, wdir, ll)); const string& ls (np.leaf ().string ()); bool wc (ls == "*" || ls == "**" || ls == "***"); @@ -968,6 +1795,12 @@ namespace build2 command_pipe::const_iterator nc (bc + 1); bool last (nc == ec); + // Make sure that stdout is not redirected if meant to be read (last_cmd + // is false) or cannot not be produced (last_cmd is true). + // + if (last && c.out && cf != nullptr) + fail (ll) << "stdout cannot be redirected"; + // True if the process path is not pre-searched and the program path // still needs to be resolved. // @@ -979,7 +1812,7 @@ namespace build2 const redirect& in ((c.in ? *c.in : env.in).effective ()); - const redirect* out (!last + const redirect* out (!last || (cf != nullptr && !last_cmd) ? nullptr // stdout is piped. : &(c.out ? *c.out : env.out).effective ()); @@ -987,72 +1820,130 @@ namespace build2 auto process_args = [&c] () -> cstrings { - cstrings args {c.program.recall_string ()}; - - for (const auto& a: c.arguments) - args.push_back (a.c_str ()); - - args.push_back (nullptr); - return args; + return build2::process_args (c.program.recall_string (), c.arguments); }; - // Prior to opening file descriptors for command input/output - // redirects let's check if the command is the exit builtin. Being a - // builtin syntactically it differs from the regular ones in a number - // of ways. It doesn't communicate with standard streams, so - // redirecting them is meaningless. It may appear only as a single - // command in a pipeline. It doesn't return any value and stops the - // script execution, so checking its exit status is meaningless as - // well. That all means we can short-circuit here calling the builtin - // and bailing out right after that. Checking that the user didn't - // specify any redirects or exit code check sounds like a right thing - // to do. - // - if (resolve && program == "exit") + // Prior to opening file descriptors for command input/output redirects + // let's check if the command is the exit, export, or timeout + // builtin. Being a builtin syntactically they differ from the regular + // ones in a number of ways. They don't communicate with standard + // streams, so redirecting them is meaningless. They may appear only as + // a single command in a pipeline. They don't return any value, so + // checking their exit status is meaningless as well. That all means we + // can short-circuit here calling the builtin and bailing out right + // after that. Checking that the user didn't specify any variables, + // timeout, redirects, or exit code check sounds like a right thing to + // do. + // + if (resolve && + (program == "exit" || program == "export" || program == "timeout")) { // In case the builtin is erroneously pipelined from the other // command, we will close stdin gracefully (reading out the stream - // content), to make sure that the command doesn't print any - // unwanted diagnostics about IO operation failure. - // - // Note that dtor will ignore any errors (which is what we want). + // content), to make sure that the command doesn't print any unwanted + // diagnostics about IO operation failure. // - ifdstream is (move (ifd), fdstream_mode::skip); + if (ifd != nullfd) + { + // Note that we can't use ifdstream dtor in the skip mode here since + // it turns the stream into the blocking mode and we won't be able + // to read out the potentially buffered stderr for the + // pipeline. Using read() is also not ideal since it performs + // parsing and allocations needlessly. This, however, is probably ok + // for such an uncommon case. + // + //ifdstream (move (ifd), fdstream_mode::skip); + + // Let's try to minimize the allocation size splitting the input + // data at whitespaces. + // + read (move (ifd), + true /* whitespace */, + false /* newline */, + false /* exact */, + [] (string&&) {}, // Just drop the string. + prev_cmd, + dl, + ll, + program.c_str ()); + } if (!first || !last) - fail (ll) << "exit builtin must be the only pipe command"; + fail (ll) << program << " builtin must be the only pipe command"; + + if (c.cwd) + fail (ll) << "current working directory cannot be specified for " + << program << " builtin"; + + if (!c.variables.empty ()) + fail (ll) << "environment variables cannot be (un)set for " + << program << " builtin"; + + if (c.timeout) + fail (ll) << "timeout cannot be specified for " << program + << " builtin"; if (c.in) - fail (ll) << "exit builtin stdin cannot be redirected"; + fail (ll) << program << " builtin stdin cannot be redirected"; if (c.out) - fail (ll) << "exit builtin stdout cannot be redirected"; + fail (ll) << program << " builtin stdout cannot be redirected"; + + if (cf != nullptr && !last_cmd) + fail (ll) << program << " builtin stdout cannot be read"; if (c.err) - fail (ll) << "exit builtin stderr cannot be redirected"; + fail (ll) << program << " builtin stderr cannot be redirected"; if (c.exit) - fail (ll) << "exit builtin exit code cannot be checked"; + fail (ll) << program << " builtin exit code cannot be checked"; if (verb >= 2) print_process (process_args ()); - exit_builtin (c.arguments, ll); // Throws exit exception. + if (program == "exit") + { + exit_builtin (c.arguments, ll); // Throws exit exception. + } + else if (program == "export") + { + export_builtin (env, c.arguments, ll); + return true; + } + else if (program == "timeout") + { + timeout_builtin (env, c.arguments, ll); + return true; + } + else + assert (false); } // Create a unique path for a command standard stream cache file. // - auto std_path = [&env, &ci, &li, &ll] (const char* n) -> path + auto std_path = [&env, ii, &li, &ci, &ll] (const char* nm) -> path { using std::to_string; - path p (n); + string s (nm); + size_t n (s.size ()); + + if (ii != nullptr) + { + // Note: reverse order (outermost to innermost). + // + for (const iteration_index* i (ii); i != nullptr; i = i->prev) + s.insert (n, "-i" + to_string (i->index)); + } // 0 if belongs to a single-line script, otherwise is the command line // number (start from one) in the script. // - if (li > 0) - p += "-" + to_string (li); + if (li != 0) + { + s += "-n"; + s += to_string (li); + } // 0 if belongs to a single-command expression, otherwise is the // command number (start from one) in the expression. @@ -1061,10 +1952,13 @@ namespace build2 // single-line script or to N-th single-command line of multi-line // script. These cases are mutually exclusive and so are unambiguous. // - if (ci > 0) - p += "-" + to_string (ci); + if (ci != 0) + { + s += "-c"; + s += to_string (ci); + } - return normalize (move (p), temp_dir (env), ll); + return normalize (path (move (s)), temp_dir (env), ll); }; // If this is the first pipeline command, then open stdin descriptor @@ -1121,6 +2015,9 @@ namespace build2 // process to hang which can be interpreted as a command failure. // @@ Both ways are quite ugly. Is there some better way to do // this? + // @@ Maybe we can create a pipe, write a byte into it, close the + // writing end, and after the process terminates make sure we can + // still read this byte out? // // Fall through. // @@ -1131,7 +2028,7 @@ namespace build2 } case redirect_type::file: { - isp = normalize (in.file.path, *env.work_dir.path, ll); + isp = normalize (in.file.path, wdir, ll); open_stdin (); break; @@ -1163,6 +2060,20 @@ namespace build2 assert (ifd.get () != -1); + // Calculate the process/builtin execution deadline. Note that we should + // also consider the left-hand side processes deadlines, not to keep + // them waiting for us and allow them to terminate not later than their + // deadlines. + // + dl = earlier (dl, env.effective_deadline ()); + + if (c.timeout) + { + deadline d (system_clock::now () + *c.timeout, c.timeout_success); + if (!dl || d < *dl) + dl = d; + } + // Prior to opening file descriptors for command outputs redirects // let's check if the command is the set builtin. Being a builtin // syntactically it differs from the regular ones in a number of ways. @@ -1181,6 +2092,9 @@ namespace build2 if (c.out) fail (ll) << "set builtin stdout cannot be redirected"; + if (cf != nullptr && !last_cmd) + fail (ll) << "set builtin stdout cannot be read"; + if (c.err) fail (ll) << "set builtin stderr cannot be redirected"; @@ -1190,10 +2104,54 @@ namespace build2 if (verb >= 2) print_process (process_args ()); - set_builtin (env, c.arguments, move (ifd), ll); + set_builtin (env, c.arguments, move (ifd), prev_cmd, dl, ll); + return true; + } + + // If this is the last command in the pipe and the command function is + // specified for it, then call it. + // + if (last && cf != nullptr && last_cmd) + { + // Must be enforced by the caller. + // + assert (!c.out && !c.err && !c.exit); + + try + { + cf (env, c.arguments, move (ifd), prev_cmd, dl, ll); + } + catch (const io_error& e) + { + diag_record dr (fail (ll)); + + dr << cmd_path (c) << ": unable to read from "; + + if (prev_cmd != nullptr) + dr << cmd_path (prev_cmd->cmd) << " output"; + else + dr << "stdin"; + + dr << ": " << e; + } + return true; } + // Propagate the pointer to the left-most command. + // + pipe_command pc (env.context, + c, + dl, + ll, + prev_cmd, + prev_cmd != nullptr ? prev_cmd->next : nullptr); + + if (prev_cmd != nullptr) + prev_cmd->next = &pc; + else + pc.next = &pc; // Points to itself. + // Open a file for command output redirect if requested explicitly // (file overwrite/append redirects) or for the purpose of the output // validation (none, here_*, file comparison redirects), register the @@ -1203,9 +2161,9 @@ namespace build2 // or null-device descriptor for merge, pass or null redirects // respectively (not opening any file). // - auto open = [&env, &ll, &std_path] (const redirect& r, - int dfd, - path& p) -> auto_fd + auto open = [&env, &wdir, &ll, &std_path, &c, &pc] (const redirect& r, + int dfd, + path& p) -> auto_fd { assert (dfd == 1 || dfd == 2); const char* what (dfd == 1 ? "stdout" : "stderr"); @@ -1223,11 +2181,34 @@ namespace build2 { try { + if (dfd == 2) // stderr? + { + fdpipe p; + if (diag_buffer::pipe (env.context) == -1) // Are we buffering? + p = fdopen_pipe (); + + // Deduce the args0 argument similar to cmd_path(). + // + // Note that we must open the diag buffer regardless of the + // diag_buffer::pipe() result. + // + pc.dbuf.open ((c.program.initial == nullptr + ? c.program.recall.string ().c_str () + : c.program.recall_string ()), + move (p.in), + fdstream_mode::non_blocking); + + if (p.out != nullfd) + return move (p.out); + + // Fall through. + } + return fddup (dfd); } catch (const io_error& e) { - fail (ll) << "unable to duplicate " << what << ": " << e; + fail (ll) << "unable to redirect " << what << ": " << e; } } @@ -1246,7 +2227,7 @@ namespace build2 // p = r.file.mode == redirect_fmode::compare ? std_path (what) - : normalize (r.file.path, *env.work_dir.path, ll); + : normalize (r.file.path, wdir, ll); m |= r.file.mode == redirect_fmode::append ? fdopen_mode::at_end @@ -1309,7 +2290,7 @@ namespace build2 // script failures investigation and, for example, for validation // "tightening". // - if (last) + if (last && out != nullptr) ofd.out = open (*out, 1, osp); else { @@ -1323,10 +2304,22 @@ namespace build2 // Merge standard streams. // bool mo (out != nullptr && out->type == redirect_type::merge); - if (mo || err.type == redirect_type::merge) + bool me (err.type == redirect_type::merge); + + if (mo || me) { + // Note that while the parser verifies that there is no stdout/stderr + // mutual redirects specified on the command line, we can still end up + // with mutual redirects here since one of such redirects can be + // provided as a default by the script environment implementation + // which the parser is not aware of at the time of parsing the command + // line. + // + if (mo && me) + fail (ll) << "stdout and stderr redirected to each other"; + auto_fd& self (mo ? ofd.out : efd); - auto_fd& other (mo ? efd : ofd.out); + auto_fd& other (mo ? efd : ofd.out); try { @@ -1340,14 +2333,419 @@ namespace build2 } } - // All descriptors should be open to the date. + // By now all descriptors should be open. + // + assert (ofd.out != nullfd && efd != nullfd); + + pc.isp = &isp; + pc.osp = &osp; + pc.esp = &esp; + + // Read out all the pipeline's buffered strerr streams watching for the + // deadline, if specified. If the deadline is reached, then terminate + // the whole pipeline, move the deadline by another 2 seconds, and + // continue reading. + // + // Note that we assume that this timeout increment is normally + // sufficient to read out the buffered data written by the already + // terminated processes. If, however, that's not the case (see + // pipe_command for the possible reasons), then we just set + // unread_stderr flag to true for such commands and bail out. + // + // Also note that this is a reduced version of the above read() function. + // + auto read_pipe = [&pc, &ll, &trace] () + { + fdselect_set fds; + for (pipe_command* c (&pc); c != nullptr; c = c->prev) + { + diag_buffer& b (c->dbuf); + + if (b.is.is_open ()) + fds.emplace_back (b.is.fd (), c); + } + + // Note that the current command deadline is the earliest (see above). + // + optional<timestamp> dlt (pc.dl ? pc.dl->value : optional<timestamp> ()); + + bool terminated (false); + + for (size_t unread (fds.size ()); unread != 0;) + { + try + { + // If a deadline is specified, then pass the timeout to fdselect(). + // + if (dlt) + { + timestamp now (system_clock::now ()); + + if (*dlt <= now || ifdselect (fds, *dlt - now) == 0) + { + if (!terminated) + { + term_pipe (&pc, trace); + terminated = true; + + dlt = system_clock::now () + chrono::seconds (2); + continue; + } + else + { + for (fdselect_state& s: fds) + { + if (s.fd != nullfd) + { + pipe_command* c (static_cast<pipe_command*> (s.data)); + + c->unread_stderr = true; + + // Let's also close the stderr stream not to confuse + // diag_buffer::close() (see read() for details). + // + try + { + c->dbuf.is.close (); + } + catch (const io_error&) {} + } + } + + break; + } + } + } + else + ifdselect (fds); + + for (fdselect_state& s: fds) + { + if (s.ready && + !static_cast<pipe_command*> (s.data)->dbuf.read ()) + { + s.fd = nullfd; + --unread; + } + } + } + catch (const io_error& e) + { + fail (ll) << "io error reading pipeline streams: " << e; + } + } + }; + + // Wait for the pipeline processes and builtins to complete, watching + // for their deadlines if present. If a deadline is reached for any of + // them, then terminate the whole pipeline. // - assert (ofd.out.get () != -1 && efd.get () != -1); + // Note: must be called after read_pipe(). + // + auto wait_pipe = [&pc, &dl, &trace] () + { + for (pipe_command* c (&pc); c != nullptr; c = c->prev) + { + try + { + if (process* p = c->proc) + { + if (!dl) + p->wait (); + else if (!timed_wait (*p, dl->value)) + term_pipe (c, trace); + } + else + { + builtin* b (c->bltn); - optional<process_exit> exit; - const builtin_info* bi (resolve - ? builtins.find (program) - : nullptr); + if (!dl) + b->wait (); + else if (!timed_wait (*b, dl->value)) + term_pipe (c, trace); + } + } + catch (const process_error& e) + { + fail (c->loc) << "unable to wait " << cmd_path (c->cmd) << ": " + << e; + } + } + }; + + // Iterate over the pipeline processes and builtins left to right, + // printing their stderr if buffered and issuing the diagnostics if the + // exit code is not available (terminated abnormally or due to a + // deadline), is unexpected, or stdout and/or stderr was not fully + // read. Throw failed at the end if the exit code for any of them is not + // available or stdout and/or stderr was not fully read. Return false if + // exit code for any of them is unexpected (the return is used, for + // example, in the if-conditions). + // + // Note: must be called after wait_pipe() and only once. + // + auto complete_pipe = [&pc, &env, diag] () + { + bool r (true); + bool fail (false); + + pipe_command* c (pc.next); // Left-most command. + assert (c != nullptr); // Since the lambda must be called once. + + for (pc.next = nullptr; c != nullptr; c = c->next) + { + // Collect the exit status, if present. + // + // Absent if the process/builtin misses the "unsuccessful" deadline. + // + optional<process_exit> exit; + + const char* w (c->bltn != nullptr ? "builtin" : "process"); + + if (c->bltn != nullptr) + { + // Note that this also handles ad hoc termination (without the + // call to term_pipe()) by the sleep builtin. + // + if (c->terminated) + { + if (c->dl && c->dl->success) + exit = process_exit (0); + } + else + exit = process_exit (c->bltn->wait ()); + + c->bltn = nullptr; + } + else if (c->proc != nullptr) + { + const process& pr (*c->proc); + +#ifndef _WIN32 + if (c->terminated && + !pr.exit->normal () && + pr.exit->signal () == SIGTERM) +#else + if (c->terminated && + !pr.exit->normal () && + pr.exit->status == DBG_TERMINATE_PROCESS) +#endif + { + if (c->dl && c->dl->success) + exit = process_exit (0); + } + else + exit = pr.exit; + + c->proc = nullptr; + } + else + assert (false); // The lambda can only be called once. + + const command& cmd (c->cmd); + const location& ll (c->loc); + + // Verify the exit status and issue the diagnostics on failure. + // + diag_record dr; + + path pr (cmd_path (cmd)); + + // Print the diagnostics if the command stdout and/or stderr are not + // fully read. + // + auto unread_output_diag = [&dr, c, w, &pr] (bool main_error) + { + if (main_error) + dr << error (c->loc) << w << ' ' << pr << ' '; + else + dr << error; + + if (c->unread_stdout) + { + dr << "stdout "; + + if (c->unread_stderr) + dr << "and "; + } + + if (c->unread_stderr) + dr << "stderr "; + + dr << "not closed after exit"; + }; + + // Fail if the process is terminated due to reaching the deadline. + // + if (!exit) + { + dr << error (ll) << w << ' ' << pr + << " terminated: execution timeout expired"; + + if (c->unread_stdout || c->unread_stderr) + unread_output_diag (false /* main_error */); + + if (verb == 1) + { + dr << info << "command line: "; + print_process (dr, *c->args); + } + + fail = true; + } + else + { + // If there is no valid exit code available by whatever reason + // then we print the proper diagnostics, dump stderr (if cached + // and not too large) and fail the whole script. Otherwise if the + // exit code is not correct then we print diagnostics if requested + // and fail the pipeline. + // + bool valid (exit->normal ()); + + // On Windows the exit code can be out of the valid codes range + // being defined as uint16_t. + // +#ifdef _WIN32 + if (valid) + valid = exit->code () < 256; +#endif + + // In the presense of a valid exit code and given stdout and + // stderr are fully read out we print the diagnostics and return + // false rather than throw. + // + // Note that there can be a race, so that the process we have + // terminated due to reaching the deadline has in fact exited + // normally. Thus, the 'unread stderr' situation can also happen + // to a successfully terminated process. If that's the case, we + // report this problem as the main error and the secondary error + // otherwise. + // + if (!valid || c->unread_stdout || c->unread_stderr) + fail = true; + + exit_comparison cmp (cmd.exit + ? cmd.exit->comparison + : exit_comparison::eq); + + uint16_t exc (cmd.exit ? cmd.exit->code : 0); + + bool success (valid && + (cmp == exit_comparison::eq) == + (exc == exit->code ())); + + if (!success) + r = false; + + if (!valid || (!success && diag)) + { + dr << error (ll) << w << ' ' << pr << ' '; + + if (!exit->normal ()) + dr << *exit; + else + { + uint16_t ec (exit->code ()); // Make sure printed as integer. + + if (!valid) + { + dr << "exit code " << ec << " out of 0-255 range"; + } + else + { + if (cmd.exit) + dr << "exit code " << ec + << (cmp == exit_comparison::eq ? " != " : " == ") + << exc; + else + dr << "exited with code " << ec; + } + } + + if (c->unread_stdout || c->unread_stderr) + unread_output_diag (false /* main_error */); + + if (verb == 1) + { + dr << info << "command line: "; + print_process (dr, *c->args); + } + + if (non_empty (*c->esp, ll) && avail_on_failure (*c->esp, env)) + dr << info << "stderr: " << *c->esp; + + if (non_empty (*c->osp, ll) && avail_on_failure (*c->osp, env)) + dr << info << "stdout: " << *c->osp; + + if (non_empty (*c->isp, ll) && avail_on_failure (*c->isp, env)) + dr << info << "stdin: " << *c->isp; + + // Print cached stderr. + // + print_file (dr, *c->esp, ll); + } + else if (c->unread_stdout || c->unread_stderr) + unread_output_diag (true /* main_error */); + } + + // Now print the buffered stderr, if present, and/or flush the + // diagnostics, if issued. + // + if (c->dbuf.is_open ()) + c->dbuf.close (move (dr)); + } + + // Fail if required. + // + if (fail) + throw failed (); + + return r; + }; + + // Close all buffered pipeline stderr streams ignoring io_error + // exceptions. + // + auto close_pipe = [&pc] () + { + for (pipe_command* c (&pc); c != nullptr; c = c->prev) + { + if (c->dbuf.is.is_open ()) + try + { + c->dbuf.is.close(); + } + catch (const io_error&) {} + } + }; + + // Derive the process/builtin CWD. + // + // If the process/builtin CWD is specified via the env pseudo-builtin, + // then use that, completing it relative to the script environment work + // directory, if it is relative. Otherwise, use the script environment + // work directory. + // + dir_path completed_cwd; + if (c.cwd && c.cwd->relative ()) + completed_cwd = wdir / *c.cwd; + + const dir_path& cwd (!completed_cwd.empty () ? completed_cwd : + c.cwd ? *c.cwd : + wdir); + + // Unless CWD is the script environment work directory (which always + // exists), verify that it exists and fail if it doesn't. + // + if (&cwd != &wdir && !exists (cwd)) + fail (ll) << "specified working directory " << cwd + << " does not exist"; + + cstrings args (process_args ()); + pc.args = &args; + + const builtin_info* bi (resolve ? builtins.find (program) : nullptr); bool success; @@ -1355,8 +2753,11 @@ namespace build2 { // Execute the builtin. // - if (verb >= 2) - print_process (process_args ()); + // Don't print the true and false builtins, since they are normally + // used for the commands execution flow control. + // + if (verb >= 2 && program != "true" && program != "false") + print_process (args); // Some of the script builtins (cp, mkdir, etc) extend libbutl // builtins (via callbacks) registering/moving cleanups for the @@ -1394,6 +2795,9 @@ namespace build2 if (cleanup_builtin (program)) cln = cleanup (); + // We also extend the sleep builtin, deactivating the thread before + // going to sleep and waking up before the deadline is reached. + // builtin_callbacks bcs { // create @@ -1555,14 +2959,32 @@ namespace build2 // sleep // - // Deactivate the thread before going to sleep. - // - [&env] (const duration& d) + [&env, &pc] (const duration& d) { + duration t (d); + const optional<timestamp>& dl (pc.dl + ? pc.dl->value + : optional<timestamp> ()); + + if (dl) + { + timestamp now (system_clock::now ()); + + if (now + t > *dl) + pc.terminated = true; + + if (*dl <= now) + return; + + duration d (*dl - now); + if (t > d) + t = d; + } + // If/when required we could probably support the precise sleep // mode (e.g., via an option). // - env.context.sched.sleep (d); + env.context.sched->sleep (t); } }; @@ -1572,16 +2994,46 @@ namespace build2 builtin b (bi->function (r, c.arguments, move (ifd), move (ofd.out), move (efd), - *env.work_dir.path, + cwd, bcs)); + pc.bltn = &b; + + // If the right-hand part of the pipe fails, then make sure we don't + // wait indefinitely in the process destructor if the deadlines are + // specified or just because a process is blocked on stderr. + // + auto g (make_exception_guard ([&pc, &close_pipe, &trace] () + { + if (pc.bltn != nullptr) + try + { + close_pipe (); + term_pipe (&pc, trace); + } + catch (const failed&) + { + // We can't do much here. + } + })); success = run_pipe (env, - nc, - ec, + nc, ec, move (ofd.in), - ci + 1, li, ll, diag); + ii, li, ci + 1, ll, diag, + cf, last_cmd, + dl, + &pc); - exit = process_exit (b.wait ()); + // Complete the pipeline execution, if not done yet. + // + if (pc.bltn != nullptr) + { + read_pipe (); + wait_pipe (); + + if (!complete_pipe ()) + success = false; + } } catch (const system_error& e) { @@ -1593,8 +3045,6 @@ namespace build2 { // Execute the process. // - cstrings args (process_args ()); - // If the process path is not pre-searched then resolve the relative // non-simple program path against the script's working directory. The // simple one will be left for the process path search machinery. Also @@ -1626,7 +3076,7 @@ namespace build2 program (path (s, 1, s.size () - 1)); } else - program (*env.work_dir.path / p); + program (wdir / p); } } catch (const invalid_path& e) @@ -1640,33 +3090,75 @@ namespace build2 ? process::path_search (args[0]) : process_path ()); + environment_vars vss; + const environment_vars& vs ( + env.merge_exported_variables (c.variables, vss)); + // Note that CWD and builtin-escaping character '^' are not printed. // - process_env pe (resolve ? pp : c.program, c.variables); + const small_vector<string, 4>& evars (vs); + process_env pe (resolve ? pp : c.program, evars); if (verb >= 2) print_process (pe, args); + // Note that stderr can only be a pipe if we are buffering the + // diagnostics. In this case also pass the reading end so it can be + // "probed" on Windows (see butl::process::pipe for details). + // process pr ( *pe.path, args.data (), - {ifd.get (), -1}, process::pipe (ofd), {-1, efd.get ()}, - env.work_dir.path->string ().c_str (), + {ifd.get (), -1}, + process::pipe (ofd), + {pc.dbuf.is.fd (), efd.get ()}, + cwd.string ().c_str (), pe.vars); + // Can't throw. + // ifd.reset (); ofd.out.reset (); efd.reset (); + pc.proc = ≺ + + // If the right-hand part of the pipe fails, then make sure we don't + // wait indefinitely in the process destructor (see above for + // details). + // + auto g (make_exception_guard ([&pc, &close_pipe, &trace] () + { + if (pc.proc != nullptr) + try + { + close_pipe (); + term_pipe (&pc, trace); + } + catch (const failed&) + { + // We can't do much here. + } + })); + success = run_pipe (env, - nc, - ec, + nc, ec, move (ofd.in), - ci + 1, li, ll, diag); + ii, li, ci + 1, ll, diag, + cf, last_cmd, + dl, + &pc); - pr.wait (); + // Complete the pipeline execution, if not done yet. + // + if (pc.proc != nullptr) + { + read_pipe (); + wait_pipe (); - exit = move (pr.exit); + if (!complete_pipe ()) + success = false; + } } catch (const process_error& e) { @@ -1679,98 +3171,23 @@ namespace build2 } } - assert (exit); - - // If the righ-hand side pipeline failed than the whole pipeline fails, - // and no further checks are required. - // - if (!success) - return false; - - // Use the program path for diagnostics (print relative, etc). - // - const path& pr (resolve - ? c.program.recall - : path (c.program.recall_string ())); // Can't throw. - - // If there is no valid exit code available by whatever reason then we - // print the proper diagnostics, dump stderr (if cached and not too - // large) and fail the whole script. Otherwise if the exit code is not - // correct then we print diagnostics if requested and fail the pipeline. - // - bool valid (exit->normal ()); - - // On Windows the exit code can be out of the valid codes range being - // defined as uint16_t. - // -#ifdef _WIN32 - if (valid) - valid = exit->code () < 256; -#endif - - exit_comparison cmp (c.exit ? c.exit->comparison : exit_comparison::eq); - uint16_t exc (c.exit ? c.exit->code : 0); - - success = valid && - (cmp == exit_comparison::eq) == (exc == exit->code ()); - - if (!valid || (!success && diag)) - { - // In the presense of a valid exit code we print the diagnostics and - // return false rather than throw. - // - diag_record d (valid ? error (ll) : fail (ll)); - - if (!exit->normal ()) - d << pr << " " << *exit; - else - { - uint16_t ec (exit->code ()); // Make sure is printed as integer. - - if (!valid) - d << pr << " exit code " << ec << " out of 0-255 range"; - else if (!success) - { - if (diag) - { - if (c.exit) - d << pr << " exit code " << ec - << (cmp == exit_comparison::eq ? " != " : " == ") << exc; - else - d << pr << " exited with code " << ec; - } - } - else - assert (false); - } - - if (non_empty (esp, ll) && avail_on_failure (esp, env)) - d << info << "stderr: " << esp; - - if (non_empty (osp, ll) && avail_on_failure (osp, env)) - d << info << "stdout: " << osp; - - if (non_empty (isp, ll) && avail_on_failure (isp, env)) - d << info << "stdin: " << isp; - - // Print cached stderr. - // - print_file (d, esp, ll); - } - - // If exit code is correct then check if the standard outputs match the - // expectations. Note that stdout is only redirected to file for the - // last command in the pipeline. + // If the pipeline or the righ-hand side outputs check failed, then no + // further checks are required. Otherwise, check if the standard outputs + // match the expectations. Note that stdout can only be redirected to + // file for the last command in the pipeline. // // The thinking behind matching stderr first is that if it mismatches, // then the program probably misbehaves (executes wrong functionality, // etc) in which case its stdout doesn't really matter. // if (success) - success = - check_output (pr, esp, isp, err, ll, env, diag, "stderr") && - (!last || - check_output (pr, osp, isp, *out, ll, env, diag, "stdout")); + { + path pr (cmd_path (c)); + + success = check_output (pr, esp, isp, err, ll, env, diag, "stderr") && + (out == nullptr || + check_output (pr, osp, isp, *out, ll, env, diag, "stdout")); + } return success; } @@ -1778,8 +3195,10 @@ namespace build2 static bool run_expr (environment& env, const command_expr& expr, - size_t li, const location& ll, - bool diag) + const iteration_index* ii, size_t li, + const location& ll, + bool diag, + const function<command_function>& cf, bool last_cmd) { // Commands are numbered sequentially throughout the expression // starting with 1. Number 0 means the command is a single one. @@ -1794,7 +3213,7 @@ namespace build2 // pipe that "switches on" the diagnostics potential printing. // command_expr::const_iterator trailing_ands; // Undefined if diag is - // disallowed. + // disallowed. if (diag) { auto i (expr.crbegin ()); @@ -1817,8 +3236,15 @@ namespace build2 // with false. // if (!((or_op && r) || (!or_op && !r))) - r = run_pipe ( - env, p.begin (), p.end (), auto_fd (), ci, li, ll, print); + { + assert (!p.empty ()); + + r = run_pipe (env, + p.begin (), p.end (), + auto_fd (), + ii, li, ci, ll, print, + cf, last_cmd); + } ci += p.size (); } @@ -1829,42 +3255,118 @@ namespace build2 void run (environment& env, const command_expr& expr, - size_t li, const location& ll) + const iteration_index* ii, size_t li, + const location& ll, + const function<command_function>& cf, + bool last_cmd) { // Note that we don't print the expression at any verbosity level // assuming that the caller does this, potentially providing some // additional information (command type, etc). // - if (!run_expr (env, expr, li, ll, true /* diag */)) + if (!run_expr (env, + expr, + ii, li, ll, + true /* diag */, + cf, last_cmd)) throw failed (); // Assume diagnostics is already printed. } bool - run_if (environment& env, - const command_expr& expr, - size_t li, const location& ll) + run_cond (environment& env, + const command_expr& expr, + const iteration_index* ii, size_t li, + const location& ll, + const function<command_function>& cf, bool last_cmd) { // Note that we don't print the expression here (see above). // - return run_expr (env, expr, li, ll, false /* diag */); + return run_expr (env, + expr, + ii, li, ll, + false /* diag */, + cf, last_cmd); } void clean (environment& env, const location& ll) { - context& ctx (env.context); + // We don't use the build2 filesystem utilities here in order to remove + // the filesystem entries regardless of the dry-run mode and also to add + // the location info to diagnostics. Other than that, these lambdas + // implement the respective utility functions semantics. + // + auto rmfile = [&ll] (const path& f) + { + try + { + rmfile_status r (try_rmfile (f)); + + if (r == rmfile_status::success && verb >= 3) + text << "rm " << f; + + return r; + } + catch (const system_error& e) + { + fail (ll) << "unable to remove file " << f << ": " << e << endf; + } + }; + + auto rmdir = [&ll] (const dir_path& d) + { + try + { + rmdir_status r (!work.sub (d) + ? try_rmdir (d) + : rmdir_status::not_empty); + + if (r == rmdir_status::success && verb >= 3) + text << "rmdir " << d; + + return r; + } + catch (const system_error& e) + { + fail (ll) << "unable to remove directory " << d << ": " << e << endf; + } + }; + + auto rmdir_r = [&ll] (const dir_path& d, bool dir) + { + if (work.sub (d)) // Don't try to remove working directory. + return rmdir_status::not_empty; + + if (!build2::entry_exists (d)) + return rmdir_status::not_exist; + + try + { + butl::rmdir_r (d, dir); + } + catch (const system_error& e) + { + fail (ll) << "unable to remove directory " << d << ": " << e << endf; + } + + if (verb >= 3) + text << "rmdir -r " << d; + + return rmdir_status::success; + }; + const dir_path& wdir (*env.work_dir.path); // Note that we operate with normalized paths here. // - // Remove special files. The order is not important as we don't - // expect directories here. + // Remove special files. The order is not important as we don't expect + // directories here. // for (const path& p: env.special_cleanups) { // Remove the file if exists. Fail otherwise. // - if (rmfile (ctx, p, 3) == rmfile_status::not_exist) + if (rmfile (p) == rmfile_status::not_exist) fail (ll) << "registered for cleanup special file " << p << " does not exist"; } @@ -1885,9 +3387,9 @@ namespace build2 // Wildcard with the last component being '***' (without trailing // separator) matches all files and sub-directories recursively as - // well as the start directories itself. So we will recursively - // remove the directories that match the parent (for the original - // path) directory wildcard. + // well as the start directories itself. So we will recursively remove + // the directories that match the parent (for the original path) + // directory wildcard. // bool recursive (cp.leaf ().representation () == "***"); const path& p (!recursive ? cp : cp.directory ()); @@ -1898,15 +3400,19 @@ namespace build2 { bool removed (false); - auto rm = [&cp, recursive, &removed, &ll, &ctx, &wdir] + auto rm = [&cp, + recursive, + &removed, + &ll, + &wdir, + &rmfile, &rmdir, &rmdir_r] (path&& pe, const string&, bool interm) { if (!interm) { - // While removing the entry we can get not_exist due to - // racing conditions, but that's ok if somebody did our job. - // Note that we still set the removed flag to true in this - // case. + // While removing the entry we can get not_exist due to racing + // conditions, but that's ok if somebody did our job. Note that + // we still set the removed flag to true in this case. // removed = true; // Will be meaningless on failure. @@ -1916,7 +3422,7 @@ namespace build2 if (!recursive) { - rmdir_status r (rmdir (ctx, d, 3)); + rmdir_status r (rmdir (d)); if (r != rmdir_status::not_empty) return true; @@ -1930,13 +3436,10 @@ namespace build2 } else { - // Don't remove the working directory (it will be removed - // by the dedicated cleanup). + // Don't remove the working directory (it will be removed by + // the dedicated cleanup). // - // Cast to uint16_t to avoid ambiguity with - // libbutl::rmdir_r(). - // - rmdir_status r (rmdir_r (ctx, d, d != wdir, 3)); + rmdir_status r (rmdir_r (d, d != wdir)); if (r != rmdir_status::not_empty) return true; @@ -1949,14 +3452,14 @@ namespace build2 } } else - rmfile (ctx, pe, 3); + rmfile (pe); } return true; }; - // Note that here we rely on the fact that recursive iterating - // goes depth-first (which make sense for the cleanup). + // Note that here we rely on the fact that recursive iterating goes + // depth-first (which make sense for the cleanup). // try { @@ -1987,9 +3490,8 @@ namespace build2 : "file"); } - // Remove the directory if exists and empty. Fail otherwise. - // Removal of non-existing directory is not an error for 'maybe' - // cleanup type. + // Remove the directory if exists and empty. Fail otherwise. Removal + // of non-existing directory is not an error for 'maybe' cleanup type. // if (p.to_directory ()) { @@ -1997,20 +3499,10 @@ namespace build2 bool wd (d == wdir); // Don't remove the working directory for the recursive cleanup - // (it will be removed by the dedicated one). - // - // Note that the root working directory contains the - // .buildignore file (see above). - // - // @@ If 'd' is a file then will fail with a diagnostics having - // no location info. Probably need to add an optional location - // parameter to rmdir() function. The same problem exists for - // a file cleanup when try to rmfile() directory instead of - // file. + // since it needs to be removed by the caller (can contain + // .buildignore file, etc). // - rmdir_status r (recursive - ? rmdir_r (ctx, d, !wd, static_cast <uint16_t> (3)) - : rmdir (ctx, d, 3)); + rmdir_status r (recursive ? rmdir_r (d, !wd) : rmdir (d)); if (r == rmdir_status::success || (r == rmdir_status::not_exist && t == cleanup_type::maybe)) @@ -2026,10 +3518,10 @@ namespace build2 print_dir (dr, d, ll); } - // Remove the file if exists. Fail otherwise. Removal of - // non-existing file is not an error for 'maybe' cleanup type. + // Remove the file if exists. Fail otherwise. Removal of non-existing + // file is not an error for 'maybe' cleanup type. // - if (rmfile (ctx, p, 3) == rmfile_status::not_exist && + if (rmfile (p) == rmfile_status::not_exist && t == cleanup_type::always) fail (ll) << "registered for cleanup file " << p << " does not exist"; @@ -2042,8 +3534,7 @@ namespace build2 try { size_t n (0); - for (const dir_entry& de: dir_iterator (p, - false /* ignore_dangling */)) + for (const dir_entry& de: dir_iterator (p, dir_iterator::no_follow)) { if (n++ < 10) dr << '\n' << (de.ltype () == entry_type::directory |