From f41599c8e9435f3dfec60b872c2b4ae31177efdd Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Sat, 10 Oct 2020 17:22:46 +0300 Subject: Add support for test timeouts --- tests/recipe/buildscript/testscript | 120 +++++++ tests/test/script/builtin/sleep.testscript | 18 + tests/test/script/runner/driver.cxx | 30 +- tests/test/script/runner/env.testscript | 29 +- tests/test/script/runner/set.testscript | 118 ++++++- tests/test/script/runner/timeout.testscript | 503 ++++++++++++++++++++++++++++ tests/test/simple/generated/buildfile | 3 +- tests/test/simple/generated/driver.cxx | 32 +- tests/test/simple/generated/testscript | 57 ++++ 9 files changed, 888 insertions(+), 22 deletions(-) create mode 100644 tests/test/script/runner/timeout.testscript (limited to 'tests') diff --git a/tests/recipe/buildscript/testscript b/tests/recipe/buildscript/testscript index 551f64a..787bafd 100644 --- a/tests/recipe/buildscript/testscript +++ b/tests/recipe/buildscript/testscript @@ -107,3 +107,123 @@ $* clean 2>- } + +: timeout +: +if ($cxx.target.class != 'windows') +{ + : update + : + { + : expired + : + { + echo 'bar' >=bar; + + cat <=buildfile; + foo: bar + % [diag=update] + {{ + cp $path($<) $path($>) + timeout 1 + ^sleep 5 + }} + EOI + + $* 2>>~%EOE% != 0; + update file{foo} + buildfile:6:3: error: ^sleep terminated: execution timeout expired + info: while updating file{foo} + %.+ + EOE + + $* clean 2>- + } + + : successful-timeout + : + { + echo 'bar' >=bar; + + cat <=buildfile; + foo: bar + % [diag=update] + {{ + cp $path($<) $path($>) + timeout --success 1 + ^sleep 5 + }} + EOI + + $* 2>>EOE; + update file{foo} + EOE + + $* clean 2>- + } + } + + : test + : + { + : expired + : + { + echo 'bar' >=bar; + + cat <=buildfile; + foo: bar + {{ + cp $path($<) $path($>) + }} + % [diag=test] test + {{ + ^sleep 5 + }} + EOI + + $* test config.test.timeout=1 2>>~%EOE% != 0; + cp file{foo} + test file{foo} + buildfile:7:3: error: ^sleep terminated: execution timeout expired + info: while testing file{foo} + %.+ + EOE + + $* test config.test.timeout=/1 2>>~%EOE% != 0; + test file{foo} + buildfile:7:3: error: ^sleep terminated: execution timeout expired + info: while testing file{foo} + %.+ + EOE + + $* clean 2>- + } + + : not-expired + : + { + echo 'bar' >=bar; + + cat <=buildfile; + foo: bar + % [diag=cp] + {{ + ^sleep 4 + cp $path($<) $path($>) + }} + % [diag=test] test + {{ + ^sleep 1 + }} + EOI + + $* test config.test.timeout=3 2>>EOE; + cp file{foo} + test file{foo} + EOE + + $* clean 2>- + } + } +} diff --git a/tests/test/script/builtin/sleep.testscript b/tests/test/script/builtin/sleep.testscript index 21ed07b..e1410ac 100644 --- a/tests/test/script/builtin/sleep.testscript +++ b/tests/test/script/builtin/sleep.testscript @@ -6,3 +6,21 @@ : success : $c <'sleep 1' && $b + +: timeout +: +{ + : failure + : + $c <'env -t 1 -- sleep 86400' && $b 2>>~%EOE% != 0 + %testscript:.*: error: sleep terminated: execution timeout expired% + %. + EOE + + : success + : + $c < // this_thread::sleep_for() +# include +#else +# include +#endif + #include // numeric_limits #include #include // abort() @@ -36,10 +43,10 @@ main (int argc, char* argv[]) // Usage: driver [-i ] (-o )* (-e )* (-f )* // (-d )* (-v )* [(-t (a|m|s|z)) | (-s )] // - // Execute actions specified by -i, -o, -e, -f, -d, and -v options in the - // order as they appear on the command line. After that terminate abnormally - // if -t option is provided, otherwise exit normally with the status - // specified by -s option (0 by default). + // Execute actions specified by -i, -o, -e, -f, -d, -v, and -l options in + // the order as they appear on the command line. After that terminate + // abnormally if -t option is provided, otherwise exit normally with the + // status specified by -s option (0 by default). // // -i // Forward stdin data to the standard stream denoted by the file @@ -62,6 +69,9 @@ main (int argc, char* argv[]) // If the specified variable is set the print its value to stdout and the // string '' otherwise. // + // -l + // Sleep the specified number of seconds. + // // -t // Abnormally terminate itself using one of the following methods: // @@ -144,6 +154,18 @@ main (int argc, char* argv[]) optional var (getenv (v)); cout << (var ? *var : "") << endl; } + else if (o == "-l") + { + size_t t (toi (v)); + + // MinGW GCC 4.9 doesn't implement this_thread so use Win32 Sleep(). + // +#ifndef _WIN32 + this_thread::sleep_for (chrono::seconds (t)); +#else + Sleep (static_cast (t * 1000)); +#endif + } else if (o == "-t") { assert (aterm == '\0' && !status); // Make sure exit method is not set. diff --git a/tests/test/script/runner/env.testscript b/tests/test/script/runner/env.testscript index 6fcedfa..ef90c3b 100644 --- a/tests/test/script/runner/env.testscript +++ b/tests/test/script/runner/env.testscript @@ -3,5 +3,30 @@ .include ../common.testscript -$c <'env abc=xyz -- $* -v abc >xyz' && $b : set -$c <'env --unset=abc -- $* -v abc >""' && env abc=xyz -- $b : unset +: variables +: +{ + $c <'env abc=xyz -- $* -v abc >xyz' && $b : set + $c <'env --unset=abc -- $* -v abc >""' && env abc=xyz -- $b : unset +} + +: timeout +: +{ + : expired + : + $c <'env --timeout 1 -- $* -l 5' && $b 2>>~%EOE% != 0 + %testscript:1:1: error: .+ terminated: execution timeout expired% + info: test id: 1 + EOE + + : not-expired + : + $c <'env --timeout 5 -- $* -l 1' && $b + + : invalid + : + $c <'env --timeout a -- $*' && $b 2>>EOE != 0 + testscript:1:15: error: env: invalid value 'a' for option '--timeout' + EOE +} diff --git a/tests/test/script/runner/set.testscript b/tests/test/script/runner/set.testscript index 9219cbb..41709e4 100644 --- a/tests/test/script/runner/set.testscript +++ b/tests/test/script/runner/set.testscript @@ -94,10 +94,28 @@ { : non-exact : - $c <'foo bar' - EOI + { + : non-empty + : + $c <'"foo" "bar"' + EOI + + : empty + : + $c <'' + EOI + + : spaces + : + $c <'' + EOI + } : exact : @@ -106,7 +124,7 @@ : $c <'foo bar ' + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar" ""' EOI : no-trailing-ws @@ -115,8 +133,22 @@ : ':' modifier. : $c <'foo bar' + set -e -w baz <:' foo bar'; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar"' + EOI + + : empty + : + $c <'' + EOI + + : spaces + : + $c <'""' EOI } } @@ -134,7 +166,7 @@ bar EOF - echo $baz >' foo bar ' + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar" ""' EOI : exact @@ -150,7 +182,7 @@ bar EOF - echo $baz >' foo bar ' + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar" "" ""' EOI : no-trailing-newline @@ -162,7 +194,7 @@ bar EOF - echo $baz >' foo bar' + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar"' EOI } } @@ -180,7 +212,7 @@ bar EOF - echo $baz >>EOO + echo ($baz[0]) >>EOO foo @@ -209,7 +241,7 @@ bar EOF - echo "$baz" >>EOO + echo ($baz[0]) >>EOO foo @@ -227,7 +259,7 @@ bar EOF - echo "$baz" >>EOO + echo ($baz[0]) >>EOO foo @@ -237,6 +269,66 @@ } } +: deadline +: +{ + : not-reached + : + $c <foo 2>| + EOI + + : set-reached + : + $c <>~%EOE% != 0 + $* -o 'foo' -l 10 | env -t 1 -- set bar + EOI + %testscript:.*: error: set terminated: execution timeout expired% + %. + EOE + + : driver-reached + : + $c <>~%EOE% != 0 + env -t 1 -- $* -o 'foo' -l 10 | set bar + EOI + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : read-some-data + : + { + s="----------------------------------------------------------------------" + s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s" + s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s" + + : failure + : + $c <>~%EOE% != 0 + echo "$s" >=f; + $* -o 'foo' -l 10 | cat f - | env -t 2 -- set bar + EOI + %testscript:.*: error: set terminated: execution timeout expired% + %. + EOE + + : success + : + : Note that the cat builtin ends up with the 'broken pipe' diagnostics or + : similar. + : + $c <=f; + timeout --success 2; + $* -o 'foo' -l 10 | cat f - 2>>~%EOE% | set bar + %cat: .+% + EOE + EOI + } +} + : attributes : { diff --git a/tests/test/script/runner/timeout.testscript b/tests/test/script/runner/timeout.testscript new file mode 100644 index 0000000..ae8f535 --- /dev/null +++ b/tests/test/script/runner/timeout.testscript @@ -0,0 +1,503 @@ +# file : tests/test/script/runner/timeout.testscript +# license : MIT; see accompanying LICENSE file + +.include ../common.testscript + +: test +: +{ + : fragment-timeout + : + { + : set + : + $c <>~%EOE% != 0 + timeout 1; + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout /10 + + { + +timeout /10 + + timeout 1; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : successful + : + $c <>~%EOE% != 0 + testscript:1:1: error: missing timeout + %. + EOE + + : invalid + : + $c <'timeout foo' && $b 2>>~%EOE% != 0 + testscript:1:1: error: invalid test fragment timeout 'foo' + %. + EOE +} + +: group +: +{ + : group-timeout + : + { + : set + : + $c <>~%EOE% != 0 + { + +timeout 1 + + $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 10/10 + + { + +timeout 1/ + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c <>~%EOE% != 0 + { + +timeout foo/ + } + EOI + testscript:2:4: error: invalid test group timeout 'foo' + %. + EOE + } + + : test-timeout + : + { + : set + : + $c <>~%EOE% != 0 + { + +timeout /1 + + $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 10/10 + + { + +timeout /1 + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c <>~%EOE% != 0 + { + +timeout /foo + } + EOI + testscript:2:4: error: invalid test timeout 'foo' + %. + EOE + } +} + +: script +: +{ + : group-timeout + : + { + : set + : + $c <>~%EOE% != 0 + +timeout 1 + + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 1 + + { + +timeout 10 + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c <>~%EOE% != 0 + +timeout foo/ + EOI + testscript:1:2: error: invalid testscript timeout 'foo' + %. + EOE + + : successful + : + $c <>~%EOE% != 0 + +timeout /1 + + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout /1 + + { + +timeout --success /1 + + { + +timeout 10/10 + + timeout 10; + env -t 10 -- $* -l 3 + } + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : successful + : + $c <>~%EOE% != 0 + { + +timeout /foo + } + EOI + testscript:2:4: error: invalid test timeout 'foo' + %. + EOE + } +} + +: config +: +{ + : operation + : + { + : set + : + $c <>~%EOE% != 0 + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 10 + + { + +timeout 10/10 + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c && $b config.test.timeout=foo 2>>EOE != 0 + error: invalid config.test.timeout test operation timeout value 'foo' + EOE + } + + : test + : + { + : set + : + $c <>~%EOE% != 0 + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 10 + + { + +timeout 10/10 + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c && $b config.test.timeout=/foo 2>>EOE != 0 + error: invalid config.test.timeout test timeout value 'foo' + EOE + } +} + +: failures +: +: Here we test that the left-hand side processes are terminated on failure. +: +{ + : set + : + $c <>~%EOE% != 0 + env -t 1 -- $* -l 86400 -o 'foo' | set --foo bar + EOI + %testscript:.*: error: set: unknown option '--foo'% + %. + EOE + + : exit + : + $c <>~%EOE% != 0 + env -t 1 -- $* -l 86400 -o 'foo' | exit 0 + EOI + %testscript:.*: error: exit builtin must be the only pipe command% + %. + EOE + + : redirect + : + $c <>~%EOE% != 0 + env -t 1 -- $* -l 86400 -o 'foo' | touch $~/foo/bar + EOI + %testscript:.*: error: touch exited with code 1% + %.+ + EOE +} + +: pipeline +: +{ + : prog-tm-prog + : + $c <'$* -l 10 | env -t 1 -- $* -i 0' && $b 2>>~%EOE% != 0 + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : tm-prog-prog + : + $c <'env -t 1 -- $* -l 10 | $* -i 0' && $b 2>>~%EOE% != 0 + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : tm-cat-prog + : + $c <'env -t 1 -- cat <"test" | $* -l 10' && $b 2>>~%EOE% != 0 + %testscript:.*: error: cat terminated: execution timeout expired% + %. + EOE + + : cat-tm-prog + : + $c <'cat <"test" | env -t 1 -- $* -l 10' && $b 2>>~%EOE% != 0 + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : tm-prog-cat + : + $c <'env -t 1 -- $* -l 10 | cat >-' && $b 2>>~%EOE% != 0 + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : tm-echo-prog + : + $c <'env -t 1 -- echo "test" | $* -l 10' && $b 2>>~%EOE% != 0 + %testscript:.*: error: echo terminated: execution timeout expired% + %. + EOE + + : successful + : + { + : prog-prog + : + $c <>~%EOE% | $* -l 10 -i 0 + %cat: unable to print stdin: .+% + EOE + EOI + } +} diff --git a/tests/test/simple/generated/buildfile b/tests/test/simple/generated/buildfile index 0809bdf..0344891 100644 --- a/tests/test/simple/generated/buildfile +++ b/tests/test/simple/generated/buildfile @@ -6,4 +6,5 @@ ./: testscript exe{driver} $b file{*.in} -exe{driver}: cxx{driver} +import libs = libbutl%lib{butl} +exe{driver}: cxx{driver} $libs diff --git a/tests/test/simple/generated/driver.cxx b/tests/test/simple/generated/driver.cxx index 1d911df..18fd0ae 100644 --- a/tests/test/simple/generated/driver.cxx +++ b/tests/test/simple/generated/driver.cxx @@ -1,24 +1,52 @@ // file : tests/test/simple/generated/driver.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file +#ifndef _WIN32 +# include +# include +#else +# include +#endif + #include #include #include using namespace std; +// If the -s option is specified, then also sleep for 5 seconds. +// int main (int argc, char* argv[]) { + int i (1); + for (; i != argc; ++i) + { + string a (argv[i]); + + if (a == "-s") + { + // MINGW GCC 4.9 doesn't implement this_thread so use Win32 Sleep(). + // +#ifndef _WIN32 + this_thread::sleep_for (chrono::seconds (5)); +#else + Sleep (5000); +#endif + } + else + break; + } + int r (0); - if (argc == 1) + if (i == argc) { cout << "1.2.3" << endl; } else { - ifstream ifs (argv[1]); + ifstream ifs (argv[i]); if (!ifs.is_open ()) cerr << "unable to open " << argv[1] << endl; diff --git a/tests/test/simple/generated/testscript b/tests/test/simple/generated/testscript index a04dccc..f6a89d8 100644 --- a/tests/test/simple/generated/testscript +++ b/tests/test/simple/generated/testscript @@ -42,3 +42,60 @@ driver = $src_root/../exe{driver} ./: file{output}: test.stdout = true file{output}: in{output} $src_root/manifest #@@ in module EOI + +: timeout +: +{ + : operation + : + { + : no-output + : + ln -s $src_base/output.in ./; + $* config.test.timeout=1 <>/~%EOE% != 0 + driver = $src_root/../exe{driver} + ./: test = $driver + ./: test.options = -s + ./: $driver + EOI + error: test dir{./} failed + % error: .+ -s terminated: execution timeout expired% + % info: test command line: .+% + EOE + + : output + : + ln -s $src_base/output.in ./; + $* config.test.timeout=1 &output &output.d <>/~%EOE% != 0 + driver = $src_root/../exe{driver} + ./: test = $driver + ./: test.options = -s + ./: $driver + ./: file{output}: test.stdout = true + file{output}: in{output} $src_root/manifest #@@ in module + EOI + error: test dir{./} failed + % error: diff .+ terminated: execution timeout expired% + % error: .+ -s terminated: execution timeout expired% + % info: test command line: .+% + EOE + } + + : test + : + { + : no-output + : + ln -s $src_base/output.in ./; + $* config.test.timeout=/1 <>/~%EOE% != 0 + driver = $src_root/../exe{driver} + ./: test = $driver + ./: test.options = -s + ./: $driver + EOI + error: test dir{./} failed + % error: .+ -s terminated: execution timeout expired% + % info: test command line: .+% + EOE + } +} -- cgit v1.1