From f41599c8e9435f3dfec60b872c2b4ae31177efdd Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Sat, 10 Oct 2020 17:22:46 +0300 Subject: Add support for test timeouts --- tests/test/script/builtin/sleep.testscript | 18 + tests/test/script/runner/driver.cxx | 30 +- tests/test/script/runner/env.testscript | 29 +- tests/test/script/runner/set.testscript | 118 ++++++- tests/test/script/runner/timeout.testscript | 503 ++++++++++++++++++++++++++++ 5 files changed, 679 insertions(+), 19 deletions(-) create mode 100644 tests/test/script/runner/timeout.testscript (limited to 'tests/test/script') diff --git a/tests/test/script/builtin/sleep.testscript b/tests/test/script/builtin/sleep.testscript index 21ed07b..e1410ac 100644 --- a/tests/test/script/builtin/sleep.testscript +++ b/tests/test/script/builtin/sleep.testscript @@ -6,3 +6,21 @@ : success : $c <'sleep 1' && $b + +: timeout +: +{ + : failure + : + $c <'env -t 1 -- sleep 86400' && $b 2>>~%EOE% != 0 + %testscript:.*: error: sleep terminated: execution timeout expired% + %. + EOE + + : success + : + $c < // this_thread::sleep_for() +# include +#else +# include +#endif + #include // numeric_limits #include #include // abort() @@ -36,10 +43,10 @@ main (int argc, char* argv[]) // Usage: driver [-i ] (-o )* (-e )* (-f )* // (-d )* (-v )* [(-t (a|m|s|z)) | (-s )] // - // Execute actions specified by -i, -o, -e, -f, -d, and -v options in the - // order as they appear on the command line. After that terminate abnormally - // if -t option is provided, otherwise exit normally with the status - // specified by -s option (0 by default). + // Execute actions specified by -i, -o, -e, -f, -d, -v, and -l options in + // the order as they appear on the command line. After that terminate + // abnormally if -t option is provided, otherwise exit normally with the + // status specified by -s option (0 by default). // // -i // Forward stdin data to the standard stream denoted by the file @@ -62,6 +69,9 @@ main (int argc, char* argv[]) // If the specified variable is set the print its value to stdout and the // string '' otherwise. // + // -l + // Sleep the specified number of seconds. + // // -t // Abnormally terminate itself using one of the following methods: // @@ -144,6 +154,18 @@ main (int argc, char* argv[]) optional var (getenv (v)); cout << (var ? *var : "") << endl; } + else if (o == "-l") + { + size_t t (toi (v)); + + // MinGW GCC 4.9 doesn't implement this_thread so use Win32 Sleep(). + // +#ifndef _WIN32 + this_thread::sleep_for (chrono::seconds (t)); +#else + Sleep (static_cast (t * 1000)); +#endif + } else if (o == "-t") { assert (aterm == '\0' && !status); // Make sure exit method is not set. diff --git a/tests/test/script/runner/env.testscript b/tests/test/script/runner/env.testscript index 6fcedfa..ef90c3b 100644 --- a/tests/test/script/runner/env.testscript +++ b/tests/test/script/runner/env.testscript @@ -3,5 +3,30 @@ .include ../common.testscript -$c <'env abc=xyz -- $* -v abc >xyz' && $b : set -$c <'env --unset=abc -- $* -v abc >""' && env abc=xyz -- $b : unset +: variables +: +{ + $c <'env abc=xyz -- $* -v abc >xyz' && $b : set + $c <'env --unset=abc -- $* -v abc >""' && env abc=xyz -- $b : unset +} + +: timeout +: +{ + : expired + : + $c <'env --timeout 1 -- $* -l 5' && $b 2>>~%EOE% != 0 + %testscript:1:1: error: .+ terminated: execution timeout expired% + info: test id: 1 + EOE + + : not-expired + : + $c <'env --timeout 5 -- $* -l 1' && $b + + : invalid + : + $c <'env --timeout a -- $*' && $b 2>>EOE != 0 + testscript:1:15: error: env: invalid value 'a' for option '--timeout' + EOE +} diff --git a/tests/test/script/runner/set.testscript b/tests/test/script/runner/set.testscript index 9219cbb..41709e4 100644 --- a/tests/test/script/runner/set.testscript +++ b/tests/test/script/runner/set.testscript @@ -94,10 +94,28 @@ { : non-exact : - $c <'foo bar' - EOI + { + : non-empty + : + $c <'"foo" "bar"' + EOI + + : empty + : + $c <'' + EOI + + : spaces + : + $c <'' + EOI + } : exact : @@ -106,7 +124,7 @@ : $c <'foo bar ' + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar" ""' EOI : no-trailing-ws @@ -115,8 +133,22 @@ : ':' modifier. : $c <'foo bar' + set -e -w baz <:' foo bar'; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar"' + EOI + + : empty + : + $c <'' + EOI + + : spaces + : + $c <'""' EOI } } @@ -134,7 +166,7 @@ bar EOF - echo $baz >' foo bar ' + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar" ""' EOI : exact @@ -150,7 +182,7 @@ bar EOF - echo $baz >' foo bar ' + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar" "" ""' EOI : no-trailing-newline @@ -162,7 +194,7 @@ bar EOF - echo $baz >' foo bar' + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar"' EOI } } @@ -180,7 +212,7 @@ bar EOF - echo $baz >>EOO + echo ($baz[0]) >>EOO foo @@ -209,7 +241,7 @@ bar EOF - echo "$baz" >>EOO + echo ($baz[0]) >>EOO foo @@ -227,7 +259,7 @@ bar EOF - echo "$baz" >>EOO + echo ($baz[0]) >>EOO foo @@ -237,6 +269,66 @@ } } +: deadline +: +{ + : not-reached + : + $c <foo 2>| + EOI + + : set-reached + : + $c <>~%EOE% != 0 + $* -o 'foo' -l 10 | env -t 1 -- set bar + EOI + %testscript:.*: error: set terminated: execution timeout expired% + %. + EOE + + : driver-reached + : + $c <>~%EOE% != 0 + env -t 1 -- $* -o 'foo' -l 10 | set bar + EOI + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : read-some-data + : + { + s="----------------------------------------------------------------------" + s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s" + s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s" + + : failure + : + $c <>~%EOE% != 0 + echo "$s" >=f; + $* -o 'foo' -l 10 | cat f - | env -t 2 -- set bar + EOI + %testscript:.*: error: set terminated: execution timeout expired% + %. + EOE + + : success + : + : Note that the cat builtin ends up with the 'broken pipe' diagnostics or + : similar. + : + $c <=f; + timeout --success 2; + $* -o 'foo' -l 10 | cat f - 2>>~%EOE% | set bar + %cat: .+% + EOE + EOI + } +} + : attributes : { diff --git a/tests/test/script/runner/timeout.testscript b/tests/test/script/runner/timeout.testscript new file mode 100644 index 0000000..ae8f535 --- /dev/null +++ b/tests/test/script/runner/timeout.testscript @@ -0,0 +1,503 @@ +# file : tests/test/script/runner/timeout.testscript +# license : MIT; see accompanying LICENSE file + +.include ../common.testscript + +: test +: +{ + : fragment-timeout + : + { + : set + : + $c <>~%EOE% != 0 + timeout 1; + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout /10 + + { + +timeout /10 + + timeout 1; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : successful + : + $c <>~%EOE% != 0 + testscript:1:1: error: missing timeout + %. + EOE + + : invalid + : + $c <'timeout foo' && $b 2>>~%EOE% != 0 + testscript:1:1: error: invalid test fragment timeout 'foo' + %. + EOE +} + +: group +: +{ + : group-timeout + : + { + : set + : + $c <>~%EOE% != 0 + { + +timeout 1 + + $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 10/10 + + { + +timeout 1/ + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c <>~%EOE% != 0 + { + +timeout foo/ + } + EOI + testscript:2:4: error: invalid test group timeout 'foo' + %. + EOE + } + + : test-timeout + : + { + : set + : + $c <>~%EOE% != 0 + { + +timeout /1 + + $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 10/10 + + { + +timeout /1 + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c <>~%EOE% != 0 + { + +timeout /foo + } + EOI + testscript:2:4: error: invalid test timeout 'foo' + %. + EOE + } +} + +: script +: +{ + : group-timeout + : + { + : set + : + $c <>~%EOE% != 0 + +timeout 1 + + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 1 + + { + +timeout 10 + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c <>~%EOE% != 0 + +timeout foo/ + EOI + testscript:1:2: error: invalid testscript timeout 'foo' + %. + EOE + + : successful + : + $c <>~%EOE% != 0 + +timeout /1 + + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout /1 + + { + +timeout --success /1 + + { + +timeout 10/10 + + timeout 10; + env -t 10 -- $* -l 3 + } + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : successful + : + $c <>~%EOE% != 0 + { + +timeout /foo + } + EOI + testscript:2:4: error: invalid test timeout 'foo' + %. + EOE + } +} + +: config +: +{ + : operation + : + { + : set + : + $c <>~%EOE% != 0 + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 10 + + { + +timeout 10/10 + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c && $b config.test.timeout=foo 2>>EOE != 0 + error: invalid config.test.timeout test operation timeout value 'foo' + EOE + } + + : test + : + { + : set + : + $c <>~%EOE% != 0 + $* -l 3 + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : reset + : + $c <>~%EOE% != 0 + +timeout 10 + + { + +timeout 10/10 + + timeout 10; + env -t 10 -- $* -l 3 + } + EOI + %testscript:.*: error: .+ terminated: execution timeout expired% + %. + EOE + + : invalid + : + $c && $b config.test.timeout=/foo 2>>EOE != 0 + error: invalid config.test.timeout test timeout value 'foo' + EOE + } +} + +: failures +: +: Here we test that the left-hand side processes are terminated on failure. +: +{ + : set + : + $c <>~%EOE% != 0 + env -t 1 -- $* -l 86400 -o 'foo' | set --foo bar + EOI + %testscript:.*: error: set: unknown option '--foo'% + %. + EOE + + : exit + : + $c <>~%EOE% != 0 + env -t 1 -- $* -l 86400 -o 'foo' | exit 0 + EOI + %testscript:.*: error: exit builtin must be the only pipe command% + %. + EOE + + : redirect + : + $c <>~%EOE% != 0 + env -t 1 -- $* -l 86400 -o 'foo' | touch $~/foo/bar + EOI + %testscript:.*: error: touch exited with code 1% + %.+ + EOE +} + +: pipeline +: +{ + : prog-tm-prog + : + $c <'$* -l 10 | env -t 1 -- $* -i 0' && $b 2>>~%EOE% != 0 + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : tm-prog-prog + : + $c <'env -t 1 -- $* -l 10 | $* -i 0' && $b 2>>~%EOE% != 0 + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : tm-cat-prog + : + $c <'env -t 1 -- cat <"test" | $* -l 10' && $b 2>>~%EOE% != 0 + %testscript:.*: error: cat terminated: execution timeout expired% + %. + EOE + + : cat-tm-prog + : + $c <'cat <"test" | env -t 1 -- $* -l 10' && $b 2>>~%EOE% != 0 + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : tm-prog-cat + : + $c <'env -t 1 -- $* -l 10 | cat >-' && $b 2>>~%EOE% != 0 + %testscript:.*: error: .+driver.* terminated: execution timeout expired% + %. + EOE + + : tm-echo-prog + : + $c <'env -t 1 -- echo "test" | $* -l 10' && $b 2>>~%EOE% != 0 + %testscript:.*: error: echo terminated: execution timeout expired% + %. + EOE + + : successful + : + { + : prog-prog + : + $c <>~%EOE% | $* -l 10 -i 0 + %cat: unable to print stdin: .+% + EOE + EOI + } +} -- cgit v1.1