Add support for test timeouts

1 files changed, 503 insertions, 0 deletions

diff --git a/tests/test/script/runner/timeout.testscript b/tests/test/script/runner/timeout.testscript
new file mode 100644
index 0000000..ae8f535
--- /dev/null
+++ b/tests/test/script/runner/timeout.testscript
@@ -0,0 +1,503 @@
+# file      : tests/test/script/runner/timeout.testscript
+# license   : MIT; see accompanying LICENSE file
+
+.include ../common.testscript
+
+: test
+:
+{
+  : fragment-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      timeout 1;
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      timeout 1;
+      timeout 0;
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout /10
+
+      {
+        +timeout /10
+
+        timeout 1;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : successful
+    :
+    $c <<EOI && $b
+      timeout --success 1;
+      $* -l 3
+      EOI
+  }
+
+  : missing
+  :
+  $c <'timeout' && $b 2>>~%EOE% != 0
+    testscript:1:1: error: missing timeout
+    %.
+    EOE
+
+  : invalid
+  :
+  $c <'timeout foo' && $b 2>>~%EOE% != 0
+    testscript:1:1: error: invalid test fragment timeout 'foo'
+    %.
+    EOE
+}
+
+: group
+:
+{
+  : group-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout 1
+
+        $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      {
+        +timeout 1
+        +timeout 0
+
+        $* -l 3
+      }
+      EOI
+
+    : override
+    :
+    : Also test slash usage inside the timeout value.
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout 10/10
+
+      {
+        +timeout 1/
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout foo/
+      }
+      EOI
+      testscript:2:4: error: invalid test group timeout 'foo'
+      %.
+      EOE
+  }
+
+  : test-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout /1
+
+        $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      {
+        +timeout /1
+        +timeout /0
+
+        $* -l 3
+      }
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout 10/10
+
+      {
+        +timeout /1
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout /foo
+      }
+      EOI
+      testscript:2:4: error: invalid test timeout 'foo'
+      %.
+      EOE
+  }
+}
+
+: script
+:
+{
+  : group-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout 1
+
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      +timeout 1
+      +timeout 0
+
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout 1
+
+      {
+        +timeout 10
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout foo/
+      EOI
+      testscript:1:2: error: invalid testscript timeout 'foo'
+      %.
+      EOE
+
+    : successful
+    :
+    $c <<EOI && $b
+      +timeout --success 1
+
+      $* -l 3
+      EOI
+  }
+
+  : test-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout /1
+
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      +timeout /1
+      +timeout /0
+
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout /1
+
+      {
+        +timeout --success /1
+
+        {
+          +timeout 10/10
+
+          timeout 10;
+          env -t 10 -- $* -l 3
+        }
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : successful
+    :
+    $c <<EOI && $b
+      +timeout --success /1
+
+      $* -l 3
+      EOI
+
+    : invalid
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout /foo
+      }
+      EOI
+      testscript:2:4: error: invalid test timeout 'foo'
+      %.
+      EOE
+  }
+}
+
+: config
+:
+{
+  : operation
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b config.test.timeout=1/10 2>>~%EOE% != 0
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b config.test.timeout=1/10 config.test.timeout=0/10
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b config.test.timeout=1/10 2>>~%EOE% != 0
+      +timeout 10
+
+      {
+        +timeout 10/10
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c && $b config.test.timeout=foo 2>>EOE != 0
+      error: invalid config.test.timeout test operation timeout value 'foo'
+      EOE
+  }
+
+  : test
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b config.test.timeout=10/1 2>>~%EOE% != 0
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b config.test.timeout=10/1 config.test.timeout=10/0
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b config.test.timeout=10/1 2>>~%EOE% != 0
+      +timeout 10
+
+      {
+        +timeout 10/10
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c && $b config.test.timeout=/foo 2>>EOE != 0
+      error: invalid config.test.timeout test timeout value 'foo'
+      EOE
+  }
+}
+
+: failures
+:
+: Here we test that the left-hand side processes are terminated on failure.
+:
+{
+  : set
+  :
+  $c <<EOI && $b 2>>~%EOE% != 0
+  env -t 1 -- $* -l 86400 -o 'foo' | set --foo bar
+  EOI
+  %testscript:.*: error: set: unknown option '--foo'%
+  %.
+  EOE
+
+  : exit
+  :
+  $c <<EOI && $b 2>>~%EOE% != 0
+  env -t 1 -- $* -l 86400 -o 'foo' | exit 0
+  EOI
+  %testscript:.*: error: exit builtin must be the only pipe command%
+  %.
+  EOE
+
+  : redirect
+  :
+  $c <<EOI && $b 2>>~%EOE% != 0
+  env -t 1 -- $* -l 86400 -o 'foo' | touch $~/foo/bar
+  EOI
+  %testscript:.*: error: touch exited with code 1%
+  %.+
+  EOE
+}
+
+: pipeline
+:
+{
+  : prog-tm-prog
+  :
+  $c <'$* -l 10 | env -t 1 -- $* -i 0' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+    %.
+    EOE
+
+  : tm-prog-prog
+  :
+  $c <'env -t 1 -- $* -l 10 | $* -i 0' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+    %.
+    EOE
+
+  : tm-cat-prog
+  :
+  $c <'env -t 1 -- cat <"test" | $* -l 10' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: cat terminated: execution timeout expired%
+    %.
+    EOE
+
+  : cat-tm-prog
+  :
+  $c <'cat <"test" | env -t 1 -- $* -l 10' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+    %.
+    EOE
+
+  : tm-prog-cat
+  :
+  $c <'env -t 1 -- $* -l 10 | cat >-' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+    %.
+    EOE
+
+  : tm-echo-prog
+  :
+  $c <'env -t 1 -- echo "test" | $* -l 10' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: echo terminated: execution timeout expired%
+    %.
+    EOE
+
+  : successful
+  :
+  {
+    : prog-prog
+    :
+    $c <<EOI && $b
+    timeout --success 1;
+    $* -l 10 | $* -i 0
+    EOI
+
+    : prog-cat
+    :
+    $c <<EOI && $b
+    timeout --success 1;
+    $* -l 10 | cat
+    EOI
+
+    : cat-prog
+    :
+    $c <<EOI && $b
+    s="----------------------------------------------------------------------";
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s";
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s";
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s";
+    timeout --success 1;
+    cat <"$s" 2>>~%EOE% | $* -l 10 -i 0
+    %cat: unable to print stdin: .+%
+    EOE
+    EOI
+  }
+}