Add support for regex in runner

4 files changed, 320 insertions, 96 deletions

diff --git a/tests/test/script/runner/buildfile b/tests/test/script/runner/buildfile
index 375be18..a7603e4 100644
--- a/tests/test/script/runner/buildfile
+++ b/tests/test/script/runner/buildfile
@@ -2,11 +2,9 @@
 # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-./: test{cleanup status} exe{driver} $b
+./: test{cleanup redirect status} exe{driver} $b
 
-# @@ Do test{*} once redirect folds in.
-#
-test{cleanup}@./ test{status}@./: target = exe{driver}
+test{*}: target = exe{driver}
 
 import libs = libbutl%lib{butl}
-exe{driver}: cxx{driver} $libs test{redirect}
+exe{driver}: cxx{driver} $libs
diff --git a/tests/test/script/runner/cleanup.test b/tests/test/script/runner/cleanup.test
index f14d02d..6f6bb95 100644
--- a/tests/test/script/runner/cleanup.test
+++ b/tests/test/script/runner/cleanup.test
@@ -6,8 +6,6 @@
 
 b += --no-column
 
-td = [string] $path.canonicalize(test/)
-
 # Valid cleanups.
 #
 # @@ TODO: $c <'$* -f a &a' && $b
@@ -93,8 +91,8 @@ $b
 : Test cleanup of non-existing file.
 :
 $c <'$* &a';
-$b 2>>"EOE" != 0
-testscript:1: error: registered for cleanup file $path.canonicalize(test/1/a) does not exist
+$b 2>>~%EOE% != 0
+%testscript:1: error: registered for cleanup file test[/\\]1[/\\]a does not exist%
 EOE
 
 : file-out-wd
@@ -102,8 +100,8 @@ EOE
 : Test explicit cleanup of a file out of the testscript working directory.
 :
 $c <'$* &../../a';
-$b 2>>"EOE" != 0
-testscript:1: error: file cleanup $path.canonicalize(../../a) is out of working directory $td
+$b 2>>~%EOE% != 0
+%testscript:1: error: file cleanup \.\.[/\\]\.\.[/\\]a is out of working directory test[/\\]%
 EOE
 
 : file-in-wd
@@ -112,24 +110,30 @@ EOE
 : directory but inside the script working directory.
 :
 $c <'$* &../a';
-$b 2>>"EOE" != 0
-testscript:1: error: registered for cleanup file $path.canonicalize(test/a) does not exist
+$b 2>>~%EOE% != 0
+%testscript:1: error: registered for cleanup file test[/\\]a does not exist%
 EOE
 
 : not-file
 :
 : Test cleanup of a directory as a file.
 :
+: Note that there is an optional trailing blank line in the regex that matches
+: the newline added by msvcrt as a part of the error description.
+:
 $c <'$* -d a &a';
-$b 2>- != 0 # @@ REGEX
+$b 2>>~%EOE% != 0
+%error: unable to remove file test[/\\]1[/\\]a: .+%
+%%?
+EOE
 
 : dir-not-exists
 :
 : Test cleanup of non-existing directory.
 :
 $c <'$* &a/';
-$b 2>>"EOE" != 0
-testscript:1: error: registered for cleanup directory ([string] $path.canonicalize(test/1/a/)) does not exist
+$b 2>>~%EOE% != 0
+%testscript:1: error: registered for cleanup directory test[/\\]1[/\\]a[/\\] does not exist%
 EOE
 
 : dir-out-wd
@@ -137,8 +141,8 @@ EOE
 : Test cleanup of a directory out of the testscript working directory.
 :
 $c <'$* &../../a/';
-$b 2>>"EOE" != 0
-testscript:1: error: directory cleanup ([string] $path.canonicalize(../../a/)) is out of working directory $td
+$b 2>>~%EOE% != 0
+%testscript:1: error: directory cleanup \.\.[/\\]\.\.[/\\]a[/\\] is out of working directory test[/\\]%
 EOE
 
 : dir-in-wd
@@ -147,8 +151,8 @@ EOE
 : working directory but inside the testscript working directory.
 :
 $c <'$* &../a/';
-$b 2>>"EOE" != 0
-testscript:1: error: registered for cleanup directory ([string] $path.canonicalize(test/a/)) does not exist
+$b 2>>~%EOE% != 0
+%testscript:1: error: registered for cleanup directory test[/\\]a[/\\] does not exist%
 EOE
 
 : dir-not-empty
@@ -156,24 +160,30 @@ EOE
 : Test cleanup of a non-empty directory.
 :
 $c <'$* -d a -f a/b &a/';
-$b 2>>"EOE" != 0
-testscript:1: error: registered for cleanup directory ([string] $path.canonicalize(test/1/a/)) is not empty
+$b 2>>~%EOE% != 0
+%testscript:1: error: registered for cleanup directory test[/\\]1[/\\]a[/\\] is not empty%
 EOE
 
 : not-dir
 :
 : Test cleanup of a file as a directory.
 :
+: Note that there is an optional trailing blank line in the regex that matches
+: the newline added by msvcrt as a part of the error description.
+:
 $c <'$* -f a &a/';
-$b 2>- != 0 # @@ REGEX
+$b 2>>~%EOE% != 0
+%error: unable to remove directory test[/\\]1[/\\]a[/\\]: .+%
+%%?
+EOE
 
 : wildcard-not-exists
 :
 : Test cleanup of a wildcard not matching any directory.
 :
 $c <'$* &a/***';
-$b 2>>"EOE" != 0
-testscript:1: error: registered for cleanup wildcard $path.canonicalize(test/1/a/***) doesn't match a directory
+$b 2>>~%EOE% != 0
+%testscript:1: error: registered for cleanup wildcard test[/\\]1[/\\]a[/\\]\*\*\* doesn't match a directory%
 EOE
 
 : wildcard-out-wd
@@ -181,8 +191,8 @@ EOE
 : Test cleanup of a wildcard out of the testscript working directory.
 :
 $c <'$* &../../a/***';
-$b 2>>"EOE" != 0
-testscript:1: error: wildcard cleanup $path.canonicalize(../../a/***) is out of working directory $td
+$b 2>>~%EOE% != 0
+%testscript:1: error: wildcard cleanup \.\.[/\\]\.\.[/\\]a[/\\]\*\*\* is out of working directory test[/\\]%
 EOE
 
 : wildcard-in-wd
@@ -192,24 +202,30 @@ EOE
 : directory.
 :
 $c <'$* &../a/***';
-$b 2>>"EOE" != 0
-testscript:1: error: registered for cleanup wildcard $path.canonicalize(test/a/***) doesn't match a directory
+$b 2>>~%EOE% != 0
+%testscript:1: error: registered for cleanup wildcard test[/\\]a[/\\]\*\*\* doesn't match a directory%
 EOE
 
 : wildcard-not-dir
 :
 : Test cleanup of a file as a wildcard.
 :
+: Note that there is an optional trailing blank line in the regex that matches
+: the newline added by msvcrt as a part of the error description.
+:
 $c <'$* -f a &a/***';
-$b 2>- != 0 # @@ REGEX
+$b 2>>~%EOE% != 0
+%error: unable to remove directory test[/\\]1[/\\]a[/\\]: .+%
+%%?
+EOE
 
 : implicit-overwrite
 :
 : Test an implicit cleanup being overwritten with the explicit one,
 :
 $c <'$* -o foo >>>a &!a';
-$b 2>>"EOE" != 0
-testscript:1: error: registered for cleanup directory ([string] $path.canonicalize(test/1/)) is not empty
+$b 2>>~%EOE% != 0
+%testscript:1: error: registered for cleanup directory test[/\\]1[/\\] is not empty%
 EOE
 
 : explicit-overwrite
@@ -220,6 +236,6 @@ $c <<EOO;
 $* &!a;
 $* -o foo >>>a
 EOO
-$b 2>>"EOE" != 0
-testscript:2: error: registered for cleanup directory ([string] $path.canonicalize(test/1/)) is not empty
+$b 2>>~%EOE% != 0
+%testscript:2: error: registered for cleanup directory test[/\\]1[/\\] is not empty%
 EOE
diff --git a/tests/test/script/runner/redirect.test b/tests/test/script/runner/redirect.test
index e8d57c1..ec3c960 100644
--- a/tests/test/script/runner/redirect.test
+++ b/tests/test/script/runner/redirect.test
@@ -2,150 +2,291 @@
 # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-# @@ Needs conversion to cat & build (then adjust buildfile).
+.include ../common.test
 
-$* -o foo >-                        : out-null
-$* -e foo 2>-                       : err-null
-$* -i 0 <foo                        : in-str
-$* -o foo >foo                      : out-str
-$* -e foo 2>foo                     : err-str
-$* -i 1 <foo >foo                   : inout-str
-$* -i 2 <foo 2>foo                  : inerr-str
-$* -i 1 -e bar <foo 1>foo 2>bar     : inout-err-str
+b += --no-column
 
+# null-redirect tests.
+#
+: out-null
+:
+$c <'$* -o foo >-';
+$b
+
+: err-null
+:
+$c <'$* -e foo 2>-';
+$b
+
+# Here-string tests.
+#
+: in-str
+:
+$c <'$* -i 0 <foo';
+$b
+
+: out-str
+:
+$c <'$* -o foo >foo';
+$b
+
+: err-str
+:
+$c <'$* -e foo 2>foo';
+$b
+
+: inout-str
+:
+$c <'$* -i 1 <foo >foo';
+$b
+
+: inout-str-fail
+:
+$c <'$* -i 1 <foo >bar';
+$b 2>>~%EOE% != 0
+%.{3}
+-bar
++foo
+%testscript:1: error: \.\.[/\\]\.\.[/\\]\.\.[/\\]driver(\.exe)? stdout doesn't match the expected output%
+%  info: stdout: test[/\\]1[/\\]stdout%
+%  info: expected stdout: test[/\\]1[/\\]stdout\.orig%
+%  info: stdin: test[/\\]1[/\\]stdin%
+EOE
+
+: inerr-str
+:
+$c <'$* -i 2 <foo 2>foo';
+$b
+
+: inout-err-str
+:
+$c <'$* -i 1 -e bar <foo 1>foo 2>bar';
+$b
+
+# Here-document tests.
+#
 : in-doc
 :
+$c <<EOI;
 $* -i 0 <<EOO
 foo
 bar
 EOO
+EOI
+$b
 
 : out-doc
 :
+$c <<EOI;
 $* -o foo -o bar >>EOO
 foo
 bar
 EOO
+EOI
+$b
 
 : err-doc
 :
+$c <<EOI;
 $* -e foo -e bar 2>>EOO
 foo
 bar
 EOO
+EOI
+$b
 
 : inout-doc
 :
-$* -i 1 <<EOI >>EOO
+$c <<EOI;
+$* -i 1 <<EOF >>EOO
 foo
 bar
-EOI
+EOF
 foo
 bar
 EOO
+EOI
+$b
 
 : inerr-doc
 :
-$* -i 2 <<EOI 2>>EOE
+$c <<EOI;
+$* -i 2 <<EOF 2>>EOE
 foo
 bar
-EOI
+EOF
 foo
 bar
 EOE
+EOI
+$b
 
 : empty-str-doc
 :
-$* -i 1 -e "" <<EOI >>EOO 2>""
-EOI
+$c <<EOI;
+$* -i 1 -e "" <<EOF >>EOO 2>""
+EOF
 EOO
+EOI
+$b
 
 : nl-containing-doc
 :
-$* -i 1 <<EOI >>EOO
+$c <<EOI;
+$* -i 1 <<EOF >>EOO
 
-EOI
+EOF
 
 EOO
+EOI
+$b
 
 # No-newline tests.
 #
-# @@ TMP Need does not compare test.
-#
 : no-newline-str
 :
-$* -i 1 <:"foo" >:"foo"
+$c <'$* -i 1 <:"foo" >:"foo"';
+$b
+
+: no-newline-empty-str
+:
+$c <'$* -i 1 <:"" >:""';
+$b
+
+: no-newline-str-fail1
+:
+$c <'$* -i 1 <:"foo" >"foo"';
+$b 2>>~/EOE/ != 0
+/.{3}
+-foo
++foo
+\ No newline at end of file
+/testscript:1: error: .+driver(\.exe)? stdout doesn't match the expected output/
+/.{3}
+EOE
 
-#\
-$* -i 1 <:"foo" >!"foo"                              # no-newline-str-fail1
-$* -i 1 <"foo"  >:!"foo"                             # no-newline-str-fail2
-#\
+: no-newline-str-fail2
+:
+$c <'$* -i 1 <"foo" >:"foo"';
+$b 2>>~/EOE/ != 0
+/.{3}
+-foo
+\ No newline at end of file
++foo
+/testscript:1: error: .+driver(\.exe)? stdout doesn't match the expected output/
+/.{3}
+EOE
 
 : no-newline-doc
 :
-$* -i 1 <<:EOI >>:EOO
+$c <<EOI;
+$* -i 1 <<:EOF >>:EOO
 foo
-EOI
+EOF
 foo
 EOO
+EOI
+$b
 
-#\
-$* -i 1 <<:EOI >>!EOO                                # no-newline-doc-fail1
+: no-newline-doc-fail1
+:
+$c <<EOI;
+$* -i 1 <<:EOF >>EOO
 foo
-EOI
+EOF
 foo
 EOO
+EOI
+$b 2>>~/EOE/ != 0
+/.{3}
+-foo
++foo
+\ No newline at end of file
+/testscript:1: error: .+driver(\.exe)? stdout doesn't match the expected output/
+/.{3}
+EOE
 
-$* -i 1 <<EOI >>:!EOO                                # no-newline-doc-fail2
+: no-newline-doc-fail2
+:
+$c <<EOI;
+$* -i 1 <<EOF >>:EOO
 foo
-EOI
+EOF
 foo
 EOO
-#\
+EOI
+$b 2>>~/EOE/ != 0
+/.{3}
+-foo
+\ No newline at end of file
++foo
+/testscript:1: error: .+driver(\.exe)? stdout doesn't match the expected output/
+/.{3}
+EOE
 
 : no-newline-empty-str-doc
 :
-$* -i 1 <<:EOI >>:EOO 2>:""
-EOI
+$c <<EOI;
+$* -i 1 <<:EOF >>:EOO 2>:""
+EOF
 EOO
+EOI
+$b
 
 : no-newline-nl-cont-doc
 :
-$* -i 1 <<:EOI >>:EOO
-
-EOI
+$c <<EOI;
+$* -i 1 <<:EOF >>:EOO
 
-EOO
+EOF
 
-: file
-:
-$* -o foo >>>out;
-$* -e bar 2>>>&out;
-$* -i 1 <<<out >>EOO
-foo
-bar
 EOO
+EOI
+$b
 
+# Merge tests.
+#
 : merge-str
 :
+$c <<EOI;
 $* -o foo -e bar 2>>EOE 1>&2
 foo
 bar
 EOE
+EOI
+$b
 
 : merge-doc
 :
-$* -i 1 <<EOI -e baz >>EOO 2>&1
+$c <<EOI;
+$* -i 1 <<EOF -e baz >>EOO 2>&1
 foo
 bar
-EOI
+EOF
 foo
 bar
 baz
 EOO
+EOI
+$b
+
+# File tests.
+#
+: file
+:
+$c <<EOI;
+$* -o foo >>>out;
+$* -e bar 2>>>&out;
+$* -i 1 <<<out >>EOO
+foo
+bar
+EOO
+EOI
+$b
 
 : merge-file
 :
+$c <<EOI;
 $* -o foo -e bar 2>&1 >>>out;
 $* -e baz -o biz 1>&2 2>>>&out;
 $* -i 1 <<<out >>EOO
@@ -154,6 +295,52 @@ bar
 baz
 biz
 EOO
+EOI
+$b
+
+# Regex tests.
+#
+: out-str-regex
+:
+{
+  : match
+  :
+  $c <'$* -o foo >~/Foo?/i';
+  $b
+
+  : fail
+  :
+  $c <'$* -o fooo >~/Foo?/i';
+  $b 2>>~%EOE% != 0
+  %testscript:1: error: \.\.[/\\]\.\.[/\\]\.\.[/\\]\.\.[/\\]driver(\.exe)? stdout doesn't match the regex%
+  %  info: stdout: test[/\\]1[/\\]stdout%
+  %  info: stdout regex: test[/\\]1[/\\]stdout\.regex%
+  EOE
+}
+
+: out-doc-regex
+:
+{
+  : match
+  :
+  $c <<EOI;
+  $* -o foo -o foo -o bar >>~/EOO/i
+  /FO*/*
+  bar
+  /*
+  EOO
+  EOI
+  $b
+
+  : match-empty
+  :
+  $c <<EOI;
+  $* >>:~/EOO/i
+  /.{0}
+  EOO
+  EOI
+  $b
+}
 
 # Builtins redirects.
 #
@@ -163,11 +350,38 @@ EOO
 : builtins
 :
 {
-  echo "abc" >-                     : out-null
-  echo "abc" 1>&2 2>-               : err-null
-  echo <foo 1>-                     : in-str
-  echo "foo" >foo                   : out-str
-  echo "foo" 2>foo 1>&2             : err-str
-  cat <foo >foo                     : inout-str
-  cat <foo 2>foo 1>&2               : inerr-str
+  : out-null
+  :
+  $c <'echo "abc" >-';
+  $b
+
+  : err-null
+  :
+  $c <'echo "abc" 1>&2 2>-';
+  $b
+
+  : in-str
+  :
+  $c <'echo <foo 1>-';
+  $b
+
+  : out-str
+  :
+  $c <'echo "foo" >foo';
+  $b
+
+  : err-str
+  :
+  $c <'echo "foo" 2>foo 1>&2';
+  $b
+
+  : inout-str
+  :
+  $c <'cat <foo >foo';
+  $b
+
+  : inerr-str
+  :
+  $c <'cat <foo 2>foo 1>&2';
+  $b
 }
diff --git a/tests/test/script/runner/status.test b/tests/test/script/runner/status.test
index e115f13..6a2c06c 100644
--- a/tests/test/script/runner/status.test
+++ b/tests/test/script/runner/status.test
@@ -6,10 +6,6 @@
 
 b += --no-column
 
-if ($cxx.target.class == "windows")
-  ext = .exe
-end
-
 # Successfull tests.
 #
 : eq-true
@@ -27,13 +23,13 @@ $b
 : eq-false
 :
 $c <'$* -s 1 == 0';
-$b 2>>"EOE" != 0
-testscript:1: error: $path.canonicalize(../../../driver$ext) exit status 1 != 0
+$b 2>>~%EOE% != 0
+%testscript:1: error: \.\.[/\\]\.\.[/\\]\.\.[/\\]driver(.exe)? exit status 1 != 0%
 EOE
 
 : ne-false
 :
 $c <'$* -s 1 != 1';
-$b 2>>"EOE" != 0
-testscript:1: error: $path.canonicalize(../../../driver$ext) exit status 1 == 1
+$b 2>>~%EOE% != 0
+%testscript:1: error: \.\.[/\\]\.\.[/\\]\.\.[/\\]driver(.exe)? exit status 1 == 1%
 EOE