diff options
37 files changed, 4634 insertions, 444 deletions
diff --git a/doc/testscript.cli b/doc/testscript.cli index 69941a6..e3a9c43 100644 --- a/doc/testscript.cli +++ b/doc/testscript.cli @@ -1418,6 +1418,7 @@ while potentially spanning several physical lines. The \c{-line} suffix here signifies a \i{logical line}, for example, a command line plus its here-document fragments. + \h#syntax-grammar|Grammar| The complete grammar of the Testscript language is presented next with the @@ -1474,33 +1475,58 @@ test: +(variable-line|command-like) variable-like: - variable-line|variable-if + variable-line|variable-flow variable-line: <variable-name> ('='|'+='|'=+') value-attributes? <value> ';'? value-attributes: '[' <key-value-pairs> ']' +variable-flow: + variable-if|variable-for|variable-while + variable-if: ('if'|'if!') command-line - variable-if-body + variable-flow-body *variable-elif ?variable-else - 'end' + 'end' ';'? variable-elif: ('elif'|'elif!') command-line - variable-if-body + variable-flow-body variable-else: 'else' - variable-if-body + variable-flow-body -variable-if-body: +variable-flow-body: *variable-like +variable-for: + variable-for-args|variable-for-stream + +variable-for-args: + 'for' variable-attributes? <variable-name> ':' \ + value-attributes? <value> + variable-flow-body + 'end' ';'? + +variable-attributes: '[' <key-value-pairs> ']' + +variable-for-stream: + (command-pipe '|')? \ + 'for' (<opt>|stdin)* variable-attributes? <variable-name> (stdin)* + variable-flow-body + 'end' ';'? + +variable-while: + 'while' command-line + variable-flow-body + 'end' ';'? + command-like: - command-line|command-if + command-line|command-flow command-line: command-expr (';'|(':' <text>))? *here-document @@ -1513,24 +1539,47 @@ command: <path>(' '+(<arg>|redirect|cleanup))* command-exit? command-exit: ('=='|'!=') <exit-status> +command-flow: + command-if|command-for|command-while + command-if: ('if'|'if!') command-line - command-if-body + command-flow-body *command-elif ?command-else 'end' (';'|(':' <text>))? command-elif: ('elif'|'elif!') command-line - command-if-body + command-flow-body command-else: 'else' - command-if-body + command-flow-body -command-if-body: +command-flow-body: *(variable-line|command-like) +command-for: + command-for-args|command-for-stream + +command-for-args: + 'for' variable-attributes? <variable-name> ':' \ + value-attributes? <value> + command-flow-body + 'end' (';'|(':' <text>))? + +command-for-stream: + (command-pipe '|')? \ + 'for' (<opt>|stdin)* variable-attributes? <variable-name> (stdin)* + command-flow-body + 'end' (';'|(':' <text>))? + +command-while: + 'while' command-line + command-flow-body + 'end' (';'|(':' <text>))? + redirect: stdin|stdout|stderr stdin: '0'?(in-redirect) @@ -1563,6 +1612,12 @@ description: +(':' <text>) \ +Note that the only purpose of having a separate (from the command flow control +constructs) variable-only flow control constructs is to remove the error-prone +requirement of having to specify \c{+} and \c{-} prefixes in group +setup/teardown. + + \h#syntax-script|Script| \ @@ -1573,6 +1628,7 @@ script: A testscript file is an implicit group scope (see \l{#model Model and Execution} for details). + \h#syntax-scope|Scope| \ @@ -1622,6 +1678,7 @@ the scopes in an \c{if-else} chain are alternative implementations of the same group/test (thus the single description). If at least one of them is a group scope, then all the others are treated as groups as well. + \h#syntax-directive|Directive| \ @@ -1654,6 +1711,7 @@ this scope should not be included again. The implementation is not required to handle links when determining if two paths are to the same file. Relative paths are assumed to be relative to the including testscript file. + \h#syntax-setup-teardown|Setup and Teardown| \ @@ -1667,11 +1725,12 @@ setup-line: '+' command-like tdown-line: '-' command-like \ -Note that variable assignments (including \c{variable-if}) do not use the +Note that variable assignments (including \c{variable-flow}) do not use the \c{'+'} and \c{'-'} prefixes. A standalone (not part of a test) variable assignment is automatically treated as a setup if no tests have yet been encountered in this scope and as a teardown otherwise. + \h#syntax-test|Test| \ @@ -1690,11 +1749,12 @@ cat <'verbose = true' >=$conf; test1 $conf \ + \h#syntax-variable|Variable| \ variable-like: - variable-line|variable-if + variable-line|variable-flow variable-line: <variable-name> ('='|'+='|'=+') value-attributes? <value> ';'? @@ -1713,25 +1773,26 @@ echo $args # foo bar fox baz The value can only be followed by \c{;} inside a test to signal the test continuation. + \h#syntax-variable-if|Variable-If| \ variable-if: ('if'|'if!') command-line - variable-if-body + variable-flow-body *variable-elif ?variable-else - 'end' + 'end' ';'? variable-elif: ('elif'|'elif!') command-line - variable-if-body + variable-flow-body variable-else: 'else' - variable-if-body + variable-flow-body -variable-if-body: +variable-flow-body: *variable-like \ @@ -1755,15 +1816,107 @@ with a ternary operator is often more concise: slash = ($cxx.target.class == 'windows' ? \\\\ : /) \ -Note also that the only purpose of having a separate (from \c{command-if}) -variable-only if-block is to remove the error-prone requirement of having to -specify \c{+} and \c{-} prefixes in group setup/teardown. + +\h#syntax-variable-for|Variable-For| + +\ +variable-for: + variable-for-args|variable-for-stream + +variable-for-args: + 'for' variable-attributes? <variable-name> ':' \ + value-attributes? <value> + variable-flow-body + 'end' ';'? + +variable-for-stream: + (command-pipe '|')? \ + 'for' (<opt>|stdin)* variable-attributes? <variable-name> (stdin)* + variable-flow-body + 'end' ';'? + +variable-flow-body: + *variable-like +\ + +A group of variables can be set in a loop, while iterating over elements of a +potentially empty list and setting the specified variable (called \i{loop +variable}) to the corresponding element on each iteration. At the end of the +iteration the loop variable contains the value of the last element, if any. + +In the first form the list results from an expression containing variable +expansions, function calls, eval contexts, and/or literal values. For example: + +\ +us = +ls = + +for v: $vs + us += $string.ucase($v) + ls += $string.lcase($v) +end +\ + +In the second form the list is read from \c{stdin} input. The input data can +be split into elements at newlines or whitespaces if \c{-n} or \c{-w} option, +respectively, is specified. This form supports the same set of options as the +\l{#builtins-set \c{set}} pseudo-builtin. For example: + +\ +us = +ls = + +cat values.txt | for -n v + us += $string.ucase($v) + ls += $string.lcase($v) +end +\ + +This example can actually be simplified as: + +\ +us = +ls = + +for -n v <=values.txt + us += $string.ucase($v) + ls += $string.lcase($v) +end +\ + + +\h#syntax-variable-while|Variable-While| + +\ +variable-while: + 'while' command-line + variable-flow-body + 'end' ';'? + +variable-flow-body: + *variable-like +\ + +A group of variables can be set in a loop, while iterating until the condition +evaluates to \c{false}. The condition \c{command-line} semantics is the same +as in \c{scope-if}. For example: + +\ +r = +i = [uint64] 0 + +while ($i != 2) + r += ($vs[$i]) + i += 1 +end +\ + \h#syntax-command|Command| \ command-like: - command-line|command-if + command-line|command-flow command-line: command-expr (';'|(':' <text>))? *here-document @@ -1778,7 +1931,7 @@ command-exit: ('=='|'!=') <exit-status> \ A command line is a command expression. If it appears directly (as opposed to -inside \c{command-if}) in a test, then it can be followed by \c{;} to signal +inside \c{command-flow}) in a test, then it can be followed by \c{;} to signal the test continuation or by \c{:} and the trailing description. A command expression can combine several command pipes with logical AND and OR @@ -1803,25 +1956,26 @@ to succeed (0 exit code). The logical result of executing a command is therefore a boolean value which is used in the higher-level constructs (pipe and expression). + \h#syntax-command-if|Command-If| \ command-if: ('if'|'if!') command-line - command-if-body + command-flow-body *command-elif ?command-else 'end' (';'|(':' <text>))? command-elif: ('elif'|'elif!') command-line - command-if-body + command-flow-body command-else: 'else' - command-if-body + command-flow-body -command-if-body: +command-flow-body: *(variable-line|command-like) \ @@ -1841,6 +1995,105 @@ end; test1 $foo \ + +\h#syntax-command-for|Command-For| + +\ +command-for: + command-for-args|command-for-stream + +command-for-args: + 'for' variable-attributes? <variable-name> ':' \ + value-attributes? <value> + command-flow-body + 'end' (';'|(':' <text>))? + +command-for-stream: + (command-pipe '|')? \ + 'for' (<opt>|stdin)* variable-attributes? <variable-name> (stdin)* + command-flow-body + 'end' (';'|(':' <text>))? + +command-flow-body: + *(variable-line|command-like) +\ + +A group of commands can be executed in a loop, while iterating over elements +of a potentially empty list and setting the specified variable (called \i{loop +variable}) to the corresponding element on each iteration. At the end of the +iteration the loop variable contains the value of the last element, if any. +Note that in a compound test, commands inside \c{command-for} must not end +with \c{;}. Rather, \c{;} may follow \c{end}. + +In the first form the list results from an expression containing variable +expansions, function calls, eval contexts, and/or literal values. For example: + +\ +ls = ; +for v: $vs + test1 $string.ucase($v) + ls += $string.lcase($v) +end; +test2 $ls +\ + +In the second form the list is read from \c{stdin} input. The input data can +be split into elements at newlines or whitespaces if \c{-n} or \c{-w} option, +respectively, is specified. This form supports the same set of options as the +\l{#builtins-set \c{set}} pseudo-builtin. For example: + +\ +ls = ; +cat values.txt | for -n v + test1 $string.ucase($v) + ls += $string.lcase($v) +end; +test2 $ls +\ + +This example can actually be simplified: + +\ +ls = ; +for -n v <=values.txt + test1 $string.ucase($v) + ls += $string.lcase($v) +end; +test2 $ls +\ + + +\h#syntax-command-while|Command-While| + +\ +command-while: + 'while' command-line + command-flow-body + 'end' (';'|(':' <text>))? + +command-flow-body: + *(variable-line|command-like) +\ + +A group of commands can be executed in a loop, while iterating until the +condition evaluates to \c{false}. The condition \c{command-line} semantics is +the same as in \c{scope-if}. Note that in a compound test, commands inside +\c{command-while} must not end with \c{;}. Rather, \c{;} may follow +\c{end}. For example: + +\ +r = ; +i = [uint64] 0; +while ($i != 2) + v = ($vs[$i]) + test1 $v + r += $v + i += 1 +end; +test2 $r +\ + + \h#syntax-redirect|Redirect| \ @@ -1969,6 +2222,7 @@ Similar to the input redirects, an output here-document redirect must be specified literally on the command line. See \l{#syntax-here-document Here Document} for details. + \h#syntax-here-document|Here-Document| \ diff --git a/libbuild2/build/script/lexer+for-loop.test.testscript b/libbuild2/build/script/lexer+for-loop.test.testscript new file mode 100644 index 0000000..3f8e6b5 --- /dev/null +++ b/libbuild2/build/script/lexer+for-loop.test.testscript @@ -0,0 +1,188 @@ +# file : libbuild2/build/script/lexer+for-loop.test.testscript +# license : MIT; see accompanying LICENSE file + +test.arguments = for-loop + +: redirect +: +{ + : pass + : + $* <"cmd <| 1>|" >>EOO + 'cmd' + <| + '1' + >| + <newline> + EOO + + : null + : + $* <"cmd <- 1>-" >>EOO + 'cmd' + <- + '1' + >- + <newline> + EOO + + : trace + : + $* <"cmd 1>!" >>EOO + 'cmd' + '1' + >! + <newline> + EOO + + : merge + : + $* <"cmd 1>&2" >>EOO + 'cmd' + '1' + >& + '2' + <newline> + EOO + + : str + : + $* <"cmd <<<=a 1>>>?b" >>EOO + 'cmd' + <<<= + 'a' + '1' + >>>? + 'b' + <newline> + EOO + + : str-nn + : + $* <"cmd <<<=:a 1>>>?:b" >>EOO + 'cmd' + <<<=: + 'a' + '1' + >>>?: + 'b' + <newline> + EOO + + : str-nn-alias + : + $* <"cmd <<<:a 1>>>?:b" >>EOO + 'cmd' + <<<: + 'a' + '1' + >>>?: + 'b' + <newline> + EOO + + : doc + : + $* <"cmd <<EOI 1>>EOO" >>EOO + 'cmd' + << + 'EOI' + '1' + >> + 'EOO' + <newline> + EOO + + : doc-nn + : + $* <"cmd <<:EOI 1>>?:EOO" >>EOO + 'cmd' + <<: + 'EOI' + '1' + >>?: + 'EOO' + <newline> + EOO + + : file-cmp + : + $* <"cmd <=in >?out 2>?err" >>EOO + 'cmd' + <= + 'in' + >? + 'out' + '2' + >? + 'err' + <newline> + EOO + + : file-write + : + $* <"cmd >=out 2>+err" >>EOO + 'cmd' + >= + 'out' + '2' + >+ + 'err' + <newline> + EOO +} + +: cleanup +: +{ + : always + : + $* <"cmd &file" >>EOO + 'cmd' + & + 'file' + <newline> + EOO + + : maybe + : + $* <"cmd &?file" >>EOO + 'cmd' + &? + 'file' + <newline> + EOO + + : never + : + $* <"cmd &!file" >>EOO + 'cmd' + &! + 'file' + <newline> + EOO +} + +: for +: +{ + : form-1 + : + $* <"for x: a" >>EOO + 'for' + 'x' + : + 'a' + <newline> + EOO + + : form-3 + : + $* <"for <<<a x" >>EOO + 'for' + <<< + 'a' + 'x' + <newline> + EOO +} diff --git a/libbuild2/build/script/lexer.cxx b/libbuild2/build/script/lexer.cxx index d849ac9..5c13239 100644 --- a/libbuild2/build/script/lexer.cxx +++ b/libbuild2/build/script/lexer.cxx @@ -78,6 +78,19 @@ namespace build2 s2 = " "; break; } + case lexer_mode::for_loop: + { + // Leading tokens of the for-loop. Like command_line but + // recognizes colon as a separator and lsbrace like value. + // + // Note that while sensing the form of the for-loop (`for x:...` + // vs `for x <...`) we need to make sure that the pre-parsed token + // types are valid for the execution phase. + // + s1 = ":=!|&<> $(#\t\n"; + s2 = " == "; + break; + } default: { // Recognize special variable names ($>, $<, $~). @@ -109,6 +122,7 @@ namespace build2 case lexer_mode::first_token: case lexer_mode::second_token: case lexer_mode::variable_line: + case lexer_mode::for_loop: r = next_line (); break; default: return base_lexer::next (); @@ -141,7 +155,8 @@ namespace build2 // if (st.lsbrace) { - assert (m == lexer_mode::variable_line); + assert (m == lexer_mode::variable_line || + m == lexer_mode::for_loop); state_.top ().lsbrace = false; // Note: st is a copy. @@ -179,11 +194,20 @@ namespace build2 case '(': return make_token (type::lparen); } + if (m == lexer_mode::for_loop) + { + switch (c) + { + case ':': return make_token (type::colon); + } + } + // Command line operator/separators. // if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { switch (c) { @@ -205,7 +229,8 @@ namespace build2 // if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { if (optional<token> t = next_cmd_op (c, sep)) return move (*t); diff --git a/libbuild2/build/script/lexer.hxx b/libbuild2/build/script/lexer.hxx index 646d3b9..313d80a 100644 --- a/libbuild2/build/script/lexer.hxx +++ b/libbuild2/build/script/lexer.hxx @@ -24,9 +24,10 @@ namespace build2 enum { command_line = base_type::value_next, - first_token, // Expires at the end of the token. - second_token, // Expires at the end of the token. - variable_line // Expires at the end of the line. + first_token, // Expires at the end of the token. + second_token, // Expires at the end of the token. + variable_line, // Expires at the end of the line. + for_loop // Used for sensing the for-loop leading tokens. }; lexer_mode () = default; diff --git a/libbuild2/build/script/lexer.test.cxx b/libbuild2/build/script/lexer.test.cxx index e496f94..d8733ba 100644 --- a/libbuild2/build/script/lexer.test.cxx +++ b/libbuild2/build/script/lexer.test.cxx @@ -35,6 +35,7 @@ namespace build2 else if (s == "second-token") m = lexer_mode::second_token; else if (s == "variable-line") m = lexer_mode::variable_line; else if (s == "variable") m = lexer_mode::variable; + else if (s == "for-loop") m = lexer_mode::for_loop; else assert (false); } diff --git a/libbuild2/build/script/parser+for.test.testscript b/libbuild2/build/script/parser+for.test.testscript index 877f958..c5f6587 100644 --- a/libbuild2/build/script/parser+for.test.testscript +++ b/libbuild2/build/script/parser+for.test.testscript @@ -16,7 +16,7 @@ cmd end EOI - buildfile:11:4: error: expected variable name instead of <newline> + buildfile:11:1: error: for: missing variable name EOE : untyped @@ -180,3 +180,461 @@ EOE } } + +: form-2 +: +: ... | for x +: +{ + : for + : + { + : status + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x != 0 + cmd + end + EOI + buildfile:11:20: error: for-loop exit code cannot be checked + EOE + + : not-last + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x | echo x + cmd + end + EOI + buildfile:11:20: error: for-loop must be last command in a pipe + EOE + + : not-last-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x|echo x + cmd + end + EOI + buildfile:11:19: error: for-loop must be last command in a pipe + EOE + + : expression-after + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x && echo x + cmd + end + EOI + buildfile:11:20: error: command expression involving for-loop + EOE + + : expression-after-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x&&echo x + cmd + end + EOI + buildfile:11:19: error: command expression involving for-loop + EOE + + : expression-before + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && echo x | for x + cmd + end + EOI + buildfile:11:24: error: command expression involving for-loop + EOE + + : expression-before-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && echo x|for x + cmd + end + EOI + buildfile:11:22: error: command expression involving for-loop + EOE + + : cleanup + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x &f + cmd + end + EOI + buildfile:11:20: error: cleanup in for-loop + EOE + + : cleanup-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x&f + cmd + end + EOI + buildfile:11:19: error: cleanup in for-loop + EOE + + : stdout-redirect + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x >a + cmd + end + EOI + buildfile:11:20: error: output redirect in for-loop + EOE + + : stdout-redirect-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x>a + cmd + end + EOI + buildfile:11:19: error: output redirect in for-loop + EOE + + : stdin-redirect + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x <a + cmd + end + EOI + buildfile:11:20: error: stdin is both piped and redirected + EOE + + : no-var + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for + cmd + end + EOI + buildfile:11:1: error: for: missing variable name + EOE + + : untyped + : + $* <<EOI >>EOO + echo 'a b' | for -w x + cmd $x + end + EOI + echo 'a b' | for -w x + EOO + + : expansion + : + $* <<EOI >>EOO + vs = a b + echo $vs | for x + cmd $x + end + EOI + echo a b | for x + EOO + + : typed-var + : + $* <<EOI >>EOO + echo 'a b' | for -w [dir_paths] x + cmd $x + end + EOI + echo 'a b' | for -w [dir_paths] x + EOO + } + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + cmd + EOI + buildfile:13:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + elif true + cmd + end + end + EOI + buildfile:12:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + echo 'a b' | for x # 1 + cmd1 $x # 2 + if ($x == "a") # 3 + cmd2 # 4 + echo x y | for y # 5 + cmd3 # 6 + end + else + cmd4 # 7 + end + cmd5 # 8 + end + cmd6 # 9 + EOI + echo 'a b' | for x # 1 + cmd6 # 9 + EOO + } +} + +: form-3 +: +: for x <... +: +{ + : for + : + { + : status + : + $* <<EOI 2>>EOE != 0 + for x <a != 0 + cmd + end + EOI + buildfile:11:10: error: for-loop exit code cannot be checked + EOE + + : not-last + : + $* <<EOI 2>>EOE != 0 + for x <a | echo x + cmd + end + EOI + buildfile:11:10: error: for-loop must be last command in a pipe + EOE + + : not-last-relex + : + $* <<EOI 2>>EOE != 0 + for <a x|echo x + cmd + end + EOI + buildfile:11:9: error: for-loop must be last command in a pipe + EOE + + : expression-after + : + $* <<EOI 2>>EOE != 0 + for x <a && echo x + cmd + end + EOI + buildfile:11:10: error: command expression involving for-loop + EOE + + : expression-after-relex + : + $* <<EOI 2>>EOE != 0 + for <a x&&echo x + cmd + end + EOI + buildfile:11:9: error: command expression involving for-loop + EOE + + : expression-before + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && for x <a + cmd + end + EOI + buildfile:11:15: error: command expression involving for-loop + EOE + + : cleanup + : + $* <<EOI 2>>EOE != 0 + for x <a &f + cmd + end + EOI + buildfile:11:10: error: cleanup in for-loop + EOE + + : cleanup-before-var + : + $* <<EOI 2>>EOE != 0 + for &f x <a + cmd + end + EOI + buildfile:11:5: error: cleanup in for-loop + EOE + + : cleanup-relex + : + $* <<EOI 2>>EOE != 0 + for <a x&f + cmd + end + EOI + buildfile:11:9: error: cleanup in for-loop + EOE + + : stdout-redirect + : + $* <<EOI 2>>EOE != 0 + for x >a + cmd + end + EOI + buildfile:11:7: error: output redirect in for-loop + EOE + + : stdout-redirect-before-var + : + $* <<EOI 2>>EOE != 0 + for >a x + cmd + end + EOI + buildfile:11:5: error: output redirect in for-loop + EOE + + : stdout-redirect-relex + : + $* <<EOI 2>>EOE != 0 + for x>a + cmd + end + EOI + buildfile:11:6: error: output redirect in for-loop + EOE + + : no-var + : + $* <<EOI 2>>EOE != 0 + for <a + cmd + end + EOI + buildfile:11:1: error: for: missing variable name + EOE + + : untyped + : + $* <<EOI >>EOO + for -w x <'a b' + cmd $x + end + EOI + for -w x <'a b' + EOO + + : expansion + : + $* <<EOI >>EOO + vs = a b + for x <$vs + cmd $x + end + EOI + for x b <a + EOO + + : typed-var + : + $* <<EOI >>EOO + for -w [dir_path] x <'a b' + cmd $x + end + EOI + for -w [dir_path] x <'a b' + EOO + } + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + cmd + EOI + buildfile:13:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + elif true + cmd + end + end + EOI + buildfile:12:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + for -w x <'a b' # 1 + cmd1 $x # 2 + if ($x == "a") # 3 + cmd2 # 4 + for -w y <'x y' # 5 + cmd3 # 6 + end + else + cmd4 # 7 + end + cmd5 # 8 + end + cmd6 # 9 + EOI + for -w x <'a b' # 1 + cmd6 # 9 + EOO + } + + : contained + : + { + : eos + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + EOI + buildfile:12:1: error: expected closing 'end' + EOE + } +} diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index bcea7e0..035ab6b 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -205,10 +205,11 @@ namespace build2 // enter: next token is peeked at (type in tt) // leave: newline - assert (!fct || - *fct == line_type::cmd_if || - *fct == line_type::cmd_while || - *fct == line_type::cmd_for); + assert (!fct || + *fct == line_type::cmd_if || + *fct == line_type::cmd_while || + *fct == line_type::cmd_for_stream || + *fct == line_type::cmd_for_args); // Determine the line type/start token. // @@ -246,50 +247,107 @@ namespace build2 break; } - case line_type::cmd_for: + // + // See pre_parse_line_start() for details. + // + case line_type::cmd_for_args: assert (false); break; + case line_type::cmd_for_stream: { - // First take care of the variable name. + // First we need to sense the next few tokens and detect which + // form of the loop we are dealing with, the first (for x: ...) + // or the third (x <...) one. Note that the second form (... | for + // x) is handled separately. + // + // @@ Do we diagnose `... | for x: ...`? + // + // If the next token doesn't introduce a variable (doesn't start + // attributes and doesn't look like a variable name), then this is + // the third form. Otherwise, if colon follows the variable name, + // then this is the first form and the third form otherwise. + // + // Note that for the third form we will need to pass the 'for' + // token as a program name to the command expression parsing + // function since it will be gone from the token stream by that + // time. Thus, we save it. // - mode (lexer_mode::normal); + // Note also that in this model it won't be possible to support + // options in the first form. + // + token pt (t); + assert (pt.type == type::word && pt.value == "for"); + mode (lexer_mode::for_loop); next_with_attributes (t, tt); - attributes_push (t, tt); - if (tt != type::word || t.qtype != quote_type::unquoted) - fail (t) << "expected variable name instead of " << t; + // Note that we also consider special variable names (those that + // don't clash with the command line elements like redirects, etc) + // to later fail gracefully. + // + string& n (t.value); - const string& n (t.value); + if (tt == type::lsbrace || // Attributes. + (tt == type::word && // Variable name. + t.qtype == quote_type::unquoted && + (n[0] == '_' || alpha (n[0]) || n == "~"))) + { + attributes_push (t, tt); - if (special_variable (n)) - fail (t) << "attempt to set '" << n << "' variable directly"; + if (tt != type::word || t.qtype != quote_type::unquoted) + fail (t) << "expected variable name instead of " << t; - // We don't pre-enter variables. - // - ln.var = nullptr; + if (special_variable (n)) + fail (t) << "attempt to set '" << n << "' special variable"; - next (t, tt); + if (lexer_->peek_char ().first == ':') + lt = line_type::cmd_for_args; + } - if (tt != type::colon) + if (lt == line_type::cmd_for_stream) // for x <... { - // @@ TMP We will need to fallback to parsing the 'for x <...' - // form instead. + // At this point `t` contains the token that follows the `for` + // token and, potentially, the attributes. Now pre-parse the + // command expression in the command_line lexer mode starting + // from this position and also passing the 'for' token as a + // program name. // - fail (t) << "expected ':' instead of " << t - << " after variable name"; + // Note that the fact that the potential attributes are already + // parsed doesn't affect the command expression pre-parsing. + // Also note that they will be available during the execution + // phase being replayed. + // + expire_mode (); // Expire the for-loop lexer mode. + + parse_command_expr_result r ( + parse_command_expr (t, tt, + lexer::redirect_aliases, + move (pt))); + + assert (r.for_loop); + + if (tt != type::newline) + fail (t) << "expected newline instead of " << t; + + parse_here_documents (t, tt, r); } + else // for x: ... + { + next (t, tt); - expire_mode (); // Expire the normal lexer mode. + assert (tt == type::colon); - // Parse the value similar to the var line type (see above). - // - mode (lexer_mode::variable_line); - parse_variable_line (t, tt); + expire_mode (); // Expire the for-loop lexer mode. - if (tt != type::newline) - fail (t) << "expected newline instead of " << t << " after for"; + // Parse the value similar to the var line type (see above). + // + mode (lexer_mode::variable_line); + parse_variable_line (t, tt); - ++level_; + if (tt != type::newline) + fail (t) << "expected newline instead of " << t << " after for"; + } + ln.var = nullptr; + ++level_; break; } case line_type::cmd_elif: @@ -321,15 +379,24 @@ namespace build2 // Fall through. case line_type::cmd: { - pair<command_expr, here_docs> p; + parse_command_expr_result r; if (lt != line_type::cmd_else && lt != line_type::cmd_end) - p = parse_command_expr (t, tt, lexer::redirect_aliases); + r = parse_command_expr (t, tt, lexer::redirect_aliases); + + if (r.for_loop) + { + lt = line_type::cmd_for_stream; + ln.var = nullptr; + + ++level_; + } if (tt != type::newline) fail (t) << "expected newline instead of " << t; - parse_here_documents (t, tt, p); + parse_here_documents (t, tt, r); + break; } } @@ -358,7 +425,8 @@ namespace build2 break; } case line_type::cmd_while: - case line_type::cmd_for: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: { tt = peek (lexer_mode::first_token); @@ -396,7 +464,8 @@ namespace build2 break; } case line_type::cmd_while: - case line_type::cmd_for: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: { fct = bt; break; @@ -466,7 +535,9 @@ namespace build2 // enter: peeked first token of next line (type in tt) // leave: newline - assert (lt == line_type::cmd_while || lt == line_type::cmd_for); + assert (lt == line_type::cmd_while || + lt == line_type::cmd_for_stream || + lt == line_type::cmd_for_args); // Parse lines until we see closing 'end'. // @@ -491,12 +562,12 @@ namespace build2 // assert (!pre_parse_); - pair<command_expr, here_docs> p ( + parse_command_expr_result pr ( parse_command_expr (t, tt, lexer::redirect_aliases)); assert (tt == type::newline); - parse_here_documents (t, tt, p); + parse_here_documents (t, tt, pr); assert (tt == type::newline); // @@ Note that currently running programs via a runner (e.g., see @@ -509,7 +580,7 @@ namespace build2 // passed to the environment constructor, similar to passing the // script deadline. // - return move (p.first); + return move (pr.expr); } // @@ -1121,6 +1192,7 @@ namespace build2 auto exec_cmd = [this] (token& t, build2::script::token_type& tt, const iteration_index* ii, size_t li, bool single, + const function<command_function>& cf, const location& ll) { // We use the 0 index to signal that this is the only command. @@ -1131,7 +1203,7 @@ namespace build2 command_expr ce ( parse_command_line (t, static_cast<token_type&> (tt))); - runner_->run (*environment_, ce, ii, li, ll); + runner_->run (*environment_, ce, ii, li, cf, ll); }; exec_lines (s.body, exec_cmd); @@ -1184,6 +1256,7 @@ namespace build2 build2::script::token_type& tt, const iteration_index* ii, size_t li, bool /* single */, + const function<command_function>& cf, const location& ll) { // Note that we never reset the line index to zero (as we do in @@ -1282,14 +1355,17 @@ namespace build2 command_expr ce ( parse_command_line (t, static_cast<token_type&> (tt))); - // Verify that this expression executes the set builtin. + // Verify that this expression executes the set builtin or is a + // for-loop. // if (find_if (ce.begin (), ce.end (), - [] (const expr_term& et) + [&cf] (const expr_term& et) { const process_path& p (et.pipe.back ().program); return p.initial == nullptr && - p.recall.string () == "set"; + (p.recall.string () == "set" || + (cf != nullptr && + p.recall.string () == "for")); }) == ce.end ()) { const replay_tokens& rt (data.scr.depdb_preamble.back ().tokens); @@ -1301,7 +1377,7 @@ namespace build2 info (rt[0].location ()) << "depdb preamble ends here"; } - runner_->run (*environment_, ce, ii, li, ll); + runner_->run (*environment_, ce, ii, li, cf, ll); } }; @@ -2336,18 +2412,36 @@ namespace build2 { // Note: depdb is disallowed inside flow control constructs. // - string s; - build2::script::run (*environment_, - cmd, - nullptr /* iteration_index */, li, - ll, - !file ? &s : nullptr); - if (!file) { - iss.str (move (s)); + function<command_function> cf ( + [&iss] + (build2::script::environment&, + const strings&, + auto_fd in, + bool pipe, + const optional<deadline>& dl, + const command& deadline_cmd, + const location& ll) + { + iss.str (stream_read (move (in), + pipe, + dl, + deadline_cmd, + ll)); + }); + + build2::script::run (*environment_, + cmd, + nullptr /* iteration_index */, li, + ll, + cf, false /* last_cmd */); + iss.exceptions (istream::badbit); } + else + build2::script::run ( + *environment_, cmd, nullptr /* iteration_index */, li, ll); } ifdstream ifs (ifdstream::badbit); @@ -2490,7 +2584,8 @@ namespace build2 environment_->set_special_variables (a); } - // When add a special variable don't forget to update lexer::word(). + // When add a special variable don't forget to update lexer::word() and + // for-loop parsing in pre_parse_line(). // bool parser:: special_variable (const string& n) noexcept diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx index 061f3f8..7e9c612 100644 --- a/libbuild2/build/script/parser.test.cxx +++ b/libbuild2/build/script/parser.test.cxx @@ -37,11 +37,32 @@ namespace build2 enter (environment&, const location&) override {} virtual void - run (environment&, + run (environment& env, const command_expr& e, const iteration_index* ii, size_t i, - const location&) override + const function<command_function>& cf, + const location& ll) override { + // If the functions is specified, then just execute it with an empty + // stdin so it can perform the housekeeping (stop replaying tokens, + // increment line index, etc). + // + if (cf != nullptr) + { + assert (e.size () == 1 && !e[0].pipe.empty ()); + + const command& c (e[0].pipe.back ()); + + // Must be enforced by the caller. + // + assert (!c.out && !c.err && !c.exit); + + cf (env, c.arguments, + fdopen_null (), false /* pipe */, + nullopt /* deadline */, c, + ll); + } + cout << e; if (line_ || iterations_) diff --git a/libbuild2/build/script/runner.cxx b/libbuild2/build/script/runner.cxx index 157fc60..c52ef66 100644 --- a/libbuild2/build/script/runner.cxx +++ b/libbuild2/build/script/runner.cxx @@ -97,25 +97,28 @@ namespace build2 run (environment& env, const command_expr& expr, const iteration_index* ii, size_t li, + const function<command_function>& cf, const location& ll) { if (verb >= 3) text << ": " << expr; // Run the expression if we are not in the dry-run mode or if it - // executes the set or exit builtin and just print the expression - // otherwise at verbosity level 2 and up. + // executes the set or exit builtin or it is a for-loop. Otherwise, + // just print the expression otherwise at verbosity level 2 and up. // if (!env.context.dry_run || find_if (expr.begin (), expr.end (), - [] (const expr_term& et) + [&cf] (const expr_term& et) { const process_path& p (et.pipe.back ().program); return p.initial == nullptr && (p.recall.string () == "set" || - p.recall.string () == "exit"); + p.recall.string () == "exit" || + (cf != nullptr && + p.recall.string () == "for")); }) != expr.end ()) - build2::script::run (env, expr, ii, li, ll); + build2::script::run (env, expr, ii, li, ll, cf); else if (verb >= 2) text << expr; } diff --git a/libbuild2/build/script/runner.hxx b/libbuild2/build/script/runner.hxx index 0652396..ec8a948 100644 --- a/libbuild2/build/script/runner.hxx +++ b/libbuild2/build/script/runner.hxx @@ -32,10 +32,14 @@ namespace build2 // Location is the start position of this command line in the script. // It can be used in diagnostics. // + // Optionally, execute the specified function instead of the last + // pipe command. + // virtual void run (environment&, const command_expr&, const iteration_index*, size_t index, + const function<command_function>&, const location&) = 0; virtual bool @@ -66,6 +70,7 @@ namespace build2 run (environment&, const command_expr&, const iteration_index*, size_t, + const function<command_function>&, const location&) override; virtual bool diff --git a/libbuild2/build/script/script.cxx b/libbuild2/build/script/script.cxx index 2e777b4..9d9b5a8 100644 --- a/libbuild2/build/script/script.cxx +++ b/libbuild2/build/script/script.cxx @@ -156,7 +156,7 @@ namespace build2 } void environment:: - set_variable (string&& nm, + set_variable (string nm, names&& val, const string& attrs, const location& ll) diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx index 2c5e6e0..ec27781 100644 --- a/libbuild2/build/script/script.hxx +++ b/libbuild2/build/script/script.hxx @@ -24,11 +24,13 @@ namespace build2 using build2::script::lines; using build2::script::redirect; using build2::script::redirect_type; + using build2::script::command; using build2::script::expr_term; using build2::script::command_expr; using build2::script::iteration_index; using build2::script::deadline; using build2::script::timeout; + using build2::script::command_function; // Forward declarations. // @@ -166,7 +168,7 @@ namespace build2 size_t exec_line = 1; virtual void - set_variable (string&& name, + set_variable (string name, names&&, const string& attrs, const location&) override; diff --git a/libbuild2/script/builtin-options.cxx b/libbuild2/script/builtin-options.cxx index 8e15ddd..9b4067b 100644 --- a/libbuild2/script/builtin-options.cxx +++ b/libbuild2/script/builtin-options.cxx @@ -1076,6 +1076,297 @@ namespace build2 return r; } + + // for_options + // + + for_options:: + for_options () + : exact_ (), + newline_ (), + whitespace_ () + { + } + + for_options:: + for_options (int& argc, + char** argv, + bool erase, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + : exact_ (), + newline_ (), + whitespace_ () + { + ::build2::build::cli::argv_scanner s (argc, argv, erase); + _parse (s, opt, arg); + } + + for_options:: + for_options (int start, + int& argc, + char** argv, + bool erase, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + : exact_ (), + newline_ (), + whitespace_ () + { + ::build2::build::cli::argv_scanner s (start, argc, argv, erase); + _parse (s, opt, arg); + } + + for_options:: + for_options (int& argc, + char** argv, + int& end, + bool erase, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + : exact_ (), + newline_ (), + whitespace_ () + { + ::build2::build::cli::argv_scanner s (argc, argv, erase); + _parse (s, opt, arg); + end = s.end (); + } + + for_options:: + for_options (int start, + int& argc, + char** argv, + int& end, + bool erase, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + : exact_ (), + newline_ (), + whitespace_ () + { + ::build2::build::cli::argv_scanner s (start, argc, argv, erase); + _parse (s, opt, arg); + end = s.end (); + } + + for_options:: + for_options (::build2::build::cli::scanner& s, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + : exact_ (), + newline_ (), + whitespace_ () + { + _parse (s, opt, arg); + } + + typedef + std::map<std::string, void (*) (for_options&, ::build2::build::cli::scanner&)> + _cli_for_options_map; + + static _cli_for_options_map _cli_for_options_map_; + + struct _cli_for_options_map_init + { + _cli_for_options_map_init () + { + _cli_for_options_map_["--exact"] = + &::build2::build::cli::thunk< for_options, &for_options::exact_ >; + _cli_for_options_map_["-e"] = + &::build2::build::cli::thunk< for_options, &for_options::exact_ >; + _cli_for_options_map_["--newline"] = + &::build2::build::cli::thunk< for_options, &for_options::newline_ >; + _cli_for_options_map_["-n"] = + &::build2::build::cli::thunk< for_options, &for_options::newline_ >; + _cli_for_options_map_["--whitespace"] = + &::build2::build::cli::thunk< for_options, &for_options::whitespace_ >; + _cli_for_options_map_["-w"] = + &::build2::build::cli::thunk< for_options, &for_options::whitespace_ >; + } + }; + + static _cli_for_options_map_init _cli_for_options_map_init_; + + bool for_options:: + _parse (const char* o, ::build2::build::cli::scanner& s) + { + _cli_for_options_map::const_iterator i (_cli_for_options_map_.find (o)); + + if (i != _cli_for_options_map_.end ()) + { + (*(i->second)) (*this, s); + return true; + } + + return false; + } + + bool for_options:: + _parse (::build2::build::cli::scanner& s, + ::build2::build::cli::unknown_mode opt_mode, + ::build2::build::cli::unknown_mode arg_mode) + { + // Can't skip combined flags (--no-combined-flags). + // + assert (opt_mode != ::build2::build::cli::unknown_mode::skip); + + bool r = false; + bool opt = true; + + while (s.more ()) + { + const char* o = s.peek (); + + if (std::strcmp (o, "--") == 0) + { + opt = false; + s.skip (); + r = true; + continue; + } + + if (opt) + { + if (_parse (o, s)) + { + r = true; + continue; + } + + if (std::strncmp (o, "-", 1) == 0 && o[1] != '\0') + { + // Handle combined option values. + // + std::string co; + if (const char* v = std::strchr (o, '=')) + { + co.assign (o, 0, v - o); + ++v; + + int ac (2); + char* av[] = + { + const_cast<char*> (co.c_str ()), + const_cast<char*> (v) + }; + + ::build2::build::cli::argv_scanner ns (0, ac, av); + + if (_parse (co.c_str (), ns)) + { + // Parsed the option but not its value? + // + if (ns.end () != 2) + throw ::build2::build::cli::invalid_value (co, v); + + s.next (); + r = true; + continue; + } + else + { + // Set the unknown option and fall through. + // + o = co.c_str (); + } + } + + // Handle combined flags. + // + char cf[3]; + { + const char* p = o + 1; + for (; *p != '\0'; ++p) + { + if (!((*p >= 'a' && *p <= 'z') || + (*p >= 'A' && *p <= 'Z') || + (*p >= '0' && *p <= '9'))) + break; + } + + if (*p == '\0') + { + for (p = o + 1; *p != '\0'; ++p) + { + std::strcpy (cf, "-"); + cf[1] = *p; + cf[2] = '\0'; + + int ac (1); + char* av[] = + { + cf + }; + + ::build2::build::cli::argv_scanner ns (0, ac, av); + + if (!_parse (cf, ns)) + break; + } + + if (*p == '\0') + { + // All handled. + // + s.next (); + r = true; + continue; + } + else + { + // Set the unknown option and fall through. + // + o = cf; + } + } + } + + switch (opt_mode) + { + case ::build2::build::cli::unknown_mode::skip: + { + s.skip (); + r = true; + continue; + } + case ::build2::build::cli::unknown_mode::stop: + { + break; + } + case ::build2::build::cli::unknown_mode::fail: + { + throw ::build2::build::cli::unknown_option (o); + } + } + + break; + } + } + + switch (arg_mode) + { + case ::build2::build::cli::unknown_mode::skip: + { + s.skip (); + r = true; + continue; + } + case ::build2::build::cli::unknown_mode::stop: + { + break; + } + case ::build2::build::cli::unknown_mode::fail: + { + throw ::build2::build::cli::unknown_argument (o); + } + } + + break; + } + + return r; + } } } diff --git a/libbuild2/script/builtin-options.hxx b/libbuild2/script/builtin-options.hxx index c7cebbc..9361d18 100644 --- a/libbuild2/script/builtin-options.hxx +++ b/libbuild2/script/builtin-options.hxx @@ -253,6 +253,90 @@ namespace build2 vector<string> clear_; bool clear_specified_; }; + + class for_options + { + public: + for_options (); + + for_options (int& argc, + char** argv, + bool erase = false, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + for_options (int start, + int& argc, + char** argv, + bool erase = false, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + for_options (int& argc, + char** argv, + int& end, + bool erase = false, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + for_options (int start, + int& argc, + char** argv, + int& end, + bool erase = false, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + for_options (::build2::build::cli::scanner&, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + // Option accessors and modifiers. + // + const bool& + exact () const; + + bool& + exact (); + + void + exact (const bool&); + + const bool& + newline () const; + + bool& + newline (); + + void + newline (const bool&); + + const bool& + whitespace () const; + + bool& + whitespace (); + + void + whitespace (const bool&); + + // Implementation details. + // + protected: + bool + _parse (const char*, ::build2::build::cli::scanner&); + + private: + bool + _parse (::build2::build::cli::scanner&, + ::build2::build::cli::unknown_mode option, + ::build2::build::cli::unknown_mode argument); + + public: + bool exact_; + bool newline_; + bool whitespace_; + }; } } diff --git a/libbuild2/script/builtin-options.ixx b/libbuild2/script/builtin-options.ixx index 8f84177..575eb95 100644 --- a/libbuild2/script/builtin-options.ixx +++ b/libbuild2/script/builtin-options.ixx @@ -153,6 +153,63 @@ namespace build2 { this->clear_specified_ = x; } + + // for_options + // + + inline const bool& for_options:: + exact () const + { + return this->exact_; + } + + inline bool& for_options:: + exact () + { + return this->exact_; + } + + inline void for_options:: + exact (const bool& x) + { + this->exact_ = x; + } + + inline const bool& for_options:: + newline () const + { + return this->newline_; + } + + inline bool& for_options:: + newline () + { + return this->newline_; + } + + inline void for_options:: + newline (const bool& x) + { + this->newline_ = x; + } + + inline const bool& for_options:: + whitespace () const + { + return this->whitespace_; + } + + inline bool& for_options:: + whitespace () + { + return this->whitespace_; + } + + inline void for_options:: + whitespace (const bool& x) + { + this->whitespace_ = x; + } } } diff --git a/libbuild2/script/builtin.cli b/libbuild2/script/builtin.cli index 50dd3a0..c993983 100644 --- a/libbuild2/script/builtin.cli +++ b/libbuild2/script/builtin.cli @@ -30,5 +30,12 @@ namespace build2 vector<string> --unset|-u; vector<string> --clear|-c; }; + + class for_options + { + bool --exact|-e; + bool --newline|-n; + bool --whitespace|-w; + }; } } diff --git a/libbuild2/script/parser.cxx b/libbuild2/script/parser.cxx index 536821b..7989c20 100644 --- a/libbuild2/script/parser.cxx +++ b/libbuild2/script/parser.cxx @@ -4,10 +4,13 @@ #include <libbuild2/script/parser.hxx> #include <cstring> // strchr() +#include <sstream> #include <libbuild2/variable.hxx> -#include <libbuild2/script/run.hxx> // exit + +#include <libbuild2/script/run.hxx> // exit, stream_reader #include <libbuild2/script/lexer.hxx> +#include <libbuild2/script/builtin-options.hxx> using namespace std; @@ -140,18 +143,20 @@ namespace build2 return nullopt; } - pair<command_expr, parser::here_docs> parser:: + parser::parse_command_expr_result parser:: parse_command_expr (token& t, type& tt, - const redirect_aliases& ra) + const redirect_aliases& ra, + optional<token>&& program) { - // enter: first token of the command line + // enter: first (or second, if program) token of the command line // leave: <newline> or unknown token command_expr expr; // OR-ed to an implied false for the first term. // - expr.push_back ({expr_operator::log_or, command_pipe ()}); + if (!pre_parse_) + expr.push_back ({expr_operator::log_or, command_pipe ()}); command c; // Command being assembled. @@ -218,8 +223,8 @@ namespace build2 // Add the next word to either one of the pending positions or to // program arguments by default. // - auto add_word = [&c, &p, &mod, &check_regex_mod, this] ( - string&& w, const location& l) + auto add_word = [&c, &p, &mod, &check_regex_mod, this] + (string&& w, const location& l) { auto add_merge = [&l, this] (optional<redirect>& r, const string& w, @@ -697,11 +702,30 @@ namespace build2 const location ll (get_location (t)); // Line location. // Keep parsing chunks of the command line until we see one of the - // "terminators" (newline, exit status comparison, etc). + // "terminators" (newline or unknown/unexpected token). // location l (ll); names ns; // Reuse to reduce allocations. + bool for_loop (false); + + if (program) + { + assert (program->type == type::word); + + // Note that here we skip all the parse_program() business since the + // program can only be one of the specially-recognized names. + // + if (program->value == "for") + for_loop = true; + else + assert (false); // Must be specially-recognized program. + + // Save the program name and continue parsing as a command. + // + add_word (move (program->value), get_location (*program)); + } + for (bool done (false); !done; l = get_location (t)) { tt = ra.resolve (tt); @@ -717,6 +741,9 @@ namespace build2 case type::equal: case type::not_equal: { + if (for_loop) + fail (l) << "for-loop exit code cannot be checked"; + if (!pre_parse_) check_pending (l); @@ -747,30 +774,39 @@ namespace build2 } case type::pipe: + if (for_loop) + fail (l) << "for-loop must be last command in a pipe"; + // Fall through. + case type::log_or: case type::log_and: + if (for_loop) + fail (l) << "command expression involving for-loop"; + // Fall through. - case type::in_pass: - case type::out_pass: + case type::clean: + if (for_loop) + fail (l) << "cleanup in for-loop"; + // Fall through. - case type::in_null: + case type::out_pass: case type::out_null: - case type::out_trace: - case type::out_merge: - - case type::in_str: - case type::in_doc: case type::out_str: case type::out_doc: - - case type::in_file: case type::out_file_cmp: case type::out_file_ovr: case type::out_file_app: + if (for_loop) + fail (l) << "output redirect in for-loop"; + // Fall through. - case type::clean: + case type::in_pass: + case type::in_null: + case type::in_str: + case type::in_doc: + case type::in_file: { if (pre_parse_) { @@ -968,6 +1004,42 @@ namespace build2 next (t, tt); break; } + case type::lsbrace: + { + // Recompose the attributes into a single command argument. + // + assert (!pre_parse_); + + attributes_push (t, tt, true /* standalone */); + + attributes as (attributes_pop ()); + assert (!as.empty ()); + + ostringstream os; + names storage; + char c ('['); + for (const attribute& a: as) + { + os << c << a.name; + + if (!a.value.null) + { + os << '='; + + storage.clear (); + to_stream (os, + reverse (a.value, storage), + quote_mode::normal, + '@'); + } + + c = ','; + } + os << ']'; + + add_word (os.str (), l); + break; + } default: { // Bail out if this is one of the unknown tokens. @@ -1053,16 +1125,33 @@ namespace build2 bool prog (p == pending::program_first || p == pending::program_next); - // Check if this is the env pseudo-builtin. + // Check if this is the env pseudo-builtin or the for-loop. // bool env (false); - if (prog && tt == type::word && t.value == "env") + if (prog && tt == type::word) { - parsed_env r (parse_env_builtin (t, tt)); - c.cwd = move (r.cwd); - c.variables = move (r.variables); - c.timeout = r.timeout; - env = true; + if (t.value == "env") + { + parsed_env r (parse_env_builtin (t, tt)); + c.cwd = move (r.cwd); + c.variables = move (r.variables); + c.timeout = r.timeout; + env = true; + } + else if (t.value == "for") + { + if (expr.size () > 1) + fail (l) << "command expression involving for-loop"; + + for_loop = true; + + // Save 'for' as a program name and continue parsing as a + // command. + // + add_word (move (t.value), l); + next (t, tt); + continue; + } } // Parse the next chunk as names to get expansion, etc. Note that @@ -1243,9 +1332,16 @@ namespace build2 switch (tt) { case type::pipe: + if (for_loop) + fail (l) << "for-loop must be last command in a pipe"; + // Fall through. + case type::log_or: case type::log_and: { + if (for_loop) + fail (l) << "command expression involving for-loop"; + // Check that the previous command makes sense. // check_command (l, tt != type::pipe); @@ -1265,30 +1361,11 @@ namespace build2 break; } - case type::in_pass: - case type::out_pass: - - case type::in_null: - case type::out_null: - - case type::out_trace: - - case type::out_merge: - - case type::in_str: - case type::out_str: - - case type::in_file: - case type::out_file_cmp: - case type::out_file_ovr: - case type::out_file_app: - { - parse_redirect (move (t), tt, l); - break; - } - case type::clean: { + if (for_loop) + fail (l) << "cleanup in for-loop"; + parse_clean (t); break; } @@ -1299,6 +1376,27 @@ namespace build2 fail (l) << "here-document redirect in expansion"; break; } + + case type::out_pass: + case type::out_null: + case type::out_trace: + case type::out_merge: + case type::out_str: + case type::out_file_cmp: + case type::out_file_ovr: + case type::out_file_app: + if (for_loop) + fail (l) << "output redirect in for-loop"; + // Fall through. + + case type::in_pass: + case type::in_null: + case type::in_str: + case type::in_file: + { + parse_redirect (move (t), tt, l); + break; + } } } @@ -1326,7 +1424,7 @@ namespace build2 expr.back ().pipe.push_back (move (c)); } - return make_pair (move (expr), move (hd)); + return parse_command_expr_result {move (expr), move (hd), for_loop}; } parser::parsed_env parser:: @@ -1575,7 +1673,7 @@ namespace build2 void parser:: parse_here_documents (token& t, type& tt, - pair<command_expr, here_docs>& p) + parse_command_expr_result& pr) { // enter: newline // leave: newline @@ -1583,7 +1681,7 @@ namespace build2 // Parse here-document fragments in the order they were mentioned on // the command line. // - for (here_doc& h: p.second) + for (here_doc& h: pr.docs) { // Switch to the here-line mode which is like single/double-quoted // string but recognized the newline as a separator. @@ -1603,7 +1701,7 @@ namespace build2 { auto i (h.redirects.cbegin ()); - command& c (p.first[i->expr].pipe[i->pipe]); + command& c (pr.expr[i->expr].pipe[i->pipe]); optional<redirect>& r (i->fd == 0 ? c.in : i->fd == 1 ? c.out : @@ -1635,7 +1733,7 @@ namespace build2 // for (++i; i != h.redirects.cend (); ++i) { - command& c (p.first[i->expr].pipe[i->pipe]); + command& c (pr.expr[i->expr].pipe[i->pipe]); optional<redirect>& ir (i->fd == 0 ? c.in : i->fd == 1 ? c.out : @@ -2062,7 +2160,7 @@ namespace build2 else if (n == "elif!") r = line_type::cmd_elifn; else if (n == "else") r = line_type::cmd_else; else if (n == "while") r = line_type::cmd_while; - else if (n == "for") r = line_type::cmd_for; + else if (n == "for") r = line_type::cmd_for_stream; else if (n == "end") r = line_type::cmd_end; else { @@ -2136,10 +2234,11 @@ namespace build2 { line_type lt (j->type); - if (lt == line_type::cmd_if || - lt == line_type::cmd_ifn || - lt == line_type::cmd_while || - lt == line_type::cmd_for) + if (lt == line_type::cmd_if || + lt == line_type::cmd_ifn || + lt == line_type::cmd_while || + lt == line_type::cmd_for_stream || + lt == line_type::cmd_for_args) ++n; // If we are nested then we just wait until we get back @@ -2164,10 +2263,8 @@ namespace build2 if (skip) { - // Note that we don't count else and end as commands. - // - // @@ Note that for the for-loop's second and third forms - // will probably need to increment li. + // Note that we don't count else, end, and 'for x: ...' as + // commands. // switch (lt) { @@ -2176,8 +2273,9 @@ namespace build2 case line_type::cmd_ifn: case line_type::cmd_elif: case line_type::cmd_elifn: - case line_type::cmd_while: ++li; break; - default: break; + case line_type::cmd_for_stream: + case line_type::cmd_while: ++li; break; + default: break; } } } @@ -2221,7 +2319,10 @@ namespace build2 single = true; } - exec_cmd (t, tt, ii, li++, single, ll); + exec_cmd (t, tt, + ii, li++, single, + nullptr /* command_function */, + ll); replay_stop (); break; @@ -2339,7 +2440,147 @@ namespace build2 break; } - case line_type::cmd_for: + case line_type::cmd_for_stream: + { + // The for-loop construct end. Set on the first iteration. + // + lines::const_iterator fe (e); + + // Let's "wrap up" all the required data into the single object + // to rely on the "small function object" optimization. + // + struct + { + lines::const_iterator i; + lines::const_iterator e; + const function<exec_set_function>& exec_set; + const function<exec_cmd_function>& exec_cmd; + const function<exec_cond_function>& exec_cond; + const function<exec_for_function>& exec_for; + const iteration_index* ii; + size_t& li; + variable_pool* var_pool; + decltype (fcend)& fce; + lines::const_iterator& fe; + } d {i, e, + exec_set, exec_cmd, exec_cond, exec_for, + ii, li, + var_pool, + fcend, + fe}; + + function<command_function> cf ( + [&d, this] + (environment& env, + const strings& args, + auto_fd in, + bool pipe, + const optional<deadline>& dl, + const command& deadline_cmd, + const location& ll) + { + namespace cli = build2::build::cli; + + try + { + // Parse arguments. + // + cli::vector_scanner scan (args); + for_options ops (scan); + + // Note: diagnostics consistent with the set builtin. + // + if (ops.whitespace () && ops.newline ()) + fail (ll) << "for: both -n|--newline and " + << "-w|--whitespace specified"; + + if (!scan.more ()) + fail (ll) << "for: missing variable name"; + + // Either attributes or variable name. + // + string a (scan.next ()); + const string* ats (!scan.more () ? nullptr : &a); + string vname (!scan.more () ? move (a) : scan.next ()); + + if (scan.more ()) + fail (ll) << "for: unexpected argument '" + << scan.next () << "'"; + + if (ats != nullptr && ats->empty ()) + fail (ll) << "for: empty variable attributes"; + + if (vname.empty ()) + fail (ll) << "for: empty variable name"; + + // Let's also diagnose the `... | for x:...` misuse which + // can probably be quite common. + // + if (vname.find (':') != string::npos) + fail (ll) << "for: ':' after variable name"; + + stream_reader sr ( + move (in), pipe, + ops.whitespace (), ops.newline (), ops.exact (), + dl, deadline_cmd, + ll); + + // Since the command pipe is parsed, we can stop + // replaying. Note that we should do this before calling + // exec_lines() for the loop body. Also note that we + // should increment the line index before that. + // + replay_stop (); + + size_t fli (++d.li); + iteration_index fi {1, d.ii}; + + for (optional<string> s; (s = sr.next ()); ) + { + d.li = fli; + + // Don't move from the variable name since it is used on + // each iteration. + // + env.set_variable (vname, + names {name (move (*s))}, + ats != nullptr ? *ats : empty_string, + ll); + + // Find the construct end, if it is not found yet. + // + if (d.fe == d.e) + d.fe = d.fce (d.i, true, false); + + if (!exec_lines (d.i + 1, d.fe, + d.exec_set, + d.exec_cmd, + d.exec_cond, + d.exec_for, + &fi, d.li, + d.var_pool)) + { + throw exit (true); + } + + fi.index++; + } + } + catch (const cli::exception& e) + { + fail (ll) << "for: " << e; + } + }); + + exec_cmd (t, tt, ii, li, false /* single */, cf, ll); + + // Position to construct end. + // + i = (fe != e ? fe : fcend (i, true, true)); + + break; + } + case line_type::cmd_for_args: { // Parse the variable name with the potential attributes. // diff --git a/libbuild2/script/parser.hxx b/libbuild2/script/parser.hxx index 3a4c46f..c402d3e 100644 --- a/libbuild2/script/parser.hxx +++ b/libbuild2/script/parser.hxx @@ -97,15 +97,34 @@ namespace build2 }; using here_docs = vector<here_doc>; - pair<command_expr, here_docs> - parse_command_expr (token&, token_type&, const redirect_aliases&); + struct parse_command_expr_result + { + command_expr expr; // Single pipe for the for-loop. + here_docs docs; + bool for_loop = false; + + parse_command_expr_result () = default; + + parse_command_expr_result (command_expr&& e, + here_docs&& h, + bool f) + : expr (move (e)), docs (move (h)), for_loop (f) {} + }; + + // Pass the first special command program name (token_type::word) if it + // is already pre-parsed. + // + parse_command_expr_result + parse_command_expr (token&, token_type&, + const redirect_aliases&, + optional<token>&& program = nullopt); command_exit parse_command_exit (token&, token_type&); void parse_here_documents (token&, token_type&, - pair<command_expr, here_docs>&); + parse_command_expr_result&); struct parsed_doc { @@ -135,6 +154,11 @@ namespace build2 // the first two tokens. Use the specified lexer mode to peek the second // token. // + // Always return the cmd_for_stream line type for the for-loop. Note + // that the for-loop form cannot be detected easily, based on the first + // two tokens. Also note that the detection can be specific for the + // script implementation (custom lexing mode, special variables, etc). + // line_type pre_parse_line_start (token&, token_type&, lexer_mode); @@ -169,6 +193,7 @@ namespace build2 using exec_cmd_function = void (token&, token_type&, const iteration_index*, size_t li, bool single, + const function<command_function>&, const location&); using exec_cond_function = bool (token&, token_type&, diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx index 81abdab..b7f3314 100644 --- a/libbuild2/script/run.cxx +++ b/libbuild2/script/run.cxx @@ -9,7 +9,8 @@ # include <libbutl/win32-utility.hxx> // DBG_TERMINATE_PROCESS #endif -#include <ios> // streamsize +#include <ios> // streamsize +#include <cstring> // strchr() #include <libbutl/regex.hxx> #include <libbutl/builtin.hxx> @@ -971,81 +972,201 @@ namespace build2 : path (c.program.recall_string ()); } - // Read out the stream content into a string. Throw io_error on the - // underlying OS error. - // - // If the execution deadline is specified, then turn the stream into the - // non-blocking mode reading its content in chunks and with a single - // operation otherwise. If the specified deadline is reached while - // reading the stream, then bail out for the successful deadline and - // fail otherwise. Note that in the former case the result will be - // incomplete, but we leave it to the caller to handle that. - // - // Note that on Windows we can only turn pipe file descriptors into the - // non-blocking mode. Thus, we have no choice but to read from - // descriptors of other types synchronously there. That implies that we - // can potentially block indefinitely reading a file and missing the - // deadline on Windows. Note though, that the user can normally rewrite - // the command, for example, `set foo <<<file` with `cat file | set foo` - // to avoid this problem. - // - static string - read (auto_fd in, + stream_reader:: + stream_reader(auto_fd&& in, #ifndef _WIN32 - bool, + bool, #else - bool pipe, + bool pipe, #endif - const optional<deadline>& dl, - const command& deadline_cmd, - const location& ll) + bool ws, bool nl, bool ex, + const optional<deadline>& dl, + const command& dc, + const location& l) + : whitespace_ (ws), + newline_ (nl), + exact_ (ex), + deadline_cmd_ (dc), + location_ (l) { - string r; - ifdstream cin; - #ifndef _WIN32 if (dl) #else if (dl && pipe) #endif { - fdselect_set fds {in.get ()}; - cin.open (move (in), fdstream_mode::non_blocking); + is_.open (move (in), fdstream_mode::non_blocking); + deadline_ = dl; + } + else + is_.open (move (in)); + } - const timestamp& dlt (dl->value); + optional<string> stream_reader:: + next () + { + if (!is_.is_open ()) + return nullopt; + + // If eos is not reached, then read and return a character. Otherwise + // close the stream and return nullopt. If the deadline is specified and + // is reached, then return nullopt for the successful deadline (as if + // eof is reached) and fail otherwise. + // + // Set the empty_ flag to false after the first character is read. + // + auto get = [this] () -> optional<char> + { + char r; - for (char buf[4096];; ) + if (deadline_) // Reading a character in the non-blocking mode. { - timestamp now (system_clock::now ()); + fdselect_set fds {is_.fd ()}; - if (dlt <= now || ifdselect (fds, dlt - now) == 0) + // Only fallback to ifdselect() if there is no character immediately + // available. + // + for (;;) { - if (!dl->success) - fail (ll) << cmd_path (deadline_cmd) - << " terminated: execution timeout expired"; - else + streamsize n (is_.readsome (&r, 1)); + + if (n == 1) break; + + if (is_.eof ()) + { + is_.close (); + return nullopt; + } + + const timestamp& dlt (deadline_->value); + timestamp now (system_clock::now ()); + + if (dlt <= now || ifdselect (fds, dlt - now) == 0) + { + is_.close (); + + if (!deadline_->success) + fail (location_) << cmd_path (deadline_cmd_) + << " terminated: execution timeout expired"; + else + return nullopt; + } + } + } + else // Reading a character in the blocking mode. + { + if (is_.peek () == ifdstream::traits_type::eof ()) + { + is_.close (); + return nullopt; } - streamsize n (cin.readsome (buf, sizeof (buf))); + is_.get (r); + } - // Bail out if eos is reached. - // - if (n == 0) - break; + empty_ = false; + return r; + }; + + if (whitespace_) // The whitespace mode. + { + const char* sep (" \n\r\t"); + + // Note that we collapse multiple consecutive whitespaces. + // + optional<char> c; + + // Skip the whitespaces. + // + while ((c = get ()) && strchr (sep, *c) != nullptr) ; - r.append (buf, n); + // Bail out for the trailing whitespace(s) or an empty stream. + // + if (!c) + { + // Return the trailing "blank" after the trailing whitespaces in the + // exact mode, unless the stream is empty. + // + return exact_ && !empty_ ? empty_string : optional<string> (); } + + // Read the word until eof or a whitespace character is encountered. + // + string r (1, *c); + while ((c = get ()) && strchr (sep, *c) == nullptr) + r += *c; + + return optional<string> (move (r)); } - else + else // The newline or no-split mode. { - cin.open (move (in)); - r = cin.read_text (); - } + // Note that we don't collapse multiple consecutive newlines. + // + // Note also that we always sanitize CRs, so in the no-split mode we + // need to loop rather than read the whole text at once. + // + optional<string> r; - cin.close (); + do + { + string l; + optional<char> c; - return r; + // Read the line until eof or newline character is encountered. + // + while ((c = get ()) && *c != '\n') + l += *c; + + // Strip the trailing CRs that can appear while, for example, + // cross-testing Windows target or as a part of msvcrt junk + // production (see above). + // + while (!l.empty () && l.back () == '\r') + l.pop_back (); + + // Append the line. + // + if (!l.empty () || // Non-empty. + c || // Empty, non-trailing. + (exact_ && // Empty, trailing, in the exact mode for + !empty_)) // non-empty stream. + { + if (newline_ || !r) + { + r = move (l); + } + else + { + *r += '\n'; + *r += l; + } + } + } + while (!newline_ && is_.is_open ()); + + return r; + } + } + + string + stream_read (auto_fd&& in, + bool pipe, + const optional<deadline>& dl, + const command& dc, + const location& ll) + { + stream_reader sr (move (in), + pipe, + false /* whitespace */, + false /* newline */, + true /* exact */, + dl, + dc, + ll); + + optional<string> s (sr.next ()); + return s ? move (*s) : empty_string; } // The set pseudo-builtin: set variable from the stdin input. @@ -1087,87 +1208,17 @@ namespace build2 if (vname.empty ()) fail (ll) << "set: empty variable name"; - // Read out the stream content into a string while keeping an eye on - // the deadline. - // - string s (read (move (in), pipe, dl, deadline_cmd, ll)); + stream_reader sr (move (in), pipe, + ops.whitespace (), ops.newline (), ops.exact (), + dl, deadline_cmd, + ll); // Parse the stream content into the variable value. // names ns; - if (!s.empty ()) - { - if (ops.whitespace ()) // The whitespace mode. - { - // Note that we collapse multiple consecutive whitespaces. - // - for (size_t p (0); p != string::npos; ) - { - // Skip the whitespaces. - // - const char* sep (" \n\r\t"); - size_t b (s.find_first_not_of (sep, p)); - - if (b != string::npos) // Word beginning. - { - size_t e (s.find_first_of (sep, b)); // Find the word end. - ns.emplace_back (string (s, b, e != string::npos ? e - b : e)); - - p = e; - } - else // Trailings whitespaces. - { - // Append the trailing "blank" after the trailing whitespaces - // in the exact mode. - // - if (ops.exact ()) - ns.emplace_back (empty_string); - - // Bail out since the end of the string is reached. - // - break; - } - } - } - else // The newline or no-split mode. - { - // Note that we don't collapse multiple consecutive newlines. - // - // Note also that we always sanitize CRs so this loop is always - // needed. - // - for (size_t p (0); p != string::npos; ) - { - size_t e (s.find ('\n', p)); - string l (s, p, e != string::npos ? e - p : e); - - // Strip the trailing CRs that can appear while, for example, - // cross-testing Windows target or as a part of msvcrt junk - // production (see above). - // - while (!l.empty () && l.back () == '\r') - l.pop_back (); - - // Append the line. - // - if (!l.empty () || // Non-empty. - e != string::npos || // Empty, non-trailing. - ops.exact ()) // Empty, trailing, in the exact mode. - { - if (ops.newline () || ns.empty ()) - ns.emplace_back (move (l)); - else - { - ns[0].value += '\n'; - ns[0].value += l; - } - } - - p = e != string::npos ? e + 1 : e; - } - } - } + for (optional<string> s; (s = sr.next ()); ) + ns.emplace_back (move (*s)); env.set_variable (move (vname), move (ns), @@ -1242,7 +1293,7 @@ namespace build2 const iteration_index* ii, size_t li, size_t ci, const location& ll, bool diag, - string* output, + const function<command_function>& cf, bool last_cmd, optional<deadline> dl = nullopt, const command* dl_cmd = nullptr, // env -t <cmd> pipe_command* prev_cmd = nullptr) @@ -1253,8 +1304,10 @@ namespace build2 // if (bc == ec) { - if (output != nullptr) + if (cf != nullptr) { + assert (!last_cmd); // Otherwise we wouldn't be here. + // The pipeline can't be empty. // assert (ifd != nullfd && prev_cmd != nullptr); @@ -1263,15 +1316,14 @@ namespace build2 try { - *output = read (move (ifd), - true /* pipe */, - dl, - dl_cmd != nullptr ? *dl_cmd : c, - ll); + cf (env, strings () /* arguments */, + move (ifd), true /* pipe */, + dl, dl_cmd != nullptr ? *dl_cmd : c, + ll); } catch (const io_error& e) { - fail (ll) << "io error reading " << cmd_path (c) << " output: " + fail (ll) << "unable to read from " << cmd_path (c) << " output: " << e; } } @@ -1329,9 +1381,10 @@ namespace build2 command_pipe::const_iterator nc (bc + 1); bool last (nc == ec); - // Make sure that stdout is not redirected if meant to be read. + // Make sure that stdout is not redirected if meant to be read (last_cmd + // is false) or cannot not be produced (last_cmd is true). // - if (last && output != nullptr && c.out) + if (last && c.out && cf != nullptr) fail (ll) << "stdout cannot be redirected"; // True if the process path is not pre-searched and the program path @@ -1345,7 +1398,7 @@ namespace build2 const redirect& in ((c.in ? *c.in : env.in).effective ()); - const redirect* out (!last || output != nullptr + const redirect* out (!last || (cf != nullptr && !last_cmd) ? nullptr // stdout is piped. : &(c.out ? *c.out : env.out).effective ()); @@ -1413,7 +1466,7 @@ namespace build2 if (c.out) fail (ll) << program << " builtin stdout cannot be redirected"; - if (output != nullptr) + if (cf != nullptr && !last_cmd) fail (ll) << program << " builtin stdout cannot be read"; if (c.err) @@ -1620,7 +1673,7 @@ namespace build2 if (c.out) fail (ll) << "set builtin stdout cannot be redirected"; - if (output != nullptr) + if (cf != nullptr && !last_cmd) fail (ll) << "set builtin stdout cannot be read"; if (c.err) @@ -1640,6 +1693,39 @@ namespace build2 return true; } + // If this is the last command in the pipe and the command function is + // specified for it, then call it. + // + if (last && cf != nullptr && last_cmd) + { + // Must be enforced by the caller. + // + assert (!c.out && !c.err && !c.exit); + + try + { + cf (env, c.arguments, + move (ifd), !first, + dl, dl_cmd != nullptr ? *dl_cmd : c, + ll); + } + catch (const io_error& e) + { + diag_record dr (fail (ll)); + + dr << cmd_path (c) << ": unable to read from "; + + if (prev_cmd != nullptr) + dr << cmd_path (prev_cmd->cmd) << " output"; + else + dr << "stdin"; + + dr << ": " << e; + } + + return true; + } + // Open a file for command output redirect if requested explicitly // (file overwrite/append redirects) or for the purpose of the output // validation (none, here_*, file comparison redirects), register the @@ -2220,7 +2306,7 @@ namespace build2 nc, ec, move (ofd.in), ii, li, ci + 1, ll, diag, - output, + cf, last_cmd, dl, dl_cmd, &pc); @@ -2347,7 +2433,7 @@ namespace build2 nc, ec, move (ofd.in), ii, li, ci + 1, ll, diag, - output, + cf, last_cmd, dl, dl_cmd, &pc); @@ -2487,7 +2573,7 @@ namespace build2 const iteration_index* ii, size_t li, const location& ll, bool diag, - string* output) + const function<command_function>& cf, bool last_cmd) { // Commands are numbered sequentially throughout the expression // starting with 1. Number 0 means the command is a single one. @@ -2532,7 +2618,7 @@ namespace build2 p.begin (), p.end (), auto_fd (), ii, li, ci, ll, print, - output); + cf, last_cmd); } ci += p.size (); @@ -2546,13 +2632,18 @@ namespace build2 const command_expr& expr, const iteration_index* ii, size_t li, const location& ll, - string* output) + const function<command_function>& cf, + bool last_cmd) { // Note that we don't print the expression at any verbosity level // assuming that the caller does this, potentially providing some // additional information (command type, etc). // - if (!run_expr (env, expr, ii, li, ll, true /* diag */, output)) + if (!run_expr (env, + expr, + ii, li, ll, + true /* diag */, + cf, last_cmd)) throw failed (); // Assume diagnostics is already printed. } @@ -2561,11 +2652,15 @@ namespace build2 const command_expr& expr, const iteration_index* ii, size_t li, const location& ll, - string* output) + const function<command_function>& cf, bool last_cmd) { // Note that we don't print the expression here (see above). // - return run_expr (env, expr, ii, li, ll, false /* diag */, output); + return run_expr (env, + expr, + ii, li, ll, + false /* diag */, + cf, last_cmd); } void diff --git a/libbuild2/script/run.hxx b/libbuild2/script/run.hxx index 01b010c..5d46d21 100644 --- a/libbuild2/script/run.hxx +++ b/libbuild2/script/run.hxx @@ -38,22 +38,24 @@ namespace build2 // Location is the start position of this command line in the script. It // can be used in diagnostics. // - // Optionally, save the command output into the referenced variable. In - // this case assume that the expression contains a single pipline. + // Optionally, execute the specified function at the end of the pipe, + // either after the last command or instead of it. // void run (environment&, const command_expr&, const iteration_index*, size_t index, const location&, - string* output = nullptr); + const function<command_function>& = nullptr, + bool last_cmd = true); bool run_cond (environment&, const command_expr&, const iteration_index*, size_t index, const location&, - string* output = nullptr); + const function<command_function>& = nullptr, + bool last_cmd = true); // Perform the registered special file cleanups in the direct order and // then the regular cleanups in the reverse order. @@ -80,6 +82,62 @@ namespace build2 // string diag_path (const dir_name_view&); + + // Read out the stream content into a string, optionally splitting the + // input data at whitespaces or newlines in which case return one + // sub-string at a time (see the set builtin options for the splitting + // semantics). Throw io_error on the underlying OS error. + // + // If the execution deadline is specified, then turn the stream into the + // non-blocking mode. If the specified deadline is reached while reading + // the stream, then bail out for the successful deadline and fail + // otherwise. Note that in the former case the result will be incomplete, + // but we leave it to the caller to handle that. + // + // Note that on Windows we can only turn pipe file descriptors into the + // non-blocking mode. Thus, we have no choice but to read from descriptors + // of other types synchronously there. That implies that we can + // potentially block indefinitely reading a file and missing the deadline + // on Windows. Note though, that the user can normally rewrite the + // command, for example, `set foo <<<file` with `cat file | set foo` to + // avoid this problem. + // + class stream_reader + { + public: + stream_reader (auto_fd&&, + bool pipe, + bool whitespace, bool newline, bool exact, + const optional<deadline>&, + const command& deadline_cmd, + const location&); + + // Return nullopt if eos is reached. + // + optional<string> + next (); + + private: + ifdstream is_; + bool whitespace_; + bool newline_; + bool exact_; + optional<deadline> deadline_; + const command& deadline_cmd_; + const location& location_; + + bool empty_ = true; // Set to false after the first character is read. + }; + + // Read the stream content using the stream reader in the no-split exact + // mode. + // + string + stream_read (auto_fd&&, + bool pipe, + const optional<deadline>&, + const command& deadline_cmd, + const location&); } } diff --git a/libbuild2/script/script.cxx b/libbuild2/script/script.cxx index 33c4c30..b8dfc68 100644 --- a/libbuild2/script/script.cxx +++ b/libbuild2/script/script.cxx @@ -20,16 +20,17 @@ namespace build2 switch (lt) { - case line_type::var: s = "variable"; break; - case line_type::cmd: s = "command"; break; - case line_type::cmd_if: s = "'if'"; break; - case line_type::cmd_ifn: s = "'if!'"; break; - case line_type::cmd_elif: s = "'elif'"; break; - case line_type::cmd_elifn: s = "'elif!'"; break; - case line_type::cmd_else: s = "'else'"; break; - case line_type::cmd_while: s = "'while'"; break; - case line_type::cmd_for: s = "'for'"; break; - case line_type::cmd_end: s = "'end'"; break; + case line_type::var: s = "variable"; break; + case line_type::cmd: s = "command"; break; + case line_type::cmd_if: s = "'if'"; break; + case line_type::cmd_ifn: s = "'if!'"; break; + case line_type::cmd_elif: s = "'elif'"; break; + case line_type::cmd_elifn: s = "'elif!'"; break; + case line_type::cmd_else: s = "'else'"; break; + case line_type::cmd_while: s = "'while'"; break; + case line_type::cmd_for_args: s = "'for'"; break; + case line_type::cmd_for_stream: s = "'for'"; break; + case line_type::cmd_end: s = "'end'"; break; } return o << s; @@ -227,7 +228,8 @@ namespace build2 case line_type::cmd_elifn: case line_type::cmd_else: case line_type::cmd_while: - case line_type::cmd_for: fc_ind += " "; break; + case line_type::cmd_for_args: + case line_type::cmd_for_stream: fc_ind += " "; break; default: break; } diff --git a/libbuild2/script/script.hxx b/libbuild2/script/script.hxx index 5eb4ee9..aa96b7f 100644 --- a/libbuild2/script/script.hxx +++ b/libbuild2/script/script.hxx @@ -28,7 +28,8 @@ namespace build2 cmd_elifn, cmd_else, cmd_while, - cmd_for, + cmd_for_args, // `for x: ...` + cmd_for_stream, // `... | for x` and `for x <...` cmd_end }; @@ -42,7 +43,7 @@ namespace build2 union { - const variable* var; // Pre-entered for line_type::var. + const variable* var; // Pre-entered for line_type::{var,cmd_for_*}. }; }; @@ -547,7 +548,7 @@ namespace build2 // Set variable value with optional (non-empty) attributes. // virtual void - set_variable (string&& name, + set_variable (string name, names&&, const string& attrs, const location&) = 0; @@ -580,6 +581,17 @@ namespace build2 ~environment () = default; }; + // Custom command function that can be executed at the end of the pipe. + // Should throw io_error on the underlying OS error. + // + using command_function = void (environment&, + const strings& args, + auto_fd in, + bool pipe, + const optional<deadline>&, + const command& deadline_cmd, + const location&); + // Helpers. // // Issue diagnostics with the specified prefix and fail if the string diff --git a/libbuild2/test/script/lexer+for-loop.test.testscript b/libbuild2/test/script/lexer+for-loop.test.testscript new file mode 100644 index 0000000..fcd12f7 --- /dev/null +++ b/libbuild2/test/script/lexer+for-loop.test.testscript @@ -0,0 +1,231 @@ +# file : libbuild2/test/script/lexer+for-loop.test.testscript +# license : MIT; see accompanying LICENSE file + +test.arguments = for-loop + +: semi +{ + : immediate + : + $* <"cmd;" >>EOO + 'cmd' + ; + <newline> + EOO + + : separated + : + $* <"cmd ;" >>EOO + 'cmd' + ; + <newline> + EOO + + : only + : + $* <";" >>EOO + ; + <newline> + EOO +} + +: colon +: +{ + : immediate + : + $* <"cmd: dsc" >>EOO + 'cmd' + : + 'dsc' + <newline> + EOO + + : separated + : + $* <"cmd :dsc" >>EOO + 'cmd' + : + 'dsc' + <newline> + EOO + + : only + : + $* <":" >>EOO + : + <newline> + EOO +} + +: redirect +: +{ + : pass + : + $* <"cmd <| 1>|" >>EOO + 'cmd' + <| + '1' + >| + <newline> + EOO + + : null + : + $* <"cmd <- 1>-" >>EOO + 'cmd' + <- + '1' + >- + <newline> + EOO + + : trace + : + $* <"cmd 1>!" >>EOO + 'cmd' + '1' + >! + <newline> + EOO + + : merge + : + $* <"cmd 1>&2" >>EOO + 'cmd' + '1' + >& + '2' + <newline> + EOO + + : str + : + $* <"cmd <a 1>b" >>EOO + 'cmd' + < + 'a' + '1' + > + 'b' + <newline> + EOO + + : str-nn + : + $* <"cmd <:a 1>:b" >>EOO + 'cmd' + <: + 'a' + '1' + >: + 'b' + <newline> + EOO + + : doc + : + $* <"cmd <<EOI 1>>EOO" >>EOO + 'cmd' + << + 'EOI' + '1' + >> + 'EOO' + <newline> + EOO + + : doc-nn + : + $* <"cmd <<:EOI 1>>:EOO" >>EOO + 'cmd' + <<: + 'EOI' + '1' + >>: + 'EOO' + <newline> + EOO + + : file-cmp + : + $* <"cmd <<<in >>>out 2>>>err" >>EOO + 'cmd' + <<< + 'in' + >>> + 'out' + '2' + >>> + 'err' + <newline> + EOO + + : file-write + : + $* <"cmd >=out 2>+err" >>EOO + 'cmd' + >= + 'out' + '2' + >+ + 'err' + <newline> + EOO +} + +: cleanup +: +{ + : always + : + $* <"cmd &file" >>EOO + 'cmd' + & + 'file' + <newline> + EOO + + : maybe + : + $* <"cmd &?file" >>EOO + 'cmd' + &? + 'file' + <newline> + EOO + + : never + : + $* <"cmd &!file" >>EOO + 'cmd' + &! + 'file' + <newline> + EOO +} + +: for +: +{ + : form-1 + : + $* <"for x: a" >>EOO + 'for' + 'x' + : + 'a' + <newline> + EOO + + : form-3 + : + $* <"for <<<a x" >>EOO + 'for' + <<< + 'a' + 'x' + <newline> + EOO +} diff --git a/libbuild2/test/script/lexer.cxx b/libbuild2/test/script/lexer.cxx index f9c8ac6..9475ad4 100644 --- a/libbuild2/test/script/lexer.cxx +++ b/libbuild2/test/script/lexer.cxx @@ -41,6 +41,12 @@ namespace build2 switch (m) { + case lexer_mode::for_loop: + { + // Leading tokens of the for-loop. Like command_line but also + // recognizes lsbrace like value. + } + // Fall through. case lexer_mode::command_line: { s1 = ":;=!|&<> $(#\t\n"; @@ -122,6 +128,7 @@ namespace build2 case lexer_mode::first_token: case lexer_mode::second_token: case lexer_mode::variable_line: + case lexer_mode::for_loop: r = next_line (); break; case lexer_mode::description_line: @@ -157,7 +164,8 @@ namespace build2 // if (st.lsbrace) { - assert (m == lexer_mode::variable_line); + assert (m == lexer_mode::variable_line || + m == lexer_mode::for_loop); state_.top ().lsbrace = false; // Note: st is a copy. @@ -197,10 +205,11 @@ namespace build2 // Line separators. // - if (m == lexer_mode::command_line || - m == lexer_mode::first_token || - m == lexer_mode::second_token || - m == lexer_mode::variable_line) + if (m == lexer_mode::command_line || + m == lexer_mode::first_token || + m == lexer_mode::second_token || + m == lexer_mode::variable_line || + m == lexer_mode::for_loop) { switch (c) { @@ -210,7 +219,8 @@ namespace build2 if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { switch (c) { @@ -222,7 +232,8 @@ namespace build2 // if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { switch (c) { @@ -244,7 +255,8 @@ namespace build2 // if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { if (optional<token> t = next_cmd_op (c, sep)) return move (*t); diff --git a/libbuild2/test/script/lexer.hxx b/libbuild2/test/script/lexer.hxx index 452e794..def269b 100644 --- a/libbuild2/test/script/lexer.hxx +++ b/libbuild2/test/script/lexer.hxx @@ -24,10 +24,11 @@ namespace build2 enum { command_line = base_type::value_next, - first_token, // Expires at the end of the token. - second_token, // Expires at the end of the token. - variable_line, // Expires at the end of the line. - description_line // Expires at the end of the line. + first_token, // Expires at the end of the token. + second_token, // Expires at the end of the token. + variable_line, // Expires at the end of the line. + description_line, // Expires at the end of the line. + for_loop // Used for sensing the for-loop leading tokens. }; lexer_mode () = default; diff --git a/libbuild2/test/script/lexer.test.cxx b/libbuild2/test/script/lexer.test.cxx index 76f102d..ef3ce4d 100644 --- a/libbuild2/test/script/lexer.test.cxx +++ b/libbuild2/test/script/lexer.test.cxx @@ -36,6 +36,7 @@ namespace build2 else if (s == "variable-line") m = lexer_mode::variable_line; else if (s == "description-line") m = lexer_mode::description_line; else if (s == "variable") m = lexer_mode::variable; + else if (s == "for-loop") m = lexer_mode::for_loop; else assert (false); } diff --git a/libbuild2/test/script/parser+for.test.testscript b/libbuild2/test/script/parser+for.test.testscript index 70c1c89..426a39b 100644 --- a/libbuild2/test/script/parser+for.test.testscript +++ b/libbuild2/test/script/parser+for.test.testscript @@ -16,7 +16,7 @@ cmd end EOI - testscript:1:4: error: expected variable name instead of <newline> + testscript:1:1: error: for: missing variable name EOE : untyped @@ -311,3 +311,703 @@ testscript:4:1: error: both leading and trailing descriptions EOE } + +: form-2 +: +: ... | for x +: +{ + : for + : + { + : status + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x != 0 + cmd + end + EOI + testscript:1:20: error: for-loop exit code cannot be checked + EOE + + : not-last + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x | echo x + cmd + end + EOI + testscript:1:20: error: for-loop must be last command in a pipe + EOE + + : not-last-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x|echo x + cmd + end + EOI + testscript:1:19: error: for-loop must be last command in a pipe + EOE + + : expression-after + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x && echo x + cmd + end + EOI + testscript:1:20: error: command expression involving for-loop + EOE + + : expression-after-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x&&echo x + cmd + end + EOI + testscript:1:19: error: command expression involving for-loop + EOE + + : expression-before + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && echo x | for x + cmd + end + EOI + testscript:1:24: error: command expression involving for-loop + EOE + + : expression-before-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && echo x|for x + cmd + end + EOI + testscript:1:22: error: command expression involving for-loop + EOE + + : cleanup + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x &f + cmd + end + EOI + testscript:1:20: error: cleanup in for-loop + EOE + + : cleanup-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x&f + cmd + end + EOI + testscript:1:19: error: cleanup in for-loop + EOE + + : stdout-redirect + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x >a + cmd + end + EOI + testscript:1:20: error: output redirect in for-loop + EOE + + : stdout-redirect-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x>a + cmd + end + EOI + testscript:1:19: error: output redirect in for-loop + EOE + + : stdin-redirect + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x <a + cmd + end + EOI + testscript:1:20: error: stdin is both piped and redirected + EOE + + : no-var + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for + cmd + end + EOI + testscript:1:1: error: for: missing variable name + EOE + + : untyped + : + $* <<EOI >>EOO + echo 'a b' | for -w x + cmd $x + end + EOI + echo 'a b' | for -w x + EOO + + : expansion + : + $* <<EOI >>EOO + vs = a b + echo $vs | for x + cmd $x + end + EOI + echo a b | for x + EOO + + : typed-var + : + $* <<EOI >>EOO + echo 'a b' | for -w [dir_path] x + cmd $x + end + EOI + echo 'a b' | for -w [dir_path] x + EOO + } + + : after-semi + : + $* -s <<EOI >>EOO + cmd1; + echo 'a b' | for x + cmd2 $x + end + EOI + { + { + cmd1 + echo 'a b' | for x + } + } + EOO + + : setup + : + $* -s <<EOI >>EOO + +echo 'a b' | for x + cmd $x + end + EOI + { + +echo 'a b' | for x + } + EOO + + : tdown + : + $* -s <<EOI >>EOO + -echo 'a b' | for x + cmd $x + end + EOI + { + -echo 'a b' | for x + } + EOO + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + cmd + EOI + testscript:3:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + elif true + cmd + end + end + EOI + testscript:2:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + echo 'a b' | for x # 1 + cmd1 $x # 2 + if ($x == "a") # 3 + cmd2 # 4 + echo x y | for y # 5 + cmd3 # 6 + end + else + cmd4 # 7 + end + cmd5 # 8 + end; + cmd6 # 9 + EOI + echo 'a b' | for x # 1 + cmd6 # 9 + EOO + } + + : contained + : + { + : semi + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + cmd; + cmd + end + EOI + testscript:2:3: error: ';' inside 'for' + EOE + + : colon-leading + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + : foo + cmd + end + EOI + testscript:2:3: error: description inside 'for' + EOE + + : colon-trailing + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + cmd : foo + end + EOI + testscript:2:3: error: description inside 'for' + EOE + + : eos + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + EOI + testscript:2:1: error: expected closing 'end' + EOE + + : scope + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + cmd + { + } + end + EOI + testscript:3:3: error: expected closing 'end' + EOE + + : setup + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + +cmd + end + EOI + testscript:2:3: error: setup command inside 'for' + EOE + + : tdown + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + -cmd + end + EOI + testscript:2:3: error: teardown command inside 'for' + EOE + } + + : leading-and-trailing-description + : + $* <<EOI 2>>EOE != 0 + : foo + echo 'a b' | for x + cmd + end : bar + EOI + testscript:4:1: error: both leading and trailing descriptions + EOE +} + +: form-3 +: +: for x <... +: +{ + : for + : + { + : status + : + $* <<EOI 2>>EOE != 0 + for x <a != 0 + cmd + end + EOI + testscript:1:10: error: for-loop exit code cannot be checked + EOE + + : not-last + : + $* <<EOI 2>>EOE != 0 + for x <a | echo x + cmd + end + EOI + testscript:1:10: error: for-loop must be last command in a pipe + EOE + + : not-last-relex + : + $* <<EOI 2>>EOE != 0 + for <a x|echo x + cmd + end + EOI + testscript:1:9: error: for-loop must be last command in a pipe + EOE + + : expression-after + : + $* <<EOI 2>>EOE != 0 + for x <a && echo x + cmd + end + EOI + testscript:1:10: error: command expression involving for-loop + EOE + + : expression-after-relex + : + $* <<EOI 2>>EOE != 0 + for <a x&&echo x + cmd + end + EOI + testscript:1:9: error: command expression involving for-loop + EOE + + : expression-before + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && for x <a + cmd + end + EOI + testscript:1:15: error: command expression involving for-loop + EOE + + : cleanup + : + $* <<EOI 2>>EOE != 0 + for x <a &f + cmd + end + EOI + testscript:1:10: error: cleanup in for-loop + EOE + + : cleanup-before-var + : + $* <<EOI 2>>EOE != 0 + for &f x <a + cmd + end + EOI + testscript:1:5: error: cleanup in for-loop + EOE + + : cleanup-relex + : + $* <<EOI 2>>EOE != 0 + for <a x&f + cmd + end + EOI + testscript:1:9: error: cleanup in for-loop + EOE + + : stdout-redirect + : + $* <<EOI 2>>EOE != 0 + for x >a + cmd + end + EOI + testscript:1:7: error: output redirect in for-loop + EOE + + : stdout-redirect-before-var + : + $* <<EOI 2>>EOE != 0 + for >a x + cmd + end + EOI + testscript:1:5: error: output redirect in for-loop + EOE + + : stdout-redirect-relex + : + $* <<EOI 2>>EOE != 0 + for x>a + cmd + end + EOI + testscript:1:6: error: output redirect in for-loop + EOE + + : no-var + : + $* <<EOI 2>>EOE != 0 + for <a + cmd + end + EOI + testscript:1:1: error: for: missing variable name + EOE + + : untyped + : + $* <<EOI >>EOO + for -w x <'a b' + cmd $x + end + EOI + for -w x <'a b' + EOO + + : expansion + : + $* <<EOI >>EOO + vs = a b + for x <$vs + cmd $x + end + EOI + for x b <a + EOO + + : typed-var + : + $* <<EOI >>EOO + for -w [dir_path] x <'a b' + cmd $x + end + EOI + for -w [dir_path] x <'a b' + EOO + } + + : after-semi + : + $* -s <<EOI >>EOO + cmd1; + for x <'a b' + cmd2 $x + end + EOI + { + { + cmd1 + for x <'a b' + } + } + EOO + + : setup + : + $* -s <<EOI >>EOO + +for x <'a b' + cmd $x + end + EOI + { + +for x <'a b' + } + EOO + + : tdown + : + $* -s <<EOI >>EOO + -for x <'a b' + cmd $x + end + EOI + { + -for x <'a b' + } + EOO + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + cmd + EOI + testscript:3:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + elif true + cmd + end + end + EOI + testscript:2:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + for -w x <'a b' # 1 + cmd1 $x # 2 + if ($x == "a") # 3 + cmd2 # 4 + for -w y <'x y' # 5 + cmd3 # 6 + end + else + cmd4 # 7 + end + cmd5 # 8 + end; + cmd6 # 9 + EOI + for -w x <'a b' # 1 + cmd6 # 9 + EOO + } + + : contained + : + { + : semi + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + cmd; + cmd + end + EOI + testscript:2:3: error: ';' inside 'for' + EOE + + : colon-leading + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + : foo + cmd + end + EOI + testscript:2:3: error: description inside 'for' + EOE + + : colon-trailing + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + cmd : foo + end + EOI + testscript:2:3: error: description inside 'for' + EOE + + : eos + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + EOI + testscript:2:1: error: expected closing 'end' + EOE + + : scope + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + cmd + { + } + end + EOI + testscript:3:3: error: expected closing 'end' + EOE + + : setup + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + +cmd + end + EOI + testscript:2:3: error: setup command inside 'for' + EOE + + : tdown + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + -cmd + end + EOI + testscript:2:3: error: teardown command inside 'for' + EOE + } + + : leading-and-trailing-description + : + $* <<EOI 2>>EOE != 0 + : foo + for x <'a b' + cmd + end : bar + EOI + testscript:4:1: error: both leading and trailing descriptions + EOE +} diff --git a/libbuild2/test/script/parser.cxx b/libbuild2/test/script/parser.cxx index f302aee..f85b185 100644 --- a/libbuild2/test/script/parser.cxx +++ b/libbuild2/test/script/parser.cxx @@ -311,10 +311,11 @@ namespace build2 // enter: next token is peeked at (type in tt) // leave: newline - assert (!fct || - *fct == line_type::cmd_if || - *fct == line_type::cmd_while || - *fct == line_type::cmd_for); + assert (!fct || + *fct == line_type::cmd_if || + *fct == line_type::cmd_while || + *fct == line_type::cmd_for_stream || + *fct == line_type::cmd_for_args); // Note: token is only peeked at. // @@ -324,6 +325,52 @@ namespace build2 // line_type lt; type st (type::eos); // Later, can only be set to plus or minus. + bool semi (false); + + // Parse the command line tail, starting from the newline or the + // potential colon/semicolon token. + // + // Note that colon and semicolon are only valid in test command lines + // and after 'end' in flow control constructs. Note that we always + // recognize them lexically, even when they are not valid tokens per + // the grammar. + // + auto parse_command_tail = [&t, &tt, &st, <, &d, &semi, &ll, this] () + { + if (tt != type::newline) + { + if (lt != line_type::cmd && lt != line_type::cmd_end) + fail (t) << "expected newline instead of " << t; + + switch (st) + { + case type::plus: fail (t) << t << " after setup command" << endf; + case type::minus: fail (t) << t << " after teardown command" << endf; + } + } + + switch (tt) + { + case type::colon: + { + if (d) + fail (ll) << "both leading and trailing descriptions"; + + d = parse_trailing_description (t, tt); + break; + } + case type::semi: + { + semi = true; + replay_pop (); // See above for the reasoning. + next (t, tt); // Get newline. + break; + } + } + + if (tt != type::newline) + fail (t) << "expected newline instead of " << t; + }; switch (tt) { @@ -371,10 +418,12 @@ namespace build2 { const string& n (t.value); + // Handle the for-loop consistently with pre_parse_line_start(). + // if (n == "if") lt = line_type::cmd_if; else if (n == "if!") lt = line_type::cmd_ifn; else if (n == "while") lt = line_type::cmd_while; - else if (n == "for") lt = line_type::cmd_for; + else if (n == "for") lt = line_type::cmd_for_stream; } break; @@ -388,8 +437,6 @@ namespace build2 // Pre-parse the line keeping track of whether it ends with a semi. // - bool semi (false); - line ln; switch (lt) { @@ -436,47 +483,80 @@ namespace build2 break; } - case line_type::cmd_for: + // + // See pre_parse_line_start() for details. + // + case line_type::cmd_for_args: assert (false); break; + case line_type::cmd_for_stream: { - // First take care of the variable name. There is no reason not to - // support variable attributes. + // First we need to sense the next few tokens and detect which + // form of the for-loop that actually is (see + // libbuild2/build/script/parser.cxx for details). // - mode (lexer_mode::normal); + token pt (t); + assert (pt.type == type::word && pt.value == "for"); + mode (lexer_mode::for_loop); next_with_attributes (t, tt); - attributes_push (t, tt); - - if (tt != type::word || t.qtype != quote_type::unquoted) - fail (t) << "expected variable name instead of " << t; string& n (t.value); - if (special_variable (n)) - fail (t) << "attempt to set '" << n << "' variable directly"; + if (tt == type::lsbrace || // Attributes. + (tt == type::word && // Variable name. + t.qtype == quote_type::unquoted && + (n[0] == '_' || + alpha (n[0]) || + n == "*" || + n == "~" || + n == "@"))) + { + attributes_push (t, tt); - ln.var = &script_->var_pool.insert (move (n)); + if (tt != type::word || t.qtype != quote_type::unquoted) + fail (t) << "expected variable name instead of " << t; - next (t, tt); + if (special_variable (n)) + fail (t) << "attempt to set '" << n << "' variable directly"; - if (tt != type::colon) + if (lexer_->peek_char ().first == ':') + lt = line_type::cmd_for_args; + } + + if (lt == line_type::cmd_for_stream) // for x <... { - // @@ TMP We will need to fallback to parsing the 'for x <...' - // form instead. - // - fail (t) << "expected ':' instead of " << t - << " after variable name"; + ln.var = nullptr; + + expire_mode (); + + parse_command_expr_result r ( + parse_command_expr (t, tt, + lexer::redirect_aliases, + move (pt))); + + assert (r.for_loop); + + parse_command_tail (); + parse_here_documents (t, tt, r); } + else // for x: ... + { + ln.var = &script_->var_pool.insert (move (n)); - expire_mode (); // Expire the normal lexer mode. + next (t, tt); - // Parse the value similar to the var line type (see above), - // except for the fact that we don't expect a trailing semicolon. - // - mode (lexer_mode::variable_line); - parse_variable_line (t, tt); + assert (tt == type::colon); - if (tt != type::newline) - fail (t) << "expected newline instead of " << t << " after for"; + expire_mode (); + + // Parse the value similar to the var line type (see above), + // except for the fact that we don't expect a trailing semicolon. + // + mode (lexer_mode::variable_line); + parse_variable_line (t, tt); + + if (tt != type::newline) + fail (t) << "expected newline instead of " << t << " after for"; + } break; } @@ -501,51 +581,20 @@ namespace build2 // Fall through. case line_type::cmd: { - pair<command_expr, here_docs> p; + parse_command_expr_result r; if (lt != line_type::cmd_else && lt != line_type::cmd_end) - p = parse_command_expr (t, tt, lexer::redirect_aliases); + r = parse_command_expr (t, tt, lexer::redirect_aliases); - // Colon and semicolon are only valid in test command lines and - // after 'end' in a flow control construct. Note that we still - // recognize them lexically, they are just not valid tokens per - // the grammar. - // - if (tt != type::newline) + if (r.for_loop) { - if (lt != line_type::cmd && lt != line_type::cmd_end) - fail (t) << "expected newline instead of " << t; - - switch (st) - { - case type::plus: fail (t) << t << " after setup command" << endf; - case type::minus: fail (t) << t << " after teardown command" << endf; - } - } - - switch (tt) - { - case type::colon: - { - if (d) - fail (ll) << "both leading and trailing descriptions"; - - d = parse_trailing_description (t, tt); - break; - } - case type::semi: - { - semi = true; - replay_pop (); // See above for the reasoning. - next (t, tt); // Get newline. - break; - } + lt = line_type::cmd_for_stream; + ln.var = nullptr; } - if (tt != type::newline) - fail (t) << "expected newline instead of " << t; + parse_command_tail (); + parse_here_documents (t, tt, r); - parse_here_documents (t, tt, p); break; } } @@ -579,7 +628,8 @@ namespace build2 break; } case line_type::cmd_while: - case line_type::cmd_for: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: { semi = pre_parse_loop (t, tt, lt, d, *ls); break; @@ -608,7 +658,8 @@ namespace build2 case line_type::cmd_if: case line_type::cmd_ifn: case line_type::cmd_while: - case line_type::cmd_for: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: { // See if this is a variable-only flow control construct. // @@ -951,7 +1002,7 @@ namespace build2 // size_t i (ls.size ()); - line_type fct; // Flow control type the block type relates to. + line_type fct; // Flow control construct type the block type relates to. switch (bt) { @@ -965,7 +1016,8 @@ namespace build2 break; } case line_type::cmd_while: - case line_type::cmd_for: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: { fct = bt; break; @@ -1068,7 +1120,9 @@ namespace build2 // enter: <newline> (previous line) // leave: <newline> - assert (lt == line_type::cmd_while || lt == line_type::cmd_for); + assert (lt == line_type::cmd_while || + lt == line_type::cmd_for_stream || + lt == line_type::cmd_for_args); tt = peek (lexer_mode::first_token); @@ -1428,7 +1482,7 @@ namespace build2 // Note: this one is only used during execution. - pair<command_expr, here_docs> p ( + parse_command_expr_result pr ( parse_command_expr (t, tt, lexer::redirect_aliases)); if (tt == type::colon) @@ -1436,10 +1490,10 @@ namespace build2 assert (tt == type::newline); - parse_here_documents (t, tt, p); + parse_here_documents (t, tt, pr); assert (tt == type::newline); - command_expr r (move (p.first)); + command_expr r (move (pr.expr)); // If the test program runner is specified, then adjust the // expressions to run test programs via this runner. @@ -1582,6 +1636,7 @@ namespace build2 auto exec_cmd = [&ct, this] (token& t, build2::script::token_type& tt, const iteration_index* ii, size_t li, bool single, + const function<command_function>& cf, const location& ll) { // We use the 0 index to signal that this is the only command. @@ -1593,7 +1648,7 @@ namespace build2 command_expr ce ( parse_command_line (t, static_cast<token_type&> (tt))); - runner_->run (*scope_, ce, ct, ii, li, ll); + runner_->run (*scope_, ce, ct, ii, li, cf, ll); }; auto exec_cond = [this] (token& t, build2::script::token_type& tt, @@ -1827,7 +1882,8 @@ namespace build2 // The rest. // - // When add a special variable don't forget to update lexer::word(). + // When add a special variable don't forget to update lexer::word() and + // for-loop parsing in pre_parse_line(). // bool parser:: special_variable (const string& n) noexcept diff --git a/libbuild2/test/script/parser.test.cxx b/libbuild2/test/script/parser.test.cxx index ab0aee9..7339346 100644 --- a/libbuild2/test/script/parser.test.cxx +++ b/libbuild2/test/script/parser.test.cxx @@ -100,11 +100,32 @@ namespace build2 } virtual void - run (scope&, + run (scope& env, const command_expr& e, command_type t, const iteration_index* ii, size_t i, - const location&) override + const function<command_function>& cf, + const location& ll) override { + // If the functions is specified, then just execute it with an empty + // stdin so it can perform the housekeeping (stop replaying tokens, + // increment line index, etc). + // + if (cf != nullptr) + { + assert (e.size () == 1 && !e[0].pipe.empty ()); + + const command& c (e[0].pipe.back ()); + + // Must be enforced by the caller. + // + assert (!c.out && !c.err && !c.exit); + + cf (env, c.arguments, + fdopen_null (), false /* pipe */, + nullopt /* deadline */, c, + ll); + } + const char* s (nullptr); switch (t) diff --git a/libbuild2/test/script/runner.cxx b/libbuild2/test/script/runner.cxx index 42eef04..340ada4 100644 --- a/libbuild2/test/script/runner.cxx +++ b/libbuild2/test/script/runner.cxx @@ -143,6 +143,7 @@ namespace build2 run (scope& sp, const command_expr& expr, command_type ct, const iteration_index* ii, size_t li, + const function<command_function>& cf, const location& ll) { // Noop for teardown commands if keeping tests output is requested. @@ -176,7 +177,7 @@ namespace build2 dr << info << "test id: " << sp.id_path.posix_string (); }); - build2::script::run (sp, expr, ii, li, ll); + build2::script::run (sp, expr, ii, li, ll, cf); } bool default_runner:: diff --git a/libbuild2/test/script/runner.hxx b/libbuild2/test/script/runner.hxx index 0309a35..687d991 100644 --- a/libbuild2/test/script/runner.hxx +++ b/libbuild2/test/script/runner.hxx @@ -48,10 +48,14 @@ namespace build2 // Location is the start position of this command line in the // testscript. It can be used in diagnostics. // + // Optionally, execute the specified function instead of the last + // pipe command. + // virtual void run (scope&, const command_expr&, command_type, const iteration_index*, size_t index, + const function<command_function>&, const location&) = 0; virtual bool @@ -88,6 +92,7 @@ namespace build2 run (scope&, const command_expr&, command_type, const iteration_index*, size_t, + const function<command_function>&, const location&) override; virtual bool diff --git a/libbuild2/test/script/script.cxx b/libbuild2/test/script/script.cxx index e10afec..bbe5326 100644 --- a/libbuild2/test/script/script.cxx +++ b/libbuild2/test/script/script.cxx @@ -115,7 +115,7 @@ namespace build2 } void scope:: - set_variable (string&& nm, + set_variable (string nm, names&& val, const string& attrs, const location& ll) diff --git a/libbuild2/test/script/script.hxx b/libbuild2/test/script/script.hxx index b75f68e..319a9e2 100644 --- a/libbuild2/test/script/script.hxx +++ b/libbuild2/test/script/script.hxx @@ -32,6 +32,7 @@ namespace build2 using build2::script::environment_vars; using build2::script::deadline; using build2::script::timeout; + using build2::script::command_function; class parser; // Required by VC for 'friend class parser' declaration. @@ -105,7 +106,7 @@ namespace build2 small_vector<const path*, 1> test_programs; void - set_variable (string&& name, + set_variable (string name, names&&, const string& attrs, const location&) override; diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx index a285b03..88ea43d 100644 --- a/libbuild2/utility.hxx +++ b/libbuild2/utility.hxx @@ -91,6 +91,7 @@ namespace build2 // <libbutl/fdstream.hxx> // + using butl::fdopen_null; using butl::open_file_or_stdin; using butl::open_file_or_stdout; diff --git a/tests/recipe/buildscript/testscript b/tests/recipe/buildscript/testscript index 0ac5d5a..531baa9 100644 --- a/tests/recipe/buildscript/testscript +++ b/tests/recipe/buildscript/testscript @@ -886,29 +886,164 @@ if $posix EOE } -: flow-control-construct +: loop : { : while : { - echo 'bar' >=bar; + : basics + : + { + echo 'bar' >=bar; - cat <<EOI >=buildfile; - foo: bar - {{ - p = $path($>) - while test -f $p != 0 - cp $path($<) $p - end - }} - EOI + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + while test -f $p != 0 + cp $path($<) $p + end + }} + EOI - $* 2>'cp file{foo}'; + $* 2>'cp file{foo}'; - cat <<<foo >'bar'; + cat <<<foo >'bar'; - $* clean 2>- + $* clean 2>- + } + + : exit + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + diag gen ($>) + + p = $path($>) + while test -f $p != 0 + touch $p + exit + cp $path($<) $p + end + }} + EOI + + $* 2>'gen file{foo.}'; + + cat <<<foo >:''; + + $* clean 2>- + } + + : error + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + diag gen ($>) + + p = $path($>) + while test -f $p != 0 + touch $p + exit 'fed up' + cp $path($<) $p + end + }} + EOI + + $* 2>>~%EOE% != 0; + gen file{foo.} + buildfile:8:5: error: fed up + %.{3} + EOE + + $* clean 2>- + } + + : depdb + : + { + : inside + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + while test -f $p != 0 + depdb hash $p + cp $path($<) $p + end + }} + EOI + + $* 2>>EOE != 0 + buildfile:5:5: error: 'depdb' call inside flow control construct + EOE + } + + : after-commands + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + while test -f $p != 0 + cp $path($<) $p + end + + depdb hash $p + }} + EOI + + $* 2>>~%EOE% != 0; + buildfile:5:5: error: disallowed command in depdb preamble + info: only variable assignments are allowed in depdb preamble + buildfile:8:3: info: depdb preamble ends here + %.{3} + EOE + + $* clean 2>- + } + + : after-vars + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($<) + + h = + while test -f $p != 0 + h += $p + end + + depdb hash $p + + cat $p >$path($>) + }} + EOI + + $* 2>'cat file{foo}'; + $* clean 2>- + } + } } : for @@ -947,6 +1082,85 @@ if $posix $* clean 2>- } + : special-var + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + rm -f $p + + for ~: $< + cat $path($f) >>$p + end + }} + EOI + + $* 2>>EOE != 0 + buildfile:6:7: error: attempt to set '~' special variable + EOE + } + + : exit + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + rm -f $p + + for f: $< + cat $path($f) >>$p + exit + end + }} + EOI + + $* 2>'cat file{foo}'; + + cat <<<foo >>EOO; + bar + EOO + + $* clean 2>- + } + + : error + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + rm -f $p + + for f: $< + cat $path($f) >>$p + exit 'fed up' + end + }} + EOI + + $* 2>>~%EOE% != 0; + cat file{foo} + buildfile:8:5: error: fed up + %.{3} + EOE + + $* clean 2>- + } + : depdb : { @@ -1031,5 +1245,456 @@ if $posix } } } + + : form-2 + : + : ... | for x + : + { + : basics + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar baz + {{ + diag gen ($>) + + p = $path($>) + rm -f $p + + echo $path($<) | for -w f + cat $f >>$p + end + }} + EOI + + $* 2>'gen file{foo.}'; + + cat <<<foo >>EOO; + bar + baz + EOO + + $* clean 2>- + } + + : special-var + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + diag gen ($>) + + p = $path($>) + rm -f $p + + echo $path($<) | for ~ + cat $f >>$p + end + }} + EOI + + $* 2>>~%EOE% != 0; + gen file{foo.} + buildfile:8:3: error: attempt to set '~' special variable + %.{3} + EOE + + $* clean 2>- + } + + : misuse + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + diag gen ($>) + + p = $path($>) + rm -f $p + + echo $path($<) | for x: + cat $f >>$p + end + }} + EOI + + $* 2>>~%EOE% != 0; + gen file{foo.} + buildfile:8:3: error: for: ':' after variable name + %.{3} + EOE + + $* clean 2>- + } + + : exit + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + diag gen ($>) + + p = $path($>) + rm -f $p + + echo $path($<) | for -w f + cat $f >>$p + exit + end + }} + EOI + + $* 2>'gen file{foo.}'; + + cat <<<foo >>EOO; + bar + EOO + + $* clean 2>- + } + + : error + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + diag gen ($>) + + p = $path($>) + rm -f $p + + echo $path($<) | for -w f + cat $f >>$p + exit 'fed up' + end + }} + EOI + + $* 2>>~%EOE% != 0; + gen file{foo.} + buildfile:10:5: error: fed up + %.{3} + EOE + + $* clean 2>- + } + + : depdb + : + { + : inside + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + echo $path($<) | for -w f + depdb hash $f + end + + p = $path($>) + rm -f $p + + echo $path($<) | for -w f + cat $f >>$p + end + }} + EOI + + $* 2>>EOE != 0 + buildfile:4:5: error: 'depdb' call inside flow control construct + EOE + } + + : after-commands + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + echo $path($<) | for -w f + echo $f >- + end + + depdb hash $p + }} + EOI + + $* 2>>~%EOE% != 0; + buildfile:4:5: error: disallowed command in depdb preamble + info: only variable assignments are allowed in depdb preamble + buildfile:7:3: info: depdb preamble ends here + %.{3} + EOE + + $* clean 2>- + } + + : after-vars + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + h = + echo $path($<) | for -w f + h += $f + end + + depdb hash $h + + diag gen ($>) + + p = $path($>) + rm -f $p + + for f: $< + cat $path($f) >>$p + end + }} + EOI + + $* 2>'gen file{foo.}'; + $* clean 2>- + } + } + } + + : form-3 + : + : for x <... + : + { + : basics + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar baz + {{ + diag gen ($>) + + p = $path($>) + rm -f $p + + for -w f <<"EOF" + $path($<) + EOF + cat $f >>$p + end + + for <<"EOF" -w f + $path($<) + EOF + cat $f >>$p + end + }} + EOI + + $* 2>'gen file{foo.}'; + + cat <<<foo >>EOO; + bar + baz + bar + baz + EOO + + $* clean 2>- + } + + : special-var + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + rm -f $p + + for ~ <<<$path($<) + cat $f >>$p + end + }} + EOI + + $* 2>>EOE != 0 + buildfile:6:6: error: attempt to set '~' special variable + EOE + } + + : exit + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + rm -f $p + + for f <<<$path($<) + cat $f >>$p + exit + end + }} + EOI + + $* 2>'cat file{foo}'; + + cat <<<foo >>EOO; + bar + EOO + + $* clean 2>- + } + + : error + : + { + echo 'bar' >=bar; + echo 'baz' >=baz; + + cat <<EOI >=buildfile; + foo: bar + {{ + p = $path($>) + rm -f $p + + for f <<<$path($<) + cat $f >>$p + exit 'fed up' + end + }} + EOI + + $* 2>>~%EOE% != 0; + cat file{foo} + buildfile:8:5: error: fed up + %.{3} + EOE + + $* clean 2>- + } + + : depdb + : + { + : inside + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + for -w f <<<$path($<) + depdb hash $f + end + + p = $path($>) + rm -f $p + + echo $path($<) | for -w f + cat $f >>$p + end + }} + EOI + + $* 2>>EOE != 0 + buildfile:4:5: error: 'depdb' call inside flow control construct + EOE + } + + : after-commands + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + for -w f <<<$path($<) + echo $f >- + end + + depdb hash a + }} + EOI + + $* 2>>~%EOE% != 0; + buildfile:4:5: error: disallowed command in depdb preamble + info: only variable assignments are allowed in depdb preamble + buildfile:7:3: info: depdb preamble ends here + %.{3} + EOE + + $* clean 2>- + } + + : after-vars + : + { + echo 'bar' >=bar; + + cat <<EOI >=buildfile; + foo: bar + {{ + h = + for -w f <<<$path($<) + h += $f + end + + depdb hash $h + + diag gen ($>) + + p = $path($>) + rm -f $p + + for f: $< + cat $path($f) >>$p + end + }} + EOI + + $* 2>'gen file{foo.}'; + $* clean 2>- + } + } + } } } diff --git a/tests/test/script/runner/for.testscript b/tests/test/script/runner/for.testscript index 21042e5..054e9ab 100644 --- a/tests/test/script/runner/for.testscript +++ b/tests/test/script/runner/for.testscript @@ -39,4 +39,379 @@ EOI testscript:1:5: error: attempt to set '*' variable directly EOE + + : exit + : + $c <<EOI && $b >>EOO + for x: a b + echo "$x" >| + exit + end + EOI + a + EOO + + : error + : + $c <<EOI && $b >>EOO 2>>EOE != 0 + for x: a b + echo "$x" >| + exit 'fed up' + end + EOI + a + EOO + testscript:3:3: error: fed up + info: test id: 1 + EOE +} + +: form-2 +: +: ... | for x +: +{ + : whitespace-split + : + $c <<EOI && $b >>EOO + echo " a b " | for -w x + echo "'$x'" >| + end + EOI + 'a' + 'b' + EOO + + : newline-split + : + $c <<EOI && $b >>EOO + cat <<EOF | for -n x + + + a + + + b + + EOF + echo "'$x'" >| + end + EOI + '' + '' + 'a' + '' + '' + 'b' + '' + EOO + + : typed + : + $c <<EOI && $b >>/EOO + echo "a b" | for -w [dir_path] x + echo $x >| + end + EOI + a/ + b/ + EOO + + : nested + : + $c <<EOI && $b >>EOO + echo "a b" | for -w x + echo "x y" | for -w y + echo "'$x $y'" >| + end + end + EOI + 'a x' + 'a y' + 'b x' + 'b y' + EOO + + : nested-diag + : + $c <<EOI && $b 2>>/~%EOE% != 0 + echo "a b" | for -w x + echo "x y" | for -w y + echo "'$x $y'" >"'a x'" + end + end + EOI + testscript:3:5: error: echo stdout doesn't match expected + info: stdout: test/1/stdout-i1-i2-n3 + info: expected stdout: test/1/stdout-i1-i2-n3.orig + info: stdout diff: test/1/stdout-i1-i2-n3.diff + %.+ + EOE + + : var-value + : + $c <<EOI && $b >>EOO + x = 'x'; + echo "a b" | for -w x + end; + echo $x >| + EOI + b + EOO + + : invalid-option + : + $c <<EOI && $b 2>>/~%EOE% != 0 + echo "a b" | for -a x + echo $x >| + end + EOI + testscript:1:1: error: for: unknown option '-a' + %. + EOE + + + : no-variable + : + $c <<EOI && $b 2>>/~%EOE% != 0 + echo "a b" | for -w + echo $x >| + end + EOI + testscript:1:1: error: for: missing variable name + %. + EOE + + : special-var + : + $c <<EOI && $b 2>>EOE != 0 + echo "a b" | for -w * + echo $* >| + end + EOI + testscript:1:1: error: attempt to set '*' variable directly + info: test id: 1 + EOE + + : misuse + : + $c <<EOI && $b 2>>EOE != 0 + echo "a b" | for v: + echo $v >| + end + EOI + testscript:1:19: error: expected newline instead of ':' + EOE + + : exit + : + $c <<EOI && $b >>EOO + for x: a b + echo "$x" >| + exit + end + EOI + a + EOO + + : error + : + $c <<EOI && $b >>EOO 2>>EOE != 0 + for x: a b + echo "$x" >| + exit 'fed up' + end + EOI + a + EOO + testscript:3:3: error: fed up + info: test id: 1 + EOE +} + +: form-3 +: +: for x <... +: +{ + : whitespace-split + : + $c <<EOI && $b >>EOO + for -w x <" a b " + echo "'$x'" >| + end + EOI + 'a' + 'b' + EOO + + : newline-split + : + $c <<EOI && $b >>EOO + for -n x <<EOF + + + a + + + b + + EOF + echo "'$x'" >| + end + EOI + '' + '' + 'a' + '' + '' + 'b' + '' + EOO + + : string-before-var + : + $c <<EOI && $b >>EOO + for <"a b" -w x + echo "'$x'" >| + end + EOI + 'a' + 'b' + EOO + + : here-doc-before-var + : + $c <<EOI && $b >>EOO + for <<EOF -n x + a + b + EOF + echo "'$x'" >| + end + EOI + 'a' + 'b' + EOO + + : typed + : + $c <<EOI && $b >>/EOO + for -w [dir_path] x <"a b" + echo $x >| + end + EOI + a/ + b/ + EOO + + : typed-no-ops + : + $c <<EOI && $b >>/EOO + for [dir_path] x <"a b" + echo $x >| + end + EOI + a b/ + EOO + + : nested + : + $c <<EOI && $b >>EOO + for -w x <"a b" + for -w y <"x y" + echo "'$x $y'" >| + end + end + EOI + 'a x' + 'a y' + 'b x' + 'b y' + EOO + + : nested-diag + : + $c <<EOI && $b 2>>/~%EOE% != 0 + for -w x <"a b" + for -w y <"x y" + echo "'$x $y'" >"'a x'" + end + end + EOI + testscript:3:5: error: echo stdout doesn't match expected + info: stdout: test/1/stdout-i1-i2-n3 + info: expected stdout: test/1/stdout-i1-i2-n3.orig + info: stdout diff: test/1/stdout-i1-i2-n3.diff + %.+ + EOE + + : var-value + : + $c <<EOI && $b >>EOO + x = 'x'; + for -w x <"a b" + end; + echo $x >| + EOI + b + EOO + + : invalid-option + : + $c <<EOI && $b 2>>/~%EOE% != 0 + for -a x <"a b" + echo $x >| + end + EOI + testscript:1:1: error: for: unknown option '-a' + %. + EOE + + + : no-variable + : + $c <<EOI && $b 2>>/~%EOE% != 0 + for -w <"a b" + echo $x >| + end + EOI + testscript:1:1: error: for: missing variable name + %. + EOE + + : special-var + : + $c <<EOI && $b 2>>EOE != 0 + for * <"a b" + echo $* >| + end + EOI + testscript:1:5: error: attempt to set '*' variable directly + EOE + + : exit + : + $c <<EOI && $b >>EOO + for x: a b + echo "$x" >| + exit + end + EOI + a + EOO + + : error + : + $c <<EOI && $b >>EOO 2>>EOE != 0 + for x: a b + echo "$x" >| + exit 'fed up' + end + EOI + a + EOO + testscript:3:3: error: fed up + info: test id: 1 + EOE } diff --git a/tests/test/script/runner/set.testscript b/tests/test/script/runner/set.testscript index b2944a3..b4c8089 100644 --- a/tests/test/script/runner/set.testscript +++ b/tests/test/script/runner/set.testscript @@ -343,6 +343,201 @@ EOE EOI } + + : split + : + : Test various splitting modes as above, but now reading the stream in the + : non-blocking mode. + : + { + : whitespace-separated-list + : + { + : non-exact + : + { + : non-empty + : + $c <<EOI && $b + timeout 10; + set -w baz <' foo bar '; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar"' + EOI + + : empty + : + $c <<EOI && $b + timeout 10; + set -w baz <:''; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'' + EOI + + : spaces + : + $c <<EOI && $b + timeout 10; + set -w baz <' '; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'' + EOI + } + + : exact + : + { + : trailing-ws + : + $c <<EOI && $b + timeout 10; + set --exact --whitespace baz <' foo bar '; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar" ""' + EOI + + : no-trailing-ws + : + : Note that we need to strip the default trailing newline as well with the + : ':' modifier. + : + $c <<EOI && $b + timeout 10; + set -e -w baz <:' foo bar'; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar"' + EOI + + : empty + : + $c <<EOI && $b + timeout 10; + set -e -w baz <:''; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'' + EOI + + : spaces + : + $c <<EOI && $b + timeout 10; + set -e -w baz <' '; + echo $regex.apply($baz, '^(.*)$', '"\1"') >'""' + EOI + } + } + + : newline-separated-list + : + { + : non-exact + : + $c <<EOI && $b + timeout 10; + set -n baz <<EOF; + + foo + + bar + + EOF + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar" ""' + EOI + + : exact + : + { + : trailing-newline + : + $c <<EOI && $b + timeout 10; + set --exact --newline baz <<EOF; + + foo + + bar + + EOF + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar" "" ""' + EOI + + : no-trailing-newline + : + $c <<EOI && $b + timeout 10; + set --exact --newline baz <<:EOF; + + foo + + bar + EOF + echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar"' + EOI + } + } + + : string + : + { + : non-exact + : + $c <<EOI && $b + timeout 10; + set baz <<EOF; + + foo + + bar + + EOF + echo ($baz[0]) >>EOO + + foo + + bar + + EOO + EOI + + : exact + : + : Note that echo adds the trailing newline, so EOF and EOO here-documents + : differ by this newline. + : + { + : trailing-newline + : + $c <<EOI && $b + timeout 10; + set -e baz <<EOF; + + foo + + bar + EOF + echo ($baz[0]) >>EOO + + foo + + bar + + EOO + EOI + + : no-trailing-newline + : + $c <<EOI && $b + timeout 10; + set -e baz <<:EOF; + + foo + + bar + EOF + echo ($baz[0]) >>EOO + + foo + + bar + EOO + EOI + } + } + } } : attributes |