From 96908ec43ed27d9f34da27b6a94a6db357465056 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Thu, 14 May 2020 14:18:50 +0300 Subject: Add build script --- bootstrap-clang.bat | 1 + bootstrap-mingw.bat | 1 + bootstrap-msvc.bat | 1 + bootstrap.gmake | 1 + bootstrap.sh | 1 + .../script/lexer+command-line.test.testscript | 152 +++++++ .../build/script/lexer+first-token.test.testscript | 30 ++ .../script/lexer+second-token.test.testscript | 53 +++ .../script/lexer+variable-line.test.testscript | 12 + .../build/script/lexer+variable.test.testscript | 25 ++ libbuild2/build/script/lexer.cxx | 254 ++++++++++++ libbuild2/build/script/lexer.hxx | 75 ++++ libbuild2/build/script/lexer.test.cxx | 77 ++++ .../build/script/parser+cleanup.test.testscript | 57 +++ .../build/script/parser+command-if.test.testscript | 395 ++++++++++++++++++ .../script/parser+command-re-parse.test.testscript | 11 + libbuild2/build/script/parser+exit.test.testscript | 26 ++ .../build/script/parser+expansion.test.testscript | 35 ++ .../script/parser+here-document.test.testscript | 212 ++++++++++ .../script/parser+here-string.test.testscript | 18 + .../build/script/parser+pipe-expr.test.testscript | 132 ++++++ .../build/script/parser+pre-parse.test.testscript | 22 + .../build/script/parser+redirect.test.testscript | 441 +++++++++++++++++++++ .../build/script/parser+regex.test.testscript | 222 +++++++++++ .../build/script/parser+variable.test.testscript | 41 ++ libbuild2/build/script/parser.cxx | 337 ++++++++++++++++ libbuild2/build/script/parser.hxx | 92 +++++ libbuild2/build/script/parser.test.cxx | 145 +++++++ libbuild2/build/script/runner.cxx | 50 +++ libbuild2/build/script/runner.hxx | 78 ++++ libbuild2/build/script/script.cxx | 156 ++++++++ libbuild2/build/script/script.hxx | 111 ++++++ libbuild2/build/script/token.cxx | 23 ++ libbuild2/build/script/token.hxx | 36 ++ libbuild2/buildfile | 3 + 35 files changed, 3326 insertions(+) create mode 100644 libbuild2/build/script/lexer+command-line.test.testscript create mode 100644 libbuild2/build/script/lexer+first-token.test.testscript create mode 100644 libbuild2/build/script/lexer+second-token.test.testscript create mode 100644 libbuild2/build/script/lexer+variable-line.test.testscript create mode 100644 libbuild2/build/script/lexer+variable.test.testscript create mode 100644 libbuild2/build/script/lexer.cxx create mode 100644 libbuild2/build/script/lexer.hxx create mode 100644 libbuild2/build/script/lexer.test.cxx create mode 100644 libbuild2/build/script/parser+cleanup.test.testscript create mode 100644 libbuild2/build/script/parser+command-if.test.testscript create mode 100644 libbuild2/build/script/parser+command-re-parse.test.testscript create mode 100644 libbuild2/build/script/parser+exit.test.testscript create mode 100644 libbuild2/build/script/parser+expansion.test.testscript create mode 100644 libbuild2/build/script/parser+here-document.test.testscript create mode 100644 libbuild2/build/script/parser+here-string.test.testscript create mode 100644 libbuild2/build/script/parser+pipe-expr.test.testscript create mode 100644 libbuild2/build/script/parser+pre-parse.test.testscript create mode 100644 libbuild2/build/script/parser+redirect.test.testscript create mode 100644 libbuild2/build/script/parser+regex.test.testscript create mode 100644 libbuild2/build/script/parser+variable.test.testscript create mode 100644 libbuild2/build/script/parser.cxx create mode 100644 libbuild2/build/script/parser.hxx create mode 100644 libbuild2/build/script/parser.test.cxx create mode 100644 libbuild2/build/script/runner.cxx create mode 100644 libbuild2/build/script/runner.hxx create mode 100644 libbuild2/build/script/script.cxx create mode 100644 libbuild2/build/script/script.hxx create mode 100644 libbuild2/build/script/token.cxx create mode 100644 libbuild2/build/script/token.hxx diff --git a/bootstrap-clang.bat b/bootstrap-clang.bat index ce58bcd..00302e9 100644 --- a/bootstrap-clang.bat +++ b/bootstrap-clang.bat @@ -63,6 +63,7 @@ set "src=build2" set "src=%src% libbuild2" set "src=%src% libbuild2\script" +set "src=%src% libbuild2\build\script" set "src=%src% libbuild2\config" set "src=%src% libbuild2\dist" set "src=%src% libbuild2\test" diff --git a/bootstrap-mingw.bat b/bootstrap-mingw.bat index 4de9789..df7e677 100644 --- a/bootstrap-mingw.bat +++ b/bootstrap-mingw.bat @@ -63,6 +63,7 @@ set "src=build2" set "src=%src% libbuild2" set "src=%src% libbuild2\script" +set "src=%src% libbuild2\build\script" set "src=%src% libbuild2\config" set "src=%src% libbuild2\dist" set "src=%src% libbuild2\test" diff --git a/bootstrap-msvc.bat b/bootstrap-msvc.bat index 5b2e0d8..3d74427 100644 --- a/bootstrap-msvc.bat +++ b/bootstrap-msvc.bat @@ -94,6 +94,7 @@ set "src=build2" set "src=%src% libbuild2" set "src=%src% libbuild2\script" +set "src=%src% libbuild2\build\script" set "src=%src% libbuild2\config" set "src=%src% libbuild2\dist" set "src=%src% libbuild2\test" diff --git a/bootstrap.gmake b/bootstrap.gmake index f46ef19..1e0e8e2 100644 --- a/bootstrap.gmake +++ b/bootstrap.gmake @@ -153,6 +153,7 @@ endif # libbuild2_sub := \ script \ +build/script \ config \ dist \ test/script \ diff --git a/bootstrap.sh b/bootstrap.sh index 6e39239..14e52cf 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -119,6 +119,7 @@ src="build2/*.cxx" src="$src libbuild2/*.cxx" src="$src libbuild2/script/*.cxx" +src="$src libbuild2/build/script/*.cxx" src="$src libbuild2/config/*.cxx" src="$src libbuild2/dist/*.cxx" src="$src libbuild2/test/*.cxx" diff --git a/libbuild2/build/script/lexer+command-line.test.testscript b/libbuild2/build/script/lexer+command-line.test.testscript new file mode 100644 index 0000000..1777583 --- /dev/null +++ b/libbuild2/build/script/lexer+command-line.test.testscript @@ -0,0 +1,152 @@ +# file : libbuild2/build/script/lexer+command-line.test.testscript +# license : MIT; see accompanying LICENSE file + +test.arguments = command-line + +: redirect +: +{ + : pass + : + $* <"cmd <| 1>|" >>EOO + 'cmd' + <| + '1' + >| + + EOO + + : null + : + $* <"cmd <- 1>-" >>EOO + 'cmd' + <- + '1' + >- + + EOO + + : trace + : + $* <"cmd 1>!" >>EOO + 'cmd' + '1' + >! + + EOO + + : merge + : + $* <"cmd 1>&2" >>EOO + 'cmd' + '1' + >& + '2' + + EOO + + : str + : + $* <"cmd b" >>EOO + 'cmd' + < + 'a' + '1' + > + 'b' + + EOO + + : str-nn + : + $* <"cmd <:a 1>:b" >>EOO + 'cmd' + <: + 'a' + '1' + >: + 'b' + + EOO + + : doc + : + $* <"cmd <>EOO" >>EOO + 'cmd' + << + 'EOI' + '1' + >> + 'EOO' + + EOO + + : doc-nn + : + $* <"cmd <<:EOI 1>>:EOO" >>EOO + 'cmd' + <<: + 'EOI' + '1' + >>: + 'EOO' + + EOO + + : file-cmp + : + $* <"cmd <<>>out 2>>>err" >>EOO + 'cmd' + <<< + 'in' + >>> + 'out' + '2' + >>> + 'err' + + EOO + + : file-write + : + $* <"cmd >=out 2>+err" >>EOO + 'cmd' + >= + 'out' + '2' + >+ + 'err' + + EOO +} + +: cleanup +: +{ + : always + : + $* <"cmd &file" >>EOO + 'cmd' + & + 'file' + + EOO + + : maybe + : + $* <"cmd &?file" >>EOO + 'cmd' + &? + 'file' + + EOO + + : never + : + $* <"cmd &!file" >>EOO + 'cmd' + &! + 'file' + + EOO +} diff --git a/libbuild2/build/script/lexer+first-token.test.testscript b/libbuild2/build/script/lexer+first-token.test.testscript new file mode 100644 index 0000000..6709e60 --- /dev/null +++ b/libbuild2/build/script/lexer+first-token.test.testscript @@ -0,0 +1,30 @@ +# file : libbuild2/build/script/lexer+first-token.test.testscript +# license : MIT; see accompanying LICENSE file + +# Note: this mode auto-expires after each token. +# +test.arguments = first-token + +: assign +: +$* <"foo=" >>EOO +'foo' +'=' + +EOO + +: append +: +$* <"foo+=" >>EOO +'foo' +'+=' + +EOO + +: prepend +: +$* <"foo=+" >>EOO +'foo' +'=+' + +EOO diff --git a/libbuild2/build/script/lexer+second-token.test.testscript b/libbuild2/build/script/lexer+second-token.test.testscript new file mode 100644 index 0000000..d5f3329 --- /dev/null +++ b/libbuild2/build/script/lexer+second-token.test.testscript @@ -0,0 +1,53 @@ +# file : libbuild2/build/script/lexer+second-token.test.testscript +# license : MIT; see accompanying LICENSE file + +# Note: this mode auto-expires after each token. +# +test.arguments = second-token + +: assign +: +$* <"=foo" >>EOO += +'foo' + +EOO + +: append +: +$* <"+= foo" >>EOO ++= +'foo' + +EOO + +: prepend +: +$* <" =+ foo" >>EOO +=+ +'foo' + +EOO + +: assign-leading +: +$* <"foo=bar" >>EOO +'foo=bar' + +EOO + +: append-leading +: +$* <"foo+= bar" >>EOO +'foo+=' +'bar' + +EOO + +: prepend-leading +: +$* <"foo =+bar" >>EOO +'foo' +'=+bar' + +EOO diff --git a/libbuild2/build/script/lexer+variable-line.test.testscript b/libbuild2/build/script/lexer+variable-line.test.testscript new file mode 100644 index 0000000..e4b5adb --- /dev/null +++ b/libbuild2/build/script/lexer+variable-line.test.testscript @@ -0,0 +1,12 @@ +# file : libbuild2/build/script/lexer+variable-line.test.testscript +# license : MIT; see accompanying LICENSE file + +test.arguments = variable-line + +: basic +: +$* <"a 'b c'" >>EOO +'a' +'b c' + +EOO diff --git a/libbuild2/build/script/lexer+variable.test.testscript b/libbuild2/build/script/lexer+variable.test.testscript new file mode 100644 index 0000000..54b0a30 --- /dev/null +++ b/libbuild2/build/script/lexer+variable.test.testscript @@ -0,0 +1,25 @@ +# file : libbuild2/build/script/lexer+variable.test.testscript +# license : MIT; see accompanying LICENSE file + +# Test handling custom variable names ($*, $~, $NN). +# +test.arguments = variable + +: primary-target +: +{ + : only + : + $* <">" >>EOO + '>' + + EOO + + : followed + : + $* <">abc" >>EOO + '>' + 'abc' + + EOO +} diff --git a/libbuild2/build/script/lexer.cxx b/libbuild2/build/script/lexer.cxx new file mode 100644 index 0000000..1f7e4fc --- /dev/null +++ b/libbuild2/build/script/lexer.cxx @@ -0,0 +1,254 @@ +// file : libbuild2/build/script/lexer.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +using namespace std; + +namespace build2 +{ + namespace build + { + namespace script + { + using type = token_type; + + void lexer:: + mode (build2::lexer_mode m, char ps, optional esc) + { + bool a (false); // attributes + + const char* s1 (nullptr); + const char* s2 (nullptr); + + bool s (true); // space + bool n (true); // newline + bool q (true); // quotes + + if (!esc) + { + assert (!state_.empty ()); + esc = state_.top ().escapes; + } + + switch (m) + { + case lexer_mode::command_line: + { + s1 = "=!|&<> $(#\t\n"; + s2 = "== "; + break; + } + case lexer_mode::first_token: + { + // First token on the script line. Like command_line but + // recognizes variable assignments as separators. + // + s1 = "=+!|&<> $(#\t\n"; + s2 = " == "; + break; + } + case lexer_mode::second_token: + { + // Second token on the script line. Like command_line but + // recognizes leading variable assignments. + // + // Note that to recognize only leading assignments we shouldn't + // add them to the separator strings (so this is identical to + // command_line). + // + s1 = "=!|&<> $(#\t\n"; + s2 = "== "; + break; + } + case lexer_mode::variable_line: + { + // Like value except we don't recognize '{'. + // + s1 = " $(#\t\n"; + s2 = " "; + break; + } + default: + { + base_lexer::mode (m, ps, esc); + return; + } + } + + assert (ps == '\0'); + state_.push (state {m, a, ps, s, n, q, *esc, s1, s2}); + } + + token lexer:: + next () + { + token r; + + switch (state_.top ().mode) + { + case lexer_mode::command_line: + case lexer_mode::first_token: + case lexer_mode::second_token: + case lexer_mode::variable_line: + r = next_line (); + break; + default: return base_lexer::next (); + } + + if (r.qtype != quote_type::unquoted) + ++quoted_; + + return r; + } + + token lexer:: + next_line () + { + bool sep (skip_spaces ()); + + xchar c (get ()); + uint64_t ln (c.line), cn (c.column); + + state st (state_.top ()); // Make copy (see first/second_token). + lexer_mode m (st.mode); + + auto make_token = [&sep, ln, cn] (type t) + { + return token (t, sep, ln, cn, token_printer); + }; + + // Handle attributes (do it first to make sure the flag is cleared + // regardless of what we return). + // + if (st.attributes) + { + assert (m == lexer_mode::variable_line); + + state_.top ().attributes = false; + + if (c == '[') + return make_token (type::lsbrace); + } + + if (eos (c)) + return make_token (type::eos); + + // Expire certain modes at the end of the token. Do it early in case + // we push any new mode (e.g., double quote). + // + if (m == lexer_mode::first_token || m == lexer_mode::second_token) + state_.pop (); + + // NOTE: remember to update mode() if adding new special characters. + + switch (c) + { + case '\n': + { + // Expire variable value mode at the end of the line. + // + if (m == lexer_mode::variable_line) + state_.pop (); + + sep = true; // Treat newline as always separated. + return make_token (type::newline); + } + + // Variable expansion, function call, and evaluation context. + // + case '$': return make_token (type::dollar); + case '(': return make_token (type::lparen); + } + + // Command line operator/separators. + // + if (m == lexer_mode::command_line || + m == lexer_mode::first_token || + m == lexer_mode::second_token) + { + switch (c) + { + // Comparison (==, !=). + // + case '=': + case '!': + { + if (peek () == '=') + { + get (); + return make_token (c == '=' ? type::equal : type::not_equal); + } + } + } + } + + // Command operators. + // + if (m == lexer_mode::command_line || + m == lexer_mode::first_token || + m == lexer_mode::second_token) + { + if (optional t = next_cmd_op (c, sep, m)) + return move (*t); + } + + // Variable assignment (=, +=, =+). + // + if (m == lexer_mode::second_token) + { + switch (c) + { + case '=': + { + if (peek () == '+') + { + get (); + return make_token (type::prepend); + } + else + return make_token (type::assign); + } + case '+': + { + if (peek () == '=') + { + get (); + return make_token (type::append); + } + } + } + } + + // Otherwise it is a word. + // + unget (c); + return word (st, sep); + } + + token lexer:: + word (state st, bool sep) + { + lexer_mode m (st.mode); + + // Customized implementation that handles special variable names ($>). + // + if (m != lexer_mode::variable) + return base_lexer::word (st, sep); + + xchar c (peek ()); + + if (c != '>') + return base_lexer::word (st, sep); + + get (); + + state_.pop (); // Expire the variable mode. + return token (string (1, c), + sep, + quote_type::unquoted, false, + c.line, c.column); + } + } + } +} diff --git a/libbuild2/build/script/lexer.hxx b/libbuild2/build/script/lexer.hxx new file mode 100644 index 0000000..cb1f5bf --- /dev/null +++ b/libbuild2/build/script/lexer.hxx @@ -0,0 +1,75 @@ +// file : libbuild2/build/script/lexer.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_BUILD_SCRIPT_LEXER_HXX +#define LIBBUILD2_BUILD_SCRIPT_LEXER_HXX + +#include +#include + +#include + +#include + +namespace build2 +{ + namespace build + { + namespace script + { + struct lexer_mode: build2::script::lexer_mode + { + using base_type = build2::script::lexer_mode; + + enum + { + command_line = base_type::value_next, + first_token, // Expires at the end of the token. + second_token, // Expires at the end of the token. + variable_line // Expires at the end of the line. + }; + + lexer_mode () = default; + lexer_mode (value_type v): base_type (v) {} + lexer_mode (build2::lexer_mode v): base_type (v) {} + }; + + class lexer: public build2::script::lexer + { + public: + using base_lexer = build2::script::lexer; + + // Note that neither the name nor escape arguments are copied. + // + lexer (istream& is, + const path_name& name, + uint64_t line, // Start line in the stream. + lexer_mode m, + const char* escapes = nullptr) + : base_lexer (is, name, line, + nullptr /* escapes */, + false /* set_mode */) + { + mode (m, '\0', escapes); + } + + virtual void + mode (build2::lexer_mode, + char = '\0', + optional = nullopt) override; + + virtual token + next () override; + + private: + token + next_line (); + + virtual token + word (state, bool) override; + }; + } + } +} + +#endif // LIBBUILD2_BUILD_SCRIPT_LEXER_HXX diff --git a/libbuild2/build/script/lexer.test.cxx b/libbuild2/build/script/lexer.test.cxx new file mode 100644 index 0000000..15c0954 --- /dev/null +++ b/libbuild2/build/script/lexer.test.cxx @@ -0,0 +1,77 @@ +// file : libbuild2/build/script/lexer.test.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include +#include + +#include +#include + +#include +#include + +using namespace std; + +namespace build2 +{ + namespace build + { + namespace script + { + // Usage: argv[0] + // + int + main (int argc, char* argv[]) + { + lexer_mode m; + { + assert (argc == 2); + string s (argv[1]); + + if (s == "command-line") m = lexer_mode::command_line; + else if (s == "first-token") m = lexer_mode::first_token; + else if (s == "second-token") m = lexer_mode::second_token; + else if (s == "variable-line") m = lexer_mode::variable_line; + else if (s == "variable") m = lexer_mode::variable; + else assert (false); + } + + try + { + cin.exceptions (istream::failbit | istream::badbit); + + // Some modes auto-expire so we need something underneath. + // + bool u (m != lexer_mode::command_line); + + path_name in (""); + lexer l (cin, in, 1 /* line */, lexer_mode::command_line); + if (u) + l.mode (m); + + // No use printing eos since we will either get it or loop forever. + // + for (token t (l.next ()); t.type != token_type::eos; t = l.next ()) + { + // Print each token on a separate line without quoting operators. + // + t.printer (cout, t, false); + cout << endl; + } + } + catch (const failed&) + { + return 1; + } + + return 0; + } + } + } +} + +int +main (int argc, char* argv[]) +{ + return build2::build::script::main (argc, argv); +} diff --git a/libbuild2/build/script/parser+cleanup.test.testscript b/libbuild2/build/script/parser+cleanup.test.testscript new file mode 100644 index 0000000..9a5af3d --- /dev/null +++ b/libbuild2/build/script/parser+cleanup.test.testscript @@ -0,0 +1,57 @@ +# file : libbuild2/build/script/parser+cleanup.test.testscript +# license : MIT; see accompanying LICENSE file + +: always +: +$* <>EOO +cmd &file +EOI +cmd &file +EOO + +: maybe +: +$* <>EOO +cmd &?file +EOI +cmd &?file +EOO + +: never +: +$* <>EOO +cmd &!file +EOI +cmd &!file +EOO + +: empty +: +$* <>EOE != 0 +cmd &"" +EOI +buildfile:11:6: error: empty cleanup path +EOE + +: missed-before +: +{ + : token + : + : Path missed before command next token + : + $* <>EOE != 0 + cmd & >file + EOI + buildfile:11:7: error: missing cleanup path + EOE + + : end + : Test path missed before end of command + : + $* <>EOE != 0 + cmd & + EOI + buildfile:11:6: error: missing cleanup path + EOE +} diff --git a/libbuild2/build/script/parser+command-if.test.testscript b/libbuild2/build/script/parser+command-if.test.testscript new file mode 100644 index 0000000..a18a885 --- /dev/null +++ b/libbuild2/build/script/parser+command-if.test.testscript @@ -0,0 +1,395 @@ +# file : libbuild2/build/script/parser+command-if.test.testscript +# license : MIT; see accompanying LICENSE file + +: if +: +{ + : true + : + $* <>EOO + if true foo + cmd1 + cmd2 + end + EOI + ? true foo + cmd1 + cmd2 + EOO + + : false + : + $* <>EOO + if false foo + cmd1 + cmd2 + end + EOI + ? false foo + EOO + + : not-true + : + $* <>EOO + if! true foo + cmd1 + cmd2 + end + EOI + ? true foo + EOO + + : not-false + : + $* <>EOO + if! false foo + cmd1 + cmd2 + end + EOI + ? false foo + cmd1 + cmd2 + EOO + + : without-command + : + $* <>EOE != 0 + if + cmd + end + EOI + buildfile:11:3: error: missing program + EOE +} + +: elif +: +{ + : true + : + $* <>EOO + if false + cmd1 + cmd2 + elif true + cmd3 + cmd4 + end + EOI + ? false + ? true + cmd3 + cmd4 + EOO + + : false + : + $* <>EOO + if false + cmd1 + cmd2 + elif false + cmd3 + cmd4 + end + EOI + ? false + ? false + EOO + + : not-true + : + $* <>EOO + if false + cmd1 + cmd2 + elif! true + cmd3 + cmd4 + end + EOI + ? false + ? true + EOO + + : not-false + : + $* <>EOO + if false + cmd1 + cmd2 + elif! false + cmd3 + cmd4 + end + EOI + ? false + ? false + cmd3 + cmd4 + EOO + + : without-if + : + $* <>EOE != 0 + cmd + elif true + cmd + end + EOI + buildfile:12:1: error: 'elif' without preceding 'if' + EOE + + : not-without-if + : + $* <>EOE != 0 + cmd + elif! true + cmd + end + EOI + buildfile:12:1: error: 'elif!' without preceding 'if' + EOE + + : after-else + : + $* <>EOE != 0 + if false + cmd + else + cmd + elif true + cmd + end + EOI + buildfile:15:1: error: 'elif' after 'else' + EOE +} + +: else +: +{ + : true + : + $* <>EOO + if false + cmd1 + cmd2 + else + cmd3 + cmd4 + end + EOI + ? false + cmd3 + cmd4 + EOO + + : false + : + $* <>EOO + if true + cmd1 + cmd2 + else + cmd3 + cmd4 + end + EOI + ? true + cmd1 + cmd2 + EOO + + : chain + : + $* <>EOO + if false + cmd + cmd + elif false + cmd + cmd + elif false + cmd + cmd + elif true + cmd1 + cmd2 + elif false + cmd + cmd + else + cmd + cmd + end + EOI + ? false + ? false + ? false + ? true + cmd1 + cmd2 + EOO + + : command-after + : + $* <>EOE != 0 + if true + cmd + else cmd + cmd + end + EOI + buildfile:13:6: error: expected newline instead of 'cmd' + EOE + + : without-if + : + $* <>EOE != 0 + cmd + else + cmd + end + EOI + buildfile:12:1: error: 'else' without preceding 'if' + EOE + + : after-else + : + $* <>EOE != 0 + if false + cmd + else + cmd + else + cmd + end + EOI + buildfile:15:1: error: 'else' after 'else' + EOE +} + +: end +{ + : without-if + : + $* <>EOE != 0 + cmd + end + EOI + buildfile:12:1: error: 'end' without preceding 'if' + EOE + + : before + { + : command + : + $* <>EOE != 0 + if true + cmd + end cmd + EOI + buildfile:13:5: error: expected newline instead of 'cmd' + EOE + } +} + +: nested +: +{ + : take + : + $* <>EOO + if true + cmd1 + if false + cmd + elif false + if true + cmd + end + else + cmd2 + end + cmd3 + end + EOI + ? true + cmd1 + ? false + ? false + cmd2 + cmd3 + EOO + + : skip + : + $* <>EOO + if false + cmd1 + if false + cmd + elif false + if true + cmd + end + else + cmd2 + end + cmd3 + else + cmd + end + EOI + ? false + cmd + EOO +} + +: contained +: +{ + : eos + : + $* <>EOE != 0 + if + EOI + buildfile:12:1: error: expected closing 'end' + EOE +} + +: line-index +: +$* -l <>EOO +if false + cmd + if true + cmd + end + cmd +elif false + cmd +else + cmd +end +EOI +? false # 1 +? false # 6 +cmd # 8 +EOO + +: var +: +$* <>EOO +if true + x = foo +else + x = bar +end +cmd $x +EOI +? true +cmd foo +EOO diff --git a/libbuild2/build/script/parser+command-re-parse.test.testscript b/libbuild2/build/script/parser+command-re-parse.test.testscript new file mode 100644 index 0000000..a59b49c --- /dev/null +++ b/libbuild2/build/script/parser+command-re-parse.test.testscript @@ -0,0 +1,11 @@ +# file : libbuild2/build/script/parser+command-re-parse.test.testscript +# license : MIT; see accompanying LICENSE file + +: double-quote +: +$* <>EOO +x = cmd \">-\" "'<-'" +$x +EOI +cmd '>-' '<-' +EOO diff --git a/libbuild2/build/script/parser+exit.test.testscript b/libbuild2/build/script/parser+exit.test.testscript new file mode 100644 index 0000000..53ee1b9 --- /dev/null +++ b/libbuild2/build/script/parser+exit.test.testscript @@ -0,0 +1,26 @@ +# file : libbuild2/build/script/parser+exit.test.testscript +# license : MIT; see accompanying LICENSE file + +: eq +: +$* <>EOO +cmd == 1 +EOI +cmd == 1 +EOO + +: ne +: +$* <>EOO +cmd!=1 +EOI +cmd != 1 +EOO + +: end +: +$* <>EOE != 0 +cmd != 1 <"foo" +EOI +buildfile:11:10: error: expected newline instead of '<' +EOE diff --git a/libbuild2/build/script/parser+expansion.test.testscript b/libbuild2/build/script/parser+expansion.test.testscript new file mode 100644 index 0000000..9f1e774 --- /dev/null +++ b/libbuild2/build/script/parser+expansion.test.testscript @@ -0,0 +1,35 @@ +# file : libbuild2/build/script/parser+expansion.test.testscript +# license : MIT; see accompanying LICENSE file + +: quote +: +: Make sure everything expanded as strings. +: +$* <>EOO +x = dir/ proj% proj%name proj%proj%dir/type{name name {name}} +cmd dir/ proj% proj%name proj%proj%dir/type{name name {name}} +cmd $x +EOI +cmd dir/ proj% proj%name proj%proj%dir/type{name name {name}} +cmd dir/ proj% proj%name proj%proj%dir/type{name name {name}} +EOO + +: unterm-quoted-seq +: +$* <>EOE != 0 +x = "'a bc" +cmd xy$x +EOI +:1:8: error: unterminated single-quoted sequence + buildfile:12:5: info: while parsing string 'xy'a bc' +EOE + +: invalid-redirect +: +$* <>EOE != 0 +x = "1>&a" +cmd $x +EOI +:1:4: error: stdout merge redirect file descriptor must be 2 + buildfile:12:5: info: while parsing string '1>&a' +EOE diff --git a/libbuild2/build/script/parser+here-document.test.testscript b/libbuild2/build/script/parser+here-document.test.testscript new file mode 100644 index 0000000..8ea6373 --- /dev/null +++ b/libbuild2/build/script/parser+here-document.test.testscript @@ -0,0 +1,212 @@ +# file : libbuild2/build/script/parser+here-document.test.testscript +# license : MIT; see accompanying LICENSE file + +: end-marker +: +{ + : missing-newline + : + $* <'cmd <<' 2>>EOE != 0 + buildfile:11:7: error: expected here-document end marker + EOE + + : missing-exit + : + $* <'cmd << != 0' 2>>EOE != 0 + buildfile:11:8: error: expected here-document end marker + EOE + + : missing-empty + : + $* <'cmd <<""' 2>>EOE != 0 + buildfile:11:7: error: expected here-document end marker + EOE + + : unseparated-expansion + : + $* <'cmd <>EOE != 0 + buildfile:11:10: error: here-document end marker must be literal + EOE + + : quoted-single-partial + : + $* <"cmd <>EOE != 0 + buildfile:11:7: error: partially-quoted here-document end marker + EOE + + : quoted-double-partial + : + $* <'cmd <<"FO"O' 2>>EOE != 0 + buildfile:11:7: error: partially-quoted here-document end marker + EOE + + : quoted-mixed + : + $* <"cmd <<\"FO\"'O'" 2>>EOE != 0 + buildfile:11:7: error: partially-quoted here-document end marker + EOE + + : unseparated + : + $* <>EOO + cmd <>EOO + cmd <<'EOF' + foo + EOF + EOI + cmd <>EOO + cmd <<"EOF" + foo + EOF + EOI + cmd <>EOO + cmd <>EOO + cmd <>EOO + cmd <>EOO + x = foo bar + cmd <<"EOF" + $x + EOF + EOI + cmd <>EOO + x = foo + cmd <<"EOF" + $x bar $x + EOF + EOI + cmd <>EOE != 0 + cmd <>EOO +cmd <>EOO +cmd <<"EOF" +'single' +"double" +b'o't"h" +('single' "double") +EOF +EOI +cmd <>EOO +cmd <"" +EOI +cmd <'' +EOO + +: empty-nn +: +$* <>EOO +cmd <:"" +EOI +cmd <:'' +EOO diff --git a/libbuild2/build/script/parser+pipe-expr.test.testscript b/libbuild2/build/script/parser+pipe-expr.test.testscript new file mode 100644 index 0000000..3dd6b1b --- /dev/null +++ b/libbuild2/build/script/parser+pipe-expr.test.testscript @@ -0,0 +1,132 @@ +# file : libbuild2/build/script/parser+pipe-expr.test.testscript +# license : MIT; see accompanying LICENSE file + +: pipe +: +$* <>EOO +cmd1 | cmd2|cmd3 +EOI +cmd1 | cmd2 | cmd3 +EOO + +: log +: +$* <>EOO +cmd1 || cmd2&&cmd3 +EOI +cmd1 || cmd2 && cmd3 +EOO + +: pipe-log +: +$* <>EOO +cmd1 | cmd2 && cmd3 | cmd4 +EOI +cmd1 | cmd2 && cmd3 | cmd4 +EOO + +: exit +: +$* <>EOO +cmd1|cmd2==1&&cmd3!=0|cmd4 +EOI +cmd1 | cmd2 == 1 && cmd3 != 0 | cmd4 +EOO + +: here-doc +: +$* <>EOO +cmd1 <>EOO2 && cmd3 <&1 | cmd4 2>>EOE4 >>EOO4 +input +one +EOI1 +ouput +two +EOO2 +input +three +EOI3 +error +four +EOE4 +output +four +EOO4 +EOI +cmd1 <>EOO2 && cmd3 <&1 | cmd4 >>EOO4 2>>EOE4 +input +one +EOI1 +ouput +two +EOO2 +input +three +EOI3 +output +four +EOO4 +error +four +EOE4 +EOO + +: leading +: +$* <>EOE != 0 +| cmd +EOI +buildfile:11:1: error: missing program +EOE + +: trailing +: +$* <>EOE != 0 +cmd && +EOI +buildfile:11:7: error: missing program +EOE + +: redirected +: +{ + : input + : + { + : first + : + $* <>EOO + cmd1 >EOE != 0 + cmd1 | cmd2 >EOO + cmd1 | cmd2 >foo + EOI + cmd1 | cmd2 >foo + EOO + + : non-last + : + $* <>EOE != 0 + cmd1 >foo | cmd2 + EOI + buildfile:11:11: error: stdout is both redirected and piped + EOE + } +} diff --git a/libbuild2/build/script/parser+pre-parse.test.testscript b/libbuild2/build/script/parser+pre-parse.test.testscript new file mode 100644 index 0000000..4aff3e8 --- /dev/null +++ b/libbuild2/build/script/parser+pre-parse.test.testscript @@ -0,0 +1,22 @@ +# file : libbuild2/build/script/parser+pre-parse.test.testscript +# license : MIT; see accompanying LICENSE file + +: attribute +: +{ + : name + : + $* <>EOE != 0 + x = [foo] + EOI + buildfile:11:5: error: unknown value attribute foo + EOE + + : name-value + : + $* <>EOE != 0 + x = [foo=bar] + EOI + buildfile:11:5: error: unknown value attribute foo=bar + EOE +} diff --git a/libbuild2/build/script/parser+redirect.test.testscript b/libbuild2/build/script/parser+redirect.test.testscript new file mode 100644 index 0000000..641381e --- /dev/null +++ b/libbuild2/build/script/parser+redirect.test.testscript @@ -0,0 +1,441 @@ +# file : libbuild2/build/script/parser+redirect.test.testscript +# license : MIT; see accompanying LICENSE file + +# @@ Add tests for redirects other than trace, here-*, file and merge. +# @@ Does it make sense to split into separate files - one per redirect type? +# + +: trace +: +{ + $* <'cmd >!' >'cmd >!' : out + $* <'cmd 2>!' >'cmd 2>!' : err +} + +: str +: +{ + : literal + : + { + : portable-path + : + $* <>EOO + cmd /bar 2>/baz + EOI + cmd /bar 2>/baz + EOO + } + + : regex + : + { + : portable-path + : + $* <>EOO + cmd >/~%foo% 2>/~%bar% + EOI + cmd >/~%foo% 2>/~%bar% + EOO + } +} + +: doc +: +{ + : literal + : + { + : portable-path + : + $* <>EOO + cmd </EOO_ 2>/EOE_ + foo + EOI_ + bar + EOO_ + baz + EOE_ + EOI + cmd </EOO_ 2>/EOE_ + foo + EOI_ + bar + EOO_ + baz + EOE_ + EOO + + : sharing + : + { + : in-out + : + $* <>EOO + cmd <<:/EOF >>:/EOF + foo + EOF + EOI + cmd <<:/EOF >>:/EOF + foo + EOF + EOO + + : different + : + { + : modifiers + : + $* <>EOE != 0 + cmd <<:/EOF >>:EOF + foo + EOF + EOI + buildfile:11:16: error: different modifiers for shared here-document 'EOF' + EOE + + : quoting + : + $* <>EOE != 0 + cmd <>"EOF" + foo + EOF + EOI + buildfile:11:13: error: different quoting for shared here-document 'EOF' + EOE + } + } + } + + : regex + : + { + : portable-path + : + $* <>EOO + cmd >/~%EOF% 2>/~%EOE% + foo + EOF + bar + EOE + EOI + cmd >/~%EOF% 2>/~%EOE% + foo + EOF + bar + EOE + EOO + + : sharing + : + { + : in-out + : + $* <>EOO + cmd >>~/EOF/ 2>>~/EOF/ + foo + EOF + EOI + cmd >>~/EOF/ 2>>~/EOF/ + foo + EOF + EOO + + : different + : + { + : introducers + : + $* <>EOE != 0 + cmd >>~/EOF/ 2>>~%EOF% + foo + EOF + EOI + buildfile:11:18: error: different introducers for shared here-document regex 'EOF' + EOE + + : flags + : + $* <>EOE != 0 + cmd >>~/EOF/ 2>>~/EOF/i + foo + EOF + EOI + buildfile:11:18: error: different global flags for shared here-document regex 'EOF' + EOE + } + } + } + + : overriding + : + { + : literal + : + { + : with + : + { + : string + : + $* <>EOO + cmd >>EOF >bar + foo + EOF + EOI + cmd >bar + EOO + + : regex + : + $* <>EOO + cmd >>FOO >>~/BAR/ + foo + FOO + bar + BAR + EOI + cmd >>~/BAR/ + bar + BAR + EOO + + : self + : + $* <>EOO + cmd >>EOF >>EOF + foo + EOF + EOI + cmd >>EOF + foo + EOF + EOO + + : different-modifiers + : + $* <>EOE != 0 + cmd >>EOF >>/EOF + foo + EOF + EOI + buildfile:11:14: error: different modifiers for shared here-document 'EOF' + EOE + } + } + + : shared + : + { + : after-sharing + : + $* <>EOO + cmd >>EOF 2>>EOF >bar + foo + EOF + EOI + cmd >bar 2>>EOF + foo + EOF + EOO + + : before-sharing + : + $* <>EOO + cmd >>EOF >bar 2>>EOF + foo + EOF + EOI + cmd >bar 2>>EOF + foo + EOF + EOO + } + } +} + +: file +: +{ + : cmp + : + $* <>EOO + cmd 0<<>>b 2>>>c + EOI + cmd <<>>b 2>>>c + EOO + + : write + : + $* <>EOO + cmd 1>=b 2>+c + EOI + cmd >=b 2>+c + EOO + + : quote + : + $* <>EOO + cmd 0<<<"a f" 1>="b f" 2>+"c f" + EOI + cmd <<<'a f' >='b f' 2>+'c f' + EOO + + : in + : + { + : missed + : + $* <>EOE !=0 + cmd <<< + EOI + buildfile:11:8: error: missing stdin file + EOE + + : empty + : + $* <>EOE !=0 + cmd <<<"" + EOI + buildfile:11:8: error: empty stdin redirect path + EOE + } + + : out + : + { + : missed + : + $* <>EOE !=0 + cmd >= + EOI + buildfile:11:7: error: missing stdout file + EOE + + : empty + : + $* <>EOE !=0 + cmd >="" + EOI + buildfile:11:7: error: empty stdout redirect path + EOE + } + + : err + : + { + : missed + : + $* <>EOE !=0 + cmd 2>= + EOI + buildfile:11:8: error: missing stderr file + EOE + + : empty + : + $* <>EOE !=0 + cmd 2>="" + EOI + buildfile:11:8: error: empty stderr redirect path + EOE + } +} + +: merge +{ + : out + : + { + : err + : + $* <>EOO + cmd 1>&2 + EOI + cmd >&2 + EOO + + : no-mutual + : + $* <>EOO + cmd 1>&2 2>&1 2>a + EOI + cmd >&2 2>a + EOO + + : not-descriptor + : + $* <>EOE != 0 + cmd 1>&a + EOI + buildfile:11:8: error: stdout merge redirect file descriptor must be 2 + EOE + + : self + : + $* <>EOE != 0 + cmd 1>&1 + EOI + buildfile:11:8: error: stdout merge redirect file descriptor must be 2 + EOE + + : missed + : + $* <>EOE != 0 + cmd 1>& + EOI + buildfile:11:8: error: missing stdout file descriptor + EOE + } + + : err + { + : out + : + $* <>EOO + cmd 2>&1 + EOI + cmd 2>&1 + EOO + + : no-mutual + : + $* <>EOO + cmd 1>&2 2>&1 >a + EOI + cmd >a 2>&1 + EOO + + : not-descriptor + : + $* <>EOE != 0 + cmd 2>&a + EOI + buildfile:11:8: error: stderr merge redirect file descriptor must be 1 + EOE + + : self + : + $* <>EOE != 0 + cmd 2>&2 + EOI + buildfile:11:8: error: stderr merge redirect file descriptor must be 1 + EOE + + : missed + : + $* <>EOE != 0 + cmd 2>& + EOI + buildfile:11:8: error: missing stderr file descriptor + EOE + } + + : mutual + : + $* <>EOE != 0 + cmd 1>&2 2>&1 + EOI + buildfile:11:14: error: stdout and stderr redirected to each other + EOE +} diff --git a/libbuild2/build/script/parser+regex.test.testscript b/libbuild2/build/script/parser+regex.test.testscript new file mode 100644 index 0000000..4a47e73 --- /dev/null +++ b/libbuild2/build/script/parser+regex.test.testscript @@ -0,0 +1,222 @@ +# file : libbuild2/build/script/parser+regex.test.testscript +# license : MIT; see accompanying LICENSE file + +: here-string +: +{ + : stdout + : + { + : missed + : + $* <'cmd >~' 2>>EOE != 0 + buildfile:11:7: error: missing stdout here-string regex + EOE + + : no-introducer + : + $* <'cmd >~""' 2>>EOE != 0 + buildfile:11:7: error: no introducer character in stdout regex redirect + EOE + + : no-term-introducer + : + $* <'cmd >~/' 2>>EOE != 0 + buildfile:11:7: error: no closing introducer character in stdout regex redirect + EOE + + : portable-path-introducer + : + $* <'cmd >/~/foo/' 2>>EOE != 0 + buildfile:11:8: error: portable path modifier and '/' introducer in stdout regex redirect + EOE + + : empty + : + $* <'cmd >~//' 2>>EOE != 0 + buildfile:11:7: error: stdout regex redirect is empty + EOE + + : no-flags + : + $* <'cmd >~/fo*/' >'cmd >~/fo*/' + + : idot + : + $* <'cmd >~/fo*/d' >'cmd >~/fo*/d' + + : icase + : + $* <'cmd >~/fo*/i' >'cmd >~/fo*/i' + + : invalid-flags1 + : + $* <'cmd >~/foo/z' 2>>EOE != 0 + buildfile:11:7: error: junk at the end of stdout regex redirect + EOE + + : invalid-flags2 + : + $* <'cmd >~/foo/iz' 2>>EOE != 0 + buildfile:11:7: error: junk at the end of stdout regex redirect + EOE + + : no-newline + : + $* <'cmd >:~/fo*/' >'cmd >:~/fo*/' + } + + : stderr + : + { + : missed + : + $* <'cmd 2>~' 2>>EOE != 0 + buildfile:11:8: error: missing stderr here-string regex + EOE + + : no-introducer + : + : Note that there is no need to reproduce all the errors as for stdout. + : All we need is to make sure that the proper description is passed to + : the parse_regex() function. + : + $* <'cmd 2>~""' 2>>EOE != 0 + buildfile:11:8: error: no introducer character in stderr regex redirect + EOE + } + + : modifier-last + : + $* <'cmd >~/x' 2>>EOE != 0 + buildfile:11:7: error: no closing introducer character in stdout regex redirect + EOE +} + +: here-doc +: +{ + : stdout + : + { + : missed + : + $* <'cmd >>~' 2>>EOE != 0 + buildfile:11:8: error: expected here-document regex end marker + EOE + + : portable-path-introducer + : + $* <>EOE != 0 + cmd >>/~/EOO/ + foo + EOO + EOI + buildfile:11:5: error: portable path modifier and '/' introducer in here-document regex end marker + EOE + + : unterminated-line-char + : + $* <>EOE != 0 + cmd >>~/EOO/ + / + EOO + EOI + buildfile:12:1: error: no syntax line characters + EOE + + : empty + : + $* <>EOE != 0 + cmd >>:~/EOO/ + EOO + EOI + buildfile:12:1: error: empty here-document regex + EOE + + : no-flags + : + $* <>EOO + cmd 2>>~/EOE/ + foo + /? + /foo/ + /foo/* + /foo/i + /foo/i* + + // + //* + EOE + EOI + cmd 2>>~/EOE/ + foo + /? + /foo/ + /foo/* + /foo/i + /foo/i* + + // + //* + EOE + EOO + + : no-newline + : + $* <'cmd >:~/fo*/' >'cmd >:~/fo*/' + $* <>EOO + cmd 2>>:~/EOE/ + foo + EOE + EOI + cmd 2>>:~/EOE/ + foo + EOE + EOO + + : end-marker-restore + : + { + : idot + : + $* <>EOO + cmd 2>>~/EOE/d + foo + EOE + EOI + cmd 2>>~/EOE/d + foo + EOE + EOO + + : icase + : + $* <>EOO + cmd 2>>~/EOE/i + foo + EOE + EOI + cmd 2>>~/EOE/i + foo + EOE + EOO + } + } + + : stderr + : + { + : missed + : + $* <'cmd 2>>~' 2>>EOE != 0 + buildfile:11:9: error: expected here-document regex end marker + EOE + } + + : modifier-last + : + $* <'cmd >>~:/FOO/' 2>>EOE != 0 + buildfile:11:5: error: no closing introducer character in here-document regex end marker + EOE +} diff --git a/libbuild2/build/script/parser+variable.test.testscript b/libbuild2/build/script/parser+variable.test.testscript new file mode 100644 index 0000000..2d9bb1e --- /dev/null +++ b/libbuild2/build/script/parser+variable.test.testscript @@ -0,0 +1,41 @@ +# file : libbuild2/build/script/parser+variable.test.testscript +# license : MIT; see accompanying LICENSE file + +: assignment +: +$* <>EOO +a = b +echo $a +EOI +echo b +EOO + +: primary-target +: +$* <>EOO +echo $> +EOI +echo driver +EOO + +: no-newline +: +$* <:'echo a' 2>>EOE != 0 +buildfile:11:7: error: expected newline instead of +EOE + +: set-primary-target +: +$* <>EOE != 0 +> = a +EOI +buildfile:11:1: error: missing program +EOE + +: empty-name +: +$* <>EOE != 0 += b +EOI +buildfile:11:1: error: missing variable name +EOE diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx new file mode 100644 index 0000000..4bdf5c7 --- /dev/null +++ b/libbuild2/build/script/parser.cxx @@ -0,0 +1,337 @@ +// file : libbuild2/build/script/parser.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include + +using namespace std; + +namespace build2 +{ + namespace build + { + namespace script + { + using type = token_type; + + // + // Pre-parse. + // + + void parser:: + pre_parse (istream& is, const path_name& pn, uint64_t line, script& s) + { + path_ = &pn; + + pre_parse_ = true; + + lexer l (is, *path_, line, lexer_mode::command_line); + set_lexer (&l); + + script_ = &s; + runner_ = nullptr; + environment_ = nullptr; + + script_->start_loc = location (*path_, line, 1); + + token t (pre_parse_script ()); + + assert (t.type == type::eos); + + script_->end_loc = get_location (t); + } + + token parser:: + pre_parse_script () + { + // enter: next token is first token of the script + // leave: eos (returned) + + token t; + type tt; + + // Parse lines until we see eos. + // + for (;;) + { + // Start lexing each line. + // + tt = peek (lexer_mode::first_token); + + // Determine the line type by peeking at the first token. + // + switch (tt) + { + case type::eos: + { + next (t, tt); + return t; + } + default: + { + pre_parse_line (t, tt); + assert (tt == type::newline); + break; + } + } + } + } + + void parser:: + pre_parse_line (token& t, type& tt, bool if_line) + { + // Determine the line type/start token. + // + line_type lt ( + pre_parse_line_start (t, tt, lexer_mode::second_token)); + + line ln; + switch (lt) + { + case line_type::var: + { + // Check if we are trying to modify any of the special variables + // ($>). + // + string& n (t.value); + + if (n == ">") + fail (t) << "attempt to set '" << n << "' variable"; + + // Pre-enter the variable. + // + ln.var = &script_->var_pool.insert (move (n)); + + next (t, tt); // Assignment kind. + + mode (lexer_mode::variable_line); + parse_variable_line (t, tt); + + if (tt != type::newline) + fail (t) << "expected newline instead of " << t; + + break; + } + case line_type::cmd_elif: + case line_type::cmd_elifn: + case line_type::cmd_else: + case line_type::cmd_end: + { + if (!if_line) + { + fail (t) << lt << " without preceding 'if'"; + } + } + // Fall through. + case line_type::cmd_if: + case line_type::cmd_ifn: + next (t, tt); // Skip to start of command. + // Fall through. + case line_type::cmd: + { + pair p; + + if (lt != line_type::cmd_else && lt != line_type::cmd_end) + p = parse_command_expr (t, tt); + + if (tt != type::newline) + fail (t) << "expected newline instead of " << t; + + parse_here_documents (t, tt, p); + break; + } + } + + assert (tt == type::newline); + + ln.type = lt; + ln.tokens = replay_data (); + script_->lines.push_back (move (ln)); + + if (lt == line_type::cmd_if || lt == line_type::cmd_ifn) + { + tt = peek (lexer_mode::first_token); + + pre_parse_if_else (t, tt); + } + } + + void parser:: + pre_parse_if_else (token& t, type& tt) + { + // enter: peeked first token of next line (type in tt) + // leave: newline + + // Parse lines until we see closing 'end'. Nested if-else blocks are + // handled recursively. + // + for (line_type bt (line_type::cmd_if); // Current block. + ; + tt = peek (lexer_mode::first_token)) + { + const location ll (get_location (peeked ())); + + if (tt == type::eos) + fail (ll) << "expected closing 'end'"; + + // Parse one line. Note that this one line can still be multiple + // lines in case of if-else. In this case we want to view it as + // cmd_if, not cmd_end. Thus remember the start position of the + // next logical line. + // + size_t i (script_->lines.size ()); + + pre_parse_line (t, tt, true /* if_line */); + assert (tt == type::newline); + + line_type lt (script_->lines[i].type); + + // First take care of 'end'. + // + if (lt == line_type::cmd_end) + return; + + // Check if-else block sequencing. + // + if (bt == line_type::cmd_else) + { + if (lt == line_type::cmd_else || + lt == line_type::cmd_elif || + lt == line_type::cmd_elifn) + fail (ll) << lt << " after " << bt; + } + + // Update current if-else block. + // + switch (lt) + { + case line_type::cmd_elif: + case line_type::cmd_elifn: bt = line_type::cmd_elif; break; + case line_type::cmd_else: bt = line_type::cmd_else; break; + default: break; + } + } + } + + command_expr parser:: + parse_command_line (token& t, type& tt) + { + // enter: first token of the command line + // leave: + + // Note: this one is only used during execution. + // + assert (!pre_parse_); + + pair p (parse_command_expr (t, tt)); + assert (tt == type::newline); + + parse_here_documents (t, tt, p); + assert (tt == type::newline); + + return move (p.first); + } + + // + // Execute. + // + + void parser:: + execute (environment& e, runner& r) + { + path_ = nullptr; // Set by replays. + + pre_parse_ = false; + + set_lexer (nullptr); + + script_ = nullptr; + runner_ = &r; + environment_ = &e; + + exec_script (); + } + + void parser:: + exec_script () + { + const script& s (environment_->script); + + runner_->enter (*environment_, s.start_loc); + + // Note that we rely on "small function object" optimization for the + // exec_*() lambdas. + // + auto exec_set = [this] (const variable& var, + token& t, build2::script::token_type& tt, + const location&) + { + next (t, tt); + type kind (tt); // Assignment kind. + + mode (lexer_mode::variable_line); + value rhs (parse_variable_line (t, tt)); + + assert (tt == type::newline); + + // Assign. + // + value& lhs (kind == type::assign + ? environment_->assign (var) + : environment_->append (var)); + + apply_value_attributes (&var, lhs, move (rhs), kind); + }; + + auto exec_cmd = [this] (token& t, build2::script::token_type& tt, + size_t li, + bool single, + const location& ll) + { + // We use the 0 index to signal that this is the only command. + // + if (single) + li = 0; + + command_expr ce ( + parse_command_line (t, static_cast (tt))); + + runner_->run (*environment_, ce, li, ll); + }; + + auto exec_if = [this] (token& t, build2::script::token_type& tt, + size_t li, + const location& ll) + { + command_expr ce ( + parse_command_line (t, static_cast (tt))); + + // Assume if-else always involves multiple commands. + // + return runner_->run_if (*environment_, ce, li, ll); + }; + + size_t li (1); + + exec_lines (s.lines.begin (), s.lines.end (), + exec_set, exec_cmd, exec_if, + li); + + runner_->leave (*environment_, s.end_loc); + } + + lookup parser:: + lookup_variable (name&& qual, string&& name, const location& loc) + { + assert (!pre_parse_); + + if (!qual.empty ()) + fail (loc) << "qualified variable name"; + + return environment_->lookup (name); + } + } + } +} diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx new file mode 100644 index 0000000..a8ffb25 --- /dev/null +++ b/libbuild2/build/script/parser.hxx @@ -0,0 +1,92 @@ +// file : libbuild2/build/script/parser.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_BUILD_SCRIPT_PARSER_HXX +#define LIBBUILD2_BUILD_SCRIPT_PARSER_HXX + +#include +#include +#include + +#include + +#include + +#include +#include + +namespace build2 +{ + namespace build + { + namespace script + { + class runner; + + class parser: public build2::script::parser + { + // Pre-parse. Issue diagnostics and throw failed in case of an error. + // + public: + parser (context& c): build2::script::parser (c) {} + + void + pre_parse (istream&, const path_name&, uint64_t line, script&); + + // Recursive descent parser. + // + // Usually (but not always) parse functions receive the token/type + // from which it should start consuming and in return the token/type + // should contain the first token that has not been consumed. + // + // Functions that are called parse_*() rather than pre_parse_*() are + // used for both stages. + // + protected: + token + pre_parse_script (); + + void + pre_parse_line (token&, token_type&, bool if_line = false); + + void + pre_parse_if_else (token&, token_type&); + + command_expr + parse_command_line (token&, token_type&); + + // Workaround for GCC 4.9 that fails to compile the base class + // protected member function call from a lambda defined in the derived + // class. + // + using build2::parser::apply_value_attributes; + + // Execute. Issue diagnostics and throw failed in case of an error. + // + public: + void + execute (environment&, runner&); + + protected: + void + exec_script (); + + // Customization hooks. + // + protected: + virtual lookup + lookup_variable (name&&, string&&, const location&) override; + + protected: + script* script_; + + // Execute state. + // + runner* runner_; + environment* environment_; + }; + } + } +} + +#endif // LIBBUILD2_BUILD_SCRIPT_PARSER_HXX diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx new file mode 100644 index 0000000..2763464 --- /dev/null +++ b/libbuild2/build/script/parser.test.cxx @@ -0,0 +1,145 @@ +// file : libbuild2/build/script/parser.test.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include + +using namespace std; + +namespace build2 +{ + namespace build + { + namespace script + { + class print_runner: public runner + { + public: + print_runner (bool line): line_ (line) {} + + virtual void + enter (environment&, const location&) override {} + + virtual void + run (environment&, + const command_expr& e, + size_t i, + const location&) override + { + cout << e; + + if (line_) + cout << " # " << i; + + cout << endl; + } + + virtual bool + run_if (environment&, + const command_expr& e, + size_t i, + const location&) override + { + cout << "? " << e; + + if (line_) + cout << " # " << i; + + cout << endl; + + return e.back ().pipe.back ().program.string () == "true"; + } + + virtual void + leave (environment&, const location&) override {} + + private: + bool line_; + }; + + // Usage: argv[0] [-l] + // + int + main (int argc, char* argv[]) + { + tracer trace ("main"); + + // Fake build system driver, default verbosity. + // + init_diag (1); + init (nullptr, argv[0]); + + // Serial execution. + // + scheduler sched (1); + global_mutexes mutexes (1); + context ctx (sched, mutexes); + + bool line (false); + + for (int i (1); i != argc; ++i) + { + string a (argv[i]); + + if (a == "-l") + line = true; + else + assert (false); + } + + try + { + cin.exceptions (istream::failbit | istream::badbit); + + // Enter mock target. Use fixed name and path so that we can use + // them in expected results. Strictly speaking target path should + // be absolute. However, the buildscript implementation doesn't + // really care. + // + file& tt ( + ctx.targets.insert (work, + dir_path (), + "driver", + string (), + trace)); + + tt.path (path ("driver")); + + // Parse and run. + // + path_name nm ("buildfile"); + + script s; + parser p (ctx); + p.pre_parse (cin, nm, 11 /* line */, s); + + environment e (s, tt); + print_runner r (line); + p.execute (e, r); + } + catch (const failed&) + { + return 1; + } + + return 0; + } + } + } +} + +int +main (int argc, char* argv[]) +{ + return build2::build::script::main (argc, argv); +} diff --git a/libbuild2/build/script/runner.cxx b/libbuild2/build/script/runner.cxx new file mode 100644 index 0000000..f52d1c0 --- /dev/null +++ b/libbuild2/build/script/runner.cxx @@ -0,0 +1,50 @@ +// file : libbuild2/build/script/runner.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include + +namespace build2 +{ + namespace build + { + namespace script + { + void default_runner:: + enter (environment&, const location&) + { + // Noop. + } + + void default_runner:: + leave (environment& env, const location& ll) + { + clean (env, ll); + } + + void default_runner:: + run (environment& env, + const command_expr& expr, + size_t li, + const location& ll) + { + if (verb >= 3) + text << ": " << expr; + + build2::script::run (env, expr, li, ll); + } + + bool default_runner:: + run_if (environment& env, + const command_expr& expr, + size_t li, const location& ll) + { + if (verb >= 3) + text << ": ?" << expr; + + return build2::script::run_if (env, expr, li, ll); + } + } + } +} diff --git a/libbuild2/build/script/runner.hxx b/libbuild2/build/script/runner.hxx new file mode 100644 index 0000000..db211df --- /dev/null +++ b/libbuild2/build/script/runner.hxx @@ -0,0 +1,78 @@ +// file : libbuild2/build/script/runner.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_BUILD_SCRIPT_RUNNER_HXX +#define LIBBUILD2_BUILD_SCRIPT_RUNNER_HXX + +#include +#include + +#include + +namespace build2 +{ + namespace build + { + struct common; + + namespace script + { + class runner + { + public: + // Location is the script start location (for diagnostics, etc). + // + virtual void + enter (environment&, const location&) = 0; + + // Index is the 1-base index of this command line in the command list. + // If it is 0 then it means there is only one command. This + // information can be used, for example, to derive file names. + // + // Location is the start position of this command line in the script. + // It can be used in diagnostics. + // + virtual void + run (environment&, + const command_expr&, + size_t index, + const location&) = 0; + + virtual bool + run_if (environment&, + const command_expr&, + size_t, + const location&) = 0; + + // Location is the script end location (for diagnostics, etc). + // + virtual void + leave (environment&, const location&) = 0; + }; + + class default_runner: public runner + { + public: + virtual void + enter (environment&, const location&) override; + + virtual void + run (environment&, + const command_expr&, + size_t, + const location&) override; + + virtual bool + run_if (environment&, + const command_expr&, + size_t, + const location&) override; + + virtual void + leave (environment&, const location&) override; + }; + } + } +} + +#endif // LIBBUILD2_BUILD_SCRIPT_RUNNER_HXX diff --git a/libbuild2/build/script/script.cxx b/libbuild2/build/script/script.cxx new file mode 100644 index 0000000..26fe006 --- /dev/null +++ b/libbuild2/build/script/script.cxx @@ -0,0 +1,156 @@ +// file : libbuild2/build/script/script.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include + +#include + +using namespace std; + +namespace build2 +{ + namespace build + { + namespace script + { + // script + // + script:: + script () + : primary_target_var (var_pool.insert (">")) + { + } + + // environment + // + static const string wd_name ("current directory"); + + environment:: + environment (const build::script::script& s, const target& pt) + : build2::script::environment ( + pt.ctx, + cast (pt.ctx.global_scope["build.host"]), + work, + wd_name), + script (s), + vars (context, false /* global */), + primary_target (pt) + { + // Set the $> variable. + // + { + value& v (assign (s.primary_target_var)); + + if (auto* t = pt.is_a ()) + v = t->path (); + else + fail << "target " << pt << " is not path-based"; + } + } + + void environment:: + set_variable (string&& nm, names&& val, const string& attrs) + { + // Set the variable value and attributes. Note that we need to aquire + // unique lock before potentially changing the script's variable + // pool. The obtained variable reference can safelly be used with no + // locking as the variable pool is an associative container + // (underneath) and we are only adding new variables into it. + // + ulock ul (script.var_pool_mutex); + + const variable& var ( + const_cast (script).var_pool. + insert (move (nm))); + + ul.unlock (); + + value& lhs (assign (var)); + + // If there are no attributes specified then the variable assignment + // is straightforward. Otherwise we will use the build2 parser helper + // function. + // + if (attrs.empty ()) + lhs.assign (move (val), &var); + else + { + build2::script::parser p (context); + p.apply_value_attributes (&var, + lhs, + value (move (val)), + attrs, + token_type::assign, + path_name ("")); + } + } + + lookup environment:: + lookup (const variable& var) const + { + auto p (vars.lookup (var)); + if (p.first != nullptr) + return lookup_type (*p.first, p.second, vars); + + return lookup_in_buildfile (var.name); + } + + lookup environment:: + lookup (const string& name) const + { + // Every variable that is ever set in a script has been pre-entered + // during pre-parse or introduced with the set builtin during + // execution. Which means that if one is not found in the script pool + // then it can only possibly be set in the buildfile. + // + // Note that we need to acquire the variable pool lock. The pool can + // be changed from multiple threads by the set builtin. The obtained + // variable pointer can safelly be used with no locking as the + // variable pool is an associative container (underneath) and we are + // only adding new variables into it. + // + const variable* pvar (nullptr); + + slock sl (script.var_pool_mutex); + pvar = script.var_pool.find (name); + sl.unlock (); + + return pvar != nullptr ? lookup (*pvar) : lookup_in_buildfile (name); + } + + lookup environment:: + lookup_in_buildfile (const string& n) const + { + // Switch to the corresponding buildfile variable. Note that we don't + // want to insert a new variable into the pool (we might be running + // in parallel). Plus, if there is no such variable, then we cannot + // possibly find any value. + // + const variable* pvar (context.var_pool.find (n)); + + if (pvar == nullptr) + return lookup_type (); + + return primary_target[*pvar]; + } + + value& environment:: + append (const variable& var) + { + auto l (lookup (var)); + + if (l.defined () && l.belongs (*this)) // Existing var. + return vars.modify (l); + + value& r (assign (var)); // NULL. + + if (l.defined ()) + r = *l; // Copy value (and type) from the outer scope. + + return r; + } + } + } +} diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx new file mode 100644 index 0000000..18ed4c4 --- /dev/null +++ b/libbuild2/build/script/script.hxx @@ -0,0 +1,111 @@ +// file : libbuild2/build/script/script.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_BUILD_SCRIPT_SCRIPT_HXX +#define LIBBUILD2_BUILD_SCRIPT_SCRIPT_HXX + +#include +#include +#include + +#include + +#include + +namespace build2 +{ + namespace build + { + namespace script + { + using build2::script::line; + using build2::script::line_type; + using build2::script::command_expr; + + // Once parsed, the script can be executed in multiple threads with the + // state (variable values, etc) maintained by the environment object. + // + class script + { + public: + build2::script::lines lines; + + variable_pool var_pool; + mutable shared_mutex var_pool_mutex; + + const variable& primary_target_var; // $> + + location start_loc; + location end_loc; + + script (); + + script (script&&) = delete; + script (const script&) = delete; + script& operator= (script&&) = delete; + script& operator= (const script&) = delete; + }; + + class environment: public build2::script::environment + { + public: + environment (const script&, const target& primary_target); + + environment (environment&&) = delete; + environment (const environment&) = delete; + environment& operator= (environment&&) = delete; + environment& operator= (const environment&) = delete; + + public: + const build::script::script& script; + + // Note that if we pass the variable name as a string, then it will + // be looked up in the wrong pool. + // + variable_map vars; + + const target& primary_target; + + virtual void + set_variable (string&& name, names&&, const string& attrs) override; + + // Variables. + // + public: + // Lookup the variable starting from this environment, then the + // primary target, and then outer buildfile scopes. + // + using lookup_type = build2::lookup; + + lookup_type + lookup (const variable&) const; + + lookup_type + lookup (const string&) const; + + // As above but only look for buildfile variables. + // + lookup_type + lookup_in_buildfile (const string&) const; + + // Return a value suitable for assignment. If the variable does not + // exist in this environment's map, then a new one with the NULL value + // is added and returned. Otherwise the existing value is returned. + // + value& + assign (const variable& var) {return vars.assign (var);} + + // Return a value suitable for append/prepend. If the variable does + // not exist in this environment's map, then outer scopes are searched + // for the same variable. If found then a new variable with the found + // value is added to the environment and returned. Otherwise this + // function proceeds as assign() above. + // + value& + append (const variable&); + }; + } + } +} + +#endif // LIBBUILD2_BUILD_SCRIPT_SCRIPT_HXX diff --git a/libbuild2/build/script/token.cxx b/libbuild2/build/script/token.cxx new file mode 100644 index 0000000..289dfd2 --- /dev/null +++ b/libbuild2/build/script/token.cxx @@ -0,0 +1,23 @@ +// file : libbuild2/build/script/token.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +using namespace std; + +namespace build2 +{ + namespace build + { + namespace script + { + void + token_printer (ostream& os, const token& t, bool d) + { + // No build script-specific tokens so far. + // + build2::script::token_printer (os, t, d); + } + } + } +} diff --git a/libbuild2/build/script/token.hxx b/libbuild2/build/script/token.hxx new file mode 100644 index 0000000..4cd90d7 --- /dev/null +++ b/libbuild2/build/script/token.hxx @@ -0,0 +1,36 @@ +// file : libbuild2/build/script/token.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_BUILD_SCRIPT_TOKEN_HXX +#define LIBBUILD2_BUILD_SCRIPT_TOKEN_HXX + +#include +#include + +#include + +namespace build2 +{ + namespace build + { + namespace script + { + struct token_type: build2::script::token_type + { + using base_type = build2::script::token_type; + + // No build script-specific tokens so far. + // + + token_type () = default; + token_type (value_type v): base_type (v) {} + token_type (build2::token_type v): base_type (v) {} + }; + + void + token_printer (ostream&, const token&, bool); + } + } +} + +#endif // LIBBUILD2_BUILD_SCRIPT_TOKEN_HXX diff --git a/libbuild2/buildfile b/libbuild2/buildfile index 9813808..5f7bc11 100644 --- a/libbuild2/buildfile +++ b/libbuild2/buildfile @@ -30,6 +30,8 @@ lib{build2}: libul{build2}: \ libul{build2}: script/{hxx ixx txx cxx}{** -*-options -**.test...} \ script/{hxx ixx cxx}{builtin-options} +libul{build2}: build/{hxx ixx txx cxx}{** -**.test...} + # Note that this won't work in libul{} since it's not installed. # lib{build2}: cxx{utility-installed}: for_install = true @@ -108,6 +110,7 @@ exe{*.test}: for t: cxx{ *.test...} \ script/cxx{**.test...} \ + build/cxx{**.test...} \ config/cxx{**.test...} \ dist/cxx{**.test...} \ install/cxx{**.test...} \ -- cgit v1.1