From 28f8338ded34f160e0083da9be4679bc778be7ca Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Fri, 25 Nov 2016 11:18:34 +0200 Subject: Distinguish token quoting type and completeness --- unit-tests/buildfile | 2 +- unit-tests/lexer/buildfile | 13 ++++ unit-tests/lexer/comment.test | 112 +++++++++++++++++++++++++++++ unit-tests/lexer/driver.cxx | 94 +++++++++++++++++++++++++ unit-tests/lexer/quoting.test | 95 +++++++++++++++++++++++++ unit-tests/test/script/lexer/comment.test | 113 ------------------------------ 6 files changed, 315 insertions(+), 114 deletions(-) create mode 100644 unit-tests/lexer/buildfile create mode 100644 unit-tests/lexer/comment.test create mode 100644 unit-tests/lexer/driver.cxx create mode 100644 unit-tests/lexer/quoting.test delete mode 100644 unit-tests/test/script/lexer/comment.test (limited to 'unit-tests') diff --git a/unit-tests/buildfile b/unit-tests/buildfile index 5d06ec7..f8cfb9d 100644 --- a/unit-tests/buildfile +++ b/unit-tests/buildfile @@ -2,6 +2,6 @@ # copyright : Copyright (c) 2014-2016 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -d = function/ test/script/ +d = function/ lexer/ test/script/ ./: $d include $d diff --git a/unit-tests/lexer/buildfile b/unit-tests/lexer/buildfile new file mode 100644 index 0000000..d9bd2df --- /dev/null +++ b/unit-tests/lexer/buildfile @@ -0,0 +1,13 @@ +# file : unit-tests/lexer/buildfile +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +#@@ Temporary until we get utility library support. +# +import libs = libbutl%lib{butl} +src = token lexer diagnostics utility variable name b-options types-parsers + +exe{driver}: cxx{driver} ../../build2/cxx{$src} $libs \ +test{comment quoting} + +include ../../build2/ diff --git a/unit-tests/lexer/comment.test b/unit-tests/lexer/comment.test new file mode 100644 index 0000000..07d7ac5 --- /dev/null +++ b/unit-tests/lexer/comment.test @@ -0,0 +1,112 @@ +# file : unit-tests/lexer/comment.test +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Single-line comments. + +$* <>:EOO # single-only +# comment +EOI +EOO + +$* <>EOO # single-first +# comment +foo +EOI +'foo' + +EOO + +$* <>EOO # single-last +foo +# comment +EOI +'foo' + +EOO + +$* <>EOO # single-few +foo +# comment +# comment +EOI +'foo' + +EOO + +$* <>EOO # single-cont +foo +# comment\\ +bar +EOI +'foo' + +'bar' + +EOO + +$* <>EOO # single-same +foo # comment +bar # comment +EOI +'foo' + +'bar' + +EOO + +# Multi-line comments. +# + +$* <>:EOO # multi-only +#\\ +comment +comment +#\\ +EOI +EOO + +$* <>:EOO # multi-empty +#\\ +#\\ +EOI +EOO + +$* <>EOO # multi-start-same +foo #\\ +comment +comment +#\\ +EOI +'foo' + +EOO + +$* <>EOO # multi-end-same +#\\ +comment +comment +foo #\\ +bar +EOI +'bar' + +EOO + +$* <>EOO # multi-end-not +#\\ +comment +#\\ not an end +foo #\\ +bar +EOI +'bar' + +EOO + +$* <>EOE != 0 # multi-unterm +#\\ +comment +EOI +stdin:3:1: error: unterminated multi-line comment +EOE diff --git a/unit-tests/lexer/driver.cxx b/unit-tests/lexer/driver.cxx new file mode 100644 index 0000000..326ac8a --- /dev/null +++ b/unit-tests/lexer/driver.cxx @@ -0,0 +1,94 @@ +// file : unit-tests/lexer/driver.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include +#include + +#include +#include + +#include +#include + +using namespace std; + +namespace build2 +{ + // Usage: argv[0] [-q] [] + // + int + main (int argc, char* argv[]) + { + bool quote (false); + lexer_mode m (lexer_mode::normal); + + for (int i (1); i != argc; ++i) + { + string a (argv[i]); + + if (a == "-q") + quote = true; + else + { + if (a == "normal") m = lexer_mode::normal; + else if (a == "variable") m = lexer_mode::variable; + else if (a == "value") m = lexer_mode::value; + else if (a == "attribute") m = lexer_mode::attribute; + else if (a == "eval") m = lexer_mode::eval; + else assert (false); + break; + } + } + + try + { + cin.exceptions (istream::failbit | istream::badbit); + + // Most alternative modes auto-expire so we need something underneath. + // + lexer l (cin, path ("stdin")); + + if (m != lexer_mode::normal) + l.mode (m); + + // No use printing eos since we will either get it or loop forever. + // + for (token t (l.next ()); t.type != token_type::eos; t = l.next ()) + { + // Print each token on a separate line without quoting operators. + // + t.printer (cout, t, false); + + if (quote) + { + char q ('\0'); + switch (t.qtype) + { + case quote_type::single: q = 'S'; break; + case quote_type::double_: q = 'D'; break; + case quote_type::mixed: q = 'M'; break; + case quote_type::unquoted: break; + } + + if (q != '\0') + cout << " [" << q << (t.qcomp ? "/C" : "/P") << ']'; + } + + cout << endl; + } + } + catch (const failed&) + { + return 1; + } + + return 0; + } +} + +int +main (int argc, char* argv[]) +{ + return build2::main (argc, argv); +} diff --git a/unit-tests/lexer/quoting.test b/unit-tests/lexer/quoting.test new file mode 100644 index 0000000..76fd904 --- /dev/null +++ b/unit-tests/lexer/quoting.test @@ -0,0 +1,95 @@ +# file : unit-tests/lexer/quoting.test +# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +test.options += -q + +: unquoted +: +$* <'foo' >>EOO +'foo' + +EOO + +: single-comp +: +$* <":'foo':" >>EOO +: +'foo' [S/C] +: + +EOO + +: double-comp +: +$* <':"foo":' >>EOO +: +'foo' [D/C] +: + +EOO + +: single-empty-comp +: +$* <"''" >>EOO +'' [S/C] + +EOO + +: double-empty-comp +: +$* <'""' >>EOO +'' [D/C] + +EOO + +: part-start-quoted +: Token start already quoted +: +$* <'"$foo"' >>EOO +'' [D/P] +\$ +'foo' [D/P] + +EOO + +: part-end-quoted +: Token end still quoted +: +$* <'"foo$"' >>EOO +'foo' [D/P] +\$ +'' [D/P] + +EOO + +: part-start-unquoted +: Token starts with unquoted character +: +$* <'f"oo"' >>EOO +'foo' [D/P] + +EOO + +: part-unquoted +: Token continous with unquoted character +: +$* <'"fo"o' >>EOO +'foo' [D/P] + +EOO + +: part-unquoted-escape +: Token continous with unquoted escaped character +: +$* <'"fo"\"' >>EOO +'fo"' [D/P] + +EOO + +: mixed +: +$* <"\"fo\"'o'" >>EOO +'foo' [M/P] + +EOO diff --git a/unit-tests/test/script/lexer/comment.test b/unit-tests/test/script/lexer/comment.test deleted file mode 100644 index 0092ed9..0000000 --- a/unit-tests/test/script/lexer/comment.test +++ /dev/null @@ -1,113 +0,0 @@ -# @@ This one should be moved to build2/lexer since we use base lexer -# functionality as is. -# -test.arguments += script-line - -# Single-line comments. - -$* <>:EOO # single-only -# comment -EOI -EOO - -$* <>EOO # single-first -# comment -foo -EOI -'foo' - -EOO - -$* <>EOO # single-last -foo -# comment -EOI -'foo' - -EOO - -$* <>EOO # single-few -foo -# comment -# comment -EOI -'foo' - -EOO - -$* <>EOO # single-cont -foo -# comment\\ -bar -EOI -'foo' - -'bar' - -EOO - -$* <>EOO # single-same -foo # comment -bar # comment -EOI -'foo' - -'bar' - -EOO - -# Multi-line comments. -# - -$* <>:EOO # multi-only -#\\ -comment -comment -#\\ -EOI -EOO - -$* <>:EOO # multi-empty -#\\ -#\\ -EOI -EOO - -$* <>EOO # multi-start-same -foo #\\ -comment -comment -#\\ -EOI -'foo' - -EOO - -$* <>EOO # multi-end-same -#\\ -comment -comment -foo #\\ -bar -EOI -'bar' - -EOO - -$* <>EOO # multi-end-not -#\\ -comment -#\\ not an end -foo #\\ -bar -EOI -'bar' - -EOO - -$* <>EOE != 0 # multi-unterm -#\\ -comment -EOI -stdin:3:1: error: unterminated multi-line comment -EOE -- cgit v1.1