aboutsummaryrefslogtreecommitdiff
path: root/unit-tests/lexer
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2016-11-25 11:18:34 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2016-11-25 11:18:34 +0200
commit28f8338ded34f160e0083da9be4679bc778be7ca (patch)
tree7bd01311683d835f946c73d7d8220f552bae718f /unit-tests/lexer
parentf32bb0aceb00cfa4bd04eea72f8fa2fe02b738b3 (diff)
Distinguish token quoting type and completeness
Diffstat (limited to 'unit-tests/lexer')
-rw-r--r--unit-tests/lexer/buildfile13
-rw-r--r--unit-tests/lexer/comment.test112
-rw-r--r--unit-tests/lexer/driver.cxx94
-rw-r--r--unit-tests/lexer/quoting.test95
4 files changed, 314 insertions, 0 deletions
diff --git a/unit-tests/lexer/buildfile b/unit-tests/lexer/buildfile
new file mode 100644
index 0000000..d9bd2df
--- /dev/null
+++ b/unit-tests/lexer/buildfile
@@ -0,0 +1,13 @@
+# file : unit-tests/lexer/buildfile
+# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+#@@ Temporary until we get utility library support.
+#
+import libs = libbutl%lib{butl}
+src = token lexer diagnostics utility variable name b-options types-parsers
+
+exe{driver}: cxx{driver} ../../build2/cxx{$src} $libs \
+test{comment quoting}
+
+include ../../build2/
diff --git a/unit-tests/lexer/comment.test b/unit-tests/lexer/comment.test
new file mode 100644
index 0000000..07d7ac5
--- /dev/null
+++ b/unit-tests/lexer/comment.test
@@ -0,0 +1,112 @@
+# file : unit-tests/lexer/comment.test
+# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Single-line comments.
+
+$* <<EOI >>:EOO # single-only
+# comment
+EOI
+EOO
+
+$* <<EOI >>EOO # single-first
+# comment
+foo
+EOI
+'foo'
+<newline>
+EOO
+
+$* <<EOI >>EOO # single-last
+foo
+# comment
+EOI
+'foo'
+<newline>
+EOO
+
+$* <<EOI >>EOO # single-few
+foo
+# comment
+# comment
+EOI
+'foo'
+<newline>
+EOO
+
+$* <<EOI >>EOO # single-cont
+foo
+# comment\\
+bar
+EOI
+'foo'
+<newline>
+'bar'
+<newline>
+EOO
+
+$* <<EOI >>EOO # single-same
+foo # comment
+bar # comment
+EOI
+'foo'
+<newline>
+'bar'
+<newline>
+EOO
+
+# Multi-line comments.
+#
+
+$* <<EOI >>:EOO # multi-only
+#\\
+comment
+comment
+#\\
+EOI
+EOO
+
+$* <<EOI >>:EOO # multi-empty
+#\\
+#\\
+EOI
+EOO
+
+$* <<EOI >>EOO # multi-start-same
+foo #\\
+comment
+comment
+#\\
+EOI
+'foo'
+<newline>
+EOO
+
+$* <<EOI >>EOO # multi-end-same
+#\\
+comment
+comment
+foo #\\
+bar
+EOI
+'bar'
+<newline>
+EOO
+
+$* <<EOI >>EOO # multi-end-not
+#\\
+comment
+#\\ not an end
+foo #\\
+bar
+EOI
+'bar'
+<newline>
+EOO
+
+$* <<EOI 2>>EOE != 0 # multi-unterm
+#\\
+comment
+EOI
+stdin:3:1: error: unterminated multi-line comment
+EOE
diff --git a/unit-tests/lexer/driver.cxx b/unit-tests/lexer/driver.cxx
new file mode 100644
index 0000000..326ac8a
--- /dev/null
+++ b/unit-tests/lexer/driver.cxx
@@ -0,0 +1,94 @@
+// file : unit-tests/lexer/driver.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <cassert>
+#include <iostream>
+
+#include <build2/types>
+#include <build2/utility>
+
+#include <build2/token>
+#include <build2/lexer>
+
+using namespace std;
+
+namespace build2
+{
+ // Usage: argv[0] [-q] [<lexer-mode>]
+ //
+ int
+ main (int argc, char* argv[])
+ {
+ bool quote (false);
+ lexer_mode m (lexer_mode::normal);
+
+ for (int i (1); i != argc; ++i)
+ {
+ string a (argv[i]);
+
+ if (a == "-q")
+ quote = true;
+ else
+ {
+ if (a == "normal") m = lexer_mode::normal;
+ else if (a == "variable") m = lexer_mode::variable;
+ else if (a == "value") m = lexer_mode::value;
+ else if (a == "attribute") m = lexer_mode::attribute;
+ else if (a == "eval") m = lexer_mode::eval;
+ else assert (false);
+ break;
+ }
+ }
+
+ try
+ {
+ cin.exceptions (istream::failbit | istream::badbit);
+
+ // Most alternative modes auto-expire so we need something underneath.
+ //
+ lexer l (cin, path ("stdin"));
+
+ if (m != lexer_mode::normal)
+ l.mode (m);
+
+ // No use printing eos since we will either get it or loop forever.
+ //
+ for (token t (l.next ()); t.type != token_type::eos; t = l.next ())
+ {
+ // Print each token on a separate line without quoting operators.
+ //
+ t.printer (cout, t, false);
+
+ if (quote)
+ {
+ char q ('\0');
+ switch (t.qtype)
+ {
+ case quote_type::single: q = 'S'; break;
+ case quote_type::double_: q = 'D'; break;
+ case quote_type::mixed: q = 'M'; break;
+ case quote_type::unquoted: break;
+ }
+
+ if (q != '\0')
+ cout << " [" << q << (t.qcomp ? "/C" : "/P") << ']';
+ }
+
+ cout << endl;
+ }
+ }
+ catch (const failed&)
+ {
+ return 1;
+ }
+
+ return 0;
+ }
+}
+
+int
+main (int argc, char* argv[])
+{
+ return build2::main (argc, argv);
+}
diff --git a/unit-tests/lexer/quoting.test b/unit-tests/lexer/quoting.test
new file mode 100644
index 0000000..76fd904
--- /dev/null
+++ b/unit-tests/lexer/quoting.test
@@ -0,0 +1,95 @@
+# file : unit-tests/lexer/quoting.test
+# copyright : Copyright (c) 2014-2016 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+test.options += -q
+
+: unquoted
+:
+$* <'foo' >>EOO
+'foo'
+<newline>
+EOO
+
+: single-comp
+:
+$* <":'foo':" >>EOO
+:
+'foo' [S/C]
+:
+<newline>
+EOO
+
+: double-comp
+:
+$* <':"foo":' >>EOO
+:
+'foo' [D/C]
+:
+<newline>
+EOO
+
+: single-empty-comp
+:
+$* <"''" >>EOO
+'' [S/C]
+<newline>
+EOO
+
+: double-empty-comp
+:
+$* <'""' >>EOO
+'' [D/C]
+<newline>
+EOO
+
+: part-start-quoted
+: Token start already quoted
+:
+$* <'"$foo"' >>EOO
+'' [D/P]
+\$
+'foo' [D/P]
+<newline>
+EOO
+
+: part-end-quoted
+: Token end still quoted
+:
+$* <'"foo$"' >>EOO
+'foo' [D/P]
+\$
+'' [D/P]
+<newline>
+EOO
+
+: part-start-unquoted
+: Token starts with unquoted character
+:
+$* <'f"oo"' >>EOO
+'foo' [D/P]
+<newline>
+EOO
+
+: part-unquoted
+: Token continous with unquoted character
+:
+$* <'"fo"o' >>EOO
+'foo' [D/P]
+<newline>
+EOO
+
+: part-unquoted-escape
+: Token continous with unquoted escaped character
+:
+$* <'"fo"\"' >>EOO
+'fo"' [D/P]
+<newline>
+EOO
+
+: mixed
+:
+$* <"\"fo\"'o'" >>EOO
+'foo' [M/P]
+<newline>
+EOO