aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2016-10-18 13:28:35 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2016-11-04 09:26:25 +0200
commite9b052c520bbf98e060ac2ec4e1e482263038dec (patch)
tree939103dff667d5de12f6ea8abfd50210ca676031
parentfc27ec48c9d63879e4b0f049060e943233cb540d (diff)
Keep track of lexer mode in parser replay mechanism
-rw-r--r--build2/parser100
-rw-r--r--build2/parser.cxx19
-rw-r--r--build2/token2
3 files changed, 97 insertions, 24 deletions
diff --git a/build2/parser b/build2/parser
index 9142cf6..89f42b8 100644
--- a/build2/parser
+++ b/build2/parser
@@ -281,7 +281,7 @@ namespace build2
peeked () const
{
assert (peeked_);
- return peek_;
+ return peek_.token;
}
void
@@ -289,13 +289,37 @@ namespace build2
{
if (replay_ != replay::play)
lexer_->mode (m, ps);
+ else
+ // As a sanity check, make sure the mode matches the next token. Note
+ // that we don't check the pair separator since it can be overriden by
+ // the lexer's mode() implementation.
+ //
+ assert (replay_i_ != replay_data_.size () &&
+ replay_data_[replay_i_].mode == m);
}
lexer_mode
mode () const
{
- assert (replay_ != replay::play);
- return lexer_->mode ();
+ if (replay_ != replay::play)
+ return lexer_->mode ();
+ else
+ {
+ assert (replay_i_ != replay_data_.size ());
+ return replay_data_[replay_i_].mode;
+ }
+ }
+
+ char
+ pair_separator () const
+ {
+ if (replay_ != replay::play)
+ return lexer_->pair_separator ();
+ else
+ {
+ assert (replay_i_ != replay_data_.size ());
+ return replay_data_[replay_i_].pair_separator;
+ }
}
void
@@ -306,9 +330,9 @@ namespace build2
}
// Token saving and replaying. Note that it can only be used in certain
- // contexts. Specifically, the lexer mode should be the same and the code
- // that parses a replay must not interact with the lexer directly (e.g.,
- // the keyword() test). For now we don't enforce any of this.
+ // contexts. Specifically, the code that parses a replay must not interact
+ // with the lexer directly (e.g., the keyword() test). For now we don't
+ // enforce any of this.
//
// Note also that the peeked token is not part of the replay, until it
// is "got".
@@ -338,13 +362,6 @@ namespace build2
replay_ = replay::stop;
}
- const token&
- replay_next ()
- {
- assert (replay_i_ != replay_data_.size ());
- return replay_data_[replay_i_++];
- }
-
struct replay_guard
{
replay_guard (parser& p, bool start = true)
@@ -371,6 +388,57 @@ namespace build2
parser* p_;
};
+ struct replay_token
+ {
+ build2::token token;
+ lexer_mode mode;
+ char pair_separator;
+ };
+
+ using replay_tokens = vector<replay_token>;
+
+ // Stop saving and get the data.
+ //
+ replay_tokens
+ replay_data ()
+ {
+ assert (replay_ == replay::save);
+
+ replay_tokens r (move (replay_data_));
+ replay_ = replay::stop;
+ replay_data_.clear ();
+
+ return r;
+ }
+
+ // Set the data and start playing.
+ //
+ void
+ replay_data (replay_tokens&& d)
+ {
+ assert (replay_ == replay::stop);
+
+ replay_data_ = move (d);
+ replay_ = replay::play;
+ }
+
+ // Implementation details, don't call directly.
+ //
+ replay_token
+ lexer_next ()
+ {
+ lexer_mode m (lexer_->mode ()); // Get it first since it may expire.
+ char ps (lexer_->pair_separator ());
+ return replay_token {lexer_->next (), m, ps};
+ }
+
+ const replay_token&
+ replay_next ()
+ {
+ assert (replay_i_ != replay_data_.size ());
+ return replay_data_[replay_i_++];
+ }
+
// Diagnostics.
//
protected:
@@ -390,12 +458,12 @@ namespace build2
target* default_target_;
names export_value_;
- token peek_ = token (token_type::eos, false, 0, 0, token_printer);
+ replay_token peek_;
bool peeked_ = false;
enum class replay {stop, save, play} replay_ = replay::stop;
- vector<token> replay_data_;
- size_t replay_i_; // Position of the next token during replay.
+ replay_tokens replay_data_;
+ size_t replay_i_; // Position of the next token during replay.
};
}
diff --git a/build2/parser.cxx b/build2/parser.cxx
index 24558ea..a81aec7 100644
--- a/build2/parser.cxx
+++ b/build2/parser.cxx
@@ -1865,8 +1865,8 @@ namespace build2
// Manually expire the value mode if we haven't reached newline/eos (where
// it expires automatically).
//
- if (lexer_->mode () == lexer_mode::value)
- lexer_->expire_mode ();
+ if (mode () == lexer_mode::value)
+ expire_mode ();
if (tt != type::rsbrace)
fail (t) << "expected ']' instead of " << t;
@@ -2540,7 +2540,7 @@ namespace build2
fail (t) << "multiple " << what << "s on the left hand side "
<< "of a pair";
- ns.back ().pair = lexer_->pair_separator ();
+ ns.back ().pair = pair_separator ();
tt = peek ();
// If the next token is separated, then we have an empty RHS. Note
@@ -3063,17 +3063,20 @@ namespace build2
type parser::
next (token& t, type& tt)
{
+ replay_token r;
+
if (peeked_)
{
- t = move (peek_);
+ r = move (peek_);
peeked_ = false;
}
else
- t = (replay_ == replay::play ? replay_next () : lexer_->next ());
+ r = replay_ != replay::play ? lexer_next () : replay_next ();
if (replay_ == replay::save)
- replay_data_.push_back (t);
+ replay_data_.push_back (r);
+ t = move (r.token);
tt = t.type;
return tt;
}
@@ -3083,10 +3086,10 @@ namespace build2
{
if (!peeked_)
{
- peek_ = (replay_ == replay::play ? replay_next () : lexer_->next ());
+ peek_ = (replay_ != replay::play ? lexer_next () : replay_next ());
peeked_ = true;
}
- return peek_.type;
+ return peek_.token.type;
}
}
diff --git a/build2/token b/build2/token
index d172e0d..517bf47 100644
--- a/build2/token
+++ b/build2/token
@@ -92,6 +92,8 @@ namespace build2
printer (&token_printer) {}
};
+ using tokens = vector<token>;
+
// Output the token value in a format suitable for diagnostics.
//
inline ostream&