diff options
| -rw-r--r-- | build2/parser | 100 | ||||
| -rw-r--r-- | build2/parser.cxx | 19 | ||||
| -rw-r--r-- | build2/token | 2 |
3 files changed, 97 insertions, 24 deletions
diff --git a/build2/parser b/build2/parser index 9142cf6..89f42b8 100644 --- a/build2/parser +++ b/build2/parser @@ -281,7 +281,7 @@ namespace build2 peeked () const { assert (peeked_); - return peek_; + return peek_.token; } void @@ -289,13 +289,37 @@ namespace build2 { if (replay_ != replay::play) lexer_->mode (m, ps); + else + // As a sanity check, make sure the mode matches the next token. Note + // that we don't check the pair separator since it can be overriden by + // the lexer's mode() implementation. + // + assert (replay_i_ != replay_data_.size () && + replay_data_[replay_i_].mode == m); } lexer_mode mode () const { - assert (replay_ != replay::play); - return lexer_->mode (); + if (replay_ != replay::play) + return lexer_->mode (); + else + { + assert (replay_i_ != replay_data_.size ()); + return replay_data_[replay_i_].mode; + } + } + + char + pair_separator () const + { + if (replay_ != replay::play) + return lexer_->pair_separator (); + else + { + assert (replay_i_ != replay_data_.size ()); + return replay_data_[replay_i_].pair_separator; + } } void @@ -306,9 +330,9 @@ namespace build2 } // Token saving and replaying. Note that it can only be used in certain - // contexts. Specifically, the lexer mode should be the same and the code - // that parses a replay must not interact with the lexer directly (e.g., - // the keyword() test). For now we don't enforce any of this. + // contexts. Specifically, the code that parses a replay must not interact + // with the lexer directly (e.g., the keyword() test). For now we don't + // enforce any of this. // // Note also that the peeked token is not part of the replay, until it // is "got". @@ -338,13 +362,6 @@ namespace build2 replay_ = replay::stop; } - const token& - replay_next () - { - assert (replay_i_ != replay_data_.size ()); - return replay_data_[replay_i_++]; - } - struct replay_guard { replay_guard (parser& p, bool start = true) @@ -371,6 +388,57 @@ namespace build2 parser* p_; }; + struct replay_token + { + build2::token token; + lexer_mode mode; + char pair_separator; + }; + + using replay_tokens = vector<replay_token>; + + // Stop saving and get the data. + // + replay_tokens + replay_data () + { + assert (replay_ == replay::save); + + replay_tokens r (move (replay_data_)); + replay_ = replay::stop; + replay_data_.clear (); + + return r; + } + + // Set the data and start playing. + // + void + replay_data (replay_tokens&& d) + { + assert (replay_ == replay::stop); + + replay_data_ = move (d); + replay_ = replay::play; + } + + // Implementation details, don't call directly. + // + replay_token + lexer_next () + { + lexer_mode m (lexer_->mode ()); // Get it first since it may expire. + char ps (lexer_->pair_separator ()); + return replay_token {lexer_->next (), m, ps}; + } + + const replay_token& + replay_next () + { + assert (replay_i_ != replay_data_.size ()); + return replay_data_[replay_i_++]; + } + // Diagnostics. // protected: @@ -390,12 +458,12 @@ namespace build2 target* default_target_; names export_value_; - token peek_ = token (token_type::eos, false, 0, 0, token_printer); + replay_token peek_; bool peeked_ = false; enum class replay {stop, save, play} replay_ = replay::stop; - vector<token> replay_data_; - size_t replay_i_; // Position of the next token during replay. + replay_tokens replay_data_; + size_t replay_i_; // Position of the next token during replay. }; } diff --git a/build2/parser.cxx b/build2/parser.cxx index 24558ea..a81aec7 100644 --- a/build2/parser.cxx +++ b/build2/parser.cxx @@ -1865,8 +1865,8 @@ namespace build2 // Manually expire the value mode if we haven't reached newline/eos (where // it expires automatically). // - if (lexer_->mode () == lexer_mode::value) - lexer_->expire_mode (); + if (mode () == lexer_mode::value) + expire_mode (); if (tt != type::rsbrace) fail (t) << "expected ']' instead of " << t; @@ -2540,7 +2540,7 @@ namespace build2 fail (t) << "multiple " << what << "s on the left hand side " << "of a pair"; - ns.back ().pair = lexer_->pair_separator (); + ns.back ().pair = pair_separator (); tt = peek (); // If the next token is separated, then we have an empty RHS. Note @@ -3063,17 +3063,20 @@ namespace build2 type parser:: next (token& t, type& tt) { + replay_token r; + if (peeked_) { - t = move (peek_); + r = move (peek_); peeked_ = false; } else - t = (replay_ == replay::play ? replay_next () : lexer_->next ()); + r = replay_ != replay::play ? lexer_next () : replay_next (); if (replay_ == replay::save) - replay_data_.push_back (t); + replay_data_.push_back (r); + t = move (r.token); tt = t.type; return tt; } @@ -3083,10 +3086,10 @@ namespace build2 { if (!peeked_) { - peek_ = (replay_ == replay::play ? replay_next () : lexer_->next ()); + peek_ = (replay_ != replay::play ? lexer_next () : replay_next ()); peeked_ = true; } - return peek_.type; + return peek_.token.type; } } diff --git a/build2/token b/build2/token index d172e0d..517bf47 100644 --- a/build2/token +++ b/build2/token @@ -92,6 +92,8 @@ namespace build2 printer (&token_printer) {} }; + using tokens = vector<token>; + // Output the token value in a format suitable for diagnostics. // inline ostream& |