Add support for token pre-processing in lexer

2 files changed, 21 insertions, 2 deletions

diff --git a/build/lexer b/build/lexer
index 13f28cb..6880d39 100644
--- a/build/lexer
+++ b/build/lexer
@@ -40,8 +40,13 @@ namespace build
   class lexer: protected butl::char_scanner
   {
   public:
-    lexer (std::istream& is, const std::string& name)
-        : char_scanner (is), fail (name) {mode_.push (lexer_mode::normal);}
+    lexer (std::istream& is,
+           const std::string& name,
+           void (*processor) (token&, const lexer&) = nullptr)
+        : char_scanner (is), fail (name), processor_ (processor)
+    {
+      mode_.push (lexer_mode::normal);
+    }
 
     const std::string&
     name () const {return fail.name_;}
@@ -74,6 +79,9 @@ namespace build
 
   private:
     token
+    next_impl ();
+
+    token
     next_eval ();
 
     token
@@ -109,6 +117,8 @@ namespace build
   private:
     fail_mark fail;
 
+    void (*processor_) (token&, const lexer&);
+
     std::stack<lexer_mode> mode_;
     char pair_separator_;
   };
diff --git a/build/lexer.cxx b/build/lexer.cxx
index b680234..79611c6 100644
--- a/build/lexer.cxx
+++ b/build/lexer.cxx
@@ -11,6 +11,15 @@ namespace build
   token lexer::
   next ()
   {
+    token t (next_impl ());
+    if (processor_ != nullptr)
+      processor_ (t, *this);
+    return t;
+  }
+
+  token lexer::
+  next_impl ()
+  {
     lexer_mode m (mode_.top ());
 
     // For some modes we have dedicated imlementations of next().