Add line processing customization hook to in::rule

2 files changed, 254 insertions, 140 deletions

diff --git a/libbuild2/in/rule.cxx b/libbuild2/in/rule.cxx
index 2569948..4eab287 100644
--- a/libbuild2/in/rule.cxx
+++ b/libbuild2/in/rule.cxx
@@ -209,39 +209,58 @@ namespace build2
           {
             // The line format is:
             //
-            // <ln> <name> <hash>
+            // <ln> <name> <hash>[/<flags>]
             //
             // Note that <name> can contain spaces (see the constraint check
-            // expressions in the version module).
+            // expressions in the version module). That's the reason why we
+            // use the `/` separator for <flags> instead of the more natural
+            // space.
             //
             char* e (nullptr);
             uint64_t ln (strtoull (s->c_str (), &e, 10));
 
-            size_t p1 (*e == ' ' ? e - s->c_str () : string::npos);
-            size_t p2 (s->rfind (' '));
+            size_t p1 (*e == ' ' ? e - s->c_str () : string::npos); // <name>
+            size_t p2 (s->rfind (' '));                             // <hash>
 
             if (p1 != string::npos && p2 != string::npos && p2 - p1 > 1)
             {
-              string n (*s, p1 + 1, p2 - p1 - 1);
+              ++p1;
+              string name (*s, p1, p2 - p1);
 
-              // Note that we have to call substitute(), not lookup() since it
-              // can be overriden with custom substitution semantics.
-              //
-              optional<string> v (
-                substitute (location (ip, ln), a, t, n, strict, null));
+              ++p2;
+              size_t p3 (s->find ('/', p2)); // <flags>
 
-              assert (v); // Rule semantics change without version increment?
+              optional<uint64_t> flags;
+              if (p3 != string::npos)
+              {
+                uint64_t v (strtoull (s->c_str () + p3 + 1, &e, 10));
+                if (*e == '\0')
+                  flags = v;
+              }
 
-              if (s->compare (p2 + 1,
-                              string::npos,
-                              sha256 (*v).string ()) == 0)
+              if (p3 == string::npos || flags)
               {
-                dd_skip++;
-                continue;
+                // Note that we have to call substitute(), not lookup() since
+                // it can be overriden with custom substitution semantics.
+                //
+                optional<string> v (
+                  substitute (location (ip, ln),
+                              a, t,
+                              name, flags,
+                              strict, null));
+
+                assert (v); // Rule semantics change without version increment?
+
+                if (s->compare (p2, p3, sha256 (*v).string ()) == 0)
+                {
+                  dd_skip++;
+                  continue;
+                }
+                else
+                  l4 ([&]{trace << name << " variable value mismatch forcing "
+                                << "update of " << t;});
               }
-              else
-                l4 ([&]{trace << n << " variable value mismatch forcing "
-                              << "update of " << t;});
+
               // Fall through.
             }
 
@@ -325,7 +344,7 @@ namespace build2
         );
 
         string s; // Reuse the buffer.
-        for (size_t ln (1);; ++ln)
+        for (uint64_t ln (1);; ++ln)
         {
           what = "read"; whom = &ip;
           if (!getline (ifs, s))
@@ -340,86 +359,11 @@ namespace build2
 
           // Not tracking column for now (see also depdb above).
           //
-          const location l (ip, ln);
-
-          // Scan the line looking for substiutions in the $<name>$ form. In
-          // the strict mode treat $$ as an escape sequence.
-          //
-          for (size_t b (0), n, d; b != (n = s.size ()); b += d)
-          {
-            d = 1;
-
-            if (s[b] != sym)
-              continue;
-
-            // Note that in the lax mode these should still be substitutions:
-            //
-            // @project@@
-            // @@project@
-
-            // Find the other end.
-            //
-            size_t e (b + 1);
-            for (; e != (n = s.size ()); ++e)
-            {
-              if (s[e] == sym)
-              {
-                if (strict && e + 1 != n && s[e + 1] == sym) // Escape.
-                  s.erase (e, 1); // Keep one, erase the other.
-                else
-                  break;
-              }
-            }
-
-            if (e == n)
-            {
-              if (strict)
-                fail (l) << "unterminated '" << sym << "'" << endf;
-
-              break;
-            }
-
-            if (e - b == 1) // Escape (or just double symbol in the lax mode).
-            {
-              if (strict)
-                s.erase (b, 1); // Keep one, erase the other.
-
-              continue;
-            }
-
-            // We have a (potential, in the lax mode) substition with b
-            // pointing to the opening symbol and e -- to the closing.
-            //
-            string name (s, b + 1, e - b -1);
-            if (optional<string> val =
-                  substitute (l, a, t, name, strict, null))
-            {
-              // Save in depdb.
-              //
-              if (dd_skip == 0)
-              {
-                // The line format is:
-                //
-                // <ln> <name> <hash>
-                //
-                string s (to_string (ln));
-                s += ' ';
-                s += name;
-                s += ' ';
-                s += sha256 (*val).string ();
-                dd.write (s);
-              }
-              else
-                --dd_skip;
-
-              // Patch the result in and adjust the delta.
-              //
-              s.replace (b, e - b + 1, *val);
-              d = val->size ();
-            }
-            else
-              d = e - b + 1; // Ignore this substitution.
-          }
+          process (location (ip, ln),
+                   a, t,
+                   dd, dd_skip,
+                   s, 0,
+                   (crlf ? "\r\n" : "\n"), sym, strict, null);
 
           what = "write"; whom = &tp;
           if (ln != 1)
@@ -462,55 +406,129 @@ namespace build2
       return prerequisite_target (&build2::search (t, p), i);
     }
 
-    string rule::
-    lookup (const location& loc,
-            action,
-            const target& t,
-            const string& n,
-            const optional<string>& null) const
+    void rule::
+    process (const location& l,
+             action a, const target& t,
+             depdb& dd, size_t dd_skip,
+             string& s, size_t b,
+             const char* nl,
+             char sym,
+             bool strict,
+             const optional<string>& null) const
     {
-      auto l (t[n]);
-
-      if (l.defined ())
+      // Scan the line looking for substiutions in the $<name>$ form. In the
+      // strict mode treat $$ as an escape sequence.
+      //
+      for (size_t n, d; b != (n = s.size ()); b += d)
       {
-        value v (*l);
+        d = 1;
 
-        if (v.null)
+        if (s[b] != sym)
+          continue;
+
+        // Note that in the lax mode these should still be substitutions:
+        //
+        // @project@@
+        // @@project@
+
+        // Find the other end.
+        //
+        size_t e (b + 1);
+        for (; e != (n = s.size ()); ++e)
         {
-          if (null)
-            return *null;
-          else
-            fail (loc) << "null value in variable '" << n << "'" <<
-              info << "use in.null to specify null value substiution string";
+          if (s[e] == sym)
+          {
+            if (strict && e + 1 != n && s[e + 1] == sym) // Escape.
+              s.erase (e, 1); // Keep one, erase the other.
+            else
+              break;
+          }
         }
 
-        // For typed values call string() for conversion.
+        if (e == n)
+        {
+          if (strict)
+            fail (l) << "unterminated '" << sym << "'";
+
+          break;
+        }
+
+        if (e - b == 1) // Escape (or just double symbol in the lax mode).
+        {
+          if (strict)
+            s.erase (b, 1); // Keep one, erase the other.
+
+          continue;
+        }
+
+        // We have a (potential, in the lax mode) substition with b pointing
+        // to the opening symbol and e -- to the closing.
         //
-        try
+        if (optional<string> val = substitute (l,
+                                               a, t,
+                                               dd, dd_skip,
+                                               string (s, b + 1, e - b -1),
+                                               nullopt /* flags */,
+                                               strict,
+                                               null))
         {
-          return convert<string> (
-            v.type == nullptr
-            ? move (v)
-            : t.ctx.functions.call (&t.base_scope (),
-                                    "string",
-                                    vector_view<value> (&v, 1),
-                                    loc));
+          replace_newlines (*val, nl);
+
+          // Patch the result in and adjust the delta.
+          //
+          s.replace (b, e - b + 1, *val);
+          d = val->size ();
         }
-        catch (const invalid_argument& e)
+        else
+          d = e - b + 1; // Ignore this substitution.
+      }
+    }
+
+    optional<string> rule::
+    substitute (const location& l,
+                action a, const target& t,
+                depdb& dd, size_t dd_skip,
+                const string& n,
+                optional<uint64_t> flags,
+                bool strict,
+                const optional<string>& null) const
+    {
+      optional<string> val (substitute (l, a, t, n, flags, strict, null));
+
+      if (val)
+      {
+        // Save in depdb.
+        //
+        if (dd_skip == 0)
         {
-          fail (loc) << e <<
-            info << "while substituting '" << n << "'" << endf;
+          // The line format is:
+          //
+          // <ln> <name> <hash>[/<flags>]
+          //
+          string s (to_string (l.line));
+          s += ' ';
+          s += n;
+          s += ' ';
+          s += sha256 (*val).string ();
+          if (flags)
+          {
+            s += '/';
+            s += to_string (*flags);
+          }
+          dd.write (s);
         }
+        else
+          --dd_skip;
       }
-      else
-        fail (loc) << "undefined variable '" << n << "'" << endf;
+
+      return val;
     }
 
     optional<string> rule::
     substitute (const location& l,
-                action a,
-                const target& t,
+                action a, const target& t,
                 const string& n,
+                optional<uint64_t> flags,
                 bool strict,
                 const optional<string>& null) const
     {
@@ -536,7 +554,53 @@ namespace build2
         }
       }
 
-      return lookup (l, a, t, n, null);
+      return lookup (l, a, t, n, flags, null);
+    }
+
+    string rule::
+    lookup (const location& loc,
+            action, const target& t,
+            const string& n,
+            optional<uint64_t> flags,
+            const optional<string>& null) const
+    {
+      assert (!flags);
+
+      auto l (t[n]);
+
+      if (l.defined ())
+      {
+        value v (*l);
+
+        if (v.null)
+        {
+          if (null)
+            return *null;
+          else
+            fail (loc) << "null value in variable '" << n << "'" <<
+              info << "use in.null to specify null value substiution string";
+        }
+
+        // For typed values call string() for conversion.
+        //
+        try
+        {
+          return convert<string> (
+            v.type == nullptr
+            ? move (v)
+            : t.ctx.functions.call (&t.base_scope (),
+                                    "string",
+                                    vector_view<value> (&v, 1),
+                                    loc));
+        }
+        catch (const invalid_argument& e)
+        {
+          fail (loc) << e <<
+            info << "while substituting '" << n << "'" << endf;
+        }
+      }
+      else
+        fail (loc) << "undefined variable '" << n << "'" << endf;
     }
   }
 }
diff --git a/libbuild2/in/rule.hxx b/libbuild2/in/rule.hxx
index 2fa1305..162ce78 100644
--- a/libbuild2/in/rule.hxx
+++ b/libbuild2/in/rule.hxx
@@ -50,14 +50,18 @@ namespace build2
       virtual target_state
       perform_update (action, const target&) const;
 
-      // Customization hooks.
+      // Customization hooks and helpers.
+      //
+      // Flags can be used by a custom implementation to alter the lookup
+      // semantics, for example, for special substitutions. Note, however,
+      // that one must make sure this semantics cannot change without changes
+      // to the .in file (see the depdb logic for details).
       //
 
       // Perform prerequisite search.
       //
       virtual prerequisite_target
-      search (action,
-              const target&,
+      search (action, const target&,
               const prerequisite_member&,
               include_type) const;
 
@@ -65,9 +69,9 @@ namespace build2
       //
       virtual string
       lookup (const location&,
-              action,
-              const target&,
+              action, const target&,
               const string& name,
+              optional<uint64_t> flags,
               const optional<string>& null) const;
 
       // Perform variable substitution. Return nullopt if it should be
@@ -75,12 +79,58 @@ namespace build2
       //
       virtual optional<string>
       substitute (const location&,
-                  action,
-                  const target&,
+                  action, const target&,
+                  const string& name,
+                  optional<uint64_t> flags,
+                  bool strict,
+                  const optional<string>& null) const;
+
+      // Call the above version and do any necessary depdb saving.
+      //
+      optional<string>
+      substitute (const location&,
+                  action, const target&,
+                  depdb& dd, size_t dd_skip,
                   const string& name,
+                  optional<uint64_t> flags,
                   bool strict,
                   const optional<string>& null) const;
 
+      // Process a line of input from the specified position performing any
+      // necessary substitutions.
+      //
+      virtual void
+      process (const location&,
+               action, const target&,
+               depdb& dd, size_t dd_skip,
+               string& line, size_t pos,
+               const char* newline,
+               char sym,
+               bool strict,
+               const optional<string>& null) const;
+
+      // Replace newlines in a multi-line value with the given newline
+      // sequence.
+      //
+      static void
+      replace_newlines (string& v, const char* newline)
+      {
+        for (size_t p (0), n; (p = v.find ('\n', p)) != string::npos; p += n)
+        {
+          n = 1;
+
+          // Deal with CRLF in the value.
+          //
+          if (p != 0 && v[p - 1] == '\r')
+          {
+            --p;
+            ++n;
+          }
+
+          v.replace (p, n, newline);
+        }
+      }
+
     protected:
       const string rule_id_;
       const string program_;