Implement in module

Given test.in containing something along these lines:

foo = $foo$

Now we can do:

using in

file{test}: in{test.in}
file{test}: foo = FOO

The alternative variable substitution symbol can be specified with the
in.symbol variable and lax (instead of the default strict) mode with
in.substitution. For example:

file{test}: in.symbol = '@'
file{test}: in.substitution = lax

6 files changed, 729 insertions, 0 deletions

diff --git a/build2/in/init.cxx b/build2/in/init.cxx
new file mode 100644
index 0000000..337722a
--- /dev/null
+++ b/build2/in/init.cxx
@@ -0,0 +1,110 @@
+// file      : build2/in/init.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <build2/in/init.hxx>
+
+#include <build2/scope.hxx>
+#include <build2/context.hxx>
+#include <build2/variable.hxx>
+#include <build2/diagnostics.hxx>
+
+#include <build2/in/rule.hxx>
+#include <build2/in/target.hxx>
+
+using namespace std;
+
+namespace build2
+{
+  namespace in
+  {
+    static const rule rule_;
+
+    bool
+    base_init (scope& rs,
+               scope&,
+               const location&,
+               unique_ptr<module_base>&,
+               bool first,
+               bool,
+               const variable_map&)
+    {
+      tracer trace ("in::base_init");
+      l5 ([&]{trace << "for " << rs.out_path ();});
+
+      assert (first);
+
+      // Enter variables.
+      //
+      {
+        auto& vp (var_pool.rw (rs));
+
+        // Alternative variable substitution symbol with '$' being the
+        // default.
+        //
+        vp.insert<string> ("in.symbol");
+
+        // Substitution mode. Valid values are 'strict' (default) and 'lax'.
+        // In the strict mode every substitution symbol is expected to start a
+        // substitution with the double symbol (e.g., $$) serving as an escape
+        // sequence.
+        //
+        // In the lax mode a pair of substitution symbols is only treated as a
+        // substitution if what's between them looks like a build2 variable
+        // name (i.e., doesn't contain spaces, etc). Everything else,
+        // including unterminated substitution symbols, is copied as is. Note
+        // also that in this mode the double symbol is not treated as an
+        // escape sequence.
+        //
+        // The lax mode is mostly useful when trying to reuse existing .in
+        // files, for example, from autoconf. Note, however, that the lax mode
+        // is still stricter than the autoconf's semantics which also leaves
+        // unknown substitutions as is.
+        //
+        vp.insert<string> ("in.substitution");
+      }
+
+      // Register target types.
+      //
+      rs.target_types.insert<in> ();
+
+      return true;
+    }
+
+    bool
+    init (scope& rs,
+          scope& bs,
+          const location& loc,
+          unique_ptr<module_base>&,
+          bool,
+          bool,
+          const variable_map&)
+    {
+      tracer trace ("in::init");
+      l5 ([&]{trace << "for " << bs.out_path ();});
+
+      // Load in.base.
+      //
+      if (!cast_false<bool> (rs["in.base.loaded"]))
+        load_module (rs, rs, "in.base", loc);
+
+      // Register rules.
+      //
+      {
+        auto& r (rs.rules);
+
+        // There are rules that are "derived" from this generic in rule in
+        // order to provide extended preprocessing functionality (see the
+        // version module for an example). To make sure they are tried first
+        // we register for path_target, not file, but in rule::match() we only
+        // match if the target is a file. A bit of a hack.
+        //
+        r.insert<path_target> (perform_update_id,   "in", rule_);
+        r.insert<path_target> (perform_clean_id,    "in", rule_);
+        r.insert<path_target> (configure_update_id, "in", rule_);
+      }
+
+      return true;
+    }
+  }
+}
diff --git a/build2/in/init.hxx b/build2/in/init.hxx
new file mode 100644
index 0000000..82879fb
--- /dev/null
+++ b/build2/in/init.hxx
@@ -0,0 +1,37 @@
+// file      : build2/in/init.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD2_IN_INIT_HXX
+#define BUILD2_IN_INIT_HXX
+
+#include <build2/types.hxx>
+#include <build2/utility.hxx>
+
+#include <build2/module.hxx>
+
+namespace build2
+{
+  namespace in
+  {
+    bool
+    base_init (scope&,
+               scope&,
+               const location&,
+               unique_ptr<module_base>&,
+               bool,
+               bool,
+               const variable_map&);
+
+    bool
+    init (scope&,
+          scope&,
+          const location&,
+          unique_ptr<module_base>&,
+          bool,
+          bool,
+          const variable_map&);
+  }
+}
+
+#endif // BUILD2_IN_INIT_HXX
diff --git a/build2/in/rule.cxx b/build2/in/rule.cxx
new file mode 100644
index 0000000..5988566
--- /dev/null
+++ b/build2/in/rule.cxx
@@ -0,0 +1,427 @@
+// file      : build2/in/rule.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <build2/in/rule.hxx>
+
+#include <cstdlib> // strtoull()
+
+#include <build2/depdb.hxx>
+#include <build2/scope.hxx>
+#include <build2/target.hxx>
+#include <build2/function.hxx>
+#include <build2/algorithm.hxx>
+#include <build2/filesystem.hxx>
+#include <build2/diagnostics.hxx>
+
+#include <build2/in/target.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace build2
+{
+  namespace in
+  {
+    bool rule::
+    match (action a, target& xt, const string&) const
+    {
+      tracer trace ("in::rule::match");
+
+      if (!xt.is_a<file> ()) // See module init() for details.
+        return false;
+
+      file& t (static_cast<file&> (xt));
+
+      bool fi (false); // Found in.
+      for (prerequisite_member p: group_prerequisite_members (a, t))
+      {
+        if (include (a, t, p) != include_type::normal) // Excluded/ad hoc.
+          continue;
+
+        fi = fi || p.is_a<in> ();
+      }
+
+      if (!fi)
+        l4 ([&]{trace << "no in file prerequisite for target " << t;});
+
+      return fi;
+    }
+
+    recipe rule::
+    apply (action a, target& xt) const
+    {
+      file& t (static_cast<file&> (xt));
+
+      // Derive the file name.
+      //
+      t.derive_path ();
+
+      // Inject dependency on the output directory.
+      //
+      inject_fsdir (a, t);
+
+      // Match prerequisite members.
+      //
+      match_prerequisite_members (a, t);
+
+      switch (a)
+      {
+      case perform_update_id: return [this] (action a, const target& t)
+        {
+          return perform_update (a, t);
+        };
+      case perform_clean_id: return &perform_clean_depdb; // Standard clean.
+      default:               return noop_recipe;          // Configure update.
+      }
+    }
+
+    string rule::
+    lookup (const location& l, const target& t, const string& n) const
+    {
+      if (auto x = t[n])
+      {
+        value v (*x);
+
+        // For typed values call string() for conversion.
+        //
+        try
+        {
+          return convert<string> (
+            v.type == nullptr
+            ? move (v)
+            : functions.call (&t.base_scope (),
+                              "string",
+                              vector_view<value> (&v, 1),
+                              l));
+        }
+        catch (const invalid_argument& e)
+        {
+          fail (l) << e <<
+            info << "while substituting '" << n << "'" << endf;
+        }
+      }
+      else
+        fail (l) << "undefined variable '" << n << "'" << endf;
+    }
+
+    optional<string> rule::
+    substitute (const location& l,
+                const target& t,
+                const string& n,
+                bool strict) const
+    {
+      // In the lax mode scan the fragment to make sure it is a variable name
+      // (that is, it can be expanded in a buildfile as just $<name>; see
+      // lexer's variable mode for details).
+      //
+      if (!strict)
+      {
+        for (size_t i (0), e (n.size ()); i != e; )
+        {
+          bool f (i == 0); // First.
+          char c (n[i++]);
+          bool l (i == e); // Last.
+
+          if (c == '_' || (f ? alpha (c) : alnum (c)))
+            continue;
+
+          if (c == '.' && !l)
+            continue;
+
+          return nullopt; // Ignore this substitution.
+        }
+      }
+
+      return lookup (l, t, n);
+    }
+
+    target_state rule::
+    perform_update (action a, const target& xt) const
+    {
+      tracer trace ("in::rule::perform_update");
+
+      const file& t (xt.as<const file&> ());
+      const path& tp (t.path ());
+
+      // Substitution symbol.
+      //
+      char sym ('$');
+      if (const string* s = cast_null<string> (t["in.symbol"]))
+      {
+        if (s->size () == 1)
+          sym = s->front ();
+        else
+          fail << "invalid substitution symbol '" << *s << "'";
+      }
+
+      // Substitution mode.
+      //
+      bool strict (true);
+      if (const string* s = cast_null<string> (t["in.substitution"]))
+      {
+        if (*s == "lax")
+          strict = false;
+        else if (*s != "strict")
+          fail << "invalid substitution mode '" << *s << "'";
+      }
+
+      // Determine if anything needs to be updated.
+      //
+      timestamp mt (t.load_mtime ());
+      auto pr (execute_prerequisites<in> (a, t, mt));
+
+      bool update (!pr.first);
+      target_state ts (update ? target_state::changed : *pr.first);
+
+      const in& i (pr.second);
+      const path& ip (i.path ());
+
+      // We use depdb to track changes to the .in file name, symbol/mode, and
+      // variable values that have been substituted.
+      //
+      depdb dd (tp + ".d");
+
+      // First should come the rule name/version.
+      //
+      if (dd.expect ("in 1") != nullptr)
+        l4 ([&]{trace << "rule mismatch forcing update of " << t;});
+
+      // Then the substitution symbol.
+      //
+      if (dd.expect (string (1, sym)) != nullptr)
+        l4 ([&]{trace << "substitution symbol mismatch forcing update of"
+                      << t;});
+
+      // Then the substitution mode.
+      //
+      if (dd.expect (strict ? "strict" : "lax") != nullptr)
+        l4 ([&]{trace << "substitution mode mismatch forcing update of"
+                      << t;});
+
+      // Then the .in file.
+      //
+      if (dd.expect (i.path ()) != nullptr)
+        l4 ([&]{trace << "in file mismatch forcing update of " << t;});
+
+      // Update if any mismatch or depdb is newer that the output.
+      //
+      if (dd.writing () || dd.mtime () > mt)
+        update = true;
+
+      // Substituted variable values.
+      //
+      // The plan is to save each substituted variable name and the hash of
+      // its value one entry per line. Plus the line location of its expansion
+      // for diagnostics.
+      //
+      // If update is true (i.e., the .in file has changes), then we simply
+      // overwrite the whole list.
+      //
+      // If update is false, then we need to read each name/hash, query and
+      // hash its current value, and compare. If hashes differ, then we need
+      // to start overwriting from this variable (the prefix of variables
+      // couldn't have changed since the .in file hasn't changed).
+      //
+      // Note that if the .in file substitutes the same variable multiple
+      // times, then we will end up with multiple entries for such a variable.
+      // For now we assume this is ok since this is probably not very common
+      // and it makes the overall logic simpler.
+      //
+      size_t dd_skip (0); // Number of "good" variable lines.
+
+      if (update)
+      {
+        // If we are still reading, mark the next line for overwriting.
+        //
+        if (dd.reading ())
+        {
+          dd.read ();  // Read the first variable line, if any.
+          dd.write (); // Mark it for overwriting.
+        }
+      }
+      else
+      {
+        while (dd.more ())
+        {
+          if (string* s = dd.read ())
+          {
+            // The line format is:
+            //
+            // <ln> <name> <hash>
+            //
+            char* e (nullptr);
+            uint64_t ln (strtoull (s->c_str (), &e, 10));
+
+            size_t p1 (*e == ' ' ? e - s->c_str () : string::npos);
+            size_t p2 (s->rfind (' '));
+
+            if (p1 != string::npos && p2 != string::npos && p2 - p1 > 1)
+            {
+              string n (*s, p1 + 1, p2 - p1 - 1);
+              string v (lookup (location (&ip, ln), t, n));
+
+              if (s->compare (p2 + 1, string::npos, sha256 (v).string ()) == 0)
+              {
+                dd_skip++;
+                continue;
+              }
+              else
+                l4 ([&]{trace << n << " variable value mismatch forcing "
+                              << "update of " << t;});
+              // Fall through.
+            }
+
+            dd.write (); // Mark this line for overwriting.
+
+            // Fall through.
+          }
+
+          break;
+        }
+      }
+
+      if (dd.writing ()) // Recheck.
+        update = true;
+
+      // If nothing changed, then we are done.
+      //
+      if (!update)
+      {
+        dd.close ();
+        return ts;
+      }
+
+      if (verb >= 2)
+        text << "in " << ip << " >" << tp;
+      else if (verb)
+        text << "in " << ip;
+
+      // Read and process the file, one line at a time.
+      //
+      const char* what;
+      const path* whom;
+      try
+      {
+        what = "open"; whom = &ip;
+        ifdstream ifs (ip, fdopen_mode::in, ifdstream::badbit);
+
+        what = "open"; whom = &tp;
+        ofdstream ofs (tp);
+        auto_rmfile arm (tp);
+
+        string s; // Reuse the buffer.
+        for (size_t ln (1);; ++ln)
+        {
+          what = "read"; whom = &ip;
+          if (!getline (ifs, s))
+            break; // Could not read anything, not even newline.
+
+          // Not tracking column for now (see also depdb above).
+          //
+          const location l (&ip, ln);
+
+          // Scan the line looking for substiutions in the $<name>$ form. In
+          // the strict mode treat $$ as an escape sequence.
+          //
+          for (size_t b (0), n, d; b != (n = s.size ()); b += d)
+          {
+            d = 1;
+
+            if (s[b] != sym)
+              continue;
+
+            // Note that in the lax mode these should still be substitutions:
+            //
+            // @project@@
+            // @@project@
+
+            // Find the other end.
+            //
+            size_t e (b + 1);
+            for (; e != (n = s.size ()); ++e)
+            {
+              if (s[e] == sym)
+              {
+                if (strict && e + 1 != n && s[e + 1] == sym) // Escape.
+                  s.erase (e, 1); // Keep one, erase the other.
+                else
+                  break;
+              }
+            }
+
+            if (e == n)
+            {
+              if (strict)
+                fail (l) << "unterminated '" << sym << "'" << endf;
+
+              break;
+            }
+
+            if (e - b == 1) // Escape (or just double symbol in the lax mode).
+            {
+              if (strict)
+                s.erase (b, 1); // Keep one, erase the other.
+
+              continue;
+            }
+
+            // We have a (potential, in the lax mode) substition with b
+            // pointing to the opening symbol and e -- to the closing.
+            //
+            string name (s, b + 1, e - b -1);
+            if (optional<string> val = substitute (l, t, name, strict))
+            {
+              // Save in depdb.
+              //
+              if (dd_skip == 0)
+              {
+                // The line format is:
+                //
+                // <ln> <name> <hash>
+                //
+                string s (to_string (ln));
+                s += ' ';
+                s += name;
+                s += ' ';
+                s += sha256 (*val).string ();
+                dd.write (s);
+              }
+              else
+                --dd_skip;
+
+              // Patch the result in and adjust the delta.
+              //
+              s.replace (b, e - b + 1, *val);
+              d = val->size ();
+            }
+            else
+              d = e - b + 1; // Ignore this substitution.
+          }
+
+          what = "write"; whom = &tp;
+          ofs << s << endl;
+        }
+
+        // Close depdb before closing the output file so its mtime is not
+        // newer than of the output.
+        //
+        dd.close ();
+
+        what = "close"; whom = &tp;
+        ofs.close ();
+        arm.cancel ();
+
+        what = "close"; whom = &ip;
+        ifs.close ();
+      }
+      catch (const io_error& e)
+      {
+        fail << "unable to " << what << ' ' << *whom << ": " << e;
+      }
+
+      t.mtime (system_clock::now ());
+      return target_state::changed;
+    }
+  }
+}
diff --git a/build2/in/rule.hxx b/build2/in/rule.hxx
new file mode 100644
index 0000000..e588fd9f
--- /dev/null
+++ b/build2/in/rule.hxx
@@ -0,0 +1,50 @@
+// file      : build2/in/rule.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD2_IN_RULE_HXX
+#define BUILD2_IN_RULE_HXX
+
+#include <build2/types.hxx>
+#include <build2/utility.hxx>
+
+#include <build2/rule.hxx>
+
+namespace build2
+{
+  namespace in
+  {
+    // Preprocess an .in file.
+    //
+    class rule: public build2::rule
+    {
+    public:
+      rule () {}
+
+      virtual bool
+      match (action, target&, const string&) const override;
+
+      virtual recipe
+      apply (action, target&) const override;
+
+      // Perform variable lookup.
+      //
+      virtual string
+      lookup (const location&, const target&, const string& name) const;
+
+      // Perform variable substitution. Return nullopt if it should be
+      // ignored.
+      //
+      virtual optional<string>
+      substitute (const location&,
+                  const target&,
+                  const string& name,
+                  bool strict) const;
+
+      target_state
+      perform_update (action, const target&) const;
+    };
+  }
+}
+
+#endif // BUILD2_IN_RULE_HXX
diff --git a/build2/in/target.cxx b/build2/in/target.cxx
new file mode 100644
index 0000000..ecf975f
--- /dev/null
+++ b/build2/in/target.cxx
@@ -0,0 +1,59 @@
+// file      : build2/in/target.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <build2/in/target.hxx>
+
+using namespace std;
+
+namespace build2
+{
+  namespace in
+  {
+    // in
+    //
+    static const target*
+    in_search (const target& xt, const prerequisite_key& cpk)
+    {
+      // If we have no extension then derive it from our target. Then delegate
+      // to file_search().
+      //
+      prerequisite_key pk (cpk);
+      optional<string>& e (pk.tk.ext);
+
+      if (!e)
+      {
+        if (const file* t = xt.is_a<file> ())
+        {
+          const string& te (t->derive_extension ());
+          e = te + (te.empty () ? "" : ".") + "in";
+        }
+        else
+          fail << "prerequisite " << pk << " for a non-file target " << xt;
+      }
+
+      return file_search (xt, pk);
+    }
+
+    static bool
+    in_pattern (const target_type&, const scope&, string&, bool)
+    {
+      fail << "pattern in in{} prerequisite" << endf;
+    }
+
+    extern const char in_ext_def[] = ""; // No extension by default.
+
+    const target_type in::static_type
+    {
+      "in",
+      &file::static_type,
+      &target_factory<in>,
+      &target_extension_fix<in_ext_def>,
+      nullptr, /* default_extension */   // Taken care of by search.
+      &in_pattern,
+      &target_print_1_ext_verb,          // Same as file.
+      &in_search,
+      false
+    };
+  }
+}
diff --git a/build2/in/target.hxx b/build2/in/target.hxx
new file mode 100644
index 0000000..90def2d
--- /dev/null
+++ b/build2/in/target.hxx
@@ -0,0 +1,46 @@
+// file      : build2/in/target.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD2_IN_TARGET_HXX
+#define BUILD2_IN_TARGET_HXX
+
+#include <build2/types.hxx>
+#include <build2/utility.hxx>
+
+#include <build2/target.hxx>
+
+namespace build2
+{
+  namespace in
+  {
+    // This is the venerable .in ("input") file that needs some kind of
+    // preprocessing.
+    //
+    // One interesting aspect of this target type is that the prerequisite
+    // search is target-dependent. Consider:
+    //
+    // hxx{version}: in{version.hxx} // version.hxx.in -> version.hxx
+    //
+    // Having to specify the header extension explicitly is inelegant. Instead
+    // what we really want to write is this:
+    //
+    // hxx{version}: in{version}
+    //
+    // But how do we know that in{version} means version.hxx.in? That's where
+    // the target-dependent search comes in: we take into account the target
+    // we are a prerequisite of.
+    //
+    class in: public file
+    {
+    public:
+      using file::file;
+
+    public:
+      static const target_type static_type;
+      virtual const target_type& dynamic_type () const {return static_type;}
+    };
+  }
+}
+
+#endif // BUILD2_IN_TARGET_HXX