Add rule for extracting C and C++ predefs

8 files changed, 518 insertions, 30 deletions

diff --git a/libbuild2/c/init.cxx b/libbuild2/c/init.cxx
index ebf0631..cb0469c 100644
--- a/libbuild2/c/init.cxx
+++ b/libbuild2/c/init.cxx
@@ -479,11 +479,11 @@ namespace build2
 
     bool
     as_cpp_init (scope& rs,
-                scope& bs,
-                const location& loc,
-                bool,
-                bool,
-                module_init_extra&)
+                 scope& bs,
+                 const location& loc,
+                 bool,
+                 bool,
+                 module_init_extra&)
     {
       tracer trace ("c::as_cpp_init");
       l5 ([&]{trace << "for " << bs;});
@@ -513,17 +513,54 @@ namespace build2
       return true;
     }
 
+    bool
+    predefs_init (scope& rs,
+                  scope& bs,
+                  const location& loc,
+                  bool,
+                  bool,
+                  module_init_extra&)
+    {
+      tracer trace ("c::predefs_init");
+      l5 ([&]{trace << "for " << bs;});
+
+      // We only support root loading (which means there can only be one).
+      //
+      if (rs != bs)
+        fail (loc) << "c.predefs module must be loaded in project root";
+
+      module* mod (rs.find_module<module> ("c"));
+
+      if (mod == nullptr)
+        fail (loc) << "c.predefs module must be loaded after c module";
+
+      // Register the c.predefs rule.
+      //
+      // Why invent a separate module instead of just always registering it in
+      // the c module? The reason is performance: this rule will be called for
+      // every C header.
+      //
+      cc::predefs_rule& r (*mod);
+
+      rs.insert_rule<h> (perform_update_id,   r.rule_name, r);
+      rs.insert_rule<h> (perform_clean_id,    r.rule_name, r);
+      rs.insert_rule<h> (configure_update_id, r.rule_name, r);
+
+      return true;
+    }
+
     static const module_functions mod_functions[] =
     {
       // NOTE: don't forget to also update the documentation in init.hxx if
       //       changing anything here.
 
-      {"c.guess",  nullptr, guess_init},
-      {"c.config", nullptr, config_init},
-      {"c.objc",   nullptr, objc_init},
-      {"c.as-cpp", nullptr, as_cpp_init},
-      {"c",        nullptr, init},
-      {nullptr,    nullptr, nullptr}
+      {"c.guess",   nullptr, guess_init},
+      {"c.config",  nullptr, config_init},
+      {"c.objc",    nullptr, objc_init},
+      {"c.as-cpp",  nullptr, as_cpp_init},
+      {"c.predefs", nullptr, predefs_init},
+      {"c",         nullptr, init},
+      {nullptr,     nullptr, nullptr}
     };
 
     const module_functions*
diff --git a/libbuild2/c/init.hxx b/libbuild2/c/init.hxx
index c3126ea..713a78a 100644
--- a/libbuild2/c/init.hxx
+++ b/libbuild2/c/init.hxx
@@ -19,13 +19,15 @@ namespace build2
     //
     // Submodules:
     //
-    // `c.guess`  -- registers and sets some variables.
-    // `c.config` -- loads c.guess and sets more variables.
-    // `c`        -- loads c.config and registers target types and rules.
-    // `c.objc`   -- registers m{} target type and enables Objective-C
-    //               compilation. Must be loaded after c.
-    // `c.as-cpp` -- registers S{} target type and enables Assembler with
-    //               C preprocessor compilation. Must be loaded after c.
+    // `c.guess`   -- registers and sets some variables.
+    // `c.config`  -- loads c.guess and sets more variables.
+    // `c`         -- loads c.config and registers target types and rules.
+    // `c.objc`    -- registers m{} target type and enables Objective-C
+    //                compilation. Must be loaded after c.
+    // `c.as-cpp`  -- registers S{} target type and enables Assembler with
+    //                C preprocessor compilation. Must be loaded after c.
+    // `c.predefs` -- registers rule for generating a C header with
+    //                predefined compiler macros. Must be loaded after c.
     //
     extern "C" LIBBUILD2_C_SYMEXPORT const module_functions*
     build2_c_load ();
diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx
index 2bf9c1b..bbdc851 100644
--- a/libbuild2/cc/compile-rule.cxx
+++ b/libbuild2/cc/compile-rule.cxx
@@ -1151,6 +1151,8 @@ namespace build2
         // Note: the leading '@' is reserved for the module map prefix (see
         // extract_modules()) and no other line must start with it.
         //
+        // NOTE: see also the predefs rule if changing anything here.
+        //
         depdb dd (tp + ".d");
 
         // First should come the rule name/version.
@@ -3639,6 +3641,8 @@ namespace build2
 
               // See perform_update() for details on the choice of options.
               //
+              // NOTE: see also the predefs rule if adding anything here.
+              //
               {
                 bool sc (find_option_prefixes (
                            {"/source-charset:", "-source-charset:"}, args));
@@ -3708,6 +3712,8 @@ namespace build2
 
               // See perform_update() for details on the choice of options.
               //
+              // NOTE: see also the predefs rule if adding anything here.
+              //
               if (!find_option_prefix ("-finput-charset=", args))
                 args.push_back ("-finput-charset=UTF-8");
 
diff --git a/libbuild2/cc/module.hxx b/libbuild2/cc/module.hxx
index dc929dd..4213516 100644
--- a/libbuild2/cc/module.hxx
+++ b/libbuild2/cc/module.hxx
@@ -17,6 +17,7 @@
 #include <libbuild2/cc/compile-rule.hxx>
 #include <libbuild2/cc/link-rule.hxx>
 #include <libbuild2/cc/install-rule.hxx>
+#include <libbuild2/cc/predefs-rule.hxx>
 
 #include <libbuild2/cc/export.hxx>
 
@@ -134,7 +135,8 @@ namespace build2
                                          public link_rule,
                                          public compile_rule,
                                          public install_rule,
-                                         public libux_install_rule
+                                         public libux_install_rule,
+                                         public predefs_rule
     {
     public:
       explicit
@@ -143,7 +145,8 @@ namespace build2
             link_rule (move (d)),
             compile_rule (move (d), rs),
             install_rule (move (d), *this),
-            libux_install_rule (move (d), *this) {}
+            libux_install_rule (move (d), *this),
+            predefs_rule (move (d)) {}
 
       void
       init (scope&,
diff --git a/libbuild2/cc/predefs-rule.cxx b/libbuild2/cc/predefs-rule.cxx
new file mode 100644
index 0000000..c8a8181
--- /dev/null
+++ b/libbuild2/cc/predefs-rule.cxx
@@ -0,0 +1,356 @@
+// file      : libbuild2/cc/predefs-rule.cxx -*- C++ -*-
+// license   : MIT; see accompanying LICENSE file
+
+#include <libbuild2/cc/predefs-rule.hxx>
+
+#include <libbuild2/depdb.hxx>
+#include <libbuild2/target.hxx>
+#include <libbuild2/context.hxx>
+#include <libbuild2/algorithm.hxx>
+#include <libbuild2/filesystem.hxx>
+#include <libbuild2/diagnostics.hxx>
+
+namespace build2
+{
+  namespace cc
+  {
+    predefs_rule::
+    predefs_rule (data&& d)
+        : common (move (d)),
+          rule_name (string (x) += ".predefs"),
+          rule_id (rule_name + " 1")
+    {
+    }
+
+    bool predefs_rule::
+    match (action, target&, const string& hint, match_extra&) const
+    {
+      // We only match with an explicit hint (failed that, we will turn every
+      // header into predefs).
+      //
+      return hint == rule_name;
+    }
+
+    recipe predefs_rule::
+    apply (action a, target& xt, match_extra&) const
+    {
+      file& t (xt.as<file> ());
+      t.derive_path ();
+
+      // Inject dependency on the output directory.
+      //
+      inject_fsdir (a, t);
+
+      if (a == perform_update_id)
+      {
+        return [this] (action a, const target& xt)
+        {
+          return perform_update (a, xt);
+        };
+      }
+      else if (a == perform_clean_id)
+      {
+        return [] (action a, const target& t)
+        {
+          // Also remove the temporary input source file in case it wasn't
+          // removed at the end of the update.
+          //
+          return perform_clean_extra (a, t.as<file> (), {".d", ".t"});
+        };
+      }
+      else
+        return noop_recipe; // Configure update.
+    }
+
+    // Filter noise, sanitize options (msvc.cxx).
+    //
+    void
+    msvc_filter_cl (diag_buffer&, const path& src);
+
+    void
+    msvc_sanitize_cl (cstrings&);
+
+    target_state predefs_rule::
+    perform_update (action a, const target& xt) const
+    {
+      tracer trace (x, "predefs_rule::perform_update");
+
+      const file& t (xt.as<file> ());
+      const path& tp (t.path ());
+
+      context& ctx (t.ctx);
+
+      const scope& rs (t.root_scope ());
+
+      // Execute prerequisites (the output directory being the only one thus
+      // not mtime checking).
+      //
+      execute_prerequisites (a, t);
+
+      // Use depdb to track changes to options, compiler, etc (similar to
+      // the compile_rule).
+      //
+      depdb dd (tp + ".d");
+      {
+        // First should come the rule name/version.
+        //
+        if (dd.expect (rule_id) != nullptr)
+          l4 ([&]{trace << "rule mismatch forcing update of " << t;});
+
+        // Then the compiler checksum.
+        //
+        if (dd.expect (cast<string> (rs[x_checksum])) != nullptr)
+          l4 ([&]{trace << "compiler mismatch forcing update of " << t;});
+
+        // Then the compiler environment checksum.
+        //
+        if (dd.expect (env_checksum) != nullptr)
+          l4 ([&]{trace << "environment mismatch forcing update of " << t;});
+
+        // Finally the options checksum (as below).
+        //
+        {
+          sha256 cs;
+          append_options (cs, t, c_coptions);
+          append_options (cs, t, x_coptions);
+          append_options (cs, cmode);
+
+          if (dd.expect (cs.string ()) != nullptr)
+            l4 ([&]{trace << "options mismatch forcing update of " << t;});
+        }
+      }
+
+      // Update if depdb mismatch.
+      //
+      bool update (dd.writing () || dd.mtime > t.load_mtime ());
+
+      dd.close ();
+
+      if (!update)
+        return target_state::unchanged; // No mtime-based prerequisites.
+
+      // Prepare the compiler command-line.
+      //
+      cstrings args {cpath.recall_string ()};
+
+      // Append compile options.
+      //
+      // Note that any command line macros that we specify with -D will end up
+      // in the predefs, which is something we don't want. So no poptions.
+      //
+      append_options (args, t, c_coptions);
+      append_options (args, t, x_coptions);
+      append_options (args, cmode);
+
+      // The output and input paths, relative to the working directory for
+      // easier to read diagnostics.
+      //
+      path relo (relative (tp));
+      path reli;
+
+      // Add compiler-specific command-line arguments.
+      //
+      switch (cclass)
+      {
+      case compiler_class::gcc:
+        {
+          // Add implied options which may affect predefs, similar to the
+          // compile rule.
+          //
+          if (!find_option_prefix ("-finput-charset=", args))
+            args.push_back ("-finput-charset=UTF-8");
+
+          if (ctype == compiler_type::clang && tsys == "win32-msvc")
+          {
+            if (!find_options ({"-nostdlib", "-nostartfiles"}, args))
+            {
+              args.push_back ("-D_MT");
+              args.push_back ("-D_DLL");
+            }
+          }
+
+          if (ctype == compiler_type::clang && cvariant == "emscripten")
+          {
+            if (x_lang == lang::cxx)
+            {
+              if (!find_option_prefix ("DISABLE_EXCEPTION_CATCHING=", args))
+              {
+                args.push_back ("-s");
+                args.push_back ("DISABLE_EXCEPTION_CATCHING=0");
+              }
+            }
+          }
+
+          args.push_back ("-E");  // Stop after the preprocessing stage.
+          args.push_back ("-dM"); // Generate #define directives.
+
+          // Output.
+          //
+          args.push_back ("-o");
+          args.push_back (relo.string ().c_str ());
+
+          // Input.
+          //
+          args.push_back ("-x");
+          switch (x_lang)
+          {
+          case lang::c:   args.push_back ("c"); break;
+          case lang::cxx: args.push_back ("c++"); break;
+          }
+
+          // With GCC and Clang we can compile /dev/null as stdin by
+          // specifying `-` and thus omitting the temporary file.
+          //
+          args.push_back ("-");
+
+          break;
+        }
+      case compiler_class::msvc:
+        {
+          // Add implied options which may affect predefs, similar to the
+          // compile rule.
+          //
+          {
+            // Note: these affect the _MSVC_EXECUTION_CHARACTER_SET, _UTF8
+            // macros.
+            //
+            bool sc (find_option_prefixes (
+                       {"/source-charset:", "-source-charset:"}, args));
+            bool ec (find_option_prefixes (
+                       {"/execution-charset:", "-execution-charset:"}, args));
+
+            if (!sc && !ec)
+              args.push_back ("/utf-8");
+            else
+            {
+              if (!sc)
+                args.push_back ("/source-charset:UTF-8");
+
+              if (!ec)
+                args.push_back ("/execution-charset:UTF-8");
+            }
+          }
+
+          if (x_lang == lang::cxx)
+          {
+            if (!find_option_prefixes ({"/EH", "-EH"}, args))
+              args.push_back ("/EHsc");
+          }
+
+          if (!find_option_prefixes ({"/MD", "/MT", "-MD", "-MT"}, args))
+            args.push_back ("/MD");
+
+          msvc_sanitize_cl (args);
+
+          args.push_back ("/nologo");
+
+          // /EP may seem like it contradicts /P but it's the recommended
+          // way to suppress `#line`s from the output of the /P option (see
+          // /P in the "MSVC Compiler Options" documentation).
+          //
+          args.push_back ("/P");  // Write preprocessor output to a file.
+          args.push_back ("/EP"); // Preprocess to stdout without `#line`s.
+
+          args.push_back ("/PD");              // Print all macro definitions.
+          args.push_back ("/Zc:preprocessor"); // Preproc. conformance mode.
+
+          // Output (note that while the /Fi: variant is only availbale
+          // starting with VS2013, /Zc:preprocessor is only available in
+          // starting from VS2019).
+          //
+          args.push_back ("/Fi:");
+          args.push_back (relo.string ().c_str ());
+
+          // Input.
+          //
+          switch (x_lang)
+          {
+          case lang::c:   args.push_back ("/TC"); break;
+          case lang::cxx: args.push_back ("/TP"); break;
+          }
+
+          // Input path.
+          //
+          // Note that with MSVC we have to use a temporary file. In
+          // particular compiling `nul` does not work.
+          //
+          reli = relo + ".t";
+          args.push_back (reli.string ().c_str ());
+
+          break;
+        }
+      }
+
+      args.push_back (nullptr);
+
+      // Run the compiler.
+      //
+      if (verb >= 2)
+        print_process (args);
+      else if (verb)
+        print_diag ((string (x_name) + "-predefs").c_str (), t);
+
+      if (!ctx.dry_run)
+      {
+        // Create an empty temporary input source file, if necessary.
+        //
+        auto_rmfile rmi;
+        if (!reli.empty ())
+        {
+          rmi = auto_rmfile (reli);
+
+          if (exists (reli, false /* follow_symlinks */))
+            rmfile (ctx, reli, 3 /* verbosity */);
+
+          touch (ctx, reli, true /* create */, 3 /* verbosity */);
+        }
+
+        try
+        {
+          // VC cl.exe sends diagnostics to stdout. It also prints the file
+          // name being compiled as the first line. So for cl.exe we filter
+          // that noise out.
+          //
+          // For other compilers also redirect stdout to stderr, in case any
+          // of them tries to pull off something similar. For sane compilers
+          // this should be harmless.
+          //
+          // We also redirect stdin to /dev/null in case that's used instead
+          // of the temporary file.
+          //
+          // Note: similar logic as in compile_rule.
+          //
+          bool filter (ctype == compiler_type::msvc);
+
+          process pr (cpath,
+                      args,
+                      -2,                                         /* stdin  */
+                      2,                                          /* stdout */
+                      diag_buffer::pipe (ctx, filter /* force */) /* stderr */);
+
+          diag_buffer dbuf (ctx, args[0], pr);
+
+          if (filter)
+            msvc_filter_cl (dbuf, reli);
+
+          dbuf.read ();
+
+          run_finish (dbuf, args, pr, 1 /* verbosity */);
+          dd.check_mtime (tp);
+        }
+        catch (const process_error& e)
+        {
+          error << "unable to execute " << args[0] << ": " << e;
+
+          if (e.child)
+            exit (1);
+
+          throw failed ();
+        }
+      }
+
+      t.mtime (system_clock::now ());
+      return target_state::changed;
+    }
+  }
+}
diff --git a/libbuild2/cc/predefs-rule.hxx b/libbuild2/cc/predefs-rule.hxx
new file mode 100644
index 0000000..60aa063
--- /dev/null
+++ b/libbuild2/cc/predefs-rule.hxx
@@ -0,0 +1,45 @@
+// file      : libbuild2/cc/predefs-rule.hxx -*- C++ -*-
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef LIBBUILD2_CC_PREDEFS_RULE_HXX
+#define LIBBUILD2_CC_PREDEFS_RULE_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/rule.hxx>
+
+#include <libbuild2/cc/types.hxx>
+#include <libbuild2/cc/common.hxx>
+
+#include <libbuild2/cc/export.hxx>
+
+namespace build2
+{
+  namespace cc
+  {
+    class LIBBUILD2_CC_SYMEXPORT predefs_rule: public rule,
+                                               virtual common
+    {
+    public:
+      const string rule_name;
+
+      explicit
+      predefs_rule (data&&);
+
+      virtual bool
+      match (action, target&, const string&, match_extra&) const override;
+
+      virtual recipe
+      apply (action, target&, match_extra&) const override;
+
+      target_state
+      perform_update (action, const target&) const;
+
+    private:
+      const string rule_id;
+    };
+  }
+}
+
+#endif // LIBBUILD2_CC_PREDEFS_RULE_HXX
diff --git a/libbuild2/cxx/init.cxx b/libbuild2/cxx/init.cxx
index 8948e6a..626c795 100644
--- a/libbuild2/cxx/init.cxx
+++ b/libbuild2/cxx/init.cxx
@@ -957,16 +957,53 @@ namespace build2
       return true;
     }
 
+    bool
+    predefs_init (scope& rs,
+                  scope& bs,
+                  const location& loc,
+                  bool,
+                  bool,
+                  module_init_extra&)
+    {
+      tracer trace ("cxx::predefs_init");
+      l5 ([&]{trace << "for " << bs;});
+
+      // We only support root loading (which means there can only be one).
+      //
+      if (rs != bs)
+        fail (loc) << "cxx.predefs module must be loaded in project root";
+
+      module* mod (rs.find_module<module> ("cxx"));
+
+      if (mod == nullptr)
+        fail (loc) << "cxx.predefs module must be loaded after cxx module";
+
+      // Register the cxx.predefs rule.
+      //
+      // Why invent a separate module instead of just always registering it in
+      // the cxx module? The reason is performance: this rule will be called
+      // for every C++ header.
+      //
+      cc::predefs_rule& r (*mod);
+
+      rs.insert_rule<hxx> (perform_update_id,   r.rule_name, r);
+      rs.insert_rule<hxx> (perform_clean_id,    r.rule_name, r);
+      rs.insert_rule<hxx> (configure_update_id, r.rule_name, r);
+
+      return true;
+    }
+
     static const module_functions mod_functions[] =
     {
       // NOTE: don't forget to also update the documentation in init.hxx if
       //       changing anything here.
 
-      {"cxx.guess",  nullptr, guess_init},
-      {"cxx.config", nullptr, config_init},
-      {"cxx.objcxx", nullptr, objcxx_init},
-      {"cxx",        nullptr, init},
-      {nullptr,      nullptr, nullptr}
+      {"cxx.guess",   nullptr, guess_init},
+      {"cxx.config",  nullptr, config_init},
+      {"cxx.objcxx",  nullptr, objcxx_init},
+      {"cxx.predefs", nullptr, predefs_init},
+      {"cxx",         nullptr, init},
+      {nullptr,       nullptr, nullptr}
     };
 
     const module_functions*
diff --git a/libbuild2/cxx/init.hxx b/libbuild2/cxx/init.hxx
index 0e42cbe..5b80fdc 100644
--- a/libbuild2/cxx/init.hxx
+++ b/libbuild2/cxx/init.hxx
@@ -19,11 +19,13 @@ namespace build2
     //
     // Submodules:
     //
-    // `cxx.guess`  -- registers and sets some variables.
-    // `cxx.config` -- loads cxx.guess and sets more variables.
-    // `cxx`        -- loads cxx.config and registers target types and rules.
-    // `cxx.objcxx` -- registers mm{} target type and enables Objective-C++
-    //                 compilation.
+    // `cxx.guess`   -- registers and sets some variables.
+    // `cxx.config`  -- loads cxx.guess and sets more variables.
+    // `cxx`         -- loads cxx.config and registers target types and rules.
+    // `cxx.objcxx`  -- registers mm{} target type and enables Objective-C++
+    //                  compilation.
+    // `cxx.predefs` -- registers rule for generating a C++ header with
+    //                  predefined compiler macros. Must be loaded after cxx.
     //
     extern "C" LIBBUILD2_CXX_SYMEXPORT const module_functions*
     build2_cxx_load ();