From f69a53d0b83f6b6448aeacb98442b90e938696f3 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Mon, 29 May 2017 14:05:21 +0200 Subject: Add ability to limit amount of preprocessing done on source The cc.preprocessed variable can be 'none' (not preprocessed), 'includes' (no depends on preprocessor, e.g., #ifdef, etc), and 'all' (the source is fully preprocessed). Note that for 'all' the source can still contain comments and line continuations. --- build2/c/init.cxx | 1 + build2/cc/common.hxx | 9 +- build2/cc/compile.cxx | 299 ++++++++++++++++++++++++++++----------- build2/cc/compile.hxx | 7 +- build2/cc/init.cxx | 11 ++ build2/cxx/init.cxx | 1 + tests/cc/preprocessed/buildfile | 8 ++ tests/cc/preprocessed/testscript | 125 ++++++++++++++++ tests/common.test | 21 +++ 9 files changed, 397 insertions(+), 85 deletions(-) create mode 100644 tests/cc/preprocessed/buildfile create mode 100644 tests/cc/preprocessed/testscript diff --git a/build2/c/init.cxx b/build2/c/init.cxx index 39ce6fe..ada8596 100644 --- a/build2/c/init.cxx +++ b/build2/c/init.cxx @@ -161,6 +161,7 @@ namespace build2 v["cc.type"], v["cc.system"], v["cc.reprocess"], + v["cc.preprocessed"], v.insert ("c.std", variable_visibility::project), diff --git a/build2/cc/common.hxx b/build2/cc/common.hxx index 135f5b1..55e6675 100644 --- a/build2/cc/common.hxx +++ b/build2/cc/common.hxx @@ -30,7 +30,7 @@ namespace build2 lang x_lang; const char* x; // Module name ("c", "cxx"). - const char* x_name; // Compiler name ("c", "c++"; also used in -x). + const char* x_name; // Compiler name ("c", "c++"). const char* x_default; // Compiler default ("gcc", "g++"). const char* x_pext; // Preprocessed source extension (".i", ".ii"). @@ -64,9 +64,10 @@ namespace build2 const variable& c_export_loptions; const variable& c_export_libs; - const variable& c_type; // cc.type - const variable& c_system; // cc.system - const variable& c_reprocess; // cc.reprocess + const variable& c_type; // cc.type + const variable& c_system; // cc.system + const variable& c_reprocess; // cc.reprocess + const variable& c_preprocessed; // cc.preprocessed const variable& x_std; diff --git a/build2/cc/compile.cxx b/build2/cc/compile.cxx index 96ab435..9fe0a5c 100644 --- a/build2/cc/compile.cxx +++ b/build2/cc/compile.cxx @@ -29,6 +29,22 @@ namespace build2 { using namespace bin; + inline bool + operator< (preprocessed l, preprocessed r) + { + return static_cast (l) < static_cast (r); + } + + preprocessed + to_preprocessed (const string& s) + { + if (s == "none") return preprocessed::none; + if (s == "includes") return preprocessed::includes; + if (s == "modules") return preprocessed::modules; + if (s == "all") return preprocessed::all; + throw invalid_argument ("invalid preprocessed value '" + s + "'"); + } + compile:: compile (data&& d) : common (move (d)), @@ -40,11 +56,12 @@ namespace build2 { explicit match_data (const prerequisite_member& s) - : src (s), mt (timestamp_unknown) {} + : src (s), pp (preprocessed::none), mt (timestamp_unknown) {} prerequisite_member src; - auto_rmfile psrc; // Preprocessed source, if any. - timestamp mt; // Target timestamp. + preprocessed pp; + auto_rmfile psrc; // Preprocessed source, if any. + timestamp mt; // Target timestamp. }; static_assert (sizeof (match_data) <= target::data_size, @@ -492,17 +509,24 @@ namespace build2 // sha256 cs; - hash_options (cs, t, c_poptions); - hash_options (cs, t, x_poptions); - - // Hash *.export.poptions from prerequisite libraries. + // This affects how we compile the source so factor it in. // - hash_lib_options (bs, cs, t, act, lo); + cs.append (&md.pp, sizeof (md.pp)); - // Extra system header dirs (last). - // - for (const dir_path& d: sys_inc_dirs) - cs.append (d.string ()); + if (md.pp != preprocessed::all) + { + hash_options (cs, t, c_poptions); + hash_options (cs, t, x_poptions); + + // Hash *.export.poptions from prerequisite libraries. + // + hash_lib_options (bs, cs, t, act, lo); + + // Extra system header dirs (last). + // + for (const dir_path& d: sys_inc_dirs) + cs.append (d.string ()); + } hash_options (cs, t, c_coptions); hash_options (cs, t, x_coptions); @@ -550,7 +574,27 @@ namespace build2 u = update (trace, act, *pt, u ? timestamp_unknown : mt) || u; } - pair p (extract_headers (act, t, lo, src, dd, u)); + // Check if the source is already preprocessed to a certain degree. + // This determines which of the following steps we perform and on + // what source (original or preprocessed). + // + if (const string* v = cast_null (t[c_preprocessed])) + try + { + md.pp = to_preprocessed (*v); + } + catch (const invalid_argument& e) + { + fail << "invalid " << c_preprocessed.name << " variable value " + << "for target " << t << ": " << e; + } + + // If we have no #include directives, then skip header dependency + // extraction. + // + pair p (auto_rmfile (), false); + if (md.pp < preprocessed::includes) + p = extract_headers (act, t, lo, src, dd, u); // If anything got updated, then we didn't rely on the cache. However, // the cached data could actually have been valid and the compiler run @@ -565,23 +609,21 @@ namespace build2 dd.close (); - // Do we have preprocessed output? + // If C++ modules support is enabled then we need to extract the + // module dependency information in addition to header dependencies + // above. // - if (!p.first.path ().empty ()) + if (u) // @@ TMP (depdb validation similar to extract_headers()). { - // If C++ modules support is enabled then we need to extract the - // module dependency information in addition to header dependencie - // above. - // - extract_modules (act, t, lo, src, p.first, dd, u); - - // If the preprocessed output is suitable for compilation and is not - // disabled, pass it along. - // - if (p.second && !cast_false (t[c_reprocess])) - md.psrc = move (p.first); + extract_modules (act, t, lo, src, p.first, md.pp, dd, u); } + // If the preprocessed output is suitable for compilation and is not + // disabled, then pass it along. + // + if (p.second && !cast_false (t[c_reprocess])) + md.psrc = move (p.first); + md.mt = u ? timestamp_nonexistent : mt; } @@ -1143,14 +1185,6 @@ namespace build2 append_std (args); - if (t.is_a ()) - { - // On Darwin, Win32 -fPIC is the default. - // - if (tclass == "linux" || tclass == "bsd") - args.push_back ("-fPIC"); - } - if (cid == compiler_id::msvc) { assert (pp != nullptr); @@ -1190,6 +1224,14 @@ namespace build2 } else { + if (t.is_a ()) + { + // On Darwin, Win32 -fPIC is the default. + // + if (tclass == "linux" || tclass == "bsd") + args.push_back ("-fPIC"); + } + // Depending on the compiler, decide whether (and how) we can // produce preprocessed output as a side effect of dependency // extraction. @@ -1935,6 +1977,7 @@ namespace build2 } } + puse = puse && !psrc.path ().empty (); return make_pair (move (psrc), puse); } @@ -1946,6 +1989,7 @@ namespace build2 lorder lo, const file& src, auto_rmfile& psrc, + preprocessed pp, depdb& /*dd*/, bool& /*updating*/) const { @@ -1966,24 +2010,29 @@ namespace build2 // For some compilers (GCC, Clang) the preporcessed output is only // partially preprocessed. For others (VC), it is already fully // preprocessed (well, almost: it still has comments but we can handle - // that). + // that). Plus, the source file might already be (sufficiently) + // preprocessed. // // So the plan is to start the compiler process that writes the fully // preprocessed output to stdout and reduce the already preprocessed // case to it. // cstrings args; + path rels; - path rels (relative (psrc.path ())); - - // This should match with how we setup preprocessing and is pretty - // similar to init_args() from extract_headers(). - // - switch (cid) + bool ps; // True if extracting from psrc. + if (pp < preprocessed::modules) { - case compiler_id::gcc: - case compiler_id::clang: + ps = !psrc.path ().empty (); + rels = relative (ps ? psrc.path () : src.path ()); + + // VC's preprocessed output, if present, is fully preprocessed. + // + if (cid != compiler_id::msvc || !ps) { + // This should match with how we setup preprocessing and is pretty + // similar to init_args() from extract_headers(). + // const scope& bs (t.base_scope ()); args.push_back (cpath.recall_string ()); @@ -2010,35 +2059,68 @@ namespace build2 append_std (args); - if (t.is_a ()) + if (cid == compiler_id::msvc) { - // On Darwin, Win32 -fPIC is the default. + args.push_back ("/nologo"); + + // See perform_update() for details on overriding the default + // exceptions and runtime. // - if (tclass == "linux" || tclass == "bsd") - args.push_back ("-fPIC"); - } + if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) + args.push_back ("/EHsc"); - // Options that trigger preprocessing of partially preprocessed - // output are a bit of a compiler-specific voodoo. - // - args.push_back ("-E"); - args.push_back ("-x"); - args.push_back (x_name); + if (!find_option_prefixes ({"/MD", "/MT"}, args)) + args.push_back ("/MD"); - if (cid == compiler_id::gcc) + args.push_back ("/E"); + args.push_back ("/C"); + args.push_back (x_lang == lang::c ? "/TC" : "/TP"); // Compile as. + } + else { - args.push_back ("-fpreprocessed"); - args.push_back ("-fdirectives-only"); + if (t.is_a ()) + { + // On Darwin, Win32 -fPIC is the default. + // + if (tclass == "linux" || tclass == "bsd") + args.push_back ("-fPIC"); + } + + // Options that trigger preprocessing of partially preprocessed + // output are a bit of a compiler-specific voodoo. + // + args.push_back ("-E"); + + if (ps) + { + const char* l (nullptr); + switch (x_lang) + { + case lang::c: l = "c"; + case lang::cxx: l = "c++"; + } + + args.push_back ("-x"); + args.push_back (l); + + if (cid == compiler_id::gcc) + { + args.push_back ("-fpreprocessed"); + args.push_back ("-fdirectives-only"); + } + } } args.push_back (rels.string ().c_str ()); args.push_back (nullptr); - break; } - case compiler_id::msvc: - break; // Already fully preprocessed. - case compiler_id::icc: - assert (false); + } + else + { + // Extracting directly from source. + // + ps = false; + rels = relative (src.path ()); } // Preprocess and parse. @@ -2051,7 +2133,8 @@ namespace build2 // Disarm the removal of the preprocessed file in case of an error. // We re-arm it below. // - psrc.cancel (); + if (ps) + psrc.cancel (); process pr; @@ -2083,7 +2166,9 @@ namespace build2 if (pr.wait ()) { - psrc = auto_rmfile (move (rels)); // Re-arm. + if (ps) + psrc = auto_rmfile (move (rels)); // Re-arm. + break; } @@ -2166,19 +2251,22 @@ namespace build2 path relo (relative (tp)); path rels (relative (s.path ())); - append_options (args, t, c_poptions); - append_options (args, t, x_poptions); + if (md.pp != preprocessed::all) + { + append_options (args, t, c_poptions); + append_options (args, t, x_poptions); - // Add *.export.poptions from prerequisite libraries. - // - append_lib_options (bs, args, t, act, lo); + // Add *.export.poptions from prerequisite libraries. + // + append_lib_options (bs, args, t, act, lo); - // Extra system header dirs (last). - // - for (const dir_path& d: sys_inc_dirs) - { - args.push_back ("-I"); - args.push_back (d.string ().c_str ()); + // Extra system header dirs (last). + // + for (const dir_path& d: sys_inc_dirs) + { + args.push_back ("-I"); + args.push_back (d.string ().c_str ()); + } } append_options (args, t, c_coptions); @@ -2271,6 +2359,8 @@ namespace build2 args.push_back (out.c_str ()); } + // Note: no way to indicate that the source if already preprocessed. + args.push_back ("/c"); // Compile only. args.push_back (x_lang == lang::c ? "/TC" : "/TP"); // Compile as. args.push_back (rels.string ().c_str ()); // Note: rely on being last. @@ -2289,6 +2379,48 @@ namespace build2 args.push_back (relo.string ().c_str ()); args.push_back ("-c"); + + if (md.pp == preprocessed::all) + { + // Note that the mode we select must still handle comments and line + // continuations. So some more compiler-specific voodoo. + // + switch (cid) + { + case compiler_id::gcc: + { + // -fdirectives-only is available since GCC 4.3.0. + // + if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) + { + args.push_back ("-fpreprocessed"); + args.push_back ("-fdirectives-only"); + } + break; + } + case compiler_id::clang: + { + // Clang handles comments and line continuations in the + // preprocessed source (it does not have -fpreprocessed). + // + const char* l (nullptr); + switch (x_lang) + { + case lang::c: l = "cpp-output"; + case lang::cxx: l = "c++-cpp-output"; + } + + args.push_back ("-x"); + args.push_back (l); + break; + } + case compiler_id::icc: + break; // Compile as normal source for now. + case compiler_id::msvc: + assert (false); + } + } + args.push_back (rels.string ().c_str ()); // Note: rely on being last. } @@ -2303,13 +2435,13 @@ namespace build2 else if (verb == 2) print_process (args); - // If we have the preprocessed output, switch to that. + // If we have the (partially) preprocessed output, switch to that. // bool psrc (!md.psrc.path ().empty ()); if (psrc) { - args.pop_back (); - args.pop_back (); + args.pop_back (); // nullptr + args.pop_back (); // rels rels = relative (md.psrc.path ()); @@ -2319,7 +2451,7 @@ namespace build2 { case compiler_id::gcc: { - // The -fpreprocessed in implied by .i/.ii. + // The -fpreprocessed is implied by .i/.ii. // args.push_back ("-fdirectives-only"); break; @@ -2328,8 +2460,15 @@ namespace build2 { // Without this Clang will treat .i/.ii as fully preprocessed. // + const char* l (nullptr); + switch (x_lang) + { + case lang::c: l = "c"; + case lang::cxx: l = "c++"; + } + args.push_back ("-x"); - args.push_back (x_name); + args.push_back (l); break; } case compiler_id::msvc: diff --git a/build2/cc/compile.hxx b/build2/cc/compile.hxx index f8e6bab..2b6fb2e 100644 --- a/build2/cc/compile.hxx +++ b/build2/cc/compile.hxx @@ -22,6 +22,11 @@ namespace build2 namespace cc { + // The order is arranged so that their integral values indicate whether + // one is a "stronger" than another. + // + enum class preprocessed: uint8_t {none, includes, modules, all}; + class compile: public rule, virtual common { public: @@ -87,7 +92,7 @@ namespace build2 depdb&, bool&) const; void extract_modules (action, file&, lorder, - const file&, auto_rmfile&, + const file&, auto_rmfile&, preprocessed, depdb&, bool&) const; private: diff --git a/build2/cc/init.cxx b/build2/cc/init.cxx index a5fcf83..f845712 100644 --- a/build2/cc/init.cxx +++ b/build2/cc/init.cxx @@ -83,6 +83,17 @@ namespace build2 v.insert ("config.cc.reprocess", true); v.insert ("cc.reprocess"); + // Ability to indicate that source is already (partially) preprocessed. + // Valid values are 'none' (not preprocessed), 'includes' (no #include + // directives in source), 'modules' (as above plus no module declaration + // depends on preprocessor, e.g., #ifdef, etc), and 'all' (the source is + // fully preprocessed). Note that for 'all' the source can still contain + // comments and line continuations. Note also that for some compilers + // (e.g., VC) there is no way to signal that the source is already + // preprocessed. + // + v.insert ("cc.preprocessed"); + return true; } diff --git a/build2/cxx/init.cxx b/build2/cxx/init.cxx index 97e5890..72a8fe5 100644 --- a/build2/cxx/init.cxx +++ b/build2/cxx/init.cxx @@ -224,6 +224,7 @@ namespace build2 v["cc.type"], v["cc.system"], v["cc.reprocess"], + v["cc.preprocessed"], v.insert ("cxx.std", variable_visibility::project), diff --git a/tests/cc/preprocessed/buildfile b/tests/cc/preprocessed/buildfile new file mode 100644 index 0000000..d812867 --- /dev/null +++ b/tests/cc/preprocessed/buildfile @@ -0,0 +1,8 @@ +# file : tests/cc/preprocessed/buildfile +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Test cc.preprocessed logic. +# + +./: test{testscript} $b diff --git a/tests/cc/preprocessed/testscript b/tests/cc/preprocessed/testscript new file mode 100644 index 0000000..cbc179e --- /dev/null +++ b/tests/cc/preprocessed/testscript @@ -0,0 +1,125 @@ +# file : tests/cc/preprocessed/testscript +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +crosstest = false +test.arguments = config.cxx="$recall($cxx.path)" update clean #@@ TMP clean + +.include ../../common.test + +# Trace filter. +# +# trace: cxx::compile::extract_(header|modules): target: .../obje{(test).o...} +# +filter = sed -n -e \ + \''s/^trace: cxx::compile::extract_([^:]+): target:[^{]+\{([^.]+).*/\1 \2/p'\' + ++cat <=build/root.build +cxx.std = latest + +using cxx + +hxx{*}: extension = hxx +cxx{*}: extension = cxx + +cxx.poptions =+ "-I$src_root" +EOI + +: none +: +: Include a header (makes sure headers are handled) which defines the +: TEST_VALUE macro. Import a non-existent module unless this macro is +: defined (makes sure modules are extracted from preprocessed source). +: Use the macro (makes sure compilation happens on preprocessed source). +: +cat <=test.hxx; + #define TEST_VALUE 0 + EOI +cat <=test.cxx; + #include + + #ifndef TEST_VALUE + import foo; + #endif + + int main () {return TEST_VALUE;} + EOI +$* --verbose 5 <&1 | $filter >>EOO #@@ &test* + exe{test}: cxx{test} + EOI + headers test + modules test + EOO + +: includes +: +cat <=test.cxx; + #ifndef TEST_VALUE + import foo; + #endif + + int main () {return TEST_VALUE;} + EOI +$* --verbose 5 <&1 | $filter >>EOO #@@ &test* + cc.preprocessed = includes + cc.poptions += -DTEST_VALUE=0 + exe{test}: cxx{test} + EOI + modules test + EOO + +: modules +: +: Define and use macro to test that compilation inclused the preprocessor. +: +cat <=test.cxx; + int main () {return TEST_VALUE;} + EOI +$* --verbose 5 <&1 | $filter >>EOO #@@ &test* + cc.preprocessed = modules + cc.poptions += -DTEST_VALUE=0 + exe{test}: cxx{test} + EOI + modules test + EOO + +: modules-extract +: +: Define macro that would have disabled the module import (makes sure +: modules are extracted directly from source). +: +cat <=test.cxx; + #define TEST_VALUE + #ifndef TEST_VALUE + import foo; + #endif + EOI +$* <>EOE != 0 ;#@@ &test* + cc.preprocessed = modules + exe{test}: cxx{test} + EOI + error: module support not yet implemented + EOE +rm -f test.o.d test.exe.obj.d #@@ TMP + +: all +: +: Test handling of comments and line continuations. Define macro on the +: command line that would have affected the result. +: +cat <=test.cxx; + // C++ comment + /* + C comment + */ + + int ma\ + in () {} + EOI +$* --verbose 5 <&1 | $filter >>EOO #@@ &test* + cc.preprocessed = all + cc.poptions += -Dmain=foo + exe{test}: cxx{test} + EOI + modules test + EOO diff --git a/tests/common.test b/tests/common.test index b8d6148..0e8ce52 100644 --- a/tests/common.test +++ b/tests/common.test @@ -5,6 +5,27 @@ # Commonly-used build system test project setup and driver command line. # +# If the includer indicated that no cross-testing should be supported, then +# use the build system driver that is building, not the one being built. +# +# In many cases expecting a cross-compiled driver to perform a native build +# under emulation is pushing things a bit too far. Plus, we have no way of +# knowing the native compiler name/path. +# +# So the idea here is to test cross-compilation with the understanding that +# the build system driver we are testing is not the one being cross-compiled +# but rather the one doing the cross-compilation. +# +if ($null($crosstest)) + crosstest = false +end + +if (!$crosstest && $test.target != $build.host) + test = $recall($build.path) +end + +# Common bootstrap.build. +# +mkdir build +cat <=build/bootstrap.build project = test -- cgit v1.1