From 934f2a9a90c5cad3cdc8a66b50c17827a3ddbcee Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Sat, 20 Jan 2018 13:46:11 +0200 Subject: Get rid of action rule override semantics Instead we now have two more or less separate match states for outer and inner parts of an action. --- build2/cc/common.cxx | 28 +- build2/cc/common.hxx | 8 +- build2/cc/compile-rule.cxx | 4631 ++++++++++++++++++++++++++++++++++++++++ build2/cc/compile-rule.hxx | 156 ++ build2/cc/compile.cxx | 4627 --------------------------------------- build2/cc/compile.hxx | 153 -- build2/cc/install-rule.cxx | 269 +++ build2/cc/install-rule.hxx | 77 + build2/cc/install.cxx | 246 --- build2/cc/install.hxx | 67 - build2/cc/link-rule.cxx | 2104 ++++++++++++++++++ build2/cc/link-rule.hxx | 166 ++ build2/cc/link.cxx | 2108 ------------------ build2/cc/link.hxx | 137 -- build2/cc/module.cxx | 33 +- build2/cc/module.hxx | 22 +- build2/cc/pkgconfig.cxx | 37 +- build2/cc/windows-manifest.cxx | 6 +- build2/cc/windows-rpath.cxx | 42 +- 19 files changed, 7492 insertions(+), 7425 deletions(-) create mode 100644 build2/cc/compile-rule.cxx create mode 100644 build2/cc/compile-rule.hxx delete mode 100644 build2/cc/compile.cxx delete mode 100644 build2/cc/compile.hxx create mode 100644 build2/cc/install-rule.cxx create mode 100644 build2/cc/install-rule.hxx delete mode 100644 build2/cc/install.cxx delete mode 100644 build2/cc/install.hxx create mode 100644 build2/cc/link-rule.cxx create mode 100644 build2/cc/link-rule.hxx delete mode 100644 build2/cc/link.cxx delete mode 100644 build2/cc/link.hxx (limited to 'build2/cc') diff --git a/build2/cc/common.cxx b/build2/cc/common.cxx index 3daada1..e4dbfe8 100644 --- a/build2/cc/common.cxx +++ b/build2/cc/common.cxx @@ -46,7 +46,7 @@ namespace build2 // void common:: process_libraries ( - action act, + action a, const scope& top_bs, linfo top_li, const dir_paths& top_sysd, @@ -222,23 +222,23 @@ namespace build2 // if (impl && !c_e_libs.defined () && !x_e_libs.defined ()) { - for (auto pt: l.prerequisite_targets) + for (const prerequisite_target& pt: l.prerequisite_targets[a]) { if (pt == nullptr) continue; - bool a; + bool la; const file* f; - if ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || - ( f = pt->is_a ())) + if ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || + ( f = pt->is_a ())) { if (sysd == nullptr) find_sysd (); if (!li) find_linfo (); - process_libraries (act, bs, *li, *sysd, - *f, a, pt.data, + process_libraries (a, bs, *li, *sysd, + *f, la, pt.data, proc_impl, proc_lib, proc_opt, true); } } @@ -275,7 +275,7 @@ namespace build2 &proc_impl, &proc_lib, &proc_opt, &sysd, &usrd, &find_sysd, &find_linfo, &sys_simple, - &bs, act, &li, this] (const lookup& lu) + &bs, a, &li, this] (const lookup& lu) { const vector* ns (cast_null> (lu)); if (ns == nullptr || ns->empty ()) @@ -300,7 +300,7 @@ namespace build2 if (sysd == nullptr) find_sysd (); if (!li) find_linfo (); - const file& t (resolve_library (act, bs, n, *li, *sysd, usrd)); + const file& t (resolve_library (a, bs, n, *li, *sysd, usrd)); if (proc_lib) { @@ -324,7 +324,7 @@ namespace build2 // @@ Where can we get the link flags? Should we try to find them // in the library's prerequisites? What about installed stuff? // - process_libraries (act, bs, *li, *sysd, + process_libraries (a, bs, *li, *sysd, t, t.is_a () || t.is_a (), 0, proc_impl, proc_lib, proc_opt, true); } @@ -402,7 +402,7 @@ namespace build2 // that's the only way to guarantee it will be up-to-date. // const file& common:: - resolve_library (action act, + resolve_library (action a, const scope& s, name n, linfo li, @@ -439,7 +439,7 @@ namespace build2 // dir_path out; prerequisite_key pk {n.proj, {tt, &n.dir, &out, &n.value, ext}, &s}; - xt = search_library_existing (act, sysd, usrd, pk); + xt = search_library_existing (a, sysd, usrd, pk); if (xt == nullptr) { @@ -454,7 +454,7 @@ namespace build2 // If this is lib{}/libu{}, pick appropriate member. // if (const libx* l = xt->is_a ()) - xt = &link_member (*l, act, li); // Pick lib*{e,a,s}{}. + xt = &link_member (*l, a, li); // Pick lib*{e,a,s}{}. return xt->as (); } diff --git a/build2/cc/common.hxx b/build2/cc/common.hxx index 5ed7173..5952df6 100644 --- a/build2/cc/common.hxx +++ b/build2/cc/common.hxx @@ -225,7 +225,7 @@ namespace build2 bool = false) const; const target* - search_library (action act, + search_library (action a, const dir_paths& sysd, optional& usrd, const prerequisite& p) const @@ -234,7 +234,7 @@ namespace build2 if (r == nullptr) { - if ((r = search_library (act, sysd, usrd, p.key ())) != nullptr) + if ((r = search_library (a, sysd, usrd, p.key ())) != nullptr) { const target* e (nullptr); if (!p.target.compare_exchange_strong ( @@ -274,12 +274,12 @@ namespace build2 bool existing = false) const; const target* - search_library_existing (action act, + search_library_existing (action a, const dir_paths& sysd, optional& usrd, const prerequisite_key& pk) const { - return search_library (act, sysd, usrd, pk, true); + return search_library (a, sysd, usrd, pk, true); } dir_paths diff --git a/build2/cc/compile-rule.cxx b/build2/cc/compile-rule.cxx new file mode 100644 index 0000000..df84547 --- /dev/null +++ b/build2/cc/compile-rule.cxx @@ -0,0 +1,4631 @@ +// file : build2/cc/compile-rule.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include // exit() +#include // strlen() + +#include +#include +#include +#include +#include +#include +#include + +#include +#include // create_project() + +#include +#include // h +#include +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace cc + { + using namespace bin; + + // module_info string serialization. + // + // The string representation is a space-separated list of module names + // with the following rules: + // + // 1. If this is a module interface unit, then the first name is the + // module name intself following by either '!' for an interface unit or + // by '+' for an implementation unit. + // + // 2. If an imported module is re-exported, then the module name is + // followed by '*'. + // + // For example: + // + // foo! foo.core* foo.base* foo.impl + // foo.base+ foo.impl + // foo.base foo.impl + // + static string + to_string (const module_info& m) + { + string s; + + if (!m.name.empty ()) + { + s += m.name; + s += m.iface ? '!' : '+'; + } + + for (const module_import& i: m.imports) + { + if (!s.empty ()) + s += ' '; + + s += i.name; + + if (i.exported) + s += '*'; + } + + return s; + } + + static module_info + to_module_info (const string& s) + { + module_info m; + + for (size_t b (0), e (0), n; (n = next_word (s, b, e, ' ')) != 0; ) + { + char c (s[e - 1]); + switch (c) + { + case '!': + case '+': + case '*': break; + default: c = '\0'; + } + + string w (s, b, n - (c == '\0' ? 0 : 1)); + + if (c == '!' || c == '+') + { + m.name = move (w); + m.iface = (c == '!'); + } + else + m.imports.push_back (module_import {move (w), c == '*', 0}); + } + + return m; + } + + // preprocessed + // + template + inline bool + operator< (preprocessed l, T r) // Template because of VC14 bug. + { + return static_cast (l) < static_cast (r); + } + + preprocessed + to_preprocessed (const string& s) + { + if (s == "none") return preprocessed::none; + if (s == "includes") return preprocessed::includes; + if (s == "modules") return preprocessed::modules; + if (s == "all") return preprocessed::all; + throw invalid_argument ("invalid preprocessed value '" + s + "'"); + } + + struct compile_rule::match_data + { + explicit + match_data (translation_type t, const prerequisite_member& s) + : type (t), src (s) {} + + translation_type type; + preprocessed pp = preprocessed::none; + bool symexport = false; // Target uses __symexport. + bool touch = false; // Target needs to be touched. + timestamp mt = timestamp_unknown; // Target timestamp. + prerequisite_member src; + auto_rmfile psrc; // Preprocessed source, if any. + path dd; // Dependency database path. + module_positions mods = {0, 0, 0}; + }; + + compile_rule:: + compile_rule (data&& d) + : common (move (d)), + rule_id (string (x) += ".compile 4") + { + static_assert (sizeof (match_data) <= target::data_size, + "insufficient space"); + } + + const char* compile_rule:: + langopt (const match_data& md) const + { + bool m (md.type == translation_type::module_iface); + //preprocessed p (md.pp); + + switch (cid) + { + case compiler_id::gcc: + { + // Ignore the preprocessed value since for GCC it is handled via + // -fpreprocessed -fdirectives-only. + // + switch (x_lang) + { + case lang::c: return "c"; + case lang::cxx: return "c++"; + } + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // Clang has *-cpp-output (but not c++-module-cpp-output) and they + // handle comments and line continuations. However, currently this + // is only by accident since these modes are essentially equivalent + // to their cpp-output-less versions. + // + switch (x_lang) + { + case lang::c: return "c"; + case lang::cxx: return m ? "c++-module" : "c++"; + } + } + case compiler_id::msvc: + { + switch (x_lang) + { + case lang::c: return "/TC"; + case lang::cxx: return "/TP"; + } + } + case compiler_id::icc: + { + switch (x_lang) + { + case lang::c: return "c"; + case lang::cxx: return "c++"; + } + } + } + + return nullptr; + } + + inline void compile_rule:: + append_symexport_options (cstrings& args, const target& t) const + { + // With VC if a BMI is compiled with dllexport, then when such BMI is + // imported, it is auto-magically treated as dllimport. Let's hope + // other compilers follow suit. + // + args.push_back (t.is_a () && tclass == "windows" + ? "-D__symexport=__declspec(dllexport)" + : "-D__symexport="); + } + + bool compile_rule:: + match (action a, target& t, const string&) const + { + tracer trace (x, "compile_rule::match"); + + bool mod (t.is_a () || t.is_a () || t.is_a ()); + + // Link-up to our group (this is the obj/bmi{} target group protocol + // which means this can be done whether we match or not). + // + if (t.group == nullptr) + t.group = &search (t, + mod ? bmi::static_type : obj::static_type, + t.dir, t.out, t.name); + + // See if we have a source file. Iterate in reverse so that a source + // file specified for a member overrides the one specified for the + // group. Also "see through" groups. + // + for (prerequisite_member p: reverse_group_prerequisite_members (a, t)) + { + if (p.is_a (mod ? *x_mod : x_src)) + { + // Save in the target's auxiliary storage. Translation type will + // be refined in apply(). + // + t.data (match_data (mod + ? translation_type::module_iface + : translation_type::plain, + p)); + return true; + } + } + + l4 ([&]{trace << "no " << x_lang << " source file for target " << t;}); + return false; + } + + // Append or hash library options from a pair of *.export.* variables + // (first one is cc.export.*) recursively, prerequisite libraries first. + // + void compile_rule:: + append_lib_options (const scope& bs, + cstrings& args, + action a, + const target& t, + linfo li) const + { + // See through utility libraries. + // + auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; + + auto opt = [&args, this] ( + const file& l, const string& t, bool com, bool exp) + { + // Note that in our model *.export.poptions are always "interface", + // even if set on liba{}/libs{}, unlike loptions. + // + if (!exp) // Ignore libux. + return; + + const variable& var ( + com + ? c_export_poptions + : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); + + append_options (args, l, var); + }; + + // In case we don't have the "small function object" optimization. + // + const function impf (imp); + const function optf (opt); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + // Should be already searched and matched for libraries. + // + if (const target* pt = p.load ()) + { + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + + bool la; + if (!((la = pt->is_a ()) || + (la = pt->is_a ()) || + pt->is_a ())) + continue; + + process_libraries (a, bs, li, sys_lib_dirs, + pt->as (), la, 0, // Hack: lflags unused. + impf, nullptr, optf); + } + } + } + + void compile_rule:: + hash_lib_options (const scope& bs, + sha256& cs, + action a, + const target& t, + linfo li) const + { + auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; + + auto opt = [&cs, this] ( + const file& l, const string& t, bool com, bool exp) + { + if (!exp) + return; + + const variable& var ( + com + ? c_export_poptions + : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); + + hash_options (cs, l, var); + }; + + // The same logic as in append_lib_options(). + // + const function impf (imp); + const function optf (opt); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + if (const target* pt = p.load ()) + { + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + + bool la; + if (!((la = pt->is_a ()) || + (la = pt->is_a ()) || + pt->is_a ())) + continue; + + process_libraries (a, bs, li, sys_lib_dirs, + pt->as (), la, 0, // Hack: lflags unused. + impf, nullptr, optf); + } + } + } + + // Append library prefixes based on the *.export.poptions variables + // recursively, prerequisite libraries first. + // + void compile_rule:: + append_lib_prefixes (const scope& bs, + prefix_map& m, + action a, + target& t, + linfo li) const + { + auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; + + auto opt = [&m, this] ( + const file& l, const string& t, bool com, bool exp) + { + if (!exp) + return; + + const variable& var ( + com + ? c_export_poptions + : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); + + append_prefixes (m, l, var); + }; + + // The same logic as in append_lib_options(). + // + const function impf (imp); + const function optf (opt); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + if (const target* pt = p.load ()) + { + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + + bool la; + if (!((la = pt->is_a ()) || + (la = pt->is_a ()) || + pt->is_a ())) + continue; + + process_libraries (a, bs, li, sys_lib_dirs, + pt->as (), la, 0, // Hack: lflags unused. + impf, nullptr, optf); + } + } + } + + // Update the target during the match phase. Return true if it has changed + // or if the passed timestamp is not timestamp_unknown and is older than + // the target. + // + // This function is used to make sure header dependencies are up to date. + // + // There would normally be a lot of headers for every source file (think + // all the system headers) and just calling execute_direct() on all of + // them can get expensive. At the same time, most of these headers are + // existing files that we will never be updating (again, system headers, + // for example) and the rule that will match them is the fallback + // file_rule. That rule has an optimization: it returns noop_recipe (which + // causes the target state to be automatically set to unchanged) if the + // file is known to be up to date. So we do the update "smartly". + // + static bool + update (tracer& trace, action a, const target& t, timestamp ts) + { + const path_target* pt (t.is_a ()); + + if (pt == nullptr) + ts = timestamp_unknown; + + target_state os (t.matched_state (a)); + + if (os == target_state::unchanged) + { + if (ts == timestamp_unknown) + return false; + else + { + // We expect the timestamp to be known (i.e., existing file). + // + timestamp mt (pt->mtime ()); + assert (mt != timestamp_unknown); + return mt > ts; + } + } + else + { + // We only want to return true if our call to execute() actually + // caused an update. In particular, the target could already have been + // in target_state::changed because of a dependency extraction run for + // some other source file. + // + // @@ MT perf: so we are going to switch the phase and execute for + // any generated header. + // + phase_switch ps (run_phase::execute); + target_state ns (execute_direct (a, t)); + + if (ns != os && ns != target_state::unchanged) + { + l6 ([&]{trace << "updated " << t + << "; old state " << os + << "; new state " << ns;}); + return true; + } + else + return ts != timestamp_unknown ? pt->newer (ts) : false; + } + } + + recipe compile_rule:: + apply (action a, target& xt) const + { + tracer trace (x, "compile_rule::apply"); + + file& t (xt.as ()); // Either obj*{} or bmi*{}. + + match_data& md (t.data ()); + bool mod (md.type == translation_type::module_iface); + + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + + otype ot (compile_type (t, mod)); + linfo li (link_info (bs, ot)); // Link info for selecting libraries. + compile_target_types tt (compile_types (ot)); + + // Derive file name from target name. + // + string e; // Primary target extension (module or object). + { + const char* o ("o"); // Object extension (.o or .obj). + + if (tsys == "win32-msvc") + { + switch (ot) + { + case otype::e: e = "exe."; break; + case otype::a: e = "lib."; break; + case otype::s: e = "dll."; break; + } + o = "obj"; + } + else if (tsys == "mingw32") + { + switch (ot) + { + case otype::e: e = "exe."; break; + case otype::a: e = "a."; break; + case otype::s: e = "dll."; break; + } + } + else if (tsys == "darwin") + { + switch (ot) + { + case otype::e: e = ""; break; + case otype::a: e = "a."; break; + case otype::s: e = "dylib."; break; + } + } + else + { + switch (ot) + { + case otype::e: e = ""; break; + case otype::a: e = "a."; break; + case otype::s: e = "so."; break; + } + } + + switch (cid) + { + case compiler_id::gcc: + { + e += mod ? "nms" : o; + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + e += mod ? "pcm" : o; + break; + } + case compiler_id::msvc: + { + e += mod ? "ifc" : o; + break; + } + case compiler_id::icc: + { + assert (!mod); + e += o; + } + } + + // If we are compiling a module, then the obj*{} is an ad hoc member + // of bmi*{}. + // + if (mod) + { + // The module interface unit can be the same as an implementation + // (e.g., foo.mxx and foo.cxx) which means obj*{} targets could + // collide. So we add the module extension to the target name. + // + target_lock obj (add_adhoc_member (a, t, tt.obj, e.c_str ())); + obj.target->as ().derive_path (o); + match_recipe (obj, group_recipe); // Set recipe and unlock. + } + } + + const path& tp (t.derive_path (e.c_str ())); + + // Inject dependency on the output directory. + // + const fsdir* dir (inject_fsdir (a, t)); + + // Match all the existing prerequisites. The injection code takes care + // of the ones it is adding. + // + // When cleaning, ignore prerequisites that are not in the same or a + // subdirectory of our project root. + // + auto& pts (t.prerequisite_targets[a]); + optional usr_lib_dirs; // Extract lazily. + + // Start asynchronous matching of prerequisites. Wait with unlocked + // phase to allow phase switching. + // + wait_guard wg (target::count_busy (), t[a].task_count, true); + + size_t start (pts.size ()); // Index of the first to be added. + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target* pt (nullptr); + + // A dependency on a library is there so that we can get its + // *.export.poptions, modules, etc. This is the "library + // meta-information protocol". See also append_lib_options(). + // + if (p.is_a () || + p.is_a () || + p.is_a () || + p.is_a ()) + { + if (a.operation () == update_id) + { + // Handle (phase two) imported libraries. We know that for such + // libraries we don't need to do match() in order to get options + // (if any, they would be set by search_library()). + // + if (p.proj ()) + { + if (search_library (a, + sys_lib_dirs, + usr_lib_dirs, + p.prerequisite) != nullptr) + continue; + } + + pt = &p.search (t); + + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + } + else + continue; + } + // + // For modules we pick only what we import which is done below so + // skip it here. One corner case is clean: we assume that someone + // else (normally library/executable) also depends on it and will + // clean it up. + // + else if (p.is_a () || p.is_a (tt.bmi)) + continue; + else + { + pt = &p.search (t); + + if (a.operation () == clean_id && !pt->dir.sub (rs.out_path ())) + continue; + } + + match_async (a, *pt, target::count_busy (), t[a].task_count); + pts.push_back (pt); + } + + wg.wait (); + + // Finish matching all the targets that we have started. + // + for (size_t i (start), n (pts.size ()); i != n; ++i) + { + const target*& pt (pts[i]); + + // Making sure a library is updated before us will only restrict + // parallelism. But we do need to match it in order to get its imports + // resolved and prerequisite_targets populated. So we match it but + // then unmatch if it is safe. And thanks to the two-pass prerequisite + // match in link::apply() it will be safe unless someone is building + // an obj?{} target directory. + // + if (build2::match ( + a, + *pt, + pt->is_a () || pt->is_a () || pt->is_a () + ? unmatch::safe + : unmatch::none)) + pt = nullptr; // Ignore in execute. + } + + // Inject additional prerequisites. We only do it when performing update + // since chances are we will have to update some of our prerequisites in + // the process (auto-generated source code). + // + if (a == perform_update_id) + { + // The cached prerequisite target should be the same as what is in + // t.prerequisite_targets since we used standard search() and match() + // above. + // + const file& src (*md.src.search (t).is_a ()); + + // Figure out if __symexport is used. While normally it is specified + // on the project root (which we cached), it can be overridden with + // a target-specific value for installed modules (which we sidebuild + // as part of our project). + // + if (modules && src.is_a (*x_mod)) + { + lookup l (src.vars[x_symexport]); + md.symexport = l ? cast (l) : symexport; + } + + // Make sure the output directory exists. + // + // Is this the right thing to do? It does smell a bit, but then we do + // worse things in inject_prerequisites() below. There is also no way + // to postpone this until update since we need to extract and inject + // header dependencies now (we don't want to be calling search() and + // match() in update), which means we need to cache them now as well. + // So the only alternative, it seems, is to cache the updates to the + // database until later which will sure complicate (and slow down) + // things. + // + if (dir != nullptr) + { + // We can do it properly by using execute_direct(). But this means + // we will be switching to the execute phase with all the associated + // overheads. At the same time, in case of update, creation of a + // directory is not going to change the external state in any way + // that would affect any parallel efforts in building the internal + // state. So we are just going to create the directory directly. + // Note, however, that we cannot modify the fsdir{} target since + // this can very well be happening in parallel. But that's not a + // problem since fsdir{}'s update is idempotent. + // + fsdir_rule::perform_update_direct (a, t); + } + + // Note: the leading '@' is reserved for the module map prefix (see + // extract_modules()) and no other line must start with it. + // + md.dd = tp + ".d"; + depdb dd (md.dd); + + // First should come the rule name/version. + // + if (dd.expect (rule_id) != nullptr) + l4 ([&]{trace << "rule mismatch forcing update of " << t;}); + + // Then the compiler checksum. Note that here we assume it + // incorporates the (default) target so that if the compiler changes + // but only in what it targets, then the checksum will still change. + // + if (dd.expect (cast (rs[x_checksum])) != nullptr) + l4 ([&]{trace << "compiler mismatch forcing update of " << t;}); + + // Then the options checksum. + // + // The idea is to keep them exactly as they are passed to the compiler + // since the order may be significant. + // + { + sha256 cs; + + // These flags affect how we compile the source and/or the format of + // depdb so factor them in. + // + cs.append (&md.pp, sizeof (md.pp)); + cs.append (&md.symexport, sizeof (md.symexport)); + + if (md.pp != preprocessed::all) + { + hash_options (cs, t, c_poptions); + hash_options (cs, t, x_poptions); + + // Hash *.export.poptions from prerequisite libraries. + // + hash_lib_options (bs, cs, a, t, li); + + // Extra system header dirs (last). + // + assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); + hash_option_values ( + cs, "-I", + sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), + [] (const dir_path& d) {return d.string ();}); + } + + hash_options (cs, t, c_coptions); + hash_options (cs, t, x_coptions); + hash_options (cs, tstd); + + if (ot == otype::s) + { + // On Darwin, Win32 -fPIC is the default. + // + if (tclass == "linux" || tclass == "bsd") + cs.append ("-fPIC"); + } + + if (dd.expect (cs.string ()) != nullptr) + l4 ([&]{trace << "options mismatch forcing update of " << t;}); + } + + // Finally the source file. + // + if (dd.expect (src.path ()) != nullptr) + l4 ([&]{trace << "source file mismatch forcing update of " << t;}); + + // If any of the above checks resulted in a mismatch (different + // compiler, options, or source file) or if the depdb is newer than + // the target (interrupted update), then do unconditional update. + // + timestamp mt; + bool u (dd.writing () || dd.mtime () > (mt = file_mtime (tp))); + if (u) + mt = timestamp_nonexistent; // Treat as if it doesn't exist. + + // Update prerequisite targets (normally just the source file). + // + // This is an unusual place and time to do it. But we have to do it + // before extracting dependencies. The reasoning for source file is + // pretty clear. What other prerequisites could we have? While + // normally they will be some other sources (as in, static content + // from src_root), it's possible they are some auto-generated stuff. + // And it's possible they affect the preprocessor result. Say some ad + // hoc/out-of-band compiler input file that is passed via the command + // line. So, to be safe, we make sure everything is up to date. + // + for (const target* pt: pts) + { + if (pt == nullptr || pt == dir) + continue; + + u = update (trace, a, *pt, u ? timestamp_unknown : mt) || u; + } + + // Check if the source is already preprocessed to a certain degree. + // This determines which of the following steps we perform and on + // what source (original or preprocessed). + // + // Note: must be set on the src target. + // + if (const string* v = cast_null (src[x_preprocessed])) + try + { + md.pp = to_preprocessed (*v); + } + catch (const invalid_argument& e) + { + fail << "invalid " << x_preprocessed.name << " variable value " + << "for target " << src << ": " << e; + } + + // If we have no #include directives, then skip header dependency + // extraction. + // + pair psrc (auto_rmfile (), false); + if (md.pp < preprocessed::includes) + psrc = extract_headers (a, bs, t, li, src, md, dd, u, mt); + + // Next we "obtain" the translation unit information. What exactly + // "obtain" entails is tricky: If things changed, then we re-parse the + // translation unit. Otherwise, we re-create this information from + // depdb. We, however, have to do it here and now in case the database + // is invalid and we still have to fallback to re-parse. + // + // Store a translation unit's checksum to detect ignorable changes + // (whitespaces, comments, etc). + // + { + string cs; + if (string* l = dd.read ()) + cs = move (*l); + else + u = true; // Database is invalid, force re-parse. + + translation_unit tu; + for (bool f (true);; f = false) + { + if (u) + { + auto p (parse_unit (a, t, li, src, psrc.first, md)); + + if (cs != p.second) + { + assert (f); // Unchanged TU has a different checksum? + dd.write (p.second); + } + else if (f) // Don't clear if it was forced. + { + // Clear the update flag and set the touch flag. Unless there + // is no object file, of course. See also the md.mt logic + // below. + // + if (mt != timestamp_nonexistent) + { + u = false; + md.touch = true; + } + } + + tu = move (p.first); + } + + if (modules) + { + if (u || !f) + { + string s (to_string (tu.mod)); + + if (f) + dd.expect (s); + else + dd.write (s); + } + else + { + if (string* l = dd.read ()) + tu.mod = to_module_info (*l); + else + { + u = true; // Database is invalid, force re-parse. + continue; + } + } + } + + break; + } + + // Make sure the translation unit type matches the resulting target + // type. + // + switch (tu.type ()) + { + case translation_type::plain: + case translation_type::module_impl: + { + if (mod) + fail << "translation unit " << src << " is not a module interface" << + info << "consider using " << x_src.name << "{} instead"; + break; + } + case translation_type::module_iface: + { + if (!mod) + fail << "translation unit " << src << " is a module interface" << + info << "consider using " << x_mod->name << "{} instead"; + break; + } + } + + md.type = tu.type (); + + // Extract the module dependency information in addition to header + // dependencies. + // + // NOTE: assumes that no further targets will be added into + // t.prerequisite_targets! + // + extract_modules (a, bs, t, li, tt, src, md, move (tu.mod), dd, u); + } + + // If anything got updated, then we didn't rely on the cache. However, + // the cached data could actually have been valid and the compiler run + // in extract_headers() as well as the code above merely validated it. + // + // We do need to update the database timestamp, however. Failed that, + // we will keep re-validating the cached data over and over again. + // + if (u && dd.reading ()) + dd.touch (); + + dd.close (); + + // If the preprocessed output is suitable for compilation and is not + // disabled, then pass it along. + // + if (psrc.second && !cast_false (t[c_reprocess])) + { + md.psrc = move (psrc.first); + + // Without modules keeping the (partially) preprocessed output + // around doesn't buy us much: if the source/headers haven't changed + // then neither will the object file. Modules make things more + // interesting: now we may have to recompile an otherwise unchanged + // translation unit because a BMI it depends on has changed. In this + // case re-processing the translation unit would be a waste and + // compiling the original source would break distributed + // compilation. + // + // Note also that the long term trend will (hopefully) be for + // modularized projects to get rid of #include's which means the + // need for producing this partially preprocessed output will + // (hopefully) gradually disappear. + // + if (modules) + md.psrc.active = false; // Keep. + } + + // Above we may have ignored changes to the translation unit. The + // problem is, unless we also update the target's timestamp, we will + // keep re-checking this on subsequent runs and it is not cheap. + // Updating the target's timestamp is not without problems either: it + // will cause a re-link on a subsequent run. So, essentially, we + // somehow need to remember two timestamps: one for checking + // "preprocessor prerequisites" above and one for checking other + // prerequisites (like modules) below. So what we are going to do is + // store the first in the target file (so we do touch it) and the + // second in depdb (which is never newer that the target). + // + md.mt = u ? timestamp_nonexistent : dd.mtime (); + } + + switch (a) + { + case perform_update_id: return [this] (action a, const target& t) + { + return perform_update (a, t); + }; + case perform_clean_id: return [this] (action a, const target& t) + { + return perform_clean (a, t); + }; + default: return noop_recipe; // Configure update. + } + } + + // Reverse-lookup target type from extension. + // + const target_type* compile_rule:: + map_extension (const scope& s, const string& n, const string& e) const + { + // We will just have to try all of the possible ones, in the "most + // likely to match" order. + // + auto test = [&s, &n, &e] (const target_type& tt) -> bool + { + // Call the extension derivation function. Here we know that it will + // only use the target type and name from the target key so we can + // pass bogus values for the rest. + // + target_key tk {&tt, nullptr, nullptr, &n, nullopt}; + + // This is like prerequisite search. + // + if (optional de = tt.default_extension (tk, s, true)) + if (*de == e) + return true; + + return false; + }; + + for (const target_type* const* p (x_inc); *p != nullptr; ++p) + if (test (**p)) return *p; + + return nullptr; + } + + void compile_rule:: + append_prefixes (prefix_map& m, const target& t, const variable& var) const + { + tracer trace (x, "compile_rule::append_prefixes"); + + // If this target does not belong to any project (e.g, an "imported as + // installed" library), then it can't possibly generate any headers for + // us. + // + const scope& bs (t.base_scope ()); + const scope* rs (bs.root_scope ()); + if (rs == nullptr) + return; + + const dir_path& out_base (t.dir); + const dir_path& out_root (rs->out_path ()); + + if (auto l = t[var]) + { + const auto& v (cast (l)); + + for (auto i (v.begin ()), e (v.end ()); i != e; ++i) + { + // -I can either be in the "-Ifoo" or "-I foo" form. For VC it can + // also be /I. + // + const string& o (*i); + + if (o.size () < 2 || (o[0] != '-' && o[0] != '/') || o[1] != 'I') + continue; + + dir_path d; + if (o.size () == 2) + { + if (++i == e) + break; // Let the compiler complain. + + d = dir_path (*i); + } + else + d = dir_path (*i, 2, string::npos); + + l6 ([&]{trace << "-I " << d;}); + + if (d.relative ()) + fail << "relative -I directory " << d + << " in variable " << var.name + << " for target " << t; + + // If we are not inside our project root, then ignore. + // + if (!d.sub (out_root)) + continue; + + // If the target directory is a sub-directory of the include + // directory, then the prefix is the difference between the + // two. Otherwise, leave it empty. + // + // The idea here is to make this "canonical" setup work auto- + // magically: + // + // 1. We include all files with a prefix, e.g., . + // 2. The library target is in the foo/ sub-directory, e.g., + // /tmp/foo/. + // 3. The poptions variable contains -I/tmp. + // + dir_path p (out_base.sub (d) ? out_base.leaf (d) : dir_path ()); + + // We use the target's directory as out_base but that doesn't work + // well for targets that are stashed in subdirectories. So as a + // heuristics we are going to also enter the outer directories of + // the original prefix. It is, however, possible, that another -I + // option after this one will produce one of these outer prefixes as + // its original prefix in which case we should override it. + // + // So we are going to assign the original prefix priority value 0 + // (highest) and then increment it for each outer prefix. + // + auto enter = [&trace, &m] (dir_path p, dir_path d, size_t prio) + { + auto j (m.find (p)); + + if (j != m.end ()) + { + prefix_value& v (j->second); + + // We used to reject duplicates but it seems this can be + // reasonably expected to work according to the order of the + // -I options. + // + // Seeing that we normally have more "specific" -I paths first, + // (so that we don't pick up installed headers, etc), we ignore + // it. + // + if (v.directory == d) + { + if (v.priority > prio) + v.priority = prio; + } + else if (v.priority <= prio) + { + if (verb >= 4) + trace << "ignoring dependency prefix " << p << '\n' + << " existing mapping to " << v.directory + << " priority " << v.priority << '\n' + << " another mapping to " << d + << " priority " << prio; + } + else + { + if (verb >= 4) + trace << "overriding dependency prefix " << p << '\n' + << " existing mapping to " << v.directory + << " priority " << v.priority << '\n' + << " new mapping to " << d + << " priority " << prio; + + v.directory = move (d); + v.priority = prio; + } + } + else + { + l6 ([&]{trace << p << " -> " << d << " priority " << prio;}); + m.emplace (move (p), prefix_value {move (d), prio}); + } + }; + + size_t prio (0); + for (bool e (false); !e; ++prio) + { + dir_path n (p.directory ()); + e = n.empty (); + enter ((e ? move (p) : p), (e ? move (d) : d), prio); + p = move (n); + } + } + } + } + + auto compile_rule:: + build_prefix_map (const scope& bs, + action a, + target& t, + linfo li) const -> prefix_map + { + prefix_map m; + + // First process our own. + // + append_prefixes (m, t, c_poptions); + append_prefixes (m, t, x_poptions); + + // Then process the include directories from prerequisite libraries. + // + append_lib_prefixes (bs, m, a, t, li); + + return m; + } + + // Return the next make prerequisite starting from the specified + // position and update position to point to the start of the + // following prerequisite or l.size() if there are none left. + // + static string + next_make (const string& l, size_t& p) + { + size_t n (l.size ()); + + // Skip leading spaces. + // + for (; p != n && l[p] == ' '; p++) ; + + // Lines containing multiple prerequisites are 80 characters max. + // + string r; + r.reserve (n); + + // Scan the next prerequisite while watching out for escape sequences. + // + for (; p != n && l[p] != ' '; p++) + { + char c (l[p]); + + if (p + 1 != n) + { + if (c == '$') + { + // Got to be another (escaped) '$'. + // + if (l[p + 1] == '$') + ++p; + } + else if (c == '\\') + { + // This may or may not be an escape sequence depending on whether + // what follows is "escapable". + // + switch (c = l[++p]) + { + case '\\': break; + case ' ': break; + default: c = '\\'; --p; // Restore. + } + } + } + + r += c; + } + + // Skip trailing spaces. + // + for (; p != n && l[p] == ' '; p++) ; + + // Skip final '\'. + // + if (p == n - 1 && l[p] == '\\') + p++; + + return r; + } + + // VC /showIncludes output. The first line is the file being compiled + // (handled by our caller). Then we have the list of headers, one per + // line, in this form (text can presumably be translated): + // + // Note: including file: C:\Program Files (x86)\[...]\iostream + // + // Finally, if we hit a non-existent header, then we end with an error + // line in this form: + // + // x.cpp(3): fatal error C1083: Cannot open include file: 'd/h.hpp': + // No such file or directory + // + // Distinguishing between the include note and the include error is + // easy: we can just check for C1083. Distinguising between the note and + // other errors/warnings is harder: an error could very well end with + // what looks like a path so we cannot look for the note but rather have + // to look for an error. Here we assume that a line containing ' CNNNN:' + // is an error. Should be robust enough in the face of language + // translation, etc. + // + // It turns out C1083 is also used when we are unable to open the main + // source file and the error line looks like this: + // + // c1xx: fatal error C1083: Cannot open source file: 's.cpp': No such + // file or directory + + // Sense whether this is an include note (return npos) or a diagnostics + // line (return postion of the NNNN code in CNNNN). + // + static inline size_t + next_show_sense (const string& l) + { + size_t p (l.find (':')); + + for (size_t n (l.size ()); + p != string::npos; + p = ++p != n ? l.find (':', p) : string::npos) + { + auto isnum = [](char c) {return c >= '0' && c <= '9';}; + + if (p > 5 && + l[p - 6] == ' ' && + l[p - 5] == 'C' && + isnum (l[p - 4]) && + isnum (l[p - 3]) && + isnum (l[p - 2]) && + isnum (l[p - 1])) + { + p -= 4; // Start of the error code. + break; + } + } + + return p; + } + + // Extract the include path from the VC /showIncludes output line. Return + // empty string if the line is not an include note or include error. Set + // the good_error flag if it is an include error (which means the process + // will terminate with the error status that needs to be ignored). + // + static string + next_show (const string& l, bool& good_error) + { + // The include error should be the last line that we handle. + // + assert (!good_error); + + size_t p (next_show_sense (l)); + if (p == string::npos) + { + // Include note. We assume the path is always at the end but need to + // handle both absolute Windows and POSIX ones. + // + // Note that VC appears to always write the absolute path to the + // included file even if it is ""-included and the source path is + // relative. Aren't we lucky today? + // + p = l.rfind (':'); + + if (p != string::npos) + { + // See if this one is part of the Windows drive letter. + // + if (p > 1 && p + 1 < l.size () && // 2 chars before, 1 after. + l[p - 2] == ' ' && + alpha (l[p - 1]) && + path::traits::is_separator (l[p + 1])) + p = l.rfind (':', p - 2); + } + + if (p != string::npos) + { + // VC uses indentation to indicate the include nesting so there + // could be any number of spaces after ':'. Skip them. + // + p = l.find_first_not_of (' ', p + 1); + } + + if (p == string::npos) + fail << "unable to parse /showIncludes include note line"; + + return string (l, p); + } + else if (l.compare (p, 4, "1083") == 0 && + l.compare (0, 5, "c1xx:") != 0 /* Not the main source file. */ ) + { + // Include error. The path is conveniently quoted with ''. + // + size_t p2 (l.rfind ('\'')); + + if (p2 != string::npos && p2 != 0) + { + size_t p1 (l.rfind ('\'', p2 - 1)); + + if (p1 != string::npos) + { + good_error = true; + return string (l, p1 + 1 , p2 - p1 - 1); + } + } + + fail << "unable to parse /showIncludes include error line" << endf; + } + else + { + // Some other error. + // + return string (); + } + } + + // Extract and inject header dependencies. Return the preprocessed source + // file as well as an indication if it is usable for compilation (see + // below for details). + // + pair compile_rule:: + extract_headers (action a, + const scope& bs, + file& t, + linfo li, + const file& src, + const match_data& md, + depdb& dd, + bool& updating, + timestamp mt) const + { + tracer trace (x, "compile_rule::extract_headers"); + + l5 ([&]{trace << "target: " << t;}); + + auto_rmfile psrc; + bool puse (true); + + // If things go wrong (and they often do in this area), give the user a + // bit extra context. + // + auto df = make_diag_frame ( + [&src](const diag_record& dr) + { + if (verb != 0) + dr << info << "while extracting header dependencies from " << src; + }); + + const scope& rs (*bs.root_scope ()); + + // Preprocess mode that preserves as much information as possible while + // still performing inclusions. Also serves as a flag indicating whether + // this compiler uses the separate preprocess and compile setup. + // + const char* pp (nullptr); + + switch (cid) + { + case compiler_id::gcc: + { + // -fdirectives-only is available since GCC 4.3.0. + // + if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) + pp = "-fdirectives-only"; + + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // -frewrite-includes is available since vanilla Clang 3.2.0. + // + // Apple Clang 5.0 is based on LLVM 3.3svn so it should have this + // option (4.2 is based on 3.2svc so it may or may not have it and, + // no, we are not going to try to find out). + // + if (cid == compiler_id::clang_apple + ? (cmaj >= 5) + : (cmaj > 3 || (cmaj == 3 && cmin >= 2))) + pp = "-frewrite-includes"; + + break; + } + case compiler_id::msvc: + { + pp = "/C"; + break; + } + case compiler_id::icc: + break; + } + + // Initialize lazily, only if required. + // + environment env; + cstrings args; + string out; // Storage. + + // Some compilers in certain modes (e.g., when also producing the + // preprocessed output) are incapable of writing the dependecy + // information to stdout. In this case we use a temporary file. + // + auto_rmfile drm; + + // Here is the problem: neither GCC nor Clang allow -MG (treat missing + // header as generated) when we produce any kind of other output (-MD). + // And that's probably for the best since otherwise the semantics gets + // pretty hairy (e.g., what is the exit code and state of the output)? + // + // One thing to note about generated headers: if we detect one, then, + // after generating it, we re-run the compiler since we need to get + // this header's dependencies. + // + // So this is how we are going to work around this problem: we first run + // with -E but without -MG. If there are any errors (maybe because of + // generated headers maybe not), we restart with -MG and without -E. If + // this fixes the error (so it was a generated header after all), then + // we have to restart at which point we go back to -E and no -MG. And we + // keep yo-yoing like this. Missing generated headers will probably be + // fairly rare occurrence so this shouldn't be too expensive. + // + // Actually, there is another error case we would like to handle: an + // outdated generated header that is now causing an error (e.g., because + // of a check that is now triggering #error or some such). So there are + // actually three error cases: outdated generated header, missing + // generated header, and some other error. To handle the outdated case + // we need the compiler to produce the dependency information even in + // case of an error. Clang does it, for VC we parse diagnostics + // ourselves, but GCC does not (but a patch has been submitted). + // + // So the final plan is then as follows: + // + // 1. Start wothout -MG and with suppressed diagnostics. + // 2. If error but we've updated a header, then repeat step 1. + // 3. Otherwise, restart with -MG and diagnostics. + // + // Note that below we don't even check if the compiler supports the + // dependency info on error. We just try to use it and if it's not + // there we ignore the io error since the compiler has failed. + // + bool args_gen; // Current state of args. + size_t args_i; // Start of the -M/-MD "tail". + + // Ok, all good then? Not so fast, the rabbit hole is deeper than it + // seems: When we run with -E we have to discard diagnostics. This is + // not a problem for errors since they will be shown on the re-run but + // it is for (preprocessor) warnings. + // + // Clang's -frewrite-includes is nice in that it preserves the warnings + // so they will be shown during the compilation of the preprocessed + // source. They are also shown during -E but that we discard. And unlike + // GCC, in Clang -M does not imply -w (disable warnings) so it would + // have been shown in -M -MG re-runs but we suppress that with explicit + // -w. All is good in the Clang land then (even -Werror works nicely). + // + // GCC's -fdirective-only, on the other hand, processes all the + // directives so they are gone from the preprocessed source. Here is + // what we are going to do to work around this: we will detect if any + // diagnostics has been written to stderr on the -E run. If that's the + // case (but the compiler indicated success) then we assume they are + // warnings and disable the use of the preprocessed output for + // compilation. This in turn will result in compilation from source + // which will display the warnings. Note that we may still use the + // preprocessed output for other things (e.g., C++ module dependency + // discovery). BTW, another option would be to collect all the + // diagnostics and then dump it if the run is successful, similar to + // the VC semantics (and drawbacks) described below. + // + // Finally, for VC, things are completely different: there is no -MG + // equivalent and we handle generated headers by analyzing the + // diagnostics. This means that unlike in the above two cases, the + // preprocessor warnings are shown during dependency extraction, not + // compilation. Not ideal but that's the best we can do. Or is it -- we + // could implement ad hoc diagnostics sensing... It appears warnings are + // in the C4000-C4999 code range though there can also be note lines + // which don't have any C-code. + // + // BTW, triggering a warning in the VC preprocessor is not easy; there + // is no #warning and pragmas are passed through to the compiler. One + // way to do it is to redefine a macro, for example: + // + // hello.cxx(4): warning C4005: 'FOO': macro redefinition + // hello.cxx(3): note: see previous definition of 'FOO' + // + // So seeing that it is hard to trigger a legitimate VC preprocessor + // warning, for now, we will just treat them as errors by adding /WX. + // + // Note: diagnostics sensing is currently only supported if dependency + // info is written to a file (see above). + // + bool sense_diag (false); + + // And here is another problem: if we have an already generated header + // in src and the one in out does not yet exist, then the compiler will + // pick the one in src and we won't even notice. Note that this is not + // only an issue with mixing in- and out-of-tree builds (which does feel + // wrong but is oh so convenient): this is also a problem with + // pre-generated headers, a technique we use to make installing the + // generator by end-users optional by shipping pre-generated headers. + // + // This is a nasty problem that doesn't seem to have a perfect solution + // (except, perhaps, C++ modules). So what we are going to do is try to + // rectify the situation by detecting and automatically remapping such + // mis-inclusions. It works as follows. + // + // First we will build a map of src/out pairs that were specified with + // -I. Here, for performance and simplicity, we will assume that they + // always come in pairs with out first and src second. We build this + // map lazily only if we are running the preprocessor and reuse it + // between restarts. + // + // With the map in hand we can then check each included header for + // potentially having a doppelganger in the out tree. If this is the + // case, then we calculate a corresponding header in the out tree and, + // (this is the most important part), check if there is a target for + // this header in the out tree. This should be fairly accurate and not + // require anything explicit from the user except perhaps for a case + // where the header is generated out of nothing (so there is no need to + // explicitly mention its target in the buildfile). But this probably + // won't be very common. + // + // One tricky area in this setup are target groups: if the generated + // sources are mentioned in the buildfile as a group, then there might + // be no header target (yet). The way we solve this is by requiring code + // generator rules to cooperate and create at least the header target as + // part of the group creation. While not all members of the group may be + // generated depending on the options (e.g., inline files might be + // suppressed), headers are usually non-optional. + // + // Note that we use path_map instead of dir_path_map to allow searching + // using path (file path). + // + using srcout_map = path_map; + srcout_map so_map; + + // The gen argument to init_args() is in/out. The caller signals whether + // to force the generated header support and on return it signals + // whether this support is enabled. The first call to init_args is + // expected to have gen false. + // + // Return NULL if the dependency information goes to stdout and a + // pointer to the temporary file path otherwise. + // + auto init_args = [&t, a, li, + &src, &md, &psrc, &sense_diag, + &rs, &bs, + pp, &env, &args, &args_gen, &args_i, &out, &drm, + &so_map, this] + (bool& gen) -> const path* + { + const path* r (nullptr); + + if (args.empty ()) // First call. + { + assert (!gen); + + // We use absolute/relative paths in the dependency output to + // distinguish existing headers from (missing) generated. Which + // means we have to (a) use absolute paths in -I and (b) pass + // absolute source path (for ""-includes). That (b) is a problem: + // if we use an absolute path, then all the #line directives will be + // absolute and all the diagnostics will have long, noisy paths + // (actually, we will still have long paths for diagnostics in + // headers). + // + // To work around this we used to pass a relative path to the source + // file and then check every relative path in the dependency output + // for existence in the source file's directory. This is not without + // issues: it is theoretically possible for a generated header that + // is <>-included and found via -I to exist in the source file's + // directory. Note, however, that this is a lot more likely to + // happen with prefix-less inclusion (e.g., ) and in this case + // we assume the file is in the project anyway. And if there is a + // conflict with a prefixed include (e.g., ), then, well, + // we will just have to get rid of quoted includes (which are + // generally a bad idea, anyway). + // + // But then this approach (relative path) fell apart further when we + // tried to implement precise changed detection: the preprocessed + // output would change depending from where it was compiled because + // of #line (which we could work around) and __FILE__/assert() + // (which we can't really do anything about). So it looks like using + // the absolute path is the lesser of all the evils (and there are + // many). + // + // Note that we detect and diagnose relative -I directories lazily + // when building the include prefix map. + // + args.push_back (cpath.recall_string ()); + + // Add *.export.poptions from prerequisite libraries. + // + append_lib_options (bs, args, a, t, li); + + append_options (args, t, c_poptions); + append_options (args, t, x_poptions); + + // Populate the src-out with the -I$out_base -I$src_base pairs. + // + { + // Try to be fast and efficient by reusing buffers as much as + // possible. + // + string ds; + + // Previous -I innermost scope if out_base plus the difference + // between the scope path and the -I path (normally empty). + // + const scope* s (nullptr); + dir_path p; + + for (auto i (args.begin ()), e (args.end ()); i != e; ++i) + { + // -I can either be in the "-Ifoo" or "-I foo" form. For VC it + // can also be /I. + // + const char* o (*i); + size_t n (strlen (o)); + + if (n < 2 || (o[0] != '-' && o[0] != '/') || o[1] != 'I') + { + s = nullptr; + continue; + } + + if (n == 2) + { + if (++i == e) + break; // Let the compiler complain. + + ds = *i; + } + else + ds.assign (o + 2, n - 2); + + if (!ds.empty ()) + { + // Note that we don't normalize the paths since it would be + // quite expensive and normally the pairs we are inerested in + // are already normalized (since they are usually specified as + // -I$src/out_*). We just need to add a trailing directory + // separator if it's not already there. + // + if (!dir_path::traits::is_separator (ds.back ())) + ds += dir_path::traits::directory_separator; + + dir_path d (move (ds), dir_path::exact); // Move the buffer in. + + // Ignore invalid paths (buffer is not moved). + // + if (!d.empty ()) + { + // Ignore any paths containing '.', '..' components. Allow + // any directory separators thought (think -I$src_root/foo + // on Windows). + // + if (d.absolute () && d.normalized (false)) + { + // If we have a candidate out_base, see if this is its + // src_base. + // + if (s != nullptr) + { + const dir_path& bp (s->src_path ()); + + if (d.sub (bp)) + { + if (p.empty () || d.leaf (bp) == p) + { + // We've got a pair. + // + so_map.emplace (move (d), s->out_path () / p); + s = nullptr; // Taken. + continue; + } + } + + // Not a pair. Fall through to consider as out_base. + // + s = nullptr; + } + + // See if this path is inside a project with an out-of- + // tree build and is in the out directory tree. + // + const scope& bs (scopes.find (d)); + if (bs.root_scope () != nullptr) + { + const dir_path& bp (bs.out_path ()); + if (bp != bs.src_path ()) + { + bool e; + if ((e = (d == bp)) || d.sub (bp)) + { + s = &bs; + if (e) + p.clear (); + else + p = d.leaf (bp); + } + } + } + } + else + s = nullptr; + + ds = move (d).string (); // Move the buffer out. + } + else + s = nullptr; + } + else + s = nullptr; + } + } + + // Extra system header dirs (last). + // + assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); + append_option_values ( + args, "-I", + sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), + [] (const dir_path& d) {return d.string ().c_str ();}); + + if (md.symexport) + append_symexport_options (args, t); + + // Some compile options (e.g., -std, -m) affect the preprocessor. + // + // Currently Clang supports importing "header modules" even when in + // the TS mode. And "header modules" support macros which means + // imports have to be resolved during preprocessing. Which poses a + // bit of a chicken and egg problem for us. For now, the workaround + // is to remove the -fmodules-ts option when preprocessing. Hopefully + // there will be a "pure modules" mode at some point. + // + + // Don't treat warnings as errors. + // + const char* werror (nullptr); + switch (cclass) + { + case compiler_class::gcc: werror = "-Werror"; break; + case compiler_class::msvc: werror = "/WX"; break; + } + + bool clang (cid == compiler_id::clang || + cid == compiler_id::clang_apple); + + append_options (args, t, c_coptions, werror); + append_options (args, t, x_coptions, werror); + append_options (args, tstd, + tstd.size () - (modules && clang ? 1 : 0)); + + switch (cclass) + { + case compiler_class::msvc: + { + assert (pp != nullptr); + + args.push_back ("/nologo"); + + // See perform_update() for details on overriding the default + // exceptions and runtime. + // + if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) + args.push_back ("/EHsc"); + + if (!find_option_prefixes ({"/MD", "/MT"}, args)) + args.push_back ("/MD"); + + args.push_back ("/P"); // Preprocess to file. + args.push_back ("/showIncludes"); // Goes to stdout (with diag). + args.push_back (pp); // /C (preserve comments). + args.push_back ("/WX"); // Warning as error (see above). + + psrc = auto_rmfile (t.path () + x_pext); + + if (cast (rs[x_version_major]) >= 18) + { + args.push_back ("/Fi:"); + args.push_back (psrc.path.string ().c_str ()); + } + else + { + out = "/Fi" + psrc.path.string (); + args.push_back (out.c_str ()); + } + + args.push_back (langopt (md)); // Compile as. + gen = args_gen = true; + break; + } + case compiler_class::gcc: + { + if (t.is_a ()) + { + // On Darwin, Win32 -fPIC is the default. + // + if (tclass == "linux" || tclass == "bsd") + args.push_back ("-fPIC"); + } + + // Depending on the compiler, decide whether (and how) we can + // produce preprocessed output as a side effect of dependency + // extraction. + // + // Note: -MM -MG skips missing <>-included. + + // Clang's -M does not imply -w (disable warnings). We also + // don't need them in the -MD case (see above) so disable for + // both. + // + if (clang) + args.push_back ("-w"); + + // Previously we used '*' as a target name but it gets expanded + // to the current directory file names by GCC (4.9) that comes + // with MSYS2 (2.4). Yes, this is the (bizarre) behavior of GCC + // being executed in the shell with -MQ '*' option and not just + // -MQ *. + // + args.push_back ("-MQ"); // Quoted target name. + args.push_back ("^"); // Old versions can't do empty target. + + args.push_back ("-x"); + args.push_back (langopt (md)); + + if (pp != nullptr) + { + // Note that the options are carefully laid out to be easy to + // override (see below). + // + args_i = args.size (); + + args.push_back ("-MD"); + args.push_back ("-E"); + args.push_back (pp); + + // Dependency output. + // + args.push_back ("-MF"); + + // GCC is not capable of writing the dependency info to + // stdout. We also need to sense the diagnostics on the -E + // runs. + // + if (cid == compiler_id::gcc) + { + // Use the .t extension (for "temporary"; .d is taken). + // + r = &(drm = auto_rmfile (t.path () + ".t")).path; + args.push_back (r->string ().c_str ()); + + sense_diag = true; + } + else + args.push_back ("-"); + + // Preprocessor output. + // + psrc = auto_rmfile (t.path () + x_pext); + args.push_back ("-o"); + args.push_back (psrc.path.string ().c_str ()); + } + else + { + args.push_back ("-M"); + args.push_back ("-MG"); // Treat missing headers as generated. + } + + gen = args_gen = (pp == nullptr); + break; + } + } + + args.push_back (src.path ().string ().c_str ()); + args.push_back (nullptr); + + // Note: only doing it here. + // + if (!env.empty ()) + env.push_back (nullptr); + } + else + { + assert (gen != args_gen); + + size_t i (args_i); + + if (gen) + { + // Overwrite. + // + args[i++] = "-M"; + args[i++] = "-MG"; + args[i++] = src.path ().string ().c_str (); + args[i] = nullptr; + + if (cid == compiler_id::gcc) + { + sense_diag = false; + } + } + else + { + // Restore. + // + args[i++] = "-MD"; + args[i++] = "-E"; + args[i++] = pp; + args[i] = "-MF"; + + if (cid == compiler_id::gcc) + { + r = &drm.path; + sense_diag = true; + } + } + + args_gen = gen; + } + + return r; + }; + + // Build the prefix map lazily only if we have non-existent files. + // Also reuse it over restarts since it doesn't change. + // + optional pfx_map; + + // If any prerequisites that we have extracted changed, then we have to + // redo the whole thing. The reason for this is auto-generated headers: + // the updated header may now include a yet-non-existent header. Unless + // we discover this and generate it (which, BTW, will trigger another + // restart since that header, in turn, can also include auto-generated + // headers), we will end up with an error during compilation proper. + // + // One complication with this restart logic is that we will see a + // "prefix" of prerequisites that we have already processed (i.e., they + // are already in our prerequisite_targets list) and we don't want to + // keep redoing this over and over again. One thing to note, however, is + // that the prefix that we have seen on the previous run must appear + // exactly the same in the subsequent run. The reason for this is that + // none of the files that it can possibly be based on have changed and + // thus it should be exactly the same. To put it another way, the + // presence or absence of a file in the dependency output can only + // depend on the previous files (assuming the compiler outputs them as + // it encounters them and it is hard to think of a reason why would + // someone do otherwise). And we have already made sure that all those + // files are up to date. And here is the way we are going to exploit + // this: we are going to keep track of how many prerequisites we have + // processed so far and on restart skip right to the next one. + // + // And one more thing: most of the time this list of headers would stay + // unchanged and extracting them by running the compiler every time is a + // bit wasteful. So we are going to cache them in the depdb. If the db + // hasn't been invalidated yet (e.g., because the compiler options have + // changed), then we start by reading from it. If anything is out of + // date then we use the same restart and skip logic to switch to the + // compiler run. + // + size_t skip_count (0); + + // Update and add a header file to the list of prerequisite targets. + // Depending on the cache flag, the file is assumed to either have come + // from the depdb cache or from the compiler run. Return whether the + // extraction process should be restarted. + // + auto add = [&trace, &pfx_map, &so_map, + a, &t, li, + &dd, &updating, &skip_count, + &bs, this] + (path f, bool cache, timestamp mt) -> bool + { + // Find or maybe insert the target. The directory is only moved + // from if insert is true. + // + auto find = [&trace, &t, this] + (dir_path&& d, path&& f, bool insert) -> const path_target* + { + // Split the file into its name part and extension. Here we can + // assume the name part is a valid filesystem name. + // + // Note that if the file has no extension, we record an empty + // extension rather than NULL (which would signify that the default + // extension should be added). + // + string e (f.extension ()); + string n (move (f).string ()); + + if (!e.empty ()) + n.resize (n.size () - e.size () - 1); // One for the dot. + + // Determine the target type. + // + const target_type* tt (nullptr); + + // See if this directory is part of any project out_root hierarchy. + // Note that this will miss all the headers that come from src_root + // (so they will be treated as generic C headers below). Generally, + // we don't have the ability to determine that some file belongs to + // src_root of some project. But that's not a problem for our + // purposes: it is only important for us to accurately determine + // target types for headers that could be auto-generated. + // + // While at it also try to determine if this target is from the src + // or out tree of said project. + // + dir_path out; + + const scope& bs (scopes.find (d)); + if (const scope* rs = bs.root_scope ()) + { + tt = map_extension (bs, n, e); + + if (bs.out_path () != bs.src_path () && d.sub (bs.src_path ())) + out = out_src (d, *rs); + } + + // If it is outside any project, or the project doesn't have such an + // extension, assume it is a plain old C header. + // + if (tt == nullptr) + { + // If the project doesn't "know" this extension then we won't + // possibly find an explicit target of this type. + // + if (!insert) + return nullptr; + + tt = &h::static_type; + } + + // Find or insert target. + // + // @@ OPT: move d, out, n + // + const target* r; + if (insert) + r = &search (t, *tt, d, out, n, &e, nullptr); + else + { + // Note that we skip any target type-specific searches (like for + // an existing file) and go straight for the target object since + // we need to find the target explicitly spelled out. + // + r = targets.find (*tt, d, out, n, e, trace); + } + + return static_cast (r); + }; + + // If it's not absolute then it either does not (yet) exist or is + // a relative ""-include (see init_args() for details). Reduce the + // second case to absolute. + // + // Note: we now always use absolute path to the translation unit so + // this no longer applies. + // +#if 0 + if (f.relative () && rels.relative ()) + { + // If the relative source path has a directory component, make sure + // it matches since ""-include will always start with that (none of + // the compilers we support try to normalize this path). Failed that + // we may end up searching for a generated header in a random + // (working) directory. + // + const string& fs (f.string ()); + const string& ss (rels.string ()); + + size_t p (path::traits::rfind_separator (ss)); + + if (p == string::npos || // No directory. + (fs.size () > p + 1 && + path::traits::compare (fs.c_str (), p, ss.c_str (), p) == 0)) + { + path t (work / f); // The rels path is relative to work. + + if (exists (t)) + f = move (t); + } + } +#endif + + const path_target* pt (nullptr); + + // If still relative then it does not exist. + // + if (f.relative ()) + { + f.normalize (); + + // This is probably as often an error as an auto-generated file, so + // trace at level 4. + // + l4 ([&]{trace << "non-existent header '" << f << "'";}); + + if (!pfx_map) + pfx_map = build_prefix_map (bs, a, t, li); + + // First try the whole file. Then just the directory. + // + // @@ Has to be a separate map since the prefix can be the same as + // the file name. + // + // auto i (pfx_map->find (f)); + + // Find the most qualified prefix of which we are a sub-path. + // + if (!pfx_map->empty ()) + { + dir_path d (f.directory ()); + auto i (pfx_map->find_sup (d)); + + if (i != pfx_map->end ()) + { + const dir_path& pd (i->second.directory); + + // If this is a prefixless mapping, then only use it if we can + // resolve it to an existing target (i.e., it is explicitly + // spelled out in a buildfile). + // + // Note that at some point we will probably have a list of + // directories. + // + pt = find (pd / d, f.leaf (), !i->first.empty ()); + if (pt != nullptr) + { + f = pd / f; + l4 ([&]{trace << "mapped as auto-generated " << f;}); + } + } + } + + if (pt == nullptr) + { + diag_record dr (fail); + dr << "header '" << f << "' not found and cannot be generated"; + //for (const auto& p: pm) + // dr << info << p.first.string () << " -> " << p.second.string (); + } + } + else + { + // We used to just normalize the path but that could result in an + // invalid path (e.g., on CentOS 7 with Clang 3.4) because of the + // symlinks. So now we realize (i.e., realpath(3)) it instead. + // Unless it comes from the depdb, in which case we've already done + // that. This is also where we handle src-out remap (again, not + // needed if cached) + // + if (!cache) + { + // While we can reasonably expect this path to exit, things do + // go south from time to time (like compiling under wine with + // file wlantypes.h included as WlanTypes.h). + // + try + { + f.realize (); + } + catch (const invalid_path&) + { + fail << "invalid header path '" << f << "'"; + } + catch (const system_error& e) + { + fail << "invalid header path '" << f << "': " << e; + } + + if (!so_map.empty ()) + { + // Find the most qualified prefix of which we are a sub-path. + // + auto i (so_map.find_sup (f)); + if (i != so_map.end ()) + { + // Ok, there is an out tree for this headers. Remap to a path + // from the out tree and see if there is a target for it. + // + dir_path d (i->second); + d /= f.leaf (i->first).directory (); + pt = find (move (d), f.leaf (), false); // d is not moved from. + + if (pt != nullptr) + { + path p (d / f.leaf ()); + l4 ([&]{trace << "remapping " << f << " to " << p;}); + f = move (p); + } + } + } + } + + if (pt == nullptr) + { + l6 ([&]{trace << "injecting " << f;}); + pt = find (f.directory (), f.leaf (), true); + } + } + + // Cache the path. + // + const path& pp (pt->path (move (f))); + + // Match to a rule. + // + // If we are reading the cache, then it is possible the file has since + // been removed (think of a header in /usr/local/include that has been + // uninstalled and now we need to use one from /usr/include). This + // will lead to the match failure which we translate to a restart. + // + if (!cache) + build2::match (a, *pt); + else if (!build2::try_match (a, *pt).first) + { + dd.write (); // Invalidate this line. + updating = true; + return true; + } + + // Update. + // + bool restart (update (trace, a, *pt, mt)); + + // Verify/add it to the dependency database. We do it after update in + // order not to add bogus files (non-existent and without a way to + // update). + // + if (!cache) + dd.expect (pp); + + // Add to our prerequisite target list. + // + t.prerequisite_targets[a].push_back (pt); + skip_count++; + + updating = updating || restart; + return restart; + }; + + // If nothing so far has invalidated the dependency database, then try + // the cached data before running the compiler. + // + bool cache (!updating); + + // See init_args() above for details on generated header support. + // + bool gen (false); + optional force_gen; + optional force_gen_skip; // Skip count at last force_gen run. + + const path* drmp (nullptr); // Points to drm.path () if active. + + for (bool restart (true); restart; cache = false) + { + restart = false; + + if (cache) + { + // If any, this is always the first run. + // + assert (skip_count == 0); + + // We should always end with a blank line. + // + for (;;) + { + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + restart = true; + break; + } + + if (l->empty ()) // Done, nothing changed. + { + // If modules are enabled, then we keep the preprocessed output + // around (see apply() for details). + // + return modules + ? make_pair (auto_rmfile (t.path () + x_pext, false), true) + : make_pair (auto_rmfile (), false); + } + + // If this header came from the depdb, make sure it is no older + // than the target (if it has changed since the target was + // updated, then the cached data is stale). + // + restart = add (path (move (*l)), true, mt); + + if (restart) + { + l6 ([&]{trace << "restarting (cache)";}); + break; + } + } + } + else + { + try + { + if (force_gen) + gen = *force_gen; + + if (args.empty () || gen != args_gen) + drmp = init_args (gen); + + if (verb >= 3) + print_process (args.data ()); // Disable pipe mode. + + process pr; + + try + { + // Assume the preprocessed output (if produced) is usable + // until proven otherwise. + // + puse = true; + + // Save the timestamp just before we start preprocessing. If + // we depend on any header that has been updated since, then + // we should assume we've "seen" the old copy and re-process. + // + timestamp pmt (system_clock::now ()); + + // If we have no generated header support, then suppress all + // diagnostics (if things go badly we will restart with this + // support). + // + if (drmp == nullptr) + { + // Dependency info goes to stdout. + // + assert (!sense_diag); + + // For VC with /P the dependency info and diagnostics all go + // to stderr so redirect it to stdout. + // + pr = process ( + cpath, + args.data (), + 0, + -1, + cclass == compiler_class::msvc ? 1 : gen ? 2 : -2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + } + else + { + // Dependency info goes to a temporary file. + // + pr = process (cpath, + args.data (), + 0, + 2, // Send stdout to stderr. + gen ? 2 : sense_diag ? -1 : -2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + + // If requested, monitor for diagnostics and if detected, mark + // the preprocessed output as unusable for compilation. + // + if (sense_diag) + { + ifdstream is (move (pr.in_efd), fdstream_mode::skip); + puse = puse && (is.peek () == ifdstream::traits_type::eof ()); + is.close (); + } + + // The idea is to reduce it to the stdout case. + // + pr.wait (); + pr.in_ofd = fdopen (*drmp, fdopen_mode::in); + } + + // We may not read all the output (e.g., due to a restart). + // Before we used to just close the file descriptor to signal to + // the other end that we are not interested in the rest. This + // works fine with GCC but Clang (3.7.0) finds this impolite and + // complains, loudly (broken pipe). So now we are going to skip + // until the end. + // + ifdstream is (move (pr.in_ofd), + fdstream_mode::text | fdstream_mode::skip, + ifdstream::badbit); + + // In some cases we may need to ignore the error return status. + // The good_error flag keeps track of that. Similarly we + // sometimes expect the error return status based on the output + // we see. The bad_error flag is for that. + // + bool good_error (false), bad_error (false); + + size_t skip (skip_count); + string l; // Reuse. + for (bool first (true), second (false); !restart; ) + { + if (eof (getline (is, l))) + break; + + l6 ([&]{trace << "header dependency line '" << l << "'";}); + + // Parse different dependency output formats. + // + switch (cclass) + { + case compiler_class::msvc: + { + if (first) + { + // The first line should be the file we are compiling. + // If it is not, then something went wrong even before + // we could compile anything (e.g., file does not + // exist). In this case the first line (and everything + // after it) is presumably diagnostics. + // + if (l != src.path ().leaf ().string ()) + { + text << l; + bad_error = true; + break; + } + + first = false; + continue; + } + + string f (next_show (l, good_error)); + + if (f.empty ()) // Some other diagnostics. + { + text << l; + bad_error = true; + break; + } + + // Skip until where we left off. + // + if (skip != 0) + { + // We can't be skipping over a non-existent header. + // + assert (!good_error); + skip--; + } + else + { + restart = add (path (move (f)), false, pmt); + + // If the header does not exist (good_error), then + // restart must be true. Except that it is possible that + // someone running in parallel has already updated it. + // In this case we must force a restart since we haven't + // yet seen what's after this at-that-time-non-existent + // header. + // + // We also need to force the target update (normally + // done by add()). + // + if (good_error) + restart = updating = true; + // + // And if we have updated the header (restart is true), + // then we may end up in this situation: an old header + // got included which caused the preprocessor to fail + // down the line. So if we are restarting, set the good + // error flag in case the process fails because of + // something like this (and if it is for a valid reason, + // then we will pick it up on the next round). + // + else if (restart) + good_error = true; + + if (restart) + l6 ([&]{trace << "restarting";}); + } + + break; + } + case compiler_class::gcc: + { + // Make dependency declaration. + // + size_t pos (0); + + if (first) + { + // Empty/invalid output should mean the wait() call + // below will return false. + // + if (l.empty () || + l[0] != '^' || l[1] != ':' || l[2] != ' ') + { + if (!l.empty ()) + text << l; + + bad_error = true; + break; + } + + first = false; + second = true; + + // While normally we would have the source file on the + // first line, if too long, it will be moved to the next + // line and all we will have on this line is "^: \". + // + if (l.size () == 4 && l[3] == '\\') + continue; + else + pos = 3; // Skip "^: ". + + // Fall through to the 'second' block. + } + + if (second) + { + second = false; + next_make (l, pos); // Skip the source file. + } + + while (pos != l.size ()) + { + string f (next_make (l, pos)); + + // Skip until where we left off. + // + if (skip != 0) + { + skip--; + continue; + } + + restart = add (path (move (f)), false, pmt); + + if (restart) + { + // The same "preprocessor may fail down the line" + // logic as above. + // + good_error = true; + + l6 ([&]{trace << "restarting";}); + break; + } + } + + break; + } + } + + if (bad_error) + break; + } + + // In case of VC, we are parsing stderr and if things go south, + // we need to copy the diagnostics for the user to see. + // + if (bad_error && cclass == compiler_class::msvc) + { + // We used to just dump the whole rdbuf but it turns out VC + // may continue writing include notes interleaved with the + // diagnostics. So we have to filter them out. + // + for (; !eof (getline (is, l)); ) + { + size_t p (next_show_sense (l)); + if (p != string::npos && l.compare (p, 4, "1083") != 0) + diag_stream_lock () << l << endl; + } + } + + is.close (); + + // This is tricky: it is possible that in parallel someone has + // generated all our missing headers and we wouldn't restart + // normally. + // + // In this case we also need to force the target update + // (normally done by add()). + // + if (force_gen && *force_gen) + { + restart = updating = true; + force_gen = false; + } + + if (pr.wait ()) + { + if (!bad_error) + continue; + + fail << "expected error exist status from " << x_lang + << " compiler"; + } + else if (pr.exit->normal ()) + { + if (good_error) // Ignore expected errors (restart). + continue; + } + + // Fall through. + } + catch (const io_error&) + { + if (pr.wait ()) + fail << "unable to read " << x_lang << " compiler header " + << "dependency output"; + + // Fall through. + } + + assert (pr.exit && !*pr.exit); + const process_exit& e (*pr.exit); + + // For normal exit we assume the child process issued some + // diagnostics. + // + if (e.normal ()) + { + // If this run was with the generated header support then we + // have issued diagnostics and it's time to give up. + // + if (gen) + throw failed (); + + // Just to recap, being here means something is wrong with the + // source: it can be a missing generated header, it can be an + // outdated generated header (e.g., some check triggered #error + // which will go away if only we updated the generated header), + // or it can be a real error that is not going away. + // + // So this is what we are going to do here: if anything got + // updated on this run (i.e., the compiler has produced valid + // dependency information even though there were errors and we + // managed to find and update a header based on this + // informaion), then we restart in the same mode hoping that + // this fixes things. Otherwise, we force the generated header + // support which will either uncover a missing generated header + // or will issue diagnostics. + // + if (restart) + l6 ([&]{trace << "trying again without generated headers";}); + else + { + // In some pathological situations (e.g., we are out of disk + // space) we may end up switching back and forth indefinitely + // without making any headway. So we use skip_count to track + // our progress. + // + if (force_gen_skip && *force_gen_skip == skip_count) + { + diag_record dr (fail); + + dr << "inconsistent " << x_lang << " compiler behavior"; + + // Show the yo-yo'ing command lines. + // + dr << info; + print_process (dr, args.data ()); // No pipes. + + init_args ((gen = true)); + dr << info << ""; + print_process (dr, args.data ()); // No pipes. + + dr << info << "perhaps you are running out of disk space?"; + } + + restart = true; + force_gen = true; + force_gen_skip = skip_count; + l6 ([&]{trace << "restarting with forced generated headers";}); + } + continue; + } + else + run_finish (args, pr); // Throws. + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + // In a multi-threaded program that fork()'ed but did not exec(), + // it is unwise to try to do any kind of cleanup (like unwinding + // the stack and running destructors). + // + if (e.child) + { + drm.cancel (); + exit (1); + } + + throw failed (); + } + } + } + + // Add the terminating blank line (we are updated depdb). + // + dd.expect (""); + + puse = puse && !psrc.path.empty (); + return make_pair (move (psrc), puse); + } + + pair compile_rule:: + parse_unit (action a, + file& t, + linfo lo, + const file& src, + auto_rmfile& psrc, + const match_data& md) const + { + tracer trace (x, "compile_rule::parse_unit"); + + // If things go wrong give the user a bit extra context. + // + auto df = make_diag_frame ( + [&src](const diag_record& dr) + { + if (verb != 0) + dr << info << "while parsing " << src; + }); + + // For some compilers (GCC, Clang) the preporcessed output is only + // partially preprocessed. For others (VC), it is already fully + // preprocessed (well, almost: it still has comments but we can handle + // that). Plus, the source file might already be (sufficiently) + // preprocessed. + // + // So the plan is to start the compiler process that writes the fully + // preprocessed output to stdout and reduce the already preprocessed + // case to it. + // + environment env; + cstrings args; + const path* sp; // Source path. + + bool ps; // True if extracting from psrc. + if (md.pp < preprocessed::modules) + { + ps = !psrc.path.empty (); + sp = &(ps ? psrc.path : src.path ()); + + // VC's preprocessed output, if present, is fully preprocessed. + // + if (cclass != compiler_class::msvc || !ps) + { + // This should match with how we setup preprocessing and is pretty + // similar to init_args() from extract_headers(). + // + args.push_back (cpath.recall_string ()); + + append_lib_options (t.base_scope (), args, a, t, lo); + + append_options (args, t, c_poptions); + append_options (args, t, x_poptions); + + assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); + append_option_values ( + args, "-I", + sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), + [] (const dir_path& d) {return d.string ().c_str ();}); + + if (md.symexport) + append_symexport_options (args, t); + + // Make sure we don't fail because of warnings. + // + // @@ Can be both -WX and /WX. + // + const char* werror (nullptr); + switch (cclass) + { + case compiler_class::gcc: werror = "-Werror"; break; + case compiler_class::msvc: werror = "/WX"; break; + } + + bool clang (cid == compiler_id::clang || + cid == compiler_id::clang_apple); + + append_options (args, t, c_coptions, werror); + append_options (args, t, x_coptions, werror); + append_options (args, tstd, + tstd.size () - (modules && clang ? 1 : 0)); + + switch (cclass) + { + case compiler_class::msvc: + { + args.push_back ("/nologo"); + + if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) + args.push_back ("/EHsc"); + + if (!find_option_prefixes ({"/MD", "/MT"}, args)) + args.push_back ("/MD"); + + args.push_back ("/E"); + args.push_back ("/C"); + args.push_back (langopt (md)); // Compile as. + + break; + } + case compiler_class::gcc: + { + if (t.is_a ()) + { + if (tclass == "linux" || tclass == "bsd") + args.push_back ("-fPIC"); + } + + // Options that trigger preprocessing of partially preprocessed + // output are a bit of a compiler-specific voodoo. + // + args.push_back ("-E"); + + if (ps) + { + args.push_back ("-x"); + args.push_back (langopt (md)); + + if (cid == compiler_id::gcc) + { + args.push_back ("-fpreprocessed"); + args.push_back ("-fdirectives-only"); + } + } + + break; + } + } + + args.push_back (sp->string ().c_str ()); + args.push_back (nullptr); + } + + if (!env.empty ()) + env.push_back (nullptr); + } + else + { + // Extracting directly from source. + // + ps = false; + sp = &src.path (); + } + + // Preprocess and parse. + // + for (;;) // Breakout loop. + try + { + // Disarm the removal of the preprocessed file in case of an error. + // We re-arm it below. + // + if (ps) + psrc.active = false; + + process pr; + + try + { + if (args.empty ()) + { + pr = process (process_exit (0)); // Successfully exited. + pr.in_ofd = fdopen (*sp, fdopen_mode::in); + } + else + { + if (verb >= 3) + print_process (args); + + // We don't want to see warnings multiple times so ignore all + // diagnostics. + // + pr = process (cpath, + args.data (), + 0, -1, -2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + } + + // Use binary mode to obtain consistent positions. + // + ifdstream is (move (pr.in_ofd), + fdstream_mode::binary | fdstream_mode::skip); + + parser p; + translation_unit tu (p.parse (is, *sp)); + + is.close (); + + if (pr.wait ()) + { + if (ps) + psrc.active = true; // Re-arm. + + // Prior to 15u5 VC was not using the 'export module' syntax so we + // use the preprequisite type to distinguish between interface and + // implementation units. + // + if (cid == compiler_id::msvc && + cmaj == 19 && cmin <= 11 && + x_mod != nullptr && src.is_a (*x_mod)) + { + // It's quite painful to guard the export with an #if/#endif so + // if it is present, "fixup" the (temporary) preprocessed output + // by getting rid of the keyword. + // + // Note: when removing this also remember to remove the test. + // + if (tu.mod.iface) + { + // We can only fixup a temporary file. + // + if (!ps) + fail (relative (src)) << "fixup requires preprocessor"; + + // Stomp out the export keyword with spaces. We are using + // std::fstream since our fdstream does not yet support + // seeking. + // + fstream os (psrc.path.string (), fstream::out | fstream::in); + auto pos (static_cast (p.export_pos)); + + if (!os.is_open () || + !os.seekp (pos) || + !os.write (" ", 6)) + fail << "unable to overwrite preprocessor output"; + } + else + tu.mod.iface = true; + } + + return pair (move (tu), p.checksum); + } + + // Fall through. + } + catch (const io_error&) + { + if (pr.wait ()) + fail << "unable to read " << x_lang << " preprocessor output"; + + // Fall through. + } + + assert (pr.exit && !*pr.exit); + const process_exit& e (*pr.exit); + + // What should we do with a normal error exit? Remember we suppressed + // the compiler's diagnostics. We used to issue a warning and continue + // with the assumption that the compilation step will fail with + // diagnostics. The problem with this approach is that we may fail + // before that because the information we return (e.g., module name) + // is bogus. So looks like failing is the only option. + // + if (e.normal ()) + { + fail << "unable to preprocess " << src << + info << "re-run with -s -V to display failing command" << + info << "then run failing command to display compiler diagnostics"; + } + else + run_finish (args, pr); // Throws. + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + } + + throw failed (); + } + + // Extract and inject module dependencies. + // + void compile_rule:: + extract_modules (action a, + const scope& bs, + file& t, + linfo li, + const compile_target_types& tt, + const file& src, + match_data& md, + module_info&& mi, + depdb& dd, + bool& updating) const + { + tracer trace (x, "compile_rule::extract_modules"); + l5 ([&]{trace << "target: " << t;}); + + // If things go wrong, give the user a bit extra context. + // + auto df = make_diag_frame ( + [&src](const diag_record& dr) + { + if (verb != 0) + dr << info << "while extracting module dependencies from " << src; + }); + + if (!modules) + { + if (!mi.name.empty () || !mi.imports.empty ()) + fail (relative (src)) << "modules support not enabled/available"; + + return; + } + + // Sanity checks. + // + // If we are compiling a module interface unit, make sure it has the + // necessary declarations. + // + if (src.is_a (*x_mod) && (mi.name.empty () || !mi.iface)) + fail << src << " is not a module interface unit"; + + // Search and match all the modules we depend on. If this is a module + // implementation unit, then treat the module itself as if it was + // imported (we insert it first since for some compilers we have to + // differentiate between this special module and real imports). Note: + // move. + // + if (!mi.iface && !mi.name.empty ()) + mi.imports.insert (mi.imports.begin (), + module_import {move (mi.name), false, 0}); + + // The change to the set of imports would have required a change to + // source code (or options). Changes to the bmi{}s themselves will be + // detected via the normal prerequisite machinery. However, the same set + // of imports could be resolved to a different set of bmi{}s (in a sense + // similar to changing the source file). To detect this we calculate and + // store a hash of all (not just direct) bmi{}'s paths. + // + sha256 cs; + + if (!mi.imports.empty ()) + md.mods = search_modules (a, bs, t, li, tt.bmi, src, mi.imports, cs); + + if (dd.expect (cs.string ()) != nullptr) + updating = true; + +#if 0 + // Save the module map for compilers that use it. + // + if (md.mods.start != 0) + { + switch (cid) + { + case compiler_id::gcc: + case compiler_id::clang: + case compiler_id::clang_apple: + { + // We don't need to redo this if the above hash hasn't changed and + // the database is valid. + // + if (dd.writing () || !dd.skip ()) + { + const auto& pts (t.prerequisite_targets); + + for (size_t i (md.mods.start); i != pts.size (); ++i) + { + if (const target* m = pts[i]) + { + // Save a variable lookup by getting the module name from + // the import list (see search_modules()). + // + dd.write ('@', false); + dd.write (mi.imports[i - md.mods.start].name, false); + dd.write ('=', false); + dd.write (m->as ().path ()); + } + } + } + break; + } + default: + break; + } + } +#endif + + // Set the cc.module_name variable if this is an interface unit. Note + // that it may seem like a good idea to set it on the bmi{} group to + // avoid duplication. We, however, cannot do it MT-safely since we don't + // match the group. + // + if (mi.iface) + { + if (value& v = t.vars.assign (c_module_name)) + assert (cast (v) == mi.name); + else + v = move (mi.name); // Note: move. + } + } + + inline bool + std_module (const string& m) + { + size_t n (m.size ()); + return (n >= 3 && + m[0] == 's' && m[1] == 't' && m[2] == 'd' && + (n == 3 || m[3] == '.')); + }; + + // Resolve imported modules to bmi*{} targets. + // + module_positions compile_rule:: + search_modules (action a, + const scope& bs, + file& t, + linfo li, + const target_type& mtt, + const file& src, + module_imports& imports, + sha256& cs) const + { + tracer trace (x, "compile_rule::search_modules"); + + // So we have a list of imports and a list of "potential" module + // prerequisites. They are potential in the sense that they may or may + // not be required by this translation unit. In other words, they are + // the pool where we can resolve actual imports. + // + // Because we may not need all of these prerequisites, we cannot just go + // ahead and match all of them (and they can even have cycles; see rule + // synthesis). This poses a bit of a problem: the only way to discover + // the module's actual name (see cc.module_name) is by matching it. + // + // One way to solve this would be to make the user specify the module + // name for each mxx{} explicitly. This will be a major pain, however. + // Another would be to require encoding of the module name in the + // interface unit file name. For example, hello.core -> hello-core.mxx. + // This is better but still too restrictive: some will want to call it + // hello_core.mxx or HelloCore.mxx (because that's their file naming + // convention) or place it in a subdirectory, say, hello/core.mxx. + // + // In the above examples one common theme about all the file names is + // that they contain, in one form or another, the "tail" of the module + // name ('core'). So what we are going to do is require that the + // interface file names contain enough of the module name tail to + // unambiguously resolve all the module imports. On our side we are + // going to implement a "fuzzy" module name to file name match. This + // should be reliable enough since we will always verify our guesses + // once we match the target and extract the actual module name. Plus, + // the user will always have the option of resolving any impasses by + // specifying the module name explicitly. + // + // So, the fuzzy match: the idea is that each match gets a score, the + // number of characters in the module name that got matched. A match + // with the highest score is used. And we use the (length + 1) for a + // match against an actual module name. + // + // For std.* modules we only accept non-fuzzy matches (think std.core vs + // some core.mxx). And if such a module is unresolved, then we assume it + // is pre-built and will be found by some other means (e.g., VC's + // IFCPATH). + // + auto match = [] (const string& f, const string& m) -> size_t + { + size_t fi (f.size ()); + size_t mi (m.size ()); + + // Scan backwards for as long as we match. Keep track of the previous + // character for case change detection. + // + for (char fc, mc, fp ('\0'), mp ('\0'); + fi != 0 && mi != 0; + fp = fc, mp = mc, --fi, --mi) + { + fc = f[fi - 1]; + mc = m[mi - 1]; + + if (casecmp (fc, mc) == 0) + continue; + + // We consider all separators equal and character case change being + // a separators. Some examples of the latter: + // + // foo.bar + // fooBAR + // FOObar + // + bool fs (fc == '_' || fc == '-' || fc == '.' || + path::traits::is_separator (fc)); + bool ms (mc == '_' || mc == '.'); + + if (fs && ms) + continue; + + // Only if one is a real separator do we consider case change. + // + if (fs || ms) + { + auto cc = [] (char c1, char c2) -> bool + { + return (alpha (c1) && + alpha (c2) && + (ucase (c1) == c1) != (ucase (c2) == c2)); + }; + + bool fa (false), ma (false); + if ((fs || (fa = cc (fp, fc))) && (ms || (ma = cc (mp, mc)))) + { + // Stay on this character if imaginary punctuation (note: cannot + // be both true). + // + if (fa) ++fi; + if (ma) ++mi; + continue; + } + } + + break; // No match. + } + + // Return the number of characters matched in the module name and not + // in the file (this may not be the same because of the imaginary + // separators). + // + return m.size () - mi; + }; + + auto& pts (t.prerequisite_targets[a]); + size_t start (pts.size ()); // Index of the first to be added. + + // We have two parallel vectors: module names/scores in imports and + // targets in prerequisite_targets (offset with start). Pre-allocate + // NULL entries in the latter. + // + size_t n (imports.size ()); + pts.resize (start + n, nullptr); + + // Oh, yes, there is one "minor" complication. It's the last one, I + // promise. It has to do with module re-exporting (export import M;). + // In this case (currently) all implementations simply treat it as a + // shallow (from the BMI's point of view) reference to the module (or an + // implicit import, if you will). Do you see where it's going? Nowever + // good, that's right. This shallow reference means that the compiler + // should be able to find BMIs for all the re-exported modules, + // recursive. The good news is we are actually in a pretty good shape to + // handle this: after match all our prerequisite BMIs will have their + // prerequisite BMIs known, recursively. The only bit that is missing is + // the re-export flag of some sorts. As well as deciding where to handle + // it: here or in append_modules(). After some meditation it became + // clear handling it here will be simpler: We need to weed out + // duplicates for which we can re-use the imports vector. And we may + // also need to save this "flattened" list of modules in depdb. + // + // Ok, so, here is the plan: + // + // 1. There is no good place in prerequisite_targets to store the + // exported flag (no, using the marking facility across match/execute + // is a bad idea). So what we are going to do is put re-exported + // bmi{}s at the back and store (in the target's data pad) the start + // position. One bad aspect about this part is that we assume those + // bmi{}s have been matched by the same rule. But let's not kid + // ourselves, there will be no other rule that matches bmi{}s. + // + // 2. Once we have matched all the bmi{}s we are importing directly + // (with all the re-exported by us at the back), we will go over them + // and copy all of their re-exported bmi{}s (using the position we + // saved on step #1). The end result will be a recursively-explored + // list of imported bmi{}s that append_modules() can simply convert + // to the list of options. + // + // One issue with this approach is that these copied targets will be + // executed which means we need to adjust their dependent counts + // (which is normally done by match). While this seems conceptually + // correct (especially if you view re-exports as implicit imports), + // it's just extra overhead (we know they will be updated). So what + // we are going to do is save another position, that of the start of + // these copied-over targets, and will only execute up to this point. + // + // And after implementing this came the reality check: all the current + // implementations require access to all the imported BMIs, not only + // re-exported. Some (like Clang) store references to imported BMI files + // so we actually don't need to pass any extra options (unless things + // get moved) but they still need access to the BMIs (and things will + // most likely have to be done differenly for distributed compilation). + // + // So the revised plan: on the off chance that some implementation will + // do it differently we will continue maintaing the imported/re-exported + // split and how much to copy-over can be made compiler specific. + // + // As a first sub-step of step #1, move all the re-exported imports to + // the end of the vector. This will make sure they end up at the end + // of prerequisite_targets. Note: the special first import, if any, + // should be unaffected. + // + sort (imports.begin (), imports.end (), + [] (const module_import& x, const module_import& y) + { + return !x.exported && y.exported; + }); + + // Go over the prerequisites once. + // + // For (direct) library prerequisites, check their prerequisite bmi{}s + // (which should be searched and matched with module names discovered; + // see the library meta-information protocol for details). + // + // For our own bmi{} prerequisites, checking if each (better) matches + // any of the imports. + + // For fuzzy check if a file name (better) resolves any of our imports + // and if so make it the new selection. For exact the name is the actual + // module name and it can only resolve one import (there are no + // duplicates). + // + // Set done to true if all the imports have now been resolved to actual + // module names (which means we can stop searching). This will happens + // if all the modules come from libraries. Which will be fairly common + // (think of all the tests) so it's worth optimizing for. + // + bool done (false); + + auto check_fuzzy = [&trace, &imports, &pts, &match, start, n] + (const target* pt, const string& name) + { + for (size_t i (0); i != n; ++i) + { + module_import& m (imports[i]); + + if (std_module (m.name)) // No fuzzy std.* matches. + continue; + + size_t n (m.name.size ()); + + if (m.score > n) // Resolved to module name. + continue; + + size_t s (match (name, m.name)); + + l5 ([&]{trace << name << " ~ " << m.name << ": " << s;}); + + if (s > m.score) + { + pts[start + i] = pt; + m.score = s; + } + } + }; + + // If resolved, return the "slot" in pts (we don't want to create a + // side build until we know we match; see below for details). + // + auto check_exact = [&trace, &imports, &pts, start, n, &done] + (const string& name) -> const target** + { + const target** r (nullptr); + done = true; + + for (size_t i (0); i != n; ++i) + { + module_import& m (imports[i]); + + size_t n (m.name.size ()); + + if (m.score > n) // Resolved to module name (no effect on done). + continue; + + if (r == nullptr) + { + size_t s (name == m.name ? n + 1 : 0); + + l5 ([&]{trace << name << " ~ " << m.name << ": " << s;}); + + if (s > m.score) + { + r = &pts[start + i].target; + m.score = s; + continue; // Scan the rest to detect if all done. + } + } + + done = false; + } + + return r; + }; + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target* pt (p.load ()); // Should be cached for libraries. + + if (pt != nullptr) + { + const target* lt (nullptr); + + if (const libx* l = pt->is_a ()) + lt = &link_member (*l, a, li); + else if (pt->is_a () || pt->is_a () || pt->is_a ()) + lt = pt; + + // If this is a library, check its bmi{}s and mxx{}s. + // + if (lt != nullptr) + { + for (const target* bt: lt->prerequisite_targets[a]) + { + if (bt == nullptr) + continue; + + // Note that here we (try) to use whatever flavor of bmi*{} is + // available. + // + // @@ MOD: BMI compatibility check. + // @@ UTL: we need to (recursively) see through libux{} (and + // also in pkgconfig_save()). + // + if (bt->is_a () || + bt->is_a () || + bt->is_a ()) + { + const string& n (cast (bt->vars[c_module_name])); + + if (const target** p = check_exact (n)) + *p = bt; + } + else if (bt->is_a (*x_mod)) + { + // This is an installed library with a list of module sources + // (the source are specified as prerequisites but the fallback + // file rule puts them into prerequisite_targets for us). + // + // The module names should be specified but if not assume + // something else is going on and ignore. + // + const string* n (cast_null (bt->vars[c_module_name])); + if (n == nullptr) + continue; + + if (const target** p = check_exact (*n)) + *p = &make_module_sidebuild (a, bs, *lt, *bt, *n); + } + else + continue; + + if (done) + break; + } + + if (done) + break; + + continue; + } + + // Fall through. + } + + // While it would have been even better not to search for a target, we + // need to get hold of the corresponding mxx{} (unlikely but possible + // for bmi{} to have a different name). + // + if (p.is_a ()) + pt = &search (t, mtt, p.key ()); // Same logic as in picking obj*{}. + else if (p.is_a (mtt)) + { + if (pt == nullptr) + pt = &p.search (t); + } + else + continue; + + // Find the mxx{} prerequisite and extract its "file name" for the + // fuzzy match unless the user specified the module name explicitly. + // + for (prerequisite_member p: group_prerequisite_members (a, *pt)) + { + if (p.is_a (*x_mod)) + { + // Check for an explicit module name. Only look for an existing + // target (which means the name can only be specified on the + // target itself, no target type/pattern-spec). + // + const target* t (p.search_existing ()); + const string* n (t != nullptr + ? cast_null (t->vars[c_module_name]) + : nullptr); + if (n != nullptr) + { + if (const target** p = check_exact (*n)) + *p = pt; + } + else + { + // Fuzzy match. + // + string f; + + // Add the directory part if it is relative. The idea is to + // include it into the module match, say hello.core vs + // hello/mxx{core}. + // + // @@ MOD: Why not for absolute? Good question. What if it + // contains special components, say, ../mxx{core}? + // + const dir_path& d (p.dir ()); + + if (!d.empty () && d.relative ()) + f = d.representation (); // Includes trailing slash. + + f += p.name (); + check_fuzzy (pt, f); + } + break; + } + } + + if (done) + break; + } + + // Diagnose unresolved modules. + // + if (!done) + { + for (size_t i (0); i != n; ++i) + { + if (pts[start + i] == nullptr && !std_module (imports[i].name)) + { + // It would have been nice to print the location of the import + // declaration. And we could save it during parsing at the expense + // of a few paths (that can be pooled). The question is what to do + // when we re-create this information from depdb? We could have + // saved the location information there but the relative paths + // (e.g., from the #line directives) could end up being wrong if + // the we re-run from a different working directory. + // + // It seems the only workable approach is to extract full location + // info during parse, not save it in depdb, when re-creating, + // fallback to just src path without any line/column information. + // This will probably cover the majority of case (most of the time + // it will be a misspelled module name, not a removal of module + // from buildfile). + // + // But at this stage this doesn't seem worth the trouble. + // + fail (relative (src)) << "unable to resolve module " + << imports[i].name; + } + } + } + + // Match in parallel and wait for completion. + // + match_members (a, t, pts, start); + + // Post-process the list of our (direct) imports. While at it, calculate + // the checksum of all (direct and indirect) bmi{} paths. + // + size_t exported (n); + size_t copied (pts.size ()); + + for (size_t i (0); i != n; ++i) + { + const module_import& m (imports[i]); + + // Determine the position of the first re-exported bmi{}. + // + if (m.exported && exported == n) + exported = i; + + const target* bt (pts[start + i]); + + if (bt == nullptr) + continue; // Unresolved (std.*). + + // Verify our guesses against extracted module names but don't waste + // time if it was a match against the actual module name. + // + const string& in (m.name); + + if (m.score <= in.size ()) + { + const string& mn (cast (bt->vars[c_module_name])); + + if (in != mn) + { + for (prerequisite_member p: group_prerequisite_members (a, *bt)) + { + if (p.is_a (*x_mod)) // Got to be there. + { + fail (relative (src)) + << "failed to correctly guess module name from " << p << + info << "guessed: " << in << + info << "actual: " << mn << + info << "consider adjusting module interface file names or" << + info << "consider specifying module name with " << x + << ".module_name"; + } + } + } + } + + // Hash (we know it's a file). + // + cs.append (static_cast (*bt).path ().string ()); + + // Copy over bmi{}s from our prerequisites weeding out duplicates. + // + if (size_t j = bt->data ().mods.start) + { + // Hard to say whether we should reserve or not. We will probably + // get quite a bit of duplications. + // + auto& bpts (bt->prerequisite_targets[a]); + for (size_t m (bpts.size ()); j != m; ++j) + { + const target* et (bpts[j]); + + if (et == nullptr) + continue; // Unresolved (std.*). + + const string& mn (cast (et->vars[c_module_name])); + + if (find_if (imports.begin (), imports.end (), + [&mn] (const module_import& i) + { + return i.name == mn; + }) == imports.end ()) + { + pts.push_back (et); + cs.append (static_cast (*et).path ().string ()); + + // Add to the list of imports for further duplicate suppression. + // We could have probably stored reference to the name (e.g., in + // score) but it's probably not worth it if we have a small + // string optimization. + // + imports.push_back (module_import {mn, true, 0}); + } + } + } + } + + if (copied == pts.size ()) // No copied tail. + copied = 0; + + if (exported == n) // No (own) re-exported imports. + exported = copied; + else + exported += start; // Rebase. + + return module_positions {start, exported, copied}; + } + + // Synthesize a dependency for building a module binary interface on + // the side. + // + const target& compile_rule:: + make_module_sidebuild (action a, + const scope& bs, + const target& lt, + const target& mt, + const string& mn) const + { + tracer trace (x, "compile_rule::make_module_sidebuild"); + + // First figure out where we are going to build. We want to avoid + // multiple sidebuilds so the outermost scope that has loaded the + // cc.config module and that is within our amalgmantion seems like a + // good place. + // + const scope& rs (*bs.root_scope ()); + const scope* as (&rs); + { + const scope* ws (as->weak_scope ()); + if (as != ws) + { + const scope* s (as); + do + { + s = s->parent_scope ()->root_scope (); + + // Use cc.core.vars as a proxy for {c,cxx}.config (a bit smelly). + // + // This is also the module that registers the scope operation + // callback that cleans up the subproject. + // + if (cast_false ((*s)["cc.core.vars.loaded"])) + as = s; + + } while (s != ws); + } + } + + // We build modules in a subproject (since there might be no full + // language support module loaded in the amalgamation, only *.config). + // So the first step is to check if the project has already been created + // and/or loaded and if not, then to go ahead and do so. + // + dir_path pd (as->out_path () / modules_sidebuild_dir /= x); + { + const scope* ps (&scopes.find (pd)); + + if (ps->out_path () != pd) + { + // Switch the phase to load then create and load the subproject. + // + phase_switch phs (run_phase::load); + + // Re-test again now that we are in exclusive phase (another thread + // could have already created and loaded the subproject). + // + ps = &scopes.find (pd); + + if (ps->out_path () != pd) + { + // The project might already be created in which case we just need + // to load it. + // + if (!is_src_root (pd)) + { + // Copy our standard and force modules. + // + string extra; + + if (const string* std = cast_null (rs[x_std])) + extra += string (x) + ".std = " + *std + '\n'; + + extra += string (x) + ".features.modules = true"; + + config::create_project ( + pd, + as->out_path ().relative (pd), /* amalgamation */ + {}, /* boot_modules */ + extra, /* root_pre */ + {string (x) + '.'}, /* root_modules */ + "", /* root_post */ + false, /* config */ + false, /* buildfile */ + "the cc module", + 2); /* verbosity */ + } + + ps = &load_project (as->rw () /* lock */, pd, pd); + } + } + + // Some sanity checks. + // +#ifndef NDEBUG + assert (ps->root ()); + const module* m (ps->modules.lookup (x)); + assert (m != nullptr && m->modules); +#endif + } + + // Next we need to come up with a file/target name that will be unique + // enough not to conflict with other modules. If we assume that within + // an amalgamation there is only one "version" of each module, then the + // module name itself seems like a good fit. We just replace '.' with + // '-'. + // + string mf; + transform (mn.begin (), mn.end (), + back_inserter (mf), + [] (char c) {return c == '.' ? '-' : c;}); + + // It seems natural to build a BMI type that corresponds to the library + // type. After all, this is where the object file part of the BMI is + // going to come from (though things will probably be different for + // module-only libraries). + // + const target_type* tt (nullptr); + switch (link_type (lt).type) + { + case otype::a: tt = &bmia::static_type; break; + case otype::s: tt = &bmis::static_type; break; + case otype::e: assert (false); + } + + // Store the BMI target in the subproject root. If the target already + // exists then we assume all this is already done (otherwise why would + // someone have created such a target). + // + if (const target* bt = targets.find ( + *tt, + pd, + dir_path (), // Always in the out tree. + mf, + nullopt, // Use default extension. + trace)) + return *bt; + + prerequisites ps; + ps.push_back (prerequisite (mt)); + + // We've added the mxx{} but it may import other modules from this + // library. Or from (direct) dependencies of this library. We add them + // all as prerequisites so that the standard module search logic can + // sort things out. This is pretty similar to what we do in link when + // synthesizing dependencies for bmi{}'s. + // + ps.push_back (prerequisite (lt)); + for (prerequisite_member p: group_prerequisite_members (a, lt)) + { + // @@ TODO: will probably need revision if using sidebuild for + // non-installed libraries (e.g., direct BMI dependencies + // will probably have to be translated to mxx{} or some such). + // + if (p.is_a () || + p.is_a () || p.is_a () || p.is_a ()) + { + ps.push_back (p.as_prerequisite ()); + } + } + + auto p (targets.insert_locked (*tt, + move (pd), + dir_path (), // Always in the out tree. + move (mf), + nullopt, // Use default extension. + true, // Implied. + trace)); + const target& bt (p.first); + + // Note that this is racy and someone might have created this target + // while we were preparing the prerequisite list. + // + if (p.second.owns_lock ()) + bt.prerequisites (move (ps)); + + return bt; + } + + // Filter cl.exe noise (msvc.cxx). + // + void + msvc_filter_cl (ifdstream&, const path& src); + + void compile_rule:: + append_modules (environment& env, + cstrings& args, + strings& stor, + action a, + const file& t, + const match_data& md) const + { + const module_positions& ms (md.mods); + assert (ms.start != 0); + + dir_path stdifc; // See the VC case below. + + auto& pts (t.prerequisite_targets[a]); + +#if 0 + switch (cid) + { + case compiler_id::gcc: + { + // Use the module map stored in depdb. + // + string s (relative (md.dd).string ()); + s.insert (0, "-fmodule-file-map=@="); + stor.push_back (move (s)); + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // In Clang the module implementation's unit .pcm is special and + // must be "loaded". + // + if (md.type == translation_type::module_impl) + { + const file& f (pts[ms.start]->as ()); + string s (relative (f.path ()).string ()); + s.insert (0, "-fmodule-file="); + stor.push_back (move (s)); + } + + // Use the module map stored in depdb for others. + // + string s (relative (md.dd).string ()); + s.insert (0, "-fmodule-file-map=@="); + stor.push_back (move (s)); + break; + } + case compiler_id::msvc: + { + for (size_t i (ms.start), n (pts.size ()); + i != n; + ++i) + { + const target* pt (pts[i]); + + if (pt == nullptr) + continue; + + // Here we use whatever bmi type has been added. And we know all + // of these are bmi's. + // + const file& f (pt->as ()); + + // In VC std.* modules can only come from a single directory + // specified with the IFCPATH environment variable or the + // /module:stdIfcDir option. + // + if (std_module (cast (f.vars[c_module_name]))) + { + dir_path d (f.path ().directory ()); + + if (stdifc.empty ()) + { + // Go one directory up since /module:stdIfcDir will look in + // either Release or Debug subdirectories. Keeping the result + // absolute feels right. + // + stor.push_back ("/module:stdIfcDir"); + stor.push_back (d.directory ().string ()); + stdifc = move (d); + } + else if (d != stdifc) // Absolute and normalized. + fail << "multiple std.* modules in different directories"; + } + else + { + stor.push_back ("/module:reference"); + stor.push_back (relative (f.path ()).string ()); + } + } + break; + } + case compiler_id::icc: + assert (false); + } +#else + size_t n (pts.size ()); + + // Clang embeds module file references so we only need to specify + // our direct imports. + // + // If/when we get the ability to specify the mapping in a file, we + // should probably pass the whole list. + // + switch (cid) + { + case compiler_id::gcc: break; // All of them. + case compiler_id::clang_apple: + case compiler_id::clang: n = ms.copied != 0 ? ms.copied : n; break; + case compiler_id::msvc: break; // All of them. + case compiler_id::icc: assert (false); + } + + for (size_t i (ms.start); i != n; ++i) + { + const target* pt (pts[i]); + + if (pt == nullptr) + continue; + + // Here we use whatever bmi type has been added. And we know all of + // these are bmi's. + // + const file& f (pt->as ()); + string s (relative (f.path ()).string ()); + + switch (cid) + { + case compiler_id::gcc: + { + s.insert (0, 1, '='); + s.insert (0, cast (f.vars[c_module_name])); + s.insert (0, "-fmodule-file="); + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // In Clang the module implementation's unit .pcm is special and + // must be "loaded". + // + if (md.type == translation_type::module_impl && i == ms.start) + s.insert (0, "-fmodule-file="); + else + { + s.insert (0, 1, '='); + s.insert (0, cast (f.vars[c_module_name])); + s.insert (0, "-fmodule-file="); + } + break; + } + case compiler_id::msvc: + { + // In VC std.* modules can only come from a single directory + // specified with the IFCPATH environment variable or the + // /module:stdIfcDir option. + // + if (std_module (cast (f.vars[c_module_name]))) + { + dir_path d (f.path ().directory ()); + + if (stdifc.empty ()) + { + // Go one directory up since /module:stdIfcDir will look in + // either Release or Debug subdirectories. Keeping the result + // absolute feels right. + // + s = d.directory ().string (); + stor.push_back ("/module:stdIfcDir"); + stdifc = move (d); + } + else + { + if (d != stdifc) // Absolute and normalized. + fail << "multiple std.* modules in different directories"; + + continue; // Skip. + } + } + else + stor.push_back ("/module:reference"); + + break; + } + case compiler_id::icc: + assert (false); + } + + stor.push_back (move (s)); + } +#endif + + // Shallow-copy storage to args. Why not do it as we go along pushing + // into storage? Because of potential reallocations. + // + for (const string& a: stor) + args.push_back (a.c_str ()); + + // VC's IFCPATH takes precedence over /module:stdIfcDir so unset it + // if we are using our own std modules. + // + if (!stdifc.empty ()) + env.push_back ("IFCPATH"); + } + + target_state compile_rule:: + perform_update (action a, const target& xt) const + { + const file& t (xt.as ()); + const path& tp (t.path ()); + + match_data md (move (t.data ())); + bool mod (md.type == translation_type::module_iface); + + // While all our prerequisites are already up-to-date, we still have to + // execute them to keep the dependency counts straight. Actually, no, we + // may also have to update the modules. + // + auto pr ( + execute_prerequisites ( + (mod ? *x_mod : x_src), + a, t, + md.mt, + [s = md.mods.start] (const target&, size_t i) + { + return s != 0 && i >= s; // Only compare timestamps for modules. + }, + md.mods.copied)); // See search_modules() for details. + + const file& s (pr.second); + const path* sp (&s.path ()); + + if (pr.first) + { + if (md.touch) + { + touch (tp, false, 2); + skip_count.fetch_add (1, memory_order_relaxed); + } + + t.mtime (md.mt); + return *pr.first; + } + + // Make sure depdb is no older than any of our prerequisites. + // + touch (md.dd, false, verb_never); + + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + + otype ot (compile_type (t, mod)); + linfo li (link_info (bs, ot)); + + environment env; + cstrings args {cpath.recall_string ()}; + + // If we are building a module, then the target is bmi*{} and its ad hoc + // member is obj*{}. + // + path relo, relm; + if (mod) + { + relm = relative (tp); + relo = relative (t.member->is_a ()->path ()); + } + else + relo = relative (tp); + + // Build the command line. + // + if (md.pp != preprocessed::all) + { + append_options (args, t, c_poptions); + append_options (args, t, x_poptions); + + // Add *.export.poptions from prerequisite libraries. + // + append_lib_options (bs, args, a, t, li); + + // Extra system header dirs (last). + // + assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); + append_option_values ( + args, "-I", + sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), + [] (const dir_path& d) {return d.string ().c_str ();}); + + if (md.symexport) + append_symexport_options (args, t); + } + + append_options (args, t, c_coptions); + append_options (args, t, x_coptions); + append_options (args, tstd); + + string out, out1; // Output options storage. + strings mods; // Module options storage. + size_t out_i (0); // Index of the -o option. + + if (cclass == compiler_class::msvc) + { + // The /F*: option variants with separate names only became available + // in VS2013/12.0. Why do we bother? Because the command line suddenly + // becomes readable. + // + uint64_t ver (cast (rs[x_version_major])); + + args.push_back ("/nologo"); + + // While we want to keep the low-level build as "pure" as possible, + // the two misguided defaults, exceptions and runtime, just have to be + // fixed. Otherwise the default build is pretty much unusable. But we + // also make sure that the user can easily disable our defaults: if we + // see any relevant options explicitly specified, we take our hands + // off. + // + // For C looks like no /EH* (exceptions supported but no C++ objects + // destroyed) is a reasonable default. + // + if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) + args.push_back ("/EHsc"); + + // The runtime is a bit more interesting. At first it may seem like a + // good idea to be a bit clever and use the static runtime if we are + // building obja{}. And for obje{} we could decide which runtime to + // use based on the library link order: if it is static-only, then we + // could assume the static runtime. But it is indeed too clever: when + // building liba{} we have no idea who is going to use it. It could be + // an exe{} that links both static and shared libraries (and is + // therefore built with the shared runtime). And to safely use the + // static runtime, everything must be built with /MT and there should + // be no DLLs in the picture. So we are going to play it safe and + // always default to the shared runtime. + // + // In a similar vein, it would seem reasonable to use the debug runtime + // if we are compiling with debug. But, again, there will be fireworks + // if we have some projects built with debug and some without and then + // we try to link them together (which is not an unreasonable thing to + // do). So by default we will always use the release runtime. + // + if (!find_option_prefixes ({"/MD", "/MT"}, args)) + args.push_back ("/MD"); + + if (md.mods.start != 0) + append_modules (env, args, mods, a, t, md); + + // The presence of /Zi or /ZI causes the compiler to write debug info + // to the .pdb file. By default it is a shared file called vcNN.pdb + // (where NN is the VC version) created (wait for it) in the current + // working directory (and not the directory of the .obj file). Also, + // because it is shared, there is a special Windows service that + // serializes access. We, of course, want none of that so we will + // create a .pdb per object file. + // + // Note that this also changes the name of the .idb file (used for + // minimal rebuild and incremental compilation): cl.exe take the /Fd + // value and replaces the .pdb extension with .idb. + // + // Note also that what we are doing here appears to be incompatible + // with PCH (/Y* options) and /Gm (minimal rebuild). + // + if (find_options ({"/Zi", "/ZI"}, args)) + { + if (ver >= 18) + args.push_back ("/Fd:"); + else + out1 = "/Fd"; + + out1 += relo.string (); + out1 += ".pdb"; + + args.push_back (out1.c_str ()); + } + + if (ver >= 18) + { + args.push_back ("/Fo:"); + args.push_back (relo.string ().c_str ()); + } + else + { + out = "/Fo" + relo.string (); + args.push_back (out.c_str ()); + } + + if (mod) + { + args.push_back ("/module:interface"); + args.push_back ("/module:output"); + args.push_back (relm.string ().c_str ()); + } + + // Note: no way to indicate that the source if already preprocessed. + + args.push_back ("/c"); // Compile only. + args.push_back (langopt (md)); // Compile as. + args.push_back (sp->string ().c_str ()); // Note: relied on being last. + } + else + { + if (ot == otype::s) + { + // On Darwin, Win32 -fPIC is the default. + // + if (tclass == "linux" || tclass == "bsd") + args.push_back ("-fPIC"); + } + + if (md.mods.start != 0) + append_modules (env, args, mods, a, t, md); + + // Note: the order of the following options is relied upon below. + // + out_i = args.size (); // Index of the -o option. + + if (mod) + { + switch (cid) + { + case compiler_id::gcc: + { + args.push_back ("-o"); + args.push_back (relo.string ().c_str ()); + + out = "-fmodule-output="; + out += relm.string (); + args.push_back (out.c_str ()); + + args.push_back ("-c"); + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + args.push_back ("-o"); + args.push_back (relm.string ().c_str ()); + args.push_back ("--precompile"); + + // Without this option Clang's .pcm will reference source files. + // In our case this file may be transient (.ii). Plus, it won't + // play nice with distributed compilation. + // + args.push_back ("-Xclang"); + args.push_back ("-fmodules-embed-all-files"); + + break; + } + case compiler_id::msvc: + case compiler_id::icc: + assert (false); + } + } + else + { + args.push_back ("-o"); + args.push_back (relo.string ().c_str ()); + args.push_back ("-c"); + } + + args.push_back ("-x"); + args.push_back (langopt (md)); + + if (md.pp == preprocessed::all) + { + // Note that the mode we select must still handle comments and line + // continuations. So some more compiler-specific voodoo. + // + switch (cid) + { + case compiler_id::gcc: + { + // -fdirectives-only is available since GCC 4.3.0. + // + if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) + { + args.push_back ("-fpreprocessed"); + args.push_back ("-fdirectives-only"); + } + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // Clang handles comments and line continuations in the + // preprocessed source (it does not have -fpreprocessed). + // + break; + } + case compiler_id::icc: + break; // Compile as normal source for now. + case compiler_id::msvc: + assert (false); + } + } + + args.push_back (sp->string ().c_str ()); + } + + args.push_back (nullptr); + + if (!env.empty ()) + env.push_back (nullptr); + + // With verbosity level 2 print the command line as if we are compiling + // the source file, not its preprocessed version (so that it's easy to + // copy and re-run, etc). Only at level 3 and above print the real deal. + // + if (verb == 1) + text << x_name << ' ' << s; + else if (verb == 2) + print_process (args); + + // If we have the (partially) preprocessed output, switch to that. + // + bool psrc (!md.psrc.path.empty ()); + bool pact (md.psrc.active); + if (psrc) + { + args.pop_back (); // nullptr + args.pop_back (); // sp + + sp = &md.psrc.path; + + // This should match with how we setup preprocessing. + // + switch (cid) + { + case compiler_id::gcc: + { + // The -fpreprocessed is implied by .i/.ii. + // + args.pop_back (); // lang() + args.pop_back (); // -x + args.push_back ("-fdirectives-only"); + break; + } + case compiler_id::clang: + case compiler_id::clang_apple: + { + // Note that without -x Clang will treat .i/.ii as fully + // preprocessed. + // + break; + } + case compiler_id::msvc: + { + // Nothing to do (/TP or /TC already there). + // + break; + } + case compiler_id::icc: + assert (false); + } + + args.push_back (sp->string ().c_str ()); + args.push_back (nullptr); + + // Let's keep the preprocessed file in case of an error but only at + // verbosity level 3 and up (when one actually sees it mentioned on + // the command line). We also have to re-arm on success (see below). + // + if (pact && verb >= 3) + md.psrc.active = false; + } + + if (verb >= 3) + print_process (args); + + try + { + // VC cl.exe sends diagnostics to stdout. It also prints the file name + // being compiled as the first line. So for cl.exe we redirect stdout + // to a pipe, filter that noise out, and send the rest to stderr. + // + // For other compilers redirect stdout to stderr, in case any of them + // tries to pull off something similar. For sane compilers this should + // be harmless. + // + bool filter (cid == compiler_id::msvc); + + process pr (cpath, + args.data (), + 0, (filter ? -1 : 2), 2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + + if (filter) + { + try + { + ifdstream is ( + move (pr.in_ofd), fdstream_mode::text, ifdstream::badbit); + + msvc_filter_cl (is, *sp); + + // If anything remains in the stream, send it all to stderr. Note + // that the eof check is important: if the stream is at eof, this + // and all subsequent writes to the diagnostics stream will fail + // (and you won't see a thing). + // + if (is.peek () != ifdstream::traits_type::eof ()) + diag_stream_lock () << is.rdbuf (); + + is.close (); + } + catch (const io_error&) {} // Assume exits with error. + } + + run_finish (args, pr); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + if (pact && verb >= 3) + md.psrc.active = true; + + // Clang's module compilation requires two separate compiler + // invocations. + // + if (mod && (cid == compiler_id::clang || + cid == compiler_id::clang_apple)) + { + // Remove the target file if this fails. If we don't do that, we will + // end up with a broken build that is up-to-date. + // + auto_rmfile rm (relm); + + // Adjust the command line. First discard everything after -o then + // build the new "tail". + // + args.resize (out_i + 1); + args.push_back (relo.string ().c_str ()); // Produce .o. + args.push_back ("-c"); // By compiling .pcm. + args.push_back ("-Wno-unused-command-line-argument"); + args.push_back (relm.string ().c_str ()); + args.push_back (nullptr); + + if (verb >= 2) + print_process (args); + + try + { + process pr (cpath, + args.data (), + 0, 2, 2, + nullptr, // CWD + env.empty () ? nullptr : env.data ()); + + run_finish (args, pr); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + rm.cancel (); + } + + // Should we go to the filesystem and get the new mtime? We know the + // file has been modified, so instead just use the current clock time. + // It has the advantage of having the subseconds precision. + // + t.mtime (system_clock::now ()); + return target_state::changed; + } + + target_state compile_rule:: + perform_clean (action a, const target& xt) const + { + const file& t (xt.as ()); + + using id = compiler_id; + + switch (cid) + { + case id::gcc: return clean_extra (a, t, {".d", x_pext, ".t"}); + case id::clang_apple: + case id::clang: return clean_extra (a, t, {".d", x_pext}); + case id::msvc: return clean_extra (a, t, {".d", x_pext, ".idb", ".pdb"}); + case id::icc: return clean_extra (a, t, {".d"}); + } + + assert (false); + return target_state::unchanged; + } + } +} diff --git a/build2/cc/compile-rule.hxx b/build2/cc/compile-rule.hxx new file mode 100644 index 0000000..6bf63bf --- /dev/null +++ b/build2/cc/compile-rule.hxx @@ -0,0 +1,156 @@ +// file : build2/cc/compile-rule.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BUILD2_CC_COMPILE_RULE_HXX +#define BUILD2_CC_COMPILE_RULE_HXX + +#include + +#include +#include + +#include +#include // auto_rmfile + +#include +#include + +namespace build2 +{ + class depdb; + + namespace cc + { + // The order is arranged so that their integral values indicate whether + // one is a "stronger" than another. + // + enum class preprocessed: uint8_t {none, includes, modules, all}; + + // Positions of the re-exported bmi{}s. See search_modules() for + // details. + // + struct module_positions + { + size_t start; // First imported bmi*{}, 0 if none. + size_t exported; // First re-exported bmi*{}, 0 if none. + size_t copied; // First copied-over bmi*{}, 0 if none. + }; + + class compile_rule: public rule, virtual common + { + public: + compile_rule (data&&); + + virtual bool + match (action, target&, const string&) const override; + + virtual recipe + apply (action, target&) const override; + + target_state + perform_update (action, const target&) const; + + target_state + perform_clean (action, const target&) const; + + private: + struct match_data; + using environment = small_vector; + + void + append_lib_options (const scope&, + cstrings&, + action, + const target&, + linfo) const; + + void + hash_lib_options (const scope&, + sha256&, + action, + const target&, + linfo) const; + + // Mapping of include prefixes (e.g., foo in ) for auto- + // generated headers to directories where they will be generated. + // + // We are using a prefix map of directories (dir_path_map) instead of + // just a map in order to also cover sub-paths (e.g., if + // we continue with the example). Specifically, we need to make sure we + // don't treat foobar as a sub-directory of foo. + // + // The priority is used to decide who should override whom. Lesser + // values are considered higher priority. See append_prefixes() for + // details. + // + // @@ The keys should be normalized. + // + struct prefix_value + { + dir_path directory; + size_t priority; + }; + using prefix_map = butl::dir_path_map; + + void + append_prefixes (prefix_map&, const target&, const variable&) const; + + void + append_lib_prefixes (const scope&, + prefix_map&, + action, + target&, + linfo) const; + + prefix_map + build_prefix_map (const scope&, action, target&, linfo) const; + + // Reverse-lookup target type from extension. + // + const target_type* + map_extension (const scope&, const string&, const string&) const; + + pair + extract_headers (action, const scope&, file&, linfo, + const file&, const match_data&, + depdb&, bool&, timestamp) const; + + pair + parse_unit (action, file&, linfo, + const file&, auto_rmfile&, const match_data&) const; + + void + extract_modules (action, const scope&, file&, linfo, + const compile_target_types&, + const file&, match_data&, + module_info&&, depdb&, bool&) const; + + module_positions + search_modules (action, const scope&, file&, linfo, + const target_type&, + const file&, module_imports&, sha256&) const; + + const target& + make_module_sidebuild (action, const scope&, const target&, + const target&, const string&) const; + + void + append_modules (environment&, cstrings&, strings&, + action, const file&, const match_data&) const; + + // Language selection option (for VC) or the value for the -x option. + // + const char* + langopt (const match_data&) const; + + void + append_symexport_options (cstrings&, const target&) const; + + private: + const string rule_id; + }; + } +} + +#endif // BUILD2_CC_COMPILE_RULE_HXX diff --git a/build2/cc/compile.cxx b/build2/cc/compile.cxx deleted file mode 100644 index 94b3478..0000000 --- a/build2/cc/compile.cxx +++ /dev/null @@ -1,4627 +0,0 @@ -// file : build2/cc/compile.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include - -#include // exit() -#include // strlen() - -#include -#include -#include -#include -#include -#include -#include - -#include -#include // create_project() - -#include -#include // h -#include -#include - -using namespace std; -using namespace butl; - -namespace build2 -{ - namespace cc - { - using namespace bin; - - // module_info string serialization. - // - // The string representation is a space-separated list of module names - // with the following rules: - // - // 1. If this is a module interface unit, then the first name is the - // module name intself following by either '!' for an interface unit or - // by '+' for an implementation unit. - // - // 2. If an imported module is re-exported, then the module name is - // followed by '*'. - // - // For example: - // - // foo! foo.core* foo.base* foo.impl - // foo.base+ foo.impl - // foo.base foo.impl - // - static string - to_string (const module_info& m) - { - string s; - - if (!m.name.empty ()) - { - s += m.name; - s += m.iface ? '!' : '+'; - } - - for (const module_import& i: m.imports) - { - if (!s.empty ()) - s += ' '; - - s += i.name; - - if (i.exported) - s += '*'; - } - - return s; - } - - static module_info - to_module_info (const string& s) - { - module_info m; - - for (size_t b (0), e (0), n; (n = next_word (s, b, e, ' ')) != 0; ) - { - char c (s[e - 1]); - switch (c) - { - case '!': - case '+': - case '*': break; - default: c = '\0'; - } - - string w (s, b, n - (c == '\0' ? 0 : 1)); - - if (c == '!' || c == '+') - { - m.name = move (w); - m.iface = (c == '!'); - } - else - m.imports.push_back (module_import {move (w), c == '*', 0}); - } - - return m; - } - - // preprocessed - // - template - inline bool - operator< (preprocessed l, T r) // Template because of VC14 bug. - { - return static_cast (l) < static_cast (r); - } - - preprocessed - to_preprocessed (const string& s) - { - if (s == "none") return preprocessed::none; - if (s == "includes") return preprocessed::includes; - if (s == "modules") return preprocessed::modules; - if (s == "all") return preprocessed::all; - throw invalid_argument ("invalid preprocessed value '" + s + "'"); - } - - struct compile::match_data - { - explicit - match_data (translation_type t, const prerequisite_member& s) - : type (t), src (s) {} - - translation_type type; - preprocessed pp = preprocessed::none; - bool symexport = false; // Target uses __symexport. - bool touch = false; // Target needs to be touched. - timestamp mt = timestamp_unknown; // Target timestamp. - prerequisite_member src; - auto_rmfile psrc; // Preprocessed source, if any. - path dd; // Dependency database path. - module_positions mods = {0, 0, 0}; - }; - - compile:: - compile (data&& d) - : common (move (d)), - rule_id (string (x) += ".compile 4") - { - static_assert (sizeof (compile::match_data) <= target::data_size, - "insufficient space"); - } - - const char* compile:: - langopt (const match_data& md) const - { - bool m (md.type == translation_type::module_iface); - //preprocessed p (md.pp); - - switch (cid) - { - case compiler_id::gcc: - { - // Ignore the preprocessed value since for GCC it is handled via - // -fpreprocessed -fdirectives-only. - // - switch (x_lang) - { - case lang::c: return "c"; - case lang::cxx: return "c++"; - } - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // Clang has *-cpp-output (but not c++-module-cpp-output) and they - // handle comments and line continuations. However, currently this - // is only by accident since these modes are essentially equivalent - // to their cpp-output-less versions. - // - switch (x_lang) - { - case lang::c: return "c"; - case lang::cxx: return m ? "c++-module" : "c++"; - } - } - case compiler_id::msvc: - { - switch (x_lang) - { - case lang::c: return "/TC"; - case lang::cxx: return "/TP"; - } - } - case compiler_id::icc: - { - switch (x_lang) - { - case lang::c: return "c"; - case lang::cxx: return "c++"; - } - } - } - - return nullptr; - } - - inline void compile:: - append_symexport_options (cstrings& args, const target& t) const - { - // With VC if a BMI is compiled with dllexport, then when such BMI is - // imported, it is auto-magically treated as dllimport. Let's hope - // other compilers follow suit. - // - args.push_back (t.is_a () && tclass == "windows" - ? "-D__symexport=__declspec(dllexport)" - : "-D__symexport="); - } - - match_result compile:: - match (action act, target& t, const string&) const - { - tracer trace (x, "compile::match"); - - bool mod (t.is_a () || t.is_a () || t.is_a ()); - - // Link-up to our group (this is the obj/bmi{} target group protocol - // which means this can be done whether we match or not). - // - if (t.group == nullptr) - t.group = &search (t, - mod ? bmi::static_type : obj::static_type, - t.dir, t.out, t.name); - - // See if we have a source file. Iterate in reverse so that a source - // file specified for a member overrides the one specified for the - // group. Also "see through" groups. - // - for (prerequisite_member p: reverse_group_prerequisite_members (act, t)) - { - if (p.is_a (mod ? *x_mod : x_src)) - { - // Save in the target's auxiliary storage. Translation type will - // be refined in apply(). - // - t.data (match_data (mod - ? translation_type::module_iface - : translation_type::plain, - p)); - return true; - } - } - - l4 ([&]{trace << "no " << x_lang << " source file for target " << t;}); - return false; - } - - // Append or hash library options from a pair of *.export.* variables - // (first one is cc.export.*) recursively, prerequisite libraries first. - // - void compile:: - append_lib_options (const scope& bs, - cstrings& args, - const target& t, - action act, - linfo li) const - { - // See through utility libraries. - // - auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; - - auto opt = [&args, this] ( - const file& l, const string& t, bool com, bool exp) - { - // Note that in our model *.export.poptions are always "interface", - // even if set on liba{}/libs{}, unlike loptions. - // - if (!exp) // Ignore libux. - return; - - const variable& var ( - com - ? c_export_poptions - : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); - - append_options (args, l, var); - }; - - // In case we don't have the "small function object" optimization. - // - const function impf (imp); - const function optf (opt); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - // Should be already searched and matched for libraries. - // - if (const target* pt = p.load ()) - { - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - - bool a; - if (!((a = pt->is_a ()) || - (a = pt->is_a ()) || - pt->is_a ())) - continue; - - process_libraries (act, bs, li, sys_lib_dirs, - pt->as (), a, 0, // Hack: lflags unused. - impf, nullptr, optf); - } - } - } - - void compile:: - hash_lib_options (const scope& bs, - sha256& cs, - const target& t, - action act, - linfo li) const - { - auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; - - auto opt = [&cs, this] ( - const file& l, const string& t, bool com, bool exp) - { - if (!exp) - return; - - const variable& var ( - com - ? c_export_poptions - : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); - - hash_options (cs, l, var); - }; - - // The same logic as in append_lib_options(). - // - const function impf (imp); - const function optf (opt); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - if (const target* pt = p.load ()) - { - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - - bool a; - if (!((a = pt->is_a ()) || - (a = pt->is_a ()) || - pt->is_a ())) - continue; - - process_libraries (act, bs, li, sys_lib_dirs, - pt->as (), a, 0, // Hack: lflags unused. - impf, nullptr, optf); - } - } - } - - // Append library prefixes based on the *.export.poptions variables - // recursively, prerequisite libraries first. - // - void compile:: - append_lib_prefixes (const scope& bs, - prefix_map& m, - target& t, - action act, - linfo li) const - { - auto imp = [] (const file& l, bool la) {return la && l.is_a ();}; - - auto opt = [&m, this] ( - const file& l, const string& t, bool com, bool exp) - { - if (!exp) - return; - - const variable& var ( - com - ? c_export_poptions - : (t == x ? x_export_poptions : var_pool[t + ".export.poptions"])); - - append_prefixes (m, l, var); - }; - - // The same logic as in append_lib_options(). - // - const function impf (imp); - const function optf (opt); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - if (const target* pt = p.load ()) - { - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - - bool a; - if (!((a = pt->is_a ()) || - (a = pt->is_a ()) || - pt->is_a ())) - continue; - - process_libraries (act, bs, li, sys_lib_dirs, - pt->as (), a, 0, // Hack: lflags unused. - impf, nullptr, optf); - } - } - } - - // Update the target during the match phase. Return true if it has changed - // or if the passed timestamp is not timestamp_unknown and is older than - // the target. - // - // This function is used to make sure header dependencies are up to date. - // - // There would normally be a lot of headers for every source file (think - // all the system headers) and just calling execute_direct() on all of - // them can get expensive. At the same time, most of these headers are - // existing files that we will never be updating (again, system headers, - // for example) and the rule that will match them is the fallback - // file_rule. That rule has an optimization: it returns noop_recipe (which - // causes the target state to be automatically set to unchanged) if the - // file is known to be up to date. So we do the update "smartly". - // - static bool - update (tracer& trace, action act, const target& t, timestamp ts) - { - const path_target* pt (t.is_a ()); - - if (pt == nullptr) - ts = timestamp_unknown; - - target_state os (t.matched_state (act)); - - if (os == target_state::unchanged) - { - if (ts == timestamp_unknown) - return false; - else - { - // We expect the timestamp to be known (i.e., existing file). - // - timestamp mt (pt->mtime ()); // @@ MT perf: know target state. - assert (mt != timestamp_unknown); - return mt > ts; - } - } - else - { - // We only want to return true if our call to execute() actually - // caused an update. In particular, the target could already have been - // in target_state::changed because of a dependency extraction run for - // some other source file. - // - // @@ MT perf: so we are going to switch the phase and execute for - // any generated header. - // - phase_switch ps (run_phase::execute); - target_state ns (execute_direct (act, t)); - - if (ns != os && ns != target_state::unchanged) - { - l6 ([&]{trace << "updated " << t - << "; old state " << os - << "; new state " << ns;}); - return true; - } - else - return ts != timestamp_unknown ? pt->newer (ts) : false; - } - } - - recipe compile:: - apply (action act, target& xt) const - { - tracer trace (x, "compile::apply"); - - file& t (xt.as ()); // Either obj*{} or bmi*{}. - - match_data& md (t.data ()); - bool mod (md.type == translation_type::module_iface); - - const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); - - otype ot (compile_type (t, mod)); - linfo li (link_info (bs, ot)); // Link info for selecting libraries. - compile_target_types tt (compile_types (ot)); - - // Derive file name from target name. - // - string e; // Primary target extension (module or object). - { - const char* o ("o"); // Object extension (.o or .obj). - - if (tsys == "win32-msvc") - { - switch (ot) - { - case otype::e: e = "exe."; break; - case otype::a: e = "lib."; break; - case otype::s: e = "dll."; break; - } - o = "obj"; - } - else if (tsys == "mingw32") - { - switch (ot) - { - case otype::e: e = "exe."; break; - case otype::a: e = "a."; break; - case otype::s: e = "dll."; break; - } - } - else if (tsys == "darwin") - { - switch (ot) - { - case otype::e: e = ""; break; - case otype::a: e = "a."; break; - case otype::s: e = "dylib."; break; - } - } - else - { - switch (ot) - { - case otype::e: e = ""; break; - case otype::a: e = "a."; break; - case otype::s: e = "so."; break; - } - } - - switch (cid) - { - case compiler_id::gcc: - { - e += mod ? "nms" : o; - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - e += mod ? "pcm" : o; - break; - } - case compiler_id::msvc: - { - e += mod ? "ifc" : o; - break; - } - case compiler_id::icc: - { - assert (!mod); - e += o; - } - } - - // If we are compiling a module, then the obj*{} is an ad hoc member - // of bmi*{}. - // - if (mod) - { - // The module interface unit can be the same as an implementation - // (e.g., foo.mxx and foo.cxx) which means obj*{} targets could - // collide. So we add the module extension to the target name. - // - target_lock obj (add_adhoc_member (act, t, tt.obj, e.c_str ())); - obj.target->as ().derive_path (o); - match_recipe (obj, group_recipe); // Set recipe and unlock. - } - } - - const path& tp (t.derive_path (e.c_str ())); - - // Inject dependency on the output directory. - // - const fsdir* dir (inject_fsdir (act, t)); - - // Match all the existing prerequisites. The injection code takes care - // of the ones it is adding. - // - // When cleaning, ignore prerequisites that are not in the same or a - // subdirectory of our project root. - // - auto& pts (t.prerequisite_targets); - optional usr_lib_dirs; // Extract lazily. - - // Start asynchronous matching of prerequisites. Wait with unlocked - // phase to allow phase switching. - // - wait_guard wg (target::count_busy (), t.task_count, true); - - size_t start (pts.size ()); // Index of the first to be added. - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target* pt (nullptr); - - // A dependency on a library is there so that we can get its - // *.export.poptions, modules, etc. This is the "library - // meta-information protocol". See also append_lib_options(). - // - if (p.is_a () || - p.is_a () || - p.is_a () || - p.is_a ()) - { - if (act.operation () == update_id) - { - // Handle (phase two) imported libraries. We know that for such - // libraries we don't need to do match() in order to get options - // (if any, they would be set by search_library()). - // - if (p.proj ()) - { - if (search_library (act, - sys_lib_dirs, - usr_lib_dirs, - p.prerequisite) != nullptr) - continue; - } - - pt = &p.search (t); - - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - } - else - continue; - } - // - // For modules we pick only what we import which is done below so - // skip it here. One corner case is clean: we assume that someone - // else (normally library/executable) also depends on it and will - // clean it up. - // - else if (p.is_a () || p.is_a (tt.bmi)) - continue; - else - { - pt = &p.search (t); - - if (act.operation () == clean_id && !pt->dir.sub (rs.out_path ())) - continue; - } - - match_async (act, *pt, target::count_busy (), t.task_count); - pts.push_back (pt); - } - - wg.wait (); - - // Finish matching all the targets that we have started. - // - for (size_t i (start), n (pts.size ()); i != n; ++i) - { - const target*& pt (pts[i]); - - // Making sure a library is updated before us will only restrict - // parallelism. But we do need to match it in order to get its imports - // resolved and prerequisite_targets populated. So we match it but - // then unmatch if it is safe. And thanks to the two-pass prerequisite - // match in link::apply() it will be safe unless someone is building - // an obj?{} target directory. - // - if (build2::match ( - act, - *pt, - pt->is_a () || pt->is_a () || pt->is_a () - ? unmatch::safe - : unmatch::none)) - pt = nullptr; // Ignore in execute. - } - - // Inject additional prerequisites. We only do it when performing update - // since chances are we will have to update some of our prerequisites in - // the process (auto-generated source code). - // - if (act == perform_update_id) - { - // The cached prerequisite target should be the same as what is in - // t.prerequisite_targets since we used standard search() and match() - // above. - // - const file& src (*md.src.search (t).is_a ()); - - // Figure out if __symexport is used. While normally it is specified - // on the project root (which we cached), it can be overridden with - // a target-specific value for installed modules (which we sidebuild - // as part of our project). - // - if (modules && src.is_a (*x_mod)) - { - lookup l (src.vars[x_symexport]); - md.symexport = l ? cast (l) : symexport; - } - - // Make sure the output directory exists. - // - // Is this the right thing to do? It does smell a bit, but then we do - // worse things in inject_prerequisites() below. There is also no way - // to postpone this until update since we need to extract and inject - // header dependencies now (we don't want to be calling search() and - // match() in update), which means we need to cache them now as well. - // So the only alternative, it seems, is to cache the updates to the - // database until later which will sure complicate (and slow down) - // things. - // - if (dir != nullptr) - { - // We can do it properly by using execute_direct(). But this means - // we will be switching to the execute phase with all the associated - // overheads. At the same time, in case of update, creation of a - // directory is not going to change the external state in any way - // that would affect any parallel efforts in building the internal - // state. So we are just going to create the directory directly. - // Note, however, that we cannot modify the fsdir{} target since - // this can very well be happening in parallel. But that's not a - // problem since fsdir{}'s update is idempotent. - // - fsdir_rule::perform_update_direct (act, t); - } - - // Note: the leading '@' is reserved for the module map prefix (see - // extract_modules()) and no other line must start with it. - // - md.dd = tp + ".d"; - depdb dd (md.dd); - - // First should come the rule name/version. - // - if (dd.expect (rule_id) != nullptr) - l4 ([&]{trace << "rule mismatch forcing update of " << t;}); - - // Then the compiler checksum. Note that here we assume it - // incorporates the (default) target so that if the compiler changes - // but only in what it targets, then the checksum will still change. - // - if (dd.expect (cast (rs[x_checksum])) != nullptr) - l4 ([&]{trace << "compiler mismatch forcing update of " << t;}); - - // Then the options checksum. - // - // The idea is to keep them exactly as they are passed to the compiler - // since the order may be significant. - // - { - sha256 cs; - - // These flags affect how we compile the source and/or the format of - // depdb so factor them in. - // - cs.append (&md.pp, sizeof (md.pp)); - cs.append (&md.symexport, sizeof (md.symexport)); - - if (md.pp != preprocessed::all) - { - hash_options (cs, t, c_poptions); - hash_options (cs, t, x_poptions); - - // Hash *.export.poptions from prerequisite libraries. - // - hash_lib_options (bs, cs, t, act, li); - - // Extra system header dirs (last). - // - assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); - hash_option_values ( - cs, "-I", - sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), - [] (const dir_path& d) {return d.string ();}); - } - - hash_options (cs, t, c_coptions); - hash_options (cs, t, x_coptions); - hash_options (cs, tstd); - - if (ot == otype::s) - { - // On Darwin, Win32 -fPIC is the default. - // - if (tclass == "linux" || tclass == "bsd") - cs.append ("-fPIC"); - } - - if (dd.expect (cs.string ()) != nullptr) - l4 ([&]{trace << "options mismatch forcing update of " << t;}); - } - - // Finally the source file. - // - if (dd.expect (src.path ()) != nullptr) - l4 ([&]{trace << "source file mismatch forcing update of " << t;}); - - // If any of the above checks resulted in a mismatch (different - // compiler, options, or source file) or if the depdb is newer than - // the target (interrupted update), then do unconditional update. - // - timestamp mt; - bool u (dd.writing () || dd.mtime () > (mt = file_mtime (tp))); - if (u) - mt = timestamp_nonexistent; // Treat as if it doesn't exist. - - // Update prerequisite targets (normally just the source file). - // - // This is an unusual place and time to do it. But we have to do it - // before extracting dependencies. The reasoning for source file is - // pretty clear. What other prerequisites could we have? While - // normally they will be some other sources (as in, static content - // from src_root), it's possible they are some auto-generated stuff. - // And it's possible they affect the preprocessor result. Say some ad - // hoc/out-of-band compiler input file that is passed via the command - // line. So, to be safe, we make sure everything is up to date. - // - for (const target* pt: pts) - { - if (pt == nullptr || pt == dir) - continue; - - u = update (trace, act, *pt, u ? timestamp_unknown : mt) || u; - } - - // Check if the source is already preprocessed to a certain degree. - // This determines which of the following steps we perform and on - // what source (original or preprocessed). - // - // Note: must be set of the src target. - // - if (const string* v = cast_null (src[x_preprocessed])) - try - { - md.pp = to_preprocessed (*v); - } - catch (const invalid_argument& e) - { - fail << "invalid " << x_preprocessed.name << " variable value " - << "for target " << src << ": " << e; - } - - // If we have no #include directives, then skip header dependency - // extraction. - // - pair psrc (auto_rmfile (), false); - if (md.pp < preprocessed::includes) - psrc = extract_headers (act, bs, t, li, src, md, dd, u, mt); - - // Next we "obtain" the translation unit information. What exactly - // "obtain" entails is tricky: If things changed, then we re-parse the - // translation unit. Otherwise, we re-create this information from - // depdb. We, however, have to do it here and now in case the database - // is invalid and we still have to fallback to re-parse. - // - // Store a translation unit's checksum to detect ignorable changes - // (whitespaces, comments, etc). - // - { - string cs; - if (string* l = dd.read ()) - cs = move (*l); - else - u = true; // Database is invalid, force re-parse. - - translation_unit tu; - for (bool f (true);; f = false) - { - if (u) - { - auto p (parse_unit (act, t, li, src, psrc.first, md)); - - if (cs != p.second) - { - assert (f); // Unchanged TU has a different checksum? - dd.write (p.second); - } - else if (f) // Don't clear if it was forced. - { - // Clear the update flag and set the touch flag. Unless there - // is no object file, of course. See also the md.mt logic - // below. - // - if (mt != timestamp_nonexistent) - { - u = false; - md.touch = true; - } - } - - tu = move (p.first); - } - - if (modules) - { - if (u || !f) - { - string s (to_string (tu.mod)); - - if (f) - dd.expect (s); - else - dd.write (s); - } - else - { - if (string* l = dd.read ()) - tu.mod = to_module_info (*l); - else - { - u = true; // Database is invalid, force re-parse. - continue; - } - } - } - - break; - } - - // Make sure the translation unit type matches the resulting target - // type. - // - switch (tu.type ()) - { - case translation_type::plain: - case translation_type::module_impl: - { - if (mod) - fail << "translation unit " << src << " is not a module interface" << - info << "consider using " << x_src.name << "{} instead"; - break; - } - case translation_type::module_iface: - { - if (!mod) - fail << "translation unit " << src << " is a module interface" << - info << "consider using " << x_mod->name << "{} instead"; - break; - } - } - - md.type = tu.type (); - - // Extract the module dependency information in addition to header - // dependencies. - // - // NOTE: assumes that no further targets will be added into - // t.prerequisite_targets! - // - extract_modules (act, bs, t, li, tt, src, md, move (tu.mod), dd, u); - } - - // If anything got updated, then we didn't rely on the cache. However, - // the cached data could actually have been valid and the compiler run - // in extract_headers() as well as the code above merely validated it. - // - // We do need to update the database timestamp, however. Failed that, - // we will keep re-validating the cached data over and over again. - // - if (u && dd.reading ()) - dd.touch (); - - dd.close (); - - // If the preprocessed output is suitable for compilation and is not - // disabled, then pass it along. - // - if (psrc.second && !cast_false (t[c_reprocess])) - { - md.psrc = move (psrc.first); - - // Without modules keeping the (partially) preprocessed output - // around doesn't buy us much: if the source/headers haven't changed - // then neither will the object file. Modules make things more - // interesting: now we may have to recompile an otherwise unchanged - // translation unit because a BMI it depends on has changed. In this - // case re-processing the translation unit would be a waste and - // compiling the original source would break distributed - // compilation. - // - // Note also that the long term trend will (hopefully) be for - // modularized projects to get rid of #include's which means the - // need for producing this partially preprocessed output will - // (hopefully) gradually disappear. - // - if (modules) - md.psrc.active = false; // Keep. - } - - // Above we may have ignored changes to the translation unit. The - // problem is, unless we also update the target's timestamp, we will - // keep re-checking this on subsequent runs and it is not cheap. - // Updating the target's timestamp is not without problems either: it - // will cause a re-link on a subsequent run. So, essentially, we - // somehow need to remember two timestamps: one for checking - // "preprocessor prerequisites" above and one for checking other - // prerequisites (like modules) below. So what we are going to do is - // store the first in the target file (so we do touch it) and the - // second in depdb (which is never newer that the target). - // - md.mt = u ? timestamp_nonexistent : dd.mtime (); - } - - switch (act) - { - case perform_update_id: return [this] (action a, const target& t) - { - return perform_update (a, t); - }; - case perform_clean_id: return [this] (action a, const target& t) - { - return perform_clean (a, t); - }; - default: return noop_recipe; // Configure update. - } - } - - // Reverse-lookup target type from extension. - // - const target_type* compile:: - map_extension (const scope& s, const string& n, const string& e) const - { - // We will just have to try all of the possible ones, in the "most - // likely to match" order. - // - auto test = [&s, &n, &e] (const target_type& tt) -> bool - { - // Call the extension derivation function. Here we know that it will - // only use the target type and name from the target key so we can - // pass bogus values for the rest. - // - target_key tk {&tt, nullptr, nullptr, &n, nullopt}; - - // This is like prerequisite search. - // - if (optional de = tt.default_extension (tk, s, true)) - if (*de == e) - return true; - - return false; - }; - - for (const target_type* const* p (x_inc); *p != nullptr; ++p) - if (test (**p)) return *p; - - return nullptr; - } - - void compile:: - append_prefixes (prefix_map& m, const target& t, const variable& var) const - { - tracer trace (x, "compile::append_prefixes"); - - // If this target does not belong to any project (e.g, an "imported as - // installed" library), then it can't possibly generate any headers for - // us. - // - const scope& bs (t.base_scope ()); - const scope* rs (bs.root_scope ()); - if (rs == nullptr) - return; - - const dir_path& out_base (t.dir); - const dir_path& out_root (rs->out_path ()); - - if (auto l = t[var]) - { - const auto& v (cast (l)); - - for (auto i (v.begin ()), e (v.end ()); i != e; ++i) - { - // -I can either be in the "-Ifoo" or "-I foo" form. For VC it can - // also be /I. - // - const string& o (*i); - - if (o.size () < 2 || (o[0] != '-' && o[0] != '/') || o[1] != 'I') - continue; - - dir_path d; - if (o.size () == 2) - { - if (++i == e) - break; // Let the compiler complain. - - d = dir_path (*i); - } - else - d = dir_path (*i, 2, string::npos); - - l6 ([&]{trace << "-I " << d;}); - - if (d.relative ()) - fail << "relative -I directory " << d - << " in variable " << var.name - << " for target " << t; - - // If we are not inside our project root, then ignore. - // - if (!d.sub (out_root)) - continue; - - // If the target directory is a sub-directory of the include - // directory, then the prefix is the difference between the - // two. Otherwise, leave it empty. - // - // The idea here is to make this "canonical" setup work auto- - // magically: - // - // 1. We include all files with a prefix, e.g., . - // 2. The library target is in the foo/ sub-directory, e.g., - // /tmp/foo/. - // 3. The poptions variable contains -I/tmp. - // - dir_path p (out_base.sub (d) ? out_base.leaf (d) : dir_path ()); - - // We use the target's directory as out_base but that doesn't work - // well for targets that are stashed in subdirectories. So as a - // heuristics we are going to also enter the outer directories of - // the original prefix. It is, however, possible, that another -I - // option after this one will produce one of these outer prefixes as - // its original prefix in which case we should override it. - // - // So we are going to assign the original prefix priority value 0 - // (highest) and then increment it for each outer prefix. - // - auto enter = [&trace, &m] (dir_path p, dir_path d, size_t prio) - { - auto j (m.find (p)); - - if (j != m.end ()) - { - prefix_value& v (j->second); - - // We used to reject duplicates but it seems this can be - // reasonably expected to work according to the order of the - // -I options. - // - // Seeing that we normally have more "specific" -I paths first, - // (so that we don't pick up installed headers, etc), we ignore - // it. - // - if (v.directory == d) - { - if (v.priority > prio) - v.priority = prio; - } - else if (v.priority <= prio) - { - if (verb >= 4) - trace << "ignoring dependency prefix " << p << '\n' - << " existing mapping to " << v.directory - << " priority " << v.priority << '\n' - << " another mapping to " << d - << " priority " << prio; - } - else - { - if (verb >= 4) - trace << "overriding dependency prefix " << p << '\n' - << " existing mapping to " << v.directory - << " priority " << v.priority << '\n' - << " new mapping to " << d - << " priority " << prio; - - v.directory = move (d); - v.priority = prio; - } - } - else - { - l6 ([&]{trace << p << " -> " << d << " priority " << prio;}); - m.emplace (move (p), prefix_value {move (d), prio}); - } - }; - - size_t prio (0); - for (bool e (false); !e; ++prio) - { - dir_path n (p.directory ()); - e = n.empty (); - enter ((e ? move (p) : p), (e ? move (d) : d), prio); - p = move (n); - } - } - } - } - - auto compile:: - build_prefix_map (const scope& bs, - target& t, - action act, - linfo li) const -> prefix_map - { - prefix_map m; - - // First process our own. - // - append_prefixes (m, t, c_poptions); - append_prefixes (m, t, x_poptions); - - // Then process the include directories from prerequisite libraries. - // - append_lib_prefixes (bs, m, t, act, li); - - return m; - } - - // Return the next make prerequisite starting from the specified - // position and update position to point to the start of the - // following prerequisite or l.size() if there are none left. - // - static string - next_make (const string& l, size_t& p) - { - size_t n (l.size ()); - - // Skip leading spaces. - // - for (; p != n && l[p] == ' '; p++) ; - - // Lines containing multiple prerequisites are 80 characters max. - // - string r; - r.reserve (n); - - // Scan the next prerequisite while watching out for escape sequences. - // - for (; p != n && l[p] != ' '; p++) - { - char c (l[p]); - - if (p + 1 != n) - { - if (c == '$') - { - // Got to be another (escaped) '$'. - // - if (l[p + 1] == '$') - ++p; - } - else if (c == '\\') - { - // This may or may not be an escape sequence depending on whether - // what follows is "escapable". - // - switch (c = l[++p]) - { - case '\\': break; - case ' ': break; - default: c = '\\'; --p; // Restore. - } - } - } - - r += c; - } - - // Skip trailing spaces. - // - for (; p != n && l[p] == ' '; p++) ; - - // Skip final '\'. - // - if (p == n - 1 && l[p] == '\\') - p++; - - return r; - } - - // VC /showIncludes output. The first line is the file being compiled - // (handled by our caller). Then we have the list of headers, one per - // line, in this form (text can presumably be translated): - // - // Note: including file: C:\Program Files (x86)\[...]\iostream - // - // Finally, if we hit a non-existent header, then we end with an error - // line in this form: - // - // x.cpp(3): fatal error C1083: Cannot open include file: 'd/h.hpp': - // No such file or directory - // - // Distinguishing between the include note and the include error is - // easy: we can just check for C1083. Distinguising between the note and - // other errors/warnings is harder: an error could very well end with - // what looks like a path so we cannot look for the note but rather have - // to look for an error. Here we assume that a line containing ' CNNNN:' - // is an error. Should be robust enough in the face of language - // translation, etc. - // - // It turns out C1083 is also used when we are unable to open the main - // source file and the error line looks like this: - // - // c1xx: fatal error C1083: Cannot open source file: 's.cpp': No such - // file or directory - - // Sense whether this is an include note (return npos) or a diagnostics - // line (return postion of the NNNN code in CNNNN). - // - static inline size_t - next_show_sense (const string& l) - { - size_t p (l.find (':')); - - for (size_t n (l.size ()); - p != string::npos; - p = ++p != n ? l.find (':', p) : string::npos) - { - auto isnum = [](char c) {return c >= '0' && c <= '9';}; - - if (p > 5 && - l[p - 6] == ' ' && - l[p - 5] == 'C' && - isnum (l[p - 4]) && - isnum (l[p - 3]) && - isnum (l[p - 2]) && - isnum (l[p - 1])) - { - p -= 4; // Start of the error code. - break; - } - } - - return p; - } - - // Extract the include path from the VC /showIncludes output line. Return - // empty string if the line is not an include note or include error. Set - // the good_error flag if it is an include error (which means the process - // will terminate with the error status that needs to be ignored). - // - static string - next_show (const string& l, bool& good_error) - { - // The include error should be the last line that we handle. - // - assert (!good_error); - - size_t p (next_show_sense (l)); - if (p == string::npos) - { - // Include note. We assume the path is always at the end but need to - // handle both absolute Windows and POSIX ones. - // - // Note that VC appears to always write the absolute path to the - // included file even if it is ""-included and the source path is - // relative. Aren't we lucky today? - // - p = l.rfind (':'); - - if (p != string::npos) - { - // See if this one is part of the Windows drive letter. - // - if (p > 1 && p + 1 < l.size () && // 2 chars before, 1 after. - l[p - 2] == ' ' && - alpha (l[p - 1]) && - path::traits::is_separator (l[p + 1])) - p = l.rfind (':', p - 2); - } - - if (p != string::npos) - { - // VC uses indentation to indicate the include nesting so there - // could be any number of spaces after ':'. Skip them. - // - p = l.find_first_not_of (' ', p + 1); - } - - if (p == string::npos) - fail << "unable to parse /showIncludes include note line"; - - return string (l, p); - } - else if (l.compare (p, 4, "1083") == 0 && - l.compare (0, 5, "c1xx:") != 0 /* Not the main source file. */ ) - { - // Include error. The path is conveniently quoted with ''. - // - size_t p2 (l.rfind ('\'')); - - if (p2 != string::npos && p2 != 0) - { - size_t p1 (l.rfind ('\'', p2 - 1)); - - if (p1 != string::npos) - { - good_error = true; - return string (l, p1 + 1 , p2 - p1 - 1); - } - } - - fail << "unable to parse /showIncludes include error line" << endf; - } - else - { - // Some other error. - // - return string (); - } - } - - // Extract and inject header dependencies. Return the preprocessed source - // file as well as an indication if it is usable for compilation (see - // below for details). - // - pair compile:: - extract_headers (action act, - const scope& bs, - file& t, - linfo li, - const file& src, - const match_data& md, - depdb& dd, - bool& updating, - timestamp mt) const - { - tracer trace (x, "compile::extract_headers"); - - l5 ([&]{trace << "target: " << t;}); - - auto_rmfile psrc; - bool puse (true); - - // If things go wrong (and they often do in this area), give the user a - // bit extra context. - // - auto df = make_diag_frame ( - [&src](const diag_record& dr) - { - if (verb != 0) - dr << info << "while extracting header dependencies from " << src; - }); - - const scope& rs (*bs.root_scope ()); - - // Preprocess mode that preserves as much information as possible while - // still performing inclusions. Also serves as a flag indicating whether - // this compiler uses the separate preprocess and compile setup. - // - const char* pp (nullptr); - - switch (cid) - { - case compiler_id::gcc: - { - // -fdirectives-only is available since GCC 4.3.0. - // - if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) - pp = "-fdirectives-only"; - - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // -frewrite-includes is available since vanilla Clang 3.2.0. - // - // Apple Clang 5.0 is based on LLVM 3.3svn so it should have this - // option (4.2 is based on 3.2svc so it may or may not have it and, - // no, we are not going to try to find out). - // - if (cid == compiler_id::clang_apple - ? (cmaj >= 5) - : (cmaj > 3 || (cmaj == 3 && cmin >= 2))) - pp = "-frewrite-includes"; - - break; - } - case compiler_id::msvc: - { - pp = "/C"; - break; - } - case compiler_id::icc: - break; - } - - // Initialize lazily, only if required. - // - environment env; - cstrings args; - string out; // Storage. - - // Some compilers in certain modes (e.g., when also producing the - // preprocessed output) are incapable of writing the dependecy - // information to stdout. In this case we use a temporary file. - // - auto_rmfile drm; - - // Here is the problem: neither GCC nor Clang allow -MG (treat missing - // header as generated) when we produce any kind of other output (-MD). - // And that's probably for the best since otherwise the semantics gets - // pretty hairy (e.g., what is the exit code and state of the output)? - // - // One thing to note about generated headers: if we detect one, then, - // after generating it, we re-run the compiler since we need to get - // this header's dependencies. - // - // So this is how we are going to work around this problem: we first run - // with -E but without -MG. If there are any errors (maybe because of - // generated headers maybe not), we restart with -MG and without -E. If - // this fixes the error (so it was a generated header after all), then - // we have to restart at which point we go back to -E and no -MG. And we - // keep yo-yoing like this. Missing generated headers will probably be - // fairly rare occurrence so this shouldn't be too expensive. - // - // Actually, there is another error case we would like to handle: an - // outdated generated header that is now causing an error (e.g., because - // of a check that is now triggering #error or some such). So there are - // actually three error cases: outdated generated header, missing - // generated header, and some other error. To handle the outdated case - // we need the compiler to produce the dependency information even in - // case of an error. Clang does it, for VC we parse diagnostics - // ourselves, but GCC does not (but a patch has been submitted). - // - // So the final plan is then as follows: - // - // 1. Start wothout -MG and with suppressed diagnostics. - // 2. If error but we've updated a header, then repeat step 1. - // 3. Otherwise, restart with -MG and diagnostics. - // - // Note that below we don't even check if the compiler supports the - // dependency info on error. We just try to use it and if it's not - // there we ignore the io error since the compiler has failed. - // - bool args_gen; // Current state of args. - size_t args_i; // Start of the -M/-MD "tail". - - // Ok, all good then? Not so fast, the rabbit hole is deeper than it - // seems: When we run with -E we have to discard diagnostics. This is - // not a problem for errors since they will be shown on the re-run but - // it is for (preprocessor) warnings. - // - // Clang's -frewrite-includes is nice in that it preserves the warnings - // so they will be shown during the compilation of the preprocessed - // source. They are also shown during -E but that we discard. And unlike - // GCC, in Clang -M does not imply -w (disable warnings) so it would - // have been shown in -M -MG re-runs but we suppress that with explicit - // -w. All is good in the Clang land then (even -Werror works nicely). - // - // GCC's -fdirective-only, on the other hand, processes all the - // directives so they are gone from the preprocessed source. Here is - // what we are going to do to work around this: we will detect if any - // diagnostics has been written to stderr on the -E run. If that's the - // case (but the compiler indicated success) then we assume they are - // warnings and disable the use of the preprocessed output for - // compilation. This in turn will result in compilation from source - // which will display the warnings. Note that we may still use the - // preprocessed output for other things (e.g., C++ module dependency - // discovery). BTW, another option would be to collect all the - // diagnostics and then dump it if the run is successful, similar to - // the VC semantics (and drawbacks) described below. - // - // Finally, for VC, things are completely different: there is no -MG - // equivalent and we handle generated headers by analyzing the - // diagnostics. This means that unlike in the above two cases, the - // preprocessor warnings are shown during dependency extraction, not - // compilation. Not ideal but that's the best we can do. Or is it -- we - // could implement ad hoc diagnostics sensing... It appears warnings are - // in the C4000-C4999 code range though there can also be note lines - // which don't have any C-code. - // - // BTW, triggering a warning in the VC preprocessor is not easy; there - // is no #warning and pragmas are passed through to the compiler. One - // way to do it is to redefine a macro, for example: - // - // hello.cxx(4): warning C4005: 'FOO': macro redefinition - // hello.cxx(3): note: see previous definition of 'FOO' - // - // So seeing that it is hard to trigger a legitimate VC preprocessor - // warning, for now, we will just treat them as errors by adding /WX. - // - // Note: diagnostics sensing is currently only supported if dependency - // info is written to a file (see above). - // - bool sense_diag (false); - - // And here is another problem: if we have an already generated header - // in src and the one in out does not yet exist, then the compiler will - // pick the one in src and we won't even notice. Note that this is not - // only an issue with mixing in- and out-of-tree builds (which does feel - // wrong but is oh so convenient): this is also a problem with - // pre-generated headers, a technique we use to make installing the - // generator by end-users optional by shipping pre-generated headers. - // - // This is a nasty problem that doesn't seem to have a perfect solution - // (except, perhaps, C++ modules). So what we are going to do is try to - // rectify the situation by detecting and automatically remapping such - // mis-inclusions. It works as follows. - // - // First we will build a map of src/out pairs that were specified with - // -I. Here, for performance and simplicity, we will assume that they - // always come in pairs with out first and src second. We build this - // map lazily only if we are running the preprocessor and reuse it - // between restarts. - // - // With the map in hand we can then check each included header for - // potentially having a doppelganger in the out tree. If this is the - // case, then we calculate a corresponding header in the out tree and, - // (this is the most important part), check if there is a target for - // this header in the out tree. This should be fairly accurate and not - // require anything explicit from the user except perhaps for a case - // where the header is generated out of nothing (so there is no need to - // explicitly mention its target in the buildfile). But this probably - // won't be very common. - // - // One tricky area in this setup are target groups: if the generated - // sources are mentioned in the buildfile as a group, then there might - // be no header target (yet). The way we solve this is by requiring code - // generator rules to cooperate and create at least the header target as - // part of the group creation. While not all members of the group may be - // generated depending on the options (e.g., inline files might be - // suppressed), headers are usually non-optional. - // - // Note that we use path_map instead of dir_path_map to allow searching - // using path (file path). - // - using srcout_map = path_map; - srcout_map so_map; - - // The gen argument to init_args() is in/out. The caller signals whether - // to force the generated header support and on return it signals - // whether this support is enabled. The first call to init_args is - // expected to have gen false. - // - // Return NULL if the dependency information goes to stdout and a - // pointer to the temporary file path otherwise. - // - auto init_args = [&t, act, li, - &src, &md, &psrc, &sense_diag, - &rs, &bs, - pp, &env, &args, &args_gen, &args_i, &out, &drm, - &so_map, this] - (bool& gen) -> const path* - { - const path* r (nullptr); - - if (args.empty ()) // First call. - { - assert (!gen); - - // We use absolute/relative paths in the dependency output to - // distinguish existing headers from (missing) generated. Which - // means we have to (a) use absolute paths in -I and (b) pass - // absolute source path (for ""-includes). That (b) is a problem: - // if we use an absolute path, then all the #line directives will be - // absolute and all the diagnostics will have long, noisy paths - // (actually, we will still have long paths for diagnostics in - // headers). - // - // To work around this we used to pass a relative path to the source - // file and then check every relative path in the dependency output - // for existence in the source file's directory. This is not without - // issues: it is theoretically possible for a generated header that - // is <>-included and found via -I to exist in the source file's - // directory. Note, however, that this is a lot more likely to - // happen with prefix-less inclusion (e.g., ) and in this case - // we assume the file is in the project anyway. And if there is a - // conflict with a prefixed include (e.g., ), then, well, - // we will just have to get rid of quoted includes (which are - // generally a bad idea, anyway). - // - // But then this approach (relative path) fell apart further when we - // tried to implement precise changed detection: the preprocessed - // output would change depending from where it was compiled because - // of #line (which we could work around) and __FILE__/assert() - // (which we can't really do anything about). So it looks like using - // the absolute path is the lesser of all the evils (and there are - // many). - // - // Note that we detect and diagnose relative -I directories lazily - // when building the include prefix map. - // - args.push_back (cpath.recall_string ()); - - // Add *.export.poptions from prerequisite libraries. - // - append_lib_options (bs, args, t, act, li); - - append_options (args, t, c_poptions); - append_options (args, t, x_poptions); - - // Populate the src-out with the -I$out_base -I$src_base pairs. - // - { - // Try to be fast and efficient by reusing buffers as much as - // possible. - // - string ds; - - // Previous -I innermost scope if out_base plus the difference - // between the scope path and the -I path (normally empty). - // - const scope* s (nullptr); - dir_path p; - - for (auto i (args.begin ()), e (args.end ()); i != e; ++i) - { - // -I can either be in the "-Ifoo" or "-I foo" form. For VC it - // can also be /I. - // - const char* o (*i); - size_t n (strlen (o)); - - if (n < 2 || (o[0] != '-' && o[0] != '/') || o[1] != 'I') - { - s = nullptr; - continue; - } - - if (n == 2) - { - if (++i == e) - break; // Let the compiler complain. - - ds = *i; - } - else - ds.assign (o + 2, n - 2); - - if (!ds.empty ()) - { - // Note that we don't normalize the paths since it would be - // quite expensive and normally the pairs we are inerested in - // are already normalized (since they are usually specified as - // -I$src/out_*). We just need to add a trailing directory - // separator if it's not already there. - // - if (!dir_path::traits::is_separator (ds.back ())) - ds += dir_path::traits::directory_separator; - - dir_path d (move (ds), dir_path::exact); // Move the buffer in. - - // Ignore invalid paths (buffer is not moved). - // - if (!d.empty ()) - { - // Ignore any paths containing '.', '..' components. Allow - // any directory separators thought (think -I$src_root/foo - // on Windows). - // - if (d.absolute () && d.normalized (false)) - { - // If we have a candidate out_base, see if this is its - // src_base. - // - if (s != nullptr) - { - const dir_path& bp (s->src_path ()); - - if (d.sub (bp)) - { - if (p.empty () || d.leaf (bp) == p) - { - // We've got a pair. - // - so_map.emplace (move (d), s->out_path () / p); - s = nullptr; // Taken. - continue; - } - } - - // Not a pair. Fall through to consider as out_base. - // - s = nullptr; - } - - // See if this path is inside a project with an out-of- - // tree build and is in the out directory tree. - // - const scope& bs (scopes.find (d)); - if (bs.root_scope () != nullptr) - { - const dir_path& bp (bs.out_path ()); - if (bp != bs.src_path ()) - { - bool e; - if ((e = (d == bp)) || d.sub (bp)) - { - s = &bs; - if (e) - p.clear (); - else - p = d.leaf (bp); - } - } - } - } - else - s = nullptr; - - ds = move (d).string (); // Move the buffer out. - } - else - s = nullptr; - } - else - s = nullptr; - } - } - - // Extra system header dirs (last). - // - assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); - append_option_values ( - args, "-I", - sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), - [] (const dir_path& d) {return d.string ().c_str ();}); - - if (md.symexport) - append_symexport_options (args, t); - - // Some compile options (e.g., -std, -m) affect the preprocessor. - // - // Currently Clang supports importing "header modules" even when in - // the TS mode. And "header modules" support macros which means - // imports have to be resolved during preprocessing. Which poses a - // bit of a chicken and egg problem for us. For now, the workaround - // is to remove the -fmodules-ts option when preprocessing. Hopefully - // there will be a "pure modules" mode at some point. - // - - // Don't treat warnings as errors. - // - const char* werror (nullptr); - switch (cclass) - { - case compiler_class::gcc: werror = "-Werror"; break; - case compiler_class::msvc: werror = "/WX"; break; - } - - bool clang (cid == compiler_id::clang || - cid == compiler_id::clang_apple); - - append_options (args, t, c_coptions, werror); - append_options (args, t, x_coptions, werror); - append_options (args, tstd, - tstd.size () - (modules && clang ? 1 : 0)); - - switch (cclass) - { - case compiler_class::msvc: - { - assert (pp != nullptr); - - args.push_back ("/nologo"); - - // See perform_update() for details on overriding the default - // exceptions and runtime. - // - if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) - args.push_back ("/EHsc"); - - if (!find_option_prefixes ({"/MD", "/MT"}, args)) - args.push_back ("/MD"); - - args.push_back ("/P"); // Preprocess to file. - args.push_back ("/showIncludes"); // Goes to stdout (with diag). - args.push_back (pp); // /C (preserve comments). - args.push_back ("/WX"); // Warning as error (see above). - - psrc = auto_rmfile (t.path () + x_pext); - - if (cast (rs[x_version_major]) >= 18) - { - args.push_back ("/Fi:"); - args.push_back (psrc.path.string ().c_str ()); - } - else - { - out = "/Fi" + psrc.path.string (); - args.push_back (out.c_str ()); - } - - args.push_back (langopt (md)); // Compile as. - gen = args_gen = true; - break; - } - case compiler_class::gcc: - { - if (t.is_a ()) - { - // On Darwin, Win32 -fPIC is the default. - // - if (tclass == "linux" || tclass == "bsd") - args.push_back ("-fPIC"); - } - - // Depending on the compiler, decide whether (and how) we can - // produce preprocessed output as a side effect of dependency - // extraction. - // - // Note: -MM -MG skips missing <>-included. - - // Clang's -M does not imply -w (disable warnings). We also - // don't need them in the -MD case (see above) so disable for - // both. - // - if (clang) - args.push_back ("-w"); - - // Previously we used '*' as a target name but it gets expanded - // to the current directory file names by GCC (4.9) that comes - // with MSYS2 (2.4). Yes, this is the (bizarre) behavior of GCC - // being executed in the shell with -MQ '*' option and not just - // -MQ *. - // - args.push_back ("-MQ"); // Quoted target name. - args.push_back ("^"); // Old versions can't do empty target. - - args.push_back ("-x"); - args.push_back (langopt (md)); - - if (pp != nullptr) - { - // Note that the options are carefully laid out to be easy to - // override (see below). - // - args_i = args.size (); - - args.push_back ("-MD"); - args.push_back ("-E"); - args.push_back (pp); - - // Dependency output. - // - args.push_back ("-MF"); - - // GCC is not capable of writing the dependency info to - // stdout. We also need to sense the diagnostics on the -E - // runs. - // - if (cid == compiler_id::gcc) - { - // Use the .t extension (for "temporary"; .d is taken). - // - r = &(drm = auto_rmfile (t.path () + ".t")).path; - args.push_back (r->string ().c_str ()); - - sense_diag = true; - } - else - args.push_back ("-"); - - // Preprocessor output. - // - psrc = auto_rmfile (t.path () + x_pext); - args.push_back ("-o"); - args.push_back (psrc.path.string ().c_str ()); - } - else - { - args.push_back ("-M"); - args.push_back ("-MG"); // Treat missing headers as generated. - } - - gen = args_gen = (pp == nullptr); - break; - } - } - - args.push_back (src.path ().string ().c_str ()); - args.push_back (nullptr); - - // Note: only doing it here. - // - if (!env.empty ()) - env.push_back (nullptr); - } - else - { - assert (gen != args_gen); - - size_t i (args_i); - - if (gen) - { - // Overwrite. - // - args[i++] = "-M"; - args[i++] = "-MG"; - args[i++] = src.path ().string ().c_str (); - args[i] = nullptr; - - if (cid == compiler_id::gcc) - { - sense_diag = false; - } - } - else - { - // Restore. - // - args[i++] = "-MD"; - args[i++] = "-E"; - args[i++] = pp; - args[i] = "-MF"; - - if (cid == compiler_id::gcc) - { - r = &drm.path; - sense_diag = true; - } - } - - args_gen = gen; - } - - return r; - }; - - // Build the prefix map lazily only if we have non-existent files. - // Also reuse it over restarts since it doesn't change. - // - optional pfx_map; - - // If any prerequisites that we have extracted changed, then we have to - // redo the whole thing. The reason for this is auto-generated headers: - // the updated header may now include a yet-non-existent header. Unless - // we discover this and generate it (which, BTW, will trigger another - // restart since that header, in turn, can also include auto-generated - // headers), we will end up with an error during compilation proper. - // - // One complication with this restart logic is that we will see a - // "prefix" of prerequisites that we have already processed (i.e., they - // are already in our prerequisite_targets list) and we don't want to - // keep redoing this over and over again. One thing to note, however, is - // that the prefix that we have seen on the previous run must appear - // exactly the same in the subsequent run. The reason for this is that - // none of the files that it can possibly be based on have changed and - // thus it should be exactly the same. To put it another way, the - // presence or absence of a file in the dependency output can only - // depend on the previous files (assuming the compiler outputs them as - // it encounters them and it is hard to think of a reason why would - // someone do otherwise). And we have already made sure that all those - // files are up to date. And here is the way we are going to exploit - // this: we are going to keep track of how many prerequisites we have - // processed so far and on restart skip right to the next one. - // - // And one more thing: most of the time this list of headers would stay - // unchanged and extracting them by running the compiler every time is a - // bit wasteful. So we are going to cache them in the depdb. If the db - // hasn't been invalidated yet (e.g., because the compiler options have - // changed), then we start by reading from it. If anything is out of - // date then we use the same restart and skip logic to switch to the - // compiler run. - // - size_t skip_count (0); - - // Update and add a header file to the list of prerequisite targets. - // Depending on the cache flag, the file is assumed to either have come - // from the depdb cache or from the compiler run. Return whether the - // extraction process should be restarted. - // - auto add = [&trace, &pfx_map, &so_map, - act, &t, li, - &dd, &updating, &skip_count, - &bs, this] - (path f, bool cache, timestamp mt) -> bool - { - // Find or maybe insert the target. The directory is only moved - // from if insert is true. - // - auto find = [&trace, &t, this] - (dir_path&& d, path&& f, bool insert) -> const path_target* - { - // Split the file into its name part and extension. Here we can - // assume the name part is a valid filesystem name. - // - // Note that if the file has no extension, we record an empty - // extension rather than NULL (which would signify that the default - // extension should be added). - // - string e (f.extension ()); - string n (move (f).string ()); - - if (!e.empty ()) - n.resize (n.size () - e.size () - 1); // One for the dot. - - // Determine the target type. - // - const target_type* tt (nullptr); - - // See if this directory is part of any project out_root hierarchy. - // Note that this will miss all the headers that come from src_root - // (so they will be treated as generic C headers below). Generally, - // we don't have the ability to determine that some file belongs to - // src_root of some project. But that's not a problem for our - // purposes: it is only important for us to accurately determine - // target types for headers that could be auto-generated. - // - // While at it also try to determine if this target is from the src - // or out tree of said project. - // - dir_path out; - - const scope& bs (scopes.find (d)); - if (const scope* rs = bs.root_scope ()) - { - tt = map_extension (bs, n, e); - - if (bs.out_path () != bs.src_path () && d.sub (bs.src_path ())) - out = out_src (d, *rs); - } - - // If it is outside any project, or the project doesn't have such an - // extension, assume it is a plain old C header. - // - if (tt == nullptr) - { - // If the project doesn't "know" this extension then we won't - // possibly find an explicit target of this type. - // - if (!insert) - return nullptr; - - tt = &h::static_type; - } - - // Find or insert target. - // - // @@ OPT: move d, out, n - // - const target* r; - if (insert) - r = &search (t, *tt, d, out, n, &e, nullptr); - else - { - // Note that we skip any target type-specific searches (like for - // an existing file) and go straight for the target object since - // we need to find the target explicitly spelled out. - // - r = targets.find (*tt, d, out, n, e, trace); - } - - return static_cast (r); - }; - - // If it's not absolute then it either does not (yet) exist or is - // a relative ""-include (see init_args() for details). Reduce the - // second case to absolute. - // - // Note: we now always use absolute path to the translation unit so - // this no longer applies. - // -#if 0 - if (f.relative () && rels.relative ()) - { - // If the relative source path has a directory component, make sure - // it matches since ""-include will always start with that (none of - // the compilers we support try to normalize this path). Failed that - // we may end up searching for a generated header in a random - // (working) directory. - // - const string& fs (f.string ()); - const string& ss (rels.string ()); - - size_t p (path::traits::rfind_separator (ss)); - - if (p == string::npos || // No directory. - (fs.size () > p + 1 && - path::traits::compare (fs.c_str (), p, ss.c_str (), p) == 0)) - { - path t (work / f); // The rels path is relative to work. - - if (exists (t)) - f = move (t); - } - } -#endif - - const path_target* pt (nullptr); - - // If still relative then it does not exist. - // - if (f.relative ()) - { - f.normalize (); - - // This is probably as often an error as an auto-generated file, so - // trace at level 4. - // - l4 ([&]{trace << "non-existent header '" << f << "'";}); - - if (!pfx_map) - pfx_map = build_prefix_map (bs, t, act, li); - - // First try the whole file. Then just the directory. - // - // @@ Has to be a separate map since the prefix can be the same as - // the file name. - // - // auto i (pfx_map->find (f)); - - // Find the most qualified prefix of which we are a sub-path. - // - if (!pfx_map->empty ()) - { - dir_path d (f.directory ()); - auto i (pfx_map->find_sup (d)); - - if (i != pfx_map->end ()) - { - const dir_path& pd (i->second.directory); - - // If this is a prefixless mapping, then only use it if we can - // resolve it to an existing target (i.e., it is explicitly - // spelled out in a buildfile). - // - // Note that at some point we will probably have a list of - // directories. - // - pt = find (pd / d, f.leaf (), !i->first.empty ()); - if (pt != nullptr) - { - f = pd / f; - l4 ([&]{trace << "mapped as auto-generated " << f;}); - } - } - } - - if (pt == nullptr) - { - diag_record dr (fail); - dr << "header '" << f << "' not found and cannot be generated"; - //for (const auto& p: pm) - // dr << info << p.first.string () << " -> " << p.second.string (); - } - } - else - { - // We used to just normalize the path but that could result in an - // invalid path (e.g., on CentOS 7 with Clang 3.4) because of the - // symlinks. So now we realize (i.e., realpath(3)) it instead. - // Unless it comes from the depdb, in which case we've already done - // that. This is also where we handle src-out remap (again, not - // needed if cached) - // - if (!cache) - { - // While we can reasonably expect this path to exit, things do - // go south from time to time (like compiling under wine with - // file wlantypes.h included as WlanTypes.h). - // - try - { - f.realize (); - } - catch (const invalid_path&) - { - fail << "invalid header path '" << f << "'"; - } - catch (const system_error& e) - { - fail << "invalid header path '" << f << "': " << e; - } - - if (!so_map.empty ()) - { - // Find the most qualified prefix of which we are a sub-path. - // - auto i (so_map.find_sup (f)); - if (i != so_map.end ()) - { - // Ok, there is an out tree for this headers. Remap to a path - // from the out tree and see if there is a target for it. - // - dir_path d (i->second); - d /= f.leaf (i->first).directory (); - pt = find (move (d), f.leaf (), false); // d is not moved from. - - if (pt != nullptr) - { - path p (d / f.leaf ()); - l4 ([&]{trace << "remapping " << f << " to " << p;}); - f = move (p); - } - } - } - } - - if (pt == nullptr) - { - l6 ([&]{trace << "injecting " << f;}); - pt = find (f.directory (), f.leaf (), true); - } - } - - // Cache the path. - // - const path& pp (pt->path (move (f))); - - // Match to a rule. - // - // If we are reading the cache, then it is possible the file has since - // been removed (think of a header in /usr/local/include that has been - // uninstalled and now we need to use one from /usr/include). This - // will lead to the match failure which we translate to a restart. - // - if (!cache) - build2::match (act, *pt); - else if (!build2::try_match (act, *pt).first) - { - dd.write (); // Invalidate this line. - updating = true; - return true; - } - - // Update. - // - bool restart (update (trace, act, *pt, mt)); - - // Verify/add it to the dependency database. We do it after update in - // order not to add bogus files (non-existent and without a way to - // update). - // - if (!cache) - dd.expect (pp); - - // Add to our prerequisite target list. - // - t.prerequisite_targets.push_back (pt); - skip_count++; - - updating = updating || restart; - return restart; - }; - - // If nothing so far has invalidated the dependency database, then try - // the cached data before running the compiler. - // - bool cache (!updating); - - // See init_args() above for details on generated header support. - // - bool gen (false); - optional force_gen; - optional force_gen_skip; // Skip count at last force_gen run. - - const path* drmp (nullptr); // Points to drm.path () if active. - - for (bool restart (true); restart; cache = false) - { - restart = false; - - if (cache) - { - // If any, this is always the first run. - // - assert (skip_count == 0); - - // We should always end with a blank line. - // - for (;;) - { - string* l (dd.read ()); - - // If the line is invalid, run the compiler. - // - if (l == nullptr) - { - restart = true; - break; - } - - if (l->empty ()) // Done, nothing changed. - { - // If modules are enabled, then we keep the preprocessed output - // around (see apply() for details). - // - return modules - ? make_pair (auto_rmfile (t.path () + x_pext, false), true) - : make_pair (auto_rmfile (), false); - } - - // If this header came from the depdb, make sure it is no older - // than the target (if it has changed since the target was - // updated, then the cached data is stale). - // - restart = add (path (move (*l)), true, mt); - - if (restart) - { - l6 ([&]{trace << "restarting (cache)";}); - break; - } - } - } - else - { - try - { - if (force_gen) - gen = *force_gen; - - if (args.empty () || gen != args_gen) - drmp = init_args (gen); - - if (verb >= 3) - print_process (args.data ()); // Disable pipe mode. - - process pr; - - try - { - // Assume the preprocessed output (if produced) is usable - // until proven otherwise. - // - puse = true; - - // Save the timestamp just before we start preprocessing. If - // we depend on any header that has been updated since, then - // we should assume we've "seen" the old copy and re-process. - // - timestamp pmt (system_clock::now ()); - - // If we have no generated header support, then suppress all - // diagnostics (if things go badly we will restart with this - // support). - // - if (drmp == nullptr) - { - // Dependency info goes to stdout. - // - assert (!sense_diag); - - // For VC with /P the dependency info and diagnostics all go - // to stderr so redirect it to stdout. - // - pr = process ( - cpath, - args.data (), - 0, - -1, - cclass == compiler_class::msvc ? 1 : gen ? 2 : -2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - } - else - { - // Dependency info goes to a temporary file. - // - pr = process (cpath, - args.data (), - 0, - 2, // Send stdout to stderr. - gen ? 2 : sense_diag ? -1 : -2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - - // If requested, monitor for diagnostics and if detected, mark - // the preprocessed output as unusable for compilation. - // - if (sense_diag) - { - ifdstream is (move (pr.in_efd), fdstream_mode::skip); - puse = puse && (is.peek () == ifdstream::traits_type::eof ()); - is.close (); - } - - // The idea is to reduce it to the stdout case. - // - pr.wait (); - pr.in_ofd = fdopen (*drmp, fdopen_mode::in); - } - - // We may not read all the output (e.g., due to a restart). - // Before we used to just close the file descriptor to signal to - // the other end that we are not interested in the rest. This - // works fine with GCC but Clang (3.7.0) finds this impolite and - // complains, loudly (broken pipe). So now we are going to skip - // until the end. - // - ifdstream is (move (pr.in_ofd), - fdstream_mode::text | fdstream_mode::skip, - ifdstream::badbit); - - // In some cases we may need to ignore the error return status. - // The good_error flag keeps track of that. Similarly we - // sometimes expect the error return status based on the output - // we see. The bad_error flag is for that. - // - bool good_error (false), bad_error (false); - - size_t skip (skip_count); - string l; // Reuse. - for (bool first (true), second (false); !restart; ) - { - if (eof (getline (is, l))) - break; - - l6 ([&]{trace << "header dependency line '" << l << "'";}); - - // Parse different dependency output formats. - // - switch (cclass) - { - case compiler_class::msvc: - { - if (first) - { - // The first line should be the file we are compiling. - // If it is not, then something went wrong even before - // we could compile anything (e.g., file does not - // exist). In this case the first line (and everything - // after it) is presumably diagnostics. - // - if (l != src.path ().leaf ().string ()) - { - text << l; - bad_error = true; - break; - } - - first = false; - continue; - } - - string f (next_show (l, good_error)); - - if (f.empty ()) // Some other diagnostics. - { - text << l; - bad_error = true; - break; - } - - // Skip until where we left off. - // - if (skip != 0) - { - // We can't be skipping over a non-existent header. - // - assert (!good_error); - skip--; - } - else - { - restart = add (path (move (f)), false, pmt); - - // If the header does not exist (good_error), then - // restart must be true. Except that it is possible that - // someone running in parallel has already updated it. - // In this case we must force a restart since we haven't - // yet seen what's after this at-that-time-non-existent - // header. - // - // We also need to force the target update (normally - // done by add()). - // - if (good_error) - restart = updating = true; - // - // And if we have updated the header (restart is true), - // then we may end up in this situation: an old header - // got included which caused the preprocessor to fail - // down the line. So if we are restarting, set the good - // error flag in case the process fails because of - // something like this (and if it is for a valid reason, - // then we will pick it up on the next round). - // - else if (restart) - good_error = true; - - if (restart) - l6 ([&]{trace << "restarting";}); - } - - break; - } - case compiler_class::gcc: - { - // Make dependency declaration. - // - size_t pos (0); - - if (first) - { - // Empty/invalid output should mean the wait() call - // below will return false. - // - if (l.empty () || - l[0] != '^' || l[1] != ':' || l[2] != ' ') - { - if (!l.empty ()) - text << l; - - bad_error = true; - break; - } - - first = false; - second = true; - - // While normally we would have the source file on the - // first line, if too long, it will be moved to the next - // line and all we will have on this line is "^: \". - // - if (l.size () == 4 && l[3] == '\\') - continue; - else - pos = 3; // Skip "^: ". - - // Fall through to the 'second' block. - } - - if (second) - { - second = false; - next_make (l, pos); // Skip the source file. - } - - while (pos != l.size ()) - { - string f (next_make (l, pos)); - - // Skip until where we left off. - // - if (skip != 0) - { - skip--; - continue; - } - - restart = add (path (move (f)), false, pmt); - - if (restart) - { - // The same "preprocessor may fail down the line" - // logic as above. - // - good_error = true; - - l6 ([&]{trace << "restarting";}); - break; - } - } - - break; - } - } - - if (bad_error) - break; - } - - // In case of VC, we are parsing stderr and if things go south, - // we need to copy the diagnostics for the user to see. - // - if (bad_error && cclass == compiler_class::msvc) - { - // We used to just dump the whole rdbuf but it turns out VC - // may continue writing include notes interleaved with the - // diagnostics. So we have to filter them out. - // - for (; !eof (getline (is, l)); ) - { - size_t p (next_show_sense (l)); - if (p != string::npos && l.compare (p, 4, "1083") != 0) - diag_stream_lock () << l << endl; - } - } - - is.close (); - - // This is tricky: it is possible that in parallel someone has - // generated all our missing headers and we wouldn't restart - // normally. - // - // In this case we also need to force the target update - // (normally done by add()). - // - if (force_gen && *force_gen) - { - restart = updating = true; - force_gen = false; - } - - if (pr.wait ()) - { - if (!bad_error) - continue; - - fail << "expected error exist status from " << x_lang - << " compiler"; - } - else if (pr.exit->normal ()) - { - if (good_error) // Ignore expected errors (restart). - continue; - } - - // Fall through. - } - catch (const io_error&) - { - if (pr.wait ()) - fail << "unable to read " << x_lang << " compiler header " - << "dependency output"; - - // Fall through. - } - - assert (pr.exit && !*pr.exit); - const process_exit& e (*pr.exit); - - // For normal exit we assume the child process issued some - // diagnostics. - // - if (e.normal ()) - { - // If this run was with the generated header support then we - // have issued diagnostics and it's time to give up. - // - if (gen) - throw failed (); - - // Just to recap, being here means something is wrong with the - // source: it can be a missing generated header, it can be an - // outdated generated header (e.g., some check triggered #error - // which will go away if only we updated the generated header), - // or it can be a real error that is not going away. - // - // So this is what we are going to do here: if anything got - // updated on this run (i.e., the compiler has produced valid - // dependency information even though there were errors and we - // managed to find and update a header based on this - // informaion), then we restart in the same mode hoping that - // this fixes things. Otherwise, we force the generated header - // support which will either uncover a missing generated header - // or will issue diagnostics. - // - if (restart) - l6 ([&]{trace << "trying again without generated headers";}); - else - { - // In some pathological situations (e.g., we are out of disk - // space) we may end up switching back and forth indefinitely - // without making any headway. So we use skip_count to track - // our progress. - // - if (force_gen_skip && *force_gen_skip == skip_count) - { - diag_record dr (fail); - - dr << "inconsistent " << x_lang << " compiler behavior"; - - // Show the yo-yo'ing command lines. - // - dr << info; - print_process (dr, args.data ()); // No pipes. - - init_args ((gen = true)); - dr << info << ""; - print_process (dr, args.data ()); // No pipes. - - dr << info << "perhaps you are running out of disk space?"; - } - - restart = true; - force_gen = true; - force_gen_skip = skip_count; - l6 ([&]{trace << "restarting with forced generated headers";}); - } - continue; - } - else - run_finish (args, pr); // Throws. - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - // In a multi-threaded program that fork()'ed but did not exec(), - // it is unwise to try to do any kind of cleanup (like unwinding - // the stack and running destructors). - // - if (e.child) - { - drm.cancel (); - exit (1); - } - - throw failed (); - } - } - } - - // Add the terminating blank line (we are updated depdb). - // - dd.expect (""); - - puse = puse && !psrc.path.empty (); - return make_pair (move (psrc), puse); - } - - pair compile:: - parse_unit (action act, - file& t, - linfo lo, - const file& src, - auto_rmfile& psrc, - const match_data& md) const - { - tracer trace (x, "compile::parse_unit"); - - // If things go wrong give the user a bit extra context. - // - auto df = make_diag_frame ( - [&src](const diag_record& dr) - { - if (verb != 0) - dr << info << "while parsing " << src; - }); - - // For some compilers (GCC, Clang) the preporcessed output is only - // partially preprocessed. For others (VC), it is already fully - // preprocessed (well, almost: it still has comments but we can handle - // that). Plus, the source file might already be (sufficiently) - // preprocessed. - // - // So the plan is to start the compiler process that writes the fully - // preprocessed output to stdout and reduce the already preprocessed - // case to it. - // - environment env; - cstrings args; - const path* sp; // Source path. - - bool ps; // True if extracting from psrc. - if (md.pp < preprocessed::modules) - { - ps = !psrc.path.empty (); - sp = &(ps ? psrc.path : src.path ()); - - // VC's preprocessed output, if present, is fully preprocessed. - // - if (cclass != compiler_class::msvc || !ps) - { - // This should match with how we setup preprocessing and is pretty - // similar to init_args() from extract_headers(). - // - args.push_back (cpath.recall_string ()); - - append_lib_options (t.base_scope (), args, t, act, lo); - - append_options (args, t, c_poptions); - append_options (args, t, x_poptions); - - assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); - append_option_values ( - args, "-I", - sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), - [] (const dir_path& d) {return d.string ().c_str ();}); - - if (md.symexport) - append_symexport_options (args, t); - - // Make sure we don't fail because of warnings. - // - // @@ Can be both -WX and /WX. - // - const char* werror (nullptr); - switch (cclass) - { - case compiler_class::gcc: werror = "-Werror"; break; - case compiler_class::msvc: werror = "/WX"; break; - } - - bool clang (cid == compiler_id::clang || - cid == compiler_id::clang_apple); - - append_options (args, t, c_coptions, werror); - append_options (args, t, x_coptions, werror); - append_options (args, tstd, - tstd.size () - (modules && clang ? 1 : 0)); - - switch (cclass) - { - case compiler_class::msvc: - { - args.push_back ("/nologo"); - - if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) - args.push_back ("/EHsc"); - - if (!find_option_prefixes ({"/MD", "/MT"}, args)) - args.push_back ("/MD"); - - args.push_back ("/E"); - args.push_back ("/C"); - args.push_back (langopt (md)); // Compile as. - - break; - } - case compiler_class::gcc: - { - if (t.is_a ()) - { - if (tclass == "linux" || tclass == "bsd") - args.push_back ("-fPIC"); - } - - // Options that trigger preprocessing of partially preprocessed - // output are a bit of a compiler-specific voodoo. - // - args.push_back ("-E"); - - if (ps) - { - args.push_back ("-x"); - args.push_back (langopt (md)); - - if (cid == compiler_id::gcc) - { - args.push_back ("-fpreprocessed"); - args.push_back ("-fdirectives-only"); - } - } - - break; - } - } - - args.push_back (sp->string ().c_str ()); - args.push_back (nullptr); - } - - if (!env.empty ()) - env.push_back (nullptr); - } - else - { - // Extracting directly from source. - // - ps = false; - sp = &src.path (); - } - - // Preprocess and parse. - // - for (;;) // Breakout loop. - try - { - // Disarm the removal of the preprocessed file in case of an error. - // We re-arm it below. - // - if (ps) - psrc.active = false; - - process pr; - - try - { - if (args.empty ()) - { - pr = process (process_exit (0)); // Successfully exited. - pr.in_ofd = fdopen (*sp, fdopen_mode::in); - } - else - { - if (verb >= 3) - print_process (args); - - // We don't want to see warnings multiple times so ignore all - // diagnostics. - // - pr = process (cpath, - args.data (), - 0, -1, -2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - } - - // Use binary mode to obtain consistent positions. - // - ifdstream is (move (pr.in_ofd), - fdstream_mode::binary | fdstream_mode::skip); - - parser p; - translation_unit tu (p.parse (is, *sp)); - - is.close (); - - if (pr.wait ()) - { - if (ps) - psrc.active = true; // Re-arm. - - // Prior to 15u5 VC was not using the 'export module' syntax so we - // use the preprequisite type to distinguish between interface and - // implementation units. - // - if (cid == compiler_id::msvc && - cmaj == 19 && cmin <= 11 && - x_mod != nullptr && src.is_a (*x_mod)) - { - // It's quite painful to guard the export with an #if/#endif so - // if it is present, "fixup" the (temporary) preprocessed output - // by getting rid of the keyword. - // - // Note: when removing this also remember to remove the test. - // - if (tu.mod.iface) - { - // We can only fixup a temporary file. - // - if (!ps) - fail (relative (src)) << "fixup requires preprocessor"; - - // Stomp out the export keyword with spaces. We are using - // std::fstream since our fdstream does not yet support - // seeking. - // - fstream os (psrc.path.string (), fstream::out | fstream::in); - auto pos (static_cast (p.export_pos)); - - if (!os.is_open () || - !os.seekp (pos) || - !os.write (" ", 6)) - fail << "unable to overwrite preprocessor output"; - } - else - tu.mod.iface = true; - } - - return pair (move (tu), p.checksum); - } - - // Fall through. - } - catch (const io_error&) - { - if (pr.wait ()) - fail << "unable to read " << x_lang << " preprocessor output"; - - // Fall through. - } - - assert (pr.exit && !*pr.exit); - const process_exit& e (*pr.exit); - - // What should we do with a normal error exit? Remember we suppressed - // the compiler's diagnostics. We used to issue a warning and continue - // with the assumption that the compilation step will fail with - // diagnostics. The problem with this approach is that we may fail - // before that because the information we return (e.g., module name) - // is bogus. So looks like failing is the only option. - // - if (e.normal ()) - { - fail << "unable to preprocess " << src << - info << "re-run with -s -V to display failing command" << - info << "then run failing command to display compiler diagnostics"; - } - else - run_finish (args, pr); // Throws. - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - if (e.child) - exit (1); - } - - throw failed (); - } - - // Extract and inject module dependencies. - // - void compile:: - extract_modules (action act, - const scope& bs, - file& t, - linfo li, - const compile_target_types& tt, - const file& src, - match_data& md, - module_info&& mi, - depdb& dd, - bool& updating) const - { - tracer trace (x, "compile::extract_modules"); - l5 ([&]{trace << "target: " << t;}); - - // If things go wrong, give the user a bit extra context. - // - auto df = make_diag_frame ( - [&src](const diag_record& dr) - { - if (verb != 0) - dr << info << "while extracting module dependencies from " << src; - }); - - if (!modules) - { - if (!mi.name.empty () || !mi.imports.empty ()) - fail (relative (src)) << "modules support not enabled/available"; - - return; - } - - // Sanity checks. - // - // If we are compiling a module interface unit, make sure it has the - // necessary declarations. - // - if (src.is_a (*x_mod) && (mi.name.empty () || !mi.iface)) - fail << src << " is not a module interface unit"; - - // Search and match all the modules we depend on. If this is a module - // implementation unit, then treat the module itself as if it was - // imported (we insert it first since for some compilers we have to - // differentiate between this special module and real imports). Note: - // move. - // - if (!mi.iface && !mi.name.empty ()) - mi.imports.insert (mi.imports.begin (), - module_import {move (mi.name), false, 0}); - - // The change to the set of imports would have required a change to - // source code (or options). Changes to the bmi{}s themselves will be - // detected via the normal prerequisite machinery. However, the same set - // of imports could be resolved to a different set of bmi{}s (in a sense - // similar to changing the source file). To detect this we calculate and - // store a hash of all (not just direct) bmi{}'s paths. - // - sha256 cs; - - if (!mi.imports.empty ()) - md.mods = search_modules (act, bs, t, li, tt.bmi, src, mi.imports, cs); - - if (dd.expect (cs.string ()) != nullptr) - updating = true; - -#if 0 - // Save the module map for compilers that use it. - // - if (md.mods.start != 0) - { - switch (cid) - { - case compiler_id::gcc: - case compiler_id::clang: - case compiler_id::clang_apple: - { - // We don't need to redo this if the above hash hasn't changed and - // the database is valid. - // - if (dd.writing () || !dd.skip ()) - { - const auto& pts (t.prerequisite_targets); - - for (size_t i (md.mods.start); i != pts.size (); ++i) - { - if (const target* m = pts[i]) - { - // Save a variable lookup by getting the module name from - // the import list (see search_modules()). - // - dd.write ('@', false); - dd.write (mi.imports[i - md.mods.start].name, false); - dd.write ('=', false); - dd.write (m->as ().path ()); - } - } - } - break; - } - default: - break; - } - } -#endif - - // Set the cc.module_name variable if this is an interface unit. Note - // that it may seem like a good idea to set it on the bmi{} group to - // avoid duplication. We, however, cannot do it MT-safely since we don't - // match the group. - // - if (mi.iface) - { - if (value& v = t.vars.assign (c_module_name)) - assert (cast (v) == mi.name); - else - v = move (mi.name); // Note: move. - } - } - - inline bool - std_module (const string& m) - { - size_t n (m.size ()); - return (n >= 3 && - m[0] == 's' && m[1] == 't' && m[2] == 'd' && - (n == 3 || m[3] == '.')); - }; - - // Resolve imported modules to bmi*{} targets. - // - module_positions compile:: - search_modules (action act, - const scope& bs, - file& t, - linfo li, - const target_type& mtt, - const file& src, - module_imports& imports, - sha256& cs) const - { - tracer trace (x, "compile::search_modules"); - - // So we have a list of imports and a list of "potential" module - // prerequisites. They are potential in the sense that they may or may - // not be required by this translation unit. In other words, they are - // the pool where we can resolve actual imports. - // - // Because we may not need all of these prerequisites, we cannot just go - // ahead and match all of them (and they can even have cycles; see rule - // synthesis). This poses a bit of a problem: the only way to discover - // the module's actual name (see cc.module_name) is by matching it. - // - // One way to solve this would be to make the user specify the module - // name for each mxx{} explicitly. This will be a major pain, however. - // Another would be to require encoding of the module name in the - // interface unit file name. For example, hello.core -> hello-core.mxx. - // This is better but still too restrictive: some will want to call it - // hello_core.mxx or HelloCore.mxx (because that's their file naming - // convention) or place it in a subdirectory, say, hello/core.mxx. - // - // In the above examples one common theme about all the file names is - // that they contain, in one form or another, the "tail" of the module - // name ('core'). So what we are going to do is require that the - // interface file names contain enough of the module name tail to - // unambiguously resolve all the module imports. On our side we are - // going to implement a "fuzzy" module name to file name match. This - // should be reliable enough since we will always verify our guesses - // once we match the target and extract the actual module name. Plus, - // the user will always have the option of resolving any impasses by - // specifying the module name explicitly. - // - // So, the fuzzy match: the idea is that each match gets a score, the - // number of characters in the module name that got matched. A match - // with the highest score is used. And we use the (length + 1) for a - // match against an actual module name. - // - // For std.* modules we only accept non-fuzzy matches (think std.core vs - // some core.mxx). And if such a module is unresolved, then we assume it - // is pre-built and will be found by some other means (e.g., VC's - // IFCPATH). - // - auto match = [] (const string& f, const string& m) -> size_t - { - size_t fi (f.size ()); - size_t mi (m.size ()); - - // Scan backwards for as long as we match. Keep track of the previous - // character for case change detection. - // - for (char fc, mc, fp ('\0'), mp ('\0'); - fi != 0 && mi != 0; - fp = fc, mp = mc, --fi, --mi) - { - fc = f[fi - 1]; - mc = m[mi - 1]; - - if (casecmp (fc, mc) == 0) - continue; - - // We consider all separators equal and character case change being - // a separators. Some examples of the latter: - // - // foo.bar - // fooBAR - // FOObar - // - bool fs (fc == '_' || fc == '-' || fc == '.' || - path::traits::is_separator (fc)); - bool ms (mc == '_' || mc == '.'); - - if (fs && ms) - continue; - - // Only if one is a real separator do we consider case change. - // - if (fs || ms) - { - auto cc = [] (char c1, char c2) -> bool - { - return (alpha (c1) && - alpha (c2) && - (ucase (c1) == c1) != (ucase (c2) == c2)); - }; - - bool fa (false), ma (false); - if ((fs || (fa = cc (fp, fc))) && (ms || (ma = cc (mp, mc)))) - { - // Stay on this character if imaginary punctuation (note: cannot - // be both true). - // - if (fa) ++fi; - if (ma) ++mi; - continue; - } - } - - break; // No match. - } - - // Return the number of characters matched in the module name and not - // in the file (this may not be the same because of the imaginary - // separators). - // - return m.size () - mi; - }; - - auto& pts (t.prerequisite_targets); - size_t start (pts.size ()); // Index of the first to be added. - - // We have two parallel vectors: module names/scores in imports and - // targets in prerequisite_targets (offset with start). Pre-allocate - // NULL entries in the latter. - // - size_t n (imports.size ()); - pts.resize (start + n, nullptr); - - // Oh, yes, there is one "minor" complication. It's the last one, I - // promise. It has to do with module re-exporting (export import M;). - // In this case (currently) all implementations simply treat it as a - // shallow (from the BMI's point of view) reference to the module (or an - // implicit import, if you will). Do you see where it's going? Nowever - // good, that's right. This shallow reference means that the compiler - // should be able to find BMIs for all the re-exported modules, - // recursive. The good news is we are actually in a pretty good shape to - // handle this: after match all our prerequisite BMIs will have their - // prerequisite BMIs known, recursively. The only bit that is missing is - // the re-export flag of some sorts. As well as deciding where to handle - // it: here or in append_modules(). After some meditation it became - // clear handling it here will be simpler: We need to weed out - // duplicates for which we can re-use the imports vector. And we may - // also need to save this "flattened" list of modules in depdb. - // - // Ok, so, here is the plan: - // - // 1. There is no good place in prerequisite_targets to store the - // exported flag (no, using the marking facility across match/execute - // is a bad idea). So what we are going to do is put re-exported - // bmi{}s at the back and store (in the target's data pad) the start - // position. One bad aspect about this part is that we assume those - // bmi{}s have been matched by the same rule. But let's not kid - // ourselves, there will be no other rule that matches bmi{}s. - // - // 2. Once we have matched all the bmi{}s we are importing directly - // (with all the re-exported by us at the back), we will go over them - // and copy all of their re-exported bmi{}s (using the position we - // saved on step #1). The end result will be a recursively-explored - // list of imported bmi{}s that append_modules() can simply convert - // to the list of options. - // - // One issue with this approach is that these copied targets will be - // executed which means we need to adjust their dependent counts - // (which is normally done by match). While this seems conceptually - // correct (especially if you view re-exports as implicit imports), - // it's just extra overhead (we know they will be updated). So what - // we are going to do is save another position, that of the start of - // these copied-over targets, and will only execute up to this point. - // - // And after implementing this came the reality check: all the current - // implementations require access to all the imported BMIs, not only - // re-exported. Some (like Clang) store references to imported BMI files - // so we actually don't need to pass any extra options (unless things - // get moved) but they still need access to the BMIs (and things will - // most likely have to be done differenly for distributed compilation). - // - // So the revised plan: on the off chance that some implementation will - // do it differently we will continue maintaing the imported/re-exported - // split and how much to copy-over can be made compiler specific. - // - // As a first sub-step of step #1, move all the re-exported imports to - // the end of the vector. This will make sure they end up at the end - // of prerequisite_targets. Note: the special first import, if any, - // should be unaffected. - // - sort (imports.begin (), imports.end (), - [] (const module_import& x, const module_import& y) - { - return !x.exported && y.exported; - }); - - // Go over the prerequisites once. - // - // For (direct) library prerequisites, check their prerequisite bmi{}s - // (which should be searched and matched with module names discovered; - // see the library meta-information protocol for details). - // - // For our own bmi{} prerequisites, checking if each (better) matches - // any of the imports. - - // For fuzzy check if a file name (better) resolves any of our imports - // and if so make it the new selection. For exact the name is the actual - // module name and it can only resolve one import (there are no - // duplicates). - // - // Set done to true if all the imports have now been resolved to actual - // module names (which means we can stop searching). This will happens - // if all the modules come from libraries. Which will be fairly common - // (think of all the tests) so it's worth optimizing for. - // - bool done (false); - - auto check_fuzzy = [&trace, &imports, &pts, &match, start, n] - (const target* pt, const string& name) - { - for (size_t i (0); i != n; ++i) - { - module_import& m (imports[i]); - - if (std_module (m.name)) // No fuzzy std.* matches. - continue; - - size_t n (m.name.size ()); - - if (m.score > n) // Resolved to module name. - continue; - - size_t s (match (name, m.name)); - - l5 ([&]{trace << name << " ~ " << m.name << ": " << s;}); - - if (s > m.score) - { - pts[start + i] = pt; - m.score = s; - } - } - }; - - // If resolved, return the "slot" in pts (we don't want to create a - // side build until we know we match; see below for details). - // - auto check_exact = [&trace, &imports, &pts, start, n, &done] - (const string& name) -> const target** - { - const target** r (nullptr); - done = true; - - for (size_t i (0); i != n; ++i) - { - module_import& m (imports[i]); - - size_t n (m.name.size ()); - - if (m.score > n) // Resolved to module name (no effect on done). - continue; - - if (r == nullptr) - { - size_t s (name == m.name ? n + 1 : 0); - - l5 ([&]{trace << name << " ~ " << m.name << ": " << s;}); - - if (s > m.score) - { - r = &pts[start + i].target; - m.score = s; - continue; // Scan the rest to detect if all done. - } - } - - done = false; - } - - return r; - }; - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target* pt (p.load ()); // Should be cached for libraries. - - if (pt != nullptr) - { - const target* lt (nullptr); - - if (const libx* l = pt->is_a ()) - lt = &link_member (*l, act, li); - else if (pt->is_a () || pt->is_a () || pt->is_a ()) - lt = pt; - - // If this is a library, check its bmi{}s and mxx{}s. - // - if (lt != nullptr) - { - for (const target* bt: lt->prerequisite_targets) - { - if (bt == nullptr) - continue; - - // Note that here we (try) to use whatever flavor of bmi*{} is - // available. - // - // @@ MOD: BMI compatibility check. - // @@ UTL: we need to (recursively) see through libux{} (and - // also in pkgconfig_save()). - // - if (bt->is_a () || - bt->is_a () || - bt->is_a ()) - { - const string& n (cast (bt->vars[c_module_name])); - - if (const target** p = check_exact (n)) - *p = bt; - } - else if (bt->is_a (*x_mod)) - { - // This is an installed library with a list of module sources - // (the source are specified as prerequisites but the fallback - // file rule puts them into prerequisite_targets for us). - // - // The module names should be specified but if not assume - // something else is going on and ignore. - // - const string* n (cast_null (bt->vars[c_module_name])); - if (n == nullptr) - continue; - - if (const target** p = check_exact (*n)) - *p = &make_module_sidebuild (act, bs, *lt, *bt, *n); - } - else - continue; - - if (done) - break; - } - - if (done) - break; - - continue; - } - - // Fall through. - } - - // While it would have been even better not to search for a target, we - // need to get hold of the corresponding mxx{} (unlikely but possible - // for bmi{} to have a different name). - // - if (p.is_a ()) - pt = &search (t, mtt, p.key ()); // Same logic as in picking obj*{}. - else if (p.is_a (mtt)) - { - if (pt == nullptr) - pt = &p.search (t); - } - else - continue; - - // Find the mxx{} prerequisite and extract its "file name" for the - // fuzzy match unless the user specified the module name explicitly. - // - for (prerequisite_member p: group_prerequisite_members (act, *pt)) - { - if (p.is_a (*x_mod)) - { - // Check for an explicit module name. Only look for an existing - // target (which means the name can only be specified on the - // target itself, no target type/pattern-spec). - // - const target* t (p.search_existing ()); - const string* n (t != nullptr - ? cast_null (t->vars[c_module_name]) - : nullptr); - if (n != nullptr) - { - if (const target** p = check_exact (*n)) - *p = pt; - } - else - { - // Fuzzy match. - // - string f; - - // Add the directory part if it is relative. The idea is to - // include it into the module match, say hello.core vs - // hello/mxx{core}. - // - // @@ MOD: Why not for absolute? Good question. What if it - // contains special components, say, ../mxx{core}? - // - const dir_path& d (p.dir ()); - - if (!d.empty () && d.relative ()) - f = d.representation (); // Includes trailing slash. - - f += p.name (); - check_fuzzy (pt, f); - } - break; - } - } - - if (done) - break; - } - - // Diagnose unresolved modules. - // - if (!done) - { - for (size_t i (0); i != n; ++i) - { - if (pts[start + i] == nullptr && !std_module (imports[i].name)) - { - // It would have been nice to print the location of the import - // declaration. And we could save it during parsing at the expense - // of a few paths (that can be pooled). The question is what to do - // when we re-create this information from depdb? We could have - // saved the location information there but the relative paths - // (e.g., from the #line directives) could end up being wrong if - // the we re-run from a different working directory. - // - // It seems the only workable approach is to extract full location - // info during parse, not save it in depdb, when re-creating, - // fallback to just src path without any line/column information. - // This will probably cover the majority of case (most of the time - // it will be a misspelled module name, not a removal of module - // from buildfile). - // - // But at this stage this doesn't seem worth the trouble. - // - fail (relative (src)) << "unable to resolve module " - << imports[i].name; - } - } - } - - // Match in parallel and wait for completion. - // - match_members (act, t, pts, start); - - // Post-process the list of our (direct) imports. While at it, calculate - // the checksum of all (direct and indirect) bmi{} paths. - // - size_t exported (n); - size_t copied (pts.size ()); - - for (size_t i (0); i != n; ++i) - { - const module_import& m (imports[i]); - - // Determine the position of the first re-exported bmi{}. - // - if (m.exported && exported == n) - exported = i; - - const target* bt (pts[start + i]); - - if (bt == nullptr) - continue; // Unresolved (std.*). - - // Verify our guesses against extracted module names but don't waste - // time if it was a match against the actual module name. - // - const string& in (m.name); - - if (m.score <= in.size ()) - { - const string& mn (cast (bt->vars[c_module_name])); - - if (in != mn) - { - for (prerequisite_member p: group_prerequisite_members (act, *bt)) - { - if (p.is_a (*x_mod)) // Got to be there. - { - fail (relative (src)) - << "failed to correctly guess module name from " << p << - info << "guessed: " << in << - info << "actual: " << mn << - info << "consider adjusting module interface file names or" << - info << "consider specifying module name with " << x - << ".module_name"; - } - } - } - } - - // Hash (we know it's a file). - // - cs.append (static_cast (*bt).path ().string ()); - - // Copy over bmi{}s from our prerequisites weeding out duplicates. - // - if (size_t j = bt->data ().mods.start) - { - // Hard to say whether we should reserve or not. We will probably - // get quite a bit of duplications. - // - for (size_t m (bt->prerequisite_targets.size ()); j != m; ++j) - { - const target* et (bt->prerequisite_targets[j]); - - if (et == nullptr) - continue; // Unresolved (std.*). - - const string& mn (cast (et->vars[c_module_name])); - - if (find_if (imports.begin (), imports.end (), - [&mn] (const module_import& i) - { - return i.name == mn; - }) == imports.end ()) - { - pts.push_back (et); - cs.append (static_cast (*et).path ().string ()); - - // Add to the list of imports for further duplicate suppression. - // We could have probably stored reference to the name (e.g., in - // score) but it's probably not worth it if we have a small - // string optimization. - // - imports.push_back (module_import {mn, true, 0}); - } - } - } - } - - if (copied == pts.size ()) // No copied tail. - copied = 0; - - if (exported == n) // No (own) re-exported imports. - exported = copied; - else - exported += start; // Rebase. - - return module_positions {start, exported, copied}; - } - - // Synthesize a dependency for building a module binary interface on - // the side. - // - const target& compile:: - make_module_sidebuild (action act, - const scope& bs, - const target& lt, - const target& mt, - const string& mn) const - { - tracer trace (x, "compile::make_module_sidebuild"); - - // First figure out where we are going to build. We want to avoid - // multiple sidebuilds so the outermost scope that has loaded the - // cc.config module and that is within our amalgmantion seems like a - // good place. - // - const scope& rs (*bs.root_scope ()); - const scope* as (&rs); - { - const scope* ws (as->weak_scope ()); - if (as != ws) - { - const scope* s (as); - do - { - s = s->parent_scope ()->root_scope (); - - // Use cc.core.vars as a proxy for {c,cxx}.config (a bit smelly). - // - // This is also the module that registers the scope operation - // callback that cleans up the subproject. - // - if (cast_false ((*s)["cc.core.vars.loaded"])) - as = s; - - } while (s != ws); - } - } - - // We build modules in a subproject (since there might be no full - // language support module loaded in the amalgamation, only *.config). - // So the first step is to check if the project has already been created - // and/or loaded and if not, then to go ahead and do so. - // - dir_path pd (as->out_path () / modules_sidebuild_dir /= x); - { - const scope* ps (&scopes.find (pd)); - - if (ps->out_path () != pd) - { - // Switch the phase to load then create and load the subproject. - // - phase_switch phs (run_phase::load); - - // Re-test again now that we are in exclusive phase (another thread - // could have already created and loaded the subproject). - // - ps = &scopes.find (pd); - - if (ps->out_path () != pd) - { - // The project might already be created in which case we just need - // to load it. - // - if (!is_src_root (pd)) - { - // Copy our standard and force modules. - // - string extra; - - if (const string* std = cast_null (rs[x_std])) - extra += string (x) + ".std = " + *std + '\n'; - - extra += string (x) + ".features.modules = true"; - - config::create_project ( - pd, - as->out_path ().relative (pd), /* amalgamation */ - {}, /* boot_modules */ - extra, /* root_pre */ - {string (x) + '.'}, /* root_modules */ - "", /* root_post */ - false, /* config */ - false, /* buildfile */ - "the cc module", - 2); /* verbosity */ - } - - ps = &load_project (as->rw () /* lock */, pd, pd); - } - } - - // Some sanity checks. - // -#ifndef NDEBUG - assert (ps->root ()); - const module* m (ps->modules.lookup (x)); - assert (m != nullptr && m->modules); -#endif - } - - // Next we need to come up with a file/target name that will be unique - // enough not to conflict with other modules. If we assume that within - // an amalgamation there is only one "version" of each module, then the - // module name itself seems like a good fit. We just replace '.' with - // '-'. - // - string mf; - transform (mn.begin (), mn.end (), - back_inserter (mf), - [] (char c) {return c == '.' ? '-' : c;}); - - // It seems natural to build a BMI type that corresponds to the library - // type. After all, this is where the object file part of the BMI is - // going to come from (though things will probably be different for - // module-only libraries). - // - const target_type* tt (nullptr); - switch (link_type (lt).type) - { - case otype::a: tt = &bmia::static_type; break; - case otype::s: tt = &bmis::static_type; break; - case otype::e: assert (false); - } - - // Store the BMI target in the subproject root. If the target already - // exists then we assume all this is already done (otherwise why would - // someone have created such a target). - // - if (const target* bt = targets.find ( - *tt, - pd, - dir_path (), // Always in the out tree. - mf, - nullopt, // Use default extension. - trace)) - return *bt; - - prerequisites ps; - ps.push_back (prerequisite (mt)); - - // We've added the mxx{} but it may import other modules from this - // library. Or from (direct) dependencies of this library. We add them - // all as prerequisites so that the standard module search logic can - // sort things out. This is pretty similar to what we do in link when - // synthesizing dependencies for bmi{}'s. - // - ps.push_back (prerequisite (lt)); - for (prerequisite_member p: group_prerequisite_members (act, lt)) - { - // @@ TODO: will probably need revision if using sidebuild for - // non-installed libraries (e.g., direct BMI dependencies - // will probably have to be translated to mxx{} or some such). - // - if (p.is_a () || - p.is_a () || p.is_a () || p.is_a ()) - { - ps.push_back (p.as_prerequisite ()); - } - } - - auto p (targets.insert_locked (*tt, - move (pd), - dir_path (), // Always in the out tree. - move (mf), - nullopt, // Use default extension. - true, // Implied. - trace)); - const target& bt (p.first); - - // Note that this is racy and someone might have created this target - // while we were preparing the prerequisite list. - // - if (p.second.owns_lock ()) - bt.prerequisites (move (ps)); - - return bt; - } - - // Filter cl.exe noise (msvc.cxx). - // - void - msvc_filter_cl (ifdstream&, const path& src); - - void compile:: - append_modules (environment& env, - cstrings& args, - strings& stor, - const file& t, - const match_data& md) const - { - const module_positions& ms (md.mods); - assert (ms.start != 0); - - dir_path stdifc; // See the VC case below. - -#if 0 - switch (cid) - { - case compiler_id::gcc: - { - // Use the module map stored in depdb. - // - string s (relative (md.dd).string ()); - s.insert (0, "-fmodule-file-map=@="); - stor.push_back (move (s)); - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // In Clang the module implementation's unit .pcm is special and - // must be "loaded". - // - if (md.type == translation_type::module_impl) - { - const file& f (t.prerequisite_targets[ms.start]->as ()); - string s (relative (f.path ()).string ()); - s.insert (0, "-fmodule-file="); - stor.push_back (move (s)); - } - - // Use the module map stored in depdb for others. - // - string s (relative (md.dd).string ()); - s.insert (0, "-fmodule-file-map=@="); - stor.push_back (move (s)); - break; - } - case compiler_id::msvc: - { - for (size_t i (ms.start), n (t.prerequisite_targets.size ()); - i != n; - ++i) - { - const target* pt (t.prerequisite_targets[i]); - - if (pt == nullptr) - continue; - - // Here we use whatever bmi type has been added. And we know all - // of these are bmi's. - // - const file& f (pt->as ()); - - // In VC std.* modules can only come from a single directory - // specified with the IFCPATH environment variable or the - // /module:stdIfcDir option. - // - if (std_module (cast (f.vars[c_module_name]))) - { - dir_path d (f.path ().directory ()); - - if (stdifc.empty ()) - { - // Go one directory up since /module:stdIfcDir will look in - // either Release or Debug subdirectories. Keeping the result - // absolute feels right. - // - stor.push_back ("/module:stdIfcDir"); - stor.push_back (d.directory ().string ()); - stdifc = move (d); - } - else if (d != stdifc) // Absolute and normalized. - fail << "multiple std.* modules in different directories"; - } - else - { - stor.push_back ("/module:reference"); - stor.push_back (relative (f.path ()).string ()); - } - } - break; - } - case compiler_id::icc: - assert (false); - } -#else - size_t n (t.prerequisite_targets.size ()); - - // Clang embeds module file references so we only need to specify - // our direct imports. - // - // If/when we get the ability to specify the mapping in a file, we - // should probably pass the whole list. - // - switch (cid) - { - case compiler_id::gcc: break; // All of them. - case compiler_id::clang_apple: - case compiler_id::clang: n = ms.copied != 0 ? ms.copied : n; break; - case compiler_id::msvc: break; // All of them. - case compiler_id::icc: assert (false); - } - - for (size_t i (ms.start); i != n; ++i) - { - const target* pt (t.prerequisite_targets[i]); - - if (pt == nullptr) - continue; - - // Here we use whatever bmi type has been added. And we know all of - // these are bmi's. - // - const file& f (pt->as ()); - string s (relative (f.path ()).string ()); - - switch (cid) - { - case compiler_id::gcc: - { - s.insert (0, 1, '='); - s.insert (0, cast (f.vars[c_module_name])); - s.insert (0, "-fmodule-file="); - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // In Clang the module implementation's unit .pcm is special and - // must be "loaded". - // - if (md.type == translation_type::module_impl && i == ms.start) - s.insert (0, "-fmodule-file="); - else - { - s.insert (0, 1, '='); - s.insert (0, cast (f.vars[c_module_name])); - s.insert (0, "-fmodule-file="); - } - break; - } - case compiler_id::msvc: - { - // In VC std.* modules can only come from a single directory - // specified with the IFCPATH environment variable or the - // /module:stdIfcDir option. - // - if (std_module (cast (f.vars[c_module_name]))) - { - dir_path d (f.path ().directory ()); - - if (stdifc.empty ()) - { - // Go one directory up since /module:stdIfcDir will look in - // either Release or Debug subdirectories. Keeping the result - // absolute feels right. - // - s = d.directory ().string (); - stor.push_back ("/module:stdIfcDir"); - stdifc = move (d); - } - else - { - if (d != stdifc) // Absolute and normalized. - fail << "multiple std.* modules in different directories"; - - continue; // Skip. - } - } - else - stor.push_back ("/module:reference"); - - break; - } - case compiler_id::icc: - assert (false); - } - - stor.push_back (move (s)); - } -#endif - - // Shallow-copy storage to args. Why not do it as we go along pushing - // into storage? Because of potential reallocations. - // - for (const string& a: stor) - args.push_back (a.c_str ()); - - // VC's IFCPATH takes precedence over /module:stdIfcDir so unset it - // if we are using our own std modules. - // - if (!stdifc.empty ()) - env.push_back ("IFCPATH"); - } - - target_state compile:: - perform_update (action act, const target& xt) const - { - const file& t (xt.as ()); - const path& tp (t.path ()); - - match_data md (move (t.data ())); - bool mod (md.type == translation_type::module_iface); - - // While all our prerequisites are already up-to-date, we still have to - // execute them to keep the dependency counts straight. Actually, no, we - // may also have to update the modules. - // - auto pr ( - execute_prerequisites ( - (mod ? *x_mod : x_src), - act, t, - md.mt, - [s = md.mods.start] (const target&, size_t i) - { - return s != 0 && i >= s; // Only compare timestamps for modules. - }, - md.mods.copied)); // See search_modules() for details. - - const file& s (pr.second); - const path* sp (&s.path ()); - - if (pr.first) - { - if (md.touch) - { - touch (tp, false, 2); - skip_count.fetch_add (1, memory_order_relaxed); - } - - t.mtime (md.mt); - return *pr.first; - } - - // Make sure depdb is no older than any of our prerequisites. - // - touch (md.dd, false, verb_never); - - const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); - - otype ot (compile_type (t, mod)); - linfo li (link_info (bs, ot)); - - environment env; - cstrings args {cpath.recall_string ()}; - - // If we are building a module, then the target is bmi*{} and its ad hoc - // member is obj*{}. - // - path relo, relm; - if (mod) - { - relm = relative (tp); - relo = relative (t.member->is_a ()->path ()); - } - else - relo = relative (tp); - - // Build the command line. - // - if (md.pp != preprocessed::all) - { - append_options (args, t, c_poptions); - append_options (args, t, x_poptions); - - // Add *.export.poptions from prerequisite libraries. - // - append_lib_options (bs, args, t, act, li); - - // Extra system header dirs (last). - // - assert (sys_inc_dirs_extra <= sys_inc_dirs.size ()); - append_option_values ( - args, "-I", - sys_inc_dirs.begin () + sys_inc_dirs_extra, sys_inc_dirs.end (), - [] (const dir_path& d) {return d.string ().c_str ();}); - - if (md.symexport) - append_symexport_options (args, t); - } - - append_options (args, t, c_coptions); - append_options (args, t, x_coptions); - append_options (args, tstd); - - string out, out1; // Output options storage. - strings mods; // Module options storage. - size_t out_i (0); // Index of the -o option. - - if (cclass == compiler_class::msvc) - { - // The /F*: option variants with separate names only became available - // in VS2013/12.0. Why do we bother? Because the command line suddenly - // becomes readable. - // - uint64_t ver (cast (rs[x_version_major])); - - args.push_back ("/nologo"); - - // While we want to keep the low-level build as "pure" as possible, - // the two misguided defaults, exceptions and runtime, just have to be - // fixed. Otherwise the default build is pretty much unusable. But we - // also make sure that the user can easily disable our defaults: if we - // see any relevant options explicitly specified, we take our hands - // off. - // - // For C looks like no /EH* (exceptions supported but no C++ objects - // destroyed) is a reasonable default. - // - if (x_lang == lang::cxx && !find_option_prefix ("/EH", args)) - args.push_back ("/EHsc"); - - // The runtime is a bit more interesting. At first it may seem like a - // good idea to be a bit clever and use the static runtime if we are - // building obja{}. And for obje{} we could decide which runtime to - // use based on the library link order: if it is static-only, then we - // could assume the static runtime. But it is indeed too clever: when - // building liba{} we have no idea who is going to use it. It could be - // an exe{} that links both static and shared libraries (and is - // therefore built with the shared runtime). And to safely use the - // static runtime, everything must be built with /MT and there should - // be no DLLs in the picture. So we are going to play it safe and - // always default to the shared runtime. - // - // In a similar vein, it would seem reasonable to use the debug runtime - // if we are compiling with debug. But, again, there will be fireworks - // if we have some projects built with debug and some without and then - // we try to link them together (which is not an unreasonable thing to - // do). So by default we will always use the release runtime. - // - if (!find_option_prefixes ({"/MD", "/MT"}, args)) - args.push_back ("/MD"); - - if (md.mods.start != 0) - append_modules (env, args, mods, t, md); - - // The presence of /Zi or /ZI causes the compiler to write debug info - // to the .pdb file. By default it is a shared file called vcNN.pdb - // (where NN is the VC version) created (wait for it) in the current - // working directory (and not the directory of the .obj file). Also, - // because it is shared, there is a special Windows service that - // serializes access. We, of course, want none of that so we will - // create a .pdb per object file. - // - // Note that this also changes the name of the .idb file (used for - // minimal rebuild and incremental compilation): cl.exe take the /Fd - // value and replaces the .pdb extension with .idb. - // - // Note also that what we are doing here appears to be incompatible - // with PCH (/Y* options) and /Gm (minimal rebuild). - // - if (find_options ({"/Zi", "/ZI"}, args)) - { - if (ver >= 18) - args.push_back ("/Fd:"); - else - out1 = "/Fd"; - - out1 += relo.string (); - out1 += ".pdb"; - - args.push_back (out1.c_str ()); - } - - if (ver >= 18) - { - args.push_back ("/Fo:"); - args.push_back (relo.string ().c_str ()); - } - else - { - out = "/Fo" + relo.string (); - args.push_back (out.c_str ()); - } - - if (mod) - { - args.push_back ("/module:interface"); - args.push_back ("/module:output"); - args.push_back (relm.string ().c_str ()); - } - - // Note: no way to indicate that the source if already preprocessed. - - args.push_back ("/c"); // Compile only. - args.push_back (langopt (md)); // Compile as. - args.push_back (sp->string ().c_str ()); // Note: relied on being last. - } - else - { - if (ot == otype::s) - { - // On Darwin, Win32 -fPIC is the default. - // - if (tclass == "linux" || tclass == "bsd") - args.push_back ("-fPIC"); - } - - if (md.mods.start != 0) - append_modules (env, args, mods, t, md); - - // Note: the order of the following options is relied upon below. - // - out_i = args.size (); // Index of the -o option. - - if (mod) - { - switch (cid) - { - case compiler_id::gcc: - { - args.push_back ("-o"); - args.push_back (relo.string ().c_str ()); - - out = "-fmodule-output="; - out += relm.string (); - args.push_back (out.c_str ()); - - args.push_back ("-c"); - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - args.push_back ("-o"); - args.push_back (relm.string ().c_str ()); - args.push_back ("--precompile"); - - // Without this option Clang's .pcm will reference source files. - // In our case this file may be transient (.ii). Plus, it won't - // play nice with distributed compilation. - // - args.push_back ("-Xclang"); - args.push_back ("-fmodules-embed-all-files"); - - break; - } - case compiler_id::msvc: - case compiler_id::icc: - assert (false); - } - } - else - { - args.push_back ("-o"); - args.push_back (relo.string ().c_str ()); - args.push_back ("-c"); - } - - args.push_back ("-x"); - args.push_back (langopt (md)); - - if (md.pp == preprocessed::all) - { - // Note that the mode we select must still handle comments and line - // continuations. So some more compiler-specific voodoo. - // - switch (cid) - { - case compiler_id::gcc: - { - // -fdirectives-only is available since GCC 4.3.0. - // - if (cmaj > 4 || (cmaj == 4 && cmin >= 3)) - { - args.push_back ("-fpreprocessed"); - args.push_back ("-fdirectives-only"); - } - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // Clang handles comments and line continuations in the - // preprocessed source (it does not have -fpreprocessed). - // - break; - } - case compiler_id::icc: - break; // Compile as normal source for now. - case compiler_id::msvc: - assert (false); - } - } - - args.push_back (sp->string ().c_str ()); - } - - args.push_back (nullptr); - - if (!env.empty ()) - env.push_back (nullptr); - - // With verbosity level 2 print the command line as if we are compiling - // the source file, not its preprocessed version (so that it's easy to - // copy and re-run, etc). Only at level 3 and above print the real deal. - // - if (verb == 1) - text << x_name << ' ' << s; - else if (verb == 2) - print_process (args); - - // If we have the (partially) preprocessed output, switch to that. - // - bool psrc (!md.psrc.path.empty ()); - bool pact (md.psrc.active); - if (psrc) - { - args.pop_back (); // nullptr - args.pop_back (); // sp - - sp = &md.psrc.path; - - // This should match with how we setup preprocessing. - // - switch (cid) - { - case compiler_id::gcc: - { - // The -fpreprocessed is implied by .i/.ii. - // - args.pop_back (); // lang() - args.pop_back (); // -x - args.push_back ("-fdirectives-only"); - break; - } - case compiler_id::clang: - case compiler_id::clang_apple: - { - // Note that without -x Clang will treat .i/.ii as fully - // preprocessed. - // - break; - } - case compiler_id::msvc: - { - // Nothing to do (/TP or /TC already there). - // - break; - } - case compiler_id::icc: - assert (false); - } - - args.push_back (sp->string ().c_str ()); - args.push_back (nullptr); - - // Let's keep the preprocessed file in case of an error but only at - // verbosity level 3 and up (when one actually sees it mentioned on - // the command line). We also have to re-arm on success (see below). - // - if (pact && verb >= 3) - md.psrc.active = false; - } - - if (verb >= 3) - print_process (args); - - try - { - // VC cl.exe sends diagnostics to stdout. It also prints the file name - // being compiled as the first line. So for cl.exe we redirect stdout - // to a pipe, filter that noise out, and send the rest to stderr. - // - // For other compilers redirect stdout to stderr, in case any of them - // tries to pull off something similar. For sane compilers this should - // be harmless. - // - bool filter (cid == compiler_id::msvc); - - process pr (cpath, - args.data (), - 0, (filter ? -1 : 2), 2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - - if (filter) - { - try - { - ifdstream is ( - move (pr.in_ofd), fdstream_mode::text, ifdstream::badbit); - - msvc_filter_cl (is, *sp); - - // If anything remains in the stream, send it all to stderr. Note - // that the eof check is important: if the stream is at eof, this - // and all subsequent writes to the diagnostics stream will fail - // (and you won't see a thing). - // - if (is.peek () != ifdstream::traits_type::eof ()) - diag_stream_lock () << is.rdbuf (); - - is.close (); - } - catch (const io_error&) {} // Assume exits with error. - } - - run_finish (args, pr); - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - if (e.child) - exit (1); - - throw failed (); - } - - if (pact && verb >= 3) - md.psrc.active = true; - - // Clang's module compilation requires two separate compiler - // invocations. - // - if (mod && (cid == compiler_id::clang || - cid == compiler_id::clang_apple)) - { - // Remove the target file if this fails. If we don't do that, we will - // end up with a broken build that is up-to-date. - // - auto_rmfile rm (relm); - - // Adjust the command line. First discard everything after -o then - // build the new "tail". - // - args.resize (out_i + 1); - args.push_back (relo.string ().c_str ()); // Produce .o. - args.push_back ("-c"); // By compiling .pcm. - args.push_back ("-Wno-unused-command-line-argument"); - args.push_back (relm.string ().c_str ()); - args.push_back (nullptr); - - if (verb >= 2) - print_process (args); - - try - { - process pr (cpath, - args.data (), - 0, 2, 2, - nullptr, // CWD - env.empty () ? nullptr : env.data ()); - - run_finish (args, pr); - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - if (e.child) - exit (1); - - throw failed (); - } - - rm.cancel (); - } - - // Should we go to the filesystem and get the new mtime? We know the - // file has been modified, so instead just use the current clock time. - // It has the advantage of having the subseconds precision. - // - t.mtime (system_clock::now ()); - return target_state::changed; - } - - target_state compile:: - perform_clean (action a, const target& xt) const - { - const file& t (xt.as ()); - - using id = compiler_id; - - switch (cid) - { - case id::gcc: return clean_extra (a, t, {".d", x_pext, ".t"}); - case id::clang_apple: - case id::clang: return clean_extra (a, t, {".d", x_pext}); - case id::msvc: return clean_extra (a, t, {".d", x_pext, ".idb", ".pdb"}); - case id::icc: return clean_extra (a, t, {".d"}); - } - - assert (false); - return target_state::unchanged; - } - } -} diff --git a/build2/cc/compile.hxx b/build2/cc/compile.hxx deleted file mode 100644 index 2878e3d..0000000 --- a/build2/cc/compile.hxx +++ /dev/null @@ -1,153 +0,0 @@ -// file : build2/cc/compile.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef BUILD2_CC_COMPILE_HXX -#define BUILD2_CC_COMPILE_HXX - -#include - -#include -#include - -#include -#include // auto_rmfile - -#include -#include - -namespace build2 -{ - class depdb; - - namespace cc - { - // The order is arranged so that their integral values indicate whether - // one is a "stronger" than another. - // - enum class preprocessed: uint8_t {none, includes, modules, all}; - - // Positions of the re-exported bmi{}s. See search_modules() for - // details. - // - struct module_positions - { - size_t start; // First imported bmi*{}, 0 if none. - size_t exported; // First re-exported bmi*{}, 0 if none. - size_t copied; // First copied-over bmi*{}, 0 if none. - }; - - class compile: public rule, virtual common - { - public: - compile (data&&); - - virtual match_result - match (action, target&, const string&) const override; - - virtual recipe - apply (action, target&) const override; - - target_state - perform_update (action, const target&) const; - - target_state - perform_clean (action, const target&) const; - - private: - struct match_data; - using environment = small_vector; - - void - append_lib_options (const scope&, - cstrings&, - const target&, - action, linfo) const; - - void - hash_lib_options (const scope&, - sha256&, - const target&, - action, linfo) const; - - // Mapping of include prefixes (e.g., foo in ) for auto- - // generated headers to directories where they will be generated. - // - // We are using a prefix map of directories (dir_path_map) instead of - // just a map in order to also cover sub-paths (e.g., if - // we continue with the example). Specifically, we need to make sure we - // don't treat foobar as a sub-directory of foo. - // - // The priority is used to decide who should override whom. Lesser - // values are considered higher priority. See append_prefixes() for - // details. - // - // @@ The keys should be normalized. - // - struct prefix_value - { - dir_path directory; - size_t priority; - }; - using prefix_map = butl::dir_path_map; - - void - append_prefixes (prefix_map&, const target&, const variable&) const; - - void - append_lib_prefixes (const scope&, - prefix_map&, - target&, - action, linfo) const; - - prefix_map - build_prefix_map (const scope&, target&, action, linfo) const; - - // Reverse-lookup target type from extension. - // - const target_type* - map_extension (const scope&, const string&, const string&) const; - - pair - extract_headers (action, const scope&, file&, linfo, - const file&, const match_data&, - depdb&, bool&, timestamp) const; - - pair - parse_unit (action, file&, linfo, - const file&, auto_rmfile&, const match_data&) const; - - void - extract_modules (action, const scope&, file&, linfo, - const compile_target_types&, - const file&, match_data&, - module_info&&, depdb&, bool&) const; - - module_positions - search_modules (action, const scope&, file&, linfo, - const target_type&, - const file&, module_imports&, sha256&) const; - - const target& - make_module_sidebuild (action, const scope&, const target&, - const target&, const string&) const; - - void - append_modules (environment&, cstrings&, strings&, - const file&, const match_data&) const; - - // Language selection option (for VC) or the value for the -x option. - // - const char* - langopt (const match_data&) const; - - void - append_symexport_options (cstrings&, const target&) const; - - private: - const string rule_id; - }; - } -} - -#endif // BUILD2_CC_COMPILE_HXX diff --git a/build2/cc/install-rule.cxx b/build2/cc/install-rule.cxx new file mode 100644 index 0000000..4e232ff --- /dev/null +++ b/build2/cc/install-rule.cxx @@ -0,0 +1,269 @@ +// file : build2/cc/install-rule.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include + +#include + +#include +#include // match() + +using namespace std; + +namespace build2 +{ + namespace cc + { + using namespace bin; + + // install_rule + // + install_rule:: + install_rule (data&& d, const link_rule& l) + : common (move (d)), link_ (l) {} + + const target* install_rule:: + filter (action a, const target& t, prerequisite_member p) const + { + // NOTE: see libux_install_rule::filter() if changing anything here. + + otype ot (link_type (t).type); + + // Don't install executable's prerequisite headers. + // + if (t.is_a () && x_header (p)) + return nullptr; + + // Here is a problem: if the user spells the obj*/bmi*{} targets + // explicitly, then the source files, including headers/modules may be + // specified as preprequisites of those targets and not of this target. + // While this can be worked around for headers by also listing them as + // prerequisites of this target, this won't work for modules (since they + // are compiled). So what we are going to do here is detect bmi*{} and + // translate them to their mxx{} (this doesn't quite work for headers + // since there would normally be several of them). + // + if (p.is_a () || p.is_a (compile_types (ot).bmi)) + { + const target& mt (p.search (t)); + + for (prerequisite_member mp: group_prerequisite_members (a, mt)) + { + if (mp.is_a (*x_mod)) + return t.is_a () ? nullptr : file_rule::filter (a, mt, mp); + } + } + + // If this is a shared library prerequisite, install it as long as it + // is in the same amalgamation as we are. + // + // Less obvious: we also want to install a static library prerequisite + // of a library (since it could be referenced from its .pc file, etc). + // + bool st (t.is_a () || t.is_a ()); // Target needs shared. + bool at (t.is_a () || t.is_a ()); // Target needs static. + + if ((st && (p.is_a () || p.is_a ())) || + (at && (p.is_a () || p.is_a ()))) + { + const target* pt (&p.search (t)); + + // If this is the lib{}/libu{} group, pick a member which we would + // link. For libu{} we want the "see through" logic. + // + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, link_info (t.base_scope (), ot)); + + if ((st && pt->is_a ()) || (at && pt->is_a ())) + return pt->in (t.weak_scope ()) ? pt : nullptr; + + // See through libux{}. Note that we are always in the same project + // (and thus amalgamation). + // + if (pt->is_a ()) + return pt; + } + + return file_rule::filter (a, t, p); + } + + bool install_rule:: + match (action a, target& t, const string& hint) const + { + // @@ How do we split the hint between the two? + // + + // We only want to handle installation if we are also the ones building + // this target. So first run link's match(). + // + return link_.match (a, t, hint) && file_rule::match (a, t, ""); + } + + recipe install_rule:: + apply (action a, target& t) const + { + recipe r (file_rule::apply (a, t)); + + if (a.operation () == update_id) + { + // Signal to the link rule that this is update for install. And if the + // update has already been executed, verify it was done for install. + // + auto& md (t.data ()); + + if (md.for_install) + { + if (!*md.for_install) + fail << "target " << t << " already updated but not for install"; + } + else + md.for_install = true; + } + else // install or uninstall + { + // Derive shared library paths and cache them in the target's aux + // storage if we are un/installing (used in *_extra() functions + // below). + // + static_assert (sizeof (link_rule::libs_paths) <= target::data_size, + "insufficient space"); + + file* f; + if ((f = t.is_a ()) != nullptr && tclass != "windows") + { + const string* p (cast_null (t["bin.lib.prefix"])); + const string* s (cast_null (t["bin.lib.suffix"])); + t.data ( + link_.derive_libs_paths (*f, + p != nullptr ? p->c_str (): nullptr, + s != nullptr ? s->c_str (): nullptr)); + } + } + + return r; + } + + bool install_rule:: + install_extra (const file& t, const install_dir& id) const + { + bool r (false); + + if (t.is_a () && tclass != "windows") + { + // Here we may have a bunch of symlinks that we need to install. + // + const scope& rs (t.root_scope ()); + auto& lp (t.data ()); + + auto ln = [&rs, &id] (const path& f, const path& l) + { + install_l (rs, id, f.leaf (), l.leaf (), false); + return true; + }; + + const path& lk (lp.link); + const path& so (lp.soname); + const path& in (lp.interm); + + const path* f (lp.real); + + if (!in.empty ()) {r = ln (*f, in) || r; f = ∈} + if (!so.empty ()) {r = ln (*f, so) || r; f = &so;} + if (!lk.empty ()) {r = ln (*f, lk) || r; } + } + + return r; + } + + bool install_rule:: + uninstall_extra (const file& t, const install_dir& id) const + { + bool r (false); + + if (t.is_a () && tclass != "windows") + { + // Here we may have a bunch of symlinks that we need to uninstall. + // + const scope& rs (t.root_scope ()); + auto& lp (t.data ()); + + auto rm = [&rs, &id] (const path& l) + { + return uninstall_f (rs, id, nullptr, l.leaf (), false); + }; + + const path& lk (lp.link); + const path& so (lp.soname); + const path& in (lp.interm); + + if (!lk.empty ()) r = rm (lk) || r; + if (!so.empty ()) r = rm (so) || r; + if (!in.empty ()) r = rm (in) || r; + } + + return r; + } + + // libux_install_rule + // + libux_install_rule:: + libux_install_rule (data&& d, const link_rule& l) + : common (move (d)), link_ (l) {} + + const target* libux_install_rule:: + filter (action a, const target& t, prerequisite_member p) const + { + // The "see through" semantics that should be parallel to install_rule + // above. In particular, here we use libue/libua/libus{} as proxies for + // exe/liba/libs{} there. + + otype ot (link_type (t).type); + + if (t.is_a () && x_header (p)) + return nullptr; + + if (p.is_a () || p.is_a (compile_types (ot).bmi)) + { + const target& mt (p.search (t)); + + for (prerequisite_member mp: group_prerequisite_members (a, mt)) + { + if (mp.is_a (*x_mod)) + return t.is_a () ? nullptr : alias_rule::filter (a, mt, mp); + } + } + + bool st (t.is_a () || t.is_a ()); // Target needs shared. + bool at (t.is_a () || t.is_a ()); // Target needs static. + + if ((st && (p.is_a () || p.is_a ())) || + (at && (p.is_a () || p.is_a ()))) + { + const target* pt (&p.search (t)); + + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, link_info (t.base_scope (), ot)); + + if ((st && pt->is_a ()) || (at && pt->is_a ())) + return pt->in (t.weak_scope ()) ? pt : nullptr; + + if (pt->is_a ()) + return pt; + } + + return alias_rule::filter (a, t, p); + } + + bool libux_install_rule:: + match (action a, target& t, const string& hint) const + { + // We only want to handle installation if we are also the ones building + // this target. So first run link's match(). + // + return link_.match (a, t, hint) && alias_rule::match (a, t, ""); + } + } +} diff --git a/build2/cc/install-rule.hxx b/build2/cc/install-rule.hxx new file mode 100644 index 0000000..ac2f93a --- /dev/null +++ b/build2/cc/install-rule.hxx @@ -0,0 +1,77 @@ +// file : build2/cc/install-rule.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BUILD2_CC_INSTALL_RULE_HXX +#define BUILD2_CC_INSTALL_RULE_HXX + +#include +#include + +#include + +#include +#include + +namespace build2 +{ + namespace cc + { + class link_rule; + + // Installation rule for exe{} and lib*{}. Here we do: + // + // 1. Signal to the link rule that this is update for install. + // + // 2. Additional filtering of prerequisites (e.g., headers of an exe{}). + // + // 3. Extra un/installation (e.g., libs{} symlinks). + // + class install_rule: public install::file_rule, virtual common + { + public: + install_rule (data&&, const link_rule&); + + virtual const target* + filter (action, const target&, prerequisite_member) const override; + + virtual bool + match (action, target&, const string&) const override; + + virtual recipe + apply (action, target&) const override; + + virtual bool + install_extra (const file&, const install_dir&) const override; + + virtual bool + uninstall_extra (const file&, const install_dir&) const override; + + private: + const link_rule& link_; + }; + + // Installation rule for libu*{}. + // + // While libu*{} themselves are not installable, we need to see through + // them in case they depend on stuff that we need to install (e.g., + // headers). Note that we use the alias_rule as a base. + // + class libux_install_rule: public install::alias_rule, virtual common + { + public: + libux_install_rule (data&&, const link_rule&); + + virtual const target* + filter (action, const target&, prerequisite_member) const override; + + virtual bool + match (action, target&, const string&) const override; + + private: + const link_rule& link_; + }; + } +} + +#endif // BUILD2_CC_INSTALL_RULE_HXX diff --git a/build2/cc/install.cxx b/build2/cc/install.cxx deleted file mode 100644 index fcaf626..0000000 --- a/build2/cc/install.cxx +++ /dev/null @@ -1,246 +0,0 @@ -// file : build2/cc/install.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include - -#include - -#include - -#include // match() -#include - -using namespace std; - -namespace build2 -{ - namespace cc - { - using namespace bin; - - // file_install - // - file_install:: - file_install (data&& d, const link& l): common (move (d)), link_ (l) {} - - const target* file_install:: - filter (action a, const target& t, prerequisite_member p) const - { - // NOTE: see also alias_install::filter() below if changing anything - // here. - - otype ot (link_type (t).type); - - // Don't install executable's prerequisite headers. - // - if (t.is_a () && x_header (p)) - return nullptr; - - // Here is a problem: if the user spells the obj*/bmi*{} targets - // explicitly, then the source files, including headers/modules may be - // specified as preprequisites of those targets and not of this target. - // While this can be worked around for headers by also listing them as - // prerequisites of this target, this won't work for modules (since they - // are compiled). So what we are going to do here is detect bmi*{} and - // translate them to their mxx{} (this doesn't quite work for headers - // since there would normally be several of them). - // - if (p.is_a () || p.is_a (compile_types (ot).bmi)) - { - const target& mt (p.search (t)); - - for (prerequisite_member mp: group_prerequisite_members (a, mt)) - { - if (mp.is_a (*x_mod)) - return t.is_a () ? nullptr : file_rule::filter (a, mt, mp); - } - } - - // If this is a shared library prerequisite, install it as long as it - // is in the same amalgamation as we are. - // - // Less obvious: we also want to install a static library prerequisite - // of a library (since it could be referenced from its .pc file, etc). - // - bool st (t.is_a () || t.is_a ()); // Target needs shared. - bool at (t.is_a () || t.is_a ()); // Target needs static. - - if ((st && (p.is_a () || p.is_a ())) || - (at && (p.is_a () || p.is_a ()))) - { - const target* pt (&p.search (t)); - - // If this is the lib{}/libu{} group, pick a member which we would - // link. For libu{} we want to the "see through" logic. - // - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, a, link_info (t.base_scope (), ot)); - - if ((st && pt->is_a ()) || (at && pt->is_a ())) - return pt->in (t.weak_scope ()) ? pt : nullptr; - - // See through libux{}. Note that we are always in the same project - // (and thus amalgamation). - // - if (pt->is_a ()) - return pt; - } - - return file_rule::filter (a, t, p); - } - - match_result file_install:: - match (action a, target& t, const string& hint) const - { - // @@ How do we split the hint between the two? - // - - // We only want to handle installation if we are also the ones building - // this target. So first run link's match(). - // - match_result r (link_.match (a, t, hint)); - return r ? file_rule::match (a, t, "") : r; - } - - recipe file_install:: - apply (action a, target& t) const - { - recipe r (file_rule::apply (a, t)); - - // Derive shared library paths and cache them in the target's aux - // storage if we are (un)installing (used in *_extra() functions below). - // - if (a.operation () == install_id || a.operation () == uninstall_id) - { - file* f; - if ((f = t.is_a ()) != nullptr && tclass != "windows") - { - const string* p (cast_null (t["bin.lib.prefix"])); - const string* s (cast_null (t["bin.lib.suffix"])); - t.data ( - link_.derive_libs_paths (*f, - p != nullptr ? p->c_str (): nullptr, - s != nullptr ? s->c_str (): nullptr)); - } - } - - return r; - } - - void file_install:: - install_extra (const file& t, const install_dir& id) const - { - if (t.is_a () && tclass != "windows") - { - // Here we may have a bunch of symlinks that we need to install. - // - const scope& rs (t.root_scope ()); - auto& lp (t.data ()); - - auto ln = [&rs, &id] (const path& f, const path& l) - { - install_l (rs, id, f.leaf (), l.leaf (), false); - }; - - const path& lk (lp.link); - const path& so (lp.soname); - const path& in (lp.interm); - - const path* f (&lp.real); - - if (!in.empty ()) {ln (*f, in); f = ∈} - if (!so.empty ()) {ln (*f, so); f = &so;} - if (!lk.empty ()) {ln (*f, lk);} - } - } - - bool file_install:: - uninstall_extra (const file& t, const install_dir& id) const - { - bool r (false); - - if (t.is_a () && tclass != "windows") - { - // Here we may have a bunch of symlinks that we need to uninstall. - // - const scope& rs (t.root_scope ()); - auto& lp (t.data ()); - - auto rm = [&rs, &id] (const path& l) - { - return uninstall_f (rs, id, nullptr, l.leaf (), false); - }; - - const path& lk (lp.link); - const path& so (lp.soname); - const path& in (lp.interm); - - if (!lk.empty ()) r = rm (lk) || r; - if (!so.empty ()) r = rm (so) || r; - if (!in.empty ()) r = rm (in) || r; - } - - return r; - } - - // alias_install - // - alias_install:: - alias_install (data&& d, const link& l): common (move (d)), link_ (l) {} - - const target* alias_install:: - filter (action a, const target& t, prerequisite_member p) const - { - // The "see through" semantics that should be parallel to file_install - // above. In particular, here we use libue/libua/libus{} as proxies for - // exe/liba/libs{} there. - - otype ot (link_type (t).type); - - if (t.is_a () && x_header (p)) - return nullptr; - - if (p.is_a () || p.is_a (compile_types (ot).bmi)) - { - const target& mt (p.search (t)); - - for (prerequisite_member mp: group_prerequisite_members (a, mt)) - { - if (mp.is_a (*x_mod)) - return t.is_a () ? nullptr : alias_rule::filter (a, mt, mp); - } - } - - bool st (t.is_a () || t.is_a ()); // Target needs shared. - bool at (t.is_a () || t.is_a ()); // Target needs static. - - if ((st && (p.is_a () || p.is_a ())) || - (at && (p.is_a () || p.is_a ()))) - { - const target* pt (&p.search (t)); - - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, a, link_info (t.base_scope (), ot)); - - if ((st && pt->is_a ()) || (at && pt->is_a ())) - return pt->in (t.weak_scope ()) ? pt : nullptr; - - if (pt->is_a ()) - return pt; - } - - return alias_rule::filter (a, t, p); - } - - match_result alias_install:: - match (action a, target& t, const string& hint) const - { - // We only want to handle installation if we are also the ones building - // this target. So first run link's match(). - // - match_result r (link_.match (a, t, hint)); - return r ? alias_rule::match (a, t, "") : r; - } - } -} diff --git a/build2/cc/install.hxx b/build2/cc/install.hxx deleted file mode 100644 index 28a0a94..0000000 --- a/build2/cc/install.hxx +++ /dev/null @@ -1,67 +0,0 @@ -// file : build2/cc/install.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef BUILD2_CC_INSTALL_HXX -#define BUILD2_CC_INSTALL_HXX - -#include -#include - -#include - -#include -#include - -namespace build2 -{ - namespace cc - { - class link; - - // Installation rule for exe{}, lib*{}, etc. - // - class file_install: public install::file_rule, virtual common - { - public: - file_install (data&&, const link&); - - virtual const target* - filter (action, const target&, prerequisite_member) const override; - - virtual match_result - match (action, target&, const string&) const override; - - virtual recipe - apply (action, target&) const override; - - virtual void - install_extra (const file&, const install_dir&) const override; - - virtual bool - uninstall_extra (const file&, const install_dir&) const override; - - private: - const link& link_; - }; - - // Installation rule for libux{}. - // - class alias_install: public install::alias_rule, virtual common - { - public: - alias_install (data&&, const link&); - - virtual const target* - filter (action, const target&, prerequisite_member) const override; - - virtual match_result - match (action, target&, const string&) const override; - - private: - const link& link_; - }; - } -} - -#endif // BUILD2_CC_INSTALL_HXX diff --git a/build2/cc/link-rule.cxx b/build2/cc/link-rule.cxx new file mode 100644 index 0000000..d06a835 --- /dev/null +++ b/build2/cc/link-rule.cxx @@ -0,0 +1,2104 @@ +// file : build2/cc/link-rule.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include // exit() + +#include +#include // file_exists() + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include // c, pc* +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace cc + { + using namespace bin; + + link_rule:: + link_rule (data&& d) + : common (move (d)), + rule_id (string (x) += ".link 1") + { + static_assert (sizeof (match_data) <= target::data_size, + "insufficient space"); + } + + bool link_rule:: + match (action a, target& t, const string& hint) const + { + tracer trace (x, "link_rule::match"); + + // NOTE: may be called multiple times (see install rules). + + ltype lt (link_type (t)); + otype ot (lt.type); + + // If this is a library, link-up to our group (this is the target group + // protocol which means this can be done whether we match or not). + // + if (lt.library ()) + { + if (t.group == nullptr) + t.group = &search (t, + lt.utility ? libu::static_type : lib::static_type, + t.dir, t.out, t.name); + } + + // Scan prerequisites and see if we can work with what we've got. Note + // that X could be C. We handle this by always checking for X first. + // + // Note also that we treat bmi{} as obj{}. + // + bool seen_x (false), seen_c (false), seen_obj (false), seen_lib (false); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + if (p.is_a (x_src) || (x_mod != nullptr && p.is_a (*x_mod))) + { + seen_x = seen_x || true; + } + else if (p.is_a ()) + { + seen_c = seen_c || true; + } + else if (p.is_a () || p.is_a ()) + { + seen_obj = seen_obj || true; + } + else if (p.is_a () || p.is_a ()) + { + if (ot != otype::e) + fail << p.type ().name << "{} as prerequisite of " << t; + + seen_obj = seen_obj || true; + } + else if (p.is_a () || p.is_a ()) + { + if (ot != otype::a) + fail << p.type ().name << "{} as prerequisite of " << t; + + seen_obj = seen_obj || true; + } + else if (p.is_a () || p.is_a ()) + { + if (ot != otype::s) + fail << p.type ().name << "{} as prerequisite of " << t; + + seen_obj = seen_obj || true; + } + else if (p.is_a () || + p.is_a () || + p.is_a () || + p.is_a ()) + { + seen_lib = seen_lib || true; + } + // If this is some other c-common source (say C++ in a C rule), then + // it will most definitely need to be compiled but we can't do that. + // + else if (p.is_a ()) + return false; + } + + if (!(seen_x || seen_c || seen_obj || seen_lib)) + return false; + + // We will only chain a C source if there is also an X source or we were + // explicitly told to. + // + if (seen_c && !seen_x && hint < x) + { + l4 ([&]{trace << "C prerequisite without " << x_lang << " or hint";}); + return false; + } + + return true; + } + + auto link_rule:: + derive_libs_paths (file& ls, const char* pfx, const char* sfx) const + -> libs_paths + { + const char* ext (nullptr); + + bool win (tclass == "windows"); + + if (win) + { + if (tsys == "mingw32") + { + if (pfx == nullptr) + pfx = "lib"; + } + + ext = "dll"; + } + else + { + if (pfx == nullptr) + pfx = "lib"; + + if (tclass == "macos") + ext = "dylib"; + else + ext = "so"; + } + + // First sort out which extension we are using. + // + const string& e (ls.derive_extension (ext)); + + auto append_ext = [&e] (path& p) + { + if (!e.empty ()) + { + p += '.'; + p += e; + } + }; + + // Figure out the version. + // + string v; + using verion_map = map; + if (const verion_map* m = cast_null (ls["bin.lib.version"])) + { + // First look for the target system. + // + auto i (m->find (tsys)); + + // Then look for the target class. + // + if (i == m->end ()) + i = m->find (tclass); + + // Then look for the wildcard. Since it is higly unlikely one can have + // a version that will work across platforms, this is only useful to + // say "all others -- no version". + // + if (i == m->end ()) + i = m->find ("*"); + + // At this stage the only platform-specific version we support is the + // "no version" override. + // + if (i != m->end () && !i->second.empty ()) + fail << i->first << "-specific bin.lib.version not yet supported"; + + // Finally look for the platform-independent version. + // + if (i == m->end ()) + i = m->find (""); + + // If we didn't find anything, fail. If the bin.lib.version was + // specified, then it should explicitly handle all the targets. + // + if (i == m->end ()) + fail << "no version for " << ctgt << " in bin.lib.version" << + info << "considere adding " << tsys << "@ or " << tclass + << "@"; + + v = i->second; + } + + // Now determine the paths. + // + path lk, so, in; + + // We start with the basic path. + // + path b (ls.dir); + path cp; // Clean pattern. + { + if (pfx == nullptr || pfx[0] == '\0') + { + b /= ls.name; + } + else + { + b /= pfx; + b += ls.name; + } + + cp = b; + cp += "?*"; // Don't match empty (like the libfoo.so symlink). + + if (sfx != nullptr) + { + b += sfx; + cp += sfx; + } + } + + append_ext (cp); + + // On Windows the real path is to libs{} and the link path is to the + // import library. + // + if (win) + { + // Usually on Windows the import library is called the same as the DLL + // but with the .lib extension. Which means it clashes with the static + // library. Instead of decorating the static library name with ugly + // suffixes (as is customary), let's use the MinGW approach (one must + // admit it's quite elegant) and call it .dll.lib. + // + lk = b; + append_ext (lk); + + libi& li (ls.member->as ()); // Note: libi is locked. + lk = li.derive_path (move (lk), tsys == "mingw32" ? "a" : "lib"); + } + else if (!v.empty ()) + { + lk = b; + append_ext (lk); + } + + if (!v.empty ()) + b += v; + + const path& re (ls.derive_path (move (b))); + + return libs_paths {move (lk), move (so), move (in), &re, move (cp)}; + } + + recipe link_rule:: + apply (action a, target& xt) const + { + tracer trace (x, "link_rule::apply"); + + file& t (xt.as ()); + + // Note that for_install is signalled by install_rule and therefore + // can only be relied upon during execute. + // + match_data& md (t.data (match_data ())); + + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + + ltype lt (link_type (t)); + otype ot (lt.type); + linfo li (link_info (bs, ot)); + + // Set the library type (C, C++, etc). + // + if (lt.library ()) + t.vars.assign (c_type) = string (x); + + // Derive file name(s) and add ad hoc group members. + // + { + target_lock libi; // Have to hold until after PDB member addition. + + const char* e (nullptr); // Extension. + const char* p (nullptr); // Prefix. + const char* s (nullptr); // Suffix. + + if (lt.utility) + { + // These are all static libraries with names indicating the kind of + // object files they contain (similar to how we name object files + // themselves). We add the 'u' extension to avoid clashes with + // real libraries/import stubs. + // + // libue libhello.u.a hello.exe.u.lib + // libua libhello.a.u.a hello.lib.u.lib + // libus libhello.so.u.a hello.dll.u.lib hello.dylib.u.lib + // + // Note that we currently don't add bin.lib.{prefix,suffix} since + // these are not installed. + // + if (tsys == "win32-msvc") + { + switch (ot) + { + case otype::e: e = "exe.u.lib"; break; + case otype::a: e = "lib.u.lib"; break; + case otype::s: e = "dll.u.lib"; break; + } + } + else + { + p = "lib"; + + if (tsys == "mingw32") + { + switch (ot) + { + case otype::e: e = "exe.u.a"; break; + case otype::a: e = "a.u.a"; break; + case otype::s: e = "dll.u.a"; break; + } + + } + else if (tsys == "darwin") + { + switch (ot) + { + case otype::e: e = "u.a"; break; + case otype::a: e = "a.u.a"; break; + case otype::s: e = "dylib.u.a"; break; + } + } + else + { + switch (ot) + { + case otype::e: e = "u.a"; break; + case otype::a: e = "a.u.a"; break; + case otype::s: e = "so.u.a"; break; + } + } + } + + t.derive_path (e, p, s); + } + else + { + if (auto l = t[ot == otype::e ? "bin.exe.prefix" : "bin.lib.prefix"]) + p = cast (l).c_str (); + if (auto l = t[ot == otype::e ? "bin.exe.suffix" : "bin.lib.suffix"]) + s = cast (l).c_str (); + + switch (ot) + { + case otype::e: + { + if (tclass == "windows") + e = "exe"; + else + e = ""; + + t.derive_path (e, p, s); + break; + } + case otype::a: + { + if (tsys == "win32-msvc") + e = "lib"; + else + { + if (p == nullptr) p = "lib"; + e = "a"; + } + + t.derive_path (e, p, s); + break; + } + case otype::s: + { + // On Windows libs{} is an ad hoc group. The libs{} itself is + // the DLL and we add libi{} import library as its member. + // + if (tclass == "windows") + libi = add_adhoc_member (a, t); + + md.libs_data = derive_libs_paths (t, p, s); + + if (libi) + match_recipe (libi, group_recipe); // Set recipe and unlock. + + break; + } + } + + // Add VC's .pdb. Note that we are looking for the link.exe /DEBUG + // option. + // + if (ot != otype::a && + tsys == "win32-msvc" && + (find_option ("/DEBUG", t, c_loptions, true) || + find_option ("/DEBUG", t, x_loptions, true))) + { + // Note: add after the import library if any. + // + target_lock pdb ( + add_adhoc_member (a, t, *bs.find_target_type ("pdb"))); + + // We call it foo.{exe,dll}.pdb rather than just foo.pdb because + // we can have both foo.exe and foo.dll in the same directory. + // + pdb.target->as ().derive_path (t.path (), "pdb"); + + match_recipe (pdb, group_recipe); // Set recipe and unlock. + } + + // Add pkg-config's .pc file. + // + // Note that we do it here regardless of whether we are installing + // or not for two reasons. Firstly, it is not easy to detect this + // situation in apply() since the for_install hasn't yet been + // communicated by install_rule. Secondly, always having the member + // takes care of cleanup automagically. The actual generation + // happens in perform_update() below. + // + if (ot != otype::e) + { + target_lock pc ( + add_adhoc_member ( + a, t, + ot == otype::a ? pca::static_type : pcs::static_type)); + + // Note that here we always use the lib name prefix, even on + // Windows with VC. The reason is the user needs a consistent name + // across platforms by which they can refere to the library. This + // is also the reason why we use the static/shared suffixes rather + // that a./.lib/.so/.dylib/.dll. + // + pc.target->as ().derive_path (nullptr, + (p == nullptr ? "lib" : p), + s); + + match_recipe (pc, group_recipe); // Set recipe and unlock. + } + } + } + + // Inject dependency on the output directory. + // + inject_fsdir (a, t); + + // Process prerequisites, pass 1: search and match prerequisite + // libraries, search obj/bmi{} targets, and search targets we do rule + // chaining for. + // + // We do libraries first in order to indicate that we will execute these + // targets before matching any of the obj/bmi{}. This makes it safe for + // compile::apply() to unmatch them and therefore not to hinder + // parallelism. + // + // We also create obj/bmi{} chain targets because we need to add + // (similar to lib{}) all the bmi{} as prerequisites to all the other + // obj/bmi{} that we are creating. Note that this doesn't mean that the + // compile rule will actually treat them all as prerequisite targets. + // Rather, they are used to resolve actual module imports. We don't + // really have to search obj{} targets here but it's the same code so we + // do it here to avoid duplication. + // + // Also, when cleaning, we ignore prerequisites that are not in the same + // or a subdirectory of our project root. + // + optional usr_lib_dirs; // Extract lazily. + compile_target_types tt (compile_types (ot)); + + auto skip = [&a, &rs] (const target*& pt) + { + if (a.operation () == clean_id && !pt->dir.sub (rs.out_path ())) + pt = nullptr; + + return pt == nullptr; + }; + + auto& pts (t.prerequisite_targets[a]); + size_t start (pts.size ()); + + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + // We pre-allocate a NULL slot for each (potential; see clean) + // prerequisite target. + // + pts.push_back (nullptr); + const target*& pt (pts.back ()); + + uint8_t m (0); // Mark: lib (0), src (1), mod (2), obj/bmi (3). + + bool mod (x_mod != nullptr && p.is_a (*x_mod)); + + if (mod || p.is_a (x_src) || p.is_a ()) + { + // Rule chaining, part 1. + // + + // Which scope shall we use to resolve the root? Unlikely, but + // possible, the prerequisite is from a different project + // altogether. So we are going to use the target's project. + // + + // If the source came from the lib{} group, then create the obj{} + // group and add the source as a prerequisite of the obj{} group, + // not the obj*{} member. This way we only need one prerequisite + // for, say, both liba{} and libs{}. The same goes for bmi{}. + // + bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. + + const target_type& rtt (mod + ? (group ? bmi::static_type : tt.bmi) + : (group ? obj::static_type : tt.obj)); + + const prerequisite_key& cp (p.key ()); // Source key. + + // Come up with the obj*/bmi*{} target. The source prerequisite + // directory can be relative (to the scope) or absolute. If it is + // relative, then use it as is. If absolute, then translate it to + // the corresponding directory under out_root. While the source + // directory is most likely under src_root, it is also possible it + // is under out_root (e.g., generated source). + // + dir_path d; + { + const dir_path& cpd (*cp.tk.dir); + + if (cpd.relative () || cpd.sub (rs.out_path ())) + d = cpd; + else + { + if (!cpd.sub (rs.src_path ())) + fail << "out of project prerequisite " << cp << + info << "specify corresponding " << rtt.name << "{} " + << "target explicitly"; + + d = rs.out_path () / cpd.leaf (rs.src_path ()); + } + } + + // obj/bmi{} is always in the out tree. Note that currently it could + // be the group -- we will pick a member in part 2 below. + // + pt = &search (t, rtt, d, dir_path (), *cp.tk.name, nullptr, cp.scope); + + // If we shouldn't clean obj{}, then it is fair to assume we + // shouldn't clean the source either (generated source will be in + // the same directory as obj{} and if not, well, go find yourself + // another build system ;-)). + // + if (skip (pt)) + continue; + + m = mod ? 2 : 1; + } + else if (p.is_a () || + p.is_a () || + p.is_a () || + p.is_a ()) + { + // Handle imported libraries. + // + // Note that since the search is rule-specific, we don't cache the + // target in the prerequisite. + // + if (p.proj ()) + pt = search_library ( + a, sys_lib_dirs, usr_lib_dirs, p.prerequisite); + + // The rest is the same basic logic as in search_and_match(). + // + if (pt == nullptr) + pt = &p.search (t); + + if (skip (pt)) + continue; + + // If this is the lib{}/libu{} group, then pick the appropriate + // member. + // + if (const libx* l = pt->is_a ()) + pt = &link_member (*l, a, li); + } + else + { + // If this is the obj{} or bmi{} target group, then pick the + // appropriate member. + // + if (p.is_a ()) pt = &search (t, tt.obj, p.key ()); + else if (p.is_a ()) pt = &search (t, tt.bmi, p.key ()); + else pt = &p.search (t); + + if (skip (pt)) + continue; + + m = 3; + } + + mark (pt, m); + } + + // Match lib{} (the only unmarked) in parallel and wait for completion. + // + match_members (a, t, pts, start); + + // Process prerequisites, pass 2: finish rule chaining but don't start + // matching anything yet since that may trigger recursive matching of + // bmi{} targets we haven't completed yet. Hairy, I know. + // + + // Parallel prerequisite_targets loop. + // + size_t i (start), n (pts.size ()); + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target*& pt (pts[i].target); + uintptr_t& pd (pts[i++].data); + + if (pt == nullptr) + continue; + + uint8_t m (unmark (pt)); // New mark: completion (1), verfication (2). + + if (m == 3) // obj/bmi{} + m = 1; // Just completion. + else if (m == 1 || m == 2) // Source/module chain. + { + bool mod (m == 2); + + m = 1; + + const target& rt (*pt); + bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. + + // If we have created a obj/bmi{} target group, pick one of its + // members; the rest would be primarily concerned with it. + // + pt = + group + ? &search (t, (mod ? tt.bmi : tt.obj), rt.dir, rt.out, rt.name) + : &rt; + + // If this obj*{} already has prerequisites, then verify they are + // "compatible" with what we are doing here. Otherwise, synthesize + // the dependency. Note that we may also end up synthesizing with + // someone beating up to it. In this case also verify. + // + bool verify (true); + + if (!pt->has_prerequisites ()) + { + prerequisites ps; + ps.push_back (p.as_prerequisite ()); // Source. + + // Add our lib*{} (see the export.* machinery for details) and + // bmi*{} (both original and chained; see module search logic) + // prerequisites. + // + // Note that we don't resolve lib{} to liba{}/libs{} here + // instead leaving it to whomever (e.g., the compile rule) will + // be needing *.export.*. One reason for doing it there is that + // the object target might be specified explicitly by the user + // in which case they will have to specify the set of lib{} + // prerequisites and it's much cleaner to do as lib{} rather + // than liba{}/libs{}. + // + // Initially, we were only adding imported libraries, but there + // is a problem with this approach: the non-imported library + // might depend on the imported one(s) which we will never "see" + // unless we start with this library. + // + // Note: have similar logic in make_module_sidebuild(). + // + size_t j (start); + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target* pt (pts[j++]); + + if (p.is_a () || + p.is_a () || p.is_a () || p.is_a () || + p.is_a () || p.is_a (tt.bmi)) + { + ps.push_back (p.as_prerequisite ()); + } + else if (x_mod != nullptr && p.is_a (*x_mod)) // Chained module. + { + // Searched during pass 1 but can be NULL or marked. + // + if (pt != nullptr && i != j) // Don't add self (note: both +1). + { + // This is sticky: pt might have come before us and if it + // was a group, then we would have picked up a member. So + // here we may have to "unpick" it. + // + bool group (j < i && !p.prerequisite.belongs (t)); + + unmark (pt); + ps.push_back (prerequisite (group ? *pt->group : *pt)); + } + } + } + + // Note: add to the group, not the member. + // + verify = !rt.prerequisites (move (ps)); + } + + if (verify) + { + // This gets a bit tricky. We need to make sure the source files + // are the same which we can only do by comparing the targets to + // which they resolve. But we cannot search ot's prerequisites -- + // only the rule that matches can. Note, however, that if all this + // works out, then our next step is to match the obj*{} target. If + // things don't work out, then we fail, in which case searching + // and matching speculatively doesn't really hurt. So we start the + // async match here and finish this verification in the "harvest" + // loop below. + // + const target_type& rtt (mod + ? (group ? bmi::static_type : tt.bmi) + : (group ? obj::static_type : tt.obj)); + + bool src (false); + for (prerequisite_member p1: group_prerequisite_members (a, *pt)) + { + // Most of the time we will have just a single source so fast- + // path that case. + // + if (p1.is_a (mod ? *x_mod : x_src) || p1.is_a ()) + { + src = true; + continue; // Check the rest of the prerequisites. + } + + // Ignore some known target types (fsdir, headers, libraries, + // modules). + // + if (p1.is_a () || + p1.is_a () || + p1.is_a () || p1.is_a () || p1.is_a () || + p1.is_a () || + p1.is_a () || p1.is_a () || p1.is_a () || + (p.is_a (mod ? *x_mod : x_src) && x_header (p1)) || + (p.is_a () && p1.is_a ())) + continue; + + fail << "synthesized dependency for prerequisite " << p + << " would be incompatible with existing target " << *pt << + info << "unexpected existing prerequisite type " << p1 << + info << "specify corresponding " << rtt.name << "{} " + << "dependency explicitly"; + } + + if (!src) + fail << "synthesized dependency for prerequisite " << p + << " would be incompatible with existing target " << *pt << + info << "no existing c/" << x_name << " source prerequisite" << + info << "specify corresponding " << rtt.name << "{} " + << "dependency explicitly"; + + m = 2; // Needs verification. + } + } + else // lib*{} + { + // If this is a static library, see if we need to link it whole. + // Note that we have to do it after match since we rely on the + // group link-up. + // + bool u; + if ((u = pt->is_a ()) || pt->is_a ()) + { + const variable& var (var_pool["bin.whole"]); // @@ Cache. + + // See the bin module for the lookup semantics discussion. Note + // that the variable is not overridable so we omit find_override() + // calls. + // + lookup l (p.prerequisite.vars[var]); + + if (!l.defined ()) + l = pt->find_original (var, true).first; + + if (!l.defined ()) + { + bool g (pt->group != nullptr); + l = bs.find_original (var, + &pt->type (), + &pt->name, + (g ? &pt->group->type () : nullptr), + (g ? &pt->group->name : nullptr)).first; + } + + if (l ? cast (*l) : u) + pd |= lflag_whole; + } + } + + mark (pt, m); + } + + // Process prerequisites, pass 3: match everything and verify chains. + // + + // Wait with unlocked phase to allow phase switching. + // + wait_guard wg (target::count_busy (), t[a].task_count, true); + + for (i = start; i != n; ++i) + { + const target*& pt (pts[i]); + + if (pt == nullptr) + continue; + + if (uint8_t m = unmark (pt)) + { + match_async (a, *pt, target::count_busy (), t[a].task_count); + mark (pt, m); + } + } + + wg.wait (); + + // The "harvest" loop: finish matching the targets we have started. Note + // that we may have bailed out early (thus the parallel i/n for-loop). + // + i = start; + for (prerequisite_member p: group_prerequisite_members (a, t)) + { + const target*& pt (pts[i++]); + + // Skipped or not marked for completion. + // + uint8_t m; + if (pt == nullptr || (m = unmark (pt)) == 0) + continue; + + build2::match (a, *pt); + + // Nothing else to do if not marked for verification. + // + if (m == 1) + continue; + + // Finish verifying the existing dependency (which is now matched) + // compared to what we would have synthesized. + // + bool mod (x_mod != nullptr && p.is_a (*x_mod)); + + for (prerequisite_member p1: group_prerequisite_members (a, *pt)) + { + if (p1.is_a (mod ? *x_mod : x_src) || p1.is_a ()) + { + // Searching our own prerequisite is ok, p1 must already be + // resolved. + // + if (&p.search (t) != &p1.search (*pt)) + { + bool group (!p.prerequisite.belongs (t)); + + const target_type& rtt (mod + ? (group ? bmi::static_type : tt.bmi) + : (group ? obj::static_type : tt.obj)); + + fail << "synthesized dependency for prerequisite " << p << " " + << "would be incompatible with existing target " << *pt << + info << "existing prerequisite " << p1 << " does not match " + << p << + info << "specify corresponding " << rtt.name << "{} " + << "dependency explicitly"; + } + + break; + } + } + } + + switch (a) + { + case perform_update_id: return [this] (action a, const target& t) + { + return perform_update (a, t); + }; + case perform_clean_id: return [this] (action a, const target& t) + { + return perform_clean (a, t); + }; + default: return noop_recipe; // Configure update. + } + } + + void link_rule:: + append_libraries (strings& args, + const file& l, bool la, lflags lf, + const scope& bs, action a, linfo li) const + { + // Note: lack of the "small function object" optimization will really + // kill us here since we are called in a loop. + // + auto imp = [] (const file&, bool la) {return la;}; + + auto lib = [&args, this] (const file* l, const string& p, lflags f, bool) + { + if (l != nullptr) + { + // On Windows a shared library is a DLL with the import library as a + // first ad hoc group member. MinGW though can link directly to DLLs + // (see search_library() for details). + // + if (l->member != nullptr && l->is_a () && tclass == "windows") + l = &l->member->as (); + + string p (relative (l->path ()).string ()); + + if (f & lflag_whole) + { + if (tsys == "win32-msvc") + { + p.insert (0, "/WHOLEARCHIVE:"); // Only available from VC14U2. + } + else if (tsys == "darwin") + { + p.insert (0, "-Wl,-force_load,"); + } + else + { + args.push_back ("-Wl,--whole-archive"); + args.push_back (move (p)); + args.push_back ("-Wl,--no-whole-archive"); + return; + } + } + + args.push_back (move (p)); + } + else + args.push_back (p); + }; + + auto opt = [&args, this] ( + const file& l, const string& t, bool com, bool exp) + { + // If we need an interface value, then use the group (lib{}). + // + if (const target* g = exp && l.is_a () ? l.group : &l) + { + const variable& var ( + com + ? (exp ? c_export_loptions : c_loptions) + : (t == x + ? (exp ? x_export_loptions : x_loptions) + : var_pool[t + (exp ? ".export.loptions" : ".loptions")])); + + append_options (args, *g, var); + } + }; + + process_libraries ( + a, bs, li, sys_lib_dirs, l, la, lf, imp, lib, opt, true); + } + + void link_rule:: + hash_libraries (sha256& cs, + bool& update, timestamp mt, + const file& l, bool la, lflags lf, + const scope& bs, action a, linfo li) const + { + auto imp = [] (const file&, bool la) {return la;}; + + struct data + { + sha256& cs; + bool& update; + timestamp mt; + } d {cs, update, mt}; + + auto lib = [&d, this] (const file* l, const string& p, lflags f, bool) + { + if (l != nullptr) + { + // Check if this library renders us out of date. + // + d.update = d.update || l->newer (d.mt); + + // On Windows a shared library is a DLL with the import library as a + // first ad hoc group member. MinGW though can link directly to DLLs + // (see search_library() for details). + // + if (l->member != nullptr && l->is_a () && tclass == "windows") + l = &l->member->as (); + + d.cs.append (f); + d.cs.append (l->path ().string ()); + } + else + d.cs.append (p); + }; + + auto opt = [&cs, this] ( + const file& l, const string& t, bool com, bool exp) + { + if (const target* g = exp && l.is_a () ? l.group : &l) + { + const variable& var ( + com + ? (exp ? c_export_loptions : c_loptions) + : (t == x + ? (exp ? x_export_loptions : x_loptions) + : var_pool[t + (exp ? ".export.loptions" : ".loptions")])); + + hash_options (cs, *g, var); + } + }; + + process_libraries ( + a, bs, li, sys_lib_dirs, l, la, lf, imp, lib, opt, true); + } + + void link_rule:: + rpath_libraries (strings& args, + const target& t, + const scope& bs, + action a, + linfo li, + bool for_install) const + { + // Use -rpath-link on targets that support it (Linux, *BSD). Note + // that we don't really need it for top-level libraries. + // + if (for_install) + { + if (tclass != "linux" && tclass != "bsd") + return; + } + + auto imp = [for_install] (const file& l, bool la) + { + // If we are not installing, then we only need to rpath interface + // libraries (they will include rpath's for their implementations) + // Otherwise, we have to do this recursively. In both cases we also + // want to see through utility libraries. + // + // The rpath-link part is tricky: ideally we would like to get only + // implementations and only of shared libraries. We are not interested + // in interfaces because we are linking their libraries explicitly. + // However, in our model there is no such thing as "implementation + // only"; it is either interface or interface and implementation. So + // we are going to rpath-link all of them which should be harmless + // except for some noise on the command line. + // + // + return (for_install ? !la : false) || l.is_a (); + }; + + // Package the data to keep within the 2-pointer small std::function + // optimization limit. + // + struct + { + strings& args; + bool for_install; + } d {args, for_install}; + + auto lib = [&d, this] (const file* l, const string& f, lflags, bool sys) + { + // We don't rpath system libraries. Why, you may ask? There are many + // good reasons and I have them written on a napkin somewhere... + // + if (sys) + return; + + if (l != nullptr) + { + if (!l->is_a ()) + return; + } + else + { + // This is an absolute path and we need to decide whether it is + // a shared or static library. Doesn't seem there is anything + // better than checking for a platform-specific extension (maybe + // we should cache it somewhere). + // + size_t p (path::traits::find_extension (f)); + + if (p == string::npos) + return; + + ++p; // Skip dot. + + bool c (true); + const char* e; + + if (tclass == "windows") {e = "dll"; c = false;} + else if (tsys == "darwin") e = "dylib"; + else e = "so"; + + if ((c + ? f.compare (p, string::npos, e) + : casecmp (f.c_str () + p, e)) != 0) + return; + } + + // Ok, if we are here then it means we have a non-system, shared + // library and its absolute path is in f. + // + string o (d.for_install ? "-Wl,-rpath-link," : "-Wl,-rpath,"); + + size_t p (path::traits::rfind_separator (f)); + assert (p != string::npos); + + o.append (f, 0, (p != 0 ? p : 1)); // Don't include trailing slash. + d.args.push_back (move (o)); + }; + + // In case we don't have the "small function object" optimization. + // + const function impf (imp); + const function libf (lib); + + for (const prerequisite_target& pt: t.prerequisite_targets[a]) + { + bool la; + const file* f; + + if ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || + ( f = pt->is_a ())) + { + if (!for_install && !la) + { + // Top-level sharen library dependency. It is either matched or + // imported so should be a cc library. + // + if (!cast_false (f->vars[c_system])) + args.push_back ( + "-Wl,-rpath," + f->path ().directory ().string ()); + } + + process_libraries (a, bs, li, sys_lib_dirs, + *f, la, pt.data, + impf, libf, nullptr); + } + } + } + + // Filter link.exe noise (msvc.cxx). + // + void + msvc_filter_link (ifdstream&, const file&, otype); + + // Translate target CPU to the link.exe/lib.exe /MACHINE option. + // + const char* + msvc_machine (const string& cpu); // msvc.cxx + + target_state link_rule:: + perform_update (action a, const target& xt) const + { + tracer trace (x, "link_rule::perform_update"); + + const file& t (xt.as ()); + const path& tp (t.path ()); + + match_data& md (t.data ()); + + // Unless the outer install rule signalled that this is update for + // install, signal back that we've performed plain update. + // + if (!md.for_install) + md.for_install = false; + + bool for_install (*md.for_install); + + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + + ltype lt (link_type (t)); + otype ot (lt.type); + linfo li (link_info (bs, ot)); + + // Update prerequisites. We determine if any relevant ones render us + // out-of-date manually below. + // + bool update (false); + timestamp mt (t.load_mtime ()); + target_state ts (straight_execute_prerequisites (a, t)); + + // If targeting Windows, take care of the manifest. + // + path manifest; // Manifest itself (msvc) or compiled object file. + timestamp rpath_timestamp (timestamp_nonexistent); // DLLs timestamp. + + if (lt.executable () && tclass == "windows") + { + // First determine if we need to add our rpath emulating assembly. The + // assembly itself is generated later, after updating the target. Omit + // it if we are updating for install. + // + if (!for_install) + rpath_timestamp = windows_rpath_timestamp (t, bs, a, li); + + pair p ( + windows_manifest (t, + rpath_timestamp != timestamp_nonexistent)); + + path& mf (p.first); + bool mf_cf (p.second); // Changed flag (timestamp resolution). + + timestamp mf_mt (file_mtime (mf)); + + if (tsys == "mingw32") + { + // Compile the manifest into the object file with windres. While we + // are going to synthesize an .rc file to pipe to windres' stdin, we + // will still use .manifest to check if everything is up-to-date. + // + manifest = mf + ".o"; + + if (mf_mt > file_mtime (manifest) || mf_cf) + { + path of (relative (manifest)); + + const process_path& rc (cast (rs["bin.rc.path"])); + + // @@ Would be good to add this to depdb (e.g,, rc changes). + // + const char* args[] = { + rc.recall_string (), + "--input-format=rc", + "--output-format=coff", + "-o", of.string ().c_str (), + nullptr}; + + if (verb >= 3) + print_process (args); + + try + { + process pr (rc, args, -1); + + try + { + ofdstream os (move (pr.out_fd)); + + // 1 is resource ID, 24 is RT_MANIFEST. We also need to escape + // Windows path backslashes. + // + os << "1 24 \""; + + const string& s (mf.string ()); + for (size_t i (0), j;; i = j + 1) + { + j = s.find ('\\', i); + os.write (s.c_str () + i, + (j == string::npos ? s.size () : j) - i); + + if (j == string::npos) + break; + + os.write ("\\\\", 2); + } + + os << "\"" << endl; + + os.close (); + } + catch (const io_error& e) + { + if (pr.wait ()) // Ignore if child failed. + fail << "unable to pipe resource file to " << args[0] + << ": " << e; + } + + run_finish (args, pr); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + update = true; // Manifest changed, force update. + } + } + else + { + manifest = move (mf); // Save for link.exe's /MANIFESTINPUT. + + if (mf_mt > mt || mf_cf) + update = true; // Manifest changed, force update. + } + } + + // Check/update the dependency database. + // + depdb dd (tp + ".d"); + + // First should come the rule name/version. + // + if (dd.expect (rule_id) != nullptr) + l4 ([&]{trace << "rule mismatch forcing update of " << t;}); + + lookup ranlib; + + // Then the linker checksum (ar/ranlib or the compiler). + // + if (lt.static_library ()) + { + ranlib = rs["bin.ranlib.path"]; + + const char* rl ( + ranlib + ? cast (rs["bin.ranlib.checksum"]).c_str () + : "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + + if (dd.expect (cast (rs["bin.ar.checksum"])) != nullptr) + l4 ([&]{trace << "ar mismatch forcing update of " << t;}); + + if (dd.expect (rl) != nullptr) + l4 ([&]{trace << "ranlib mismatch forcing update of " << t;}); + } + else + { + // For VC we use link.exe directly. + // + const string& cs ( + cast ( + rs[tsys == "win32-msvc" + ? var_pool["bin.ld.checksum"] + : x_checksum])); + + if (dd.expect (cs) != nullptr) + l4 ([&]{trace << "linker mismatch forcing update of " << t;}); + } + + // Next check the target. While it might be incorporated into the linker + // checksum, it also might not (e.g., VC link.exe). + // + if (dd.expect (ctgt.string ()) != nullptr) + l4 ([&]{trace << "target mismatch forcing update of " << t;}); + + // Start building the command line. While we don't yet know whether we + // will really need it, we need to hash it to find out. So the options + // are to either replicate the exact process twice, first for hashing + // then for building or to go ahead and start building and hash the + // result. The first approach is probably more efficient while the + // second is simpler. Let's got with the simpler for now (actually it's + // kind of a hybrid). + // + cstrings args {nullptr}; // Reserve one for config.bin.ar/config.x. + + // Storage. + // + string soname1, soname2; + strings sargs; + + if (lt.static_library ()) + { + if (tsys == "win32-msvc") ; + else + { + // If the user asked for ranlib, don't try to do its function with + // -s. Some ar implementations (e.g., the LLVM one) don't support + // leading '-'. + // + args.push_back (ranlib ? "rc" : "rcs"); + } + } + else + { + if (tsys == "win32-msvc") + { + // We are using link.exe directly so don't pass the compiler + // options. + } + else + { + append_options (args, t, c_coptions); + append_options (args, t, x_coptions); + append_options (args, tstd); + } + + append_options (args, t, c_loptions); + append_options (args, t, x_loptions); + + // Extra system library dirs (last). + // + // @@ /LIBPATH:, not /LIBPATH + // + assert (sys_lib_dirs_extra <= sys_lib_dirs.size ()); + append_option_values ( + args, + cclass == compiler_class::msvc ? "/LIBPATH:" : "-L", + sys_lib_dirs.begin () + sys_lib_dirs_extra, sys_lib_dirs.end (), + [] (const dir_path& d) {return d.string ().c_str ();}); + + // Handle soname/rpath. + // + if (tclass == "windows") + { + // Limited emulation for Windows with no support for user-defined + // rpaths. + // + auto l (t["bin.rpath"]); + + if (l && !l->empty ()) + fail << ctgt << " does not support rpath"; + } + else + { + // Set soname. + // + if (lt.shared_library ()) + { + const libs_paths& paths (md.libs_data); + const string& leaf (paths.effect_soname ().leaf ().string ()); + + if (tclass == "macos") + { + // With Mac OS 10.5 (Leopard) Apple finally caved in and gave us + // a way to emulate vanilla -rpath. + // + // It may seem natural to do something different on update for + // install. However, if we don't make it @rpath, then the user + // won't be able to use config.bin.rpath for installed libraries. + // + soname1 = "-install_name"; + soname2 = "@rpath/" + leaf; + } + else + soname1 = "-Wl,-soname," + leaf; + + if (!soname1.empty ()) + args.push_back (soname1.c_str ()); + + if (!soname2.empty ()) + args.push_back (soname2.c_str ()); + } + + // Add rpaths. We used to first add the ones specified by the user + // so that they take precedence. But that caused problems if we have + // old versions of the libraries sitting in the rpath location + // (e.g., installed libraries). And if you think about this, it's + // probably correct to prefer libraries that we explicitly imported + // to the ones found via rpath. + // + // Note also that if this is update for install, then we don't add + // rpath of the imported libraries (i.e., we assume they are also + // installed). But we add -rpath-link for some platforms. + // + rpath_libraries (sargs, t, bs, a, li, for_install); + + if (auto l = t["bin.rpath"]) + for (const dir_path& p: cast (l)) + sargs.push_back ("-Wl,-rpath," + p.string ()); + } + } + + // All the options should now be in. Hash them and compare with the db. + // + { + sha256 cs; + + for (size_t i (1); i != args.size (); ++i) + cs.append (args[i]); + + for (size_t i (0); i != sargs.size (); ++i) + cs.append (sargs[i]); + + if (dd.expect (cs.string ()) != nullptr) + l4 ([&]{trace << "options mismatch forcing update of " << t;}); + } + + // Finally, hash and compare the list of input files. + // + // Should we capture actual files or their checksum? The only good + // reason for capturing actual files is diagnostics: we will be able + // to pinpoint exactly what is causing the update. On the other hand, + // the checksum is faster and simpler. And we like simple. + // + { + sha256 cs; + + for (const prerequisite_target& p: t.prerequisite_targets[a]) + { + const target* pt (p.target); + + // If this is bmi*{}, then obj*{} is its ad hoc member. + // + if (modules) + { + if (pt->is_a () || pt->is_a () || pt->is_a ()) + pt = pt->member; + } + + const file* f; + bool la (false), ls (false); + + if ((f = pt->is_a ()) || + (f = pt->is_a ()) || + (f = pt->is_a ()) || + (!lt.static_library () && // @@ UTL: TODO libua to liba link. + ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || + (ls = (f = pt->is_a ()))))) + { + // Link all the dependent interface libraries (shared) or interface + // and implementation (static), recursively. + // + // Also check if any of them render us out of date. The tricky + // case is, say, a utility library (static) that depends on a + // shared library. When the shared library is updated, there is no + // reason to re-archive the utility but those who link the utility + // have to "see through" the changes in the shared library. + // + if (la || ls) + { + hash_libraries (cs, update, mt, *f, la, p.data, bs, a, li); + f = nullptr; // Timestamp checked by hash_libraries(). + } + else + cs.append (f->path ().string ()); + } + else + f = pt->is_a (); // Consider executable mtime (e.g., linker). + + // Check if this input renders us out of date. + // + if (f != nullptr) + update = update || f->newer (mt); + } + + // Treat it as input for both MinGW and VC (mtime checked above). + // + if (!manifest.empty ()) + cs.append (manifest.string ()); + + // Treat .libs as inputs, not options. + // + if (!lt.static_library ()) + { + hash_options (cs, t, c_libs); + hash_options (cs, t, x_libs); + } + + if (dd.expect (cs.string ()) != nullptr) + l4 ([&]{trace << "file set mismatch forcing update of " << t;}); + } + + // If any of the above checks resulted in a mismatch (different linker, + // options or input file set), or if the database is newer than the + // target (interrupted update) then force the target update. Also note + // this situation in the "from scratch" flag. + // + bool scratch (false); + if (dd.writing () || dd.mtime () > mt) + scratch = update = true; + + dd.close (); + + // (Re)generate pkg-config's .pc file. While the target itself might be + // up-to-date from a previous run, there is no guarantee that .pc exists + // or also up-to-date. So to keep things simple we just regenerate it + // unconditionally. + // + // Also, if you are wondering why don't we just always produce this .pc, + // install or no install, the reason is unless and until we are updating + // for install, we have no idea where-to things will be installed. + // + if (for_install) + { + bool la; + const file* f; + + if ((la = (f = t.is_a ())) || + ( f = t.is_a ())) + pkgconfig_save (a, *f, la); + } + + // If nothing changed, then we are done. + // + if (!update) + return ts; + + // Ok, so we are updating. Finish building the command line. + // + string out, out1, out2, out3; // Storage. + + // Translate paths to relative (to working directory) ones. This results + // in easier to read diagnostics. + // + path relt (relative (tp)); + + const process_path* ld (nullptr); + if (lt.static_library ()) + { + ld = &cast (rs["bin.ar.path"]); + + if (tsys == "win32-msvc") + { + // lib.exe has /LIBPATH but it's not clear/documented what it's used + // for. Perhaps for link-time code generation (/LTCG)? If that's the + // case, then we may need to pass *.loptions. + // + args.push_back ("/NOLOGO"); + + // Add /MACHINE. + // + args.push_back (msvc_machine (cast (rs[x_target_cpu]))); + + out = "/OUT:" + relt.string (); + args.push_back (out.c_str ()); + } + else + args.push_back (relt.string ().c_str ()); + } + else + { + // The options are usually similar enough to handle executables + // and shared libraries together. + // + if (tsys == "win32-msvc") + { + // Using link.exe directly. + // + ld = &cast (rs["bin.ld.path"]); + args.push_back ("/NOLOGO"); + + if (ot == otype::s) + args.push_back ("/DLL"); + + // Add /MACHINE. + // + args.push_back (msvc_machine (cast (rs[x_target_cpu]))); + + // Unless explicitly enabled with /INCREMENTAL, disable incremental + // linking (it is implicitly enabled if /DEBUG is specified). The + // reason is the .ilk file: its name cannot be changed and if we + // have, say, foo.exe and foo.dll, then they will end up stomping on + // each other's .ilk's. + // + // So the idea is to disable it by default but let the user request + // it explicitly if they are sure their project doesn't suffer from + // the above issue. We can also have something like 'incremental' + // config initializer keyword for this. + // + // It might also be a good idea to ask Microsoft to add an option. + // + if (!find_option ("/INCREMENTAL", args, true)) + args.push_back ("/INCREMENTAL:NO"); + + if (cid == compiler_id::clang) + { + // According to Clang's MSVC.cpp, we shall link libcmt.lib (static + // multi-threaded runtime) unless -nostdlib or -nostartfiles is + // specified. + // + if (!find_options ({"-nostdlib", "-nostartfiles"}, t, c_coptions) && + !find_options ({"-nostdlib", "-nostartfiles"}, t, x_coptions)) + args.push_back ("/DEFAULTLIB:libcmt.lib"); + } + + // If you look at the list of libraries Visual Studio links by + // default, it includes everything and a couple of kitchen sinks + // (winspool32.lib, ole32.lib, odbc32.lib, etc) while we want to + // keep our low-level build as pure as possible. However, there seem + // to be fairly essential libraries that are not linked by link.exe + // by default (use /VERBOSE:LIB to see the list). For example, MinGW + // by default links advapi32, shell32, user32, and kernel32. And so + // we follow suit and make sure those are linked. advapi32 and + // kernel32 are already on the default list and we only need to add + // the other two. + // + // The way we are going to do it is via the /DEFAULTLIB option + // rather than specifying the libraries as normal inputs (as VS + // does). This way the user can override our actions with the + // /NODEFAULTLIB option. + // + args.push_back ("/DEFAULTLIB:shell32.lib"); + args.push_back ("/DEFAULTLIB:user32.lib"); + + // Take care of the manifest (will be empty for the DLL). + // + if (!manifest.empty ()) + { + out3 = "/MANIFESTINPUT:"; + out3 += relative (manifest).string (); + args.push_back ("/MANIFEST:EMBED"); + args.push_back (out3.c_str ()); + } + + if (ot == otype::s) + { + // On Windows libs{} is the DLL and its first ad hoc group member + // is the import library. + // + // This will also create the .exp export file. Its name will be + // derived from the import library by changing the extension. + // Lucky for us -- there is no option to name it. + // + auto& imp (t.member->as ()); + out2 = "/IMPLIB:" + relative (imp.path ()).string (); + args.push_back (out2.c_str ()); + } + + // If we have /DEBUG then name the .pdb file. It is either the first + // (exe) or the second (dll) ad hoc group member. + // + if (find_option ("/DEBUG", args, true)) + { + auto& pdb ( + (ot == otype::e ? t.member : t.member->member)->as ()); + out1 = "/PDB:" + relative (pdb.path ()).string (); + args.push_back (out1.c_str ()); + } + + // @@ An executable can have an import library and VS seems to + // always name it. I wonder what would trigger its generation? + // Could it be the presence of export symbols? Yes, link.exe will + // generate the import library iff there are exported symbols. + // Which means there could be a DLL without an import library + // (which we currently don't handle very well). + // + out = "/OUT:" + relt.string (); + args.push_back (out.c_str ()); + } + else + { + switch (cclass) + { + case compiler_class::gcc: + { + ld = &cpath; + + // Add the option that triggers building a shared library and + // take care of any extras (e.g., import library). + // + if (ot == otype::s) + { + if (tclass == "macos") + args.push_back ("-dynamiclib"); + else + args.push_back ("-shared"); + + if (tsys == "mingw32") + { + // On Windows libs{} is the DLL and its first ad hoc group + // member is the import library. + // + auto& imp (t.member->as ()); + out = "-Wl,--out-implib=" + relative (imp.path ()).string (); + args.push_back (out.c_str ()); + } + } + + args.push_back ("-o"); + args.push_back (relt.string ().c_str ()); + break; + } + case compiler_class::msvc: assert (false); + } + } + } + + args[0] = ld->recall_string (); + + // The same logic as during hashing above. + // + for (const prerequisite_target& p: t.prerequisite_targets[a]) + { + const target* pt (p.target); + + if (modules) + { + if (pt->is_a () || pt->is_a () || pt->is_a ()) + pt = pt->member; + } + + const file* f; + bool la (false), ls (false); + + if ((f = pt->is_a ()) || + (f = pt->is_a ()) || + (f = pt->is_a ()) || + (!lt.static_library () && // @@ UTL: TODO libua to liba link. + ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || + (ls = (f = pt->is_a ()))))) + { + // Link all the dependent interface libraries (shared) or interface + // and implementation (static), recursively. + // + if (la || ls) + append_libraries (sargs, *f, la, p.data, bs, a, li); + else + sargs.push_back (relative (f->path ()).string ()); // string()&& + } + } + + // For MinGW manifest is an object file. + // + if (!manifest.empty () && tsys == "mingw32") + sargs.push_back (relative (manifest).string ()); + + // Shallow-copy sargs to args. Why not do it as we go along pushing into + // sargs? Because of potential reallocations. + // + for (const string& a: sargs) + args.push_back (a.c_str ()); + + if (!lt.static_library ()) + { + append_options (args, t, c_libs); + append_options (args, t, x_libs); + } + + args.push_back (nullptr); + + // Cleanup old (versioned) libraries. + // + if (lt.shared_library ()) + { + const libs_paths& paths (md.libs_data); + const path& p (paths.clean); + + if (!p.empty ()) + try + { + if (verb >= 4) // Seeing this with -V doesn't really add any value. + text << "rm " << p; + + auto rm = [&paths, this] (path&& m, const string&, bool interm) + { + if (!interm) + { + // Filter out paths that have one of the current paths as a + // prefix. + // + auto test = [&m] (const path& p) + { + const string& s (p.string ()); + return s.empty () || m.string ().compare (0, s.size (), s) != 0; + }; + + if (test (*paths.real) && + test (paths.interm) && + test (paths.soname) && + test (paths.link)) + { + try_rmfile (m); + try_rmfile (m + ".d"); + + if (tsys == "win32-msvc") + { + try_rmfile (m.base () += ".ilk"); + try_rmfile (m += ".pdb"); + } + } + } + return true; + }; + + path_search (p, rm, dir_path (), false); // Don't follow symlinks. + } + catch (const system_error&) {} // Ignore errors. + } + else if (lt.static_library ()) + { + // We use relative paths to the object files which means we may end + // up with different ones depending on CWD and some implementation + // treat them as different archive members. So remote the file to + // be sure. Note that we ignore errors leaving it to the achiever + // to complain. + // + if (mt != timestamp_nonexistent) + try_rmfile (relt, true); + } + + if (verb >= 2) + print_process (args); + else if (verb) + text << "ld " << t; + + try + { + // VC tools (both lib.exe and link.exe) send diagnostics to stdout. + // Also, link.exe likes to print various gratuitous messages. So for + // link.exe we redirect stdout to a pipe, filter that noise out, and + // send the rest to stderr. + // + // For lib.exe (and any other insane compiler that may try to pull off + // something like this) we are going to redirect stdout to stderr. For + // sane compilers this should be harmless. + // + bool filter (tsys == "win32-msvc" && !lt.static_library ()); + + process pr (*ld, args.data (), 0, (filter ? -1 : 2)); + + if (filter) + { + try + { + ifdstream is ( + move (pr.in_ofd), fdstream_mode::text, ifdstream::badbit); + + msvc_filter_link (is, t, ot); + + // If anything remains in the stream, send it all to stderr. Note + // that the eof check is important: if the stream is at eof, this + // and all subsequent writes to the diagnostics stream will fail + // (and you won't see a thing). + // + if (is.peek () != ifdstream::traits_type::eof ()) + diag_stream_lock () << is.rdbuf (); + + is.close (); + } + catch (const io_error&) {} // Assume exits with error. + } + + run_finish (args, pr); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + // In a multi-threaded program that fork()'ed but did not exec(), + // it is unwise to try to do any kind of cleanup (like unwinding + // the stack and running destructors). + // + if (e.child) + exit (1); + + throw failed (); + } + + // Remove the target file if any of the subsequent actions fail. If we + // don't do that, we will end up with a broken build that is up-to-date. + // + auto_rmfile rm (relt); + + if (ranlib) + { + const process_path& rl (cast (ranlib)); + + const char* args[] = { + rl.recall_string (), + relt.string ().c_str (), + nullptr}; + + if (verb >= 2) + print_process (args); + + run (rl, args); + } + + if (tclass == "windows") + { + // For Windows generate rpath-emulating assembly (unless updating for + // install). + // + if (lt.executable () && !for_install) + windows_rpath_assembly (t, bs, a, li, + cast (rs[x_target_cpu]), + rpath_timestamp, + scratch); + } + else if (lt.shared_library ()) + { + // For shared libraries we may need to create a bunch of symlinks. + // + auto ln = [] (const path& f, const path& l) + { + if (verb >= 3) + text << "ln -sf " << f << ' ' << l; + + try + { + if (file_exists (l, false)) // The -f part. + try_rmfile (l); + + mksymlink (f, l); + } + catch (const system_error& e) + { + fail << "unable to create symlink " << l << ": " << e; + } + }; + + const libs_paths& paths (md.libs_data); + + const path& lk (paths.link); + const path& so (paths.soname); + const path& in (paths.interm); + + const path* f (paths.real); + + if (!in.empty ()) {ln (f->leaf (), in); f = ∈} + if (!so.empty ()) {ln (f->leaf (), so); f = &so;} + if (!lk.empty ()) {ln (f->leaf (), lk);} + } + + rm.cancel (); + + // Should we go to the filesystem and get the new mtime? We know the + // file has been modified, so instead just use the current clock time. + // It has the advantage of having the subseconds precision. + // + t.mtime (system_clock::now ()); + return target_state::changed; + } + + target_state link_rule:: + perform_clean (action a, const target& xt) const + { + const file& t (xt.as ()); + ltype lt (link_type (t)); + + if (lt.executable ()) + { + if (tclass == "windows") + { + if (tsys == "mingw32") + return clean_extra ( + a, t, {".d", ".dlls/", ".manifest.o", ".manifest"}); + else + // Assuming it's VC or alike. Clean up .ilk in case the user + // enabled incremental linking (note that .ilk replaces .exe). + // + return clean_extra ( + a, t, {".d", ".dlls/", ".manifest", "-.ilk"}); + } + } + else if (lt.shared_library ()) + { + if (tclass == "windows") + { + // Assuming it's VC or alike. Clean up .exp and .ilk. + // + // Note that .exp is based on the .lib, not .dll name. And with + // versioning their bases may not be the same. + // + if (tsys != "mingw32") + return clean_extra (a, t, {{".d", "-.ilk"}, {"-.exp"}}); + } + else + { + // Here we can have a bunch of symlinks that we need to remove. If + // the paths are empty, then they will be ignored. + // + const libs_paths& paths (t.data ().libs_data); + + return clean_extra (a, t, {".d", + paths.link.string ().c_str (), + paths.soname.string ().c_str (), + paths.interm.string ().c_str ()}); + } + } + // For static library it's just the defaults. + + return clean_extra (a, t, {".d"}); + } + } +} diff --git a/build2/cc/link-rule.hxx b/build2/cc/link-rule.hxx new file mode 100644 index 0000000..ba40410 --- /dev/null +++ b/build2/cc/link-rule.hxx @@ -0,0 +1,166 @@ +// file : build2/cc/link-rule.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BUILD2_CC_LINK_RULE_HXX +#define BUILD2_CC_LINK_RULE_HXX + +#include + +#include +#include + +#include + +#include +#include + +namespace build2 +{ + namespace cc + { + class link_rule: public rule, virtual common + { + public: + link_rule (data&&); + + virtual bool + match (action, target&, const string&) const override; + + virtual recipe + apply (action, target&) const override; + + target_state + perform_update (action, const target&) const; + + target_state + perform_clean (action, const target&) const; + + private: + friend class install_rule; + friend class libux_install_rule; + + // Shared library paths. + // + struct libs_paths + { + // If any (except real) is empty, then it is the same as the next + // one. Except for intermediate, for which empty indicates that it is + // not used. + // + // The libs{} path is always the real path. On Windows the link path + // is the import library. + // + path link; // What we link: libfoo.so + path soname; // SONAME: libfoo-1.so, libfoo.so.1 + path interm; // Intermediate: libfoo.so.1.2 + const path* real; // Real: libfoo.so.1.2.3 + + inline const path& + effect_link () const {return link.empty () ? effect_soname () : link;} + + inline const path& + effect_soname () const {return soname.empty () ? *real : soname;} + + // Cleanup pattern used to remove previous versions. If empty, no + // cleanup is performed. The above (current) names are automatically + // filtered out. + // + path clean; + }; + + libs_paths + derive_libs_paths (file&, const char*, const char*) const; + + struct match_data + { + // The "for install" condition is signalled to us by install_rule when + // it is matched for the update operation. It also verifies that if we + // have already been executed, then it was for install. + // + // This has an interesting implication: it means that this rule cannot + // be used to update targets during match. Specifically, we cannot be + // executed for group resolution purposes (not a problem) nor as part + // of the generated source update. The latter case can be a problem: + // imagine a code generator that itself may need to be updated before + // it can be used to re-generate some out-of-date source code. As an + // aside, note that even if we were somehow able to communicate the + // "for install" in this case, the result of such an update may not + // actually be "usable" (e.g., not runnable because of the missing + // rpaths). There is another prominent case where the result may not + // be usable: cross-compilation. + // + // So the current (admittedly fuzzy) thinking is that a project shall + // not try to use its own build for update since it may not be usable + // (because of cross-compilations, being "for install", etc). Instead, + // it should rely on another, "usable" build of itself (this, BTW, is + // related to bpkg's build-time vs run-time dependencies). + // + optional for_install; + + libs_paths libs_data; + }; + + // Library handling. + // + void + append_libraries (strings&, + const file&, bool, lflags, + const scope&, action, linfo) const; + + void + hash_libraries (sha256&, + bool&, timestamp, + const file&, bool, lflags, + const scope&, action, linfo) const; + + void + rpath_libraries (strings&, + const target&, + const scope&, action, linfo, + bool) const; + + // Windows rpath emulation (windows-rpath.cxx). + // + struct windows_dll + { + const string& dll; + const string* pdb; // NULL if none. + string pdb_storage; + + bool operator< (const windows_dll& y) const {return dll < y.dll;} + }; + + using windows_dlls = std::set; + + timestamp + windows_rpath_timestamp (const file&, + const scope&, + action, linfo) const; + + windows_dlls + windows_rpath_dlls (const file&, const scope&, action, linfo) const; + + void + windows_rpath_assembly (const file&, const scope&, action, linfo, + const string&, + timestamp, + bool) const; + + // Windows-specific (windows-manifest.cxx). + // + pair + windows_manifest (const file&, bool rpath_assembly) const; + + // pkg-config's .pc file generation (pkgconfig.cxx). + // + void + pkgconfig_save (action, const file&, bool) const; + + private: + const string rule_id; + }; + } +} + +#endif // BUILD2_CC_LINK_RULE_HXX diff --git a/build2/cc/link.cxx b/build2/cc/link.cxx deleted file mode 100644 index f69d549..0000000 --- a/build2/cc/link.cxx +++ /dev/null @@ -1,2108 +0,0 @@ -// file : build2/cc/link.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#include - -#include -#include // exit() - -#include -#include // file_exists() - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include // c, pc* -#include - -using namespace std; -using namespace butl; - -namespace build2 -{ - namespace cc - { - using namespace bin; - - link:: - link (data&& d) - : common (move (d)), - rule_id (string (x) += ".link 1") - { - } - - match_result link:: - match (action act, target& t, const string& hint) const - { - tracer trace (x, "link::match"); - - // @@ TODO: - // - // - if path already assigned, verify extension? - // - // @@ Q: - // - // - if there is no .o, are we going to check if the one derived - // from target exist or can be built? A: No. - // What if there is a library. Probably ok if static, not if shared, - // (i.e., a utility library). - // - - ltype lt (link_type (t)); - otype ot (lt.type); - - // If this is a library, link-up to our group (this is the target group - // protocol which means this can be done whether we match or not). - // - if (lt.library ()) - { - if (t.group == nullptr) - t.group = &search (t, - lt.utility ? libu::static_type : lib::static_type, - t.dir, t.out, t.name); - } - - // Scan prerequisites and see if we can work with what we've got. Note - // that X could be C. We handle this by always checking for X first. - // - // Note also that we treat bmi{} as obj{}. - // - bool seen_x (false), seen_c (false), seen_obj (false), seen_lib (false); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - if (p.is_a (x_src) || (x_mod != nullptr && p.is_a (*x_mod))) - { - seen_x = seen_x || true; - } - else if (p.is_a ()) - { - seen_c = seen_c || true; - } - else if (p.is_a () || p.is_a ()) - { - seen_obj = seen_obj || true; - } - else if (p.is_a () || p.is_a ()) - { - if (ot != otype::e) - fail << p.type ().name << "{} as prerequisite of " << t; - - seen_obj = seen_obj || true; - } - else if (p.is_a () || p.is_a ()) - { - if (ot != otype::a) - fail << p.type ().name << "{} as prerequisite of " << t; - - seen_obj = seen_obj || true; - } - else if (p.is_a () || p.is_a ()) - { - if (ot != otype::s) - fail << p.type ().name << "{} as prerequisite of " << t; - - seen_obj = seen_obj || true; - } - else if (p.is_a () || - p.is_a () || - p.is_a () || - p.is_a ()) - { - seen_lib = seen_lib || true; - } - // If this is some other c-common source (say C++ in a C rule), then - // it will most definitely need to be compiled but we can't do that. - // - else if (p.is_a ()) - return false; - } - - if (!(seen_x || seen_c || seen_obj || seen_lib)) - return false; - - // We will only chain a C source if there is also an X source or we were - // explicitly told to. - // - if (seen_c && !seen_x && hint < x) - { - l4 ([&]{trace << "C prerequisite without " << x_lang << " or hint";}); - return false; - } - - return true; - } - - auto link:: - derive_libs_paths (file& ls, const char* pfx, const char* sfx) const - -> libs_paths - { - const char* ext (nullptr); - - bool win (tclass == "windows"); - - if (win) - { - if (tsys == "mingw32") - { - if (pfx == nullptr) - pfx = "lib"; - } - - ext = "dll"; - } - else - { - if (pfx == nullptr) - pfx = "lib"; - - if (tclass == "macos") - ext = "dylib"; - else - ext = "so"; - } - - // First sort out which extension we are using. - // - const string& e (ls.derive_extension (ext)); - - auto append_ext = [&e] (path& p) - { - if (!e.empty ()) - { - p += '.'; - p += e; - } - }; - - // Figure out the version. - // - string v; - using verion_map = map; - if (const verion_map* m = cast_null (ls["bin.lib.version"])) - { - // First look for the target system. - // - auto i (m->find (tsys)); - - // Then look for the target class. - // - if (i == m->end ()) - i = m->find (tclass); - - // Then look for the wildcard. Since it is higly unlikely one can have - // a version that will work across platforms, this is only useful to - // say "all others -- no version". - // - if (i == m->end ()) - i = m->find ("*"); - - // At this stage the only platform-specific version we support is the - // "no version" override. - // - if (i != m->end () && !i->second.empty ()) - fail << i->first << "-specific bin.lib.version not yet supported"; - - // Finally look for the platform-independent version. - // - if (i == m->end ()) - i = m->find (""); - - // If we didn't find anything, fail. If the bin.lib.version was - // specified, then it should explicitly handle all the targets. - // - if (i == m->end ()) - fail << "no version for " << ctgt << " in bin.lib.version" << - info << "considere adding " << tsys << "@ or " << tclass - << "@"; - - v = i->second; - } - - // Now determine the paths. - // - path lk, so, in; - - // We start with the basic path. - // - path b (ls.dir); - path cp; // Clean pattern. - { - if (pfx == nullptr || pfx[0] == '\0') - { - b /= ls.name; - } - else - { - b /= pfx; - b += ls.name; - } - - cp = b; - cp += "?*"; // Don't match empty (like the libfoo.so symlink). - - if (sfx != nullptr) - { - b += sfx; - cp += sfx; - } - } - - append_ext (cp); - - // On Windows the real path is to libs{} and the link path is to the - // import library. - // - if (win) - { - // Usually on Windows the import library is called the same as the DLL - // but with the .lib extension. Which means it clashes with the static - // library. Instead of decorating the static library name with ugly - // suffixes (as is customary), let's use the MinGW approach (one must - // admit it's quite elegant) and call it .dll.lib. - // - lk = b; - append_ext (lk); - - libi& li (ls.member->as ()); // Note: libi is locked. - lk = li.derive_path (move (lk), tsys == "mingw32" ? "a" : "lib"); - } - else if (!v.empty ()) - { - lk = b; - append_ext (lk); - } - - if (!v.empty ()) - b += v; - - const path& re (ls.derive_path (move (b))); - - return libs_paths {move (lk), move (so), move (in), re, move (cp)}; - } - - recipe link:: - apply (action act, target& xt) const - { - static_assert (sizeof (link::libs_paths) <= target::data_size, - "insufficient space"); - - tracer trace (x, "link::apply"); - - file& t (xt.as ()); - - const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); - - ltype lt (link_type (t)); - otype ot (lt.type); - linfo li (link_info (bs, ot)); - - // Set the library type (C, C++, etc). - // - if (lt.library ()) - t.vars.assign (c_type) = string (x); - - // Derive file name(s) and add ad hoc group members. - // - { - target_lock libi; // Have to hold until after PDB member addition. - - const char* e (nullptr); // Extension. - const char* p (nullptr); // Prefix. - const char* s (nullptr); // Suffix. - - if (lt.utility) - { - // These are all static libraries with names indicating the kind of - // object files they contain (similar to how we name object files - // themselves). We add the 'u' extension to avoid clashes with - // real libraries/import stubs. - // - // libue libhello.u.a hello.exe.u.lib - // libua libhello.a.u.a hello.lib.u.lib - // libus libhello.so.u.a hello.dll.u.lib hello.dylib.u.lib - // - // Note that we currently don't add bin.lib.{prefix,suffix} since - // these are not installed. - // - if (tsys == "win32-msvc") - { - switch (ot) - { - case otype::e: e = "exe.u.lib"; break; - case otype::a: e = "lib.u.lib"; break; - case otype::s: e = "dll.u.lib"; break; - } - } - else - { - p = "lib"; - - if (tsys == "mingw32") - { - switch (ot) - { - case otype::e: e = "exe.u.a"; break; - case otype::a: e = "a.u.a"; break; - case otype::s: e = "dll.u.a"; break; - } - - } - else if (tsys == "darwin") - { - switch (ot) - { - case otype::e: e = "u.a"; break; - case otype::a: e = "a.u.a"; break; - case otype::s: e = "dylib.u.a"; break; - } - } - else - { - switch (ot) - { - case otype::e: e = "u.a"; break; - case otype::a: e = "a.u.a"; break; - case otype::s: e = "so.u.a"; break; - } - } - } - - t.derive_path (e, p, s); - } - else - { - if (auto l = t[ot == otype::e ? "bin.exe.prefix" : "bin.lib.prefix"]) - p = cast (l).c_str (); - if (auto l = t[ot == otype::e ? "bin.exe.suffix" : "bin.lib.suffix"]) - s = cast (l).c_str (); - - switch (ot) - { - case otype::e: - { - if (tclass == "windows") - e = "exe"; - else - e = ""; - - t.derive_path (e, p, s); - break; - } - case otype::a: - { - if (tsys == "win32-msvc") - e = "lib"; - else - { - if (p == nullptr) p = "lib"; - e = "a"; - } - - t.derive_path (e, p, s); - break; - } - case otype::s: - { - // On Windows libs{} is an ad hoc group. The libs{} itself is - // the DLL and we add libi{} import library as its member. - // - if (tclass == "windows") - libi = add_adhoc_member (act, t); - - t.data (derive_libs_paths (t, p, s)); // Cache in target. - - if (libi) - match_recipe (libi, group_recipe); // Set recipe and unlock. - - break; - } - } - - // Add VC's .pdb. Note that we are looking for the link.exe /DEBUG - // option. - // - if (ot != otype::a && - tsys == "win32-msvc" && - (find_option ("/DEBUG", t, c_loptions, true) || - find_option ("/DEBUG", t, x_loptions, true))) - { - // Note: add after the import library if any. - // - target_lock pdb ( - add_adhoc_member (act, t, *bs.find_target_type ("pdb"))); - - // We call it foo.{exe,dll}.pdb rather than just foo.pdb because - // we can have both foo.exe and foo.dll in the same directory. - // - pdb.target->as ().derive_path (t.path (), "pdb"); - - match_recipe (pdb, group_recipe); // Set recipe and unlock. - } - - // Add pkg-config's .pc file. - // - // Note that we do it here regardless of whether we are installing - // or not for two reasons. Firstly, it is not easy to detect this - // situation in apply() since the action may (and is) overridden to - // unconditional install. Secondly, always having the member takes - // care of cleanup automagically. The actual generation happens in - // the install rule. - // - if (ot != otype::e) - { - target_lock pc ( - add_adhoc_member ( - act, t, - ot == otype::a ? pca::static_type : pcs::static_type)); - - // Note that here we always use the lib name prefix, even on - // Windows with VC. The reason is the user needs a consistent name - // across platforms by which they can refere to the library. This - // is also the reason why we use the static/shared suffixes rather - // that a./.lib/.so/.dylib/.dll. - // - pc.target->as ().derive_path (nullptr, - (p == nullptr ? "lib" : p), - s); - - match_recipe (pc, group_recipe); // Set recipe and unlock. - } - } - } - - // Inject dependency on the output directory. - // - inject_fsdir (act, t); - - // Process prerequisites, pass 1: search and match prerequisite - // libraries, search obj/bmi{} targets, and search targets we do rule - // chaining for. - // - // We do libraries first in order to indicate that we will execute these - // targets before matching any of the obj/bmi{}. This makes it safe for - // compile::apply() to unmatch them and therefore not to hinder - // parallelism. - // - // We also create obj/bmi{} chain targets because we need to add - // (similar to lib{}) all the bmi{} as prerequisites to all the other - // obj/bmi{} that we are creating. Note that this doesn't mean that the - // compile rule will actually treat them all as prerequisite targets. - // Rather, they are used to resolve actual module imports. We don't - // really have to search obj{} targets here but it's the same code so we - // do it here to avoid duplication. - // - // Also, when cleaning, we ignore prerequisites that are not in the same - // or a subdirectory of our project root. - // - optional usr_lib_dirs; // Extract lazily. - compile_target_types tt (compile_types (ot)); - - auto skip = [&act, &rs] (const target*& pt) - { - if (act.operation () == clean_id && !pt->dir.sub (rs.out_path ())) - pt = nullptr; - - return pt == nullptr; - }; - - size_t start (t.prerequisite_targets.size ()); - - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - // We pre-allocate a NULL slot for each (potential; see clean) - // prerequisite target. - // - t.prerequisite_targets.push_back (nullptr); - const target*& pt (t.prerequisite_targets.back ()); - - uint8_t m (0); // Mark: lib (0), src (1), mod (2), obj/bmi (3). - - bool mod (x_mod != nullptr && p.is_a (*x_mod)); - - if (mod || p.is_a (x_src) || p.is_a ()) - { - // Rule chaining, part 1. - // - - // Which scope shall we use to resolve the root? Unlikely, but - // possible, the prerequisite is from a different project - // altogether. So we are going to use the target's project. - // - - // If the source came from the lib{} group, then create the obj{} - // group and add the source as a prerequisite of the obj{} group, - // not the obj*{} member. This way we only need one prerequisite - // for, say, both liba{} and libs{}. The same goes for bmi{}. - // - bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. - - const target_type& rtt (mod - ? (group ? bmi::static_type : tt.bmi) - : (group ? obj::static_type : tt.obj)); - - const prerequisite_key& cp (p.key ()); // Source key. - - // Come up with the obj*/bmi*{} target. The source prerequisite - // directory can be relative (to the scope) or absolute. If it is - // relative, then use it as is. If absolute, then translate it to - // the corresponding directory under out_root. While the source - // directory is most likely under src_root, it is also possible it - // is under out_root (e.g., generated source). - // - dir_path d; - { - const dir_path& cpd (*cp.tk.dir); - - if (cpd.relative () || cpd.sub (rs.out_path ())) - d = cpd; - else - { - if (!cpd.sub (rs.src_path ())) - fail << "out of project prerequisite " << cp << - info << "specify corresponding " << rtt.name << "{} " - << "target explicitly"; - - d = rs.out_path () / cpd.leaf (rs.src_path ()); - } - } - - // obj/bmi{} is always in the out tree. Note that currently it could - // be the group -- we will pick a member in part 2 below. - // - pt = &search (t, rtt, d, dir_path (), *cp.tk.name, nullptr, cp.scope); - - // If we shouldn't clean obj{}, then it is fair to assume we - // shouldn't clean the source either (generated source will be in - // the same directory as obj{} and if not, well, go find yourself - // another build system ;-)). - // - if (skip (pt)) - continue; - - m = mod ? 2 : 1; - } - else if (p.is_a () || - p.is_a () || - p.is_a () || - p.is_a ()) - { - // Handle imported libraries. - // - // Note that since the search is rule-specific, we don't cache the - // target in the prerequisite. - // - if (p.proj ()) - pt = search_library ( - act, sys_lib_dirs, usr_lib_dirs, p.prerequisite); - - // The rest is the same basic logic as in search_and_match(). - // - if (pt == nullptr) - pt = &p.search (t); - - if (skip (pt)) - continue; - - // If this is the lib{}/libu{} group, then pick the appropriate - // member. - // - if (const libx* l = pt->is_a ()) - pt = &link_member (*l, act, li); - } - else - { - // If this is the obj{} or bmi{} target group, then pick the - // appropriate member. - // - if (p.is_a ()) pt = &search (t, tt.obj, p.key ()); - else if (p.is_a ()) pt = &search (t, tt.bmi, p.key ()); - else pt = &p.search (t); - - if (skip (pt)) - continue; - - m = 3; - } - - mark (pt, m); - } - - // Match lib{} (the only unmarked) in parallel and wait for completion. - // - match_members (act, t, t.prerequisite_targets, start); - - // Process prerequisites, pass 2: finish rule chaining but don't start - // matching anything yet since that may trigger recursive matching of - // bmi{} targets we haven't completed yet. Hairy, I know. - // - - // Parallel prerequisite_targets loop. - // - size_t i (start), n (t.prerequisite_targets.size ()); - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target*& pt (t.prerequisite_targets[i].target); - uintptr_t& pd (t.prerequisite_targets[i++].data); - - if (pt == nullptr) - continue; - - uint8_t m (unmark (pt)); // New mark: completion (1), verfication (2). - - if (m == 3) // obj/bmi{} - m = 1; // Just completion. - else if (m == 1 || m == 2) // Source/module chain. - { - bool mod (m == 2); - - m = 1; - - const target& rt (*pt); - bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. - - // If we have created a obj/bmi{} target group, pick one of its - // members; the rest would be primarily concerned with it. - // - pt = - group - ? &search (t, (mod ? tt.bmi : tt.obj), rt.dir, rt.out, rt.name) - : &rt; - - // If this obj*{} already has prerequisites, then verify they are - // "compatible" with what we are doing here. Otherwise, synthesize - // the dependency. Note that we may also end up synthesizing with - // someone beating up to it. In this case also verify. - // - bool verify (true); - - if (!pt->has_prerequisites ()) - { - prerequisites ps; - ps.push_back (p.as_prerequisite ()); // Source. - - // Add our lib*{} (see the export.* machinery for details) and - // bmi*{} (both original and chained; see module search logic) - // prerequisites. - // - // Note that we don't resolve lib{} to liba{}/libs{} here - // instead leaving it to whomever (e.g., the compile rule) will - // be needing *.export.*. One reason for doing it there is that - // the object target might be specified explicitly by the user - // in which case they will have to specify the set of lib{} - // prerequisites and it's much cleaner to do as lib{} rather - // than liba{}/libs{}. - // - // Initially, we were only adding imported libraries, but there - // is a problem with this approach: the non-imported library - // might depend on the imported one(s) which we will never "see" - // unless we start with this library. - // - // Note: have similar logic in make_module_sidebuild(). - // - size_t j (start); - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target* pt (t.prerequisite_targets[j++]); - - if (p.is_a () || - p.is_a () || p.is_a () || p.is_a () || - p.is_a () || p.is_a (tt.bmi)) - { - ps.push_back (p.as_prerequisite ()); - } - else if (x_mod != nullptr && p.is_a (*x_mod)) // Chained module. - { - // Searched during pass 1 but can be NULL or marked. - // - if (pt != nullptr && i != j) // Don't add self (note: both +1). - { - // This is sticky: pt might have come before us and if it - // was a group, then we would have picked up a member. So - // here we may have to "unpick" it. - // - bool group (j < i && !p.prerequisite.belongs (t)); - - unmark (pt); - ps.push_back (prerequisite (group ? *pt->group : *pt)); - } - } - } - - // Note: add to the group, not the member. - // - verify = !rt.prerequisites (move (ps)); - } - - if (verify) - { - // This gets a bit tricky. We need to make sure the source files - // are the same which we can only do by comparing the targets to - // which they resolve. But we cannot search ot's prerequisites -- - // only the rule that matches can. Note, however, that if all this - // works out, then our next step is to match the obj*{} target. If - // things don't work out, then we fail, in which case searching - // and matching speculatively doesn't really hurt. So we start the - // async match here and finish this verification in the "harvest" - // loop below. - // - const target_type& rtt (mod - ? (group ? bmi::static_type : tt.bmi) - : (group ? obj::static_type : tt.obj)); - - bool src (false); - for (prerequisite_member p1: group_prerequisite_members (act, *pt)) - { - // Most of the time we will have just a single source so fast- - // path that case. - // - if (p1.is_a (mod ? *x_mod : x_src) || p1.is_a ()) - { - src = true; - continue; // Check the rest of the prerequisites. - } - - // Ignore some known target types (fsdir, headers, libraries, - // modules). - // - if (p1.is_a () || - p1.is_a () || - p1.is_a () || p1.is_a () || p1.is_a () || - p1.is_a () || - p1.is_a () || p1.is_a () || p1.is_a () || - (p.is_a (mod ? *x_mod : x_src) && x_header (p1)) || - (p.is_a () && p1.is_a ())) - continue; - - fail << "synthesized dependency for prerequisite " << p - << " would be incompatible with existing target " << *pt << - info << "unexpected existing prerequisite type " << p1 << - info << "specify corresponding " << rtt.name << "{} " - << "dependency explicitly"; - } - - if (!src) - fail << "synthesized dependency for prerequisite " << p - << " would be incompatible with existing target " << *pt << - info << "no existing c/" << x_name << " source prerequisite" << - info << "specify corresponding " << rtt.name << "{} " - << "dependency explicitly"; - - m = 2; // Needs verification. - } - } - else // lib*{} - { - // If this is a static library, see if we need to link it whole. - // Note that we have to do it after match since we rely on the - // group link-up. - // - bool u; - if ((u = pt->is_a ()) || pt->is_a ()) - { - const variable& var (var_pool["bin.whole"]); // @@ Cache. - - // See the bin module for the lookup semantics discussion. Note - // that the variable is not overridable so we omit find_override() - // calls. - // - lookup l (p.prerequisite.vars[var]); - - if (!l.defined ()) - l = pt->find_original (var, true).first; - - if (!l.defined ()) - { - bool g (pt->group != nullptr); - l = bs.find_original (var, - &pt->type (), - &pt->name, - (g ? &pt->group->type () : nullptr), - (g ? &pt->group->name : nullptr)).first; - } - - if (l ? cast (*l) : u) - pd |= lflag_whole; - } - } - - mark (pt, m); - } - - // Process prerequisites, pass 3: match everything and verify chains. - // - - // Wait with unlocked phase to allow phase switching. - // - wait_guard wg (target::count_busy (), t.task_count, true); - - for (i = start; i != n; ++i) - { - const target*& pt (t.prerequisite_targets[i]); - - if (pt == nullptr) - continue; - - if (uint8_t m = unmark (pt)) - { - match_async (act, *pt, target::count_busy (), t.task_count); - mark (pt, m); - } - } - - wg.wait (); - - // The "harvest" loop: finish matching the targets we have started. Note - // that we may have bailed out early (thus the parallel i/n for-loop). - // - i = start; - for (prerequisite_member p: group_prerequisite_members (act, t)) - { - const target*& pt (t.prerequisite_targets[i++]); - - // Skipped or not marked for completion. - // - uint8_t m; - if (pt == nullptr || (m = unmark (pt)) == 0) - continue; - - build2::match (act, *pt); - - // Nothing else to do if not marked for verification. - // - if (m == 1) - continue; - - // Finish verifying the existing dependency (which is now matched) - // compared to what we would have synthesized. - // - bool mod (x_mod != nullptr && p.is_a (*x_mod)); - - for (prerequisite_member p1: group_prerequisite_members (act, *pt)) - { - if (p1.is_a (mod ? *x_mod : x_src) || p1.is_a ()) - { - // Searching our own prerequisite is ok, p1 must already be - // resolved. - // - if (&p.search (t) != &p1.search (*pt)) - { - bool group (!p.prerequisite.belongs (t)); - - const target_type& rtt (mod - ? (group ? bmi::static_type : tt.bmi) - : (group ? obj::static_type : tt.obj)); - - fail << "synthesized dependency for prerequisite " << p << " " - << "would be incompatible with existing target " << *pt << - info << "existing prerequisite " << p1 << " does not match " - << p << - info << "specify corresponding " << rtt.name << "{} " - << "dependency explicitly"; - } - - break; - } - } - } - - switch (act) - { - case perform_update_id: return [this] (action a, const target& t) - { - return perform_update (a, t); - }; - case perform_clean_id: return [this] (action a, const target& t) - { - return perform_clean (a, t); - }; - default: return noop_recipe; // Configure update. - } - } - - void link:: - append_libraries (strings& args, - const file& l, bool la, lflags lf, - const scope& bs, action act, linfo li) const - { - // Note: lack of the "small function object" optimization will really - // kill us here since we are called in a loop. - // - auto imp = [] (const file&, bool la) {return la;}; - - auto lib = [&args, this] (const file* l, const string& p, lflags f, bool) - { - if (l != nullptr) - { - // On Windows a shared library is a DLL with the import library as a - // first ad hoc group member. MinGW though can link directly to DLLs - // (see search_library() for details). - // - if (l->member != nullptr && l->is_a () && tclass == "windows") - l = &l->member->as (); - - string p (relative (l->path ()).string ()); - - if (f & lflag_whole) - { - if (tsys == "win32-msvc") - { - p.insert (0, "/WHOLEARCHIVE:"); // Only available from VC14U2. - } - else if (tsys == "darwin") - { - p.insert (0, "-Wl,-force_load,"); - } - else - { - args.push_back ("-Wl,--whole-archive"); - args.push_back (move (p)); - args.push_back ("-Wl,--no-whole-archive"); - return; - } - } - - args.push_back (move (p)); - } - else - args.push_back (p); - }; - - auto opt = [&args, this] ( - const file& l, const string& t, bool com, bool exp) - { - // If we need an interface value, then use the group (lib{}). - // - if (const target* g = exp && l.is_a () ? l.group : &l) - { - const variable& var ( - com - ? (exp ? c_export_loptions : c_loptions) - : (t == x - ? (exp ? x_export_loptions : x_loptions) - : var_pool[t + (exp ? ".export.loptions" : ".loptions")])); - - append_options (args, *g, var); - } - }; - - process_libraries ( - act, bs, li, sys_lib_dirs, l, la, lf, imp, lib, opt, true); - } - - void link:: - hash_libraries (sha256& cs, - bool& update, timestamp mt, - const file& l, bool la, lflags lf, - const scope& bs, action act, linfo li) const - { - auto imp = [] (const file&, bool la) {return la;}; - - struct data - { - sha256& cs; - bool& update; - timestamp mt; - } d {cs, update, mt}; - - auto lib = [&d, this] (const file* l, const string& p, lflags f, bool) - { - if (l != nullptr) - { - // Check if this library renders us out of date. - // - d.update = d.update || l->newer (d.mt); - - // On Windows a shared library is a DLL with the import library as a - // first ad hoc group member. MinGW though can link directly to DLLs - // (see search_library() for details). - // - if (l->member != nullptr && l->is_a () && tclass == "windows") - l = &l->member->as (); - - d.cs.append (f); - d.cs.append (l->path ().string ()); - } - else - d.cs.append (p); - }; - - auto opt = [&cs, this] ( - const file& l, const string& t, bool com, bool exp) - { - if (const target* g = exp && l.is_a () ? l.group : &l) - { - const variable& var ( - com - ? (exp ? c_export_loptions : c_loptions) - : (t == x - ? (exp ? x_export_loptions : x_loptions) - : var_pool[t + (exp ? ".export.loptions" : ".loptions")])); - - hash_options (cs, *g, var); - } - }; - - process_libraries ( - act, bs, li, sys_lib_dirs, l, la, lf, imp, lib, opt, true); - } - - void link:: - rpath_libraries (strings& args, - const target& t, - const scope& bs, - action act, - linfo li, - bool for_install) const - { - // Use -rpath-link on targets that support it (Linux, *BSD). Note - // that we don't really need it for top-level libraries. - // - if (for_install) - { - if (tclass != "linux" && tclass != "bsd") - return; - } - - auto imp = [for_install] (const file& l, bool la) - { - // If we are not installing, then we only need to rpath interface - // libraries (they will include rpath's for their implementations) - // Otherwise, we have to do this recursively. In both cases we also - // want to see through utility libraries. - // - // The rpath-link part is tricky: ideally we would like to get only - // implementations and only of shared libraries. We are not interested - // in interfaces because we are linking their libraries explicitly. - // However, in our model there is no such thing as "implementation - // only"; it is either interface or interface and implementation. So - // we are going to rpath-link all of them which should be harmless - // except for some noise on the command line. - // - // - return (for_install ? !la : false) || l.is_a (); - }; - - // Package the data to keep within the 2-pointer small std::function - // optimization limit. - // - struct - { - strings& args; - bool for_install; - } d {args, for_install}; - - auto lib = [&d, this] (const file* l, const string& f, lflags, bool sys) - { - // We don't rpath system libraries. Why, you may ask? There are many - // good reasons and I have them written on a napkin somewhere... - // - if (sys) - return; - - if (l != nullptr) - { - if (!l->is_a ()) - return; - } - else - { - // This is an absolute path and we need to decide whether it is - // a shared or static library. Doesn't seem there is anything - // better than checking for a platform-specific extension (maybe - // we should cache it somewhere). - // - size_t p (path::traits::find_extension (f)); - - if (p == string::npos) - return; - - ++p; // Skip dot. - - bool c (true); - const char* e; - - if (tclass == "windows") {e = "dll"; c = false;} - else if (tsys == "darwin") e = "dylib"; - else e = "so"; - - if ((c - ? f.compare (p, string::npos, e) - : casecmp (f.c_str () + p, e)) != 0) - return; - } - - // Ok, if we are here then it means we have a non-system, shared - // library and its absolute path is in f. - // - string o (d.for_install ? "-Wl,-rpath-link," : "-Wl,-rpath,"); - - size_t p (path::traits::rfind_separator (f)); - assert (p != string::npos); - - o.append (f, 0, (p != 0 ? p : 1)); // Don't include trailing slash. - d.args.push_back (move (o)); - }; - - // In case we don't have the "small function object" optimization. - // - const function impf (imp); - const function libf (lib); - - for (auto pt: t.prerequisite_targets) - { - bool a; - const file* f; - - if ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || - ( f = pt->is_a ())) - { - if (!for_install && !a) - { - // Top-level sharen library dependency. It is either matched or - // imported so should be a cc library. - // - if (!cast_false (f->vars[c_system])) - args.push_back ( - "-Wl,-rpath," + f->path ().directory ().string ()); - } - - process_libraries (act, bs, li, sys_lib_dirs, - *f, a, pt.data, - impf, libf, nullptr); - } - } - } - - // Filter link.exe noise (msvc.cxx). - // - void - msvc_filter_link (ifdstream&, const file&, otype); - - // Translate target CPU to the link.exe/lib.exe /MACHINE option. - // - const char* - msvc_machine (const string& cpu); // msvc.cxx - - target_state link:: - perform_update (action act, const target& xt) const - { - tracer trace (x, "link::perform_update"); - - auto oop (act.outer_operation ()); - bool for_install (oop == install_id || oop == uninstall_id); - - const file& t (xt.as ()); - const path& tp (t.path ()); - - const scope& bs (t.base_scope ()); - const scope& rs (*bs.root_scope ()); - - ltype lt (link_type (t)); - otype ot (lt.type); - linfo li (link_info (bs, ot)); - - // Update prerequisites. We determine if any relevant ones render us - // out-of-date manually below. - // - bool update (false); - timestamp mt (t.load_mtime ()); - target_state ts (straight_execute_prerequisites (act, t)); - - // If targeting Windows, take care of the manifest. - // - path manifest; // Manifest itself (msvc) or compiled object file. - timestamp rpath_timestamp (timestamp_nonexistent); // DLLs timestamp. - - if (lt.executable () && tclass == "windows") - { - // First determine if we need to add our rpath emulating assembly. The - // assembly itself is generated later, after updating the target. Omit - // it if we are updating for install. - // - if (!for_install) - rpath_timestamp = windows_rpath_timestamp (t, bs, act, li); - - pair p ( - windows_manifest (t, - rpath_timestamp != timestamp_nonexistent)); - - path& mf (p.first); - bool mf_cf (p.second); // Changed flag (timestamp resolution). - - timestamp mf_mt (file_mtime (mf)); - - if (tsys == "mingw32") - { - // Compile the manifest into the object file with windres. While we - // are going to synthesize an .rc file to pipe to windres' stdin, we - // will still use .manifest to check if everything is up-to-date. - // - manifest = mf + ".o"; - - if (mf_mt > file_mtime (manifest) || mf_cf) - { - path of (relative (manifest)); - - const process_path& rc (cast (rs["bin.rc.path"])); - - // @@ Would be good to add this to depdb (e.g,, rc changes). - // - const char* args[] = { - rc.recall_string (), - "--input-format=rc", - "--output-format=coff", - "-o", of.string ().c_str (), - nullptr}; - - if (verb >= 3) - print_process (args); - - try - { - process pr (rc, args, -1); - - try - { - ofdstream os (move (pr.out_fd)); - - // 1 is resource ID, 24 is RT_MANIFEST. We also need to escape - // Windows path backslashes. - // - os << "1 24 \""; - - const string& s (mf.string ()); - for (size_t i (0), j;; i = j + 1) - { - j = s.find ('\\', i); - os.write (s.c_str () + i, - (j == string::npos ? s.size () : j) - i); - - if (j == string::npos) - break; - - os.write ("\\\\", 2); - } - - os << "\"" << endl; - - os.close (); - } - catch (const io_error& e) - { - if (pr.wait ()) // Ignore if child failed. - fail << "unable to pipe resource file to " << args[0] - << ": " << e; - } - - run_finish (args, pr); - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - if (e.child) - exit (1); - - throw failed (); - } - - update = true; // Manifest changed, force update. - } - } - else - { - manifest = move (mf); // Save for link.exe's /MANIFESTINPUT. - - if (mf_mt > mt || mf_cf) - update = true; // Manifest changed, force update. - } - } - - // Check/update the dependency database. - // - depdb dd (tp + ".d"); - - // First should come the rule name/version. - // - if (dd.expect (rule_id) != nullptr) - l4 ([&]{trace << "rule mismatch forcing update of " << t;}); - - lookup ranlib; - - // Then the linker checksum (ar/ranlib or the compiler). - // - if (lt.static_library ()) - { - ranlib = rs["bin.ranlib.path"]; - - const char* rl ( - ranlib - ? cast (rs["bin.ranlib.checksum"]).c_str () - : "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); - - if (dd.expect (cast (rs["bin.ar.checksum"])) != nullptr) - l4 ([&]{trace << "ar mismatch forcing update of " << t;}); - - if (dd.expect (rl) != nullptr) - l4 ([&]{trace << "ranlib mismatch forcing update of " << t;}); - } - else - { - // For VC we use link.exe directly. - // - const string& cs ( - cast ( - rs[tsys == "win32-msvc" - ? var_pool["bin.ld.checksum"] - : x_checksum])); - - if (dd.expect (cs) != nullptr) - l4 ([&]{trace << "linker mismatch forcing update of " << t;}); - } - - // Next check the target. While it might be incorporated into the linker - // checksum, it also might not (e.g., VC link.exe). - // - if (dd.expect (ctgt.string ()) != nullptr) - l4 ([&]{trace << "target mismatch forcing update of " << t;}); - - // Start building the command line. While we don't yet know whether we - // will really need it, we need to hash it to find out. So the options - // are to either replicate the exact process twice, first for hashing - // then for building or to go ahead and start building and hash the - // result. The first approach is probably more efficient while the - // second is simpler. Let's got with the simpler for now (actually it's - // kind of a hybrid). - // - cstrings args {nullptr}; // Reserve one for config.bin.ar/config.x. - - // Storage. - // - string soname1, soname2; - strings sargs; - - if (lt.static_library ()) - { - if (tsys == "win32-msvc") ; - else - { - // If the user asked for ranlib, don't try to do its function with - // -s. Some ar implementations (e.g., the LLVM one) don't support - // leading '-'. - // - args.push_back (ranlib ? "rc" : "rcs"); - } - } - else - { - if (tsys == "win32-msvc") - { - // We are using link.exe directly so don't pass the compiler - // options. - } - else - { - append_options (args, t, c_coptions); - append_options (args, t, x_coptions); - append_options (args, tstd); - } - - append_options (args, t, c_loptions); - append_options (args, t, x_loptions); - - // Extra system library dirs (last). - // - // @@ /LIBPATH:, not /LIBPATH - // - assert (sys_lib_dirs_extra <= sys_lib_dirs.size ()); - append_option_values ( - args, - cclass == compiler_class::msvc ? "/LIBPATH:" : "-L", - sys_lib_dirs.begin () + sys_lib_dirs_extra, sys_lib_dirs.end (), - [] (const dir_path& d) {return d.string ().c_str ();}); - - // Handle soname/rpath. - // - if (tclass == "windows") - { - // Limited emulation for Windows with no support for user-defined - // rpaths. - // - auto l (t["bin.rpath"]); - - if (l && !l->empty ()) - fail << ctgt << " does not support rpath"; - } - else - { - // Set soname. - // - if (lt.shared_library ()) - { - const libs_paths& paths (t.data ()); - const string& leaf (paths.effect_soname ().leaf ().string ()); - - if (tclass == "macos") - { - // With Mac OS 10.5 (Leopard) Apple finally caved in and gave us - // a way to emulate vanilla -rpath. - // - // It may seem natural to do something different on update for - // install. However, if we don't make it @rpath, then the user - // won't be able to use config.bin.rpath for installed libraries. - // - soname1 = "-install_name"; - soname2 = "@rpath/" + leaf; - } - else - soname1 = "-Wl,-soname," + leaf; - - if (!soname1.empty ()) - args.push_back (soname1.c_str ()); - - if (!soname2.empty ()) - args.push_back (soname2.c_str ()); - } - - // Add rpaths. We used to first add the ones specified by the user - // so that they take precedence. But that caused problems if we have - // old versions of the libraries sitting in the rpath location - // (e.g., installed libraries). And if you think about this, it's - // probably correct to prefer libraries that we explicitly imported - // to the ones found via rpath. - // - // Note also that if this is update for install, then we don't add - // rpath of the imported libraries (i.e., we assume they are also - // installed). But we add -rpath-link for some platforms. - // - rpath_libraries (sargs, t, bs, act, li, for_install); - - if (auto l = t["bin.rpath"]) - for (const dir_path& p: cast (l)) - sargs.push_back ("-Wl,-rpath," + p.string ()); - } - } - - // All the options should now be in. Hash them and compare with the db. - // - { - sha256 cs; - - for (size_t i (1); i != args.size (); ++i) - cs.append (args[i]); - - for (size_t i (0); i != sargs.size (); ++i) - cs.append (sargs[i]); - - if (dd.expect (cs.string ()) != nullptr) - l4 ([&]{trace << "options mismatch forcing update of " << t;}); - } - - // Finally, hash and compare the list of input files. - // - // Should we capture actual files or their checksum? The only good - // reason for capturing actual files is diagnostics: we will be able - // to pinpoint exactly what is causing the update. On the other hand, - // the checksum is faster and simpler. And we like simple. - // - { - sha256 cs; - - for (auto p: t.prerequisite_targets) - { - const target* pt (p.target); - - // If this is bmi*{}, then obj*{} is its ad hoc member. - // - if (modules) - { - if (pt->is_a () || pt->is_a () || pt->is_a ()) - pt = pt->member; - } - - const file* f; - bool a (false), s (false); - - if ((f = pt->is_a ()) || - (f = pt->is_a ()) || - (f = pt->is_a ()) || - (!lt.static_library () && // @@ UTL: TODO libua to liba link. - ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || - (s = (f = pt->is_a ()))))) - { - // Link all the dependent interface libraries (shared) or interface - // and implementation (static), recursively. - // - // Also check if any of them render us out of date. The tricky - // case is, say, a utility library (static) that depends on a - // shared library. When the shared library is updated, there is no - // reason to re-archive the utility but those who link the utility - // have to "see through" the changes in the shared library. - // - if (a || s) - { - hash_libraries (cs, update, mt, *f, a, p.data, bs, act, li); - f = nullptr; // Timestamp checked by hash_libraries(). - } - else - cs.append (f->path ().string ()); - } - else - f = pt->is_a (); // Consider executable mtime (e.g., linker). - - // Check if this input renders us out of date. - // - if (f != nullptr) - update = update || f->newer (mt); - } - - // Treat it as input for both MinGW and VC (mtime checked above). - // - if (!manifest.empty ()) - cs.append (manifest.string ()); - - // Treat .libs as inputs, not options. - // - if (!lt.static_library ()) - { - hash_options (cs, t, c_libs); - hash_options (cs, t, x_libs); - } - - if (dd.expect (cs.string ()) != nullptr) - l4 ([&]{trace << "file set mismatch forcing update of " << t;}); - } - - // If any of the above checks resulted in a mismatch (different linker, - // options or input file set), or if the database is newer than the - // target (interrupted update) then force the target update. Also note - // this situation in the "from scratch" flag. - // - bool scratch (false); - if (dd.writing () || dd.mtime () > mt) - scratch = update = true; - - dd.close (); - - // (Re)generate pkg-config's .pc file. While the target itself might be - // up-to-date from a previous run, there is no guarantee that .pc exists - // or also up-to-date. So to keep things simple we just regenerate it - // unconditionally. - // - // Also, if you are wondering why don't we just always produce this .pc, - // install or no install, the reason is unless and until we are updating - // for install, we have no idea where to things will be installed. - // - if (for_install) - { - bool a; - const file* f; - - if ((a = (f = t.is_a ())) || - ( f = t.is_a ())) - { - // @@ Hack: this should really be in install:update_extra() where we - // (should) what we are installing and what not. - // - if (rs["install.root"]) - pkgconfig_save (act, *f, a); - } - } - - // If nothing changed, then we are done. - // - if (!update) - return ts; - - // Ok, so we are updating. Finish building the command line. - // - string out, out1, out2, out3; // Storage. - - // Translate paths to relative (to working directory) ones. This results - // in easier to read diagnostics. - // - path relt (relative (tp)); - - const process_path* ld (nullptr); - if (lt.static_library ()) - { - ld = &cast (rs["bin.ar.path"]); - - if (tsys == "win32-msvc") - { - // lib.exe has /LIBPATH but it's not clear/documented what it's used - // for. Perhaps for link-time code generation (/LTCG)? If that's the - // case, then we may need to pass *.loptions. - // - args.push_back ("/NOLOGO"); - - // Add /MACHINE. - // - args.push_back (msvc_machine (cast (rs[x_target_cpu]))); - - out = "/OUT:" + relt.string (); - args.push_back (out.c_str ()); - } - else - args.push_back (relt.string ().c_str ()); - } - else - { - // The options are usually similar enough to handle executables - // and shared libraries together. - // - if (tsys == "win32-msvc") - { - // Using link.exe directly. - // - ld = &cast (rs["bin.ld.path"]); - args.push_back ("/NOLOGO"); - - if (ot == otype::s) - args.push_back ("/DLL"); - - // Add /MACHINE. - // - args.push_back (msvc_machine (cast (rs[x_target_cpu]))); - - // Unless explicitly enabled with /INCREMENTAL, disable incremental - // linking (it is implicitly enabled if /DEBUG is specified). The - // reason is the .ilk file: its name cannot be changed and if we - // have, say, foo.exe and foo.dll, then they will end up stomping on - // each other's .ilk's. - // - // So the idea is to disable it by default but let the user request - // it explicitly if they are sure their project doesn't suffer from - // the above issue. We can also have something like 'incremental' - // config initializer keyword for this. - // - // It might also be a good idea to ask Microsoft to add an option. - // - if (!find_option ("/INCREMENTAL", args, true)) - args.push_back ("/INCREMENTAL:NO"); - - if (cid == compiler_id::clang) - { - // According to Clang's MSVC.cpp, we shall link libcmt.lib (static - // multi-threaded runtime) unless -nostdlib or -nostartfiles is - // specified. - // - if (!find_options ({"-nostdlib", "-nostartfiles"}, t, c_coptions) && - !find_options ({"-nostdlib", "-nostartfiles"}, t, x_coptions)) - args.push_back ("/DEFAULTLIB:libcmt.lib"); - } - - // If you look at the list of libraries Visual Studio links by - // default, it includes everything and a couple of kitchen sinks - // (winspool32.lib, ole32.lib, odbc32.lib, etc) while we want to - // keep our low-level build as pure as possible. However, there seem - // to be fairly essential libraries that are not linked by link.exe - // by default (use /VERBOSE:LIB to see the list). For example, MinGW - // by default links advapi32, shell32, user32, and kernel32. And so - // we follow suit and make sure those are linked. advapi32 and - // kernel32 are already on the default list and we only need to add - // the other two. - // - // The way we are going to do it is via the /DEFAULTLIB option - // rather than specifying the libraries as normal inputs (as VS - // does). This way the user can override our actions with the - // /NODEFAULTLIB option. - // - args.push_back ("/DEFAULTLIB:shell32.lib"); - args.push_back ("/DEFAULTLIB:user32.lib"); - - // Take care of the manifest (will be empty for the DLL). - // - if (!manifest.empty ()) - { - out3 = "/MANIFESTINPUT:"; - out3 += relative (manifest).string (); - args.push_back ("/MANIFEST:EMBED"); - args.push_back (out3.c_str ()); - } - - if (ot == otype::s) - { - // On Windows libs{} is the DLL and its first ad hoc group member - // is the import library. - // - // This will also create the .exp export file. Its name will be - // derived from the import library by changing the extension. - // Lucky for us -- there is no option to name it. - // - auto& imp (t.member->as ()); - out2 = "/IMPLIB:" + relative (imp.path ()).string (); - args.push_back (out2.c_str ()); - } - - // If we have /DEBUG then name the .pdb file. It is either the first - // (exe) or the second (dll) ad hoc group member. - // - if (find_option ("/DEBUG", args, true)) - { - auto& pdb ( - (ot == otype::e ? t.member : t.member->member)->as ()); - out1 = "/PDB:" + relative (pdb.path ()).string (); - args.push_back (out1.c_str ()); - } - - // @@ An executable can have an import library and VS seems to - // always name it. I wonder what would trigger its generation? - // Could it be the presence of export symbols? Yes, link.exe will - // generate the import library iff there are exported symbols. - // Which means there could be a DLL without an import library - // (which we currently don't handle very well). - // - out = "/OUT:" + relt.string (); - args.push_back (out.c_str ()); - } - else - { - switch (cclass) - { - case compiler_class::gcc: - { - ld = &cpath; - - // Add the option that triggers building a shared library and - // take care of any extras (e.g., import library). - // - if (ot == otype::s) - { - if (tclass == "macos") - args.push_back ("-dynamiclib"); - else - args.push_back ("-shared"); - - if (tsys == "mingw32") - { - // On Windows libs{} is the DLL and its first ad hoc group - // member is the import library. - // - auto& imp (t.member->as ()); - out = "-Wl,--out-implib=" + relative (imp.path ()).string (); - args.push_back (out.c_str ()); - } - } - - args.push_back ("-o"); - args.push_back (relt.string ().c_str ()); - break; - } - case compiler_class::msvc: assert (false); - } - } - } - - args[0] = ld->recall_string (); - - // The same logic as during hashing above. - // - for (auto p: t.prerequisite_targets) - { - const target* pt (p.target); - - if (modules) - { - if (pt->is_a () || pt->is_a () || pt->is_a ()) - pt = pt->member; - } - - const file* f; - bool a (false), s (false); - - if ((f = pt->is_a ()) || - (f = pt->is_a ()) || - (f = pt->is_a ()) || - (!lt.static_library () && // @@ UTL: TODO libua to liba link. - ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || - (s = (f = pt->is_a ()))))) - { - // Link all the dependent interface libraries (shared) or interface - // and implementation (static), recursively. - // - if (a || s) - append_libraries (sargs, *f, a, p.data, bs, act, li); - else - sargs.push_back (relative (f->path ()).string ()); // string()&& - } - } - - // For MinGW manifest is an object file. - // - if (!manifest.empty () && tsys == "mingw32") - sargs.push_back (relative (manifest).string ()); - - // Shallow-copy sargs to args. Why not do it as we go along pushing into - // sargs? Because of potential reallocations. - // - for (const string& a: sargs) - args.push_back (a.c_str ()); - - if (!lt.static_library ()) - { - append_options (args, t, c_libs); - append_options (args, t, x_libs); - } - - args.push_back (nullptr); - - // Cleanup old (versioned) libraries. - // - if (lt.shared_library ()) - { - const libs_paths& paths (t.data ()); - const path& p (paths.clean); - - if (!p.empty ()) - try - { - if (verb >= 4) // Seeing this with -V doesn't really add any value. - text << "rm " << p; - - auto rm = [&paths, this] (path&& m, const string&, bool interm) - { - if (!interm) - { - // Filter out paths that have one of the current paths as a - // prefix. - // - auto test = [&m] (const path& p) - { - const string& s (p.string ()); - return s.empty () || m.string ().compare (0, s.size (), s) != 0; - }; - - if (test (paths.real) && - test (paths.interm) && - test (paths.soname) && - test (paths.link)) - { - try_rmfile (m); - try_rmfile (m + ".d"); - - if (tsys == "win32-msvc") - { - try_rmfile (m.base () += ".ilk"); - try_rmfile (m += ".pdb"); - } - } - } - return true; - }; - - path_search (p, rm, dir_path (), false); // Don't follow symlinks. - } - catch (const system_error&) {} // Ignore errors. - } - else if (lt.static_library ()) - { - // We use relative paths to the object files which means we may end - // up with different ones depending on CWD and some implementation - // treat them as different archive members. So remote the file to - // be sure. Note that we ignore errors leaving it to the achiever - // to complain. - // - if (mt != timestamp_nonexistent) - try_rmfile (relt, true); - } - - if (verb >= 2) - print_process (args); - else if (verb) - text << "ld " << t; - - try - { - // VC tools (both lib.exe and link.exe) send diagnostics to stdout. - // Also, link.exe likes to print various gratuitous messages. So for - // link.exe we redirect stdout to a pipe, filter that noise out, and - // send the rest to stderr. - // - // For lib.exe (and any other insane compiler that may try to pull off - // something like this) we are going to redirect stdout to stderr. For - // sane compilers this should be harmless. - // - bool filter (tsys == "win32-msvc" && !lt.static_library ()); - - process pr (*ld, args.data (), 0, (filter ? -1 : 2)); - - if (filter) - { - try - { - ifdstream is ( - move (pr.in_ofd), fdstream_mode::text, ifdstream::badbit); - - msvc_filter_link (is, t, ot); - - // If anything remains in the stream, send it all to stderr. Note - // that the eof check is important: if the stream is at eof, this - // and all subsequent writes to the diagnostics stream will fail - // (and you won't see a thing). - // - if (is.peek () != ifdstream::traits_type::eof ()) - diag_stream_lock () << is.rdbuf (); - - is.close (); - } - catch (const io_error&) {} // Assume exits with error. - } - - run_finish (args, pr); - } - catch (const process_error& e) - { - error << "unable to execute " << args[0] << ": " << e; - - // In a multi-threaded program that fork()'ed but did not exec(), - // it is unwise to try to do any kind of cleanup (like unwinding - // the stack and running destructors). - // - if (e.child) - exit (1); - - throw failed (); - } - - // Remove the target file if any of the subsequent actions fail. If we - // don't do that, we will end up with a broken build that is up-to-date. - // - auto_rmfile rm (relt); - - if (ranlib) - { - const process_path& rl (cast (ranlib)); - - const char* args[] = { - rl.recall_string (), - relt.string ().c_str (), - nullptr}; - - if (verb >= 2) - print_process (args); - - run (rl, args); - } - - if (tclass == "windows") - { - // For Windows generate rpath-emulating assembly (unless updating for - // install). - // - if (lt.executable () && !for_install) - windows_rpath_assembly (t, bs, act, li, - cast (rs[x_target_cpu]), - rpath_timestamp, - scratch); - } - else if (lt.shared_library ()) - { - // For shared libraries we may need to create a bunch of symlinks. - // - auto ln = [] (const path& f, const path& l) - { - if (verb >= 3) - text << "ln -sf " << f << ' ' << l; - - try - { - if (file_exists (l, false)) // The -f part. - try_rmfile (l); - - mksymlink (f, l); - } - catch (const system_error& e) - { - fail << "unable to create symlink " << l << ": " << e; - } - }; - - const libs_paths& paths (t.data ()); - - const path& lk (paths.link); - const path& so (paths.soname); - const path& in (paths.interm); - - const path* f (&paths.real); - - if (!in.empty ()) {ln (f->leaf (), in); f = ∈} - if (!so.empty ()) {ln (f->leaf (), so); f = &so;} - if (!lk.empty ()) {ln (f->leaf (), lk);} - } - - rm.cancel (); - - // Should we go to the filesystem and get the new mtime? We know the - // file has been modified, so instead just use the current clock time. - // It has the advantage of having the subseconds precision. - // - t.mtime (system_clock::now ()); - return target_state::changed; - } - - target_state link:: - perform_clean (action act, const target& xt) const - { - const file& t (xt.as ()); - ltype lt (link_type (t)); - - if (lt.executable ()) - { - if (tclass == "windows") - { - if (tsys == "mingw32") - return clean_extra ( - act, t, {".d", ".dlls/", ".manifest.o", ".manifest"}); - else - // Assuming it's VC or alike. Clean up .ilk in case the user - // enabled incremental linking (note that .ilk replaces .exe). - // - return clean_extra ( - act, t, {".d", ".dlls/", ".manifest", "-.ilk"}); - } - } - else if (lt.shared_library ()) - { - if (tclass == "windows") - { - // Assuming it's VC or alike. Clean up .exp and .ilk. - // - // Note that .exp is based on the .lib, not .dll name. And with - // versioning their bases may not be the same. - // - if (tsys != "mingw32") - return clean_extra (act, t, {{".d", "-.ilk"}, {"-.exp"}}); - } - else - { - // Here we can have a bunch of symlinks that we need to remove. If - // the paths are empty, then they will be ignored. - // - const libs_paths& paths (t.data ()); - - return clean_extra (act, t, {".d", - paths.link.string ().c_str (), - paths.soname.string ().c_str (), - paths.interm.string ().c_str ()}); - } - } - // For static library it's just the defaults. - - return clean_extra (act, t, {".d"}); - } - } -} diff --git a/build2/cc/link.hxx b/build2/cc/link.hxx deleted file mode 100644 index c26102d..0000000 --- a/build2/cc/link.hxx +++ /dev/null @@ -1,137 +0,0 @@ -// file : build2/cc/link.hxx -*- C++ -*- -// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd -// license : MIT; see accompanying LICENSE file - -#ifndef BUILD2_CC_LINK_HXX -#define BUILD2_CC_LINK_HXX - -#include - -#include -#include - -#include - -#include -#include - -namespace build2 -{ - namespace cc - { - class link: public rule, virtual common - { - public: - link (data&&); - - virtual match_result - match (action, target&, const string&) const override; - - virtual recipe - apply (action, target&) const override; - - target_state - perform_update (action, const target&) const; - - target_state - perform_clean (action, const target&) const; - - private: - friend class file_install; - friend class alias_install; - - // Shared library paths. - // - struct libs_paths - { - // If any (except real) is empty, then it is the same as the next - // one. Except for intermediate, for which empty indicates that it is - // not used. - // - // The libs{} path is always the real path. On Windows the link path - // is the import library. - // - const path link; // What we link: libfoo.so - const path soname; // SONAME: libfoo-1.so, libfoo.so.1 - const path interm; // Intermediate: libfoo.so.1.2 - const path& real; // Real: libfoo.so.1.2.3 - - inline const path& - effect_link () const {return link.empty () ? effect_soname () : link;} - - inline const path& - effect_soname () const {return soname.empty () ? real : soname;} - - // Cleanup pattern used to remove previous versions. If empty, no - // cleanup is performed. The above (current) names are automatically - // filtered out. - // - const path clean; - }; - - libs_paths - derive_libs_paths (file&, const char*, const char*) const; - - // Library handling. - // - void - append_libraries (strings&, - const file&, bool, lflags, - const scope&, action, linfo) const; - - void - hash_libraries (sha256&, - bool&, timestamp, - const file&, bool, lflags, - const scope&, action, linfo) const; - - void - rpath_libraries (strings&, - const target&, - const scope&, action, linfo, - bool) const; - - // Windows rpath emulation (windows-rpath.cxx). - // - struct windows_dll - { - const string& dll; - const string* pdb; // NULL if none. - string pdb_storage; - - bool operator< (const windows_dll& y) const {return dll < y.dll;} - }; - - using windows_dlls = std::set; - - timestamp - windows_rpath_timestamp (const file&, - const scope&, - action, linfo) const; - - windows_dlls - windows_rpath_dlls (const file&, const scope&, action, linfo) const; - - void - windows_rpath_assembly (const file&, const scope&, action, linfo, - const string&, - timestamp, - bool) const; - - // Windows-specific (windows-manifest.cxx). - // - pair - windows_manifest (const file&, bool rpath_assembly) const; - - // pkg-config's .pc file generation (pkgconfig.cxx). - // - void - pkgconfig_save (action, const file&, bool) const; - - private: - const string rule_id; - }; - } -} - -#endif // BUILD2_CC_LINK_HXX diff --git a/build2/cc/module.cxx b/build2/cc/module.cxx index c56bca9..ae64220 100644 --- a/build2/cc/module.cxx +++ b/build2/cc/module.cxx @@ -499,8 +499,8 @@ namespace build2 // We register for configure so that we detect unresolved imports // during configuration rather that later, e.g., during update. // - const compile& cr (*this); - const link& lr (*this); + const compile_rule& cr (*this); + const link_rule& lr (*this); r.insert (perform_update_id, x_compile, cr); r.insert (perform_clean_id, x_compile, cr); @@ -559,26 +559,27 @@ namespace build2 // if (install_loaded) { - const file_install& fr (*this); - const alias_install& ar (*this); + const install_rule& ir (*this); - r.insert (perform_install_id, x_install, fr); - r.insert (perform_uninstall_id, x_uninstall, fr); + r.insert (perform_install_id, x_install, ir); + r.insert (perform_uninstall_id, x_uninstall, ir); - r.insert (perform_install_id, x_install, fr); - r.insert (perform_uninstall_id, x_uninstall, fr); + r.insert (perform_install_id, x_install, ir); + r.insert (perform_uninstall_id, x_uninstall, ir); - r.insert (perform_install_id, x_install, fr); - r.insert (perform_uninstall_id, x_uninstall, fr); + r.insert (perform_install_id, x_install, ir); + r.insert (perform_uninstall_id, x_uninstall, ir); - r.insert (perform_install_id, x_install, ar); - r.insert (perform_uninstall_id, x_uninstall, ar); + const libux_install_rule& lr (*this); - r.insert (perform_install_id, x_install, ar); - r.insert (perform_uninstall_id, x_uninstall, ar); + r.insert (perform_install_id, x_install, lr); + r.insert (perform_uninstall_id, x_uninstall, lr); - r.insert (perform_install_id, x_install, ar); - r.insert (perform_uninstall_id, x_uninstall, ar); + r.insert (perform_install_id, x_install, lr); + r.insert (perform_uninstall_id, x_uninstall, lr); + + r.insert (perform_install_id, x_install, lr); + r.insert (perform_uninstall_id, x_uninstall, lr); } } } diff --git a/build2/cc/module.hxx b/build2/cc/module.hxx index de61611..58aa184 100644 --- a/build2/cc/module.hxx +++ b/build2/cc/module.hxx @@ -13,9 +13,9 @@ #include -#include -#include -#include +#include +#include +#include namespace build2 { @@ -76,19 +76,19 @@ namespace build2 }; class module: public module_base, public virtual common, - link, - compile, - file_install, - alias_install + link_rule, + compile_rule, + install_rule, + libux_install_rule { public: explicit module (data&& d) : common (move (d)), - link (move (d)), - compile (move (d)), - file_install (move (d), *this), - alias_install (move (d), *this) {} + link_rule (move (d)), + compile_rule (move (d)), + install_rule (move (d), *this), + libux_install_rule (move (d), *this) {} void init (scope&, const location&, const variable_map&); diff --git a/build2/cc/pkgconfig.cxx b/build2/cc/pkgconfig.cxx index 0ffd135..697a60e 100644 --- a/build2/cc/pkgconfig.cxx +++ b/build2/cc/pkgconfig.cxx @@ -26,8 +26,8 @@ #include #include -#include -#include +#include +#include using namespace std; using namespace butl; @@ -451,7 +451,7 @@ namespace build2 // #ifndef BUILD2_BOOTSTRAP bool common:: - pkgconfig_load (action act, + pkgconfig_load (action a, const scope& s, lib& lt, liba* at, @@ -592,12 +592,13 @@ namespace build2 // Extract --cflags and set them as lib?{}:export.poptions. Note that we // still pass --static in case this is pkgconf which has Cflags.private. // - auto parse_cflags = [&trace, this] (target& t, const pkgconf& pc, bool a) + auto parse_cflags = + [&trace, this] (target& t, const pkgconf& pc, bool la) { strings pops; bool arg (false); - for (auto& o: pc.cflags (a)) + for (auto& o: pc.cflags (la)) { if (arg) { @@ -646,8 +647,8 @@ namespace build2 // Parse --libs into loptions/libs (interface and implementation). If // ps is not NULL, add each resolves library target as a prerequisite. // - auto parse_libs = [act, &s, top_sysd, this] - (target& t, const pkgconf& pc, bool a, prerequisites* ps) + auto parse_libs = [a, &s, top_sysd, this] + (target& t, const pkgconf& pc, bool la, prerequisites* ps) { strings lops; vector libs; @@ -664,7 +665,7 @@ namespace build2 // library names (without -l) after seeing an unknown option. // bool arg (false), first (true), known (true), have_L; - for (auto& o: pc.libs (a)) + for (auto& o: pc.libs (la)) { if (arg) { @@ -726,10 +727,10 @@ namespace build2 // Space-separated list of escaped library flags. // - auto lflags = [&pc, a] () -> string + auto lflags = [&pc, la] () -> string { string r; - for (const auto& o: pc.libs (a)) + for (const auto& o: pc.libs (la)) { if (!r.empty ()) r += ' '; @@ -831,7 +832,7 @@ namespace build2 prerequisite_key pk { nullopt, {&lib::static_type, &out, &out, &name, nullopt}, &s}; - if (const target* lt = search_library (act, top_sysd, usrd, pk)) + if (const target* lt = search_library (a, top_sysd, usrd, pk)) { // We used to pick a member but that doesn't seem right since the // same target could be used with different link orders. @@ -1112,8 +1113,8 @@ namespace build2 #endif - void link:: - pkgconfig_save (action act, const file& l, bool la) const + void link_rule:: + pkgconfig_save (action a, const file& l, bool la) const { tracer trace (x, "pkgconfig_save"); @@ -1258,7 +1259,7 @@ namespace build2 os << " -L" << escape (ld.string ()); // Now process ourselves as if we were being linked to something (so - // pretty similar to link::append_libraries()). + // pretty similar to link_rule::append_libraries()). // bool priv (false); auto imp = [&priv] (const file&, bool la) {return priv && la;}; @@ -1307,7 +1308,7 @@ namespace build2 // linfo li {otype::e, la ? lorder::a_s : lorder::s_a}; - process_libraries (act, bs, li, sys_lib_dirs, + process_libraries (a, bs, li, sys_lib_dirs, l, la, 0, // Link flags. imp, lib, opt, true); os << endl; @@ -1317,7 +1318,7 @@ namespace build2 os << "Libs.private:"; priv = true; - process_libraries (act, bs, li, sys_lib_dirs, + process_libraries (a, bs, li, sys_lib_dirs, l, la, 0, // Link flags. imp, lib, opt, false); os << endl; @@ -1339,7 +1340,7 @@ namespace build2 }; vector modules; - for (const target* pt: l.prerequisite_targets) + for (const target* pt: l.prerequisite_targets[a]) { // @@ UTL: we need to (recursively) see through libux{} (and // also in search_modules()). @@ -1354,7 +1355,7 @@ namespace build2 // the first mxx{} target that we see. // const target* mt (nullptr); - for (const target* t: pt->prerequisite_targets) + for (const target* t: pt->prerequisite_targets[a]) { if ((mt = t->is_a (*x_mod))) break; diff --git a/build2/cc/windows-manifest.cxx b/build2/cc/windows-manifest.cxx index 4393fbf..ae33f66 100644 --- a/build2/cc/windows-manifest.cxx +++ b/build2/cc/windows-manifest.cxx @@ -9,7 +9,7 @@ #include #include -#include +#include using namespace std; using namespace butl; @@ -39,10 +39,10 @@ namespace build2 // file corresponding to the exe{} target. Return the manifest file path // as well as whether it was changed. // - pair link:: + pair link_rule:: windows_manifest (const file& t, bool rpath_assembly) const { - tracer trace (x, "link::windows_manifest"); + tracer trace (x, "link_rule::windows_manifest"); const scope& rs (t.root_scope ()); diff --git a/build2/cc/windows-rpath.cxx b/build2/cc/windows-rpath.cxx index b28ce42..8854542 100644 --- a/build2/cc/windows-rpath.cxx +++ b/build2/cc/windows-rpath.cxx @@ -13,7 +13,7 @@ #include -#include +#include using namespace std; using namespace butl; @@ -46,10 +46,10 @@ namespace build2 // Return the greatest (newest) timestamp of all the DLLs that we will be // adding to the assembly or timestamp_nonexistent if there aren't any. // - timestamp link:: + timestamp link_rule:: windows_rpath_timestamp (const file& t, const scope& bs, - action act, + action a, linfo li) const { timestamp r (timestamp_nonexistent); @@ -103,19 +103,19 @@ namespace build2 r = t; }; - for (auto pt: t.prerequisite_targets) + for (const prerequisite_target& pt: t.prerequisite_targets[a]) { if (pt == nullptr) continue; - bool a; + bool la; const file* f; - if ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || // See through. + if ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || // See through. ( f = pt->is_a ())) - process_libraries (act, bs, li, sys_lib_dirs, - *f, a, pt.data, + process_libraries (a, bs, li, sys_lib_dirs, + *f, la, pt.data, imp, lib, nullptr, true); } @@ -125,10 +125,10 @@ namespace build2 // Like *_timestamp() but actually collect the DLLs (and weed out the // duplicates). // - auto link:: + auto link_rule:: windows_rpath_dlls (const file& t, const scope& bs, - action act, + action a, linfo li) const -> windows_dlls { windows_dlls r; @@ -193,19 +193,19 @@ namespace build2 } }; - for (auto pt: t.prerequisite_targets) + for (const prerequisite_target& pt: t.prerequisite_targets[a]) { if (pt == nullptr) continue; - bool a; + bool la; const file* f; - if ((a = (f = pt->is_a ())) || - (a = (f = pt->is_a ())) || // See through. - ( f = pt->is_a ())) - process_libraries (act, bs, li, sys_lib_dirs, - *f, a, pt.data, + if ((la = (f = pt->is_a ())) || + (la = (f = pt->is_a ())) || // See through. + ( f = pt->is_a ())) + process_libraries (a, bs, li, sys_lib_dirs, + *f, la, pt.data, imp, lib, nullptr, true); } @@ -223,10 +223,10 @@ namespace build2 // unnecessary work by comparing the DLLs timestamp against the assembly // manifest file. // - void link:: + void link_rule:: windows_rpath_assembly (const file& t, const scope& bs, - action act, + action a, linfo li, const string& tcpu, timestamp ts, @@ -264,7 +264,7 @@ namespace build2 windows_dlls dlls; if (!empty) - dlls = windows_rpath_dlls (t, bs, act, li); + dlls = windows_rpath_dlls (t, bs, a, li); // Clean the assembly directory and make sure it exists. Maybe it would // have been faster to overwrite the existing manifest rather than -- cgit v1.1