From f6319b258bb478e19d4a17852a8406e6b1119b87 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Sat, 30 Jan 2021 11:06:22 +0200 Subject: Rework include translation support See the config.cxx.translate_include variable documentation in cxx/init.cxx for details. --- doc/manual.cli | 4 +- libbuild2/c/init.cxx | 10 ++- libbuild2/cc/common.hxx | 10 +-- libbuild2/cc/compile-rule.cxx | 193 +++++++++++++++++++++++------------------- libbuild2/cc/guess.cxx | 175 ++++++++++++++++++++++++++++++++++++++ libbuild2/cc/guess.hxx | 10 +++ libbuild2/cc/init.cxx | 8 ++ libbuild2/cc/link-rule.cxx | 2 +- libbuild2/cc/module.cxx | 136 ++++++++++++++++------------- libbuild2/cc/module.hxx | 5 +- libbuild2/cc/pkgconfig.cxx | 174 ++++++++++++++++++++++++++++++------- libbuild2/cc/types.cxx | 189 +++++++++++++++++++++++++++++++++++++++++ libbuild2/cc/types.hxx | 77 +++++++++++++++++ libbuild2/cc/utility.cxx | 53 ++++++++++++ libbuild2/cc/utility.hxx | 26 ++++++ libbuild2/cxx/init.cxx | 44 ++++++++-- libbuild2/dist/operation.cxx | 2 - libbuild2/parser.cxx | 1 - libbuild2/script/run.cxx | 1 - libbuild2/utility.hxx | 5 ++ 20 files changed, 925 insertions(+), 200 deletions(-) create mode 100644 libbuild2/cc/types.cxx diff --git a/doc/manual.cli b/doc/manual.cli index d7cf92f..e09aef8 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -6552,8 +6552,8 @@ config.cxx.aoptions config.cxx.libs cxx.libs -config.cxx.translatable_headers - cxx.translatable_headers +config.cxx.translate_include + cxx.translate_include \ diff --git a/libbuild2/c/init.cxx b/libbuild2/c/init.cxx index 227afb2..afd8f88 100644 --- a/libbuild2/c/init.cxx +++ b/libbuild2/c/init.cxx @@ -175,7 +175,7 @@ namespace build2 vp.insert ("config.c.loptions"), vp.insert ("config.c.aoptions"), vp.insert ("config.c.libs"), - nullptr /* config.c.translatable_headers */, + nullptr /* config.c.translate_include */, vp.insert ("c.path"), vp.insert ("c.mode"), @@ -192,7 +192,7 @@ namespace build2 vp.insert ("c.aoptions"), vp.insert ("c.libs"), - nullptr /* c.translatable_headers */, + nullptr /* c.translate_include */, vp["cc.poptions"], vp["cc.coptions"], @@ -220,6 +220,7 @@ namespace build2 vp["cc.type"], vp["cc.system"], vp["cc.module_name"], + vp["cc.importable"], vp["cc.reprocess"], vp.insert ("c.preprocessed"), // See cxx.preprocessed. @@ -259,7 +260,8 @@ namespace build2 // Alias some cc. variables as c. // - vp.insert_alias (d.c_runtime, "c.runtime"); + vp.insert_alias (d.c_runtime, "c.runtime"); + vp.insert_alias (d.c_importable, "c.importable"); auto& m (extra.set_module (new config_module (move (d)))); m.guess (rs, loc, extra.hints); @@ -363,7 +365,7 @@ namespace build2 }; auto& m (extra.set_module (new module (move (d)))); - m.init (rs, loc, extra.hints); + m.init (rs, loc, extra.hints, *cm.x_info); return true; } diff --git a/libbuild2/cc/common.hxx b/libbuild2/cc/common.hxx index f072591..856c0ce 100644 --- a/libbuild2/cc/common.hxx +++ b/libbuild2/cc/common.hxx @@ -64,7 +64,7 @@ namespace build2 const variable& config_x_loptions; const variable& config_x_aoptions; const variable& config_x_libs; - const variable* config_x_translatable_headers; + const variable* config_x_translate_include; const variable& x_path; // Compiler process path. const variable& x_mode; // Compiler mode options. @@ -79,7 +79,7 @@ namespace build2 const variable& x_loptions; const variable& x_aoptions; const variable& x_libs; - const variable* x_translatable_headers; + const variable* x_translate_include; const variable& c_poptions; // cc.* const variable& c_coptions; @@ -107,6 +107,7 @@ namespace build2 const variable& c_type; // cc.type const variable& c_system; // cc.system const variable& c_module_name; // cc.module_name + const variable& c_importable; // cc.importable const variable& c_reprocess; // cc.reprocess const variable& x_preprocessed; // x.preprocessed @@ -172,8 +173,7 @@ namespace build2 bool modules; // x.features.modules bool symexport; // x.features.symexport - const strings* xlate_hdr; // x.translatable_headers (NULL if - // unused/empty). + build2::cc::importable_headers* importable_headers; // The order of sys_*_dirs is the mode entries first, followed by the // compiler built-in entries, and finished off with any extra entries @@ -252,7 +252,7 @@ namespace build2 ctgt (tgt), tsys (ctgt.system), tclass (ctgt.class_), modules (fm), symexport (fs), - xlate_hdr (nullptr), + importable_headers (nullptr), sys_lib_dirs (sld), sys_inc_dirs (sid), sys_mod_dirs (smd), sys_lib_dirs_mode (slm), sys_inc_dirs_mode (sim), sys_mod_dirs_mode (smm), diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx index b96c39d..531f2a4 100644 --- a/libbuild2/cc/compile-rule.cxx +++ b/libbuild2/cc/compile-rule.cxx @@ -943,9 +943,30 @@ namespace build2 if (ut == unit_type::module_intf) // Note: still unrefined. cs.append (&md.symexport, sizeof (md.symexport)); - if (xlate_hdr != nullptr) - append_options (cs, *xlate_hdr); - + // If we track translate_include then we should probably also track + // the cc.importable flag for each header we include, which would be + // quite heavy-handed indeed. Or maybe we shouldn't bother with this + // at all: after all include translation is an optimization so why + // rebuild an otherwise up-to-date target? + // +#if 0 + if (modules) + { + // While there is also the companion importable_headers map, it's + // unlikely to change in a way that affects us without changes to + // other things that we track (e.g., compiler version, etc). + // + if (const auto* v = cast_null ( + t[x_translate_include])) + { + for (const auto& p: *v) + { + cs.append (p.first); + cs.append (!p.second || *p.second); + } + } + } +#endif if (md.pp != preprocessed::all) { append_options (cs, t, x_poptions); @@ -1813,6 +1834,10 @@ namespace build2 size_t header_units = 0; // Number of header units imported. module_imports& imports; // Unused (potentially duplicate suppression). + // Include translation (looked up lazily). + // + optional translatable_headers; + small_vector batch; // Reuse buffers. module_mapper_state (size_t s, module_imports& i) @@ -2094,21 +2119,85 @@ namespace build2 // Now handle INCLUDE and IMPORT differences. // - const string& hp (ht->path ().string ()); + const path& hp (ht->path ()); + const string& hs (hp.string ()); // Reduce include translation to the import case. // - if (!imp && xlate_hdr != nullptr) + if (!imp) { - auto i (lower_bound ( - xlate_hdr->begin (), xlate_hdr->end (), - hp, - [] (const string& x, const string& y) - { - return path_traits::compare (x, y) < 0; - })); + if (!st.translatable_headers) + st.translatable_headers = + cast_null (t[x_translate_include]); + + if (*st.translatable_headers != nullptr) + { + auto& ths (**st.translatable_headers); + + // First look for the header path in the translatable headers + // itself. + // + auto i (ths.find (hs)), ie (ths.end ()); - imp = (i != xlate_hdr->end () && *i == hp); + // Next look it up in the importable headers and then look up + // the associated groups in the translatable headers. + // + if (i == ie) + { + slock l (importable_headers->mutex); + auto& ihs (importable_headers->header_map); + + auto j (ihs.find (hp)), je (ihs.end ()); + + if (j != je) + { + // The groups are ordered from the most to least + // specific. + // + for (const string& g: j->second) + if ((i = ths.find (g)) != ie) + break; + } + + // Finally look for the `all` groups. + // + if (i == ie) + { + i = ths.find (header_group_all_importable); + + if (i != ie) + { + // See if this header is marked as importable. + // + if (lookup l = (*ht)[c_importable]) + { + if (!cast (l)) + i = ie; + } + else if (j != je) + { + // See if this is one of ad hoc *-importable groups + // (currently only std-importable). + // + const auto& gs (j->second); + if (find (gs.begin (), + gs.end (), + header_group_std_importable) == gs.end ()) + i = ie; + } + else + i = ie; + } + + if (i == ie) + i = ths.find (header_group_all); + } + } + + // Translate if we found an entry and it's not false. + // + imp = (i != ie && (!i->second || *i->second)); + } } if (imp) @@ -2151,7 +2240,7 @@ namespace build2 } catch (const failed&) { - r = "ERROR 'unable to update header unit "; r += hp; r += '\''; + r = "ERROR 'unable to update header unit "; r += hs; r += '\''; continue; } } @@ -2160,7 +2249,7 @@ namespace build2 if (skip) st.skip--; else - dd.expect (hp); + dd.expect (hs); // Confusingly, TRUE means include textually and FALSE means we // don't know. @@ -2925,82 +3014,12 @@ namespace build2 } else { - // We used to just normalize the path but that could result in an - // invalid path (e.g., for some system/compiler headers on CentOS 7 - // with Clang 3.4) because of the symlinks (if a directory component - // is a symlink, then any following `..` are resolved relative to the - // target; see path::normalize() for background). - // - // Initially, to fix this, we realized (i.e., realpath(3)) it instead. - // But that turned out also not to be quite right since now we have - // all the symlinks resolved: conceptually it feels correct to keep - // the original header names since that's how the user chose to - // arrange things and practically this is how the compilers see/report - // them (e.g., the GCC module mapper). - // - // So now we have a pretty elaborate scheme where we try to use the - // normalized path if possible and fallback to realized. Normalized - // paths will work for situations where `..` does not cross symlink - // boundaries, which is the sane case. And for the insane case we only - // really care about out-of-project files (i.e., system/compiler - // headers). In other words, if you have the insane case inside your - // project, then you are on your own. - // - // All of this is unless the path comes from the depdb, in which case + // Normalize the path unless it comes from the depdb, in which case // we've already done that (normally). This is also where we handle // src-out remap (again, not needed if cached). // if (!cache || norm) - { - // Interestingly, on most paltforms and with most compilers (Clang - // on Linux being a notable exception) most system/compiler headers - // are already normalized. - // - path_abnormality a (f.abnormalities ()); - if (a != path_abnormality::none) - { - // While we can reasonably expect this path to exit, things do go - // south from time to time (like compiling under wine with file - // wlantypes.h included as WlanTypes.h). - // - try - { - // If we have any parent components, then we have to verify the - // normalized path matches realized. - // - path r; - if ((a & path_abnormality::parent) == path_abnormality::parent) - { - r = f; - r.realize (); - } - - try - { - f.normalize (); - - // Note that we might still need to resolve symlinks in the - // normalized path. - // - if (!r.empty () && f != r && path (f).realize () != r) - f = move (r); - } - catch (const invalid_path&) - { - assert (!r.empty ()); // Shouldn't have failed if no `..`. - f = move (r); // Fallback to realize. - } - } - catch (const invalid_path&) - { - fail << "invalid header path '" << f.string () << "'"; - } - catch (const system_error& e) - { - fail << "invalid header path '" << f.string () << "': " << e; - } - } - } + normalize_header (f); if (!cache) { diff --git a/libbuild2/cc/guess.cxx b/libbuild2/cc/guess.cxx index 9afa29e..d068c20 100644 --- a/libbuild2/cc/guess.cxx +++ b/libbuild2/cc/guess.cxx @@ -3218,5 +3218,180 @@ namespace build2 return r; } + + // Table 23 [tab:headers.cpp]. + // + // In the future we will probably have to maintain per-standard additions. + // + static const char* std_importable[] = { + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + }; + + // Table 24 ([tab:headers.cpp.c]) + // + static const char* std_non_importable[] = { + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + }; + + void + guess_std_importable_headers (const compiler_info& ci, + const dir_paths& sys_inc_dirs, + importable_headers& hs) + { + hs.group_map.emplace (header_group_std, 0); + hs.group_map.emplace (header_group_std_importable, 0); + + // For better performance we make compiler-specific assumptions. + // + // For example, we can assume that all these headers are found in the + // same header search directory. This is at least the case for GCC's + // libstdc++. + // + // Note also that some headers could be missing. For example, + // is currently not provided by GCC. Though entering missing headers + // should be harmless. + // + pair* p; + auto add_groups = [&p] (bool imp) + { + if (imp) + p->second.push_back (header_group_std_importable); // More specific. + + p->second.push_back (header_group_std); + }; + + if (ci.id.type != compiler_type::gcc) + { + for (const char* f: std_importable) + if ((p = hs.insert_angle (sys_inc_dirs, f)) != nullptr) + add_groups (true); + + for (const char* f: std_non_importable) + if ((p = hs.insert_angle (sys_inc_dirs, f)) != nullptr) + add_groups (false); + } + else + { + p = hs.insert_angle (sys_inc_dirs, std_importable[0]); + assert (p != nullptr); + + add_groups (true); + + dir_path d (p->first.directory ()); + + auto add_header = [&hs, &d, &p, add_groups] (const char* f, bool imp) + { + path fp (d); + fp.combine (f + 1, strlen (f) - 2, '\0'); // Assuming simple. + + p = &hs.insert_angle (move (fp), f); + add_groups (imp); + }; + + for (size_t i (1); + i != sizeof (std_importable) / sizeof (std_importable[0]); + ++i) + add_header (std_importable[i], true); + + for (const char* f: std_non_importable) + add_header (f, false); + } + } } } diff --git a/libbuild2/cc/guess.hxx b/libbuild2/cc/guess.hxx index 868e925..a141fa3 100644 --- a/libbuild2/cc/guess.hxx +++ b/libbuild2/cc/guess.hxx @@ -265,6 +265,16 @@ namespace build2 const string& cid, const string& pattern, const strings& mode); + + // Insert importable/non-importable C++ standard library headers + // ([headers]/4). + // + // Note that the importable_headers instance should be unique-locked. + // + void + guess_std_importable_headers (const compiler_info&, + const dir_paths& sys_inc_dirs, + importable_headers&); } } diff --git a/libbuild2/cc/init.cxx b/libbuild2/cc/init.cxx index bb50d07..e8b0e6f 100644 --- a/libbuild2/cc/init.cxx +++ b/libbuild2/cc/init.cxx @@ -146,6 +146,14 @@ namespace build2 // vp.insert ("cc.module_name", v_t); + // Importable header marker (normally set via the x.importable alias). + // + // Note that while at first it might seem like a good idea to allow + // setting it on a scope, that will cause translation of inline/template + // includes which is something we definitely don't want. + // + vp.insert ("cc.importable", v_t); + // Ability to disable using preprocessed output for compilation. // vp.insert ("config.cc.reprocess"); diff --git a/libbuild2/cc/link-rule.cxx b/libbuild2/cc/link-rule.cxx index f500389..9b75e8d 100644 --- a/libbuild2/cc/link-rule.cxx +++ b/libbuild2/cc/link-rule.cxx @@ -858,7 +858,7 @@ namespace build2 // else { - if (!p.is_a () && !p.is_a ()) + if (!p.is_a () && !p.is_a () && !x_header (p, true)) { // @@ Temporary hack until we get the default outer operation // for update. This allows operations like test and install to diff --git a/libbuild2/cc/module.cxx b/libbuild2/cc/module.cxx index 8241a01..1709c23 100644 --- a/libbuild2/cc/module.cxx +++ b/libbuild2/cc/module.cxx @@ -3,6 +3,7 @@ #include +#include #include // left, setw() #include @@ -426,20 +427,21 @@ namespace build2 translate_std (xi, tt, rs, mode, v); } - // config.x.translatable_header + // config.x.translate_include // // It's still fuzzy whether specifying (or maybe tweaking) this list in // the configuration will be a common thing to do so for now we use - // omitted. It's also probably too early to think whether we should have - // the cc.* version and what the semantics should be. + // omitted. // - if (x_translatable_headers != nullptr) + if (x_translate_include != nullptr) { - lookup l (lookup_config (rs, *config_x_translatable_headers)); - - // @@ MODHDR: if(modules) ? - // - rs.assign (x_translatable_headers) += cast_null (l); + if (lookup l = lookup_config (rs, *config_x_translate_include)) + { + // @@ MODHDR: if(modules) ? Yes. + // + rs.assign (x_translate_include).prepend ( + cast (l)); + } } // Extract system header/library search paths from the compiler and @@ -718,8 +720,19 @@ namespace build2 } } + // Global cache of ad hoc importable headers. + // + // The key is a hash of the system header search directories + // (sys_inc_dirs) where we search for the headers. + // + static map importable_headers_cache; + static mutex importable_headers_mutex; + void module:: - init (scope& rs, const location& loc, const variable_map&) + init (scope& rs, + const location& loc, + const variable_map&, + const compiler_info& xi) { tracer trace (x, "init"); @@ -740,71 +753,76 @@ namespace build2 // load_module (rs, rs, "cc.core", loc); - // Process, sort, and cache (in this->xlate_hdr) translatable headers. - // Keep the cache NULL if unused or empty. + // Search include translation headers and groups. // - // @@ MODHDR TODO: support exclusions entries (e.g., -)? - // - if (modules && x_translatable_headers != nullptr) + if (modules) { - strings* ih (cast_null (rs.assign (x_translatable_headers))); + { + sha256 k; + for (const dir_path& d: sys_inc_dirs) + k.append (d.string ()); + + mlock l (importable_headers_mutex); + importable_headers = &importable_headers_cache[k.string ()]; + } + + auto& hs (*importable_headers); - if (ih != nullptr && !ih->empty ()) + ulock ul (hs.mutex); + + if (hs.group_map.find (header_group_std) == hs.group_map.end ()) + guess_std_importable_headers (xi, sys_inc_dirs, hs); + + // Process x.translate_include. + // + const variable& var (*x_translate_include); + if (auto* v = cast_null (rs[var])) { - // Translate <>-style header names to absolute paths using the - // compiler's include search paths. Otherwise complete and normalize - // since when searching in this list we always use the absolute and - // normalized header target path. - // - for (string& h: *ih) + for (const auto& p: *v) { - if (h.empty ()) - continue; + const string& k (p.first); - path f; - if (h.front () == '<' && h.back () == '>') + if (k.front () == '<' && k.back () == '>') { - h.pop_back (); - h.erase (0, 1); + if (path_pattern (k)) + { + size_t n (hs.insert_angle_pattern (sys_inc_dirs, k)); - for (const dir_path& d: sys_inc_dirs) + l5 ([&]{trace << "pattern " << k << " searched to " << n + << " headers";}); + } + else { - if (file_exists ((f = d, f /= h), - true /* follow_symlinks */, - true /* ignore_errors */)) - goto found; + // What should we do if not found? While we can fail, this + // could be too drastic if, for example, the header is + // "optional" and may or may not be present/used. So for now + // let's ignore (we could have also removed it from the map as + // an indication). + // + const auto* r (hs.insert_angle (sys_inc_dirs, k)); + + l5 ([&]{trace << "header " << k << " searched to " + << (r ? r->first.string ().c_str () : "");}); } - - // What should we do if not found? While we can fail, this could - // be too drastic if, for example, the header is "optional" and - // may or may not be present/used. So for now let's restore the - // original form to aid debugging (it can't possibly match any - // absolute path). + } + else if (path_traits::find_separator (k) == string::npos) + { + // Group name. // - h.insert (0, 1, '<'); - h.push_back ('>'); - continue; - - found: - ; // Fall through. + if (k != header_group_all_importable && + k != header_group_std_importable && + k != header_group_all && + k != header_group_std) + fail (loc) << "unknown header group '" << k << "' in " << var; } else { - f = path (move (h)); - - if (f.relative ()) - f.complete (); + // Absolute and normalized header path. + // + if (!path_traits::absolute (k)) + fail (loc) << "relative header path '" << k << "' in " << var; } - - // @@ MODHDR: should we use the more elaborate but robust - // normalize/realize scheme so the we get the same - // path? Feels right. - f.normalize (); - h = move (f).string (); } - - sort (ih->begin (), ih->end ()); - xlate_hdr = ih; } } diff --git a/libbuild2/cc/module.hxx b/libbuild2/cc/module.hxx index 85b7158..ac170da 100644 --- a/libbuild2/cc/module.hxx +++ b/libbuild2/cc/module.hxx @@ -103,7 +103,10 @@ namespace build2 libux_install_rule (move (d), *this) {} void - init (scope&, const location&, const variable_map&); + init (scope&, + const location&, + const variable_map&, + const compiler_info&); }; } } diff --git a/libbuild2/cc/pkgconfig.cxx b/libbuild2/cc/pkgconfig.cxx index 6a1b3c8..be7674a 100644 --- a/libbuild2/cc/pkgconfig.cxx +++ b/libbuild2/cc/pkgconfig.cxx @@ -1142,15 +1142,17 @@ namespace build2 return r; }; - // Parse modules and add them to the prerequisites. + // Parse modules, enter them as targets, and add them to the + // prerequisites. // - auto parse_modules = [&trace, &next, &s, this] - (const pkgconf& pc, prerequisites& ps) + auto parse_modules = [&trace, this, + &next, &s, <] (const pkgconf& pc, + prerequisites& ps) { - string mstr (pc.variable ("cxx_modules")); + string val (pc.variable ("cxx_modules")); string m; - for (size_t b (0), e (0); !(m = next (mstr, b, e)).empty (); ) + for (size_t b (0), e (0); !(m = next (val, b, e)).empty (); ) { // The format is = with `..` used as a partition // separator (see pkgconfig_save() for details). @@ -1159,7 +1161,7 @@ namespace build2 if (p == string::npos || p == 0 || // Empty name. p == m.size () - 1) // Empty path. - fail << "invalid module information in '" << mstr << "'" << + fail << "invalid module information in '" << val << "'" << info << "while parsing pkg-config --variable=cxx_modules " << pc.path; @@ -1193,12 +1195,12 @@ namespace build2 // If the target already exists, then setting its variables is not // MT-safe. So currently we only do it if we have the lock (and thus - // nobody can see this target yet) assuming that this has already + // nobody can see this target yet) verifying that this has already // been done otherwise. // // @@ This is not quite correct, though: this target could already // exist but for a "different purpose" (e.g., it could be used as - // a header). + // a header). Well, maybe it shouldn't. // // @@ Could setting it in the rule-specific vars help? (But we // are not matching a rule for it.) Note that we are setting @@ -1211,7 +1213,7 @@ namespace build2 // Set module properties. Note that if unspecified we should still // set them to their default values since the hosting project may - // have them set to incompatible value. + // have them set to incompatible values. // { value& v (mt.vars.assign (x_preprocessed)); // NULL @@ -1224,11 +1226,68 @@ namespace build2 tl.second.unlock (); } + else + { + if (!mt.vars[c_module_name]) + fail << "unexpected metadata for module target " << mt << + info << "module is expected to have assigned name" << + info << "make sure this module is used via " << lt + << " prerequisite"; + } ps.push_back (prerequisite (mt)); } }; + // Parse importable headers and enter them as targets. + // + auto parse_headers = [&trace, this, + &next, &s, <] (const pkgconf& pc, + const target_type& tt, + const char* lang) + { + string var (string (lang) + "_importable_headers"); + string val (pc.variable (var)); + + string h; + for (size_t b (0), e (0); !(h = next (val, b, e)).empty (); ) + { + path hp (move (h)); + path hf (hp.leaf ()); + + auto tl ( + s.ctx.targets.insert_locked ( + tt, + hp.directory (), + dir_path (), + hf.base ().string (), + hf.extension (), + target_decl::implied, + trace)); + + target& ht (tl.first); + + // If the target already exists, then setting its variables is not + // MT-safe. So currently we only do it if we have the lock (and thus + // nobody can see this target yet) verifying that this has already + // been done otherwise. + // + if (tl.second.owns_lock ()) + { + ht.vars.assign (c_importable) = true; + tl.second.unlock (); + } + else + { + if (!cast_false (ht.vars[c_importable])) + fail << "unexpected metadata for existing header target " << ht << + info << "header is expected to be marked importable" << + info << "make sure this header is used via " << lt + << " prerequisite"; + } + } + }; + // For now we only populate prerequisites for lib{}. To do it for // liba{} would require weeding out duplicates that are already in // lib{}. @@ -1291,13 +1350,21 @@ namespace build2 parse_cflags (*st, spc, false); // For now we assume static and shared variants export the same set of - // modules. While technically possible, having a different set will most - // likely lead to all sorts of complications (at least for installed - // libraries) and life is short. + // modules/importable headers. While technically possible, having + // different sets will most likely lead to all sorts of complications + // (at least for installed libraries) and life is short. // if (modules) + { parse_modules (ipc, prs); + // We treat headers outside of any project as C headers (see + // enter_header() for details). + // + parse_headers (ipc, h::static_type /* **x_hdr */, x); + parse_headers (ipc, h::static_type, "c"); + } + assert (!lt.has_prerequisites ()); if (!prs.empty ()) lt.prerequisites (move (prs)); @@ -1614,32 +1681,46 @@ namespace build2 } } - // If we have modules, list them in the modules variable. We also save - // some extra info about them (yes, the rabbit hole runs deep). This - // code is pretty similar to compiler::search_modules(). + // If we have modules and/or importable headers, list them in the + // respective variables. We also save some extra info about modules + // (yes, the rabbit hole runs deep). This code is pretty similar to + // compiler::search_modules(). + // + // Note that we want to convey the importable headers information even + // if modules are not enabled. // - if (modules) { struct module { string name; path file; - string pp; + string preprocessed; bool symexport; }; - vector modules; + vector mods; + + // If we were to ever support another C-based language (e.g., + // Objective-C) and libraries that can use a mix of languages (e.g., + // C++ and Objective-C), then we would need to somehow reverse- + // lookup header target type to language. Let's hope we don't. + // + vector x_hdrs; + vector c_hdrs; // Note that the prerequisite targets are in the member, not the - // group (for now we don't support different sets of modules for - // static/shared library; see load above for details). + // group (for now we don't support different sets of modules/headers + // for static/shared library; see load above for details). // for (const target* pt: l.prerequisite_targets[a]) { + if (pt == nullptr) + continue; + // @@ UTL: we need to (recursively) see through libu*{} (and // also in search_modules()). // - if (pt != nullptr && pt->is_a ()) + if (modules && pt->is_a ()) { // What we have is a binary module interface. What we need is // a module interface source it was built from. We assume it's @@ -1666,7 +1747,7 @@ namespace build2 if (const string* v = cast_null ((*mt)[x_preprocessed])) pp = *v; - modules.push_back ( + mods.push_back ( module { cast (pt->state[a].vars[c_module_name]), move (p), @@ -1674,9 +1755,21 @@ namespace build2 symexport }); } + else if (pt->is_a (**x_hdr) || pt->is_a ()) + { + if (cast_false ((*pt)[c_importable])) + { + path p (install::resolve_file (pt->as ())); + + if (p.empty ()) // Not installed. + continue; + + (pt->is_a () ? c_hdrs : x_hdrs).push_back (move (p)); + } + } } - if (!modules.empty ()) + if (size_t n = mods.size ()) { os << endl << "cxx_modules ="; @@ -1688,7 +1781,7 @@ namespace build2 // for example hello.print..impl. While in the variable values we // can use `:`, for consistency we use `..` there as well. // - for (module& m: modules) + for (module& m: mods) { size_t p (m.name.find (':')); if (p != string::npos) @@ -1696,7 +1789,8 @@ namespace build2 // Module names shouldn't require escaping. // - os << ' ' << m.name << '=' << escape (m.file.string ()); + os << (n != 1 ? " \\\n" : " ") + << m.name << '=' << escape (m.file.string ()); } os << endl; @@ -1705,16 +1799,38 @@ namespace build2 // // _module_. = // - for (const module& m: modules) + for (const module& m: mods) { - if (!m.pp.empty ()) - os << "cxx_module_preprocessed." << m.name << " = " << m.pp - << endl; + if (!m.preprocessed.empty ()) + os << "cxx_module_preprocessed." << m.name << " = " + << m.preprocessed << endl; if (m.symexport) os << "cxx_module_symexport." << m.name << " = true" << endl; } } + + if (size_t n = c_hdrs.size ()) + { + os << endl + << "c_importable_headers ="; + + for (const path& h: c_hdrs) + os << (n != 1 ? " \\\n" : " ") << escape (h.string ()); + + os << endl; + } + + if (size_t n = x_hdrs.size ()) + { + os << endl + << x << "_importable_headers ="; + + for (const path& h: x_hdrs) + os << (n != 1 ? " \\\n" : " ") << escape (h.string ()); + + os << endl; + } } os.close (); diff --git a/libbuild2/cc/types.cxx b/libbuild2/cc/types.cxx new file mode 100644 index 0000000..666b048 --- /dev/null +++ b/libbuild2/cc/types.cxx @@ -0,0 +1,189 @@ +// file : libbuild2/cc/types.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include + +using namespace std; + +namespace build2 +{ + namespace cc + { + const string header_group_all ("all"); + const string header_group_all_importable ("all-importable"); + const string header_group_std ("std"); + const string header_group_std_importable ("std-importable"); + + // Find the position where the group should be inserted unless the group + // is already there. + // + using groups = importable_headers::groups; + + static inline optional + find_angle (const groups& gs, const string& g) + { + for (auto i (gs.begin ()); i != gs.end (); ++i) + { + // After last angle-bracket file. + // + if (i->front () != '<' || i->back () != '>' || path_pattern (*i)) + return i; + + if (*i == g) + return nullopt; + } + + return gs.begin (); + } + + static inline optional + find_angle_pattern (const groups& gs, const string& g) + { + for (auto i (gs.begin ()); i != gs.end (); ++i) + { + // After last angle-bracket file pattern. + // + if (i->front () != '<' || i->back () != '>') + return i; + + if (*i == g) + return nullopt; + } + + return gs.begin (); + } + + auto importable_headers:: + insert_angle (const dir_paths& sys_inc_dirs, + const string& s) -> pair* + { + assert (s.front () == '<' && s.back () == '>'); + + // First see if it has already been inserted. + // + auto i (group_map.find (s)); + if (i == group_map.end ()) + { + path f (s, 1, s.size () - 2); + + path p; + for (const dir_path& d: sys_inc_dirs) + { + if (file_exists ((p = d, p /= f), + true /* follow_symlinks */, + true /* ignore_errors */)) + goto found; + } + + return nullptr; + + found: + normalize_header (p); + + // Note that it's possible this header has already been entered as + // part of a different group. + // + auto j (header_map.emplace (move (p), groups {}).first); + + if (auto p = find_angle (j->second, s)) + j->second.insert (*p, s); + + i = group_map.emplace (s, reinterpret_cast (&*j)).first; + } + + return reinterpret_cast*> (i->second); + } + + auto importable_headers:: + insert_angle (path p, const string& s) -> pair& + { + assert (s.front () == '<' && s.back () == '>'); + + // First see if it has already been inserted. + // + auto i (group_map.find (s)); + if (i == group_map.end ()) + { + // Note that it's possible this header has already been entered as + // part of a different group. + // + auto j (header_map.emplace (move (p), groups {}).first); + + if (auto p = find_angle (j->second, s)) + j->second.insert (*p, s); + + i = group_map.emplace (s, reinterpret_cast (&*j)).first; + } + + return *reinterpret_cast*> (i->second); + } + + size_t importable_headers:: + insert_angle_pattern (const dir_paths& sys_inc_dirs, const string& pat) + { + assert (pat.front () == '<' && pat.back () == '>' && path_pattern (pat)); + + // First see if it has already been inserted. + // + auto i (group_map.find (pat)); + if (i == group_map.end ()) + { + path f (pat, 1, pat.size () - 2); + + struct data + { + uintptr_t n; + const string& pat; + const dir_path* dir; + } d {0, pat, nullptr}; + + auto process = [&d, this] (path&& pe, const string&, bool interm) + { + if (interm) + return true; + + path p (*d.dir / pe); + normalize_header (p); + + string s (move (pe).string ()); + s.insert (0, 1, '<'); + s.push_back ('>'); + + // Note that it's possible this header has already been entered as + // part of a different group. + // + auto j (header_map.emplace (move (p), groups {}).first); + + if (auto p = find_angle (j->second, s)) + j->second.insert (*p, move (s)); + + if (auto p = find_angle_pattern (j->second, d.pat)) + j->second.insert (*p, d.pat); + + d.n++; + return true; + }; + + for (const dir_path& dir: sys_inc_dirs) + { + d.dir = &dir; + + try + { + path_search (f, process, dir); + } + catch (const system_error& e) + { + fail << "unable to scan " << dir << ": " << e; + } + } + + i = group_map.emplace (pat, d.n).first; + } + + return static_cast (i->second); + } + } +} diff --git a/libbuild2/cc/types.hxx b/libbuild2/cc/types.hxx index 20b67c2..70a6340 100644 --- a/libbuild2/cc/types.hxx +++ b/libbuild2/cc/types.hxx @@ -4,6 +4,9 @@ #ifndef LIBBUILD2_CC_TYPES_HXX #define LIBBUILD2_CC_TYPES_HXX +#include +#include + #include #include @@ -81,6 +84,80 @@ namespace build2 build2::cc::module_info module_info; }; + // Ad hoc (as opposed to marked with x.importable) importable headers. + // + // Note that these are only searched for in the system header search + // directories (sys_inc_dirs). + // + struct importable_headers + { + mutable shared_mutex mutex; + + using groups = small_vector; + + // Map of groups (e.g., std, , ) that have already + // been inserted. + // + // For angle-bracket file groups (e.g., ), the value is a + // pointer to the corresponding header_map element. For angle-bracket + // file pattern groups (e.g., ), the value is the number + // of files in the group. + // + // Note that while the case-sensitivity of header names in #include + // directives is implementation-defined, our group names are case- + // sensitive (playing loose with the case will lead to portability + // issues sooner or later so we don't bother with any more elborate + // solutions). + // + std::unordered_map group_map; + + // Map of absolute and normalized header paths to groups (e.g., std, + // , ) to which they belong. The groups are + // ordered from the most to least specific (e.g., then std). + // + std::unordered_map header_map; + + // Note that all these functions assume the instance is unique-locked. + // + + // Search for and insert an angle-bracket file, for example , + // making it belong to the angle-bracket file group itself. Return the + // pointer to the corresponding header_map element if the file has been + // found and NULL otherwise (so can be used as bool). + // + pair* + insert_angle (const dir_paths& sys_inc_dirs, const string& file); + + // As above but for a manually-searched absolute and normalized path. + // + pair& + insert_angle (path, const string& file); + + // Search for and insert an angle-bracket file pattern, for example + // , making each header belong to the angle-bracket file + // group (e.g., ) and the angle-bracket file pattern + // group itself. Return the number of files found that match the + // pattern. + // + size_t + insert_angle_pattern (const dir_paths& sys_inc_dirs, const string& pat); + }; + + // Headers and header groups whose inclusion should or should not be + // translated to the corresponding header unit imports. + // + // The key is either an absolute and normalized header path or a reference + // to an importable_headers group (e.g., , std). + // + using translatable_headers = std::map>; + + // Special translatable header groups. + // + extern const string header_group_all; + extern const string header_group_all_importable; + extern const string header_group_std; + extern const string header_group_std_importable; + // Compiler language. // enum class lang {c, cxx}; diff --git a/libbuild2/cc/utility.cxx b/libbuild2/cc/utility.cxx index 283e1b4..ffe3e03 100644 --- a/libbuild2/cc/utility.cxx +++ b/libbuild2/cc/utility.cxx @@ -17,5 +17,58 @@ namespace build2 const dir_path module_build_dir (dir_path (module_dir) /= "build"); const dir_path module_build_modules_dir ( dir_path (module_build_dir) /= "modules"); + + void + normalize_header (path& f) + { + // Interestingly, on most paltforms and with most compilers (Clang on + // Linux being a notable exception) most system/compiler headers are + // already normalized. + // + path_abnormality a (f.abnormalities ()); + if (a != path_abnormality::none) + { + // While we can reasonably expect this path to exit, things do go + // south from time to time (like compiling under wine with file + // wlantypes.h included as WlanTypes.h). + // + try + { + // If we have any parent components, then we have to verify the + // normalized path matches realized. + // + path r; + if ((a & path_abnormality::parent) == path_abnormality::parent) + { + r = f; + r.realize (); + } + + try + { + f.normalize (); + + // Note that we might still need to resolve symlinks in the + // normalized path. + // + if (!r.empty () && f != r && path (f).realize () != r) + f = move (r); + } + catch (const invalid_path&) + { + assert (!r.empty ()); // Shouldn't have failed if no `..`. + f = move (r); // Fallback to realize. + } + } + catch (const invalid_path&) + { + fail << "invalid header path '" << f.string () << "'"; + } + catch (const system_error& e) + { + fail << "invalid header path '" << f.string () << "': " << e; + } + } + } } } diff --git a/libbuild2/cc/utility.hxx b/libbuild2/cc/utility.hxx index a856fd0..42e53e3 100644 --- a/libbuild2/cc/utility.hxx +++ b/libbuild2/cc/utility.hxx @@ -48,6 +48,32 @@ namespace build2 compile_target_types compile_types (otype); + + // Normalize an absolute path to an existing header. + // + // We used to just normalize the path but that could result in an invalid + // path (e.g., for some system/compiler headers on CentOS 7 with Clang + // 3.4) because of the symlinks (if a directory component is a symlink, + // then any following `..` are resolved relative to the target; see + // path::normalize() for background). + // + // Initially, to fix this, we realized (i.e., realpath(3)) it instead. + // But that turned out also not to be quite right since now we have all + // the symlinks resolved: conceptually it feels correct to keep the + // original header names since that's how the user chose to arrange things + // and practically this is how the compilers see/report them (e.g., the + // GCC module mapper). + // + // So now we have a pretty elaborate scheme where we try to use the + // normalized path if possible and fallback to realized. Normalized paths + // will work for situations where `..` does not cross symlink boundaries, + // which is the sane case. And for the insane case we only really care + // about out-of-project files (i.e., system/compiler headers). In other + // words, if you have the insane case inside your project, then you are on + // your own. + // + void + normalize_header (path&); } } diff --git a/libbuild2/cxx/init.cxx b/libbuild2/cxx/init.cxx index 52e1ba2..876f5d8 100644 --- a/libbuild2/cxx/init.cxx +++ b/libbuild2/cxx/init.cxx @@ -431,16 +431,42 @@ namespace build2 vp.insert ("config.cxx.aoptions"), vp.insert ("config.cxx.libs"), - // List of translatable headers. Inclusions of such headers are + // Headers and header groups whose inclusion should or should not be // translated to the corresponding header unit imports. // // A header can be specified either as an absolute and normalized path - // or as a <>-style include name. The latter kind is automatically - // translated to the absolute form based on the compiler's system (as - // opposed to -I) header search paths. Note also that all entries must - // be specified before loading the cxx module. + // or as a <>-style include file or file pattern (for example, + // , ). The latter kind is automatically + // resolved to the absolute form based on the compiler's system (as + // opposed to project's) header search paths. // - &vp.insert ("config.cxx.translatable_headers"), + // Currently recognized header groups are: + // + // std-importable -- translate importable standard library headers + // std -- translate all standard library headers + // all-importable -- translate all importable headers + // all -- translate all headers + // + // Note that a header may belong to multiple groups which are looked + // up from the most to least specific, for example: , + // std-importable, std, all-importable, all. + // + // A header or group can also be excluded from being translated, for + // example: + // + // std-importable @false + // + // The config.cxx.translate_include value is prepended (merged with + // override) into cxx.translate_include while loading the cxx.config + // module. The headers and header groups in cxx.translate_include are + // resolved while loading the cxx module. For example: + // + // cxx.translate_include = @false # Can be overriden. + // using cxx.config + // cxx.translate_include =+ @false # Cannot be overriden. + // using cxx + // + &vp.insert ("config.cxx.translate_include"), vp.insert ("cxx.path"), vp.insert ("cxx.mode"), @@ -457,7 +483,7 @@ namespace build2 vp.insert ("cxx.aoptions"), vp.insert ("cxx.libs"), - &vp.insert ("cxx.translatable_headers"), + &vp.insert ("cxx.translate_include"), vp["cc.poptions"], vp["cc.coptions"], @@ -485,6 +511,7 @@ namespace build2 vp["cc.type"], vp["cc.system"], vp["cc.module_name"], + vp["cc.importable"], vp["cc.reprocess"], // Ability to signal that source is already (partially) preprocessed. @@ -539,6 +566,7 @@ namespace build2 // vp.insert_alias (d.c_runtime, "cxx.runtime"); vp.insert_alias (d.c_module_name, "cxx.module_name"); + vp.insert_alias (d.c_importable, "cxx.importable"); auto& m (extra.set_module (new config_module (move (d)))); m.guess (rs, loc, extra.hints); @@ -662,7 +690,7 @@ namespace build2 }; auto& m (extra.set_module (new module (move (d)))); - m.init (rs, loc, extra.hints); + m.init (rs, loc, extra.hints, *cm.x_info); return true; } diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx index 2fa953d..07f3b42 100644 --- a/libbuild2/dist/operation.cxx +++ b/libbuild2/dist/operation.cxx @@ -6,8 +6,6 @@ #include #include -#include - #include #include #include diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index 951759b..def4654 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -7,7 +7,6 @@ #include // cout #include // path_search -#include #include #include diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx index 58ba23d..0b49dea 100644 --- a/libbuild2/script/run.cxx +++ b/libbuild2/script/run.cxx @@ -15,7 +15,6 @@ #include #include // fdopen_mode, fddup() #include // path_search() -#include #include #include diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx index a07d928..3ffc7b2 100644 --- a/libbuild2/utility.hxx +++ b/libbuild2/utility.hxx @@ -18,6 +18,7 @@ #include // combine_hash(), reverse_iterate(), etc #include +#include #include #include @@ -94,6 +95,10 @@ namespace build2 using butl::open_file_or_stdin; using butl::open_file_or_stdout; + // + // + using butl::path_pattern; + // Diagnostics state (verbosity level, etc; see ). // // Note on naming of values (here and in the global state below) that come -- cgit v1.1