From 9dadc037cdf49b8e6d869d4226e2afeadaa0780a Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Sun, 28 Aug 2016 15:03:29 +0200 Subject: Factor library search/processing out to cc::common --- build2/cc/common.cxx | 839 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 839 insertions(+) create mode 100644 build2/cc/common.cxx (limited to 'build2/cc/common.cxx') diff --git a/build2/cc/common.cxx b/build2/cc/common.cxx new file mode 100644 index 0000000..e0db91e --- /dev/null +++ b/build2/cc/common.cxx @@ -0,0 +1,839 @@ +// file : build2/cc/common.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2016 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include // import() +#include +#include +#include +#include +#include +#include + +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace cc + { + using namespace bin; + + // Recursively process prerequisite libraries. If proc_impl returns false, + // then only process interface (*.export.libs), otherwise -- interface and + // implementation (prerequisite and from *.libs, unless overriden). + // + // Note that here we assume that an interface library is also an + // implementation (since we don't use *.export.libs in static link). We + // currently have this restriction to make sure the target in + // *.export.libs is up-to-date (which will happen automatically if it is + // listed as a prerequisite of this library). + // + // Storing a reference to library path in proc_lib is legal (it comes + // either from the target's path or from one of the *.libs variables + // neither of which should change on this run). + // + void common:: + process_libraries ( + scope& top_bs, + lorder top_lo, + const dir_paths& top_sysd, + file& l, + bool la, + const function& proc_impl, // Implementation? + const function& proc_lib, // True if system library. + const function& proc_opt, // *.export. + bool self /*= false*/) const // Call proc_lib on l? + { + // Determine if an absolute path is to a system library. Note that + // we assume both paths to be normalized. + // + auto sys = [] (const dir_paths& sysd, const string& p) -> bool + { + size_t pn (p.size ()); + + for (const dir_path& d: sysd) + { + const string& ds (d.string ()); // Can be "/", otherwise no slash. + size_t dn (ds.size ()); + + if (pn > dn && + p.compare (0, dn, ds) == 0 && + (path::traits::is_separator (ds[dn - 1]) || + path::traits::is_separator (p[dn]))) + return true; + } + + return false; + }; + + // See what type of library this is (C, C++, etc). Use it do decide + // which x.libs variable name to use. If it's unknown, then we only + // look into prerequisites. + // + const string* t (cast_null (l.vars[c_type])); + + if (self && proc_lib) + { + const string& p (l.path ().string ()); + + bool s (t != nullptr // If cc library (matched or imported). + ? cast_false (l.vars[c_system]) + : sys (top_sysd, p)); + + proc_lib (&l, p, s); + } + + bool impl (proc_impl && proc_impl (l, la)); + bool cc, same; + + auto& vp (var_pool); + lookup c_e_libs; + lookup x_e_libs; + + if (t != nullptr) + { + cc = *t == "cc"; + same = !cc && *t == x; + + // The explicit export override should be set on the liba/libs{} + // target itself. Note also that we only check for *.libs. If one + // doesn't have any libraries but needs to set, say, *.loptions, then + // *.libs should be set to NULL or empty (this is why we check for + // the result being defined). + // + if (impl) + c_e_libs = l.vars[c_export_libs]; // Override. + else if (l.group != nullptr) // lib{} group. + c_e_libs = l.group->vars[c_export_libs]; + + if (!cc) + { + const variable& var (same + ? x_export_libs + : vp[*t + ".export.libs"]); + + if (impl) + x_e_libs = l.vars[var]; // Override. + else if (l.group != nullptr) // lib{} group. + x_e_libs = l.group->vars[var]; + } + } + + scope& bs (t == nullptr || cc ? top_bs : l.base_scope ()); + optional lo; // Calculate lazily. + const dir_paths* sysd (nullptr); // Resolve lazily. + + // Find system search directories corresponding to this library, i.e., + // from its project and for its type (C, C++, etc). + // + auto find_sysd = [&top_sysd, t, cc, same, &bs, &sysd, this] () + { + // Use the search dirs corresponding to this library scope/type. + // + sysd = (t == nullptr || cc) + ? &top_sysd // Imported library, use importer's sysd. + : &cast ( + bs.root_scope ()->vars[same + ? x_sys_lib_dirs + : var_pool[*t + ".sys_lib_dirs"]]); + }; + + auto find_lo = [top_lo, t, cc, &bs, &l, &lo, this] () + { + lo = (t == nullptr || cc) ? top_lo : link_order (bs, link_type (l)); + }; + + // Only go into prerequisites (implementation) if instructed and we are + // not using explicit export. Otherwise, interface dependencies come + // from the lib{}:*.export.libs below. + // + if (impl && !c_e_libs.defined () && !x_e_libs.defined ()) + { + for (target* p: l.prerequisite_targets) + { + bool a; + file* f; + + if ((a = (f = p->is_a ()) != nullptr) + || (f = p->is_a ()) != nullptr) + { + if (sysd == nullptr) find_sysd (); + if (!lo) find_lo (); + + process_libraries (bs, *lo, *sysd, + *f, a, + proc_impl, proc_lib, proc_opt, true); + } + } + } + + // Process libraries (recursively) from *.export.libs (of type names) + // handling import, etc. + // + // If it is not a C-common library, then it probably doesn't have any of + // the *.libs and we are done. + // + if (t == nullptr) + return; + + optional usrd; // Extract lazily. + + // Determine if a "simple path" is a system library. + // + auto sys_simple = [&sysd, &sys, &find_sysd] (const string& p) -> bool + { + bool s (!path::traits::absolute (p)); + + if (!s) + { + if (sysd == nullptr) find_sysd (); + + s = sys (*sysd, p); + } + + return s; + }; + + auto proc_int = [&l, + &proc_impl, &proc_lib, &proc_opt, + &sysd, &usrd, + &find_sysd, &find_lo, &sys, &sys_simple, + &bs, &lo, this] (const lookup& lu) + { + const names* ns (cast_null (lu)); + if (ns == nullptr || ns->empty ()) + return; + + for (const name& n: *ns) + { + if (n.simple ()) + { + // This is something like -lpthread or shell32.lib so should be a + // valid path. But it can also be an absolute library path (e.g., + // something that in the future will come from our -static/-shared + // .pc files. + // + if (proc_lib) + proc_lib (nullptr, n.value, sys_simple (n.value)); + } + else + { + // This is a potentially project-qualified target. + // + if (sysd == nullptr) find_sysd (); + if (!lo) find_lo (); + + file& t (resolve_library (n, bs, *lo, *sysd, usrd)); + + if (proc_lib) + { + // This can happen if the target is mentioned in *.export.libs + // (i.e., it is an interface dependency) but not in the + // library's prerequisites (i.e., it is not an implementation + // dependency). + // + // Note that we used to just check for path being assigned but + // on Windows import-installed DLLs may legally have empty + // paths. + // + if (t.mtime (false) == timestamp_unknown) + fail << "interface dependency " << t << " is out of date" << + info << "mentioned in *.export.libs of target " << l << + info << "is it a prerequisite of " << l << "?"; + } + + // Process it recursively. + // + process_libraries (bs, *lo, *sysd, + t, t.is_a (), + proc_impl, proc_lib, proc_opt, true); + } + } + }; + + // Process libraries from *.libs (of type strings). + // + auto proc_imp = [&proc_lib, &sys_simple] (const lookup& lu) + { + const strings* ns (cast_null (lu)); + if (ns == nullptr || ns->empty ()) + return; + + for (const string& n: *ns) + { + // This is something like -lpthread or shell32.lib so should be a + // valid path. + // + proc_lib (nullptr, n, sys_simple (n)); + } + }; + + // If all we know is it's a C-common library, then in both cases we only + // look for cc.export.libs. + // + if (cc) + { + if (proc_opt) proc_opt (l, *t, true, true); + if (c_e_libs) proc_int (c_e_libs); + } + else + { + if (impl) + { + // Interface and implementation: as discussed above, we can have two + // situations: overriden export or default export. + // + if (c_e_libs.defined () || x_e_libs.defined ()) + { + // NOTE: should this not be from l.vars rather than l? Or perhaps + // we can assume non-common values will be set on libs{}/liba{}. + // + if (proc_opt) proc_opt (l, *t, true, true); + if (c_e_libs) proc_int (c_e_libs); + + if (proc_opt) proc_opt (l, *t, false, true); + if (x_e_libs) proc_int (x_e_libs); + } + else + { + // For default export we use the same options/libs as were used to + // build the library. Since libraries in (non-export) *.libs are + // not targets, we don't need to recurse. + // + if (proc_opt) proc_opt (l, *t, true, false); + if (proc_lib) proc_imp (l[c_libs]); + + if (proc_opt) proc_opt (l, *t, false, false); + if (proc_lib) proc_imp (l[same ? x_libs : vp[*t + ".libs"]]); + } + } + else + { + // Interface: only add *.export.* (interface dependencies). + // + if (proc_opt) proc_opt (l, *t, true, true); + if (c_e_libs) proc_int (c_e_libs); + + if (proc_opt) proc_opt (l, *t, false, true); + if (x_e_libs) proc_int (x_e_libs); + } + } + } + + // The name can be an absolute target name (e.g., /tmp/libfoo/lib{foo}) or + // a potentially project-qualified relative target name (e.g., + // libfoo%lib{foo}). + // + // Note that the scope, search paths, and the link order should all be + // derived from the library target that mentioned this name. This way we + // will select exactly the same target as the library's matched rule and + // that's the only way to guarantee it will be up-to-date. + // + file& common:: + resolve_library (name n, + scope& s, + lorder lo, + const dir_paths& sysd, + optional& usrd) const + { + if (n.type != "lib" && n.type != "liba" && n.type != "libs") + fail << "target name " << n << " is not a library"; + + target* xt (nullptr); + + if (n.dir.absolute () && !n.qualified ()) + { + // Search for an existing target with this name "as if" it was a + // prerequisite. + // + xt = &search (move (n), s); + } + else + { + // This is import. + // + const string* ext; + const target_type* tt (s.find_target_type (n, ext)); // Changes name. + + if (tt == nullptr) + fail << "unknown target type '" << n.type << "' in library " << n; + + // @@ OUT: for now we assume out is undetermined, just like in + // search (name, scope). + // + dir_path out; + prerequisite_key pk {n.proj, {tt, &n.dir, &out, &n.value, ext}, &s}; + xt = search_library (sysd, usrd, pk); + + if (xt == nullptr) + { + if (n.qualified ()) + xt = &import (pk); + else + fail << "unable to find library " << pk; + } + } + + // If this is lib{}, pick appropriate member. + // + if (lib* l = xt->is_a ()) + xt = &link_member (*l, lo); // Pick liba{} or libs{}. + + return static_cast (*xt); + } + + // Note that pk's scope should not be NULL (even if dir is absolute). If + // sys is not NULL, then store there an inidication of whether this is a + // system library. + // + target* common:: + search_library (const dir_paths& sysd, + optional& usrd, + const prerequisite_key& p) const + { + tracer trace (x, "search_library"); + + assert (p.scope != nullptr); + + // @@ This is hairy enough to warrant a separate implementation for + // Windows. + // + bool l (p.is_a ()); + const string* ext (l ? nullptr : p.tk.ext); // Only for liba/libs. + + // Then figure out what we need to search for. + // + const string& name (*p.tk.name); + + // liba + // + path an; + const string* ae (nullptr); + + if (l || p.is_a ()) + { + // We are trying to find a library in the search paths extracted from + // the compiler. It would only be natural if we used the library + // prefix/extension that correspond to this compiler and/or its + // target. + // + // Unlike MinGW, VC's .lib/.dll.lib naming is by no means standard and + // we might need to search for other names. In fact, there is no + // reliable way to guess from the file name what kind of library it + // is, static or import and we will have to do deep inspection of such + // alternative names. However, if we did find .dll.lib, then we can + // assume that .lib is the static library without any deep inspection + // overhead. + // + const char* e (""); + + if (cid == "msvc") + { + an = path (name); + e = "lib"; + } + else + { + an = path ("lib" + name); + e = "a"; + } + + ae = ext == nullptr + ? &extension_pool.find (e) + : ext; + + if (!ae->empty ()) + { + an += '.'; + an += *ae; + } + } + + // libs + // + path sn; + const string* se (nullptr); + + if (l || p.is_a ()) + { + const char* e (""); + + if (cid == "msvc") + { + sn = path (name); + e = "dll.lib"; + } + else + { + sn = path ("lib" + name); + + if (tsys == "darwin") e = "dylib"; + else if (tsys == "mingw32") e = "dll.a"; // See search code below. + else e = "so"; + } + + se = ext == nullptr + ? &extension_pool.find (e) + : ext; + + if (!se->empty ()) + { + sn += '.'; + sn += *se; + } + } + + // Now search. + // + liba* a (nullptr); + libs* s (nullptr); + + path f; // Reuse the buffer. + const dir_path* pd (nullptr); + + auto search =[&a, &s, + &an, &ae, + &sn, &se, + &name, ext, + &p, &f, &trace, this] (const dir_path& d) -> bool + { + timestamp mt; + + // libs + // + // Look for the shared library first. The order is important for VC: + // only if we found .dll.lib can we safely assumy that just .lib is a + // static library. + // + if (!sn.empty ()) + { + f = d; + f /= sn; + mt = file_mtime (f); + + if (mt != timestamp_nonexistent) + { + // On Windows what we found is the import library which we need + // to make the first ad hoc member of libs{}. + // + if (tclass == "windows") + { + s = &targets.insert (d, dir_path (), name, nullptr, trace); + + if (s->member == nullptr) + { + libi& i ( + targets.insert (d, dir_path (), name, se, trace)); + + if (i.path ().empty ()) + i.path (move (f)); + + i.mtime (mt); + + // Presumably there is a DLL somewhere, we just don't know + // where (and its possible we might have to look for one if we + // decide we need to do rpath emulation for installed + // libraries as well). We will represent this as empty path + // but valid timestamp (aka "trust me, it's there"). + // + s->mtime (mt); + s->member = &i; + } + } + else + { + s = &targets.insert (d, dir_path (), name, se, trace); + + if (s->path ().empty ()) + s->path (move (f)); + + s->mtime (mt); + } + } + else if (ext == nullptr && tsys == "mingw32") + { + // Above we searched for the import library (.dll.a) but if it's + // not found, then we also search for the .dll (unless the + // extension was specified explicitly) since we can link to it + // directly. Note also that the resulting libs{} would end up + // being the .dll. + // + se = &extension_pool.find ("dll"); + f = f.base (); // Remove .a from .dll.a. + mt = file_mtime (f); + + if (mt != timestamp_nonexistent) + { + s = &targets.insert (d, dir_path (), name, se, trace); + + if (s->path ().empty ()) + s->path (move (f)); + + s->mtime (mt); + } + } + } + + // liba + // + // If we didn't find .dll.lib then we cannot assume .lib is static. + // + if (!an.empty () && (s != nullptr || cid != "msvc")) + { + f = d; + f /= an; + + if ((mt = file_mtime (f)) != timestamp_nonexistent) + { + // Enter the target. Note that because the search paths are + // normalized, the result is automatically normalized as well. + // + // Note that this target is outside any project which we treat + // as out trees. + // + a = &targets.insert (d, dir_path (), name, ae, trace); + + if (a->path ().empty ()) + a->path (move (f)); + + a->mtime (mt); + } + } + + // Alternative search for VC. + // + if (cid == "msvc") + { + scope& rs (*p.scope->root_scope ()); + const process_path& ld (cast (rs["bin.ld.path"])); + + if (s == nullptr && !sn.empty ()) + s = msvc_search_shared (ld, d, p); + + if (a == nullptr && !an.empty ()) + a = msvc_search_static (ld, d, p); + } + + return a != nullptr || s != nullptr; + }; + + // First try user directories (i.e., -L). + // + bool sys (false); + + if (!usrd) + usrd = extract_library_dirs (*p.scope); + + for (const dir_path& d: *usrd) + { + if (search (d)) + { + pd = &d; + break; + } + } + + // Next try system directories (i.e., those extracted from the compiler). + // + if (pd == nullptr) + { + for (const dir_path& d: sysd) + { + if (search (d)) + { + pd = &d; + break; + } + } + + sys = true; + } + + if (pd == nullptr) + return nullptr; + + // Add the "using static/shared library" macro (used, for example, to + // handle DLL export). The absence of either of these macros would mean + // some other build system that cannot distinguish between the two (and + // no pkg-config information). + // + auto add_macro = [this] (target& t, const char* suffix) + { + // If there is already a value (either in cc.export or x.export), + // don't add anything: we don't want to be accumulating defines nor + // messing with custom values. And if we are adding, then use the + // generic cc.export. + // + // The only way we could already have this value is if this same + // library was also imported as a project (as opposed to installed). + // Unlikely but possible. In this case the values were set by the + // export stub and we shouldn't touch them. + // + if (!t.vars[x_export_poptions]) + { + auto p (t.vars.insert (c_export_poptions)); + + if (p.second) + { + // The "standard" macro name will be LIB_{STATIC,SHARED}, + // where is the target name. Here we want to strike a + // balance between being unique and not too noisy. + // + string d ("-DLIB"); + + auto upcase_sanitize = [] (char c) + { + return (c == '-' || c == '+' || c == '.') ? '_' : ucase (c); + }; + + transform (t.name.begin (), + t.name.end (), + back_inserter (d), + upcase_sanitize); + + d += '_'; + d += suffix; + + strings o; + o.push_back (move (d)); + p.first.get () = move (o); + } + } + }; + + // Enter (or find) the lib{} target group. Note that we must be careful + // here since its possible we have already imported some of its members. + // + lib& lt ( + targets.insert ( + *pd, dir_path (), name, l ? p.tk.ext : nullptr, trace)); + + // It should automatically link-up to the members we have found. + // + assert (a == nullptr || lt.a == a); + assert (s == nullptr || lt.s == s); + + // Update the bin.lib variable to indicate what's available. + // + const char* bl (lt.a != nullptr + ? (lt.s != nullptr ? "both" : "static") + : "shared"); + lt.assign ("bin.lib") = bl; + + target* r (l ? < : (p.is_a () ? static_cast (a) : s)); + + // Mark as a "cc" library (unless already marked) and set the system + // flag. + // + auto mark_cc = [sys, this] (target& t) -> bool + { + auto p (t.vars.insert (c_type)); + + if (p.second) + { + p.first.get () = string ("cc"); + + if (sys) + t.vars.assign (c_system) = true; + } + + return p.second; + }; + + // If the library already has cc.type, then assume it was either already + // imported or was matched by a rule. + // + if (a != nullptr && !mark_cc (*a)) + a = nullptr; + + if (s != nullptr && !mark_cc (*s)) + s = nullptr; + + if (a != nullptr || s != nullptr) + { + // Try to extract library information from pkg-config. We only add the + // default macro if we could not extract more precise information. The + // idea is that when we auto-generate .pc files, we will copy those + // macros (or custom ones) from *.export.poptions. + // + if (pkgconfig == nullptr || + !pkgconfig_extract (*p.scope, lt, a, s, p.proj, name, *pd, sysd)) + { + if (a != nullptr) add_macro (*a, "STATIC"); + if (s != nullptr) add_macro (*s, "SHARED"); + } + } + + return r; + } + + dir_paths common:: + extract_library_dirs (scope& bs) const + { + dir_paths r; + + // Extract user-supplied search paths (i.e., -L, /LIBPATH). + // + auto extract = [&r, this] (const value& val) + { + const auto& v (cast (val)); + + for (auto i (v.begin ()), e (v.end ()); i != e; ++i) + { + const string& o (*i); + + dir_path d; + + if (cid == "msvc") + { + // /LIBPATH: (case-insensitive). + // + if ((o[0] == '/' || o[0] == '-') && + (i->compare (1, 8, "LIBPATH:") == 0 || + i->compare (1, 8, "libpath:") == 0)) + d = dir_path (*i, 9, string::npos); + else + continue; + } + else + { + // -L can either be in the "-L" or "-L " form. + // + if (*i == "-L") + { + if (++i == e) + break; // Let the compiler complain. + + d = dir_path (*i); + } + else if (i->compare (0, 2, "-L") == 0) + d = dir_path (*i, 2, string::npos); + else + continue; + } + + // Ignore relative paths. Or maybe we should warn? + // + if (!d.relative ()) + r.push_back (move (d)); + } + }; + + if (auto l = bs[c_loptions]) extract (*l); + if (auto l = bs[x_loptions]) extract (*l); + + return r; + } + } +} -- cgit v1.1