From d64ae97f6865bc25d496485622530e2a090c2eb4 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 21 Aug 2019 12:11:48 +0200 Subject: Implement dynamic loading of build system modules --- build2/b.cxx | 85 +++++++------- libbuild2/buildfile | 11 ++ libbuild2/context.cxx | 5 +- libbuild2/file.cxx | 2 +- libbuild2/file.hxx | 2 +- libbuild2/function.hxx | 14 +-- libbuild2/module.cxx | 307 +++++++++++++++++++++++++++++++++++++++++++------ libbuild2/module.hxx | 49 +++++--- libbuild2/scope.hxx | 2 +- libbuild2/utility.cxx | 5 + libbuild2/utility.hxx | 6 +- 11 files changed, 381 insertions(+), 107 deletions(-) diff --git a/build2/b.cxx b/build2/b.cxx index 15844dc..e7f11f2 100644 --- a/build2/b.cxx +++ b/build2/b.cxx @@ -484,56 +484,59 @@ main (int argc, char* argv[]) SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); #endif - // Register builtin modules. + // Load builtin modules. // { - using mf = module_functions; - auto& bm (builtin_modules); - - auto reg = [] (module_load_function* lf) + auto load = [] (module_load_function* lf) { for (const module_functions* i (lf ()); i->name != nullptr; ++i) - builtin_modules[i->name] = *i; + loaded_modules[i->name] = i; }; - reg (&config::build2_config_load); - reg (&dist::build2_dist_load); - reg (&test::build2_test_load); - reg (&install::build2_install_load); - - reg (&version::build2_version_load); - reg (&in::build2_in_load); - - bm["bin.vars"] = mf {"bin.vars", nullptr, &bin::vars_init}; - bm["bin.config"] = mf {"bin.config", nullptr, &bin::config_init}; - bm["bin"] = mf {"bin", nullptr, &bin::init}; - bm["bin.ar.config"] = mf {"bin.ar.config", nullptr, &bin::ar_config_init}; - bm["bin.ar"] = mf {"bin.ar", nullptr, &bin::ar_init}; - bm["bin.ld.config"] = mf {"bin.ld.config", nullptr, &bin::ld_config_init}; - bm["bin.ld"] = mf {"bin.ld", nullptr, &bin::ld_init}; - bm["bin.rc.config"] = mf {"bin.rc.config", nullptr, &bin::rc_config_init}; - bm["bin.rc"] = mf {"bin.rc", nullptr, &bin::rc_init}; - - bm["cc.core.vars"] = mf {"cc.core.vars", nullptr, &cc::core_vars_init}; - bm["cc.core.guess"] = mf {"cc.core.guess", nullptr, &cc::core_guess_init}; - bm["cc.core.config"] = mf {"cc.core.config", nullptr, &cc::core_config_init}; - bm["cc.core"] = mf {"cc.core", nullptr, &cc::core_init}; - bm["cc.config"] = mf {"cc.config", nullptr, &cc::config_init}; - bm["cc"] = mf {"cc", nullptr, &cc::init}; - - bm["c.guess"] = mf {"c.guess", nullptr, &c::guess_init}; - bm["c.config"] = mf {"c.config", nullptr, &c::config_init}; - bm["c"] = mf {"c", nullptr, &c::init}; - - bm["cxx.guess"] = mf {"cxx.guess", nullptr, &cxx::guess_init}; - bm["cxx.config"] = mf {"cxx.config", nullptr, &cxx::config_init}; - bm["cxx"] = mf {"cxx", nullptr, &cxx::init}; + // @@ TMP + // +#define TMP_LOAD(N, S, I) \ + static const module_functions N {S, nullptr, &I}; \ + loaded_modules[S] = &N + + load (&config::build2_config_load); + load (&dist::build2_dist_load); + load (&test::build2_test_load); + load (&install::build2_install_load); + + load (&version::build2_version_load); + load (&in::build2_in_load); + + TMP_LOAD (bin_vars, "bin.vars", bin::vars_init); + TMP_LOAD (bin_config, "bin.config", bin::config_init); + TMP_LOAD (bin, "bin", bin::init); + TMP_LOAD (bin_ar_config, "bin.ar.config", bin::ar_config_init); + TMP_LOAD (bin_ar, "bin.ar", bin::ar_init); + TMP_LOAD (bin_ld_config, "bin.ld.config", bin::ld_config_init); + TMP_LOAD (bin_ld, "bin.ld", bin::ld_init); + TMP_LOAD (bin_rc_config, "bin.rc.config", bin::rc_config_init); + TMP_LOAD (bin_rc, "bin.rc", bin::rc_init); + + TMP_LOAD (cc_core_vars, "cc.core.vars", cc::core_vars_init); + TMP_LOAD (cc_core_guess, "cc.core.guess", cc::core_guess_init); + TMP_LOAD (cc_core_config, "cc.core.config", cc::core_config_init); + TMP_LOAD (cc_core, "cc.core", cc::core_init); + TMP_LOAD (cc_config, "cc.config", cc::config_init); + TMP_LOAD (cc, "cc", cc::init); + + TMP_LOAD (c_guess, "c.guess", c::guess_init); + TMP_LOAD (c_config, "c.config", c::config_init); + TMP_LOAD (c, "c", c::init); + + TMP_LOAD (cxx_guess, "cxx.guess", cxx::guess_init); + TMP_LOAD (cxx_config, "cxx.config", cxx::config_init); + TMP_LOAD (cxx, "cxx", cxx::init); #ifndef BUILD2_BOOTSTRAP - bm["cli.config"] = mf {"cli.config", nullptr, &cli::config_init}; - bm["cli"] = mf {"cli", nullptr, &cli::init}; + TMP_LOAD (cli_config, "cli.config", cli::config_init); + TMP_LOAD (cli, "cli", cli::init); - reg (&bash::build2_bash_load); + load (&bash::build2_bash_load); #endif } diff --git a/libbuild2/buildfile b/libbuild2/buildfile index 6539a01..7d91e08 100644 --- a/libbuild2/buildfile +++ b/libbuild2/buildfile @@ -2,6 +2,9 @@ # copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file +# NOTE: remember to update bundled_modules in libbuild2/modules.cxx if adding +# a new module. +# ./: lib{build2} bash/ in/ version/ import int_libs = libbutl%lib{butl} @@ -13,6 +16,9 @@ lib{build2}: libul{build2}: {hxx ixx txx cxx}{* -config -version -*.test...} \ # tests loop below). Note that the build system core can still function # without them or with their alternative implementations. # +# NOTE: remember to update import_modules() in libbuild2/modules.cxx if adding +# a new such module. +# for m: config dist install test libul{build2}: $m/{hxx ixx txx cxx}{** -**-options -**.test...} @@ -67,8 +73,13 @@ obja{context}: cxx.poptions += -DLIBBUILD2_STATIC_BUILD objs{context}: cxx.poptions += -DLIBBUILD2_SHARED_BUILD if ($cxx.target.class != "windows") +{ cxx.libs += -lpthread + if ($cxx.target.class != "bsd") + cxx.libs += -ldl +} + # Export options. # lib{build2}: diff --git a/libbuild2/context.cxx b/libbuild2/context.cxx index e7f9751..2f5e2af 100644 --- a/libbuild2/context.cxx +++ b/libbuild2/context.cxx @@ -461,10 +461,7 @@ namespace build2 // Build system interface version. In particular, it is embedded into // build system modules as load_suffix. // - set ("build.version.interface", - v.pre_release () - ? v.string_project_id () - : to_string (v.major ()) + '.' + to_string (v.minor ())); + set ("build.version.interface", build_version_interface); // Allow detection (for example, in tests) whether this is a staged // toolchain. diff --git a/libbuild2/file.cxx b/libbuild2/file.cxx index 66f05f7..1da9397 100644 --- a/libbuild2/file.cxx +++ b/libbuild2/file.cxx @@ -1170,7 +1170,7 @@ namespace build2 if (scope* rs = root.parent_scope ()->root_scope ()) load_root (*rs); - // Finish off loading bootstrapped modules. + // Finish off initializing bootstrapped modules. // for (auto& p: root.root_extra->modules) { diff --git a/libbuild2/file.hxx b/libbuild2/file.hxx index 4d668fe..48d1b63 100644 --- a/libbuild2/file.hxx +++ b/libbuild2/file.hxx @@ -163,7 +163,7 @@ namespace build2 LIBBUILD2_SYMEXPORT bool bootstrapped (scope& root); - // Execute pre/post-bootstrap hooks. Similar to bootstrap_out/sr(), should + // Execute pre/post-bootstrap hooks. Similar to bootstrap_out/src(), should // only be called once per project bootstrap. // LIBBUILD2_SYMEXPORT void diff --git a/libbuild2/function.hxx b/libbuild2/function.hxx index 6b2bfe1..51c17c0 100644 --- a/libbuild2/function.hxx +++ b/libbuild2/function.hxx @@ -429,10 +429,10 @@ namespace build2 #endif }; - // Cast data/thunk. + // Cast data/thunk for functions. // template - struct function_cast + struct function_cast_func { // A pointer to a standard layout struct is a pointer to its first data // member, which in our case is the cast thunk. @@ -468,7 +468,7 @@ namespace build2 // argument. // template - struct function_cast + struct function_cast_func { struct data { @@ -500,7 +500,7 @@ namespace build2 // Specialization for void return type. In this case we return NULL value. // template - struct function_cast + struct function_cast_func { struct data { @@ -528,7 +528,7 @@ namespace build2 }; template - struct function_cast + struct function_cast_func { struct data { @@ -753,7 +753,7 @@ namespace build2 operator= (R (*impl) (A...)) && { using args = function_args; - using cast = function_cast; + using cast = function_cast_func; insert (move (name), function_overload ( @@ -770,7 +770,7 @@ namespace build2 operator= (R (*impl) (const scope*, A...)) && { using args = function_args; - using cast = function_cast; + using cast = function_cast_func; insert (move (name), function_overload ( diff --git a/libbuild2/module.cxx b/libbuild2/module.cxx index 50530f2..1feb121 100644 --- a/libbuild2/module.cxx +++ b/libbuild2/module.cxx @@ -4,23 +4,259 @@ #include +#ifndef _WIN32 +# include +#else +# include +#endif + +#include // import() #include #include #include +// Core modules bundled with libbuild2. +// +#include +#include +#include +#include + using namespace std; +using namespace butl; namespace build2 { - available_module_map builtin_modules; + loaded_module_map loaded_modules; + + // Sorted array of bundled modules (excluding core modules bundled with + // libbuild2; see below). + // + static const char* bundled_modules[] = { + "bash", + "in", + "version" + }; + + static inline bool + bundled_module (const string& mod) + { + return binary_search ( + bundled_modules, + bundled_modules + sizeof (bundled_modules) / sizeof (*bundled_modules), + mod); + } + + static module_load_function* + import_module (scope& /*bs*/, + const string& mod, + const location& loc, + bool boot, + bool opt) + { + // Take care of core modules that are bundled with libbuild2 in case they + // are not pre-loaded by the driver. + // + if (mod == "config") return &config::build2_config_load; + else if (mod == "dist") return &dist::build2_dist_load; + else if (mod == "install") return &install::build2_install_load; + else if (mod == "test") return &test::build2_test_load; + + bool bundled (bundled_module (mod)); + + // Importing external modules during bootstrap is problematic: we haven't + // loaded config.build nor entered all the variable overrides so it's not + // clear what import() can do except confuse matters. So this requires + // more thinking. + // + if (boot && !bundled) + { + fail (loc) << "unable to load build system module " << mod << + info << "loading external modules during bootstrap is not yet " + << "supported"; + } + + path lib; + +#if 0 + // See if we can import a target for this module. + // + // Check if one of the bundled modules, if so, the project name is + // build2, otherwise -- libbuild2-. + // + // The target we are looking for is %lib{build2-}. + // + name tgt ( + import (bs, + name (bundled ? "build2" : "libbuild2-" + mod, + dir_path (), + "lib", + "build2-" + mod), + loc)); + + if (!tgt.qualified ()) + { + // Switch the phase and update the target. This will also give us the + // shared library path. + // + // @@ TODO + // + } + else +#endif + { + // No luck. Form the shared library name (incorporating build system + // core version) and try using system-default search (installed, rpath, + // etc). + + // @@ This is unfortunate: it would have been nice to do something + // similar to what we've done for exe{}. While libs{} is in the bin + // module, we could bring it in (we've done it for exe{}). The + // problems are: it is intertwined with its group (lib{}) and we + // don't have any mechanisms to deal with prefixes, only extensions. + // + const char* pfx; + const char* sfx; +#if defined(_WIN32) + pfx = "build2-"; sfx = ".dll"; +#elif defined(__APPLE__) + pfx = "libbuild2-"; sfx = ".dylib"; +#else + pfx = "libbuild2-"; sfx = ".so"; +#endif + + lib = path (pfx + mod + '-' + build_version_interface + sfx); + } + + string sym (sanitize_identifier ("build2_" + mod + "_load")); + + // Note that we don't unload our modules since it's not clear what would + // the benefit be. + // + module_load_function* r (nullptr); + +#ifndef _WIN32 + // Use RTLD_NOW instead of RTLD_LAZY to both speed things up (we are going + // to use this module now) and to detect any symbol mismatches. + // + if (void* h = dlopen (lib.string ().c_str (), RTLD_NOW | RTLD_GLOBAL)) + { + r = function_cast (dlsym (h, sym.c_str ())); + + // I don't think we should ignore this even if optional. + // + if (r == nullptr) + fail (loc) << "unable to lookup " << sym << " in build system module " + << mod << " (" << lib << "): " << dlerror (); + } + else if (!opt) + fail (loc) << "unable to load build system module " << mod + << " (" << lib << "): " << dlerror (); +#else + if (HMODULE h = LoadLibrary (lib.string ().c_str ())) + { + r = function_cast ( + GetProcAddress (h, sym.c_str ())); + + if (r == nullptr) + fail (loc) << "unable to lookup " << sym << " in build system module " + << mod << " (" << lib << "): " << win32::last_error_msg (); + } + else if (!opt) + fail (loc) << "unable to load build system module " << mod + << " (" << lib << "): " << win32::last_error_msg (); +#endif + + return r; + } + + static const module_functions* + find_module (scope& bs, + const string& smod, + const location& loc, + bool boot, + bool opt) + { + // Optional modules and submodules sure make this logic convoluted. So we + // divide it into two parts: (1) find or insert an entry (for submodule + // or, failed that, for the main module, the latter potentially NULL) and + // (2) analyze the entry and issue diagnostics. + // + auto i (loaded_modules.find (smod)), e (loaded_modules.end ()); + + if (i == e) + { + // If this is a submodule, get the main module name. + // + string mmod (smod, 0, smod.find ('.')); + + if (mmod != smod) + i = loaded_modules.find (mmod); + + if (i == e) + { + module_load_function* f (import_module (bs, mmod, loc, boot, opt)); + + if (f != nullptr) + { + // Enter all the entries noticing which one is our submodule. If + // none are, then we notice the main module. + // + for (const module_functions* j (f ()); j->name != nullptr; ++j) + { + const string& n (j->name); + + auto p (loaded_modules.emplace (n, j)); + + if (!p.second) + fail (loc) << "build system submodule name " << n << " of main " + << "module " << mmod << " is already in use"; + + if (n == smod || (i == e && n == mmod)) + i = p.first; + } + + // We should at least have the main module. + // + if (i == e) + fail (loc) << "invalid function list in build system module " + << mmod; + } + else + i = loaded_modules.emplace (move (mmod), nullptr).first; + } + } + + // Now the iterator points to a submodule or to the main module, the + // latter potentially NULL. + // + if (!opt) + { + if (i->second == nullptr) + { + fail (loc) << "unable to load build system module " << i->first; + } + else if (i->first != smod) + { + fail (loc) << "build system module " << i->first << " has no " + << "submodule " << smod; + } + } + + // Note that if the main module exists but has no such submodule, we + // return NULL rather than fail (think of an older version of a module + // that doesn't implement some extra functionality). + // + return i->second; + } void - boot_module (scope& rs, const string& name, const location& loc) + boot_module (scope& rs, const string& mod, const location& loc) { - // First see if this modules has already been loaded for this project. + // First see if this modules has already been booted for this project. // - loaded_module_map& lm (rs.root_extra->modules); - auto i (lm.find (name)); + module_map& lm (rs.root_extra->modules); + auto i (lm.find (mod)); if (i != lm.end ()) { @@ -35,58 +271,48 @@ namespace build2 // Otherwise search for this module. // - auto j (builtin_modules.find (name)); - - if (j == builtin_modules.end ()) - fail (loc) << "unknown module " << name; - - const module_functions& mf (j->second); + const module_functions& mf ( + *find_module (rs, mod, loc, true /* boot */, false /* optional */)); if (mf.boot == nullptr) - fail (loc) << "module " << name << " shouldn't be loaded in bootstrap"; + fail (loc) << "build system module " << mod << " should not be loaded " + << "during bootstrap"; - i = lm.emplace (name, + i = lm.emplace (mod, module_state {true, false, mf.init, nullptr, loc}).first; i->second.first = mf.boot (rs, loc, i->second.module); - rs.assign (var_pool.rw (rs).insert (name + ".booted")) = true; + rs.assign (var_pool.rw (rs).insert (mod + ".booted")) = true; } bool - load_module (scope& rs, + init_module (scope& rs, scope& bs, - const string& name, + const string& mod, const location& loc, bool opt, const variable_map& hints) { - // First see if this modules has already been loaded for this project. + // First see if this modules has already been inited for this project. // - loaded_module_map& lm (rs.root_extra->modules); - auto i (lm.find (name)); + module_map& lm (rs.root_extra->modules); + auto i (lm.find (mod)); bool f (i == lm.end ()); if (f) { // Otherwise search for this module. // - auto j (builtin_modules.find (name)); - - if (j == builtin_modules.end ()) + if (const module_functions* mf = find_module ( + rs, mod, loc, false /* boot */, opt)) { - if (!opt) - fail (loc) << "unknown module " << name; - } - else - { - const module_functions& mf (j->second); - - if (mf.boot != nullptr) - fail (loc) << "module " << name << " should be loaded in bootstrap"; + if (mf->boot != nullptr) + fail (loc) << "build system module " << mod << " should be loaded " + << "during bootstrap"; i = lm.emplace ( - name, - module_state {false, false, mf.init, nullptr, loc}).first; + mod, + module_state {false, false, mf->init, nullptr, loc}).first; } } else @@ -103,11 +329,15 @@ namespace build2 // Note: pattern-typed in context.cxx:reset() as project-visibility // variables of type bool. // + // We call the variable 'loaded' rather than 'inited' because it is + // buildfile-visible (where we use the term "load a module"; see the note + // on terminology above) + // auto& vp (var_pool.rw (rs)); - value& lv (bs.assign (vp.insert (name + ".loaded"))); - value& cv (bs.assign (vp.insert (name + ".configured"))); + value& lv (bs.assign (vp.insert (mod + ".loaded"))); + value& cv (bs.assign (vp.insert (mod + ".configured"))); - bool l; // Loaded. + bool l; // Loaded (initialized). bool c; // Configured. // Suppress duplicate init() calls for the same module in the same scope. @@ -122,7 +352,7 @@ namespace build2 if (!opt) { if (!l) - fail (loc) << "unknown module " << name; + fail (loc) << "unable to load build system module " << mod; // We don't have original diagnostics. We could call init() again so // that it can issue it. But that means optional modules must be @@ -130,7 +360,8 @@ namespace build2 // simple for now. // if (!c) - fail (loc) << "module " << name << " failed to configure"; + fail (loc) << "build system module " << mod << " failed to " + << "configure"; } } else diff --git a/libbuild2/module.hxx b/libbuild2/module.hxx index 8343e11..200e52f 100644 --- a/libbuild2/module.hxx +++ b/libbuild2/module.hxx @@ -17,6 +17,12 @@ namespace build2 { + // A few high-level notes on the terminology: From the user's perspective, + // the module is "loaded" (with the `using` directive). From the + // implementation's perspectives, the module library is "loaded" and the + // module is "bootstrapped" (or "booted" for short) and then "initialized" + // (or "inited"). + class scope; class location; @@ -76,7 +82,7 @@ namespace build2 extern "C" using module_load_function = const module_functions* (); - // Loaded modules state. + // Module state. // struct module_state { @@ -87,7 +93,7 @@ namespace build2 const location loc; // Boot location. }; - struct loaded_module_map: std::map + struct module_map: std::map { template T* @@ -100,15 +106,15 @@ namespace build2 } }; - // Load and boot the specified module. + // Boot the specified module loading its library if necessary. // LIBBUILD2_SYMEXPORT void boot_module (scope& root, const string& name, const location&); - // Load (if not already loaded) and initialize the specified module. Used - // by the parser but also by some modules to load prerequisite modules. - // Return true if the module was both successfully loaded and configured - // (false can only be returned if optional). + // Init the specified module loading its library if necessary. Used by the + // parser but also by some modules to init prerequisite modules. Return true + // if the module was both successfully loaded and configured (false can only + // be returned if optional is true). // // The config_hints variable map can be used to pass configuration hints // from one module to another. For example, the cxx modude may pass the @@ -117,20 +123,37 @@ namespace build2 // its tools). // LIBBUILD2_SYMEXPORT bool - load_module (scope& root, + init_module (scope& root, scope& base, const string& name, const location&, bool optional = false, const variable_map& config_hints = variable_map ()); - // Builtin modules. + // An alias to use from other modules (we could also distinguish between + // boot and init). + // + // @@ TODO: maybe incorporate the .loaded variable check we have all over + // (it's not clear if init_module() already has this semantics)? + // + inline bool + load_module (scope& root, + scope& base, + const string& name, + const location& loc, + bool optional = false, + const variable_map& config_hints = variable_map ()) + { + return init_module (root, base, name, loc, optional, config_hints); + } + + // Loaded modules (as in libraries). // - // @@ Maybe this should be renamed to loaded modules? - // @@ We can also change it to std::map + // A NULL entry for the main module indicates that a module library was not + // found. // - using available_module_map = std::map; - LIBBUILD2_SYMEXPORT extern available_module_map builtin_modules; + using loaded_module_map = std::map; + LIBBUILD2_SYMEXPORT extern loaded_module_map loaded_modules; } #endif // LIBBUILD2_MODULE_HXX diff --git a/libbuild2/scope.hxx b/libbuild2/scope.hxx index 455bcc6..f69e822 100644 --- a/libbuild2/scope.hxx +++ b/libbuild2/scope.hxx @@ -298,7 +298,7 @@ namespace build2 // Modules. // - loaded_module_map modules; + module_map modules; // Variable override cache (see above). // diff --git a/libbuild2/utility.cxx b/libbuild2/utility.cxx index 79545cc..cacb464 100644 --- a/libbuild2/utility.cxx +++ b/libbuild2/utility.cxx @@ -74,6 +74,11 @@ namespace build2 process_path argv0; const standard_version build_version (LIBBUILD2_VERSION_STR); + const string build_version_interface ( + build_version.pre_release () + ? build_version.string_project_id () + : (to_string (build_version.major ()) + '.' + + to_string (build_version.minor ()))); bool dry_run_option; optional mtime_check_option; diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx index c251b64..ed94a08 100644 --- a/libbuild2/utility.hxx +++ b/libbuild2/utility.hxx @@ -71,10 +71,13 @@ namespace build2 using butl::trim; using butl::next_word; + using butl::sanitize_identifier; using butl::make_guard; using butl::make_exception_guard; + using butl::function_cast; + using butl::getenv; using butl::setenv; using butl::unsetenv; @@ -136,9 +139,10 @@ namespace build2 // LIBBUILD2_SYMEXPORT extern process_path argv0; - // Build system driver version and check. + // Build system core version and interface version. // LIBBUILD2_SYMEXPORT extern const standard_version build_version; + LIBBUILD2_SYMEXPORT extern const string build_version_interface; LIBBUILD2_SYMEXPORT extern bool dry_run_option; // --dry-run -- cgit v1.1