diff options
Diffstat (limited to 'libbuild2/config')
| -rw-r--r-- | libbuild2/config/functions.cxx | 62 | ||||
| -rw-r--r-- | libbuild2/config/host-config.cxx.in | 3 | ||||
| -rw-r--r-- | libbuild2/config/init.cxx | 518 | ||||
| -rw-r--r-- | libbuild2/config/module.cxx | 13 | ||||
| -rw-r--r-- | libbuild2/config/module.hxx | 98 | ||||
| -rw-r--r-- | libbuild2/config/operation.cxx | 522 | ||||
| -rw-r--r-- | libbuild2/config/operation.hxx | 15 | ||||
| -rw-r--r-- | libbuild2/config/types.hxx | 25 | ||||
| -rw-r--r-- | libbuild2/config/utility.cxx | 72 | ||||
| -rw-r--r-- | libbuild2/config/utility.hxx | 182 | ||||
| -rw-r--r-- | libbuild2/config/utility.ixx | 13 | ||||
| -rw-r--r-- | libbuild2/config/utility.txx | 4 |
12 files changed, 1284 insertions, 243 deletions
diff --git a/libbuild2/config/functions.cxx b/libbuild2/config/functions.cxx index 5e85238..b1a61a2 100644 --- a/libbuild2/config/functions.cxx +++ b/libbuild2/config/functions.cxx @@ -7,6 +7,7 @@ #include <libbuild2/function.hxx> #include <libbuild2/variable.hxx> +#include <libbuild2/config/module.hxx> #include <libbuild2/config/operation.hxx> using namespace std; @@ -20,6 +21,58 @@ namespace build2 { function_family f (m, "config"); + // $config.origin() + // + // Return the origin of the value of the specified configuration + // variable. Possible result values and their semantics are as follows: + // + // undefined + // The variable is undefined. + // + // default + // The variable has the default value from the config directive (or + // as specified by a module). + // + // buildfile + // The variable has the value from a buildfile, normally config.build + // but could also be from file(s) specified with config.config.load. + // + // override + // The variable has the command line override value. Note that if + // the override happens to be append/prepend, then the value could + // incorporate the original value. + // + // Note that the variable must be specified as a name and not as an + // expansion (i.e., without $). + // + // Note that this function is not pure. + // + f.insert (".origin", false) += [] (const scope* s, names name) + { + if (s == nullptr) + fail << "config.origin() called out of scope" << endf; + + // Only look in the root scope since that's the only config.* + // variables we generally consider. + // + s = s->root_scope (); + + if (s == nullptr) + fail << "config.origin() called out of project" << endf; + + switch (origin (*s, convert<string> (move (name))).first) + { + case variable_origin::undefined: return "undefined"; + case variable_origin::default_: return "default"; + case variable_origin::buildfile: return "buildfile"; + case variable_origin::override_: return "override"; + } + + return ""; // Should not reach. + }; + + // $config.save() + // // Return the configuration file contents as a string, similar to the // config.config.save variable functionality. // @@ -39,6 +92,14 @@ namespace build2 if (s == nullptr) fail << "config.save() called out of project" << endf; + // See save_config() for details. + // + assert (s->ctx.phase == run_phase::load); + const module* mod (s->find_module<module> (module::name)); + + if (mod == nullptr) + fail << "config.save() called without config module"; + ostringstream os; // Empty project set should is ok as long as inherit is false. @@ -47,6 +108,7 @@ namespace build2 save_config (*s, os, path_name ("config.save()"), false /* inherit */, + *mod, ps); return os.str (); diff --git a/libbuild2/config/host-config.cxx.in b/libbuild2/config/host-config.cxx.in index 9e3e0c2..6b1ce77 100644 --- a/libbuild2/config/host-config.cxx.in +++ b/libbuild2/config/host-config.cxx.in @@ -9,5 +9,8 @@ namespace build2 // extern const char host_config[] = R"###($host_config$)###"; extern const char build2_config[] = R"###($build2_config$)###"; + + extern const char host_config_no_warnings[] = R"###($host_config_no_warnings$)###"; + extern const char build2_config_no_warnings[] = R"###($build2_config_no_warnings$)###"; } } diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx index 1513a47..2f134c4 100644 --- a/libbuild2/config/init.cxx +++ b/libbuild2/config/init.cxx @@ -4,6 +4,7 @@ #include <libbuild2/config/init.hxx> #include <sstream> +#include <cstdlib> // getenv() #include <libbuild2/file.hxx> #include <libbuild2/rule.hxx> @@ -25,9 +26,66 @@ namespace build2 { namespace config { + static const file_rule file_rule_ (true /* check_type */); + void functions (function_map&); // functions.cxx + // Custom save function for config.config.environment. + // + // It tries to optimize the storage in subprojects by appending the + // difference (compared to the amalgamation's values) instead of storing + // the entire values. + // + static pair<names_view, const char*> + save_environment (const value& d, const value* b, names& storage) + { + if (b == nullptr) + return make_pair (reverse (d, storage, true /* reduce */), "="); + + // The plan is to iterator over environment variables adding those that + // are not in base to storage. There is, however, a complication: we may + // see multiple entries for the same variable and only the last entry + // should have effect. So we need to iterate in reverse and check if + // we've already seen this variable. + // + const strings& ds (d.as<strings> ()); + const strings& bs (b->as<strings> ()); + + for (auto i (ds.rbegin ()), e (ds.rend ()); i != e; ++i) + { + // Note that we must only consider variable names (up to first '=' if + // any). + // + const string& v (*i); + size_t p (v.find ('=')); + + // Check if we have already seen this variable. + // + if (find_if (ds.rbegin (), i, + [&v, p] (const string& v1) + { + return saved_environment::compare ( + v, v1, p, v1.find ('=')); + }) != i) + continue; + + // Check if there is the same value in base. + // + auto j (find_if (bs.rbegin (), bs.rend (), + [&v, p] (const string& v1) + { + return saved_environment::compare ( + v, v1, p, v1.find ('=')); + })); + + if (j == bs.rend () || *j != v) + storage.push_back (name (v)); + } + + return make_pair (names_view (storage), "+="); + } + void boot (scope& rs, const location&, module_boot_extra& extra) { @@ -44,7 +102,10 @@ namespace build2 // reserved to not be valid module names (`build`). We also currently // treat `import` as special. // - auto& vp (rs.var_pool ()); + // All the variables we enter are qualified so go straight for the + // public variable pool. + // + auto& vp (rs.var_pool (true /* public */)); // NOTE: all config.** variables are by default made (via a pattern) to // be overridable with global visibility. So we must override this if a @@ -57,7 +118,7 @@ namespace build2 // as a command line override. // // Note: must be entered during bootstrap since we need it in - // configure_execute(). + // configure_execute() even for the forward case. // vp.insert<path> ("config.config.save", true /* ovr */); @@ -93,8 +154,11 @@ namespace build2 // // Use the NULL value to clear. // - auto& c_p (vp.insert<vector<pair<string, string>>> ( - "config.config.persist", true /* ovr */, v_p)); + // Note: must be entered during bootstrap since we need it in create's + // save_config(). + // + vp.insert<vector<pair<string, string>>> ( + "config.config.persist", true /* ovr */, v_p); // Only create the module if we are configuring, creating, or // disfiguring or if it was requested with config.config.module (useful @@ -116,17 +180,11 @@ namespace build2 if (!d) { - // Used as a variable prefix by configure_execute(). - // - vp.insert ("config"); - // Adjust priority for the config module and import pseudo-module so // that their variables come first in config.build. // m.save_module ("config", INT32_MIN); m.save_module ("import", INT32_MIN); - - m.save_variable (c_p, save_null_omitted); } } @@ -152,6 +210,9 @@ namespace build2 #ifndef BUILD2_BOOTSTRAP extern const char host_config[]; extern const char build2_config[]; + + extern const char host_config_no_warnings[]; + extern const char build2_config_no_warnings[]; #endif bool @@ -170,20 +231,165 @@ namespace build2 return true; } + context& ctx (rs.ctx); + l5 ([&]{trace << "for " << rs;}); - auto& vp (rs.var_pool ()); + // If we have the module, then we are configuring (or the project wishes + // to call $config.save(); we don't get here on disfigure). + // + module* m (extra.module != nullptr + ? &extra.module_as<module> () + : nullptr); + + auto& vp (rs.var_pool (true /* public */)); // Note: config.* is pattern-typed to global visibility. // const auto v_p (variable_visibility::project); - auto& c_l (vp.insert<paths> ("config.config.load", true /* ovr */)); auto& c_v (vp.insert<uint64_t> ("config.version", false /*ovr*/, v_p)); + auto& c_l (vp.insert<paths> ("config.config.load", true /* ovr */)); + + // Omit loading the configuration from the config.build file (it is + // still loaded from config.config.load if specified). Similar to + // config.config.load, only values specified on this project's root + // scope and global scope are considered. + // + // Note that this variable is not saved in config.build and is expected + // to always be specified as a command line override. + // + auto& c_u (vp.insert<bool> ("config.config.unload", true /*ovr*/)); - // Load config.build if one exists followed by extra files specified in - // config.config.load (we don't need to worry about disfigure since we - // will never be init'ed). + // Configuration variables to disfigure. + // + // The exact semantics is to ignore these variables when loading + // config.build (and any files specified in config.config.load), letting + // them to take on the default values (more precisely, the current + // implementation undefined them after loading config.build). See also + // config.config.unload. + // + // Besides names, variables can also be specified as patterns in the + // config.<prefix>.(*|**)[<suffix>] form where `*` matches single + // component names (i.e., `foo` but not `foo.bar`), and `**` matches + // single and multi-component names. Currently only single wildcard (`*` + // or `**`) is supported. Additionally, a pattern in the + // config.<prefix>(*|**) form (i.e., without `.` after <prefix>) matches + // config.<prefix>.(*|**) plus config.<prefix> itself (but not + // config.<prefix>foo). + // + // For example, to disfigure all the project configuration variables + // (while preserving all the module configuration variables; note + // quoting to prevent pattern expansion): + // + // b config.config.disfigure="'config.hello**'" + // + // Note that this variable is not saved in config.build and is expected + // to always be specified as a command line override. + // + // We also had the idea of using NULL values as a more natural way to + // undefine a configuration variable, which would only work for non- + // nullable variables (such as project configuration variables) or for + // those where NULL is the default value (most of the others). However, + // this cannot work in our model since we cannot reset a NULL override + // to a default value. So setting the variable itself to some special + // value does not seem to be an option and we have to convey this in + // some other way, such as in config.config.disfigure. Another idea is + // to invent a parallel set of variables, such as disfig.*, that can be + // used for that (though they would still have to be specified with some + // dummy value, for example disfig.hello.fancy=). On the other hand, + // this desire to disfigure individual variables does not seem to be + // very common (we lived without it for years without noticing). So + // it's not clear we need to do something like disfig.* which has a + // wiff of hack to it. + // + auto& c_d (vp.insert<strings> ("config.config.disfigure", true /*ovr*/)); + + // Hermetic configurations. + // + // A hermetic configuration stores environment variables that affect the + // project in config.config.environment. + // + // Note that this is essentially a tri-state value: true means keep + // hermetizing (save the environment in config.config.environment), + // false means keep de-hermetizing (clear config.config.environment) and + // undefined/NULL means don't touch config.config.environment. + // + // During reconfiguration things stay hermetic unless re-hermetization + // is explicitly requested with config.config.hermetic.reload=true (or + // de-hermetization is requested with config.config.hermetic=false). + // + // Use the NULL value to clear. + // + auto& c_h (vp.insert<bool> ("config.config.hermetic", true /* ovr */)); + + if (m != nullptr) + m->save_variable (c_h, save_null_omitted); + + // Request hermetic configuration re-hermetization. + // + // Note that this variable is not saved in config.build and is expected + // to always be specified as a command line override. + // + auto& c_h_r ( + vp.insert<bool> ("config.config.hermetic.reload", true /* ovr */)); + + // Hermetic configuration environment variables inclusion/exclusion. + // + // This configuration variable can be used to include additional or + // exclude existing environment variables into/from the list that should + // be saved in order to make the configuration hermetic. For example: + // + // config.config.hermetic.environment="LANG PATH@false" + // + // Use the NULL or empty value to clear. + // + auto& c_h_e ( + vp.insert<hermetic_environment> ("config.config.hermetic.environment")); + + if (m != nullptr) + m->save_variable (c_h_e, save_null_omitted | save_empty_omitted); + + // Configuration environment variables. + // + // Environment variables used by tools (e.g., compilers), buildfiles + // (e.g., $getenv()), and the build system itself (e.g., to locate + // tools) in ways that affect the build result are in essence part of + // the project configuration. + // + // This variable allows storing environment variable overrides that + // should be applied to the environment when executing tools, etc., as + // part of a project build. Specifically, it contains a list of + // environment variable "sets" (<name>=<value>) and "unsets" (<name>). + // If multiple entries are specified for the same environment variable, + // the last entry has effect. For example: + // + // config.config.environment="LC_ALL=C LANG" + // + // Note that a subproject inherits overrides from its amalgamation (this + // semantics is the result of the way we optimize the storage of this + // variable in subproject's config.build; the thinking is that if a + // variable is not overridden by the subproject then it doesn't affect + // the build result and therefore it's irrelevant whether it has a value + // that came from the original environment of from the amalgamation + // override). + // + // Use the NULL or empty value to clear. + // + // @@ We could use =<name> as a "pass-through" instruction (e.g., if + // we need to use original value in subproject). + // + auto& c_e (vp.insert<strings> ("config.config.environment", true /* ovr */)); + + if (m != nullptr) + m->save_variable (c_e, + save_null_omitted | save_empty_omitted | save_base, + &save_environment); + + // Load config.build if one exists (and unless config.config.unload is + // specified) followed by extra files specified in config.config.load + // (we don't need to worry about disfigure since we will never be + // init'ed). // auto load_config = [&rs, &c_v] (istream& is, const path_name& in, @@ -228,15 +434,37 @@ namespace build2 auto load_config_file = [&load_config] (const path& f, const location& l) { path_name fn (f); - ifdstream ifs; - load_config (open_file_or_stdin (fn, ifs), fn, l); + try + { + ifdstream ifs; + load_config (open_file_or_stdin (fn, ifs), fn, l); + } + catch (const io_error& e) + { + fail << "unable to read buildfile " << fn << ": " << e; + } }; + // Load config.build unless requested not to. + // { - path f (config_file (rs)); + // The same semantics as in config.config.load below. + // + bool u; + { + lookup l (rs[c_u]); + u = (l && + (l.belongs (rs) || l.belongs (ctx.global_scope)) && + cast_false<bool> (l)); + } + + if (!u) + { + path f (config_file (rs)); - if (exists (f)) - load_config_file (f, l); + if (exists (f)) + load_config_file (f, l); + } } if (lookup l = rs[c_l]) @@ -251,7 +479,7 @@ namespace build2 // b ./config.config.load=.../config.build # this project // b !config.config.load=.../config.build # every project // - if (l.belongs (rs) || l.belongs (rs.ctx.global_scope)) + if (l.belongs (rs) || l.belongs (ctx.global_scope)) { for (const path& f: cast<paths> (l)) { @@ -259,14 +487,23 @@ namespace build2 const string& s (f.string ()); - if (s[0] != '~') + if (s.empty ()) + fail << "empty path in config.config.load"; + else if (s[0] != '~') load_config_file (f, l); - else if (s == "~host" || s == "~build2") + else if (s == "~host" || s == "~host-no-warnings" || + s == "~build2" || s == "~build2-no-warnings") { #ifdef BUILD2_BOOTSTRAP assert (false); #else - istringstream is (s[1] == 'h' ? host_config : build2_config); + istringstream is (s[1] == 'h' + ? (s.size () == 5 + ? host_config + : host_config_no_warnings) + : (s.size () == 7 + ? build2_config + : build2_config_no_warnings)); load_config (is, path_name (s), l); #endif } @@ -277,33 +514,235 @@ namespace build2 } } - // Cache the config.config.persist value, if any. + // Undefine variables specified with config.config.disfigure. // - if (extra.module != nullptr) + if (const strings* ns = cast_null<strings> (rs[c_d])) { - auto& m (extra.module_as<module> ()); + auto p (rs.vars.lookup_namespace ("config")); + + for (auto i (p.first); i != p.second; ) + { + const variable& var (i->first); + + // This can be one of the overrides (__override, __prefix, etc), + // which we skip. + // + if (!var.override ()) + { + bool m (false); + + for (const string& n: *ns) + { + if (n.compare (0, 7, "config.") != 0) + fail << "config.* variable expected in " + << "config.config.disfigure instead of '" << n << "'"; + + size_t p (n.find ('*')); + + if (p == string::npos) + { + if ((m = var.name == n)) + break; + } + else + { + // Pattern in one of these forms: + // + // config.<prefix>.(*|**)[<suffix>] + // config.<prefix>(*|**) + // + // BTW, an alternative way to handle this would be to + // translate it to a path and use our path_match() machinery, + // similar to how we do it for build config include/exclude. + // Perhaps one day when/if we decide to support multiple + // wildcards. + // + if (p == 7) + fail << "config.<prefix>* pattern expected in " + << "config.config.disfigure instead of '" << n << "'"; + + bool r (n[p + 1] == '*'); // Recursive. + + size_t pe; // Prefix end/size. + if (n[p - 1] != '.') + { + // Second form should have no suffix. + // + if (p + (r ? 2 : 1) != n.size ()) + fail << "config.<prefix>(*|**) pattern expected in " + << "config.config.disfigure instead of '" << n << "'"; + + // Match just <prefix>. + // + if ((m = n.compare (0, p, var.name) == 0)) + break; + + pe = p; + } + else + pe = p - 1; + + // Match <prefix> followed by `.`. + // + if (n.compare (0, pe, var.name, 0, pe) != 0 || + var.name[pe] != '.') + continue; + + // Match suffix. + // + size_t sb (p + (r ? 2 : 1)); // Suffix begin. + size_t sn (n.size () - sb); // Suffix size. + + size_t te; // Stem end. + if (sn == 0) // No suffix. + te = var.name.size (); + else + { + if (var.name.size () < pe + 1 + sn) // Too short. + continue; + + te = var.name.size () - sn; + + if (n.compare (sb, sn, var.name, te, sn) != 0) + continue; + } + + // Match stem. + // + if ((m = r || var.name.find ('.', pe + 1) >= te)) + break; + } + } + + if (m) + { + i = rs.vars.erase (i); // Undefine. + continue; + } + } - m.persist = - cast_null<vector<pair<string, string>>> ( - rs["config.config.persist"]); + ++i; + } } - // Register alias and fallback rule for the configure meta-operation. + // Save and cache the config.config.persist value, if any. // - // We need this rule for out-of-any-project dependencies (e.g., - // libraries imported from /usr/lib). We are registring it on the - // global scope similar to builtin rules. + if (m != nullptr) + { + auto& c_p (*vp.find ("config.config.persist")); + m->save_variable (c_p, save_null_omitted); + m->persist = cast_null<vector<pair<string, string>>> (rs[c_p]); + } + + // If we are configuring, handle config.config.hermetic. + // + // The overall plan is to either clear config.config.environment (if + // c.c.h=false) or populate it with the values that affect this project + // (if c.c.h=true). We have to do it half here (because c.c.e is used as + // a source for the project environment and we would naturally want the + // semantics to be equivalent to what will be saved in config.build) and + // half in configure_execute() (because that's where we have the final + // list of all the environment variables we need to save). + // + // So here we must deal with the cases where the current c.c.e value + // will be changed: either cleared (c.c.h=false) or set to new values + // from the "outer" environment (c.c.h.reload=true). Note also that even + // then a c.c.e value from an amalgamation, if any, should be in effect. + // + if (ctx.current_mif->id == configure_id && + (!cast_true<bool> (rs[c_h]) || // c.c.h=false + cast_false<bool> (rs[c_h_r]))) // c.c.h.r=true + { + rs.vars.erase (c_e); // Undefine. + } + + // Copy config.config.environment to scope::root_extra::environment and + // calculate its checksum. // - rs.global_scope ().insert_rule<mtime_target> ( - configure_id, 0, "config.file", file_rule::instance); + // Note that we store shallow copies that point to the c.c.environment + // value which means it should not change. + // + if (const strings* src = cast_null<strings> (rs[c_e])) + { + sha256 cs; + vector<const char*>& dst (rs.root_extra->environment); + + // The idea is to only copy entries that are effective, that is those + // that actually override something in the environment. This should be + // both more efficient and less noisy (e.g., if we need to print this + // in diagnostics). + // + // Note that config.config.environment may contain duplicates and the + // last entry should have effect. + // + // Note also that we use std::getenv() instead of butl::getenv() to + // disregard any thread environment overrides. + // + for (auto i (src->rbegin ()), e (src->rend ()); i != e; ++i) + { + // Note that we must only consider variable names (up to first '=' + // if any). + // + const string& v (*i); + size_t p (v.find ('=')); + + // Check if we have already seen this variable. + // + if (find_if (src->rbegin (), i, + [&v, p] (const string& v1) + { + return saved_environment::compare ( + v, v1, p, v1.find ('=')); + }) != i) + continue; + + // If it's an unset, see if it actually unsets anything. + // + if (p == string::npos) + { + if (std::getenv (v.c_str ()) == nullptr) + continue; + } + // + // And if it's a set, see if it sets a different value. + // + else + { + const char* v1 (std::getenv (string (v, 0, p).c_str ())); + if (v1 != nullptr && v.compare (p + 1, string::npos, v1) == 0) + continue; + } + + dst.push_back (v.c_str ()); + cs.append (v); + } + + if (!dst.empty ()) + { + dst.push_back (nullptr); + rs.root_extra->environment_checksum = cs.string (); + } + } - //@@ outer + // Register alias and fallback rule for the configure meta-operation. + // rs.insert_rule<alias> (configure_id, 0, "config.alias", alias_rule::instance); // This allows a custom configure rule while doing nothing by default. // - rs.insert_rule<target> (configure_id, 0, "config", noop_rule::instance); - rs.insert_rule<file> (configure_id, 0, "config.file", noop_rule::instance); + rs.insert_rule<target> (configure_id, 0, "config.noop", noop_rule::instance); + + // We need this rule for out-of-any-project dependencies (for example, + // libraries imported from /usr/lib). We are registering it on the + // global scope similar to builtin rules. + // + // Note: use target instead of anything more specific (such as + // mtime_target) in order not to take precedence over the rules above. + // + // See a similar rule in the dist module. + // + rs.global_scope ().insert_rule<target> ( + configure_id, 0, "config.file", file_rule_); return true; } @@ -320,6 +759,7 @@ namespace build2 // Initialize the config entry points in the build system core. // config_save_variable = &module::save_variable; + config_save_environment = &module::save_environment; config_save_module = &module::save_module; config_preprocess_create = &preprocess_create; config_configure_post = &module::configure_post; diff --git a/libbuild2/config/module.cxx b/libbuild2/config/module.cxx index bf68c4e..713d30c 100644 --- a/libbuild2/config/module.cxx +++ b/libbuild2/config/module.cxx @@ -12,7 +12,9 @@ namespace build2 namespace config { bool module:: - save_variable (const variable& var, optional<uint64_t> flags) + save_variable (const variable& var, + optional<uint64_t> flags, + save_variable_function* save) { const string& n (var.name); @@ -43,7 +45,7 @@ namespace build2 return false; } - sv.push_back (saved_variable {var, flags}); + sv.push_back (saved_variable {var, flags, save}); return true; } @@ -54,6 +56,13 @@ namespace build2 m->save_variable (var, flags); } + void module:: + save_environment (scope& rs, const char* var) + { + if (module* m = rs.find_module<module> (module::name)) + m->save_environment (var); + } + bool module:: save_module (const char* name, int prio) { diff --git a/libbuild2/config/module.hxx b/libbuild2/config/module.hxx index 857a30c..8d3ff67 100644 --- a/libbuild2/config/module.hxx +++ b/libbuild2/config/module.hxx @@ -4,9 +4,9 @@ #ifndef LIBBUILD2_CONFIG_MODULE_HXX #define LIBBUILD2_CONFIG_MODULE_HXX -#include <map> +#include <cstring> // strncmp() -#include <libbutl/prefix-map.mxx> +#include <libbutl/prefix-map.hxx> #include <libbuild2/types.hxx> #include <libbuild2/utility.hxx> @@ -28,10 +28,19 @@ namespace build2 // saved in the order populated. If flags are absent, then this variable // was marked as "unsaved" (always transient). // + // The optional save function can be used to implement custom variable + // saving, for example, as a difference appended to the base value. The + // second half of the result is the assignment operator to use. + // + using save_variable_function = + pair<names_view, const char*> (const value&, + const value* base, + names& storage); struct saved_variable { reference_wrapper<const variable> var; optional<uint64_t> flags; + save_variable_function* save; }; struct saved_variables: vector<saved_variable> @@ -43,7 +52,7 @@ namespace build2 const_iterator find (const variable& var) const { - return std::find_if ( + return find_if ( begin (), end (), [&var] (const saved_variable& v) {return var == v.var;}); @@ -55,7 +64,7 @@ namespace build2 // Priority order with INT32_MIN being the highest. Modules with the // same priority are saved in the order inserted. // - std::multimap<std::int32_t, const_iterator> order; + multimap<std::int32_t, const_iterator> order; pair<iterator, bool> insert (string name, int prio = 0) @@ -69,14 +78,77 @@ namespace build2 } }; - struct module: build2::module + // List of environment variable names that effect this project. + // + // Note that on Windows environment variable names are case-insensitive. + // + struct saved_environment: vector<string> + { + // Compare environment variable names. + // + static inline bool + compare (const string& x, + const string& y, + size_t xn = string::npos, + size_t yn = string::npos) + { + if (xn == string::npos) xn = x.size (); + if (yn == string::npos) yn = y.size (); + + return xn == yn && +#ifdef _WIN32 + icasecmp (x.c_str (), y.c_str (), xn) == 0 +#else + strncmp (x.c_str (), y.c_str (), xn) == 0 +#endif + ; + } + + iterator + find (const string& v) + { + return find_if ( + begin (), + end (), + [&v] (const string& v1) {return compare (v, v1);}); + } + + const_iterator + find (const string& v) const + { + return find_if ( + begin (), + end (), + [&v] (const string& v1) {return compare (v, v1);}); + } + + void + insert (string v) + { + if (find (v) == end ()) + push_back (move (v)); + } + + void + erase (const string& v) + { + auto i (find (v)); + if (i != end ()) + vector<string>::erase (i); + } + }; + + class module: public build2::module { + public: config::saved_modules saved_modules; // Return true if variable/module were newly inserted. // bool - save_variable (const variable&, optional<uint64_t> flags); + save_variable (const variable&, + optional<uint64_t> flags, + save_variable_function* = nullptr); static void save_variable (scope&, const variable&, optional<uint64_t>); @@ -88,7 +160,7 @@ namespace build2 save_module (scope&, const char*, int); const saved_variable* - find_variable (const variable& var) + find_variable (const variable& var) const { auto i (saved_modules.find_sup (var.name)); if (i != saved_modules.end ()) @@ -101,6 +173,18 @@ namespace build2 return nullptr; } + void + save_environment (const char* var) + { + saved_environment.insert (var); + } + + static void + save_environment (scope&, const char*); + + config::saved_environment saved_environment; + strings old_environment; + // Configure/disfigure hooks. // static bool diff --git a/libbuild2/config/operation.cxx b/libbuild2/config/operation.cxx index b9856be..150bf1a 100644 --- a/libbuild2/config/operation.cxx +++ b/libbuild2/config/operation.cxx @@ -42,7 +42,7 @@ namespace build2 ofs << "# Created automatically by the config module." << endl << "#" << endl << "src_root = "; - to_stream (ofs, name (src_root), true /* quote */, '@'); + to_stream (ofs, name (src_root), quote_mode::normal, '@'); ofs << endl; ofs.close (); @@ -61,8 +61,10 @@ namespace build2 path f (src_root / rs.root_extra->out_root_file); - if (verb) - text << (verb >= 2 ? "cat >" : "save ") << f; + if (verb >= 2) + text << "cat >" << f; + else if (verb) + print_diag ("save", f); try { @@ -71,7 +73,7 @@ namespace build2 ofs << "# Created automatically by the config module." << endl << "#" << endl << "out_root = "; - to_stream (ofs, name (out_root), true /* quote */, '@'); + to_stream (ofs, name (out_root), quote_mode::normal, '@'); ofs << endl; ofs.close (); @@ -82,8 +84,6 @@ namespace build2 } } - using project_set = set<const scope*>; // Use pointers to get comparison. - // Return (first) whether an unused/inherited variable should be saved // according to the config.config.persist value and (second) whether the // user should be warned about it. @@ -134,7 +134,8 @@ namespace build2 bool r; if (c.compare (p, 4 , "save") == 0) r = true; else if (c.compare (p, 4 , "drop") == 0) r = false; - else fail << "invalid config.config.persist action '" << c << "'"; + else fail << "invalid config.config.persist action '" << c << "'" + << endf; bool w (false); if ((p += 4) != c.size ()) @@ -159,24 +160,26 @@ namespace build2 // If inherit is false, then don't rely on inheritance from outer scopes. // + // @@ We are modifying the module (marking additional variables as saved) + // and this function can be called from a buildfile (probably only + // during serial execution but still). + // + // We could also be configuring multiple projects (including from + // pkg_configure() in bpkg) but feels like we should be ok since we + // only modify this project's root scope data which should not affect + // any other project. + // + // See also save_environment() for a similar issue. + // void save_config (const scope& rs, ostream& os, const path_name& on, bool inherit, + const module& mod, const project_set& projects) { context& ctx (rs.ctx); - // @@ We are modifying the module (marking additional variables as - // saved) and this function can be called from a buildfile (probably - // only during serial execution but still). - // - module* mod (rs.find_module<module> (module::name)); - - if (mod == nullptr) - fail (on) << "no configuration information available during this " - << "meta-operation"; - names storage; auto info_value = [&storage] (diag_record& dr, const value& v) mutable @@ -186,7 +189,7 @@ namespace build2 if (v) { storage.clear (); - dr << "'" << reverse (v, storage) << "'"; + dr << "'" << reverse (v, storage, true /* reduce */) << "'"; } else dr << "[null]"; @@ -214,9 +217,11 @@ namespace build2 // saved according to config.config.persist potentially warning if the // variable would otherwise be dropped. // + // Note: go straight for the public variable pool. + // auto& vp (ctx.var_pool); - for (auto p (rs.vars.lookup_namespace (*vp.find ("config"))); + for (auto p (rs.vars.lookup_namespace ("config")); p.first != p.second; ++p.first) { @@ -228,26 +233,59 @@ namespace build2 if (size_t n = var->override ()) var = vp.find (string (var->name, 0, n)); + const string& name (var->name); + // Skip special variables. // - if (var->name == "config.booted" || - var->name == "config.loaded" || - var->name == "config.configured" || - var->name.compare (0, 14, "config.config.") == 0) + if (name == "config.booted" || + name == "config.loaded" || + name == "config.configured" || + name.compare (0, 14, "config.config.") == 0) continue; - if (mod->find_variable (*var)) // Saved or unsaved. + if (mod.find_variable (*var)) // Saved or unsaved. continue; + // Skip config.**.develop variables (see parser::parse_config() for + // details). + // + // In a sense, this variable is always "available" but if the + // package does not distinguish between development and consumption, + // then specifying config.*.develop=true should be noop. + // + { + size_t p (name.rfind ('.')); + if (p != 6 && name.compare (p + 1, string::npos, "develop") == 0) + continue; + } + + // A common reason behind an unused config.import.* value is an + // unused dependency. That is, there is depends in manifest but no + // import in buildfile (or import could be conditional in which case + // depends should also be conditional). So let's suggest this + // possibility. Note that the project name may have been sanitized + // to a variable name. Oh, well, better than nothing. + // + auto info_import = [] (diag_record& dr, const string& var) + { + if (var.compare (0, 14, "config.import.") == 0) + { + size_t p (var.find ('.', 14)); + + dr << info << "potentially unused dependency on " + << string (var, 14, p == string::npos ? p : p - 14); + } + }; + const value& v (p.first->second); pair<bool, bool> r (save_config_variable (*var, - mod->persist, + mod.persist, false /* inherited */, true /* unused */)); if (r.first) // save { - mod->save_variable (*var, 0); + const_cast<module&> (mod).save_variable (*var, 0); if (r.second) // warn { @@ -266,6 +304,7 @@ namespace build2 diag_record dr; dr << warn (on) << "saving no longer used variable " << *var; + info_import (dr, var->name); if (verb >= 2) info_value (dr, v); } @@ -276,6 +315,7 @@ namespace build2 { diag_record dr; dr << warn (on) << "dropping no longer used variable " << *var; + info_import (dr, var->name); info_value (dr, v); } } @@ -283,12 +323,23 @@ namespace build2 // Save config variables. // - for (auto p: mod->saved_modules.order) + for (auto p: mod.saved_modules.order) { const string& sname (p.second->first); const saved_variables& svars (p.second->second); - bool first (true); // Separate modules with a blank line. + // Separate modules with a blank line. + // + auto first = [v = true] () mutable + { + if (v) + { + v = false; + return "\n"; + } + return ""; + }; + for (const saved_variable& sv: svars) { if (!sv.flags) // unsaved @@ -308,7 +359,13 @@ namespace build2 // inherited. We might also not have any value at all (see // unconfigured()). // - if (!l.defined () || (l->null && flags & save_null_omitted)) + // Note that we must check for null() before attempting any + // further tests. + // + if (!l.defined () || + (l->null ? flags & save_null_omitted : + l->empty () ? flags & save_empty_omitted : + (flags & save_false_omitted) != 0 && !cast<bool> (*l))) continue; // Handle inherited from outer scope values. @@ -317,90 +374,97 @@ namespace build2 // we save the inherited values regardless of whether they are // used or not. // - if (inherit && !(l.belongs (rs) || l.belongs (ctx.global_scope))) + const value* base (nullptr); + if (inherit) { - // This is presumably an inherited value. But it could also be - // some left-over garbage. For example, an amalgamation could - // have used a module but then dropped it while its config - // values are still lingering in config.build. They are probably - // still valid and we should probably continue using them but we - // definitely want to move them to our config.build since they - // will be dropped from the amalgamation's config.build on the - // next reconfigure. Let's also warn the user just in case, - // unless there is no module and thus we couldn't really check - // (the latter could happen when calling $config.save() during - // other meta-operations, though it passes false for inherit). + // Return true if the specified value can be inherited from. // - // There is also another case that falls under this now that - // overrides are by default amalgamation-wide rather than just - // "project and subprojects": we may be (re-)configuring a - // subproject but the override is now set on the outer project's - // root. - // - bool found (false), checked (true); - const scope* r (&rs); - while ((r = r->parent_scope ()->root_scope ()) != nullptr) + auto find_inherited = [&on, &projects, + &info_value, + &sname, &rs, &var] (const lookup& org, + const lookup& ovr) { - if (l.belongs (*r)) + const lookup& l (ovr); + + // This is presumably an inherited value. But it could also be + // some left-over garbage. For example, an amalgamation could + // have used a module but then dropped it while its config + // values are still lingering in config.build. They are + // probably still valid and we should probably continue using + // them but we definitely want to move them to our + // config.build since they will be dropped from the + // amalgamation's config.build on the next reconfigure. Let's + // also warn the user just in case, unless there is no module + // and thus we couldn't really check (the latter could happen + // when calling $config.save() during other meta-operations, + // though it passes false for inherit). + // + // There is also another case that falls under this now that + // overrides are by default amalgamation-wide rather than just + // "project and subprojects": we may be (re-)configuring a + // subproject but the override is now set on the outer + // project's root. + // + bool found (false), checked (true); + const scope* r (&rs); + while ((r = r->parent_scope ()->root_scope ()) != nullptr) { - // Find the config module (might not be there). - // - if (auto* m = r->find_module<const module> (module::name)) + if (l.belongs (*r)) { - // Find the corresponding saved module. + // Find the config module (might not be there). // - auto i (m->saved_modules.find (sname)); - - if (i != m->saved_modules.end ()) + if (auto* m = r->find_module<const module> (module::name)) { - // Find the variable. + // Find the corresponding saved module. // - const saved_variables& sv (i->second); - found = sv.find (var) != sv.end (); + auto i (m->saved_modules.find (sname)); - // If not marked as saved, check whether overriden via - // config.config.persist. - // - if (!found && m->persist != nullptr) + if (i != m->saved_modules.end ()) { - found = save_config_variable ( - var, - m->persist, - false /* inherited */, - true /* unused */).first; + // Find the variable. + // + const saved_variables& sv (i->second); + found = sv.find (var) != sv.end (); + + // If not marked as saved, check whether overriden via + // config.config.persist. + // + if (!found && m->persist != nullptr) + { + found = save_config_variable ( + var, + m->persist, + false /* inherited */, + true /* unused */).first; + } + + // Handle that other case: if this is an override but + // the outer project itself is not being configured, + // then we need to save this override. + // + // One problem with using the already configured + // project set is that the outer project may be + // configured only after us in which case both + // projects will save the value. But perhaps this is a + // feature, not a bug since this is how project-local + // (%) override behaves. + // + if (found && + org != ovr && + projects.find (r) == projects.end ()) + found = false; } - - // Handle that other case: if this is an override but - // the outer project itself is not being configured, - // then we need to save this override. - // - // One problem with using the already configured project - // set is that the outer project may be configured only - // after us in which case both projects will save the - // value. But perhaps this is a feature, not a bug since - // this is how project-local (%) override behaves. - // - if (found && - org.first != ovr.first && - projects.find (r) == projects.end ()) - found = false; } - } - else - checked = false; + else + checked = false; - break; + break; + } } - } - if (found) - { - // Inherited. - // - continue; - } - else - { + if (found) + return true; + // If this value is not defined in a project's root scope, // then something is broken. // @@ -413,7 +477,7 @@ namespace build2 // our own. One special case where we don't want to warn the // user is if the variable is overriden. // - if (checked && org.first == ovr.first) + if (checked && org == ovr) { diag_record dr; dr << warn (on) << "saving previously inherited variable " @@ -425,6 +489,39 @@ namespace build2 if (verb >= 2) info_value (dr, *l); } + + return false; + }; + + // Inherit as-is. + // + if (!l.belongs (rs) && + !l.belongs (ctx.global_scope) && + find_inherited (org.first, ovr.first)) + continue; + else if (flags & save_base) + { + // See if we can base our value on inherited. + // + if (const scope* ors = rs.parent_scope ()->root_scope ()) + { + pair<lookup, size_t> org (ors->lookup_original (var)); + pair<lookup, size_t> ovr (var.overrides == nullptr + ? org + : ors->lookup_override (var, org)); + const lookup& l (ovr.first); + + // We cannot base anything on an empty value. + // + if (l && !l->empty ()) + { + // @@ It's not clear we want the checks/diagnostics in + // this case. + // + if (find_inherited (org.first, ovr.first)) + base = l.value; + } + } } } @@ -442,44 +539,42 @@ namespace build2 continue; } - // If we got here then we are saving this variable. Handle the - // blank line. - // - if (first) - { - os << endl; - first = false; - } - // Handle the save_default_commented flag. // - if ((org.first.defined () && org.first->extra) && // Default value. - org.first == ovr.first && // Not overriden. + if (org.first.defined () && org.first->extra == 1 && // Default. + org.first == ovr.first && // No override. (flags & save_default_commented) != 0) { - os << '#' << n << " =" << endl; + os << first () << '#' << n << " =" << endl; continue; } - if (v) + if (v.null) { - storage.clear (); - names_view ns (reverse (v, storage)); + os << first () << n << " = [null]" << endl; + continue; + } - os << n; + storage.clear (); + pair<names_view, const char*> p ( + sv.save != nullptr + ? sv.save (v, base, storage) + : make_pair (reverse (v, storage, true /* reduce */), "=")); - if (ns.empty ()) - os << " ="; - else - { - os << " = "; - to_stream (os, ns, true /* quote */, '@'); - } + // Might becomes empty after a custom save function had at it. + // + if (p.first.empty () && (flags & save_empty_omitted)) + continue; + + os << first () << n << ' ' << p.second; - os << endl; + if (!p.first.empty ()) + { + os << ' '; + to_stream (os, p.first, quote_mode::normal, '@'); } - else - os << n << " = [null]" << endl; + + os << endl; } } } @@ -493,6 +588,7 @@ namespace build2 save_config (const scope& rs, const path& f, bool inherit, + const module& mod, const project_set& projects) { path_name fn (f); @@ -500,13 +596,16 @@ namespace build2 if (f.string () == "-") fn.name = "<stdout>"; - if (verb) - text << (verb >= 2 ? "cat >" : "save ") << fn; + if (verb >= 2) + text << "cat >" << fn; + else if (verb) + print_diag ("save", fn); try { ofdstream ofs; - save_config (rs, open_file_or_stdout (fn, ofs), fn, inherit, projects); + save_config ( + rs, open_file_or_stdout (fn, ofs), fn, inherit, mod, projects); ofs.close (); } catch (const io_error& e) @@ -515,10 +614,86 @@ namespace build2 } } + // Update config.config.environment value for a hermetic configuration. + // + // @@ We are modifying the module. See also save_config() for a similar + // issue. + // + static void + save_environment (scope& rs, module& mod) + { + // Here we have two parts: (1) get the list of environment variables we + // need to save and (2) save their values in config.config.environment. + // + // The saved_environment list should by now contain all the project- + // specific environment variables. To that we add builtin defaults and + // then filter the result against config.config.hermetic.environment + // inclusions/exclusions. + // + auto& vars (mod.saved_environment); + + vars.insert ("PATH"); + +#if defined(_WIN32) +#elif defined(__APPLE__) + vars.insert ("DYLD_LIBRARY_PATH"); +#else // Linux, FreeBSD, NetBSD, OpenBSD + vars.insert ("LD_LIBRARY_PATH"); +#endif + + for (const pair<string, optional<bool>>& p: + cast_empty<hermetic_environment> ( + rs["config.config.hermetic.environment"])) + { + if (!p.second || *p.second) + vars.insert (p.first); + else + vars.erase (p.first); + } + + // Get the values. + // + strings vals; + { + // Set the project environment before querying the values. Note that + // the logic in init() makes sure that this is all we need to do to + // handle reload (in which case we should still be using + // config.config.environment from amalgamation, if any). + // + auto_project_env penv (rs); + + for (const string& var: vars) + { + if (optional<string> val = getenv (var)) + { + vals.push_back (var + '=' + *val); + } + else + vals.push_back (var); // Unset. + } + } + + // Note: go straight for the public variable pool. + // + value& v (rs.assign (*rs.ctx.var_pool.find ("config.config.environment"))); + + // Note that setting new config.config.environment value invalidates the + // project's environment (scope::root_extra::environment) which could be + // queried in the post-configuration hook. We could re-initialize it but + // the c.c.e value from amalgamation could be referenced by subprojects. + // So instead it seems easier to just save the old value in the module. + // + if (v) + mod.old_environment = move (v.as<strings> ()); + + v = move (vals); + } + static void configure_project (action a, const scope& rs, const variable* c_s, // config.config.save + const module& mod, project_set& projects) { tracer trace ("configure_project"); @@ -538,7 +713,7 @@ namespace build2 // if (out_root != src_root) { - mkdir_p (out_root / rs.root_extra->build_dir); + mkdir_p (out_root / rs.root_extra->build_dir, 1); mkdir (out_root / rs.root_extra->bootstrap_dir, 2); } @@ -548,6 +723,12 @@ namespace build2 { l5 ([&]{trace << "completely configuring " << out_root;}); + // Save the environment if this configuration is hermetic (see init() + // for the other half of this logic). + // + if (cast_false<bool> (rs["config.config.hermetic"])) + save_environment (const_cast<scope&> (rs), const_cast<module&> (mod)); + // Save src-root.build unless out_root is the same as src. // if (c_s == nullptr && out_root != src_root) @@ -572,12 +753,17 @@ namespace build2 // may end up leaving it in partially configured state. // if (c_s == nullptr) - save_config (rs, config_file (rs), true /* inherit */, projects); + save_config (rs, config_file (rs), true /* inherit */, mod, projects); else { lookup l (rs[*c_s]); if (l && (l.belongs (rs) || l.belongs (ctx.global_scope))) { + const path& f (cast<path> (l)); + + if (f.empty ()) + fail << "empty path in " << *c_s; + // While writing the complete configuration seems like a natural // default, there might be a desire to take inheritance into // account (if, say, we are exporting at multiple levels). One can @@ -585,7 +771,7 @@ namespace build2 // still want to support this mode somehow in the future (it seems // like an override of config.config.persist should do the trick). // - save_config (rs, cast<path> (l), false /* inherit */, projects); + save_config (rs, f, false /* inherit */, mod, projects); } } } @@ -596,11 +782,8 @@ namespace build2 if (c_s == nullptr) { - if (module* m = rs.find_module<module> (module::name)) - { - for (auto hook: m->configure_post_) - hook (a, rs); - } + for (auto hook: mod.configure_post_) + hook (a, rs); } // Configure subprojects that have been loaded. @@ -611,15 +794,19 @@ namespace build2 { const dir_path& pd (p.second); dir_path out_nroot (out_root / pd); - const scope& nrs (ctx.scopes.find (out_nroot)); + const scope& nrs (ctx.scopes.find_out (out_nroot)); - // @@ Strictly speaking we need to check whether the config module - // was loaded for this subproject. + // Skip this subproject if it is not loaded or doesn't use the + // config module. // - if (nrs.out_path () != out_nroot) // This subproject not loaded. - continue; + if (nrs.out_path () == out_nroot) + { + if (const module* m = nrs.find_module<module> (module::name)) + { + configure_project (a, nrs, c_s, *m, projects); + } + } - configure_project (a, nrs, c_s, projects); } } } @@ -651,7 +838,7 @@ namespace build2 for (auto p: *ps) { dir_path out_nroot (out_root / p.second); - const scope& nrs (ctx.scopes.find (out_nroot)); + const scope& nrs (ctx.scopes.find_out (out_nroot)); assert (nrs.out_path () == out_nroot); configure_forward (nrs, projects); @@ -662,11 +849,13 @@ namespace build2 operation_id (*pre) (const values&, meta_operation_id, const location&); static operation_id - configure_operation_pre (const values&, operation_id o) + configure_operation_pre (context&, const values&, operation_id o) { // Don't translate default to update. In our case unspecified // means configure everything. // + // Note: see pkg_configure() in bpkg if changing anything here. + // return o; } @@ -701,8 +890,10 @@ namespace build2 } static void - configure_pre (const values& params, const location& l) + configure_pre (context&, const values& params, const location& l) { + // Note: see pkg_configure() in bpkg if changing anything here. + // forward (params, "configure", l); // Validate. } @@ -722,11 +913,13 @@ namespace build2 // create_bootstrap_inner (rs); - if (rs.out_path () == rs.src_path ()) + if (rs.out_eq_src ()) fail (l) << "forwarding to source directory " << rs.src_path (); } else - load (params, rs, buildfile, out_base, src_base, l); // Normal load. + // Normal load. + // + perform_load (params, rs, buildfile, out_base, src_base, l); } static void @@ -746,7 +939,7 @@ namespace build2 ts.push_back (&rs); } else - search (params, rs, bs, bf, tk, l, ts); // Normal search. + perform_search (params, rs, bs, bf, tk, l, ts); // Normal search. } static void @@ -766,6 +959,8 @@ namespace build2 context& ctx (fwd ? ts[0].as<scope> ().ctx : ts[0].as<target> ().ctx); + // Note: go straight for the public variable pool. + // const variable* c_s (ctx.var_pool.find ("config.config.save")); if (c_s->overrides == nullptr) @@ -820,16 +1015,28 @@ namespace build2 ctx.current_operation (*oif); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, true /* inner */, location ()); + phase_lock pl (ctx, run_phase::match); - match (action (configure_id, id), t); + match_sync (action (configure_id, id), t); + + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, true /* inner */); } } - configure_project (a, *rs, c_s, projects); + configure_project (a, + *rs, + c_s, + *rs->find_module<module> (module::name), + projects); } } } + // NOTE: see pkg_configure() in bpkg if changing anything here. + // const meta_operation_info mo_configure { configure_id, "configure", @@ -879,7 +1086,7 @@ namespace build2 { const dir_path& pd (p.second); dir_path out_nroot (out_root / pd); - const scope& nrs (ctx.scopes.find (out_nroot)); + const scope& nrs (ctx.scopes.find_out (out_nroot)); assert (nrs.out_path () == out_nroot); // See disfigure_load(). r = disfigure_project (a, nrs, projects) || r; @@ -905,7 +1112,7 @@ namespace build2 } } - if (module* m = rs.find_module<module> (module::name)) + if (const module* m = rs.find_module<module> (module::name)) { for (auto hook: m->disfigure_pre_) r = hook (a, rs) || r; @@ -987,7 +1194,7 @@ namespace build2 for (auto p: *ps) { dir_path out_nroot (out_root / p.second); - const scope& nrs (ctx.scopes.find (out_nroot)); + const scope& nrs (ctx.scopes.find_out (out_nroot)); assert (nrs.out_path () == out_nroot); r = disfigure_forward (nrs, projects) || r; @@ -1004,13 +1211,13 @@ namespace build2 } static void - disfigure_pre (const values& params, const location& l) + disfigure_pre (context&, const values& params, const location& l) { forward (params, "disfigure", l); // Validate. } static operation_id - disfigure_operation_pre (const values&, operation_id o) + disfigure_operation_pre (context&, const values&, operation_id o) { // Don't translate default to update. In our case unspecified // means disfigure everything. @@ -1128,6 +1335,8 @@ namespace build2 // Add the default config.config.persist value unless there is a custom // one (specified as a command line override). // + // Note: go straight for the public variable pool. + // const variable& var (*ctx.var_pool.find ("config.config.persist")); if (!rs[var].defined ()) @@ -1244,7 +1453,8 @@ namespace build2 string ("config"), /* config_module */ nullopt, /* config_file */ true, /* buildfile */ - "the create meta-operation"); + "the create meta-operation", + 1 /* verbosity */); save_config (ctx, d); } diff --git a/libbuild2/config/operation.hxx b/libbuild2/config/operation.hxx index a887eb7..1662941 100644 --- a/libbuild2/config/operation.hxx +++ b/libbuild2/config/operation.hxx @@ -4,8 +4,6 @@ #ifndef LIBBUILD2_CONFIG_OPERATION_HXX #define LIBBUILD2_CONFIG_OPERATION_HXX -#include <set> - #include <libbuild2/types.hxx> #include <libbuild2/utility.hxx> @@ -15,8 +13,10 @@ namespace build2 { namespace config { - extern const meta_operation_info mo_configure; - extern const meta_operation_info mo_disfigure; + class module; + + LIBBUILD2_SYMEXPORT extern const meta_operation_info mo_configure; + LIBBUILD2_SYMEXPORT extern const meta_operation_info mo_disfigure; const string& preprocess_create (context&, @@ -27,7 +27,7 @@ namespace build2 // Configuration exporting. // - using project_set = std::set<const scope*>; // Pointers for comparison. + using project_set = set<const scope*>; // Pointers for comparison. // If inherit is false, then don't rely on inheritance from outer scopes // (used for config.config.save/$config.save()). In this case the already @@ -37,7 +37,12 @@ namespace build2 save_config (const scope& rs, ostream&, const path_name&, bool inherit, + const module&, const project_set&); + + // See config.config.hermetic.environment. + // + using hermetic_environment = vector<pair<string, optional<bool>>>; } } diff --git a/libbuild2/config/types.hxx b/libbuild2/config/types.hxx new file mode 100644 index 0000000..3cdc5e3 --- /dev/null +++ b/libbuild2/config/types.hxx @@ -0,0 +1,25 @@ +// file : libbuild2/config/types.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_CONFIG_TYPES_HXX +#define LIBBUILD2_CONFIG_TYPES_HXX + +#include <libbuild2/types.hxx> + +namespace build2 +{ + namespace config + { + // The origin of the value of a configuration variable. + // + enum class variable_origin + { + undefined, // Undefined. + default_, // Default value from the config directive. + buildfile, // Value from a buildfile, normally config.build. + override_ // Value from a command line override. + }; + } +} + +#endif // LIBBUILD2_CONFIG_TYPES_HXX diff --git a/libbuild2/config/utility.cxx b/libbuild2/config/utility.cxx index f777c08..6574367 100644 --- a/libbuild2/config/utility.cxx +++ b/libbuild2/config/utility.cxx @@ -8,6 +8,7 @@ using namespace std; namespace build2 { void (*config_save_variable) (scope&, const variable&, optional<uint64_t>); + void (*config_save_environment) (scope&, const char*); void (*config_save_module) (scope&, const char*, int); const string& (*config_preprocess_create) (context&, values&, @@ -20,7 +21,7 @@ namespace build2 namespace config { pair<lookup, bool> - lookup_config_impl (scope& rs, const variable& var) + lookup_config_impl (scope& rs, const variable& var, uint64_t sflags) { // This is a stripped-down version of the default value case. @@ -31,7 +32,7 @@ namespace build2 // Treat an inherited value that was set to default as new. // - if (l.defined () && l->extra) + if (l.defined () && l->extra == 1) n = true; if (var.overrides != nullptr) @@ -70,7 +71,7 @@ namespace build2 } if (l.defined ()) - save_variable (rs, var); + save_variable (rs, var, sflags); return pair<lookup, bool> (l, n); } @@ -80,7 +81,9 @@ namespace build2 const string& n, initializer_list<const char*> ig) { - auto& vp (rs.var_pool ()); + // Note: go straight for the public variable pool. + // + auto& vp (rs.ctx.var_pool); // Search all outer scopes for any value in this namespace. // @@ -90,7 +93,7 @@ namespace build2 // any original values, they will be "visible"; see find_override() for // details. // - const variable& ns (vp.insert ("config." + n)); + const string ns ("config." + n); for (scope* s (&rs); s != nullptr; s = s->parent_scope ()) { for (auto p (s->vars.lookup_namespace (ns)); @@ -106,12 +109,12 @@ namespace build2 auto match_tail = [&ns, v] (const char* t) { - return v->name.compare (ns.name.size () + 1, string::npos, t) == 0; + return v->name.compare (ns.size () + 1, string::npos, t) == 0; }; // Ignore config.*.configured and user-supplied names. // - if (v->name.size () <= ns.name.size () || + if (v->name.size () <= ns.size () || (!match_tail ("configured") && find_if (ig.begin (), ig.end (), match_tail) == ig.end ())) return true; @@ -127,7 +130,7 @@ namespace build2 // Pattern-typed as bool. // const variable& var ( - rs.var_pool ().insert ("config." + n + ".configured")); + rs.var_pool (true).insert ("config." + n + ".configured")); save_variable (rs, var); @@ -141,7 +144,7 @@ namespace build2 // Pattern-typed as bool. // const variable& var ( - rs.var_pool ().insert ("config." + n + ".configured")); + rs.var_pool (true).insert ("config." + n + ".configured")); save_variable (rs, var); @@ -155,5 +158,56 @@ namespace build2 else return false; } + + pair<variable_origin, lookup> + origin (const scope& rs, const string& n) + { + // Note: go straight for the public variable pool. + // + const variable* var (rs.ctx.var_pool.find (n)); + + if (var == nullptr) + { + if (n.compare (0, 7, "config.") != 0) + throw invalid_argument ("config.* variable expected"); + + return make_pair (variable_origin::undefined, lookup ()); + } + + return origin (rs, *var); + } + + pair<variable_origin, lookup> + origin (const scope& rs, const variable& var) + { + // Make sure this is a config.* variable. This could matter since we + // rely on the semantics of value::extra. We could also detect + // special variables like config.booted, some config.config.*, etc., + // (see config_save() for details) but that seems harmless. + // + if (var.name.compare (0, 7, "config.") != 0) + throw invalid_argument ("config.* variable expected"); + + return origin (rs, var, rs.lookup_original (var)); + } + + pair<variable_origin, lookup> + origin (const scope& rs, const variable& var, pair<lookup, size_t> org) + { + pair<lookup, size_t> ovr (var.overrides == nullptr + ? org + : rs.lookup_override (var, org)); + + if (!ovr.first.defined ()) + return make_pair (variable_origin::undefined, lookup ()); + + if (org.first != ovr.first) + return make_pair (variable_origin::override_, ovr.first); + + return make_pair (org.first->extra == 1 + ? variable_origin::default_ + : variable_origin::buildfile, + org.first); + } } } diff --git a/libbuild2/config/utility.hxx b/libbuild2/config/utility.hxx index 0429555..1e2ff53 100644 --- a/libbuild2/config/utility.hxx +++ b/libbuild2/config/utility.hxx @@ -11,6 +11,8 @@ #include <libbuild2/scope.hxx> #include <libbuild2/variable.hxx> +#include <libbuild2/config/types.hxx> + #include <libbuild2/export.hxx> namespace build2 @@ -36,6 +38,9 @@ namespace build2 (*config_save_variable) (scope&, const variable&, optional<uint64_t>); LIBBUILD2_SYMEXPORT extern void + (*config_save_environment) (scope&, const char*); + + LIBBUILD2_SYMEXPORT extern void (*config_save_module) (scope&, const char*, int); LIBBUILD2_SYMEXPORT extern const string& @@ -53,10 +58,22 @@ namespace build2 namespace config { - // Mark the variable to be saved during configuration. + // Mark a variable to be saved during configuration. + // + // Note: the save_*_omitted flags work best when undefined or (one of) the + // omitted value(s) is the default (see a note in lookup_config() + // documentation for details). + // + // The below lookup_*() functions mark the default value by setting + // value::extra to 1. Note that it's exactly 1 and not "not 0" since other + // values could have other meaning (see, for example, package skeleton + // in bpkg). // const uint64_t save_default_commented = 0x01; // Based on value::extra. const uint64_t save_null_omitted = 0x02; // Treat NULL as undefined. + const uint64_t save_empty_omitted = 0x04; // Treat empty as undefined. + const uint64_t save_false_omitted = 0x08; // Treat false as undefined. + const uint64_t save_base = 0x10; // Custom save with base. inline void save_variable (scope& rs, const variable& var, uint64_t flags = 0) @@ -65,7 +82,7 @@ namespace build2 config_save_variable (rs, var, flags); } - // Mark the variable as "unsaved" (always transient). + // Mark a variable as "unsaved" (always transient). // // Such variables are not very common and are usually used to control the // process of configuration itself. @@ -77,6 +94,93 @@ namespace build2 config_save_variable (rs, var, nullopt); } + // Mark an environment variable to be saved during hermetic configuration. + // + // Some notes/suggestions on saving environment variables for tools (e.g., + // compilers, etc): + // + // 1. We want to save variables that affect the result (e.g., build + // output) rather than byproducts (e.g., diagnostics). + // + // 2. Environment variables are often poorly documented (and not always in + // the ENVIRONMENT section; sometimes they are mentioned together with + // the corresponding option). A sensible approach in this case is to + // save documented (and perhaps well-known undocumented) variables -- + // the user can always save additional variables if necessary. The way + // to discover undocumented environment variables is to grep the source + // code. + // + // 3. Sometime environment variables only affect certain modes of a tool. + // If such modes are not used, then there is no need to save the + // corresponding variables. + // + // 4. Finally, there could be environment variables that are incompatible + // with what we are doing (e.g., they change the mode of operation or + // some such; see GCC's DEPENDENCIES_OUTPUT for example). The two ways + // to deal with this is either clear them for each invocation or, if + // that's too burdensome and there is no good reason to have the build + // system invoked with such variables, detect their presence and fail. + // Note that unsetting them for the entire build system process is not + // an option since that would be racy. + // + // See also build2::hash_environment(). + // + inline void + save_environment (scope& rs, const string& var) + { + if (config_save_environment != nullptr) + config_save_environment (rs, var.c_str ()); + } + + inline void + save_environment (scope& rs, const char* var) + { + if (config_save_environment != nullptr) + config_save_environment (rs, var); + } + + inline void + save_environment (scope& rs, initializer_list<const char*> vars) + { + if (config_save_environment != nullptr) + { + for (const char* var: vars) + config_save_environment (rs, var); + } + } + + inline void + save_environment (scope& rs, const cstrings& vars) + { + if (config_save_environment != nullptr) + { + for (const char* var: vars) + config_save_environment (rs, var); + } + } + + inline void + save_environment (scope& rs, const strings& vars) + { + if (config_save_environment != nullptr) + { + for (const string& var: vars) + config_save_environment (rs, var.c_str ()); + } + } + + // A NULL-terminated list of variables (may itself be NULL). + // + inline void + save_environment (scope& rs, const char* const* vars) + { + if (vars != nullptr && config_save_environment != nullptr) + { + for (; *vars != nullptr; ++vars) + config_save_environment (rs, *vars); + } + } + // Establish module save order/priority with INT32_MIN being the highest. // Modules with the same priority are saved in the order inserted. // @@ -136,9 +240,11 @@ namespace build2 // // Unlike the rest of the lookup_config() versions, this one leaves the // unspecified value as undefined rather than setting it to a default - // value. This can be useful when we don't have a default value or in case - // we want the mentioning of the variable to be omitted from persistent - // storage (e.g., a config file) if the default value is used. + // value (in this case it also doesn't mark the variable for saving with + // the specified flags). This can be useful when we don't have a default + // value or in case we want the mentioning of the variable to be omitted + // from persistent storage (e.g., a config file) if the default value is + // used. // // Note also that we can first do the lookup without the default value and // then, if there is no value, call the version with the default value and @@ -147,27 +253,38 @@ namespace build2 // expensive. It is also ok to call both versions multiple times provided // the flags are the same. // - // @@ Should we pass flags and interpret save_null_omitted to treat null - // as undefined? Sounds logical. - // lookup - lookup_config (scope& rs, const variable&); + lookup_config (scope& rs, + const variable&, + uint64_t save_flags = 0); lookup - lookup_config (bool& new_value, scope& rs, const variable&); + lookup_config (bool& new_value, + scope& rs, + const variable&, + uint64_t save_flags = 0); // Note that the variable is expected to have already been entered. // inline lookup - lookup_config (scope& rs, const string& var) + lookup_config (scope& rs, + const string& var, + uint64_t save_flags = 0) { - return lookup_config (rs, rs.ctx.var_pool[var]); + // Note: go straight for the public variable pool. + // + return lookup_config (rs, rs.ctx.var_pool[var], save_flags); } inline lookup - lookup_config (bool& new_value, scope& rs, const string& var) + lookup_config (bool& new_value, + scope& rs, + const string& var, + uint64_t save_flags = 0) { - return lookup_config (new_value, rs, rs.ctx.var_pool[var]); + // Note: go straight for the public variable pool. + // + return lookup_config (new_value, rs, rs.ctx.var_pool[var], save_flags); } // Lookup a config.* variable value and, if the value is undefined, set it @@ -195,8 +312,14 @@ namespace build2 // or from the command line (i.e., it is inherited from the amalgamation), // then its value is "overridden" to the default value on this root scope. // - // @@ Should save_null_omitted be interpreted to treat null as undefined? - // Sounds logical. + // Note that while it may seem logical, these functions do not + // "reinterpret" defined values according to the save_*_omitted flags (for + // example, by returning the default value if the defined value is NULL + // and the save_null_omitted flag is specified). This is because such a + // reinterpretation may cause a diversion between the returned value and + // the re-queried config.* variable value if the defined value came from + // an override. To put another way, the save_*_omitted flags are purely to + // reduce the noise in config.build. // template <typename T> lookup @@ -248,6 +371,8 @@ namespace build2 uint64_t save_flags = 0, bool override = false) { + // Note: go straight for the public variable pool. + // return lookup_config (rs, rs.ctx.var_pool[var], std::forward<T> (default_value), // VC14 @@ -264,6 +389,8 @@ namespace build2 uint64_t save_flags = 0, bool override = false) { + // Note: go straight for the public variable pool. + // return lookup_config (new_value, rs, rs.ctx.var_pool[var], @@ -308,7 +435,7 @@ namespace build2 const V* cv ( cast_null<V> ( lookup_config (rs, - rs.var_pool ().insert<V> ("config." + var), + rs.var_pool (true).insert<V> ("config." + var), std::forward<T> (default_value)))); // VC14 value& v (bs.assign<V> (move (var))); @@ -326,7 +453,7 @@ namespace build2 const V* cv ( cast_null<V> ( lookup_config (rs, - rs.var_pool ().insert<V> ("config." + var), + rs.var_pool (true).insert<V> ("config." + var), std::forward<T> (default_value)))); // VC14 value& v (bs.append<V> (move (var))); @@ -389,6 +516,25 @@ namespace build2 // LIBBUILD2_SYMEXPORT bool unconfigured (scope& rs, const string& var, bool value); + + // Return the origin of the value of the specified configuration variable + // plus the value itself. See $config.origin() for details. + // + // Throws invalid_argument if the passed variable is not config.*. + // + LIBBUILD2_SYMEXPORT pair<variable_origin, lookup> + origin (const scope& rs, const string& name); + + LIBBUILD2_SYMEXPORT pair<variable_origin, lookup> + origin (const scope& rs, const variable&); + + // As above but using the result of scope::lookup_original() or + // semantically equivalent (e.g., lookup_namespace()). + // + // Note that this version does not check that the variable is config.*. + // + LIBBUILD2_SYMEXPORT pair<variable_origin, lookup> + origin (const scope& rs, const variable&, pair<lookup, size_t> original); } } diff --git a/libbuild2/config/utility.ixx b/libbuild2/config/utility.ixx index 79d5470..d8348bd 100644 --- a/libbuild2/config/utility.ixx +++ b/libbuild2/config/utility.ixx @@ -6,22 +6,25 @@ namespace build2 namespace config { LIBBUILD2_SYMEXPORT pair<lookup, bool> - lookup_config_impl (scope&, const variable&); + lookup_config_impl (scope&, const variable&, uint64_t); template <typename T> pair<lookup, bool> lookup_config_impl (scope&, const variable&, T&&, uint64_t, bool); inline lookup - lookup_config (scope& rs, const variable& var) + lookup_config (scope& rs, const variable& var, uint64_t sflags) { - return lookup_config_impl (rs, var).first; + return lookup_config_impl (rs, var, sflags).first; } inline lookup - lookup_config (bool& new_value, scope& rs, const variable& var) + lookup_config (bool& new_value, + scope& rs, + const variable& var, + uint64_t sflags) { - auto r (lookup_config_impl (rs, var)); + auto r (lookup_config_impl (rs, var, sflags)); new_value = new_value || r.second; return r.first; } diff --git a/libbuild2/config/utility.txx b/libbuild2/config/utility.txx index b88f76c..71e41fd 100644 --- a/libbuild2/config/utility.txx +++ b/libbuild2/config/utility.txx @@ -58,7 +58,7 @@ namespace build2 if (!l.defined () || (def_ovr && !l.belongs (rs))) { value& v (rs.assign (var) = std::forward<T> (def_val)); // VC14 - v.extra = true; // Default value flag. + v.extra = 1; // Default value flag. n = (sflags & save_default_commented) == 0; // Absence means default. l = lookup (v, var, rs); @@ -66,7 +66,7 @@ namespace build2 } // Treat an inherited value that was set to default as new. // - else if (l->extra) + else if (l->extra == 1) n = (sflags & save_default_commented) == 0; // Absence means default. if (var.overrides != nullptr) |