From b3bc26dc284cf73e97b88c9979d49368d07e686c Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Fri, 26 Mar 2021 16:03:29 +0200 Subject: Add support for propagating project environment --- libbuild2/adhoc-rule-cxx.cxx | 4 ++- libbuild2/algorithm.cxx | 76 ++++++++++++++++++++++++++------------- libbuild2/config/init.cxx | 86 ++++++++++++++++++++++++++++++++++++++++++-- libbuild2/dist/operation.cxx | 7 ++++ libbuild2/file.cxx | 37 ++++++++++++++++--- libbuild2/file.hxx | 4 +++ libbuild2/module.cxx | 4 ++- libbuild2/parser.cxx | 44 +++++++++++++++-------- libbuild2/parser.hxx | 5 +-- libbuild2/scope.hxx | 30 ++++++++++++++++ libbuild2/script/run.cxx | 4 +-- libbuild2/target.cxx | 4 +++ libbuild2/utility.hxx | 3 +- 13 files changed, 254 insertions(+), 54 deletions(-) diff --git a/libbuild2/adhoc-rule-cxx.cxx b/libbuild2/adhoc-rule-cxx.cxx index 4bf67f0..92af269 100644 --- a/libbuild2/adhoc-rule-cxx.cxx +++ b/libbuild2/adhoc-rule-cxx.cxx @@ -291,8 +291,10 @@ namespace build2 create_module_context (ctx, loc); } - // "Switch" to the module context. + // Clear current project's environment and "switch" to the module + // context. // + auto_thread_env penv (nullptr); context& ctx (*t.ctx.module_context); // Mark the queue so that we don't work any tasks that may already be diff --git a/libbuild2/algorithm.cxx b/libbuild2/algorithm.cxx index 3f83ddd..5e93935 100644 --- a/libbuild2/algorithm.cxx +++ b/libbuild2/algorithm.cxx @@ -326,6 +326,14 @@ namespace build2 const rule_match* match_rule (action a, target& t, const rule* skip, bool try_match) { + const scope& bs (t.base_scope ()); + + // Match rules in project environment. + // + auto_project_env penv; + if (const scope* rs = bs.root_scope ()) + penv = auto_project_env (*rs); + // First check for an ad hoc recipe. // if (!t.adhoc_recipes.empty ()) @@ -402,8 +410,6 @@ namespace build2 meta_operation_id mo (a.meta_operation ()); operation_id o (a.inner () ? a.operation () : a.outer_operation ()); - const scope& bs (t.base_scope ()); - for (auto tt (&t.type ()); tt != nullptr; tt = tt->base) { // Search scopes outwards, stopping at the project root. @@ -593,6 +599,14 @@ namespace build2 target& t, const pair>& m) { + const scope& bs (t.base_scope ()); + + // Apply rules in project environment. + // + auto_project_env penv; + if (const scope* rs = bs.root_scope ()) + penv = auto_project_env (*rs); + auto df = make_diag_frame ( [a, &t, &m](const diag_record& dr) { @@ -1120,7 +1134,7 @@ namespace build2 static target_state execute_recipe (action a, target& t, const recipe& r) { - target_state ts (target_state::unknown); + target_state ts (target_state::unchanged); try { @@ -1155,31 +1169,43 @@ namespace build2 op_s = nullptr; // Ignore. } - // Pre operations. - // - // Note that here we assume the dir{} target cannot be part of a group - // and as a result we (a) don't try to avoid calling post callbacks in - // case of a group failure and (b) merge the pre and post states with - // the group state. - // - if (op_s != nullptr) + if (r != nullptr || op_s != nullptr) { - for (auto i (op_p.first); i != op_p.second; ++i) - if (const auto& f = i->second.pre) - ts |= f (a, *op_s, *op_t); - } + const scope& bs (t.base_scope ()); - // Recipe. - // - ts |= r != nullptr ? r (a, t) : target_state::unchanged; + // Execute recipe/callbacks in project environment. + // + auto_project_env penv; + if (const scope* rs = bs.root_scope ()) + penv = auto_project_env (*rs); - // Post operations. - // - if (op_s != nullptr) - { - for (auto i (op_p.first); i != op_p.second; ++i) - if (const auto& f = i->second.post) - ts |= f (a, *op_s, *op_t); + // Pre operations. + // + // Note that here we assume the dir{} target cannot be part of a group + // and as a result we (a) don't try to avoid calling post callbacks in + // case of a group failure and (b) merge the pre and post states with + // the group state. + // + if (op_s != nullptr) + { + for (auto i (op_p.first); i != op_p.second; ++i) + if (const auto& f = i->second.pre) + ts |= f (a, *op_s, *op_t); + } + + // Recipe. + // + if (r != nullptr) + ts |= r (a, t); + + // Post operations. + // + if (op_s != nullptr) + { + for (auto i (op_p.first); i != op_p.second; ++i) + if (const auto& f = i->second.post) + ts |= f (a, *op_s, *op_t); + } } // See the recipe documentation for details on what's going on here. diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx index 9911670..1e6b36b 100644 --- a/libbuild2/config/init.cxx +++ b/libbuild2/config/init.cxx @@ -4,6 +4,8 @@ #include #include +#include // getenv() +#include // strncmp() #include #include @@ -28,6 +30,24 @@ namespace build2 void functions (function_map&); // functions.cxx + // Compare environment variable names. Note that on Windows they are + // case-insensitive. + // + static inline bool + compare_env (const string& x, size_t xn, const string& y, size_t yn) + { + if (xn == string::npos) xn = x.size (); + if (yn == string::npos) yn = y.size (); + + return xn == yn && +#ifdef _WIN32 + icasecmp (x.c_str (), y.c_str (), xn) == 0 +#else + strncmp (x.c_str (), y.c_str (), xn) == 0 +#endif + ; + } + // Custom save function for config.config.environment. // // It tries to optimize the storage in subprojects by appending the @@ -62,7 +82,7 @@ namespace build2 if (find_if (ds.rbegin (), i, [&v, p] (const string& v1) { - return v.compare (0, p, v1, 0, v1.find ('=')) == 0; + return compare_env (v, p, v1, v1.find ('=')); }) != i) continue; @@ -71,7 +91,7 @@ namespace build2 auto j (find_if (bs.rbegin (), bs.rend (), [&v, p] (const string& v1) { - return v.compare (0, p, v1, 0, v1.find ('=')) == 0; + return compare_env (v, p, v1, v1.find ('=')); })); if (j == bs.rend () || *j != v) @@ -375,6 +395,68 @@ namespace build2 rs["config.config.persist"]); } + // Copy config.config.environment to scope::root_extra::environment. + // + // Note that we store shallow copies that point to the c.c.environment + // value which means it shall not change. + // + if (const strings* src = cast_null ( + rs["config.config.environment"])) + { + vector& dst (rs.root_extra->environment); + + // The idea is to only copy entries that are effective, that is those + // that actually override something in the environment. This should be + // both more efficient and less noisy (e.g., if we decide to print + // this in diagnostics). + // + // Note that config.config.environment may contain duplicates and the + // last entry should have effect. + // + // Note also that we use std::getenv() instead of butl::getenv() to + // disregard any thread environment overrides. + // + for (auto i (src->rbegin ()), e (src->rend ()); i != e; ++i) + { + // Note that we must only consider variable names (up to first '=' + // if any). + // + const string& v (*i); + size_t p (v.find ('=')); + + // Check if we have already seen this variable. + // + if (find_if (src->rbegin (), i, + [&v, p] (const string& v1) + { + return compare_env (v, p, v1, v1.find ('=')); + }) != i) + continue; + + // If it's an unset, see if it actually unsets anything. + // + if (p == string::npos) + { + if (std::getenv (v.c_str ()) == nullptr) + continue; + } + // + // And if it's a set, see if it sets a different value. + // + else + { + const char* v1 (std::getenv (string (v, 0, p).c_str ())); + if (v1 != nullptr && v.compare (p + 1, string::npos, v1) == 0) + continue; + } + + dst.push_back (v.c_str ()); + } + + if (!dst.empty ()) + dst.push_back (nullptr); + } + // Register alias and fallback rule for the configure meta-operation. // // We need this rule for out-of-any-project dependencies (e.g., diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx index f04a809..edb8162 100644 --- a/libbuild2/dist/operation.cxx +++ b/libbuild2/dist/operation.cxx @@ -426,6 +426,10 @@ namespace build2 add_subdir (rs, dir_path (), files); } + // Apply project environment. + // + auto_project_env penv (rs); + dir_path td (dist_root / dir_path (dist_package)); // Clean up the target directory. @@ -502,7 +506,10 @@ namespace build2 } if (path_match (t.path ().leaf ().string (), pat.leaf ().string ())) + { + auto_project_env penv (*srs); cb.function (r, *srs, cb.data); + } } if (prog) diff --git a/libbuild2/file.cxx b/libbuild2/file.cxx index 4f24dd9..e96142d 100644 --- a/libbuild2/file.cxx +++ b/libbuild2/file.cxx @@ -609,7 +609,8 @@ namespace build2 {}, /* operations */ {}, /* modules */ {}, /* override_cache */ - {}} /* target_types */); + {}, /* target_types */ + {}} /* environment */); // Enter built-in meta-operation and operation names. Loading of // modules (via the src bootstrap; see below) can result in @@ -1404,6 +1405,10 @@ namespace build2 optional altn; if (!bootstrapped (rs)) { + // Clear current project's environment. + // + auto_project_env penv (nullptr); + value& v (bootstrap_out (rs, altn)); if (!v) @@ -1494,9 +1499,16 @@ namespace build2 } }; - init_modules (module_boot_init::before_first); - init_modules (module_boot_init::before_second); - init_modules (module_boot_init::before); + { + init_modules (module_boot_init::before_first); + + // Project environment should now be in effect. + // + auto_project_env penv (root); + + init_modules (module_boot_init::before_second); + init_modules (module_boot_init::before); + } // Load hooks and root.build. // @@ -1521,7 +1533,10 @@ namespace build2 // Finish off initializing bootstrapped modules (after mode). // - init_modules (module_boot_init::after); + { + auto_project_env penv (root); + init_modules (module_boot_init::after); + } // Print the project configuration report, similar to how we do it in // build system modules. @@ -1638,6 +1653,10 @@ namespace build2 if (!bootstrapped (rs)) { + // Clear current project's environment. + // + auto_project_env penv (nullptr); + optional altn; bootstrap_out (rs, altn); setup_root (rs, forwarded); @@ -1743,6 +1762,10 @@ namespace build2 return nullopt; } + // Clear current project's environment for good measure. + // + auto_project_env penv (nullptr); + // Note: to ease handling (think patching third-party code) we will always // specify the --build2-metadata option in this single-argument form. // @@ -2339,6 +2362,10 @@ namespace build2 } } + // Clear current project's environment. + // + auto_project_env penv (nullptr); + for (const scope* proot (nullptr); ; proot = root) { bool top (proot == nullptr); diff --git a/libbuild2/file.hxx b/libbuild2/file.hxx index 0c4ad62..1c8e57e 100644 --- a/libbuild2/file.hxx +++ b/libbuild2/file.hxx @@ -145,6 +145,10 @@ namespace build2 // false or the new scope is not in any project, then NULL is returned in // second. // + // Note that if switching the scope involved switching the project, then you + // may also need to switch to the new project's environment (see + // root_extra::environment). + // LIBBUILD2_SYMEXPORT pair switch_scope (scope& root, const dir_path& out_base, bool project = true); diff --git a/libbuild2/module.cxx b/libbuild2/module.cxx index eb4395d..fc50aef 100644 --- a/libbuild2/module.cxx +++ b/libbuild2/module.cxx @@ -412,8 +412,10 @@ namespace build2 // ctx.module_context->modules_lock = ctx.modules_lock; - // "Switch" to the module context. + // Clear current project's environment and "switch" to the module + // context. // + auto_thread_env penv (nullptr); context& ctx (*bs.ctx.module_context); // Mark the queue so that we don't work any tasks that may already be diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index def4654..90c865b 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -49,7 +49,8 @@ namespace build2 class parser::enter_scope { public: - enter_scope (): p_ (nullptr), r_ (nullptr), s_ (nullptr), b_ (nullptr) {} + enter_scope () + : p_ (nullptr), r_ (nullptr), s_ (nullptr), b_ (nullptr) {} enter_scope (parser& p, dir_path&& d) : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_) @@ -75,7 +76,15 @@ namespace build2 if (n) d.normalize (); - p.switch_scope (d); + e_ = p.switch_scope (d); + } + + // As above but for already absolute and normalized directory. + // + enter_scope (parser& p, const dir_path& d, bool) + : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_) + { + e_ = p.switch_scope (d); } ~enter_scope () @@ -101,6 +110,7 @@ namespace build2 r_ = x.r_; s_ = x.s_; b_ = x.b_; + e_ = move (x.e_); x.p_ = nullptr; } return *this; @@ -114,6 +124,7 @@ namespace build2 scope* r_; scope* s_; const dir_path* b_; // Pattern base. + auto_project_env e_; }; class parser::enter_target @@ -269,6 +280,13 @@ namespace build2 pbase_ = scope_->src_path_; + // Note that root_ may not be a project root (see parse_export_stub()). + // + auto_project_env penv ( + stage_ != stage::boot && root_ != nullptr && root_->root_extra != nullptr + ? auto_project_env (*root_) + : auto_project_env ()); + if (path_->path != nullptr) enter_buildfile (*path_->path); // Note: needs scope_. @@ -2076,10 +2094,7 @@ namespace build2 // out the absolute buildfile path since we may switch the project // root and src_root with it (i.e., include into a sub-project). // - scope* ors (root_); - scope* ocs (scope_); - const dir_path* opb (pbase_); - switch_scope (out_base); + enter_scope sg (*this, out_base, true /* absolute & normalized */); if (root_ == nullptr) fail (l) << "out of project include from " << out_base; @@ -2095,9 +2110,6 @@ namespace build2 if (!root_->buildfiles.insert (p).second) // Note: may be "new" root. { l5 ([&]{trace (l) << "skipping already included " << p;}); - pbase_ = opb; - scope_ = ocs; - root_ = ors; continue; } @@ -2113,10 +2125,6 @@ namespace build2 { fail (l) << "unable to read buildfile " << p << ": " << e; } - - pbase_ = opb; - scope_ = ocs; - root_ = ors; } next_after_newline (t, tt); @@ -6896,11 +6904,13 @@ namespace build2 assert (pre_parse_); } - void parser:: + auto_project_env parser:: switch_scope (const dir_path& d) { tracer trace ("parser::switch_scope", &path_); + auto_project_env r; + // Switching the project during bootstrap can result in bizarre nesting // with unexpected loading order (e.g., config.build are loaded from inner // to outter rather than the expected reverse). On the other hand, it can @@ -6917,6 +6927,10 @@ namespace build2 if (proj && p.second != root_) { root_ = p.second; + + if (root_ != nullptr) + r = auto_project_env (*root_); + l5 ([&] { if (root_ != nullptr) @@ -6925,6 +6939,8 @@ namespace build2 trace << "switching to out of project scope"; }); } + + return r; } void parser:: diff --git a/libbuild2/parser.hxx b/libbuild2/parser.hxx index 3efb94a..915e34b 100644 --- a/libbuild2/parser.hxx +++ b/libbuild2/parser.hxx @@ -528,9 +528,10 @@ namespace build2 // Switch to a new current scope. Note that this function might also have // to switch to a new root scope if the new current scope is in another - // project. So both must be saved and restored. + // project. So both must be saved and restored. In case of a new root, it + // also switches to the new project's environment. // - void + auto_project_env switch_scope (const dir_path& out_base); void diff --git a/libbuild2/scope.hxx b/libbuild2/scope.hxx index b8ee15a..9f88dc9 100644 --- a/libbuild2/scope.hxx +++ b/libbuild2/scope.hxx @@ -488,6 +488,19 @@ namespace build2 // Target types. // target_type_map target_types; + + // Environment variable overrides. + // + // These overrides should be applied to the environment when running + // tools (e.g., compilers) or querying environment variables from the + // buildfiles and by the build system itself. Populated by the config + // module and is not available during bootstrap (more precisely, not + // available until before_first modules have been initialized). The list + // is either empty of NULL-terminated. + // + // See also auto_project_env below. + // + vector environment; }; unique_ptr root_extra; @@ -575,6 +588,23 @@ namespace build2 return to_stream (os, s.out_path (), true /* representation */); } + // Automatic project environment setup/cleanup. + // + struct auto_project_env: auto_thread_env + { + auto_project_env () = default; + + explicit + auto_project_env (nullptr_t p) // Clear current environment. + : auto_thread_env (p) {} + + explicit + auto_project_env (const scope& rs) + : auto_thread_env (rs.root_extra->environment.empty () + ? nullptr + : rs.root_extra->environment.data ()) {} + }; + // Return the src/out directory corresponding to the given out/src. The // passed directory should be a sub-directory of out/src_root. // diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx index 0b49dea..3faa35c 100644 --- a/libbuild2/script/run.cxx +++ b/libbuild2/script/run.cxx @@ -2126,8 +2126,8 @@ namespace build2 // Note that CWD and builtin-escaping character '^' are not printed. // - const small_vector& evs (vs); - process_env pe (resolve ? pp : c.program, evs); + const small_vector& evars (vs); + process_env pe (resolve ? pp : c.program, evars); if (verb >= 2) print_process (pe, args); diff --git a/libbuild2/target.cxx b/libbuild2/target.cxx index 2ac5f9a..fec522c 100644 --- a/libbuild2/target.cxx +++ b/libbuild2/target.cxx @@ -1209,6 +1209,10 @@ namespace build2 { // Ok, no luck, switch the scope. // + // Note that we don't need to do anything for the project's + // environment: source_once() will take care of it itself and + // search_implied() is not affected. + // pair sp ( switch_scope (*s.rw ().root_scope (), out_base)); diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx index 4cfdf15..6616f78 100644 --- a/libbuild2/utility.hxx +++ b/libbuild2/utility.hxx @@ -81,8 +81,7 @@ namespace build2 using butl::function_cast; using butl::getenv; - using butl::setenv; - using butl::unsetenv; + using butl::auto_thread_env; using butl::throw_generic_error; using butl::throw_system_error; -- cgit v1.1