Add support for propagating project environment

13 files changed, 254 insertions, 54 deletions

diff --git a/libbuild2/adhoc-rule-cxx.cxx b/libbuild2/adhoc-rule-cxx.cxx
index 4bf67f0..92af269 100644
--- a/libbuild2/adhoc-rule-cxx.cxx
+++ b/libbuild2/adhoc-rule-cxx.cxx
@@ -291,8 +291,10 @@ namespace build2
         create_module_context (ctx, loc);
       }
 
-      // "Switch" to the module context.
+      // Clear current project's environment and "switch" to the module
+      // context.
       //
+      auto_thread_env penv (nullptr);
       context& ctx (*t.ctx.module_context);
 
       // Mark the queue so that we don't work any tasks that may already be
diff --git a/libbuild2/algorithm.cxx b/libbuild2/algorithm.cxx
index 3f83ddd..5e93935 100644
--- a/libbuild2/algorithm.cxx
+++ b/libbuild2/algorithm.cxx
@@ -326,6 +326,14 @@ namespace build2
   const rule_match*
   match_rule (action a, target& t, const rule* skip, bool try_match)
   {
+    const scope& bs (t.base_scope ());
+
+    // Match rules in project environment.
+    //
+    auto_project_env penv;
+    if (const scope* rs = bs.root_scope ())
+      penv = auto_project_env (*rs);
+
     // First check for an ad hoc recipe.
     //
     if (!t.adhoc_recipes.empty ())
@@ -402,8 +410,6 @@ namespace build2
     meta_operation_id mo (a.meta_operation ());
     operation_id o (a.inner () ? a.operation () : a.outer_operation ());
 
-    const scope& bs (t.base_scope ());
-
     for (auto tt (&t.type ()); tt != nullptr; tt = tt->base)
     {
       // Search scopes outwards, stopping at the project root.
@@ -593,6 +599,14 @@ namespace build2
               target& t,
               const pair<const string, reference_wrapper<const rule>>& m)
   {
+    const scope& bs (t.base_scope ());
+
+    // Apply rules in project environment.
+    //
+    auto_project_env penv;
+    if (const scope* rs = bs.root_scope ())
+      penv = auto_project_env (*rs);
+
     auto df = make_diag_frame (
       [a, &t, &m](const diag_record& dr)
       {
@@ -1120,7 +1134,7 @@ namespace build2
   static target_state
   execute_recipe (action a, target& t, const recipe& r)
   {
-    target_state ts (target_state::unknown);
+    target_state ts (target_state::unchanged);
 
     try
     {
@@ -1155,31 +1169,43 @@ namespace build2
           op_s = nullptr; // Ignore.
       }
 
-      // Pre operations.
-      //
-      // Note that here we assume the dir{} target cannot be part of a group
-      // and as a result we (a) don't try to avoid calling post callbacks in
-      // case of a group failure and (b) merge the pre and post states with
-      // the group state.
-      //
-      if (op_s != nullptr)
+      if (r != nullptr || op_s != nullptr)
       {
-        for (auto i (op_p.first); i != op_p.second; ++i)
-          if (const auto& f = i->second.pre)
-            ts |= f (a, *op_s, *op_t);
-      }
+        const scope& bs (t.base_scope ());
 
-      // Recipe.
-      //
-      ts |= r != nullptr ? r (a, t) : target_state::unchanged;
+        // Execute recipe/callbacks in project environment.
+        //
+        auto_project_env penv;
+        if (const scope* rs = bs.root_scope ())
+          penv = auto_project_env (*rs);
 
-      // Post operations.
-      //
-      if (op_s != nullptr)
-      {
-        for (auto i (op_p.first); i != op_p.second; ++i)
-          if (const auto& f = i->second.post)
-            ts |= f (a, *op_s, *op_t);
+        // Pre operations.
+        //
+        // Note that here we assume the dir{} target cannot be part of a group
+        // and as a result we (a) don't try to avoid calling post callbacks in
+        // case of a group failure and (b) merge the pre and post states with
+        // the group state.
+        //
+        if (op_s != nullptr)
+        {
+          for (auto i (op_p.first); i != op_p.second; ++i)
+            if (const auto& f = i->second.pre)
+              ts |= f (a, *op_s, *op_t);
+        }
+
+        // Recipe.
+        //
+        if (r != nullptr)
+          ts |= r (a, t);
+
+        // Post operations.
+        //
+        if (op_s != nullptr)
+        {
+          for (auto i (op_p.first); i != op_p.second; ++i)
+            if (const auto& f = i->second.post)
+              ts |= f (a, *op_s, *op_t);
+        }
       }
 
       // See the recipe documentation for details on what's going on here.
diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx
index 9911670..1e6b36b 100644
--- a/libbuild2/config/init.cxx
+++ b/libbuild2/config/init.cxx
@@ -4,6 +4,8 @@
 #include <libbuild2/config/init.hxx>
 
 #include <sstream>
+#include <cstdlib> // getenv()
+#include <cstring> // strncmp()
 
 #include <libbuild2/file.hxx>
 #include <libbuild2/rule.hxx>
@@ -28,6 +30,24 @@ namespace build2
     void
     functions (function_map&); // functions.cxx
 
+    // Compare environment variable names. Note that on Windows they are
+    // case-insensitive.
+    //
+    static inline bool
+    compare_env (const string& x, size_t xn, const string& y, size_t yn)
+    {
+      if (xn == string::npos) xn = x.size ();
+      if (yn == string::npos) yn = y.size ();
+
+      return xn == yn &&
+#ifdef _WIN32
+        icasecmp (x.c_str (), y.c_str (), xn) == 0
+#else
+        strncmp (x.c_str (), y.c_str (), xn) == 0
+#endif
+        ;
+    }
+
     // Custom save function for config.config.environment.
     //
     // It tries to optimize the storage in subprojects by appending the
@@ -62,7 +82,7 @@ namespace build2
         if (find_if (ds.rbegin (), i,
                      [&v, p] (const string& v1)
                      {
-                       return v.compare (0, p, v1, 0, v1.find ('=')) == 0;
+                       return compare_env (v, p, v1, v1.find ('='));
                      }) != i)
           continue;
 
@@ -71,7 +91,7 @@ namespace build2
         auto j (find_if (bs.rbegin (), bs.rend (),
                          [&v, p] (const string& v1)
                          {
-                           return v.compare (0, p, v1, 0, v1.find ('=')) == 0;
+                           return compare_env (v, p, v1, v1.find ('='));
                          }));
 
         if (j == bs.rend () || *j != v)
@@ -375,6 +395,68 @@ namespace build2
             rs["config.config.persist"]);
       }
 
+      // Copy config.config.environment to scope::root_extra::environment.
+      //
+      // Note that we store shallow copies that point to the c.c.environment
+      // value which means it shall not change.
+      //
+      if (const strings* src = cast_null<strings> (
+            rs["config.config.environment"]))
+      {
+        vector<const char*>& dst (rs.root_extra->environment);
+
+        // The idea is to only copy entries that are effective, that is those
+        // that actually override something in the environment. This should be
+        // both more efficient and less noisy (e.g., if we decide to print
+        // this in diagnostics).
+        //
+        // Note that config.config.environment may contain duplicates and the
+        // last entry should have effect.
+        //
+        // Note also that we use std::getenv() instead of butl::getenv() to
+        // disregard any thread environment overrides.
+        //
+        for (auto i (src->rbegin ()), e (src->rend ()); i != e; ++i)
+        {
+          // Note that we must only consider variable names (up to first '='
+          // if any).
+          //
+          const string& v (*i);
+          size_t p (v.find ('='));
+
+          // Check if we have already seen this variable.
+          //
+          if (find_if (src->rbegin (), i,
+                       [&v, p] (const string& v1)
+                       {
+                         return compare_env (v, p, v1, v1.find ('='));
+                       }) != i)
+            continue;
+
+          // If it's an unset, see if it actually unsets anything.
+          //
+          if (p == string::npos)
+          {
+            if (std::getenv (v.c_str ()) == nullptr)
+              continue;
+          }
+          //
+          // And if it's a set, see if it sets a different value.
+          //
+          else
+          {
+            const char* v1 (std::getenv (string (v, 0, p).c_str ()));
+            if (v1 != nullptr && v.compare (p + 1, string::npos, v1) == 0)
+              continue;
+          }
+
+          dst.push_back (v.c_str ());
+        }
+
+        if (!dst.empty ())
+          dst.push_back (nullptr);
+      }
+
       // Register alias and fallback rule for the configure meta-operation.
       //
       // We need this rule for out-of-any-project dependencies (e.g.,
diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx
index f04a809..edb8162 100644
--- a/libbuild2/dist/operation.cxx
+++ b/libbuild2/dist/operation.cxx
@@ -426,6 +426,10 @@ namespace build2
         add_subdir (rs, dir_path (), files);
       }
 
+      // Apply project environment.
+      //
+      auto_project_env penv (rs);
+
       dir_path td (dist_root / dir_path (dist_package));
 
       // Clean up the target directory.
@@ -502,7 +506,10 @@ namespace build2
           }
 
           if (path_match (t.path ().leaf ().string (), pat.leaf ().string ()))
+          {
+            auto_project_env penv (*srs);
             cb.function (r, *srs, cb.data);
+          }
         }
 
         if (prog)
diff --git a/libbuild2/file.cxx b/libbuild2/file.cxx
index 4f24dd9..e96142d 100644
--- a/libbuild2/file.cxx
+++ b/libbuild2/file.cxx
@@ -609,7 +609,8 @@ namespace build2
         {}, /* operations */
         {}, /* modules */
         {}, /* override_cache */
-        {}} /* target_types */);
+        {}, /* target_types */
+        {}} /* environment */);
 
     // Enter built-in meta-operation and operation names. Loading of
     // modules (via the src bootstrap; see below) can result in
@@ -1404,6 +1405,10 @@ namespace build2
         optional<bool> altn;
         if (!bootstrapped (rs))
         {
+          // Clear current project's environment.
+          //
+          auto_project_env penv (nullptr);
+
           value& v (bootstrap_out (rs, altn));
 
           if (!v)
@@ -1494,9 +1499,16 @@ namespace build2
       }
     };
 
-    init_modules (module_boot_init::before_first);
-    init_modules (module_boot_init::before_second);
-    init_modules (module_boot_init::before);
+    {
+      init_modules (module_boot_init::before_first);
+
+      // Project environment should now be in effect.
+      //
+      auto_project_env penv (root);
+
+      init_modules (module_boot_init::before_second);
+      init_modules (module_boot_init::before);
+    }
 
     // Load hooks and root.build.
     //
@@ -1521,7 +1533,10 @@ namespace build2
 
     // Finish off initializing bootstrapped modules (after mode).
     //
-    init_modules (module_boot_init::after);
+    {
+      auto_project_env penv (root);
+      init_modules (module_boot_init::after);
+    }
 
     // Print the project configuration report, similar to how we do it in
     // build system modules.
@@ -1638,6 +1653,10 @@ namespace build2
 
     if (!bootstrapped (rs))
     {
+      // Clear current project's environment.
+      //
+      auto_project_env penv (nullptr);
+
       optional<bool> altn;
       bootstrap_out (rs, altn);
       setup_root (rs, forwarded);
@@ -1743,6 +1762,10 @@ namespace build2
         return nullopt;
     }
 
+    // Clear current project's environment for good measure.
+    //
+    auto_project_env penv (nullptr);
+
     // Note: to ease handling (think patching third-party code) we will always
     // specify the --build2-metadata option in this single-argument form.
     //
@@ -2339,6 +2362,10 @@ namespace build2
       }
     }
 
+    // Clear current project's environment.
+    //
+    auto_project_env penv (nullptr);
+
     for (const scope* proot (nullptr); ; proot = root)
     {
       bool top (proot == nullptr);
diff --git a/libbuild2/file.hxx b/libbuild2/file.hxx
index 0c4ad62..1c8e57e 100644
--- a/libbuild2/file.hxx
+++ b/libbuild2/file.hxx
@@ -145,6 +145,10 @@ namespace build2
   // false or the new scope is not in any project, then NULL is returned in
   // second.
   //
+  // Note that if switching the scope involved switching the project, then you
+  // may also need to switch to the new project's environment (see
+  // root_extra::environment).
+  //
   LIBBUILD2_SYMEXPORT pair<scope&, scope*>
   switch_scope (scope& root, const dir_path& out_base, bool project = true);
 
diff --git a/libbuild2/module.cxx b/libbuild2/module.cxx
index eb4395d..fc50aef 100644
--- a/libbuild2/module.cxx
+++ b/libbuild2/module.cxx
@@ -412,8 +412,10 @@ namespace build2
       //
       ctx.module_context->modules_lock = ctx.modules_lock;
 
-      // "Switch" to the module context.
+      // Clear current project's environment and "switch" to the module
+      // context.
       //
+      auto_thread_env penv (nullptr);
       context& ctx (*bs.ctx.module_context);
 
       // Mark the queue so that we don't work any tasks that may already be
diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx
index def4654..90c865b 100644
--- a/libbuild2/parser.cxx
+++ b/libbuild2/parser.cxx
@@ -49,7 +49,8 @@ namespace build2
   class parser::enter_scope
   {
   public:
-    enter_scope (): p_ (nullptr), r_ (nullptr), s_ (nullptr), b_ (nullptr) {}
+    enter_scope ()
+        : p_ (nullptr), r_ (nullptr), s_ (nullptr), b_ (nullptr) {}
 
     enter_scope (parser& p, dir_path&& d)
         : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_)
@@ -75,7 +76,15 @@ namespace build2
       if (n)
         d.normalize ();
 
-      p.switch_scope (d);
+      e_ = p.switch_scope (d);
+    }
+
+    // As above but for already absolute and normalized directory.
+    //
+    enter_scope (parser& p, const dir_path& d, bool)
+        : p_ (&p), r_ (p.root_), s_ (p.scope_), b_ (p.pbase_)
+    {
+      e_ = p.switch_scope (d);
     }
 
     ~enter_scope ()
@@ -101,6 +110,7 @@ namespace build2
         r_ = x.r_;
         s_ = x.s_;
         b_ = x.b_;
+        e_ = move (x.e_);
         x.p_ = nullptr;
       }
       return *this;
@@ -114,6 +124,7 @@ namespace build2
     scope* r_;
     scope* s_;
     const dir_path* b_; // Pattern base.
+    auto_project_env e_;
   };
 
   class parser::enter_target
@@ -269,6 +280,13 @@ namespace build2
 
     pbase_ = scope_->src_path_;
 
+    // Note that root_ may not be a project root (see parse_export_stub()).
+    //
+    auto_project_env penv (
+      stage_ != stage::boot && root_ != nullptr && root_->root_extra != nullptr
+      ? auto_project_env (*root_)
+      : auto_project_env ());
+
     if (path_->path != nullptr)
       enter_buildfile (*path_->path); // Note: needs scope_.
 
@@ -2076,10 +2094,7 @@ namespace build2
       // out the absolute buildfile path since we may switch the project
       // root and src_root with it (i.e., include into a sub-project).
       //
-      scope* ors (root_);
-      scope* ocs (scope_);
-      const dir_path* opb (pbase_);
-      switch_scope (out_base);
+      enter_scope sg (*this, out_base, true /* absolute & normalized */);
 
       if (root_ == nullptr)
         fail (l) << "out of project include from " << out_base;
@@ -2095,9 +2110,6 @@ namespace build2
       if (!root_->buildfiles.insert (p).second) // Note: may be "new" root.
       {
         l5 ([&]{trace (l) << "skipping already included " << p;});
-        pbase_ = opb;
-        scope_ = ocs;
-        root_ = ors;
         continue;
       }
 
@@ -2113,10 +2125,6 @@ namespace build2
       {
         fail (l) << "unable to read buildfile " << p << ": " << e;
       }
-
-      pbase_ = opb;
-      scope_ = ocs;
-      root_ = ors;
     }
 
     next_after_newline (t, tt);
@@ -6896,11 +6904,13 @@ namespace build2
     assert (pre_parse_);
   }
 
-  void parser::
+  auto_project_env parser::
   switch_scope (const dir_path& d)
   {
     tracer trace ("parser::switch_scope", &path_);
 
+    auto_project_env r;
+
     // Switching the project during bootstrap can result in bizarre nesting
     // with unexpected loading order (e.g., config.build are loaded from inner
     // to outter rather than the expected reverse). On the other hand, it can
@@ -6917,6 +6927,10 @@ namespace build2
     if (proj && p.second != root_)
     {
       root_ = p.second;
+
+      if (root_ != nullptr)
+        r = auto_project_env (*root_);
+
       l5 ([&]
           {
             if (root_ != nullptr)
@@ -6925,6 +6939,8 @@ namespace build2
               trace << "switching to out of project scope";
           });
     }
+
+    return r;
   }
 
   void parser::
diff --git a/libbuild2/parser.hxx b/libbuild2/parser.hxx
index 3efb94a..915e34b 100644
--- a/libbuild2/parser.hxx
+++ b/libbuild2/parser.hxx
@@ -528,9 +528,10 @@ namespace build2
 
     // Switch to a new current scope. Note that this function might also have
     // to switch to a new root scope if the new current scope is in another
-    // project. So both must be saved and restored.
+    // project. So both must be saved and restored. In case of a new root, it
+    // also switches to the new project's environment.
     //
-    void
+    auto_project_env
     switch_scope (const dir_path& out_base);
 
     void
diff --git a/libbuild2/scope.hxx b/libbuild2/scope.hxx
index b8ee15a..9f88dc9 100644
--- a/libbuild2/scope.hxx
+++ b/libbuild2/scope.hxx
@@ -488,6 +488,19 @@ namespace build2
       // Target types.
       //
       target_type_map target_types;
+
+      // Environment variable overrides.
+      //
+      // These overrides should be applied to the environment when running
+      // tools (e.g., compilers) or querying environment variables from the
+      // buildfiles and by the build system itself. Populated by the config
+      // module and is not available during bootstrap (more precisely, not
+      // available until before_first modules have been initialized). The list
+      // is either empty of NULL-terminated.
+      //
+      // See also auto_project_env below.
+      //
+      vector<const char*> environment;
     };
 
     unique_ptr<root_extra_type> root_extra;
@@ -575,6 +588,23 @@ namespace build2
     return to_stream (os, s.out_path (), true /* representation */);
   }
 
+  // Automatic project environment setup/cleanup.
+  //
+  struct auto_project_env: auto_thread_env
+  {
+    auto_project_env () = default;
+
+    explicit
+    auto_project_env (nullptr_t p) // Clear current environment.
+        : auto_thread_env (p) {}
+
+    explicit
+    auto_project_env (const scope& rs)
+        : auto_thread_env (rs.root_extra->environment.empty ()
+                           ? nullptr
+                           : rs.root_extra->environment.data ()) {}
+  };
+
   // Return the src/out directory corresponding to the given out/src. The
   // passed directory should be a sub-directory of out/src_root.
   //
diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx
index 0b49dea..3faa35c 100644
--- a/libbuild2/script/run.cxx
+++ b/libbuild2/script/run.cxx
@@ -2126,8 +2126,8 @@ namespace build2
 
           // Note that CWD and builtin-escaping character '^' are not printed.
           //
-          const small_vector<string, 4>& evs (vs);
-          process_env pe (resolve ? pp : c.program, evs);
+          const small_vector<string, 4>& evars (vs);
+          process_env pe (resolve ? pp : c.program, evars);
 
           if (verb >= 2)
             print_process (pe, args);
diff --git a/libbuild2/target.cxx b/libbuild2/target.cxx
index 2ac5f9a..fec522c 100644
--- a/libbuild2/target.cxx
+++ b/libbuild2/target.cxx
@@ -1209,6 +1209,10 @@ namespace build2
       {
         // Ok, no luck, switch the scope.
         //
+        // Note that we don't need to do anything for the project's
+        // environment: source_once() will take care of it itself and
+        // search_implied() is not affected.
+        //
         pair<scope&, scope*> sp (
           switch_scope (*s.rw ().root_scope (), out_base));
 
diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx
index 4cfdf15..6616f78 100644
--- a/libbuild2/utility.hxx
+++ b/libbuild2/utility.hxx
@@ -81,8 +81,7 @@ namespace build2
   using butl::function_cast;
 
   using butl::getenv;
-  using butl::setenv;
-  using butl::unsetenv;
+  using butl::auto_thread_env;
 
   using butl::throw_generic_error;
   using butl::throw_system_error;