Add base functionality for hermetic build configurationshermetic

5 files changed, 325 insertions, 105 deletions

diff --git a/libbuild2/config/functions.cxx b/libbuild2/config/functions.cxx
index 5e85238..398512c 100644
--- a/libbuild2/config/functions.cxx
+++ b/libbuild2/config/functions.cxx
@@ -7,6 +7,7 @@
 #include <libbuild2/function.hxx>
 #include <libbuild2/variable.hxx>
 
+#include <libbuild2/config/module.hxx>
 #include <libbuild2/config/operation.hxx>
 
 using namespace std;
@@ -39,6 +40,11 @@ namespace build2
         if (s == nullptr)
           fail << "config.save() called out of project" << endf;
 
+        module* mod (s->find_module<module> (module::name));
+
+        if (mod == nullptr)
+          fail << "config.save() called without config module";
+
         ostringstream os;
 
         // Empty project set should is ok as long as inherit is false.
@@ -47,6 +53,7 @@ namespace build2
         save_config (*s,
                      os, path_name ("config.save()"),
                      false /* inherit */,
+                     *mod,
                      ps);
 
         return os.str ();
diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx
index 1e6b36b..950579a 100644
--- a/libbuild2/config/init.cxx
+++ b/libbuild2/config/init.cxx
@@ -5,7 +5,6 @@
 
 #include <sstream>
 #include <cstdlib> // getenv()
-#include <cstring> // strncmp()
 
 #include <libbuild2/file.hxx>
 #include <libbuild2/rule.hxx>
@@ -30,24 +29,6 @@ namespace build2
     void
     functions (function_map&); // functions.cxx
 
-    // Compare environment variable names. Note that on Windows they are
-    // case-insensitive.
-    //
-    static inline bool
-    compare_env (const string& x, size_t xn, const string& y, size_t yn)
-    {
-      if (xn == string::npos) xn = x.size ();
-      if (yn == string::npos) yn = y.size ();
-
-      return xn == yn &&
-#ifdef _WIN32
-        icasecmp (x.c_str (), y.c_str (), xn) == 0
-#else
-        strncmp (x.c_str (), y.c_str (), xn) == 0
-#endif
-        ;
-    }
-
     // Custom save function for config.config.environment.
     //
     // It tries to optimize the storage in subprojects by appending the
@@ -82,7 +63,8 @@ namespace build2
         if (find_if (ds.rbegin (), i,
                      [&v, p] (const string& v1)
                      {
-                       return compare_env (v, p, v1, v1.find ('='));
+                       return saved_environment::compare (
+                         v, v1, p, v1.find ('='));
                      }) != i)
           continue;
 
@@ -91,7 +73,8 @@ namespace build2
         auto j (find_if (bs.rbegin (), bs.rend (),
                          [&v, p] (const string& v1)
                          {
-                           return compare_env (v, p, v1, v1.find ('='));
+                           return saved_environment::compare (
+                             v, v1, p, v1.find ('='));
                          }));
 
         if (j == bs.rend () || *j != v)
@@ -130,7 +113,7 @@ namespace build2
       // as a command line override.
       //
       // Note: must be entered during bootstrap since we need it in
-      // configure_execute().
+      // configure_execute() even for the forward case.
       //
       vp.insert<path> ("config.config.save", true /* ovr */);
 
@@ -166,39 +149,11 @@ namespace build2
       //
       // Use the NULL value to clear.
       //
-      auto& c_p (vp.insert<vector<pair<string, string>>> (
-                   "config.config.persist", true /* ovr */, v_p));
-
-      // Configuration environment variables.
-      //
-      // Environment variables used by tools (e.g., compilers), buildfiles
-      // (e.g., $getenv()), and the build system itself (e.g., to locate
-      // tools) in ways that affect the build result are in essence part of
-      // the project configuration.
-      //
-      // This variable allows storing environment variable overrides that
-      // should be applied to the environment when executing tools, etc., as
-      // part of a project build. Specifically, it contains a list of
-      // environment variable "sets" (<name>=<value>) and "unsets" (<name>).
-      // If multiple entries are specified for the same environment variable,
-      // the last entry has effect. For example:
-      //
-      // config.config.environment="LC_ALL=C LANG"
-      //
-      // Note that a subproject inherits overrides from its amalgamation (this
-      // semantics is the result of the way we optimize the storage of this
-      // variable in subproject's config.build; the thinking is that if a
-      // variable is not overridden by the subproject then it doesn't affect
-      // the build result and therefore it's irrelevant whether it has a value
-      // that came from the original environment of from the amalgamation
-      // override).
-      //
-      // Use the NULL or empty value to clear.
-      //
-      // @@ We could use =<name> as a "pass-through" instruction (e.g., if
-      //    we need to use original value in subproject).
+      // Note: must be entered during bootstrap since we need it in create's
+      // save_config().
       //
-      auto& c_e (vp.insert<strings> ("config.config.environment", true /* ovr */));
+      vp.insert<vector<pair<string, string>>> (
+        "config.config.persist", true /* ovr */, v_p);
 
       // Only create the module if we are configuring, creating, or
       // disfiguring or if it was requested with config.config.module (useful
@@ -229,11 +184,6 @@ namespace build2
           //
           m.save_module ("config", INT32_MIN);
           m.save_module ("import", INT32_MIN);
-
-          m.save_variable (c_p, save_null_omitted);
-          m.save_variable (c_e,
-                           save_null_omitted | save_empty_omitted | save_base,
-                           &save_environment);
         }
       }
 
@@ -277,16 +227,106 @@ namespace build2
         return true;
       }
 
+      context& ctx (rs.ctx);
+
       l5 ([&]{trace << "for " << rs;});
 
+      // If we have the module, then we are configuring (or the project wishes
+      // to call $config.save(); we don't get here on disfigure).
+      //
+      module* m (extra.module != nullptr
+                 ? &extra.module_as<module> ()
+                 : nullptr);
+
       auto& vp (rs.var_pool ());
 
       // Note: config.* is pattern-typed to global visibility.
       //
       const auto v_p (variable_visibility::project);
 
-      auto& c_l (vp.insert<paths> ("config.config.load", true /* ovr */));
       auto& c_v (vp.insert<uint64_t> ("config.version", false /*ovr*/, v_p));
+      auto& c_l (vp.insert<paths> ("config.config.load", true /* ovr */));
+
+      // Hermetic configurations.
+      //
+      // A hermetic configuration stores environment variables that affect the
+      // project in config.config.environment.
+      //
+      // Note that this is essentially a tri-state value: true means keep
+      // hermetizing (save the environment in config.config.environment),
+      // false means keep de-hermetizing (clear config.config.environment) and
+      // undefined/NULL means don't touch config.config.environment.
+      //
+      // During reconfiguration things stay hermetic unless re-hermetization
+      // is explicitly requested with config.config.hermetic.reload=true (or
+      // de-hermetization is requested with config.config.hermetic=false).
+      //
+      // Use the NULL value to clear.
+      //
+      auto& c_h (vp.insert<bool> ("config.config.hermetic", true /* ovr */));
+
+      if (m != nullptr)
+        m->save_variable (c_h, save_null_omitted);
+
+      // Request hermetic configuration re-hermetization.
+      //
+      // Note that this variable is not saved in config.build and is expected
+      // to always be specified as a command line override.
+      //
+      auto& c_h_r (
+        vp.insert<bool> ("config.config.hermetic.reload", true /* ovr */));
+
+      // Hermetic configuration environment variables inclusion/exclusion.
+      //
+      // This configuration variable can be used to include additional or
+      // exclude existing environment variables into/from the list that should
+      // be saved in order to make the configuration hermetic. For example:
+      //
+      // config.config.hermetic.environment="LANG PATH@false"
+      //
+      // Use the NULL or empty value to clear.
+      //
+      auto& c_h_e (
+        vp.insert<hermetic_environment> ("config.config.hermetic.environment"));
+
+      if (m != nullptr)
+        m->save_variable (c_h_e, save_null_omitted | save_empty_omitted);
+
+      // Configuration environment variables.
+      //
+      // Environment variables used by tools (e.g., compilers), buildfiles
+      // (e.g., $getenv()), and the build system itself (e.g., to locate
+      // tools) in ways that affect the build result are in essence part of
+      // the project configuration.
+      //
+      // This variable allows storing environment variable overrides that
+      // should be applied to the environment when executing tools, etc., as
+      // part of a project build. Specifically, it contains a list of
+      // environment variable "sets" (<name>=<value>) and "unsets" (<name>).
+      // If multiple entries are specified for the same environment variable,
+      // the last entry has effect. For example:
+      //
+      // config.config.environment="LC_ALL=C LANG"
+      //
+      // Note that a subproject inherits overrides from its amalgamation (this
+      // semantics is the result of the way we optimize the storage of this
+      // variable in subproject's config.build; the thinking is that if a
+      // variable is not overridden by the subproject then it doesn't affect
+      // the build result and therefore it's irrelevant whether it has a value
+      // that came from the original environment of from the amalgamation
+      // override).
+      //
+      // Use the NULL or empty value to clear.
+      //
+      // @@ We could use =<name> as a "pass-through" instruction (e.g., if
+      //    we need to use original value in subproject).
+      //
+      auto& c_e (vp.insert<strings> ("config.config.environment", true /* ovr */));
+
+      if (m != nullptr)
+        m->save_variable (c_e,
+                          save_null_omitted | save_empty_omitted | save_base,
+                          &save_environment);
 
       // Load config.build if one exists followed by extra files specified in
       // config.config.load (we don't need to worry about disfigure since we
@@ -358,7 +398,7 @@ namespace build2
         // b ./config.config.load=.../config.build  # this project
         // b  !config.config.load=.../config.build  # every project
         //
-        if (l.belongs (rs) || l.belongs (rs.ctx.global_scope))
+        if (l.belongs (rs) || l.belongs (ctx.global_scope))
         {
           for (const path& f: cast<paths> (l))
           {
@@ -384,31 +424,50 @@ namespace build2
         }
       }
 
-      // Cache the config.config.persist value, if any.
+      // Save and cache the config.config.persist value, if any.
       //
-      if (extra.module != nullptr)
+      if (m != nullptr)
       {
-        auto& m (extra.module_as<module> ());
+        auto& c_p (*vp.find ("config.config.persist"));
+        m->save_variable (c_p, save_null_omitted);
+        m->persist = cast_null<vector<pair<string, string>>> (rs[c_p]);
+      }
 
-        m.persist =
-          cast_null<vector<pair<string, string>>> (
-            rs["config.config.persist"]);
+      // If we are configuring, handle config.config.hermetic.
+      //
+      // The overall plan is to either clear config.config.environment (if
+      // c.c.h=false) or populate it with the values that affect this project
+      // (if c.c.h=true). We have to do it half here (because c.c.e is used as
+      // a source for the project environment and we would naturally want the
+      // semantics to be equivalent to what will be saved in config.build) and
+      // half in configure_execute() (because that's where we have the final
+      // list of all the environment variables we need to save).
+      //
+      // So here we must deal with the cases where the current c.c.e value
+      // will be changed: either cleared (c.c.h=false) or set to new values
+      // from the "outer" environment (c.c.h.reload=true). Note also that even
+      // then a c.c.e value from an amalgamation, if any, should be in effect.
+      //
+      if (ctx.current_mif->id == configure_id &&
+          (!cast_true<bool> (rs[c_h]) ||  // c.c.h=false
+           cast_false<bool> (rs[c_h_r]))) // c.c.h.r=true
+      {
+        rs.vars.erase (c_e); // Undefine.
       }
 
       // Copy config.config.environment to scope::root_extra::environment.
       //
       // Note that we store shallow copies that point to the c.c.environment
-      // value which means it shall not change.
+      // value which means it should not change.
       //
-      if (const strings* src = cast_null<strings> (
-            rs["config.config.environment"]))
+      if (const strings* src = cast_null<strings> (rs[c_e]))
       {
         vector<const char*>& dst (rs.root_extra->environment);
 
         // The idea is to only copy entries that are effective, that is those
         // that actually override something in the environment. This should be
-        // both more efficient and less noisy (e.g., if we decide to print
-        // this in diagnostics).
+        // both more efficient and less noisy (e.g., if we need to print this
+        // in diagnostics).
         //
         // Note that config.config.environment may contain duplicates and the
         // last entry should have effect.
@@ -429,7 +488,8 @@ namespace build2
           if (find_if (src->rbegin (), i,
                        [&v, p] (const string& v1)
                        {
-                         return compare_env (v, p, v1, v1.find ('='));
+                         return saved_environment::compare (
+                           v, v1, p, v1.find ('='));
                        }) != i)
             continue;
 
diff --git a/libbuild2/config/module.hxx b/libbuild2/config/module.hxx
index b8b5d07..92fed32 100644
--- a/libbuild2/config/module.hxx
+++ b/libbuild2/config/module.hxx
@@ -4,6 +4,8 @@
 #ifndef LIBBUILD2_CONFIG_MODULE_HXX
 #define LIBBUILD2_CONFIG_MODULE_HXX
 
+#include <cstring> // strncmp()
+
 #include <libbutl/prefix-map.mxx>
 
 #include <libbuild2/types.hxx>
@@ -50,7 +52,7 @@ namespace build2
       const_iterator
       find (const variable& var) const
       {
-        return std::find_if (
+        return find_if (
           begin (),
           end (),
           [&var] (const saved_variable& v) {return var == v.var;});
@@ -76,8 +78,69 @@ namespace build2
       }
     };
 
-    struct module: build2::module
+    // List of environment variable names that effect this project.
+    //
+    // Note that on Windows environment variable names are case-insensitive.
+    //
+    struct saved_environment: vector<string>
     {
+      // Compare environment variable names.
+      //
+      static inline bool
+      compare (const string& x,
+               const string& y,
+               size_t xn = string::npos,
+               size_t yn = string::npos)
+      {
+        if (xn == string::npos) xn = x.size ();
+        if (yn == string::npos) yn = y.size ();
+
+        return xn == yn &&
+#ifdef _WIN32
+          icasecmp (x.c_str (), y.c_str (), xn) == 0
+#else
+          strncmp (x.c_str (), y.c_str (), xn) == 0
+#endif
+          ;
+      }
+
+      iterator
+      find (const string& v)
+      {
+        return find_if (
+          begin (),
+          end (),
+          [&v] (const string& v1) {return compare (v, v1);});
+      }
+
+      const_iterator
+      find (const string& v) const
+      {
+        return find_if (
+          begin (),
+          end (),
+          [&v] (const string& v1) {return compare (v, v1);});
+      }
+
+      void
+      insert (string v)
+      {
+        if (find (v) == end ())
+          push_back (move (v));
+      }
+
+      void
+      erase (const string& v)
+      {
+        auto i (find (v));
+        if (i != end ())
+          vector<string>::erase (i);
+      }
+    };
+
+    class module: public build2::module
+    {
+    public:
       config::saved_modules saved_modules;
 
       // Return true if variable/module were newly inserted.
@@ -110,6 +173,9 @@ namespace build2
         return nullptr;
       }
 
+      config::saved_environment saved_environment;
+      strings old_environment;
+
       // Configure/disfigure hooks.
       //
       static bool
diff --git a/libbuild2/config/operation.cxx b/libbuild2/config/operation.cxx
index c5e8004..760f998 100644
--- a/libbuild2/config/operation.cxx
+++ b/libbuild2/config/operation.cxx
@@ -157,24 +157,19 @@ namespace build2
 
     // If inherit is false, then don't rely on inheritance from outer scopes.
     //
+    // @@ We are modifying the module (marking additional variables as saved)
+    //    and this function can be called from a buildfile (probably only
+    //    during serial execution but still).
+    //
     void
     save_config (const scope& rs,
                  ostream& os, const path_name& on,
                  bool inherit,
+                 module& mod,
                  const project_set& projects)
     {
       context& ctx (rs.ctx);
 
-      // @@ We are modifying the module (marking additional variables as
-      // saved) and this function can be called from a buildfile (probably
-      // only during serial execution but still).
-      //
-      module* mod (rs.find_module<module> (module::name));
-
-      if (mod == nullptr)
-        fail (on) << "no configuration information available during this "
-                  << "meta-operation";
-
       names storage;
 
       auto info_value = [&storage] (diag_record& dr, const value& v) mutable
@@ -234,18 +229,18 @@ namespace build2
               var->name.compare (0, 14, "config.config.") == 0)
             continue;
 
-          if (mod->find_variable (*var)) // Saved or unsaved.
+          if (mod.find_variable (*var)) // Saved or unsaved.
             continue;
 
           const value& v (p.first->second);
 
           pair<bool, bool> r (save_config_variable (*var,
-                                                    mod->persist,
+                                                    mod.persist,
                                                     false /* inherited */,
                                                     true  /* unused */));
           if (r.first) // save
           {
-            mod->save_variable (*var, 0);
+            mod.save_variable (*var, 0);
 
             if (r.second) // warn
             {
@@ -281,7 +276,7 @@ namespace build2
 
         // Save config variables.
         //
-        for (auto p: mod->saved_modules.order)
+        for (auto p: mod.saved_modules.order)
         {
           const string& sname (p.second->first);
           const saved_variables& svars (p.second->second);
@@ -542,6 +537,7 @@ namespace build2
     save_config (const scope& rs,
                  const path& f,
                  bool inherit,
+                 module& mod,
                  const project_set& projects)
     {
       path_name fn (f);
@@ -555,7 +551,8 @@ namespace build2
       try
       {
         ofdstream ofs;
-        save_config (rs, open_file_or_stdout (fn, ofs), fn, inherit, projects);
+        save_config (
+          rs, open_file_or_stdout (fn, ofs), fn, inherit, mod, projects);
         ofs.close ();
       }
       catch (const io_error& e)
@@ -564,10 +561,81 @@ namespace build2
       }
     }
 
+    // Update config.config.environment value for a hermetic configuration.
+    //
+    static void
+    save_environment (scope& rs, module& mod)
+    {
+      // Here we have two parts: (1) get the list of environment variables we
+      // need to save and (2) save their values in config.config.environment.
+      //
+      // The saved_environment list should by now contain all the project-
+      // specific environment variables. To that we add builtin defaults and
+      // then filter the result against config.config.hermetic.environment
+      // inclusions/exclusions.
+      //
+      auto& vars (mod.saved_environment);
+
+      vars.insert ("PATH");
+
+#if   defined(_WIN32)
+#elif defined(__APPLE__)
+      vars.insert ("DYLD_LIBRARY_PATH");
+#else // Linux, FreeBSD, NetBSD, OpenBSD
+      vars.insert ("LD_LIBRARY_PATH");
+#endif
+
+      for (const pair<string, optional<bool>>& p:
+             cast_empty<hermetic_environment> (
+               rs["config.config.hermetic.environment"]))
+      {
+        if (!p.second || *p.second)
+          vars.insert (p.first);
+        else
+          vars.erase (p.first);
+      }
+
+      // Get the values.
+      //
+      strings vals;
+      {
+        // Set the project environment before querying the values. Note that
+        // the logic in init() makes sure that this is all we need to do to
+        // handle reload (in which case we should still be using
+        // config.config.environment from amalgamation, if any).
+        //
+        auto_project_env penv (rs);
+
+        for (const string& var: vars)
+        {
+          if (optional<string> val = getenv (var))
+          {
+            vals.push_back (var + '=' + *val);
+          }
+          else
+            vals.push_back (var); // Unset.
+        }
+      }
+
+      value& v (rs.assign (*rs.ctx.var_pool.find ("config.config.environment")));
+
+      // Note that setting new config.config.environment value invalidates the
+      // project's environment (scope::root_extra::environment) which could be
+      // queried in the post-configuration hook. We could re-initialize it but
+      // the c.c.e value from amalgamation could be referenced by subprojects.
+      // So instead it seems easier to just save the old value in the module.
+      //
+      if (v)
+        mod.old_environment = move (v.as<strings> ());
+
+      v = move (vals);
+    }
+
     static void
     configure_project (action a,
-                       const scope& rs,
+                       scope& rs,
                        const variable* c_s, // config.config.save
+                       module& mod,
                        project_set& projects)
     {
       tracer trace ("configure_project");
@@ -597,6 +665,12 @@ namespace build2
       {
         l5 ([&]{trace << "completely configuring " << out_root;});
 
+        // Save the environment if this configuration is hermetic (see init()
+        // for the other half of this logic).
+        //
+        if (cast_false<bool> (rs["config.config.hermetic"]))
+          save_environment (rs, mod);
+
         // Save src-root.build unless out_root is the same as src.
         //
         if (c_s == nullptr && out_root != src_root)
@@ -621,7 +695,7 @@ namespace build2
         // may end up leaving it in partially configured state.
         //
         if (c_s == nullptr)
-          save_config (rs, config_file (rs), true /* inherit */, projects);
+          save_config (rs, config_file (rs), true /* inherit */, mod, projects);
         else
         {
           lookup l (rs[*c_s]);
@@ -634,7 +708,8 @@ namespace build2
             // still want to support this mode somehow in the future (it seems
             // like an override of config.config.persist should do the trick).
             //
-            save_config (rs, cast<path> (l), false /* inherit */, projects);
+            save_config (
+              rs, cast<path> (l), false /* inherit */, mod, projects);
           }
         }
       }
@@ -645,11 +720,8 @@ namespace build2
 
       if (c_s == nullptr)
       {
-        if (module* m = rs.find_module<module> (module::name))
-        {
-          for (auto hook: m->configure_post_)
-            hook (a, rs);
-        }
+        for (auto hook: mod.configure_post_)
+          hook (a, rs);
       }
 
       // Configure subprojects that have been loaded.
@@ -660,15 +732,19 @@ namespace build2
         {
           const dir_path& pd (p.second);
           dir_path out_nroot (out_root / pd);
-          const scope& nrs (ctx.scopes.find_out (out_nroot));
+          scope& nrs (ctx.scopes.find_out (out_nroot).rw ());
 
-          // @@ Strictly speaking we need to check whether the config module
-          //    was loaded for this subproject.
+          // Skip this subproject if it is not loaded or doesn't use the
+          // config module.
           //
-          if (nrs.out_path () != out_nroot) // This subproject not loaded.
-            continue;
+          if (nrs.out_path () == out_nroot)
+          {
+            if (module* m = rs.find_module<module> (module::name))
+            {
+              configure_project (a, nrs, c_s, *m, projects);
+            }
+          }
 
-          configure_project (a, nrs, c_s, projects);
         }
       }
     }
@@ -874,7 +950,11 @@ namespace build2
             }
           }
 
-          configure_project (a, *rs, c_s, projects);
+          configure_project (a,
+                             rs->rw (),
+                             c_s,
+                             *rs->find_module<module> (module::name),
+                             projects);
         }
       }
     }
diff --git a/libbuild2/config/operation.hxx b/libbuild2/config/operation.hxx
index a887eb7..ded3c8b 100644
--- a/libbuild2/config/operation.hxx
+++ b/libbuild2/config/operation.hxx
@@ -15,6 +15,8 @@ namespace build2
 {
   namespace config
   {
+    class module;
+
     extern const meta_operation_info mo_configure;
     extern const meta_operation_info mo_disfigure;
 
@@ -37,7 +39,12 @@ namespace build2
     save_config (const scope& rs,
                  ostream&, const path_name&,
                  bool inherit,
+                 module&,
                  const project_set&);
+
+    // See config.config.hermetic.environment.
+    //
+    using hermetic_environment = vector<pair<string, optional<bool>>>;
   }
 }