Implement config.config.environment storage

5 files changed, 252 insertions, 99 deletions

diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx
index 1513a47..9911670 100644
--- a/libbuild2/config/init.cxx
+++ b/libbuild2/config/init.cxx
@@ -28,6 +28,59 @@ namespace build2
     void
     functions (function_map&); // functions.cxx
 
+    // Custom save function for config.config.environment.
+    //
+    // It tries to optimize the storage in subprojects by appending the
+    // difference (compared to the amalgamation's values) instead of storing
+    // the entire values.
+    //
+    static pair<names_view, const char*>
+    save_environment (const value& d, const value* b, names& storage)
+    {
+      if (b == nullptr)
+        return make_pair (reverse (d, storage), "=");
+
+      // The plan is to iterator over environment variables adding those that
+      // are not in base to storage. There is, however, a complication: we may
+      // see multiple entries for the same variable and only the last entry
+      // should have effect. So we need to iterate in reverse and check if
+      // we've already seen this variable.
+      //
+      const strings& ds (d.as<strings> ());
+      const strings& bs (b->as<strings> ());
+
+      for (auto i (ds.rbegin ()), e (ds.rend ()); i != e; ++i)
+      {
+        // Note that we must only consider variable names (up to first '=' if
+        // any).
+        //
+        const string& v (*i);
+        size_t p (v.find ('='));
+
+        // Check if we have already seen this variable.
+        //
+        if (find_if (ds.rbegin (), i,
+                     [&v, p] (const string& v1)
+                     {
+                       return v.compare (0, p, v1, 0, v1.find ('=')) == 0;
+                     }) != i)
+          continue;
+
+        // Check if there is the same value in base.
+        //
+        auto j (find_if (bs.rbegin (), bs.rend (),
+                         [&v, p] (const string& v1)
+                         {
+                           return v.compare (0, p, v1, 0, v1.find ('=')) == 0;
+                         }));
+
+        if (j == bs.rend () || *j != v)
+          storage.push_back (name (v));
+      }
+
+      return make_pair (names_view (storage), "+=");
+    }
+
     void
     boot (scope& rs, const location&, module_boot_extra& extra)
     {
@@ -96,6 +149,37 @@ namespace build2
       auto& c_p (vp.insert<vector<pair<string, string>>> (
                    "config.config.persist", true /* ovr */, v_p));
 
+      // Configuration environment variables.
+      //
+      // Environment variables used by tools (e.g., compilers), buildfiles
+      // (e.g., $getenv()), and the build system itself (e.g., to locate
+      // tools) in ways that affect the build result are in essence part of
+      // the project configuration.
+      //
+      // This variable allows storing environment variable overrides that
+      // should be applied to the environment when executing tools, etc., as
+      // part of a project build. Specifically, it contains a list of
+      // environment variable "sets" (<name>=<value>) and "unsets" (<name>).
+      // If multiple entries are specified for the same environment variable,
+      // the last entry has effect. For example:
+      //
+      // config.config.environment="LC_ALL=C LANG"
+      //
+      // Note that a subproject inherits overrides from its amalgamation (this
+      // semantics is the result of the way we optimize the storage of this
+      // variable in subproject's config.build; the thinking is that if a
+      // variable is not overridden by the subproject then it doesn't affect
+      // the build result and therefore it's irrelevant whether it has a value
+      // that came from the original environment of from the amalgamation
+      // override).
+      //
+      // Use the NULL or empty value to clear.
+      //
+      // @@ We could use =<name> as a "pass-through" instruction (e.g., if
+      //    we need to use original value in subproject).
+      //
+      auto& c_e (vp.insert<strings> ("config.config.environment", true /* ovr */));
+
       // Only create the module if we are configuring, creating, or
       // disfiguring or if it was requested with config.config.module (useful
       // if we need to call $config.save() during other meta-operations).
@@ -127,6 +211,9 @@ namespace build2
           m.save_module ("import", INT32_MIN);
 
           m.save_variable (c_p, save_null_omitted);
+          m.save_variable (c_e,
+                           save_null_omitted | save_empty_omitted | save_base,
+                           &save_environment);
         }
       }
 
diff --git a/libbuild2/config/module.cxx b/libbuild2/config/module.cxx
index bf68c4e..c84b1fa 100644
--- a/libbuild2/config/module.cxx
+++ b/libbuild2/config/module.cxx
@@ -12,7 +12,9 @@ namespace build2
   namespace config
   {
     bool module::
-    save_variable (const variable& var, optional<uint64_t> flags)
+    save_variable (const variable& var,
+                   optional<uint64_t> flags,
+                   save_variable_function* save)
     {
       const string& n (var.name);
 
@@ -43,7 +45,7 @@ namespace build2
         return false;
       }
 
-      sv.push_back (saved_variable {var, flags});
+      sv.push_back (saved_variable {var, flags, save});
       return true;
     }
 
diff --git a/libbuild2/config/module.hxx b/libbuild2/config/module.hxx
index 3afe721..b8b5d07 100644
--- a/libbuild2/config/module.hxx
+++ b/libbuild2/config/module.hxx
@@ -26,10 +26,19 @@ namespace build2
     // saved in the order populated. If flags are absent, then this variable
     // was marked as "unsaved" (always transient).
     //
+    // The optional save function can be used to implement custom variable
+    // saving, for example, as a difference appended to the base value. The
+    // second half of the result is the assignment operator to use.
+    //
+    using save_variable_function =
+      pair<names_view, const char*> (const value&,
+                                     const value* base,
+                                     names& storage);
     struct saved_variable
     {
       reference_wrapper<const variable> var;
       optional<uint64_t> flags;
+      save_variable_function* save;
     };
 
     struct saved_variables: vector<saved_variable>
@@ -74,7 +83,9 @@ namespace build2
       // Return true if variable/module were newly inserted.
       //
       bool
-      save_variable (const variable&, optional<uint64_t> flags);
+      save_variable (const variable&,
+                     optional<uint64_t> flags,
+                     save_variable_function* = nullptr);
 
       static void
       save_variable (scope&, const variable&, optional<uint64_t>);
diff --git a/libbuild2/config/operation.cxx b/libbuild2/config/operation.cxx
index d30de4d..c5e8004 100644
--- a/libbuild2/config/operation.cxx
+++ b/libbuild2/config/operation.cxx
@@ -286,7 +286,18 @@ namespace build2
           const string& sname (p.second->first);
           const saved_variables& svars (p.second->second);
 
-          bool first (true); // Separate modules with a blank line.
+          // Separate modules with a blank line.
+          //
+          auto first = [v = true] () mutable
+          {
+            if (v)
+            {
+              v = false;
+              return "\n";
+            }
+            return "";
+          };
+
           for (const saved_variable& sv: svars)
           {
             if (!sv.flags) // unsaved
@@ -306,7 +317,9 @@ namespace build2
             // inherited. We might also not have any value at all (see
             // unconfigured()).
             //
-            if (!l.defined () || (l->null && flags & save_null_omitted))
+            if (!l.defined () ||
+                (l->null     ? flags & save_null_omitted  :
+                 l->empty () ? flags & save_empty_omitted : false))
               continue;
 
             // Handle inherited from outer scope values.
@@ -315,90 +328,97 @@ namespace build2
             // we save the inherited values regardless of whether they are
             // used or not.
             //
-            if (inherit && !(l.belongs (rs) || l.belongs (ctx.global_scope)))
+            const value* base (nullptr);
+            if (inherit)
             {
-              // This is presumably an inherited value. But it could also be
-              // some left-over garbage. For example, an amalgamation could
-              // have used a module but then dropped it while its config
-              // values are still lingering in config.build. They are probably
-              // still valid and we should probably continue using them but we
-              // definitely want to move them to our config.build since they
-              // will be dropped from the amalgamation's config.build on the
-              // next reconfigure. Let's also warn the user just in case,
-              // unless there is no module and thus we couldn't really check
-              // (the latter could happen when calling $config.save() during
-              // other meta-operations, though it passes false for inherit).
-              //
-              // There is also another case that falls under this now that
-              // overrides are by default amalgamation-wide rather than just
-              // "project and subprojects": we may be (re-)configuring a
-              // subproject but the override is now set on the outer project's
-              // root.
+              // Return true if the specified value can be inherited from.
               //
-              bool found (false), checked (true);
-              const scope* r (&rs);
-              while ((r = r->parent_scope ()->root_scope ()) != nullptr)
+              auto find_inherited = [&on, &projects,
+                                     &info_value,
+                                     &sname, &rs, &var] (const lookup& org,
+                                                         const lookup& ovr)
               {
-                if (l.belongs (*r))
+                const lookup& l (ovr);
+
+                // This is presumably an inherited value. But it could also be
+                // some left-over garbage. For example, an amalgamation could
+                // have used a module but then dropped it while its config
+                // values are still lingering in config.build. They are
+                // probably still valid and we should probably continue using
+                // them but we definitely want to move them to our
+                // config.build since they will be dropped from the
+                // amalgamation's config.build on the next reconfigure. Let's
+                // also warn the user just in case, unless there is no module
+                // and thus we couldn't really check (the latter could happen
+                // when calling $config.save() during other meta-operations,
+                // though it passes false for inherit).
+                //
+                // There is also another case that falls under this now that
+                // overrides are by default amalgamation-wide rather than just
+                // "project and subprojects": we may be (re-)configuring a
+                // subproject but the override is now set on the outer
+                // project's root.
+                //
+                bool found (false), checked (true);
+                const scope* r (&rs);
+                while ((r = r->parent_scope ()->root_scope ()) != nullptr)
                 {
-                  // Find the config module (might not be there).
-                  //
-                  if (auto* m = r->find_module<const module> (module::name))
+                  if (l.belongs (*r))
                   {
-                    // Find the corresponding saved module.
+                    // Find the config module (might not be there).
                     //
-                    auto i (m->saved_modules.find (sname));
-
-                    if (i != m->saved_modules.end ())
+                    if (auto* m = r->find_module<const module> (module::name))
                     {
-                      // Find the variable.
+                      // Find the corresponding saved module.
                       //
-                      const saved_variables& sv (i->second);
-                      found = sv.find (var) != sv.end ();
+                      auto i (m->saved_modules.find (sname));
 
-                      // If not marked as saved, check whether overriden via
-                      // config.config.persist.
-                      //
-                      if (!found && m->persist != nullptr)
+                      if (i != m->saved_modules.end ())
                       {
-                        found = save_config_variable (
-                          var,
-                          m->persist,
-                          false /* inherited */,
-                          true  /* unused */).first;
+                        // Find the variable.
+                        //
+                        const saved_variables& sv (i->second);
+                        found = sv.find (var) != sv.end ();
+
+                        // If not marked as saved, check whether overriden via
+                        // config.config.persist.
+                        //
+                        if (!found && m->persist != nullptr)
+                        {
+                          found = save_config_variable (
+                            var,
+                            m->persist,
+                            false /* inherited */,
+                            true  /* unused */).first;
+                        }
+
+                        // Handle that other case: if this is an override but
+                        // the outer project itself is not being configured,
+                        // then we need to save this override.
+                        //
+                        // One problem with using the already configured
+                        // project set is that the outer project may be
+                        // configured only after us in which case both
+                        // projects will save the value. But perhaps this is a
+                        // feature, not a bug since this is how project-local
+                        // (%) override behaves.
+                        //
+                        if (found &&
+                            org != ovr &&
+                            projects.find (r) == projects.end ())
+                          found = false;
                       }
-
-                      // Handle that other case: if this is an override but
-                      // the outer project itself is not being configured,
-                      // then we need to save this override.
-                      //
-                      // One problem with using the already configured project
-                      // set is that the outer project may be configured only
-                      // after us in which case both projects will save the
-                      // value. But perhaps this is a feature, not a bug since
-                      // this is how project-local (%) override behaves.
-                      //
-                      if (found &&
-                          org.first != ovr.first &&
-                          projects.find (r) == projects.end ())
-                        found = false;
                     }
-                  }
-                  else
-                    checked = false;
+                    else
+                      checked = false;
 
-                  break;
+                    break;
+                  }
                 }
-              }
 
-              if (found)
-              {
-                // Inherited.
-                //
-                continue;
-              }
-              else
-              {
+                if (found)
+                  return true;
+
                 // If this value is not defined in a project's root scope,
                 // then something is broken.
                 //
@@ -411,7 +431,7 @@ namespace build2
                 // our own. One special case where we don't want to warn the
                 // user is if the variable is overriden.
                 //
-                if (checked && org.first == ovr.first)
+                if (checked && org == ovr)
                 {
                   diag_record dr;
                   dr << warn (on) << "saving previously inherited variable "
@@ -423,6 +443,39 @@ namespace build2
                   if (verb >= 2)
                     info_value (dr, *l);
                 }
+
+                return false;
+              };
+
+              // Inherit as-is.
+              //
+              if (!l.belongs (rs) &&
+                  !l.belongs (ctx.global_scope) &&
+                  find_inherited (org.first, ovr.first))
+                continue;
+              else if (flags & save_base)
+              {
+                // See if we can base our value on inherited.
+                //
+                if (const scope* ors = rs.parent_scope ()->root_scope ())
+                {
+                  pair<lookup, size_t> org (ors->lookup_original (var));
+                  pair<lookup, size_t> ovr (var.overrides == nullptr
+                                            ? org
+                                            : ors->lookup_override (var, org));
+                  const lookup& l (ovr.first);
+
+                  // We cannot base anything on an empty value.
+                  //
+                  if (l && !l->empty ())
+                  {
+                    // @@ It's not clear we want the checks/diagnostics in
+                    //    this case.
+                    //
+                    if (find_inherited (org.first, ovr.first))
+                      base = l.value;
+                  }
+                }
               }
             }
 
@@ -440,44 +493,42 @@ namespace build2
                 continue;
             }
 
-            // If we got here then we are saving this variable. Handle the
-            // blank line.
-            //
-            if (first)
-            {
-              os << endl;
-              first = false;
-            }
-
             // Handle the save_default_commented flag.
             //
             if ((org.first.defined () && org.first->extra) && // Default value.
                 org.first == ovr.first &&                     // Not overriden.
                 (flags & save_default_commented) != 0)
             {
-              os << '#' << n << " =" << endl;
+              os << first () << '#' << n << " =" << endl;
               continue;
             }
 
-            if (v)
+            if (v.null)
             {
-              storage.clear ();
-              names_view ns (reverse (v, storage));
+              os << first () << n << " = [null]" << endl;
+              continue;
+            }
 
-              os << n;
+            storage.clear ();
+            pair<names_view, const char*> p (
+              sv.save != nullptr
+              ? sv.save (v, base, storage)
+              : make_pair (reverse (v, storage), "="));
 
-              if (ns.empty ())
-                os << " =";
-              else
-              {
-                os << " = ";
-                to_stream (os, ns, true /* quote */, '@');
-              }
+            // Might becomes empty after a custom save function had at it.
+            //
+            if (p.first.empty () && (flags & save_empty_omitted))
+              continue;
 
-              os << endl;
+            os << first () << n << ' ' << p.second;
+
+            if (!p.first.empty ())
+            {
+              os << ' ';
+              to_stream (os, p.first, true /* quote */, '@');
             }
-            else
-              os << n << " = [null]" << endl;
+
+            os << endl;
           }
         }
       }
diff --git a/libbuild2/config/utility.hxx b/libbuild2/config/utility.hxx
index 0429555..45fdaba 100644
--- a/libbuild2/config/utility.hxx
+++ b/libbuild2/config/utility.hxx
@@ -57,6 +57,8 @@ namespace build2
     //
     const uint64_t save_default_commented = 0x01; // Based on value::extra.
     const uint64_t save_null_omitted      = 0x02; // Treat NULL as undefined.
+    const uint64_t save_empty_omitted     = 0x04; // Treat empty as undefined.
+    const uint64_t save_base              = 0x08; // Custom save with base.
 
     inline void
     save_variable (scope& rs, const variable& var, uint64_t flags = 0)