Initial support for amalgamation/subprojects

For now both need to be manually specified in src bootstrap. At
this stage main() loads any outer root scopes while include loads
any inner.

8 files changed, 277 insertions, 60 deletions

diff --git a/build/b.cxx b/build/b.cxx
index d7a5638..49abe5c 100644
--- a/build/b.cxx
+++ b/build/b.cxx
@@ -89,41 +89,39 @@ namespace build
     return path ();
   }
 
+  // Create and bootstrap outer root scopes, if any. Loading is
+  // done by root_pre().
+  //
   static void
-  bootstrap_out (scope& root)
+  create_outer_roots (scope& root)
   {
-    path bf (root.path () / path ("build/bootstrap/src-root.build"));
+    auto v (root.ro_variables ()["amalgamation"]);
 
-    if (!file_exists (bf))
+    if (!v)
       return;
 
-    //@@ TODO: if bootstrap files can source other bootstrap files
-    //   (the way to express dependecies), then we need a way to
-    //   prevent multiple sourcing. We handle it here but we still
-    //   need something like source_once (once [scope] source).
-    //
-    source_once (bf, root, root);
-  }
+    const path& d (v.as<const path&> ());
+    path out_root (root.path () / d);
+    path src_root (root.src_path () / d);
+    out_root.normalize ();
+    src_root.normalize ();
 
-  // Return true if we loaded anything.
-  //
-  static bool
-  bootstrap_src (scope& root)
-  {
-    tracer trace ("bootstrap_src");
-
-    path bf (root.src_path () / path ("build/bootstrap.build"));
+    scope& rs (create_root (out_root, src_root));
 
-    if (!file_exists (bf))
-      return false;
+    bootstrap_out (rs);
 
-    // We assume that bootstrap out cannot load this file explicitly. It
-    // feels wrong to allow this since that makes the whole bootstrap
-    // process hard to reason about. But we may try to bootstrap the
-    // same root scope multiple time.
+    // Check if the bootstrap process changed src_root.
     //
-    source_once (bf, root, root);
-    return true;
+    const path& p (rs.variables["src_root"].as<const path&> ());
+
+    if (src_root != p)
+      fail << "bootstrapped src_root " << p << " does not match "
+           << "amalgamated " << src_root;
+
+    rs.src_path_ = &p;
+
+    bootstrap_src (rs);
+    create_outer_roots (rs);
   }
 }
 
@@ -502,31 +500,11 @@ main (int argc, char* argv[])
           // files, if any. Note that we might already have done this
           // as a result of one of the preceding target processing.
           //
-          scope& rs (scopes.insert (out_root, true).first);
-
-          // Enter built-in meta-operation and operation names. Note that
-          // the order of registration should match the id constants; see
-          // <operation> for details. Loading of modules (via the src_root
-          // bootstrap; see below) can result in additional names being
-          // added.
-          //
-          if (rs.meta_operations.empty ())
-          {
-            assert (rs.meta_operations.insert (perform) == perform_id);
-
-            assert (rs.operations.insert (default_) == default_id);
-            assert (rs.operations.insert (update) == update_id);
-            assert (rs.operations.insert (clean) == clean_id);
-          }
-
-          rs.variables["out_root"] = out_root;
-
-          // If we know src_root, add that variable as well. This could
+          // If we know src_root, set that variable as well. This could
           // be of use to the bootstrap file (other than src-root.build,
           // which, BTW, doesn't need to exist if src_root == out_root).
           //
-          if (!src_root.empty ())
-            rs.variables["src_root"] = src_root;
+          scope& rs (create_root (out_root, src_root));
 
           bootstrap_out (rs);
 
@@ -587,6 +565,34 @@ main (int argc, char* argv[])
           //
           bool bootstrapped (bootstrap_src (rs));
 
+          // Check that out_root that we have found is the innermost root
+          // for this project. If it is not, then it means we are trying
+          // to load a disfigured sub-project and that we do not support.
+          // Why don't we support it? Because things are already complex
+          // enough here.
+          //
+          if (auto v = rs.ro_variables ()["subprojects"])
+          {
+            for (const name& n: v.as<const list_value&> ().data)
+            {
+              // Should be a list of directories.
+              //
+              if (!n.type.empty () || !n.value.empty () || n.dir.empty ())
+                fail << "expected directory in subprojects variable "
+                     << "instead of " << n;
+
+              if (out_base.sub (out_root / n.dir))
+                fail << tn << " is in a subproject of " << out_root <<
+                  info << "explicitly specify src_base for this target";
+            }
+          }
+
+          // Create and bootstrap outer roots if any. Loading is done
+          // by root_pre() (that would normally be called by the meta-
+          // operation's load() callback below).
+          //
+          create_outer_roots (rs);
+
           // The src bootstrap should have loaded all the modules that
           // may add new meta/operations. So at this stage they should
           // all be known. We store the combined action id in uint8_t;
diff --git a/build/file b/build/file
index 669d040..9975e06 100644
--- a/build/file
+++ b/build/file
@@ -25,6 +25,23 @@ namespace build
   void
   source_once (const path& buildfile, scope& root, scope& base, scope& once);
 
+  // Create project's root scope. Only set the src_root variable is the
+  // passed src_root value is not empty.
+  //
+  scope&
+  create_root (const path& out_root, const path& src_root);
+
+  // Bootstrap the project's root scope, the out part.
+  //
+  void
+  bootstrap_out (scope& root);
+
+  // Bootstrap the project's root scope, the src part. Return true if
+  // we loaded anything.
+  //
+  bool
+  bootstrap_src (scope& root);
+
   // Load project's root[-pre].build unless already loaded.
   //
   void
diff --git a/build/file.cxx b/build/file.cxx
index 7bb8bb6..0b0220d 100644
--- a/build/file.cxx
+++ b/build/file.cxx
@@ -53,16 +53,111 @@ namespace build
     source (bf, root, base);
   }
 
-  void
-  root_pre (scope& root)
+  scope&
+  create_root (const path& out_root, const path& src_root)
   {
-    tracer trace ("root_pre");
+    scope& rs (scopes.insert (out_root, true).first);
+
+    // Enter built-in meta-operation and operation names. Note that
+    // the order of registration should match the id constants; see
+    // <operation> for details. Loading of modules (via the src
+    // bootstrap; see below) can result in additional names being
+    // added.
+    //
+    if (rs.meta_operations.empty ())
+    {
+      assert (rs.meta_operations.insert (perform) == perform_id);
 
-    path bf (root.src_path () / path ("build/root.build"));
+      assert (rs.operations.insert (default_) == default_id);
+      assert (rs.operations.insert (update) == update_id);
+      assert (rs.operations.insert (clean) == clean_id);
+    }
+
+    // If this is already a root scope, verify that things are
+    // consistent.
+    //
+    {
+      auto v (rs.variables["out_root"]);
+
+      if (!v)
+        v = out_root;
+      else
+      {
+        const path& p (v.as<const path&> ());
+
+        if (p != out_root)
+          fail << "new out_root " << out_root << " does not match "
+               << "existing " << p;
+      }
+    }
+
+    if (!src_root.empty ())
+    {
+      auto v (rs.variables["src_root"]);
+
+      if (!v)
+        v = src_root;
+      else
+      {
+        const path& p (v.as<const path&> ());
+
+        if (p != src_root)
+          fail << "new src_root " << src_root << " does not match "
+               << "existing " << p;
+      }
+    }
+
+    return rs;
+  }
+
+  void
+  bootstrap_out (scope& root)
+  {
+    path bf (root.path () / path ("build/bootstrap/src-root.build"));
 
     if (!file_exists (bf))
       return;
 
+    //@@ TODO: if bootstrap files can source other bootstrap files
+    //   (the way to express dependecies), then we need a way to
+    //   prevent multiple sourcing. We handle it here but we still
+    //   need something like source_once (once [scope] source).
+    //
     source_once (bf, root, root);
   }
+
+  bool
+  bootstrap_src (scope& root)
+  {
+    tracer trace ("bootstrap_src");
+
+    path bf (root.src_path () / path ("build/bootstrap.build"));
+
+    if (!file_exists (bf))
+      return false;
+
+    // We assume that bootstrap out cannot load this file explicitly. It
+    // feels wrong to allow this since that makes the whole bootstrap
+    // process hard to reason about. But we may try to bootstrap the
+    // same root scope multiple time.
+    //
+    source_once (bf, root, root);
+    return true;
+  }
+
+  void
+  root_pre (scope& root)
+  {
+    tracer trace ("root_pre");
+
+    // First load outer roots, if any.
+    //
+    if (scope* rs = root.parent_scope ()->root_scope ())
+      root_pre (*rs);
+
+    path bf (root.src_path () / path ("build/root.build"));
+
+    if (file_exists (bf))
+      source_once (bf, root, root);
+  }
 }
diff --git a/build/parser b/build/parser
index 4423e25..1597432 100644
--- a/build/parser
+++ b/build/parser
@@ -83,6 +83,9 @@ namespace build
     void
     process_default_target (token&);
 
+    void
+    create_inner_roots (const path&);
+
     // Lexer.
     //
   private:
diff --git a/build/parser.cxx b/build/parser.cxx
index 1347ebd..2845143 100644
--- a/build/parser.cxx
+++ b/build/parser.cxx
@@ -20,6 +20,7 @@
 #include <build/prerequisite>
 #include <build/variable>
 #include <build/module>
+#include <build/file>
 #include <build/diagnostics>
 #include <build/context>
 
@@ -44,12 +45,12 @@ namespace build
     root_ = &root;
     default_target_ = nullptr;
 
-    out_root_ = &root["out_root"].as<const path&> ();
+    out_root_ = &root.ro_variables ()["out_root"].as<const path&> ();
 
     // During bootstrap we may not know src_root yet.
     //
     {
-      auto v (root["src_root"]);
+      auto v (root.ro_variables ()["src_root"]);
       src_root_ = v ? &v.as<const path&> () : nullptr;
     }
 
@@ -435,6 +436,65 @@ namespace build
       fail (t) << "expected newline instead of " << t;
   }
 
+  // Create, bootstrap, and load outer root scopes, if any. Also
+  // update the parser state to point to the innermost root scope.
+  //
+  void parser::
+  create_inner_roots (const path& out_base)
+  {
+    auto v (root_->ro_variables ()["subprojects"]);
+
+    if (!v)
+      return;
+
+    for (const name& n: v.as<const list_value&> ().data)
+    {
+      // Should be a list of directories.
+      //
+      if (!n.type.empty () || !n.value.empty () || n.dir.empty ())
+        fail << "expected directory in subprojects variable "
+             << "instead of " << n;
+
+      path out_root (*out_root_ / n.dir);
+
+      if (!out_base.sub (out_root))
+        continue;
+
+      path src_root (*src_root_ / n.dir);
+      scope& rs (create_root (out_root, src_root));
+
+      bootstrap_out (rs);
+
+      // Check if the bootstrap process changed src_root.
+      //
+      const path& p (rs.variables["src_root"].as<const path&> ());
+
+      if (src_root != p)
+        fail << "bootstrapped src_root " << p << " does not match "
+             << "subproject " << src_root;
+
+      rs.src_path_ = &p;
+
+      bootstrap_src (rs);
+
+      // Load the root scope.
+      //
+      root_pre (rs);
+
+      // Update parser state.
+      //
+      root_ = &rs;
+      out_root_ = &rs.variables["out_root"].as<const path&> ();
+      src_root_ = &p;
+
+      // See if there are more inner roots.
+      //
+      create_inner_roots (out_base);
+
+      break; // Can only be in one sub-project.
+    }
+  }
+
   void parser::
   include (token& t, token_type& tt)
   {
@@ -510,6 +570,12 @@ namespace build
         out_base = out_src (src_base, *out_root_, *src_root_);
       }
 
+      // Create, bootstrap and load root scope(s) of subproject(s) that
+      // this out_base belongs to.
+      //
+      scope* ors (root_);
+      create_inner_roots (out_base);
+
       ifstream ifs (p.string ());
 
       if (!ifs.is_open ())
@@ -553,6 +619,13 @@ namespace build
       scope_ = os;
       lexer_ = ol;
       path_ = op;
+
+      if (root_ != ors)
+      {
+        root_ = ors;
+        out_root_ = &root_->ro_variables ()["out_root"].as<const path&> ();
+        src_root_ = &root_->ro_variables ()["src_root"].as<const path&> ();
+      }
     }
 
     if (tt == type::newline)
diff --git a/build/scope b/build/scope
index 057fbc1..2b621e7 100644
--- a/build/scope
+++ b/build/scope
@@ -42,9 +42,9 @@ namespace build
     bool
     root () const {return root_ == this;}
 
-    // Variable lookup. Note that this find, not find or insert like
-    // in the variable_map, because we also search in outer scopes.
-    // For the latter use the variables map directly.
+    // Variable lookup. Note that this is find, not find or insert like
+    // in the variable_map, because we also search in outer scopes. For
+    // the latter use the variables map directly.
     //
   public:
     value_proxy
@@ -57,6 +57,10 @@ namespace build
 
   public:
     variable_map variables;
+
+    const variable_map&
+    ro_variables () const {return variables;}
+
     prerequisite_set prerequisites;
 
     // Meta/operations supported by this project (set on the root
diff --git a/build/scope.cxx b/build/scope.cxx
index cdca747..68eba3d 100644
--- a/build/scope.cxx
+++ b/build/scope.cxx
@@ -64,10 +64,10 @@ namespace build
           if (p == nullptr)
             p = c.parent_;
 
-          if (root)
+          if (root && c.root_ == p->root_) // No intermediate root.
             c.root_ = &s;
 
-          if (p == c.parent_) // A scope without an intermediate parent.
+          if (p == c.parent_) // No intermediate parent.
             c.parent_ = &s;
         }
 
@@ -90,7 +90,9 @@ namespace build
       for (++r.first; r.first != r.second; ++r.first)
       {
         scope& c (r.first->second);
-        c.root_ = &s;
+
+        if (c.root_ == s.root_) // No intermediate root.
+          c.root_ = &s;
       }
 
       s.root_ = &s;
diff --git a/build/variable b/build/variable
index 500cc00..aca0bed 100644
--- a/build/variable
+++ b/build/variable
@@ -240,6 +240,23 @@ namespace build
       return value_proxy (&base::operator[] (v), &scope_);
     }
 
+    value_proxy
+    operator[] (const std::string& v) const
+    {
+      return operator[] (variable_pool.find (v));
+    }
+
+    value_proxy
+    operator[] (const variable& v) const
+    {
+      auto i (find (v));
+      return i != end ()
+        // @@ To do this properly we seem to need ro_value_proxy.
+        //
+        ? value_proxy (&const_cast<value_ptr&> (i->second), &scope_)
+        : value_proxy (nullptr, nullptr);
+    }
+
     std::pair<iterator, iterator>
     find_namespace (const std::string& ns)
     {