Add ability to serialize compilation/linking in cc rules

Specifically, both the C/C++ compiler and link rules now recognize the
cc.serialize boolean variable which instructs them to compiler/link
serially with regards to any other recipe.

This is primarily useful when compiling large translation units or linking
large binaries that require so much memory that doing that in parallel with
other compilation/linking jobs is likely to summon the OOM killer. For
example:

obj{memory-hog}: cc.serialize = true

6 files changed, 45 insertions, 8 deletions

diff --git a/libbuild2/c/init.cxx b/libbuild2/c/init.cxx
index 10b3d6a..8bc2f7d 100644
--- a/libbuild2/c/init.cxx
+++ b/libbuild2/c/init.cxx
@@ -345,6 +345,7 @@ namespace build2
         vp["cc.module_name"],
         vp["cc.importable"],
         vp["cc.reprocess"],
+        vp["cc.serialize"],
 
         vp.insert<string>   ("c.preprocessed"), // See cxx.preprocessed.
         nullptr,                                // No __symexport (no modules).
diff --git a/libbuild2/cc/common.hxx b/libbuild2/cc/common.hxx
index 6347005..cb85632 100644
--- a/libbuild2/cc/common.hxx
+++ b/libbuild2/cc/common.hxx
@@ -117,6 +117,7 @@ namespace build2
       const variable& c_module_name;  // cc.module_name
       const variable& c_importable;   // cc.importable
       const variable& c_reprocess;    // cc.reprocess
+      const variable& c_serialize;    // cc.serialize
 
       const variable& x_preprocessed; // x.preprocessed
       const variable* x_symexport;    // x.features.symexport
diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx
index d490c8e..2e4775e 100644
--- a/libbuild2/cc/compile-rule.cxx
+++ b/libbuild2/cc/compile-rule.cxx
@@ -7830,6 +7830,14 @@ namespace build2
       if (!env.empty ())
         env.push_back (nullptr);
 
+      // We have no choice but to serialize early if we want the command line
+      // printed shortly before actually executing the compiler. Failed that,
+      // it may look like we are still executing in parallel.
+      //
+      scheduler::alloc_guard jobs_ag;
+      if (!ctx.dry_run && cast_false<bool> (t[c_serialize]))
+        jobs_ag = scheduler::alloc_guard (*ctx.sched, phase_unlock (nullptr));
+
       // With verbosity level 2 print the command line as if we are compiling
       // the source file, not its preprocessed version (so that it's easy to
       // copy and re-run, etc). Only at level 3 and above print the real deal.
@@ -7994,6 +8002,8 @@ namespace build2
           throw failed ();
         }
 
+        jobs_ag.deallocate ();
+
         if (md.deferred_failure)
           fail << "expected error exit status from " << x_lang << " compiler";
       }
diff --git a/libbuild2/cc/init.cxx b/libbuild2/cc/init.cxx
index 1bddbb2..e124450 100644
--- a/libbuild2/cc/init.cxx
+++ b/libbuild2/cc/init.cxx
@@ -185,6 +185,13 @@ namespace build2
       //
       vp.insert<bool> ("cc.reprocess");
 
+      // Execute serially with regards to any other recipe. This is primarily
+      // useful when compiling large translation units or linking large
+      // binaries that require so much memory that doing that in parallel with
+      // other compilation/linking jobs is likely to summon the OOM killer.
+      //
+      vp.insert<bool> ("cc.serialize");
+
       // Register scope operation callback.
       //
       // It feels natural to clean up sidebuilds as a post operation but that
diff --git a/libbuild2/cc/link-rule.cxx b/libbuild2/cc/link-rule.cxx
index 704fb47..08a60b9 100644
--- a/libbuild2/cc/link-rule.cxx
+++ b/libbuild2/cc/link-rule.cxx
@@ -3969,6 +3969,14 @@ namespace build2
           try_rmfile (relt, true);
       }
 
+      // We have no choice but to serialize early if we want the command line
+      // printed shortly before actually executing the linker. Failed that, it
+      // may look like we are still executing in parallel.
+      //
+      scheduler::alloc_guard jobs_ag;
+      if (!ctx.dry_run && cast_false<bool> (t[c_serialize]))
+        jobs_ag = scheduler::alloc_guard (*ctx.sched, phase_unlock (nullptr));
+
       if (verb == 1)
         print_diag (lt.static_library () ? "ar" : "ld", t);
       else if (verb == 2)
@@ -3989,10 +3997,15 @@ namespace build2
       //
       // Note that we are not going to bother with oargs for this.
       //
+      // Note also that we now have scheduler::serialize() which allows us to
+      // block until full parallelism is available (this mode can currently
+      // be forced with cc.serialize=true; maybe we should invent something
+      // like config.cc.link_serialize or some such which can be used when
+      // LTO is enabled).
+      //
       string jobs_arg;
-      scheduler::alloc_guard jobs_extra;
 
-      if (!lt.static_library ())
+      if (!ctx.dry_run && !lt.static_library ())
       {
         switch (ctype)
         {
@@ -4008,8 +4021,10 @@ namespace build2
             auto i (find_option_prefix ("-flto", args.rbegin (), args.rend ()));
             if (i != args.rend () && strcmp (*i, "-flto=auto") == 0)
             {
-              jobs_extra = scheduler::alloc_guard (*ctx.sched, 0);
-              jobs_arg = "-flto=" + to_string (1 + jobs_extra.n);
+              if (jobs_ag.n == 0) // Might already have (see above).
+                jobs_ag = scheduler::alloc_guard (*ctx.sched, 0);
+
+              jobs_arg = "-flto=" + to_string (1 + jobs_ag.n);
               *i = jobs_arg.c_str ();
             }
             break;
@@ -4027,8 +4042,10 @@ namespace build2
                 strcmp (*i, "-flto=thin") == 0 &&
                 !find_option_prefix ("-flto-jobs=", args))
             {
-              jobs_extra = scheduler::alloc_guard (*ctx.sched, 0);
-              jobs_arg = "-flto-jobs=" + to_string (1 + jobs_extra.n);
+              if (jobs_ag.n == 0) // Might already have (see above).
+                jobs_ag = scheduler::alloc_guard (*ctx.sched, 0);
+
+              jobs_arg = "-flto-jobs=" + to_string (1 + jobs_ag.n);
               args.insert (i.base (), jobs_arg.c_str ()); // After -flto=thin.
             }
             break;
@@ -4200,8 +4217,6 @@ namespace build2
             if (!e)
               throw failed ();
           }
-
-          jobs_extra.deallocate ();
         }
         catch (const process_error& e)
         {
@@ -4281,6 +4296,8 @@ namespace build2
         }
       }
 
+      jobs_ag.deallocate ();
+
       // For Windows generate (or clean up) rpath-emulating assembly.
       //
       if (tclass == "windows")
diff --git a/libbuild2/cxx/init.cxx b/libbuild2/cxx/init.cxx
index 7eedb4a..8159d18 100644
--- a/libbuild2/cxx/init.cxx
+++ b/libbuild2/cxx/init.cxx
@@ -951,6 +951,7 @@ namespace build2
         vp["cc.module_name"],
         vp["cc.importable"],
         vp["cc.reprocess"],
+        vp["cc.serialize"],
 
         // Ability to signal that source is already (partially) preprocessed.
         // Valid values are 'none' (not preprocessed), 'includes' (no #include