Rework tool importation along with cli module

Specifically, now config.<tool> (like config.cli) is handled by the import
machinery (it is like a shorter alias for config.import.<tool>.<tool>.exe
that we already had). And the cli module now uses that instead of custom
logic.

This also adds support for uniform tool metadata extraction that is handled by
the import machinery. As a result, a tool that follows the "build2 way" can be
imported with metadata by the buildfile and/or corresponding module without
any tool-specific code or brittleness associated with parsing --version or
similar outputs. See the cli tool/module for details.

Finally, two new flavors of the import directive are now supported: import!
triggers immediate importation skipping any rule-specific logic while import?
is optional import (analogous to using?). Note that optional import is always
immediate. There is also the import-specific metadata attribute which can be
specified for these two import flavors in order to trigger metadata
importation. For example:

import? [metadata] cli = cli%exe{cli}

if ($cli != [null])
  info "cli version $($cli:cli.version)"

1 files changed, 173 insertions, 262 deletions

diff --git a/build2/cli/init.cxx b/build2/cli/init.cxx
index c68a62f..8f9df4a 100644
--- a/build2/cli/init.cxx
+++ b/build2/cli/init.cxx
@@ -3,6 +3,7 @@
 
 #include <build2/cli/init.hxx>
 
+#include <libbuild2/file.hxx>
 #include <libbuild2/scope.hxx>
 #include <libbuild2/target.hxx>
 #include <libbuild2/variable.hxx>
@@ -12,337 +13,246 @@
 
 #include <libbuild2/cxx/target.hxx>
 
-#include <build2/cli/target.hxx>
 #include <build2/cli/rule.hxx>
-
-using namespace std;
-using namespace butl;
+#include <build2/cli/module.hxx>
+#include <build2/cli/target.hxx>
 
 namespace build2
 {
   namespace cli
   {
-    static const compile_rule compile_rule_;
+    // Remaining issues/semantics change:
+    //
+    // @@ Unconfigured caching.
+    //
+    // @@ Default-found cli used to result in config.cli=cli and now it's just
+    //    omitted (and default-not-found -- in config.cli.configured=false).
+    //
+    //    - Writing any default will take precedence over config.import.cli.
+    //      In fact, this duality is a bigger problem: if we have a config
+    //      that uses config.cli there is no way to reconfigure it to use
+    //      config.import.cli.
+    //
+    //    - We could have saved it commented.
+    //
+    //    - We could do this at the module level only since we also have
+    //      config.cli.options?
+    //
+    //    - Note that in the CLI compiler itself we now rely on default cli
+    //      being NULL/undefined. So if faving, should probably be commented
+    //      out. BUT: it will still be defined, so will need to be defined
+    //      NULL. Note also that long term the CLI compiler will not use the
+    //      module relying on an ad hoc recipe instead.
+    //
+    //    ! Maybe reserving NULL (instead of making it the same as NULL) for
+    //      this "configured to default" state and saving commented is not a
+    //      bad idea. Feels right to have some marker in config.build that
+    //      things are in effect. And I believe if config.import.cli is
+    //      specified, it will just be dropped.
 
     bool
-    config_init (scope& rs,
-                 scope& bs,
-                 const location& l,
-                 bool first,
-                 bool optional,
-                 module_init_extra&)
+    guess_init (scope& rs,
+                scope& bs,
+                const location& loc,
+                bool,
+                bool optional,
+                module_init_extra& extra)
     {
-      tracer trace ("cli::config_init");
-      l5 ([&]{trace << "for " << bs;});
+      tracer trace ("cli::guess_init");
+      l5 ([&]{trace << "for " << rs;});
 
-      // Enter variables.
+      // We only support root loading (which means there can only be one).
       //
-      if (first)
-      {
-        auto& vp (rs.var_pool ());
-
-        // The special config.cli=false value is recognized as an explicit
-        // request to leave the module unconfigured.
-        //
-        vp.insert<path>    ("config.cli");
-        vp.insert<strings> ("config.cli.options");
+      if (rs != bs)
+        fail (loc) << "cli.guess module must be loaded in project root";
 
-        //@@ TODO: split version into componets (it is stdver).
-        //
-        vp.insert<process_path> ("cli.path");
-        vp.insert<string>       ("cli.version");
-        vp.insert<string>       ("cli.checksum");
-        vp.insert<strings>      ("cli.options");
-      }
-
-      // Configuration.
+      // Adjust module config.build save priority (code generator).
       //
-      // The plan is as follows: try to configure the module. If this fails,
-      // we are using default values, and the module is optional, leave it
-      // unconfigured.
+      config::save_module (rs, "cli", 150);
+
+      // Enter metadata variables.
       //
-      using config::lookup_config;
-      using config::specified_config;
+      auto& vp (rs.var_pool ());
 
+      auto& v_ver (vp.insert<string> ("cli.version"));
+      auto& v_sum (vp.insert<string> ("cli.checksum"));
 
-      // First take care of the explicit request by the user to leave the
+      // Import the CLI compiler target.
+      //
+      // Note that the special config.cli=false value (recognized by the
+      // import machinery) is treated as an explicit request to leave the
       // module unconfigured.
       //
-      bool conf (true);
+      bool new_cfg (false);
+      pair<const exe*, import_kind> ir (
+        import_direct<exe> (
+          new_cfg,
+          rs,
+          name ("cli", dir_path (), "exe", "cli"), // cli%exe{cli}
+          true      /* phase2 */,
+          optional,
+          true      /* metadata */,
+          loc,
+          "module load"));
+
+      const exe* tgt (ir.first);
+
+      // Extract metadata.
+      //
+      auto* ver (tgt != nullptr ? &cast<string> (tgt->vars[v_ver]) : nullptr);
+      auto* sum (tgt != nullptr ? &cast<string> (tgt->vars[v_sum]) : nullptr);
 
-      if (const path* p = cast_null<path> (rs["config.cli"]))
+      // Print the report.
+      //
+      // If this is a configuration with new values, then print the report
+      // at verbosity level 2 and up (-v).
+      //
+      if (verb >= (new_cfg ? 2 : 3))
       {
-        conf = p->string () != "false";
+        diag_record dr (text);
+        dr << "cli " << project (rs) << '@' << rs << '\n';
 
-        if (!conf && !optional)
-          fail (l) << "non-optional module requested to be left unconfigured";
+        if (tgt != nullptr)
+          dr << "  cli        " << ir << '\n'
+             << "  version    " << *ver << '\n'
+             << "  checksum   " << *sum;
+        else
+          dr << "  cli        " << "not found, leaving unconfigured";
       }
 
-      if (conf)
-      {
-        // Otherwise we will only honor optional if the user didn't specify
-        // any cli configuration explicitly.
-        //
-        optional = optional && !specified_config (rs, "cli");
+      if (tgt == nullptr)
+        return false;
 
-        // If the configuration says we are unconfigured, then we should't
-        // re-run tests, etc. But we may still need to print the config
-        // report.
-        //
-        conf = !optional || !config::unconfigured (rs, "cli");
-      }
+      // The cli variable (untyped) is an imported compiler target name.
+      //
+      rs.assign ("cli") = tgt->as_name ();
+      rs.assign (v_sum) = *sum;
+      rs.assign (v_ver) = *ver;
 
-      if (first)
       {
-        // config.cli
-        //
-        process_path pp;
+        standard_version v (*ver);
 
-        // Return version or empty string if the cli executable is not found
-        // or is not the command line interface compiler.
-        //
-        // @@ This needs some more thinking/cleanup. Specifically, what does
-        //    it mean "cli not found"? Is it just not found in PATH? That plus
-        //    was not able to execute (e.g., some shared libraries missing)?
-        //    That plus cli that we found is something else?
-        //
-        auto test = [optional, &pp] (const path& cli) -> string
-        {
-          const char* args[] = {cli.string ().c_str (), "--version", nullptr};
-
-          // @@ TODO: redo using run_start()/run_finish() or even
-          //    run<string>(). We have the ability to ignore exit code and
-          //    redirect STDERR to STDOUT.
-
-          try
-          {
-            // Only search in PATH (specifically, omitting the current
-            // executable's directory on Windows).
-            //
-            pp = process::path_search (cli,
-                                       true        /* init */,
-                                       dir_path () /* fallback */,
-                                       true        /* path_only */);
-            args[0] = pp.recall_string ();
-
-            if (verb >= 3)
-              print_process (args);
-
-            process pr (pp, args, 0, -1); // Open pipe to stdout.
-
-            try
-            {
-              ifdstream is (move (pr.in_ofd), fdstream_mode::skip);
-
-              // The version should be the last word on the first line. But
-              // also check the prefix since there are other things called
-              // 'cli', for example, "Mono JIT compiler".
-              //
-              string v;
-              getline (is, v);
-
-              if (v.compare (0, 37,
-                             "CLI (command line interface compiler)") == 0)
-              {
-                size_t p (v.rfind (' '));
-
-                if (p == string::npos)
-                  fail << "unexpected output from " << cli;
-
-                v.erase (0, p + 1);
-              }
-              else
-              {
-                if (!optional)
-                  fail << cli << " is not command line interface compiler" <<
-                    info << "use config.cli to override";
-
-                v.clear ();
-              }
-
-              is.close (); // Don't block the other end.
-
-              if (pr.wait ())
-                return v;
-
-              // Presumably issued diagnostics. Fall through.
-            }
-            catch (const io_error&)
-            {
-              pr.wait ();
-
-              // Fall through.
-            }
-
-            // Fall through.
-          }
-          catch (const process_error& e)
-          {
-            // In some cases this is not enough (e.g., the runtime linker
-            // will print scary errors if some shared libraries are not
-            // found). So it would be good to redirect child's STDERR.
-            //
-            if (!optional)
-              error << "unable to execute " << args[0] << ": " << e <<
-                info << "use config.cli to override";
-
-            if (e.child)
-              exit (1);
-
-            // Fall through.
-          }
-
-          return string (); // Not found.
-        };
+        rs.assign<uint64_t> ("cli.version.number") = v.version;
+        rs.assign<uint64_t> ("cli.version.major") = v.major ();
+        rs.assign<uint64_t> ("cli.version.minor") = v.minor ();
+        rs.assign<uint64_t> ("cli.version.patch") = v.patch ();
+      }
 
-        // Adjust module priority (code generator).
-        //
-        config::save_module (rs, "cli", 150);
+      // Cache some values in the module for easier access in the rule.
+      //
+      extra.set_module (new module (data {*tgt, *sum}));
 
-        string ver;            // Empty means unconfigured.
-        path cli ("cli");      // Default value.
-        bool new_cfg (false);  // New configuration.
+      return true;
+    }
 
-        if (optional)
-        {
-          // Test the default value before setting any config.cli.* values
-          // so that if we fail to configure, nothing will be written to
-          // config.build.
-          //
-          if (conf)
-          {
-            ver = test (cli);
-
-            if (ver.empty ())
-            {
-              conf = false;
-              new_cfg = true;
-            }
-            else
-            {
-              auto l (lookup_config (new_cfg, rs, "config.cli", cli));
-              assert (new_cfg && cast<path> (l) == cli);
-            }
-          }
-        }
-        else
-        {
-          cli = cast<path> (lookup_config (new_cfg, rs, "config.cli", cli));
-          ver = test (cli);
+    bool
+    config_init (scope& rs,
+                 scope& bs,
+                 const location& loc,
+                 bool,
+                 bool optional,
+                 module_init_extra& extra)
+    {
+      tracer trace ("cli::config_init");
+      l5 ([&]{trace << "for " << rs;});
 
-          if (ver.empty ())
-            throw failed (); // Diagnostics already issued.
-        }
+      // We only support root loading (which means there can only be one).
+      //
+      if (rs != bs)
+        fail (loc) << "cli.config module must be loaded in project root";
 
-        string checksum;
-        if (conf)
-        {
-          // Hash the compiler path and version.
-          //
-          sha256 cs;
-          cs.append (pp.effect_string ());
-          cs.append (ver);
-          checksum = cs.string ();
-        }
-        else
-        {
-          // Note that we are unconfigured so that we don't keep re-testing
-          // this on each run.
-          //
-          new_cfg = config::unconfigured (rs, "cli", true) || new_cfg;
-        }
-
-        // If this is a configuration with new values, then print the report
-        // at verbosity level 2 and up (-v).
-        //
-        if (verb >= (new_cfg ? 2 : 3))
-        {
-          diag_record dr (text);
-          dr << "cli " << project (rs) << '@' << rs << '\n';
-
-          if (conf)
-            dr << "  cli        " << pp << '\n'
-               << "  version    " << ver << '\n'
-               << "  checksum   " << checksum;
-          else
-            dr << "  cli        " << "not found, leaving unconfigured";
-        }
-
-        if (conf)
-        {
-          rs.assign ("cli.path")     = move (pp);
-          rs.assign ("cli.version")  = move (ver);
-          rs.assign ("cli.checksum") = move (checksum);
-        }
+      // Load cli.guess and share its module instance as ours.
+      //
+      if (const shared_ptr<build2::module>* r = load_module (
+            rs, rs, "cli.guess", loc, optional, extra.hints))
+      {
+        extra.module = *r;
       }
-
-      if (conf)
+      else
       {
-        // config.cli.options
+        // This can happen if someone already optionally loaded cli.guess
+        // and it has failed to configure.
         //
-        // This one is optional. We also merge it into the corresponding cli.*
-        // variables. See the cc module for more information on this merging
-        // semantics and some of its tricky aspects.
-        //
-        bs.assign ("cli.options") += cast_null<strings> (
-          lookup_config (rs, "config.cli.options", nullptr));
+        if (!optional)
+          fail (loc) << "cli could not be configured" <<
+            info << "re-run with -V for more information";
+
+        return false;
       }
 
-      return conf;
+      // Configuration.
+      //
+      using config::append_config;
+
+      // config.cli.options
+      //
+      // Note that we merge it into the corresponding cli.* variable.
+      //
+      append_config<strings> (rs, rs, "cli.options", nullptr);
+
+      return true;
     }
 
     bool
     init (scope& rs,
           scope& bs,
-          const location& l,
-          bool first,
+          const location& loc,
+          bool,
           bool optional,
           module_init_extra& extra)
     {
       tracer trace ("cli::init");
-      l5 ([&]{trace << "for " << bs;});
+      l5 ([&]{trace << "for " << rs;});
+
+      // We only support root loading (which means there can only be one).
+      //
+      if (rs != bs)
+        fail (loc) << "cli module must be loaded in project root";
 
       // Make sure the cxx module has been loaded since we need its targets
       // types (?xx{}). Note that we don't try to load it ourselves because of
       // the non-trivial variable merging semantics. So it is better to let
-      // the user load cxx explicitly.
+      // the user load cxx explicitly. @@ Not sure the reason still holds
+      // though it might still make sense to expect the user to load cxx.
       //
-      if (!cast_false<bool> (bs["cxx.loaded"]))
-        fail (l) << "cxx module must be loaded before cli";
+      if (!cast_false<bool> (rs["cxx.loaded"]))
+        fail (loc) << "cxx module must be loaded before cli";
 
-      // Load cli.config.
+      // Load cli.config and get its module instance.
       //
-      if (!cast_false<bool> (bs["cli.config.loaded"]))
+      if (const shared_ptr<build2::module>* r = load_module (
+            rs, rs, "cli.config", loc, optional, extra.hints))
       {
-        if (!init_module (rs, bs, "cli.config", l, optional, extra.hints))
-          return false;
+        extra.module = *r;
       }
-      else if (!cast_false<bool> (bs["cli.config.configured"]))
+      else
       {
+        // This can happen if someone already optionally loaded cli.config
+        // and it has failed to configure.
+        //
         if (!optional)
-          fail (l) << "cli module could not be configured" <<
+          fail (loc) << "cli could not be configured" <<
             info << "re-run with -V for more information";
 
         return false;
       }
 
+      auto& m (extra.module_as<module> ());
+
       // Register target types.
       //
-      if (first)
-      {
-        rs.insert_target_type<cli> ();
-        rs.insert_target_type<cli_cxx> ();
-      }
+      rs.insert_target_type<cli> ();
+      rs.insert_target_type<cli_cxx> ();
 
       // Register our rules.
       //
       {
-        auto reg = [&bs] (meta_operation_id mid, operation_id oid)
+        auto reg = [&rs, &m] (meta_operation_id mid, operation_id oid)
         {
-          bs.insert_rule<cli_cxx>  (mid, oid, "cli.compile", compile_rule_);
-          bs.insert_rule<cxx::hxx> (mid, oid, "cli.compile", compile_rule_);
-          bs.insert_rule<cxx::cxx> (mid, oid, "cli.compile", compile_rule_);
-          bs.insert_rule<cxx::ixx> (mid, oid, "cli.compile", compile_rule_);
+          rs.insert_rule<cli_cxx>  (mid, oid, "cli.compile", m);
+          rs.insert_rule<cxx::hxx> (mid, oid, "cli.compile", m);
+          rs.insert_rule<cxx::cxx> (mid, oid, "cli.compile", m);
+          rs.insert_rule<cxx::ixx> (mid, oid, "cli.compile", m);
         };
 
         reg (perform_id, update_id);
@@ -366,6 +276,7 @@ namespace build2
       // NOTE: don't forget to also update the documentation in init.hxx if
       //       changing anything here.
 
+      {"cli.guess",  nullptr, guess_init},
       {"cli.config", nullptr, config_init},
       {"cli",        nullptr, init},
       {nullptr,      nullptr, nullptr}