Rework tool importation along with cli module

Specifically, now config.<tool> (like config.cli) is handled by the import
machinery (it is like a shorter alias for config.import.<tool>.<tool>.exe
that we already had). And the cli module now uses that instead of custom
logic.

This also adds support for uniform tool metadata extraction that is handled by
the import machinery. As a result, a tool that follows the "build2 way" can be
imported with metadata by the buildfile and/or corresponding module without
any tool-specific code or brittleness associated with parsing --version or
similar outputs. See the cli tool/module for details.

Finally, two new flavors of the import directive are now supported: import!
triggers immediate importation skipping any rule-specific logic while import?
is optional import (analogous to using?). Note that optional import is always
immediate. There is also the import-specific metadata attribute which can be
specified for these two import flavors in order to trigger metadata
importation. For example:

import? [metadata] cli = cli%exe{cli}

if ($cli != [null])
  info "cli version $($cli:cli.version)"

6 files changed, 231 insertions, 283 deletions

diff --git a/build2/cli/init.cxx b/build2/cli/init.cxx
index c68a62f..8f9df4a 100644
--- a/build2/cli/init.cxx
+++ b/build2/cli/init.cxx
@@ -3,6 +3,7 @@
 
 #include <build2/cli/init.hxx>
 
+#include <libbuild2/file.hxx>
 #include <libbuild2/scope.hxx>
 #include <libbuild2/target.hxx>
 #include <libbuild2/variable.hxx>
@@ -12,337 +13,246 @@
 
 #include <libbuild2/cxx/target.hxx>
 
-#include <build2/cli/target.hxx>
 #include <build2/cli/rule.hxx>
-
-using namespace std;
-using namespace butl;
+#include <build2/cli/module.hxx>
+#include <build2/cli/target.hxx>
 
 namespace build2
 {
   namespace cli
   {
-    static const compile_rule compile_rule_;
+    // Remaining issues/semantics change:
+    //
+    // @@ Unconfigured caching.
+    //
+    // @@ Default-found cli used to result in config.cli=cli and now it's just
+    //    omitted (and default-not-found -- in config.cli.configured=false).
+    //
+    //    - Writing any default will take precedence over config.import.cli.
+    //      In fact, this duality is a bigger problem: if we have a config
+    //      that uses config.cli there is no way to reconfigure it to use
+    //      config.import.cli.
+    //
+    //    - We could have saved it commented.
+    //
+    //    - We could do this at the module level only since we also have
+    //      config.cli.options?
+    //
+    //    - Note that in the CLI compiler itself we now rely on default cli
+    //      being NULL/undefined. So if faving, should probably be commented
+    //      out. BUT: it will still be defined, so will need to be defined
+    //      NULL. Note also that long term the CLI compiler will not use the
+    //      module relying on an ad hoc recipe instead.
+    //
+    //    ! Maybe reserving NULL (instead of making it the same as NULL) for
+    //      this "configured to default" state and saving commented is not a
+    //      bad idea. Feels right to have some marker in config.build that
+    //      things are in effect. And I believe if config.import.cli is
+    //      specified, it will just be dropped.
 
     bool
-    config_init (scope& rs,
-                 scope& bs,
-                 const location& l,
-                 bool first,
-                 bool optional,
-                 module_init_extra&)
+    guess_init (scope& rs,
+                scope& bs,
+                const location& loc,
+                bool,
+                bool optional,
+                module_init_extra& extra)
     {
-      tracer trace ("cli::config_init");
-      l5 ([&]{trace << "for " << bs;});
+      tracer trace ("cli::guess_init");
+      l5 ([&]{trace << "for " << rs;});
 
-      // Enter variables.
+      // We only support root loading (which means there can only be one).
       //
-      if (first)
-      {
-        auto& vp (rs.var_pool ());
-
-        // The special config.cli=false value is recognized as an explicit
-        // request to leave the module unconfigured.
-        //
-        vp.insert<path>    ("config.cli");
-        vp.insert<strings> ("config.cli.options");
+      if (rs != bs)
+        fail (loc) << "cli.guess module must be loaded in project root";
 
-        //@@ TODO: split version into componets (it is stdver).
-        //
-        vp.insert<process_path> ("cli.path");
-        vp.insert<string>       ("cli.version");
-        vp.insert<string>       ("cli.checksum");
-        vp.insert<strings>      ("cli.options");
-      }
-
-      // Configuration.
+      // Adjust module config.build save priority (code generator).
       //
-      // The plan is as follows: try to configure the module. If this fails,
-      // we are using default values, and the module is optional, leave it
-      // unconfigured.
+      config::save_module (rs, "cli", 150);
+
+      // Enter metadata variables.
       //
-      using config::lookup_config;
-      using config::specified_config;
+      auto& vp (rs.var_pool ());
 
+      auto& v_ver (vp.insert<string> ("cli.version"));
+      auto& v_sum (vp.insert<string> ("cli.checksum"));
 
-      // First take care of the explicit request by the user to leave the
+      // Import the CLI compiler target.
+      //
+      // Note that the special config.cli=false value (recognized by the
+      // import machinery) is treated as an explicit request to leave the
       // module unconfigured.
       //
-      bool conf (true);
+      bool new_cfg (false);
+      pair<const exe*, import_kind> ir (
+        import_direct<exe> (
+          new_cfg,
+          rs,
+          name ("cli", dir_path (), "exe", "cli"), // cli%exe{cli}
+          true      /* phase2 */,
+          optional,
+          true      /* metadata */,
+          loc,
+          "module load"));
+
+      const exe* tgt (ir.first);
+
+      // Extract metadata.
+      //
+      auto* ver (tgt != nullptr ? &cast<string> (tgt->vars[v_ver]) : nullptr);
+      auto* sum (tgt != nullptr ? &cast<string> (tgt->vars[v_sum]) : nullptr);
 
-      if (const path* p = cast_null<path> (rs["config.cli"]))
+      // Print the report.
+      //
+      // If this is a configuration with new values, then print the report
+      // at verbosity level 2 and up (-v).
+      //
+      if (verb >= (new_cfg ? 2 : 3))
       {
-        conf = p->string () != "false";
+        diag_record dr (text);
+        dr << "cli " << project (rs) << '@' << rs << '\n';
 
-        if (!conf && !optional)
-          fail (l) << "non-optional module requested to be left unconfigured";
+        if (tgt != nullptr)
+          dr << "  cli        " << ir << '\n'
+             << "  version    " << *ver << '\n'
+             << "  checksum   " << *sum;
+        else
+          dr << "  cli        " << "not found, leaving unconfigured";
       }
 
-      if (conf)
-      {
-        // Otherwise we will only honor optional if the user didn't specify
-        // any cli configuration explicitly.
-        //
-        optional = optional && !specified_config (rs, "cli");
+      if (tgt == nullptr)
+        return false;
 
-        // If the configuration says we are unconfigured, then we should't
-        // re-run tests, etc. But we may still need to print the config
-        // report.
-        //
-        conf = !optional || !config::unconfigured (rs, "cli");
-      }
+      // The cli variable (untyped) is an imported compiler target name.
+      //
+      rs.assign ("cli") = tgt->as_name ();
+      rs.assign (v_sum) = *sum;
+      rs.assign (v_ver) = *ver;
 
-      if (first)
       {
-        // config.cli
-        //
-        process_path pp;
+        standard_version v (*ver);
 
-        // Return version or empty string if the cli executable is not found
-        // or is not the command line interface compiler.
-        //
-        // @@ This needs some more thinking/cleanup. Specifically, what does
-        //    it mean "cli not found"? Is it just not found in PATH? That plus
-        //    was not able to execute (e.g., some shared libraries missing)?
-        //    That plus cli that we found is something else?
-        //
-        auto test = [optional, &pp] (const path& cli) -> string
-        {
-          const char* args[] = {cli.string ().c_str (), "--version", nullptr};
-
-          // @@ TODO: redo using run_start()/run_finish() or even
-          //    run<string>(). We have the ability to ignore exit code and
-          //    redirect STDERR to STDOUT.
-
-          try
-          {
-            // Only search in PATH (specifically, omitting the current
-            // executable's directory on Windows).
-            //
-            pp = process::path_search (cli,
-                                       true        /* init */,
-                                       dir_path () /* fallback */,
-                                       true        /* path_only */);
-            args[0] = pp.recall_string ();
-
-            if (verb >= 3)
-              print_process (args);
-
-            process pr (pp, args, 0, -1); // Open pipe to stdout.
-
-            try
-            {
-              ifdstream is (move (pr.in_ofd), fdstream_mode::skip);
-
-              // The version should be the last word on the first line. But
-              // also check the prefix since there are other things called
-              // 'cli', for example, "Mono JIT compiler".
-              //
-              string v;
-              getline (is, v);
-
-              if (v.compare (0, 37,
-                             "CLI (command line interface compiler)") == 0)
-              {
-                size_t p (v.rfind (' '));
-
-                if (p == string::npos)
-                  fail << "unexpected output from " << cli;
-
-                v.erase (0, p + 1);
-              }
-              else
-              {
-                if (!optional)
-                  fail << cli << " is not command line interface compiler" <<
-                    info << "use config.cli to override";
-
-                v.clear ();
-              }
-
-              is.close (); // Don't block the other end.
-
-              if (pr.wait ())
-                return v;
-
-              // Presumably issued diagnostics. Fall through.
-            }
-            catch (const io_error&)
-            {
-              pr.wait ();
-
-              // Fall through.
-            }
-
-            // Fall through.
-          }
-          catch (const process_error& e)
-          {
-            // In some cases this is not enough (e.g., the runtime linker
-            // will print scary errors if some shared libraries are not
-            // found). So it would be good to redirect child's STDERR.
-            //
-            if (!optional)
-              error << "unable to execute " << args[0] << ": " << e <<
-                info << "use config.cli to override";
-
-            if (e.child)
-              exit (1);
-
-            // Fall through.
-          }
-
-          return string (); // Not found.
-        };
+        rs.assign<uint64_t> ("cli.version.number") = v.version;
+        rs.assign<uint64_t> ("cli.version.major") = v.major ();
+        rs.assign<uint64_t> ("cli.version.minor") = v.minor ();
+        rs.assign<uint64_t> ("cli.version.patch") = v.patch ();
+      }
 
-        // Adjust module priority (code generator).
-        //
-        config::save_module (rs, "cli", 150);
+      // Cache some values in the module for easier access in the rule.
+      //
+      extra.set_module (new module (data {*tgt, *sum}));
 
-        string ver;            // Empty means unconfigured.
-        path cli ("cli");      // Default value.
-        bool new_cfg (false);  // New configuration.
+      return true;
+    }
 
-        if (optional)
-        {
-          // Test the default value before setting any config.cli.* values
-          // so that if we fail to configure, nothing will be written to
-          // config.build.
-          //
-          if (conf)
-          {
-            ver = test (cli);
-
-            if (ver.empty ())
-            {
-              conf = false;
-              new_cfg = true;
-            }
-            else
-            {
-              auto l (lookup_config (new_cfg, rs, "config.cli", cli));
-              assert (new_cfg && cast<path> (l) == cli);
-            }
-          }
-        }
-        else
-        {
-          cli = cast<path> (lookup_config (new_cfg, rs, "config.cli", cli));
-          ver = test (cli);
+    bool
+    config_init (scope& rs,
+                 scope& bs,
+                 const location& loc,
+                 bool,
+                 bool optional,
+                 module_init_extra& extra)
+    {
+      tracer trace ("cli::config_init");
+      l5 ([&]{trace << "for " << rs;});
 
-          if (ver.empty ())
-            throw failed (); // Diagnostics already issued.
-        }
+      // We only support root loading (which means there can only be one).
+      //
+      if (rs != bs)
+        fail (loc) << "cli.config module must be loaded in project root";
 
-        string checksum;
-        if (conf)
-        {
-          // Hash the compiler path and version.
-          //
-          sha256 cs;
-          cs.append (pp.effect_string ());
-          cs.append (ver);
-          checksum = cs.string ();
-        }
-        else
-        {
-          // Note that we are unconfigured so that we don't keep re-testing
-          // this on each run.
-          //
-          new_cfg = config::unconfigured (rs, "cli", true) || new_cfg;
-        }
-
-        // If this is a configuration with new values, then print the report
-        // at verbosity level 2 and up (-v).
-        //
-        if (verb >= (new_cfg ? 2 : 3))
-        {
-          diag_record dr (text);
-          dr << "cli " << project (rs) << '@' << rs << '\n';
-
-          if (conf)
-            dr << "  cli        " << pp << '\n'
-               << "  version    " << ver << '\n'
-               << "  checksum   " << checksum;
-          else
-            dr << "  cli        " << "not found, leaving unconfigured";
-        }
-
-        if (conf)
-        {
-          rs.assign ("cli.path")     = move (pp);
-          rs.assign ("cli.version")  = move (ver);
-          rs.assign ("cli.checksum") = move (checksum);
-        }
+      // Load cli.guess and share its module instance as ours.
+      //
+      if (const shared_ptr<build2::module>* r = load_module (
+            rs, rs, "cli.guess", loc, optional, extra.hints))
+      {
+        extra.module = *r;
       }
-
-      if (conf)
+      else
       {
-        // config.cli.options
+        // This can happen if someone already optionally loaded cli.guess
+        // and it has failed to configure.
         //
-        // This one is optional. We also merge it into the corresponding cli.*
-        // variables. See the cc module for more information on this merging
-        // semantics and some of its tricky aspects.
-        //
-        bs.assign ("cli.options") += cast_null<strings> (
-          lookup_config (rs, "config.cli.options", nullptr));
+        if (!optional)
+          fail (loc) << "cli could not be configured" <<
+            info << "re-run with -V for more information";
+
+        return false;
       }
 
-      return conf;
+      // Configuration.
+      //
+      using config::append_config;
+
+      // config.cli.options
+      //
+      // Note that we merge it into the corresponding cli.* variable.
+      //
+      append_config<strings> (rs, rs, "cli.options", nullptr);
+
+      return true;
     }
 
     bool
     init (scope& rs,
           scope& bs,
-          const location& l,
-          bool first,
+          const location& loc,
+          bool,
           bool optional,
           module_init_extra& extra)
     {
       tracer trace ("cli::init");
-      l5 ([&]{trace << "for " << bs;});
+      l5 ([&]{trace << "for " << rs;});
+
+      // We only support root loading (which means there can only be one).
+      //
+      if (rs != bs)
+        fail (loc) << "cli module must be loaded in project root";
 
       // Make sure the cxx module has been loaded since we need its targets
       // types (?xx{}). Note that we don't try to load it ourselves because of
       // the non-trivial variable merging semantics. So it is better to let
-      // the user load cxx explicitly.
+      // the user load cxx explicitly. @@ Not sure the reason still holds
+      // though it might still make sense to expect the user to load cxx.
       //
-      if (!cast_false<bool> (bs["cxx.loaded"]))
-        fail (l) << "cxx module must be loaded before cli";
+      if (!cast_false<bool> (rs["cxx.loaded"]))
+        fail (loc) << "cxx module must be loaded before cli";
 
-      // Load cli.config.
+      // Load cli.config and get its module instance.
       //
-      if (!cast_false<bool> (bs["cli.config.loaded"]))
+      if (const shared_ptr<build2::module>* r = load_module (
+            rs, rs, "cli.config", loc, optional, extra.hints))
       {
-        if (!init_module (rs, bs, "cli.config", l, optional, extra.hints))
-          return false;
+        extra.module = *r;
       }
-      else if (!cast_false<bool> (bs["cli.config.configured"]))
+      else
       {
+        // This can happen if someone already optionally loaded cli.config
+        // and it has failed to configure.
+        //
         if (!optional)
-          fail (l) << "cli module could not be configured" <<
+          fail (loc) << "cli could not be configured" <<
             info << "re-run with -V for more information";
 
         return false;
       }
 
+      auto& m (extra.module_as<module> ());
+
       // Register target types.
       //
-      if (first)
-      {
-        rs.insert_target_type<cli> ();
-        rs.insert_target_type<cli_cxx> ();
-      }
+      rs.insert_target_type<cli> ();
+      rs.insert_target_type<cli_cxx> ();
 
       // Register our rules.
       //
       {
-        auto reg = [&bs] (meta_operation_id mid, operation_id oid)
+        auto reg = [&rs, &m] (meta_operation_id mid, operation_id oid)
         {
-          bs.insert_rule<cli_cxx>  (mid, oid, "cli.compile", compile_rule_);
-          bs.insert_rule<cxx::hxx> (mid, oid, "cli.compile", compile_rule_);
-          bs.insert_rule<cxx::cxx> (mid, oid, "cli.compile", compile_rule_);
-          bs.insert_rule<cxx::ixx> (mid, oid, "cli.compile", compile_rule_);
+          rs.insert_rule<cli_cxx>  (mid, oid, "cli.compile", m);
+          rs.insert_rule<cxx::hxx> (mid, oid, "cli.compile", m);
+          rs.insert_rule<cxx::cxx> (mid, oid, "cli.compile", m);
+          rs.insert_rule<cxx::ixx> (mid, oid, "cli.compile", m);
         };
 
         reg (perform_id, update_id);
@@ -366,6 +276,7 @@ namespace build2
       // NOTE: don't forget to also update the documentation in init.hxx if
       //       changing anything here.
 
+      {"cli.guess",  nullptr, guess_init},
       {"cli.config", nullptr, config_init},
       {"cli",        nullptr, init},
       {nullptr,      nullptr, nullptr}
diff --git a/build2/cli/init.hxx b/build2/cli/init.hxx
index d5998f5..1c54316 100644
--- a/build2/cli/init.hxx
+++ b/build2/cli/init.hxx
@@ -17,8 +17,9 @@ namespace build2
     //
     // Submodules:
     //
-    // `cli.config` -- registers variables.
-    // `cli`        -- loads cli.config and registers target types and rules.
+    // `cli.guess`  -- set variables describing the compiler.
+    // `cli.config` -- load `cli.guess` and set the rest of the variables.
+    // `cli`        -- load `cli.config` and register targets and rules.
     //
     extern "C" const module_functions*
     build2_cli_load ();
diff --git a/build2/cli/module.hxx b/build2/cli/module.hxx
new file mode 100644
index 0000000..70f6ba8
--- /dev/null
+++ b/build2/cli/module.hxx
@@ -0,0 +1,30 @@
+// file      : build2/cli/module.hxx -*- C++ -*-
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD2_CLI_MODULE_HXX
+#define BUILD2_CLI_MODULE_HXX
+
+#include <libbuild2/types.hxx>
+#include <libbuild2/utility.hxx>
+
+#include <libbuild2/module.hxx>
+
+#include <build2/cli/rule.hxx>
+
+namespace build2
+{
+  namespace cli
+  {
+    class module: public build2::module,
+                  public virtual data,
+                  public compile_rule
+    {
+    public:
+      explicit
+      module (data&& d)
+          : data (move (d)), compile_rule (move (d)) {}
+    };
+  }
+}
+
+#endif // BUILD2_CLI_MODULE_HXX
diff --git a/build2/cli/rule.cxx b/build2/cli/rule.cxx
index 9917f1a..3109689 100644
--- a/build2/cli/rule.cxx
+++ b/build2/cli/rule.cxx
@@ -13,9 +13,6 @@
 
 #include <build2/cli/target.hxx>
 
-using namespace std;
-using namespace butl;
-
 namespace build2
 {
   namespace cli
@@ -166,11 +163,17 @@ namespace build2
         //
         match_prerequisite_members (a, t);
 
-        //@@ TODO: inject dependency on exe{cli}.
+        // For update inject dependency on the CLI compiler target.
+        //
+        if (a == perform_update_id)
+          inject (a, t, ctgt);
 
         switch (a)
         {
-        case perform_update_id: return &perform_update;
+        case perform_update_id: return [this] (action a, const target& t)
+          {
+            return perform_update (a, t);
+          };
         case perform_clean_id:  return &perform_clean_group_depdb;
         default:                return noop_recipe; // Configure/dist update.
         }
@@ -206,7 +209,7 @@ namespace build2
     }
 
     target_state compile_rule::
-    perform_update (action a, const target& xt)
+    perform_update (action a, const target& xt) const
     {
       tracer trace ("cli::compile_rule::perform_update");
 
@@ -215,7 +218,6 @@ namespace build2
       // timestamp, depdb, etc.
       //
       const cli_cxx& t (xt.as<cli_cxx> ());
-      const scope& rs (t.root_scope ());
       const path& tp (t.h->path ());
 
       // Update prerequisites and determine if any relevant ones render us
@@ -242,7 +244,7 @@ namespace build2
 
         // Then the compiler checksum.
         //
-        if (dd.expect (cast<string> (rs["cli.checksum"])) != nullptr)
+        if (dd.expect (csum) != nullptr)
           l4 ([&]{trace << "compiler mismatch forcing update of " << t;});
 
         // Then the options checksum.
@@ -277,9 +279,8 @@ namespace build2
       path relo (relative (t.dir));
       path rels (relative (s.path ()));
 
-      const process_path& cli (cast<process_path> (rs["cli.path"]));
-
-      cstrings args {cli.recall_string ()};
+      const process_path& pp (ctgt.process_path ());
+      cstrings args {pp.recall_string ()};
 
       // See if we need to pass --output-{prefix,suffix}
       //
@@ -323,7 +324,7 @@ namespace build2
 
       if (!t.ctx.dry_run)
       {
-        run (cli, args);
+        run (pp, args);
         dd.check_mtime (tp);
       }
 
diff --git a/build2/cli/rule.hxx b/build2/cli/rule.hxx
index aa3b8fa..64f1614 100644
--- a/build2/cli/rule.hxx
+++ b/build2/cli/rule.hxx
@@ -13,12 +13,20 @@ namespace build2
 {
   namespace cli
   {
+    // Cached data shared between rules and the module.
+    //
+    struct data
+    {
+      const exe&    ctgt; // CLI compiler target.
+      const string& csum; // CLI compiler checksum.
+    };
+
     // @@ Redo as two separate rules?
     //
-    class compile_rule: public rule
+    class compile_rule: public rule, virtual data
     {
     public:
-      compile_rule () {}
+      compile_rule (data&& d): data (move (d)) {}
 
       virtual bool
       match (action, target&, const string&) const override;
@@ -26,8 +34,8 @@ namespace build2
       virtual recipe
       apply (action, target&) const override;
 
-      static target_state
-      perform_update (action, const target&);
+      target_state
+      perform_update (action, const target&) const;
     };
   }
 }
diff --git a/build2/cli/target.cxx b/build2/cli/target.cxx
index 09f3e10..ca16044 100644
--- a/build2/cli/target.cxx
+++ b/build2/cli/target.cxx
@@ -5,9 +5,6 @@
 
 #include <libbuild2/context.hxx>
 
-using namespace std;
-using namespace butl;
-
 namespace build2
 {
   namespace cli