Implement support for narrowing down tests (config.test)

20 files changed, 667 insertions, 228 deletions

diff --git a/build2/bin/init.cxx b/build2/bin/init.cxx
index e1b812b..eaded2a 100644
--- a/build2/bin/init.cxx
+++ b/build2/bin/init.cxx
@@ -112,7 +112,7 @@ namespace build2
       {
         value& v (b.assign ("bin.lib"));
         if (!v)
-          v = required (r, "config.bin.lib", "both").first;
+          v = *required (r, "config.bin.lib", "both").first;
       }
 
       // config.bin.exe.lib
@@ -120,7 +120,7 @@ namespace build2
       {
         value& v (b.assign ("bin.exe.lib"));
         if (!v)
-          v = required (r, "config.bin.exe.lib", exe_lib).first;
+          v = *required (r, "config.bin.exe.lib", exe_lib).first;
       }
 
       // config.bin.liba.lib
@@ -128,7 +128,7 @@ namespace build2
       {
         value& v (b.assign ("bin.liba.lib"));
         if (!v)
-          v = required (r, "config.bin.liba.lib", liba_lib).first;
+          v = *required (r, "config.bin.liba.lib", liba_lib).first;
       }
 
       // config.bin.libs.lib
@@ -136,7 +136,7 @@ namespace build2
       {
         value& v (b.assign ("bin.libs.lib"));
         if (!v)
-          v = required (r, "config.bin.libs.lib", libs_lib).first;
+          v = *required (r, "config.bin.libs.lib", libs_lib).first;
       }
 
       // config.bin.rpath
@@ -154,16 +154,16 @@ namespace build2
       // that might have been specified before loading the module.
       //
       {
-        const value* p (omitted (r, "config.bin.prefix").first);
-        const value* s (omitted (r, "config.bin.suffix").first);
+        lookup p (omitted (r, "config.bin.prefix").first);
+        lookup s (omitted (r, "config.bin.suffix").first);
 
-        auto set = [&r, &b] (const char* bv, const char* cv, const value* v)
+        auto set = [&r, &b] (const char* bv, const char* cv, lookup l)
         {
-          if (const value* o = omitted (r, cv).first)
-            v = o;
+          if (lookup o = omitted (r, cv).first)
+            l = o;
 
-          if (v != nullptr)
-            b.assign (bv) = *v;
+          if (l)
+            b.assign (bv) = *l;
         };
 
         set ("bin.lib.prefix", "config.bin.lib.prefix", p);
@@ -186,21 +186,21 @@ namespace build2
           // mechanism.
           //
           auto p (omitted (r, var));
-          const value* v (p.first);
+          lookup l (p.first);
 
           // Then see if there is a config hint (e.g., from the C++ module).
           //
           bool hint (false);
-          if (v == nullptr)
+          if (!l)
           {
-            if (auto l = hints[var])
+            if (auto hl = hints[var])
             {
-              v = l.value;
+              l = hl;
               hint = true;
             }
           }
 
-          if (v == nullptr)
+          if (!l)
             fail (loc) << "unable to determine binutils target" <<
               info << "consider specifying it with " << var <<
               info << "or first load a module that can provide it as a hint, "
@@ -208,7 +208,7 @@ namespace build2
 
           // Split/canonicalize the target.
           //
-          string s (cast<string> (*v));
+          string s (cast<string> (l));
 
           // Did the user ask us to use config.sub? If this is a hinted value,
           // then we assume it has already been passed through config.sub.
@@ -262,22 +262,22 @@ namespace build2
           // mechanism.
           //
           auto p (omitted (r, var));
-          const value* v (p.first);
+          lookup l (p.first);
 
           // Then see if there is a config hint (e.g., from the C++ module).
           //
-          if (v == nullptr)
+          if (!l)
           {
-            if (auto l = hints[var])
-              v = l.value;
+            if (auto hl = hints[var])
+              l = hl;
           }
 
           // For ease of use enter it as bin.pattern (since it can come from
           // different places).
           //
-          if (v != nullptr)
+          if (l)
           {
-            const string& s (cast<string> (*v));
+            const string& s (cast<string> (l));
 
             if (s.empty () ||
                 (!path::traits::is_separator (s.back ()) &&
diff --git a/build2/buildfile b/build2/buildfile
index 83f29f8..6d497ca 100644
--- a/build2/buildfile
+++ b/build2/buildfile
@@ -79,7 +79,9 @@ exe{b}:                                                   \
     install/{hxx         cxx}{ operation                } \
     install/{hxx         cxx}{ rule                     } \
     install/{hxx            }{ utility                  } \
+       test/{hxx         cxx}{ common                   } \
        test/{hxx         cxx}{ init                     } \
+       test/{hxx            }{ module                   } \
        test/{hxx         cxx}{ operation                } \
        test/{hxx         cxx}{ rule                     } \
        test/{hxx         cxx}{ target                   } \
diff --git a/build2/cc/module.cxx b/build2/cc/module.cxx
index 40bd454..e63f740 100644
--- a/build2/cc/module.cxx
+++ b/build2/cc/module.cxx
@@ -55,7 +55,7 @@ namespace build2
       //
       auto p (config::omitted (rs, config_x));
 
-      if (p.first == nullptr)
+      if (!p.first)
       {
         // If someone already loaded cc.core.config then use its toolchain
         // id and (optional) pattern to guess an appropriate default (e.g.,
@@ -71,13 +71,11 @@ namespace build2
         // user changes the source of the pattern, this one will get updated
         // as well.
         //
-        auto p1 (config::required (rs,
-                                   config_x,
-                                   d,
-                                   false,
-                                   cc_loaded ? config::save_commented : 0));
-        p.first = &p1.first.get ();
-        p.second = p1.second;
+        p = config::required (rs,
+                              config_x,
+                              d,
+                              false,
+                              cc_loaded ? config::save_commented : 0);
       }
 
       // Figure out which compiler we are dealing with, its target, etc.
diff --git a/build2/config/utility b/build2/config/utility
index 1f1fba2..6c18715 100644
--- a/build2/config/utility
+++ b/build2/config/utility
@@ -28,14 +28,16 @@ namespace build2
     // the value is "new", that is, it was set to the default value (inherited
     // or not, including overrides). We also treat command line overrides
     // (inherited or not) as new. This flag is usually used to test that the
-    // new value is valid, print report, etc.
+    // new value is valid, print report, etc. We return the value as lookup
+    // (always defined) to pass alone its location (could be used to detect
+    // inheritance, etc).
     //
     // Note also that if save_flags has save_commented, then a default value
     // is never considered "new" since for such variables absence of a value
     // means the default value.
     //
     template <typename T>
-    pair<reference_wrapper<const value>, bool>
+    pair<lookup, bool>
     required (scope& root,
               const variable&,
               const T& default_value,
@@ -43,7 +45,7 @@ namespace build2
               uint64_t save_flags = 0);
 
     template <typename T>
-    inline pair<reference_wrapper<const value>, bool>
+    inline pair<lookup, bool>
     required (scope& root,
               const string& name,
               const T& default_value,
@@ -54,7 +56,7 @@ namespace build2
         root, var_pool[name], default_value, override, save_flags);
     }
 
-    inline pair<reference_wrapper<const value>, bool>
+    inline pair<lookup, bool>
     required (scope& root,
               const string& name,
               const char* default_value,
@@ -65,32 +67,31 @@ namespace build2
         root, name, string (default_value), override, save_flags);
     }
 
-    // As above, but leave the unspecified value as undefined (and return
-    // NULL pointer) rather than setting it to the default value.
+    // As above, but leave the unspecified value as undefined rather than
+    // setting it to the default value.
     //
     // This can be useful when we don't have a default value but may figure
     // out some fallback. See config.bin.target for an example.
     //
-    pair<const value*, bool>
+    pair<lookup, bool>
     omitted (scope& root, const variable&);
 
-    inline pair<const value*, bool>
+    inline pair<lookup, bool>
     omitted (scope& root, const string& name)
     {
       return omitted (root, var_pool[name]);
     }
 
-    // Set, if necessary, an optional config.* variable. In particular,
-    // an unspecified variable is set to NULL which is used to distinguish
-    // between the "configured as unspecified" and "not yet configured"
-    // cases.
+    // Set, if necessary, an optional config.* variable. In particular, an
+    // unspecified variable is set to NULL which is used to distinguish
+    // between the "configured as unspecified" and "not yet configured" cases.
     //
-    // Return the value, which can be NULL.
+    // Return the value (as always defined lookup), which can be NULL.
     //
-    const value&
+    lookup
     optional (scope& root, const variable&);
 
-    inline const value&
+    inline lookup
     optional (scope& root, const string& var)
     {
       return optional (root, var_pool[var]);
diff --git a/build2/config/utility.cxx b/build2/config/utility.cxx
index dac11c2..7a3fa9c 100644
--- a/build2/config/utility.cxx
+++ b/build2/config/utility.cxx
@@ -14,7 +14,7 @@ namespace build2
 {
   namespace config
   {
-    pair<const value*, bool>
+    pair<lookup, bool>
     omitted (scope& r, const variable& var)
     {
       // This is a stripped-down version of the required() twisted
@@ -44,12 +44,12 @@ namespace build2
       }
 
       if (l.defined () && current_mif->id == configure_id)
-          save_variable (r, var);
+        save_variable (r, var);
 
-      return pair<const value*, bool> (l.value, n);
+      return pair<lookup, bool> (l, n);
     }
 
-    const value&
+    lookup
     optional (scope& r, const variable& var)
     {
       if (current_mif->id == configure_id)
@@ -57,8 +57,8 @@ namespace build2
 
       auto l (r[var]);
       return l.defined ()
-        ? *l
-        : r.assign (var); // NULL.
+        ? l
+        : lookup (r.assign (var), r); // NULL.
     }
 
     bool
diff --git a/build2/config/utility.txx b/build2/config/utility.txx
index b6859d0..1434406 100644
--- a/build2/config/utility.txx
+++ b/build2/config/utility.txx
@@ -10,7 +10,7 @@ namespace build2
   namespace config
   {
     template <typename T>
-    pair<reference_wrapper<const value>, bool>
+    pair<lookup, bool>
     required (scope& root,
               const variable& var,
               const T& def_val,
@@ -60,7 +60,7 @@ namespace build2
         }
       }
 
-      return pair<reference_wrapper<const value>, bool> (*l, n);
+      return pair<lookup, bool> (l, n);
     }
   }
 }
diff --git a/build2/dist/init.cxx b/build2/dist/init.cxx
index 14e1887..f185145 100644
--- a/build2/dist/init.cxx
+++ b/build2/dist/init.cxx
@@ -112,8 +112,8 @@ namespace build2
 
         if (s)
         {
-          if (const value& cv = config::optional (r, "config.dist.root"))
-            v = cast<dir_path> (cv); // Strip abs_dir_path.
+          if (lookup l = config::optional (r, "config.dist.root"))
+            v = cast<dir_path> (l); // Strip abs_dir_path.
         }
       }
 
@@ -124,10 +124,10 @@ namespace build2
 
         if (s)
         {
-          if (const value& cv = config::required (r,
-                                                  "config.dist.cmd",
-                                                  path ("install")).first)
-            v = run_search (cast<path> (cv), true);
+          if (lookup l = config::required (r,
+                                           "config.dist.cmd",
+                                           path ("install")).first)
+            v = run_search (cast<path> (l), true);
         }
       }
 
@@ -138,8 +138,8 @@ namespace build2
 
         if (s)
         {
-          if (const value& cv = config::optional (r, "config.dist.archives"))
-            v = cv;
+          if (lookup l = config::optional (r, "config.dist.archives"))
+            v = *l;
         }
       }
 
diff --git a/build2/install/init.cxx b/build2/install/init.cxx
index bb54bcd..5281d38 100644
--- a/build2/install/init.cxx
+++ b/build2/install/init.cxx
@@ -46,7 +46,7 @@ namespace build2
              bool override = false)
     {
       string vn;
-      const value* cv (nullptr);
+      lookup l;
 
       bool global (*name == '\0');
 
@@ -61,10 +61,10 @@ namespace build2
         vn += var;
         const variable& vr (var_pool.insert<CT> (move (vn), true));
 
-        cv = dv != nullptr
-          ? &config::required (r, vr, *dv, override).first.get ()
+        l = dv != nullptr
+          ? config::required (r, vr, *dv, override).first
           : (global
-             ? &config::optional (r, vr)
+             ? config::optional (r, vr)
              : config::omitted (r, vr).first);
       }
 
@@ -80,8 +80,8 @@ namespace build2
 
       if (spec)
       {
-        if (cv != nullptr && *cv)
-          v = cast<T> (*cv); // Strip CT to T.
+        if (l)
+          v = cast<T> (l); // Strip CT to T.
       }
       else
       {
diff --git a/build2/pkgconfig/init.cxx b/build2/pkgconfig/init.cxx
index 8d50def..ff386bb 100644
--- a/build2/pkgconfig/init.cxx
+++ b/build2/pkgconfig/init.cxx
@@ -59,9 +59,9 @@ namespace build2
 
       auto p (config::omitted (rs, c_x));
 
-      if (const value* v = p.first)
+      if (p.first)
       {
-        const path& x (cast<path> (*v));
+        const path& x (cast<path> (p.first));
 
         try
         {
diff --git a/build2/test/common b/build2/test/common
new file mode 100644
index 0000000..e9213b6
--- /dev/null
+++ b/build2/test/common
@@ -0,0 +1,44 @@
+// file      : build2/test/common -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD2_TEST_COMMON
+#define BUILD2_TEST_COMMON
+
+#include <build2/types>
+#include <build2/utility>
+
+#include <build2/target>
+
+namespace build2
+{
+  namespace test
+  {
+    struct common
+    {
+      // The config.test query interface.
+      //
+      const names* test_ = nullptr; // The config.test value if any.
+      scope*       root_ = nullptr; // The root scope for target resolution.
+
+      // Return true if the specified alias target should pass-through to it
+      // prerequisites.
+      //
+      bool
+      pass (target& alias_target) const;
+
+      // Return true if the specified target should be tested.
+      //
+      bool
+      test (target& test_target) const;
+
+      // Return true if the specified target should be tested with the
+      // specified testscript test (or group).
+      //
+      bool
+      test (target& test_target, const path& id_path) const;
+    };
+  }
+}
+
+#endif // BUILD2_TEST_COMMON
diff --git a/build2/test/common.cxx b/build2/test/common.cxx
new file mode 100644
index 0000000..afd52e6
--- /dev/null
+++ b/build2/test/common.cxx
@@ -0,0 +1,214 @@
+// file      : build2/test/common.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <build2/test/common>
+
+#include <build2/target>
+#include <build2/algorithm>
+
+using namespace std;
+
+namespace build2
+{
+  namespace test
+  {
+    // Determine if we have the target (first), id path (second), or both (in
+    // which case we also advance the iterator).
+    //
+    static pair<const name*, const name*>
+    sense (names::const_iterator& i)
+    {
+      const name* tn (nullptr);
+      const name* pn (nullptr);
+
+      if (i->pair)
+      {
+        tn = &*i++;
+        pn = &*i;
+      }
+      else
+      {
+        // If it has a type (exe{hello}) or a directory (basics/), then
+        // we assume it is a target.
+        //
+        (i->typed () || !i->dir.empty () ? tn : pn) = &*i;
+      }
+
+      // Validate the target.
+      //
+      if (tn != nullptr)
+      {
+        if (tn->qualified ())
+          fail << "project-qualified target '" << *tn << " in config.test";
+      }
+
+      // Validate the id path.
+      //
+      if (pn != nullptr)
+      {
+        if (!pn->simple () || pn->empty ())
+          fail << "invalid id path '" << *pn << " in config.test";
+      }
+
+      return make_pair (tn, pn);
+    }
+
+    bool common::
+    pass (target& a) const
+    {
+      if (test_ == nullptr)
+        return true;
+
+      // We need to "enable" aliases that "lead up" to the targets we are
+      // interested in. So see if any target is in a subdirectory of this
+      // alias.
+      //
+      // If we don't see any targets (e.g., only id paths), then we assume all
+      // targets match and therefore we always pass.
+      //
+      bool r (true);
+
+      // Directory part from root to this alias (the same in src and out).
+      //
+      const dir_path d (a.out_dir ().leaf (root_->out_path ()));
+
+      for (auto i (test_->begin ()); i != test_->end (); ++i)
+      {
+        if (const name* n = sense (i).first)
+        {
+          // Reset result to false if no match (but we have seen a target).
+          //
+          r = n->dir.sub (d);
+
+          // See test() below for details on this special case.
+          //
+          if (!r && !n->typed ())
+            r = d.sub (n->dir);
+
+          if (r)
+            break;
+        }
+      }
+
+      return r;
+    }
+
+    bool common::
+    test (target& t) const
+    {
+      if (test_ == nullptr)
+        return true;
+
+      // If we don't see any targets (e.g., only id paths), then we assume
+      // all of them match.
+      //
+      bool r (true);
+
+      // Directory part from root to this alias (the same in src and out).
+      //
+      const dir_path d (t.out_dir ().leaf (root_->out_path ()));
+      const target_type& tt (t.type ());
+
+      for (auto i (test_->begin ()); i != test_->end (); ++i)
+      {
+        if (const name* n = sense (i).first)
+        {
+          // Reset result to false if no match (but we have seen a target).
+          //
+
+          // When specifying a directory, for example, config.tests=tests/,
+          // one would intuitively expect that all the tests under it will
+          // run. But that's not what will happen with the below test: while
+          // the dir{tests/} itself will match, any target underneath won't.
+          // So we are going to handle this type if a target specially by
+          // making it match any target in or under it.
+          //
+          // Note that we only do this for tests/, not dir{tests/} since it is
+          // not always the semantics that one wants. Sometimes one may want
+          // to run tests (scripts) just for the tests/ target but not for any
+          // of its prerequisites. So dir{tests/} is a way to disable this
+          // special logic.
+          //
+          // Note: the same code as in test() below.
+          //
+          if (!n->typed ())
+            r = d.sub (n->dir);
+          else
+            // First quickly and cheaply weed out names that cannot possibly
+            // match. Only then search for a target as if it was a prerequisite,
+            // which can be expensive.
+            //
+            r =
+              t.name == n->value &&       // Name matches.
+              tt.name == n->type &&       // Target type matches.
+              d == n->dir &&              // Directory matches.
+              &search (*n, *root_) == &t;
+
+          if (r)
+            break;
+        }
+      }
+
+      return r;
+    }
+
+    bool common::
+    test (target& t, const path& id) const
+    {
+      if (test_ == nullptr)
+        return true;
+
+      // If we don't see any id paths (e.g., only targets), then we assume
+      // all of them match.
+      //
+      bool r (true);
+
+      // Directory part from root to this alias (the same in src and out).
+      //
+      const dir_path d (t.out_dir ().leaf (root_->out_path ()));
+      const target_type& tt (t.type ());
+
+      for (auto i (test_->begin ()); i != test_->end (); ++i)
+      {
+        auto p (sense (i));
+
+        if (const name* n = p.second)
+        {
+          // If there is a target, check that it matches ours.
+          //
+          if (const name* n = p.first)
+          {
+            // Note: the same code as in test() above.
+            //
+            bool r;
+
+            if (!n->typed ())
+              r = d.sub (n->dir);
+            else
+              r =
+                t.name == n->value &&
+                tt.name == n->type &&
+                d == n->dir &&
+                &search (*n, *root_) == &t;
+
+            if (!r)
+              continue; // Not our target.
+          }
+
+          // If the id (group) "leads up" to what we want to run or we
+          // (group) lead up to the id, then match.
+          //
+          const path p (n->value);
+
+          // Reset result to false if no match (but we have seen an id path).
+          //
+          if ((r = p.sub (id) || id.sub (p)))
+            break;
+        }
+      }
+
+      return r;
+    }
+  }
+}
diff --git a/build2/test/init.cxx b/build2/test/init.cxx
index f592568..4338231 100644
--- a/build2/test/init.cxx
+++ b/build2/test/init.cxx
@@ -9,7 +9,9 @@
 #include <build2/rule>
 #include <build2/diagnostics>
 
-#include <build2/test/rule>
+#include <build2/config/utility>
+
+#include <build2/test/module>
 #include <build2/test/target>
 #include <build2/test/operation>
 
@@ -20,9 +22,6 @@ namespace build2
 {
   namespace test
   {
-    static rule rule_;
-    static alias_rule alias_rule_;
-
     void
     boot (scope& rs, const location&, unique_ptr<module_base>&)
     {
@@ -39,6 +38,16 @@ namespace build2
       //
       auto& vp (var_pool);
 
+      // Tests to execute.
+      //
+      // Specified as <target>@<path-id> pairs with both sides being optional.
+      // The variable is untyped (we want a list of name-pairs), overridable,
+      // and inheritable. The target is relative (in essence a prerequisite)
+      // which is resolved from the (root) scope where the config.test value
+      // is defined.
+      //
+      vp.insert ("config.test", true);
+
       // Note: none are overridable.
       //
       // The test variable is a name which can be a path (with the
@@ -76,7 +85,7 @@ namespace build2
     init (scope& rs,
           scope&,
           const location& l,
-          unique_ptr<module_base>&,
+          unique_ptr<module_base>& mod,
           bool first,
           bool,
           const variable_map& config_hints)
@@ -92,16 +101,39 @@ namespace build2
       const dir_path& out_root (rs.out_path ());
       l5 ([&]{trace << "for " << out_root;});
 
-      assert (config_hints.empty ()); // We don't known any hints.
+      assert (mod == nullptr);
+      mod.reset (new module ());
+      module& m (static_cast<module&> (*mod));
 
-      //@@ TODO: Need ability to specify extra diff options (e.g.,
-      //   --strip-trailing-cr, now hardcoded).
+      // Configure.
+      //
+      assert (config_hints.empty ()); // We don't known any hints.
 
       // Adjust module priority so that the config.test.* values are saved at
       // the end of config.build.
       //
-      // if (s)
-      //   config::save_module (r, "test", INT32_MAX);
+      config::save_module (rs, "test", INT32_MAX);
+
+      // config.test
+      //
+      if (lookup l = config::omitted (rs, "config.test").first)
+      {
+        // Figure out which root scope it came from.
+        //
+        scope* s (&rs);
+        for (;
+             s != nullptr && !l.belongs (*s);
+             s = s->parent_scope ()->root_scope ())
+        assert (s != nullptr);
+
+        m.test_ = &cast<names> (l);
+        m.root_ = s;
+      }
+
+      //@@ TODO: Need ability to specify extra diff options (e.g.,
+      //   --strip-trailing-cr, now hardcoded).
+      //
+      //@@ TODO: Pring report.
 
       // Register target types.
       //
@@ -114,18 +146,19 @@ namespace build2
       // Register rules.
       //
       {
-        auto& r (rs.rules);
+        rule& r (m);
+        alias_rule& ar (m);
 
         // Register our test running rule.
         //
-        r.insert<target> (perform_test_id, "test", rule_);
-        r.insert<alias> (perform_test_id, "test", alias_rule_);
+        rs.rules.insert<target> (perform_test_id, "test", r);
+        rs.rules.insert<alias> (perform_test_id, "test", ar);
 
         // Register our rule for the dist meta-operation. We need to do this
         // because we may have ad hoc prerequisites (test input/output files)
         // that need to be entered into the target list.
         //
-        r.insert<target> (dist_id, test_id, "test", rule_);
+        rs.rules.insert<target> (dist_id, test_id, "test", r);
       }
 
       return true;
diff --git a/build2/test/module b/build2/test/module
new file mode 100644
index 0000000..49b3031
--- /dev/null
+++ b/build2/test/module
@@ -0,0 +1,26 @@
+// file      : build2/test/module -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD2_TEST_MODULE
+#define BUILD2_TEST_MODULE
+
+#include <build2/types>
+#include <build2/utility>
+
+#include <build2/module>
+
+#include <build2/test/rule>
+#include <build2/test/common>
+
+namespace build2
+{
+  namespace test
+  {
+    struct module: module_base, virtual common, rule, alias_rule
+    {
+    };
+  }
+}
+
+#endif // BUILD2_TEST_MODULE
diff --git a/build2/test/rule b/build2/test/rule
index e3fc864..20ad905 100644
--- a/build2/test/rule
+++ b/build2/test/rule
@@ -11,34 +11,40 @@
 #include <build2/rule>
 #include <build2/operation>
 
+#include <build2/test/common>
+
 namespace build2
 {
   namespace test
   {
-    class rule: public build2::rule
+    class rule_common: public build2::rule, protected virtual common
     {
     public:
       virtual match_result
       match (action, target&, const string&) const override;
 
+      target_state
+      perform_script (action, target&) const;
+    };
+
+    class rule: public rule_common
+    {
+    public:
       virtual recipe
       apply (action, target&) const override;
 
       static target_state
-      perform_script (action, target&);
-
-      static target_state
       perform_test (action, target&);
     };
 
-    class alias_rule: public rule
+    class alias_rule: public rule_common
     {
     public:
       virtual recipe
       apply (action, target&) const override;
 
-      static target_state
-      perform_test (action, target&);
+      target_state
+      perform_test (action, target&) const;
     };
   }
 }
diff --git a/build2/test/rule.cxx b/build2/test/rule.cxx
index 4201f37..c50f035 100644
--- a/build2/test/rule.cxx
+++ b/build2/test/rule.cxx
@@ -25,14 +25,16 @@ namespace build2
   {
     struct match_data
     {
-      bool test = false;
-      bool script = false;
+      bool pass; // Pass-through to prerequsites (for alias only).
+      bool test;
+
+      bool script;
     };
 
     static_assert (sizeof (match_data) <= target::data_size,
                    "insufficient space");
 
-    match_result rule::
+    match_result rule_common::
     match (action a, target& t, const string&) const
     {
       // The (admittedly twisted) logic of this rule tries to achieve the
@@ -47,64 +49,69 @@ namespace build2
       // (which, if not testable, it will noop).
       //
       // And to add a bit more complexity, we want to handle aliases slightly
-      // differently: we don't want to ignore their prerequisites if the alias
-      // is not testable since their prerequisites could be.
+      // differently: we may not want to ignore their prerequisites if the
+      // alias is not testable since their prerequisites could be.
 
-      match_data md;
+      match_data md {t.is_a<alias> () && pass (t), false, false};
 
-      // We have two very different cases: testscript and simple test (plus it
-      // may not be a testable target at all). So as the first step determine
-      // which case this is.
-      //
-      // If we have any prerequisites of the test{} type, then this is the
-      // testscript case.
-      //
-      for (prerequisite_member p: group_prerequisite_members (a, t))
+      if (test (t))
       {
-        if (p.is_a<testscript> ())
+        // We have two very different cases: testscript and simple test (plus
+        // it may not be a testable target at all). So as the first step
+        // determine which case this is.
+        //
+        // If we have any prerequisites of the test{} type, then this is the
+        // testscript case.
+        //
+        for (prerequisite_member p: group_prerequisite_members (a, t))
         {
-          md.script = true;
+          if (p.is_a<testscript> ())
+          {
+            md.script = true;
 
-          // We treat this target as testable unless the test variable is
-          // explicitly set to false.
-          //
-          const name* n (cast_null<name> (t["test"]));
-          md.test = n == nullptr || !n->simple () || n->value != "false";
-          break;
+            // We treat this target as testable unless the test variable is
+            // explicitly set to false.
+            //
+            const name* n (cast_null<name> (t["test"]));
+            md.test = n == nullptr || !n->simple () || n->value != "false";
+            break;
+          }
         }
-      }
 
-      // If this is not a script, then determine if it is a simple test.
-      // Ignore aliases and testscripts files themselves at the outset.
-      //
-      if (!md.script && !t.is_a<alias> () && !t.is_a<testscript> ())
-      {
-        // For the simple case whether this is a test is controlled by the
-        // test variable. Also, it feels redundant to specify, say, "test =
-        // true" and "test.output = test.out" -- the latter already says this
-        // is a test.
+        // If this is not a script, then determine if it is a simple test.
+        // Ignore aliases and testscripts files themselves at the outset.
         //
+        if (!md.script && !t.is_a<alias> () && !t.is_a<testscript> ())
+        {
+          // For the simple case whether this is a test is controlled by the
+          // test variable. Also, it feels redundant to specify, say, "test =
+          // true" and "test.output = test.out" -- the latter already says this
+          // is a test.
+          //
 
-        // Use lookup depths to figure out who "overrides" whom.
-        //
-        auto p (t.find ("test"));
-        const name* n (cast_null<name> (p.first));
+          // Use lookup depths to figure out who "overrides" whom.
+          //
+          auto p (t.find ("test"));
+          const name* n (cast_null<name> (p.first));
 
-        if (n != nullptr && n->simple () && n->value != "false")
-          md.test = true;
-        else
-        {
-          auto test = [&t, &p] (const char* var)
+          // Note that test can be set to an "override" target.
+          //
+          if (n != nullptr && (!n->simple () || n->value != "false"))
+            md.test = true;
+          else
           {
-            return t.find (var).second < p.second;
-          };
-
-          md.test =
-            test ("test.input")     ||
-            test ("test.output")    ||
-            test ("test.roundtrip") ||
-            test ("test.options")   ||
-            test ("test.arguments");
+            auto test = [&t, &p] (const char* var)
+            {
+              return t.find (var).second < p.second;
+            };
+
+            md.test =
+              test ("test.input")     ||
+              test ("test.output")    ||
+              test ("test.roundtrip") ||
+              test ("test.options")   ||
+              test ("test.arguments");
+          }
         }
       }
 
@@ -137,7 +144,7 @@ namespace build2
       // Change the recipe action to (update, 0) (i.e., "unconditional
       // update") to make sure we won't match any prerequisites.
       //
-      if (md.test && a.operation () == update_id)
+      if (a.operation () == update_id && (md.pass || md.test))
         mr.recipe_action = action (a.meta_operation (), update_id);
 
       // Note that we match even if this target is not testable so that we can
@@ -157,6 +164,9 @@ namespace build2
       //
       assert (!md.test || md.script);
 
+      if (!md.pass && !md.test)
+        return noop_recipe;
+
       // If this is the update pre-operation then simply redirect to the
       // standard alias rule.
       //
@@ -175,7 +185,9 @@ namespace build2
 
       // If not a test then also redirect to the alias rule.
       //
-      return md.test ? perform_test : default_recipe;
+      return md.test
+        ? [this] (action a, target& t) {return perform_test (a, t);}
+        : default_recipe;
     }
 
     recipe rule::
@@ -206,7 +218,7 @@ namespace build2
             t.prerequisite_targets.push_back (&p.search ());
         }
 
-        return &perform_script;
+        return [this] (action a, target& t) {return perform_script (a, t);};
       }
       else
       {
@@ -350,29 +362,33 @@ namespace build2
       }
     }
 
-    target_state rule::
-    perform_script (action, target& t)
+    target_state rule_common::
+    perform_script (action, target& t) const
     {
       // Figure out whether the testscript file is called 'testscript', in
       // which case it should be the only one.
       //
-      optional<bool> one;
-      for (target* pt: t.prerequisite_targets)
+      bool one;
       {
-        // In case we are using the alias rule's list (see above).
-        //
-        if (testscript* ts = pt->is_a<testscript> ())
+        optional<bool> o;
+        for (target* pt: t.prerequisite_targets)
         {
-          bool r (ts->name == "testscript");
+          // In case we are using the alias rule's list (see above).
+          //
+          if (testscript* ts = pt->is_a<testscript> ())
+          {
+            bool r (ts->name == "testscript");
 
-          if ((r && one) || (!r && one && *one))
-            fail << "both 'testscript' and other names specified for " << t;
+            if ((r && o) || (!r && o && *o))
+              fail << "both 'testscript' and other names specified for " << t;
 
-          one = r;
+            o = r;
+          }
         }
-      }
 
-      assert (one); // We should have a testscript or we wouldn't be here.
+        assert (o); // We should have a testscript or we wouldn't be here.
+        one = *o;
+      }
 
       // Calculate root working directory. It is in the out_base of the target
       // and is called just test for dir{} targets and test-<target-name> for
@@ -402,45 +418,58 @@ namespace build2
 
       if (exists (wd))
       {
-        bool e (empty (wd));
-
         warn << "working directory " << wd << " exists "
-             << (e ? "" : "and is not empty ") << "at the beginning "
+             << (empty (wd) ? "" : "and is not empty ") << "at the beginning "
              << "of the test";
 
-        if (!e)
-          build2::rmdir_r (wd, false, 2);
+        // Remove the directory itself not to confuse the runner which tries
+        // to detect when tests stomp on each others feet.
+        //
+        build2::rmdir_r (wd, true, 2);
       }
-      else if (!*one)
-        mkdir (wd, 2);
+
+      // Delay actually creating the directory in case all the tests are
+      // ignored (via config.test).
+      //
+      bool mk (!one);
 
       // Run all the testscripts.
       //
-      auto run = [&t, &wd] (testscript& ts)
+      for (target* pt: t.prerequisite_targets)
       {
-        if (verb)
+        if (testscript* ts = pt->is_a<testscript> ())
         {
-          const auto& tt (cast<target_triplet> (t["test.target"]));
-          text << "test " << t << " with " << ts << " on " << tt;
-        }
+          // If this is just the testscript, then its id path is empty (and
+          // it can only be ignored by ignoring the test target, which makes
+          // sense since it's the only testscript file).
+          //
+          if (one || test (t, path (ts->name)))
+          {
+            if (mk)
+            {
+              mkdir (wd, 2);
+              mk = false;
+            }
 
-        script::parser p;
-        script::script s (t, ts, wd);
-        p.pre_parse (s);
+            if (verb)
+            {
+              const auto& tt (cast<target_triplet> (t["test.target"]));
+              text << "test " << t << " with " << *ts << " on " << tt;
+            }
 
-        script::default_runner r;
-        p.execute (s, r);
-      };
+            script::parser p;
+            script::script s (t, *ts, wd);
+            p.pre_parse (s);
 
-      for (target* pt: t.prerequisite_targets)
-      {
-        if (testscript* ts = pt->is_a<testscript> ())
-          run (*ts);
+            script::default_runner r (*this);
+            p.execute (s, r);
+          }
+        }
       }
 
       // Cleanup.
       //
-      if (!*one)
+      if (!one && !mk)
       {
         if (!empty (wd))
           fail << "working directory " << wd << " is not empty at the "
@@ -471,10 +500,9 @@ namespace build2
       for (next++; *next != nullptr; next++) ;
       next++;
 
-      // Redirect stdout to a pipe unless we are last, in which
-      // case redirect it to stderr.
+      // Redirect stdout to a pipe unless we are last.
       //
-      int out (*next == nullptr ? 2 : -1);
+      int out (*next != nullptr ? -1 : 1);
       bool pr, wr;
 
       try
@@ -519,28 +547,89 @@ namespace build2
     }
 
     target_state rule::
-    perform_test (action, target& t)
+    perform_test (action, target& tt)
     {
       // @@ Would be nice to print what signal/core was dumped.
       //
-      // @@ Doesn't have to be a file target if we have test.cmd (or
-      //    just use test which is now path).
+
+      // See if we have the test executable override.
       //
+      path p;
+      {
+        // Note that the test variable's visibility is target.
+        //
+        lookup l (tt["test"]);
+
+        // Note that we have similar code for scripted tests.
+        //
+        target* t (nullptr);
+
+        if (l.defined ())
+        {
+          const name* n (cast_null<name> (l));
+
+          if (n == nullptr)
+            fail << "invalid test executable override: null value";
+          else if (n->empty ())
+            fail << "invalid test executable override: empty value";
+          else if (n->simple ())
+          {
+            // Ignore the special 'true' value.
+            //
+            if (n->value != "true")
+              p = path (n->value);
+            else
+              t = &tt;
+          }
+          else if (n->directory ())
+            fail << "invalid test executable override: '" << *n << "'";
+          else
+          {
+            // Must be a target name.
+            //
+            // @@ OUT: what if this is a @-qualified pair or names?
+            //
+            t = &search (*n, tt.base_scope ());
+          }
+        }
+        else
+          // By default we set it to the test target's path.
+          //
+          t = &tt;
+
+        if (t != nullptr)
+        {
+          if (auto* pt = t->is_a<path_target> ())
+          {
+            // Do some sanity checks: the target better be up-to-date with
+            // an assigned path.
+            //
+            p = pt->path ();
 
-      file& ft (static_cast<file&> (t));
-      assert (!ft.path ().empty ()); // Should have been assigned by update.
+            if (p.empty ())
+              fail << "target " << *pt << " specified in the test variable "
+                   << "is out of date" <<
+                info << "consider specifying it as a prerequisite of " << tt;
+          }
+          else
+            fail << "target " << *t << (t != &tt
+                                        ? " specified in the test variable "
+                                        : " requested to be tested ")
+                 << "is not path-based";
+        }
+      }
 
-      process_path fpp (run_search (ft.path (), true));
-      cstrings args {fpp.recall_string ()};
+      process_path pp (run_search (p, true));
+      cstrings args {pp.recall_string ()};
 
       // Do we have options?
       //
-      if (auto l = t["test.options"])
+      if (auto l = tt["test.options"])
         append_options (args, cast<strings> (l));
 
       // Do we have input?
       //
-      auto& pts (t.prerequisite_targets);
+      auto& pts (tt.prerequisite_targets);
       if (pts.size () != 0 && pts[0] != nullptr)
       {
         file& it (static_cast<file&> (*pts[0]));
@@ -551,7 +640,7 @@ namespace build2
       //
       else
       {
-        if (auto l = t["test.arguments"])
+        if (auto l = tt["test.arguments"])
           append_options (args, cast<strings> (l));
       }
 
@@ -581,10 +670,10 @@ namespace build2
       if (verb >= 2)
         print_process (args);
       else if (verb)
-        text << "test " << t;
+        text << "test " << tt;
 
       diag_record dr;
-      if (!run_test (t, dr, args.data ()))
+      if (!run_test (tt, dr, args.data ()))
       {
         dr << info << "test command line: ";
         print_process (dr, args);
@@ -595,7 +684,7 @@ namespace build2
     }
 
     target_state alias_rule::
-    perform_test (action a, target& t)
+    perform_test (action a, target& t) const
     {
       // Run the alias recipe first then the test.
       //
diff --git a/build2/test/script/parser b/build2/test/script/parser
index 8f30a8c..9ad5fe9 100644
--- a/build2/test/script/parser
+++ b/build2/test/script/parser
@@ -136,7 +136,11 @@ namespace build2
         //
       public:
         void
-        execute (script& s, runner& r) {if (!s.empty ()) execute (s, s, r);}
+        execute (script& s, runner& r)
+        {
+          if (!s.empty ())
+            execute (s, s, r);
+        }
 
         void
         execute (scope&, script&, runner&);
@@ -195,7 +199,7 @@ namespace build2
         lexer* lexer_;
         string id_prefix_; // Auto-derived id prefix.
 
-        // Parse state.
+        // Execute state.
         //
         runner* runner_;
         scope* scope_;
diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx
index da072dc..f250d27 100644
--- a/build2/test/script/parser.cxx
+++ b/build2/test/script/parser.cxx
@@ -2,11 +2,11 @@
 // copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
 // license   : MIT; see accompanying LICENSE file
 
+#include <build2/test/script/parser>
+
 #include <sstream>
 #include <cstring> // strstr()
 
-#include <build2/test/script/parser>
-
 #include <build2/scheduler>
 
 #include <build2/test/script/lexer>
@@ -2743,6 +2743,14 @@ namespace build2
 
           for (unique_ptr<scope>& chain: g->scopes)
           {
+            // Check if this scope is ignored (e.g., via config.test).
+            //
+            if (!runner_->test (*chain))
+            {
+              chain.reset ();
+              continue;
+            }
+
             // Pick a scope from the if-else chain.
             //
             // In fact, we are going to drop all but the selected (if any)
diff --git a/build2/test/script/runner b/build2/test/script/runner
index 5e05255..7b932b9 100644
--- a/build2/test/script/runner
+++ b/build2/test/script/runner
@@ -16,11 +16,18 @@ namespace build2
 {
   namespace test
   {
+    class common;
+
     namespace script
     {
       class runner
       {
       public:
+        // Return false if this test/group should be skipped.
+        //
+        virtual bool
+        test (scope&) const = 0;
+
         // Location is the scope start location (for diagnostics, etc).
         //
         virtual void
@@ -49,6 +56,12 @@ namespace build2
       class default_runner: public runner
       {
       public:
+        explicit
+        default_runner (const common& c): common_ (c) {}
+
+        virtual bool
+        test (scope& s) const override;
+
         virtual void
         enter (scope&, const location&) override;
 
@@ -60,6 +73,9 @@ namespace build2
 
         virtual void
         leave (scope&, const location&) override;
+
+      private:
+        const common& common_;
       };
     }
   }
diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx
index 64c6e87..522dedd 100644
--- a/build2/test/script/runner.cxx
+++ b/build2/test/script/runner.cxx
@@ -10,6 +10,9 @@
 #include <butl/fdstream> // fdopen_mode, fdnull(), fddup()
 
 #include <build2/filesystem>
+
+#include <build2/test/common>
+
 #include <build2/test/script/builtin>
 
 using namespace std;
@@ -310,21 +313,26 @@ namespace build2
         }
       }
 
+      bool default_runner::
+      test (scope& s) const
+      {
+        return common_.test (s.root->test_target, s.id_path);
+      }
+
       void default_runner::
       enter (scope& sp, const location&)
       {
-        if (!exists (sp.wd_path))
-          // @@ Shouldn't we add an optional location parameter to mkdir() and
-          // alike utility functions so the failure message can contain
-          // location info?
-          //
-          mkdir (sp.wd_path, 2);
-        else
-          // Scope working directory shall be empty (the script working
-          // directory is cleaned up by the test rule prior the script
-          // execution).
-          //
-          assert (empty (sp.wd_path));
+        // Scope working directory shall be empty (the script working
+        // directory is cleaned up by the test rule prior the script
+        // execution).
+        //
+        // @@ Shouldn't we add an optional location parameter to mkdir() and
+        // alike utility functions so the failure message can contain
+        // location info?
+        //
+        if (mkdir (sp.wd_path, 2) == mkdir_status::already_exists)
+          fail << "working directory " << sp.wd_path << " already exists" <<
+            info << "are tests stomping on each other's feet?";
 
         // We don't change the current directory here but indicate that the
         // scope test commands will be executed in that directory.
diff --git a/build2/test/script/script.cxx b/build2/test/script/script.cxx
index 661ec7f..2a34f66 100644
--- a/build2/test/script/script.cxx
+++ b/build2/test/script/script.cxx
@@ -424,24 +424,11 @@ namespace build2
 
       // script
       //
-      static inline string
-      script_id (const path& p)
-      {
-        string r (p.leaf ().string ());
-
-        if (r == "testscript")
-          return string ();
-
-        size_t n (path::traits::find_extension (r));
-        assert (n != string::npos);
-        r.resize (n);
-        return r;
-      }
-
       script::
       script (target& tt, testscript& st, const dir_path& rwd)
-          : group (script_id (st.path ())),
-            test_target (tt), script_target (st)
+          : group (st.name == "testscript" ? string () : st.name),
+            test_target (tt),
+            script_target (st)
       {
         // Set the script working dir ($~) to $out_base/test/<id> (id_path
         // for root is just the id which is empty if st is 'testscript').
@@ -458,7 +445,10 @@ namespace build2
           //
           lookup l (find_in_buildfile ("test", false));
 
+          // Note that we have similar code for simple tests.
+          //
           target* t (nullptr);
+
           if (l.defined ())
           {
             const name* n (cast_null<name> (l));