Add support for $path_search() and $path_match()

13 files changed, 467 insertions, 38 deletions

diff --git a/build2/buildfile b/build2/buildfile
index 6d48718..8525eb8 100644
--- a/build2/buildfile
+++ b/build2/buildfile
@@ -16,6 +16,7 @@ exe{b}:                                                   \
             {hxx     txx cxx}{ filesystem               } \
             {hxx         cxx}{ function                 } \
             {            cxx}{ functions-builtin        } \
+            {            cxx}{ functions-filesystem     } \
             {            cxx}{ functions-path           } \
             {            cxx}{ functions-process-path   } \
             {            cxx}{ functions-string         } \
diff --git a/build2/function.cxx b/build2/function.cxx
index 9f2fb9c..399d679 100644
--- a/build2/function.cxx
+++ b/build2/function.cxx
@@ -301,9 +301,10 @@ namespace build2
   function_map functions;
 
   void builtin_functions ();        // functions-builtin.cxx
-  void string_functions ();         // functions-string.cxx
+  void filesystem_functions ();     // functions-filesystem.cxx
   void path_functions ();           // functions-path.cxx
   void process_path_functions ();   // functions-process-path.cxx
+  void string_functions ();         // functions-string.cxx
   void target_triplet_functions (); // functions-target-triplet.cxx
 
   struct functions_init
@@ -311,9 +312,10 @@ namespace build2
     functions_init ()
     {
       builtin_functions ();
-      string_functions ();
+      filesystem_functions ();
       path_functions ();
       process_path_functions ();
+      string_functions ();
       target_triplet_functions ();
     }
   };
diff --git a/build2/functions-builtin.cxx b/build2/functions-builtin.cxx
index f9bbda5..d6d9501 100644
--- a/build2/functions-builtin.cxx
+++ b/build2/functions-builtin.cxx
@@ -47,16 +47,7 @@ namespace build2
 
     f["getenv"] = [](names name)
     {
-      // Note that if the name size is greater than one, we will fail with the
-      // proper diagnostics.
-      //
-      assert (!name.empty ());
-
-      string n (name.size () == 1
-                ? convert<string> (move (name[0]))
-                : convert<string> (move (name)));
-
-      return getenv (n);
+      return getenv (convert<string> (move (name)));
     };
   }
 }
diff --git a/build2/functions-filesystem.cxx b/build2/functions-filesystem.cxx
new file mode 100644
index 0000000..d9e74b7
--- /dev/null
+++ b/build2/functions-filesystem.cxx
@@ -0,0 +1,274 @@
+// file      : build2/functions-filesystem.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <libbutl/filesystem.hxx>
+
+#include <build2/function.hxx>
+#include <build2/variable.hxx>
+
+using namespace std;
+
+namespace build2
+{
+  // Return paths of filesystem entries that match the pattern. See
+  // path_search() overloads (below) for details.
+  //
+  static names
+  path_search (const path& pattern, const optional<dir_path>& start)
+  {
+    names r;
+    auto add = [&r] (path&& p, const std::string&, bool interm) -> bool
+    {
+      // Canonicalizing paths seems to be the right thing to do. Otherwise, we
+      // can end up with different separators in the same path on Windows.
+      //
+      if (!interm)
+        r.emplace_back (
+          value_traits<path>::reverse (move (p.canonicalize ())));
+
+      return true;
+    };
+
+    // Print paths "as is" in the diagnostics.
+    //
+    try
+    {
+      if (pattern.absolute ())
+        path_search (pattern, add);
+      else
+      {
+        // An absolute start directory must be specified for the relative
+        // pattern.
+        //
+        if (!start || start->relative ())
+        {
+          diag_record dr (fail);
+
+          if (!start)
+            dr << "start directory is not specified";
+          else
+            dr << "start directory '" << start->representation ()
+               << "' is relative";
+
+          dr << info << "pattern '" << pattern.representation ()
+             << "' is relative";
+        }
+
+        path_search (pattern, add, *start);
+      }
+    }
+    catch (const system_error& e)
+    {
+      diag_record d (fail);
+      d << "unable to scan";
+
+      // If the pattern is absolute, then the start directory is not used, and
+      // so printing it would be misleading.
+      //
+      if (start && pattern.relative ())
+        d << " '" << start->representation () << "'";
+
+      d << ": " << e
+        << info << "pattern: '" << pattern.representation () << "'";
+    }
+
+    return r;
+  }
+
+  using butl::path_match;
+
+  // Return true if a path for a filesystem entry matches the pattern. See
+  // path_match() overloads (below) for details.
+  //
+  static bool
+  path_match (const path& pattern,
+              const path& entry,
+              const optional<dir_path>& start)
+  {
+    // If pattern and entry are both either absolute or relative and
+    // non-empty, and the first pattern component is not a self-matching
+    // wildcard, then ignore the start directory.
+    //
+    bool rel (pattern.relative () == entry.relative () &&
+              !pattern.empty () && !entry.empty ());
+
+    bool self (!pattern.empty () &&
+               (*pattern.begin ()).find ("***") != string::npos);
+
+    if (rel && !self)
+      return path_match (pattern, entry);
+
+    // The start directory must be specified and be absolute.
+    //
+    if (!start || start->relative ())
+    {
+      diag_record dr (fail);
+
+      // Print paths "as is".
+      //
+      if (!start)
+        dr << "start directory is not specified";
+      else
+        dr << "start directory path '" << start->representation ()
+           << "' is relative";
+
+      dr << info << "pattern: '" << pattern.representation () << "'"
+         << info << "entry: '" << entry.representation () << "'";
+    }
+
+    return path_match (pattern, entry, *start);
+  }
+
+  void
+  filesystem_functions ()
+  {
+    function_family f ("filesystem");
+
+    // path_search
+    //
+    // Return filesystem paths that match the pattern. If the pattern is an
+    // absolute path, then the start directory is ignored (if present).
+    // Otherwise, the start directory must be specified and be absolute.
+    //
+    f["path_search"] = [](path pattern, optional<dir_path> start)
+    {
+      return path_search (pattern, start);
+    };
+
+    f["path_search"] = [](path pattern, names start)
+    {
+      return path_search (pattern, convert<dir_path> (move (start)));
+    };
+
+    f["path_search"] = [](names pattern, optional<dir_path> start)
+    {
+      return path_search (convert<path> (move (pattern)), start);
+    };
+
+    f["path_search"] = [](names pattern, names start)
+    {
+      return path_search (convert<path>     (move (pattern)),
+                          convert<dir_path> (move (start)));
+    };
+
+    // path_match
+    //
+    // Match a filesystem entry name against a name pattern (both are strings),
+    // or a filesystem entry path against a path pattern. For the latter case
+    // the start directory may also be required (see below). The semantics of
+    // the pattern and name/entry arguments is determined according to the
+    // following rules:
+    //
+    // - The arguments must be of the string or path types, or be untyped.
+    //
+    // - If one of the arguments is typed, then the other one must be of the
+    //   same type or be untyped. In the later case, an untyped argument is
+    //   converted to the type of the other argument.
+    //
+    // - If both arguments are untyped and the start directory is specified,
+    //   then the arguments are converted to the path type.
+    //
+    // - If both arguments are untyped and the start directory is not
+    //   specified, then, if one of the arguments is syntactically a path (the
+    //   value contains a directory separator), convert them to the path type,
+    //   otherwise to the string type (match as names).
+    //
+    // If pattern and entry paths are both either absolute or relative and
+    // non-empty, and the first pattern component is not a self-matching
+    // wildcard (doesn't contain ***), then the start directory is not
+    // required, and is ignored if specified. Otherwise, the start directory
+    // must be specified and be an absolute path.
+    //
+    // Name matching.
+    //
+    f["path_match"] = [](string pattern, string name)
+    {
+      return path_match (pattern, name);
+    };
+
+    f["path_match"] = [](string pattern, names name)
+    {
+      return path_match (pattern, convert<string> (move (name)));
+    };
+
+    f["path_match"] = [](names pattern, string name)
+    {
+      return path_match (convert<string> (move (pattern)), name);
+    };
+
+    // Path matching.
+    //
+    //                   path      path      *
+    //
+    f["path_match"] = [](path pat, path ent, optional<dir_path> start)
+    {
+      return path_match (pat, ent, start);
+    };
+
+    f["path_match"] = [](path pat, path ent, names start)
+    {
+      return path_match (pat, ent, convert<dir_path> (move (start)));
+    };
+
+    //                   path      untyped    *
+    //
+    f["path_match"] = [](path pat, names ent, optional<dir_path> start)
+    {
+      return path_match (pat, convert<path> (move (ent)), start);
+    };
+
+    f["path_match"] = [](path pat, names ent, names start)
+    {
+      return path_match (pat,
+                         convert<path> (move (ent)),
+                         convert<dir_path> (move (start)));
+    };
+
+    //                   untyped    path      *
+    //
+    f["path_match"] = [](names pat, path ent, optional<dir_path> start)
+    {
+      return path_match (convert<path> (move (pat)), ent, start);
+    };
+
+    f["path_match"] = [](names pat, path ent, names start)
+    {
+      return path_match (convert<path> (move (pat)),
+                         ent,
+                         convert<dir_path> (move (start)));
+    };
+
+    // The semantics depends on the presence of the start directory or the
+    // first two argument syntactic representation.
+    //
+    //                   untyped    untyped    *
+    //
+    f["path_match"] = [](names pat, names ent, optional<dir_path> start)
+    {
+      auto path_arg = [] (const names& a) -> bool
+      {
+        return a.size () == 1 &&
+        (a[0].directory () ||
+         a[0].value.find_first_of (path::traits::directory_separators) !=
+         string::npos);
+      };
+
+      return start || path_arg (pat) || path_arg (ent)
+        ? path_match (convert<path> (move (pat)),   // Match as paths.
+                      convert<path> (move (ent)),
+                      start)
+        : path_match (convert<string> (move (pat)), // Match as names.
+                      convert<string> (move (ent)));
+    };
+
+    f["path_match"] = [](names pat, names ent, names start)
+    {
+      // Match as paths.
+      //
+      return path_match (convert<path> (move (pat)),
+                         convert<path> (move (ent)),
+                         convert<dir_path> (move (start)));
+    };
+  }
+}
diff --git a/tests/function/filesystem/buildfile b/tests/function/filesystem/buildfile
new file mode 100644
index 0000000..ade9a68
--- /dev/null
+++ b/tests/function/filesystem/buildfile
@@ -0,0 +1,5 @@
+# file      : tests/function/filesystem/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: test{testscript} $b
diff --git a/tests/function/filesystem/testscript b/tests/function/filesystem/testscript
new file mode 100644
index 0000000..5c8a9a6
--- /dev/null
+++ b/tests/function/filesystem/testscript
@@ -0,0 +1,155 @@
+# file      : tests/function/filesystem/testscript
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+.include ../../common.test
+
+: path-search
+:
+{
+  +mkdir -p a/b
+
+  : pattern-path
+  :
+  : Test overloads for pattern being of the path type.
+  :
+  {
+    : start-abs-dir
+    :
+    $* <'print $path_search([path] "b**/", [dir_path] $src_base/../../a)' >/'b/'
+
+    : start-relative-dir
+    :
+    $* <'print $path_search([path] "b**/", [dir_path] ../../a)' 2>>~%EOE% != 0
+    %error: start directory '.+' is relative%
+    %  info: pattern '.+' is relative%
+    EOE
+
+    : start-untyped
+    :
+    $* <'print $path_search([path] "b**/", $src_base/../../a)' >/'b/'
+
+    : start-multiple-names
+    :
+    $* <'print $path_search([path] "b**/", a b)' 2>>EOE != 0
+    error: invalid argument: invalid dir_path value: multiple names
+    EOE
+  }
+
+  : pattern-untyped
+  :
+  : Test overloads for pattern being untyped.
+  :
+  {
+    : start-dir
+    :
+    $* <'print $path_search("b**/", [dir_path] $src_base/../../a)' >/'b/'
+
+    : start-untyped
+    :
+    $* <'print $path_search("b**/", $src_base/../../a)' >/'b/'
+
+    : abs-pattern
+    :
+    if ($test.target == $build.host)
+    {
+      touch b;
+      $* <'print $path_search("$src_base/b*")' >>/"EOO"
+      $out_base/test/$@/b
+      EOO
+    }
+  }
+
+  : pattern-multiple-names
+  :
+  {
+    : dir
+    :
+    touch b;
+    $* <'print $path_search(a b, $src_base)' 2>>EOE != 0
+    error: invalid argument: invalid path value: multiple names
+    EOE
+  }
+}
+
+: path-match
+:
+{
+  : string
+  :
+  : Test overloads for at least one of the first two arguments being of the
+  : string type.
+  :
+  {
+    : string-string
+    :
+    $* <'print $path_match([string] "b*", [string] "b")' >'true'
+
+    : string-untyped
+    :
+    $* <'print $path_match([string] "b*", "b")' >'true'
+
+    : untyped-string
+    :
+    $* <'print $path_match("b*", [string] "b")' >'true'
+
+    : string-path
+    :
+    $* <'print $path_match([string] "b*", [path] "b")' 2>>~/EOE/ != 0
+    <stdin>:1:8: error: unmatched call to path_match(string, path)
+    /.{11}
+    EOE
+  }
+
+  : path
+  :
+  : Test overloads for at least one of the first two arguments being of the
+  : path type.
+  :
+  {
+    : path-path
+    :
+    $* <'print $path_match([path] "b**", [path] "a/b")' >'true'
+
+    : path-path-untyped
+    :
+    $* <'print $path_match([path] "b**", [path] "a/b", "$src_base")' >'true'
+
+    : path-untyped
+    :
+    $* <'print $path_match([path] "b**", "a/b")' >'true'
+
+    : path-untyped-dir
+    :
+    $* <'print $path_match([path] "b**", "a/b", $src_base)' >'true'
+
+    : untyped-path
+    :
+    $* <'print $path_match("b**", [path] "a/b")' >'true'
+  }
+
+  : untyped
+  :
+  : Test overloads for the first two arguments being untyped.
+  :
+  {
+    : converted-to-strings
+    :
+    $* <'print $path_match("b**", "b")' >'true'
+
+    : converted-to-paths-due-to
+    {
+      : pattern
+      :
+      $* <'print $path_match("b**/", "a/b/")' >'true'
+
+      : entry
+      :
+      $* <'print $path_match("b**", "a/b")' >'true'
+
+      : start-dir
+      :
+      $* <'print $path_match("s***/", "", "$src_base")' >'true'
+    }
+  }
+}
diff --git a/unit-tests/cc/lexer/buildfile b/unit-tests/cc/lexer/buildfile
index ff4e0b3..3152e77 100644
--- a/unit-tests/cc/lexer/buildfile
+++ b/unit-tests/cc/lexer/buildfile
@@ -5,11 +5,11 @@
 #@@ Temporary until we get utility library support.
 #
 import libs = libbutl%lib{butl}
-src = cc/lexer token lexer diagnostics utility variable name b-options types-parsers \
-context scope parser target operation rule prerequisite file module function \
-functions-builtin functions-path functions-process-path functions-string \
-functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation module} spec
+src = cc/lexer token lexer diagnostics utility variable name b-options \
+types-parsers context scope parser target operation rule prerequisite file \
+module function functions-builtin functions-filesystem functions-path \
+functions-process-path functions-string functions-target-triplet algorithm \
+search dump filesystem scheduler config/{utility init operation module} spec
 
 exe{driver}: cxx{driver} ../../../build2/cxx{$src} ../../../build2/liba{b} \
 $libs test{*}
diff --git a/unit-tests/cc/parser/buildfile b/unit-tests/cc/parser/buildfile
index 5d20367..bb1ad53 100644
--- a/unit-tests/cc/parser/buildfile
+++ b/unit-tests/cc/parser/buildfile
@@ -5,10 +5,11 @@
 #@@ Temporary until we get utility library support.
 #
 import libs = libbutl%lib{butl}
-src = cc/{lexer parser} token lexer diagnostics utility variable name b-options types-parsers \
-context scope parser target operation rule prerequisite file module function \
-functions-builtin functions-path functions-process-path functions-string \
-functions-target-triplet algorithm search dump filesystem scheduler \
+src = cc/{lexer parser} token lexer diagnostics utility variable name    \
+b-options types-parsers context scope parser target operation rule       \
+prerequisite file module function functions-builtin functions-filesystem \
+functions-path functions-process-path functions-string                   \
+functions-target-triplet algorithm search dump filesystem scheduler      \
 config/{utility init operation module} spec
 
 exe{driver}: cxx{driver} ../../../build2/cxx{$src} ../../../build2/liba{b} \
diff --git a/unit-tests/function/buildfile b/unit-tests/function/buildfile
index e61737c..c0cd26f 100644
--- a/unit-tests/function/buildfile
+++ b/unit-tests/function/buildfile
@@ -7,9 +7,9 @@
 import libs = libbutl%lib{butl}
 src = token lexer diagnostics utility variable name b-options types-parsers \
 context scope parser target operation rule prerequisite file module function \
-functions-builtin functions-path functions-process-path functions-string \
-functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation module} spec
+functions-builtin functions-filesystem functions-path functions-process-path \
+functions-string functions-target-triplet algorithm search dump filesystem \
+scheduler config/{utility init operation module} spec
 
 exe{driver}: cxx{driver} ../../build2/cxx{$src} ../../build2/liba{b} $libs test{call syntax}
 
diff --git a/unit-tests/lexer/buildfile b/unit-tests/lexer/buildfile
index d3bbbfe..3d4e3ed 100644
--- a/unit-tests/lexer/buildfile
+++ b/unit-tests/lexer/buildfile
@@ -7,9 +7,9 @@
 import libs = libbutl%lib{butl}
 src = token lexer diagnostics utility variable name b-options types-parsers \
 context scope parser target operation rule prerequisite file module function \
-functions-builtin functions-path functions-process-path functions-string \
-functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation module} spec
+functions-builtin functions-filesystem functions-path functions-process-path \
+functions-string functions-target-triplet algorithm search dump filesystem \
+scheduler config/{utility init operation module} spec
 
 exe{driver}: cxx{driver} ../../build2/cxx{$src} ../../build2/liba{b} $libs \
 test{*}
diff --git a/unit-tests/scheduler/buildfile b/unit-tests/scheduler/buildfile
index 0a12414..22361b5 100644
--- a/unit-tests/scheduler/buildfile
+++ b/unit-tests/scheduler/buildfile
@@ -7,9 +7,9 @@
 import libs = libbutl%lib{butl}
 src = token lexer diagnostics utility variable name b-options types-parsers \
 context scope parser target operation rule prerequisite file module function \
-functions-builtin functions-path functions-process-path functions-string \
-functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation module} spec
+functions-builtin functions-filesystem functions-path functions-process-path \
+functions-string functions-target-triplet algorithm search dump filesystem \
+scheduler config/{utility init operation module} spec
 
 exe{driver}: cxx{driver} ../../build2/cxx{$src} ../../build2/liba{b} $libs
 
diff --git a/unit-tests/test/script/lexer/buildfile b/unit-tests/test/script/lexer/buildfile
index c5f3d2a..2070089 100644
--- a/unit-tests/test/script/lexer/buildfile
+++ b/unit-tests/test/script/lexer/buildfile
@@ -7,9 +7,9 @@
 import libs = libbutl%lib{butl}
 src = token lexer diagnostics utility variable name b-options types-parsers \
 context scope parser target operation rule prerequisite file module function \
-functions-builtin functions-path functions-process-path functions-string \
-functions-target-triplet algorithm search dump filesystem scheduler \
-config/{utility init operation module} test/script/{token lexer} spec
+functions-builtin functions-filesystem functions-path functions-process-path \
+functions-string functions-target-triplet algorithm search dump filesystem \
+scheduler config/{utility init operation module} test/script/{token lexer} spec
 
 exe{driver}: cxx{driver} ../../../../build2/cxx{$src} ../../../../build2/liba{b} $libs \
 test{command-line first-token second-token command-expansion variable-line \
diff --git a/unit-tests/test/script/parser/buildfile b/unit-tests/test/script/parser/buildfile
index 5ca2d0b..634120c 100644
--- a/unit-tests/test/script/parser/buildfile
+++ b/unit-tests/test/script/parser/buildfile
@@ -6,11 +6,11 @@
 #
 import libs = libbutl%lib{butl}
 src = token lexer parser diagnostics utility variable name context target \
-scope prerequisite file module operation rule b-options algorithm search \
-filesystem function functions-builtin functions-path functions-process-path \
-functions-string functions-target-triplet config/{utility init operation module} \
-dump types-parsers test/{target script/{token lexer parser regex script}} \
-scheduler spec
+scope prerequisite file module operation rule b-options algorithm search  \
+filesystem function functions-builtin functions-filesystem functions-path \
+functions-process-path functions-string functions-target-triplet          \
+config/{utility init operation module} dump types-parsers                 \
+test/{target script/{token lexer parser regex script}} scheduler spec
 
 exe{driver}: cxx{driver} ../../../../build2/cxx{$src} ../../../../build2/liba{b} $libs \
 test{cleanup command-if command-re-parse description directive exit \