From 57b10c06925d0bdf6ffb38488ee908f085109e95 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Thu, 4 Jul 2019 19:12:15 +0300 Subject: Move config, dist, test, and install modules into library --- libbuild2/test/common.cxx | 220 ++ libbuild2/test/common.hxx | 72 + libbuild2/test/init.cxx | 231 ++ libbuild2/test/init.hxx | 36 + libbuild2/test/module.hxx | 37 + libbuild2/test/operation.cxx | 55 + libbuild2/test/operation.hxx | 22 + libbuild2/test/rule.cxx | 882 +++++ libbuild2/test/rule.hxx | 67 + libbuild2/test/script/builtin.cxx | 1979 +++++++++++ libbuild2/test/script/builtin.hxx | 74 + .../script/lexer+command-expansion.test.testscript | 248 ++ .../test/script/lexer+command-line.test.testscript | 208 ++ .../script/lexer+description-line.test.testscript | 33 + .../test/script/lexer+first-token.test.testscript | 97 + .../test/script/lexer+second-token.test.testscript | 68 + .../script/lexer+variable-line.test.testscript | 28 + .../test/script/lexer+variable.test.testscript | 70 + libbuild2/test/script/lexer.cxx | 551 ++++ libbuild2/test/script/lexer.hxx | 94 + libbuild2/test/script/lexer.test.cxx | 85 + .../test/script/parser+cleanup.test.testscript | 58 + .../test/script/parser+command-if.test.testscript | 548 ++++ .../script/parser+command-re-parse.test.testscript | 12 + .../test/script/parser+description.test.testscript | 486 +++ .../test/script/parser+directive.test.testscript | 74 + libbuild2/test/script/parser+exit.test.testscript | 27 + .../test/script/parser+expansion.test.testscript | 36 + .../script/parser+here-document.test.testscript | 213 ++ .../test/script/parser+here-string.test.testscript | 19 + .../test/script/parser+include.test.testscript | 104 + .../test/script/parser+pipe-expr.test.testscript | 133 + .../test/script/parser+pre-parse.test.testscript | 23 + .../test/script/parser+redirect.test.testscript | 356 ++ libbuild2/test/script/parser+regex.test.testscript | 223 ++ .../test/script/parser+scope-if.test.testscript | 554 ++++ libbuild2/test/script/parser+scope.test.testscript | 280 ++ .../script/parser+setup-teardown.test.testscript | 151 + libbuild2/test/script/parser.cxx | 3451 ++++++++++++++++++++ libbuild2/test/script/parser.hxx | 250 ++ libbuild2/test/script/parser.test.cxx | 245 ++ libbuild2/test/script/regex.cxx | 440 +++ libbuild2/test/script/regex.hxx | 703 ++++ libbuild2/test/script/regex.ixx | 35 + libbuild2/test/script/regex.test.cxx | 302 ++ libbuild2/test/script/runner.cxx | 1891 +++++++++++ libbuild2/test/script/runner.hxx | 101 + libbuild2/test/script/script.cxx | 741 +++++ libbuild2/test/script/script.hxx | 559 ++++ libbuild2/test/script/script.ixx | 60 + libbuild2/test/script/token.cxx | 57 + libbuild2/test/script/token.hxx | 65 + libbuild2/test/target.cxx | 63 + libbuild2/test/target.hxx | 31 + 54 files changed, 17448 insertions(+) create mode 100644 libbuild2/test/common.cxx create mode 100644 libbuild2/test/common.hxx create mode 100644 libbuild2/test/init.cxx create mode 100644 libbuild2/test/init.hxx create mode 100644 libbuild2/test/module.hxx create mode 100644 libbuild2/test/operation.cxx create mode 100644 libbuild2/test/operation.hxx create mode 100644 libbuild2/test/rule.cxx create mode 100644 libbuild2/test/rule.hxx create mode 100644 libbuild2/test/script/builtin.cxx create mode 100644 libbuild2/test/script/builtin.hxx create mode 100644 libbuild2/test/script/lexer+command-expansion.test.testscript create mode 100644 libbuild2/test/script/lexer+command-line.test.testscript create mode 100644 libbuild2/test/script/lexer+description-line.test.testscript create mode 100644 libbuild2/test/script/lexer+first-token.test.testscript create mode 100644 libbuild2/test/script/lexer+second-token.test.testscript create mode 100644 libbuild2/test/script/lexer+variable-line.test.testscript create mode 100644 libbuild2/test/script/lexer+variable.test.testscript create mode 100644 libbuild2/test/script/lexer.cxx create mode 100644 libbuild2/test/script/lexer.hxx create mode 100644 libbuild2/test/script/lexer.test.cxx create mode 100644 libbuild2/test/script/parser+cleanup.test.testscript create mode 100644 libbuild2/test/script/parser+command-if.test.testscript create mode 100644 libbuild2/test/script/parser+command-re-parse.test.testscript create mode 100644 libbuild2/test/script/parser+description.test.testscript create mode 100644 libbuild2/test/script/parser+directive.test.testscript create mode 100644 libbuild2/test/script/parser+exit.test.testscript create mode 100644 libbuild2/test/script/parser+expansion.test.testscript create mode 100644 libbuild2/test/script/parser+here-document.test.testscript create mode 100644 libbuild2/test/script/parser+here-string.test.testscript create mode 100644 libbuild2/test/script/parser+include.test.testscript create mode 100644 libbuild2/test/script/parser+pipe-expr.test.testscript create mode 100644 libbuild2/test/script/parser+pre-parse.test.testscript create mode 100644 libbuild2/test/script/parser+redirect.test.testscript create mode 100644 libbuild2/test/script/parser+regex.test.testscript create mode 100644 libbuild2/test/script/parser+scope-if.test.testscript create mode 100644 libbuild2/test/script/parser+scope.test.testscript create mode 100644 libbuild2/test/script/parser+setup-teardown.test.testscript create mode 100644 libbuild2/test/script/parser.cxx create mode 100644 libbuild2/test/script/parser.hxx create mode 100644 libbuild2/test/script/parser.test.cxx create mode 100644 libbuild2/test/script/regex.cxx create mode 100644 libbuild2/test/script/regex.hxx create mode 100644 libbuild2/test/script/regex.ixx create mode 100644 libbuild2/test/script/regex.test.cxx create mode 100644 libbuild2/test/script/runner.cxx create mode 100644 libbuild2/test/script/runner.hxx create mode 100644 libbuild2/test/script/script.cxx create mode 100644 libbuild2/test/script/script.hxx create mode 100644 libbuild2/test/script/script.ixx create mode 100644 libbuild2/test/script/token.cxx create mode 100644 libbuild2/test/script/token.hxx create mode 100644 libbuild2/test/target.cxx create mode 100644 libbuild2/test/target.hxx (limited to 'libbuild2/test') diff --git a/libbuild2/test/common.cxx b/libbuild2/test/common.cxx new file mode 100644 index 0000000..11c5d90 --- /dev/null +++ b/libbuild2/test/common.cxx @@ -0,0 +1,220 @@ +// file : libbuild2/test/common.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include + +using namespace std; + +namespace build2 +{ + namespace test + { + // Determine if we have the target (first), id path (second), or both (in + // which case we also advance the iterator). + // + static pair + sense (names::const_iterator& i) + { + const name* tn (nullptr); + const name* pn (nullptr); + + if (i->pair) + { + tn = &*i++; + pn = &*i; + } + else + { + // If it has a type (exe{hello}) or a directory (basics/), then + // we assume it is a target. + // + (i->typed () || !i->dir.empty () ? tn : pn) = &*i; + } + + // Validate the target. + // + if (tn != nullptr) + { + if (tn->qualified ()) + fail << "project-qualified target '" << *tn << " in config.test"; + } + + // Validate the id path. + // + if (pn != nullptr) + { + if (!pn->simple () || pn->empty ()) + fail << "invalid id path '" << *pn << " in config.test"; + } + + return make_pair (tn, pn); + } + + bool common:: + pass (const target& a) const + { + if (test_ == nullptr) + return true; + + // We need to "enable" aliases that "lead up" to the targets we are + // interested in. So see if any target is in a subdirectory of this + // alias. + // + // If we don't see any targets (e.g., only id paths), then we assume all + // targets match and therefore we always pass. + // + bool r (true); + + // Directory part from root to this alias (the same in src and out). + // + const dir_path d (a.out_dir ().leaf (root_->out_path ())); + + for (auto i (test_->begin ()); i != test_->end (); ++i) + { + if (const name* n = sense (i).first) + { + // Reset result to false if no match (but we have seen a target). + // + r = n->dir.sub (d); + + // See test() below for details on this special case. + // + if (!r && !n->typed ()) + r = d.sub (n->dir); + + if (r) + break; + } + } + + return r; + } + + bool common:: + test (const target& t) const + { + if (test_ == nullptr) + return true; + + // If we don't see any targets (e.g., only id paths), then we assume + // all of them match. + // + bool r (true); + + // Directory part from root to this alias (the same in src and out). + // + const dir_path d (t.out_dir ().leaf (root_->out_path ())); + const target_type& tt (t.type ()); + + for (auto i (test_->begin ()); i != test_->end (); ++i) + { + if (const name* n = sense (i).first) + { + // Reset result to false if no match (but we have seen a target). + // + + // When specifying a directory, for example, config.tests=tests/, + // one would intuitively expect that all the tests under it will + // run. But that's not what will happen with the below test: while + // the dir{tests/} itself will match, any target underneath won't. + // So we are going to handle this type if a target specially by + // making it match any target in or under it. + // + // Note that we only do this for tests/, not dir{tests/} since it is + // not always the semantics that one wants. Sometimes one may want + // to run tests (scripts) just for the tests/ target but not for any + // of its prerequisites. So dir{tests/} is a way to disable this + // special logic. + // + // Note: the same code as in test() below. + // + if (!n->typed ()) + r = d.sub (n->dir); + else + // First quickly and cheaply weed out names that cannot possibly + // match. Only then search for a target (as if it was a + // prerequisite), which can be expensive. + // + // We cannot specify an src target in config.test since we used + // the pair separator for ids. As a result, we search for both + // out and src targets. + // + r = + t.name == n->value && // Name matches. + tt.name == n->type && // Target type matches. + d == n->dir && // Directory matches. + (search_existing (*n, *root_) == &t || + search_existing (*n, *root_, d) == &t); + + if (r) + break; + } + } + + return r; + } + + bool common:: + test (const target& t, const path& id) const + { + if (test_ == nullptr) + return true; + + // If we don't see any id paths (e.g., only targets), then we assume + // all of them match. + // + bool r (true); + + // Directory part from root to this alias (the same in src and out). + // + const dir_path d (t.out_dir ().leaf (root_->out_path ())); + const target_type& tt (t.type ()); + + for (auto i (test_->begin ()); i != test_->end (); ++i) + { + auto p (sense (i)); + + if (const name* n = p.second) + { + // If there is a target, check that it matches ours. + // + if (const name* n = p.first) + { + // Note: the same code as in test() above. + // + bool r; + + if (!n->typed ()) + r = d.sub (n->dir); + else + r = + t.name == n->value && + tt.name == n->type && + d == n->dir && + (search_existing (*n, *root_) == &t || + search_existing (*n, *root_, d) == &t); + + if (!r) + continue; // Not our target. + } + + // If the id (group) "leads up" to what we want to run or we + // (group) lead up to the id, then match. + // + const path p (n->value); + + // Reset result to false if no match (but we have seen an id path). + // + if ((r = p.sub (id) || id.sub (p))) + break; + } + } + + return r; + } + } +} diff --git a/libbuild2/test/common.hxx b/libbuild2/test/common.hxx new file mode 100644 index 0000000..5bb78ee --- /dev/null +++ b/libbuild2/test/common.hxx @@ -0,0 +1,72 @@ +// file : libbuild2/test/common.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_TEST_COMMON_HXX +#define LIBBUILD2_TEST_COMMON_HXX + +#include +#include + +#include + +namespace build2 +{ + namespace test + { + enum class output_before {fail, warn, clean}; + enum class output_after {clean, keep}; + + struct common_data + { + const variable& config_test; + const variable& config_test_output; + + const variable& var_test; + const variable& test_options; + const variable& test_arguments; + + const variable& test_stdin; + const variable& test_stdout; + const variable& test_roundtrip; + const variable& test_input; + + const variable& test_target; + }; + + struct common: common_data + { + // The config.test.output values. + // + output_before before = output_before::warn; + output_after after = output_after::clean; + + // The config.test query interface. + // + const names* test_ = nullptr; // The config.test value if any. + scope* root_ = nullptr; // The root scope for target resolution. + + // Return true if the specified alias target should pass-through to its + // prerequisites. + // + bool + pass (const target& alias_target) const; + + // Return true if the specified target should be tested. + // + bool + test (const target& test_target) const; + + // Return true if the specified target should be tested with the + // specified testscript test (or group). + // + bool + test (const target& test_target, const path& id_path) const; + + explicit + common (common_data&& d): common_data (move (d)) {} + }; + } +} + +#endif // LIBBUILD2_TEST_COMMON_HXX diff --git a/libbuild2/test/init.cxx b/libbuild2/test/init.cxx new file mode 100644 index 0000000..3d13acc --- /dev/null +++ b/libbuild2/test/init.cxx @@ -0,0 +1,231 @@ +// file : libbuild2/test/init.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include // script::regex::init() + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace test + { + bool + boot (scope& rs, const location&, unique_ptr& mod) + { + tracer trace ("test::boot"); + + l5 ([&]{trace << "for " << rs;}); + + // Register our operations. + // + rs.insert_operation (test_id, op_test); + rs.insert_operation (update_for_test_id, op_update_for_test); + + // Enter module variables. Do it during boot in case they get assigned + // in bootstrap.build. + // + auto& vp (var_pool.rw (rs)); + + common_data d { + + // Tests to execute. + // + // Specified as @ pairs with both sides being + // optional. The variable is untyped (we want a list of name-pairs), + // overridable, and inheritable. The target is relative (in essence a + // prerequisite) which is resolved from the (root) scope where the + // config.test value is defined. + // + vp.insert ("config.test", true), + + // Test working directory before/after cleanup (see Testscript spec + // for semantics). + // + vp.insert ("config.test.output", true), + + // The test variable is a name which can be a path (with the + // true/false special values) or a target name. + // + // Note: none are overridable. + // + vp.insert ("test", variable_visibility::target), + vp.insert ("test.options", variable_visibility::project), + vp.insert ("test.arguments", variable_visibility::project), + + // Prerequisite-specific. + // + // test.stdin and test.stdout can be used to mark a prerequisite as a + // file to redirect stdin from and to compare stdout to, respectively. + // test.roundtrip is a shortcut to mark a prerequisite as both stdin + // and stdout. + // + // Prerequisites marked with test.input are treated as additional test + // inputs: they are made sure to be up to date and their paths are + // passed as additional command line arguments (after test.options and + // test.arguments). Their primary use is to pass inputs that may have + // varying file names/paths, for example: + // + // exe{parent}: exe{child}: test.input = true + // + // Note that currently this mechanism is only available to simple + // tests though we could also support it for testscript (e.g., by + // appending the input paths to test.arguments or by passing them in a + // separate test.inputs variable). + // + vp.insert ("test.stdin", variable_visibility::prereq), + vp.insert ("test.stdout", variable_visibility::prereq), + vp.insert ("test.roundtrip", variable_visibility::prereq), + vp.insert ("test.input", variable_visibility::prereq), + + // Test target platform. + // + vp.insert ("test.target", variable_visibility::project) + }; + + // These are only used in testscript. + // + vp.insert ("test.redirects", variable_visibility::project); + vp.insert ("test.cleanups", variable_visibility::project); + + // Unless already set, default test.target to build.host. Note that it + // can still be overriden by the user, e.g., in root.build. + // + { + value& v (rs.assign (d.test_target)); + + if (!v || v.empty ()) + v = cast ((*global_scope)["build.host"]); + } + + mod.reset (new module (move (d))); + return false; + } + + bool + init (scope& rs, + scope&, + const location& l, + unique_ptr& mod, + bool first, + bool, + const variable_map& config_hints) + { + tracer trace ("test::init"); + + if (!first) + { + warn (l) << "multiple test module initializations"; + return true; + } + + const dir_path& out_root (rs.out_path ()); + l5 ([&]{trace << "for " << out_root;}); + + assert (mod != nullptr); + module& m (static_cast (*mod)); + + // Configure. + // + assert (config_hints.empty ()); // We don't known any hints. + + // Adjust module priority so that the config.test.* values are saved at + // the end of config.build. + // + config::save_module (rs, "test", INT32_MAX); + + // config.test + // + if (lookup l = config::omitted (rs, m.config_test).first) + { + // Figure out which root scope it came from. + // + scope* s (&rs); + for (; + s != nullptr && !l.belongs (*s); + s = s->parent_scope ()->root_scope ()) + assert (s != nullptr); + + m.test_ = &cast (l); + m.root_ = s; + } + + // config.test.output + // + if (lookup l = config::omitted (rs, m.config_test_output).first) + { + const name_pair& p (cast (l)); + + // If second half is empty, then first is the after value. + // + const name& a (p.second.empty () ? p.first : p.second); // after + const name& b (p.second.empty () ? p.second : p.first); // before + + // Parse and validate. + // + if (!b.simple ()) + fail << "invalid config.test.output before value '" << b << "'"; + + if (!a.simple ()) + fail << "invalid config.test.output after value '" << a << "'"; + + if (a.value == "clean") m.after = output_after::clean; + else if (a.value == "keep") m.after = output_after::keep; + else fail << "invalid config.test.output after value '" << a << "'"; + + if (b.value == "fail") m.before = output_before::fail; + else if (b.value == "warn") m.before = output_before::warn; + else if (b.value == "clean") m.before = output_before::clean; + else if (b.value == "") m.before = output_before::clean; + else fail << "invalid config.test.output before value '" << b << "'"; + } + + //@@ TODO: Need ability to specify extra diff options (e.g., + // --strip-trailing-cr, now hardcoded). + // + //@@ TODO: Pring report. + + // Register target types. + // + { + auto& t (rs.target_types); + + auto& tt (t.insert ()); + t.insert_file ("testscript", tt); + } + + // Register our test running rule. + // + { + default_rule& dr (m); + + rs.rules.insert (perform_test_id, "test", dr); + rs.rules.insert (perform_test_id, "test", dr); + } + + return true; + } + + module_functions + build2_test_load () + { + script::regex::init (); + + return module_functions {&boot, &init}; + } + } +} diff --git a/libbuild2/test/init.hxx b/libbuild2/test/init.hxx new file mode 100644 index 0000000..a76b720 --- /dev/null +++ b/libbuild2/test/init.hxx @@ -0,0 +1,36 @@ +// file : libbuild2/test/init.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_TEST_INIT_HXX +#define LIBBUILD2_TEST_INIT_HXX + +#include +#include + +#include + +#include + +namespace build2 +{ + namespace test + { + bool + boot (scope&, const location&, unique_ptr&); + + bool + init (scope&, + scope&, + const location&, + unique_ptr&, + bool, + bool, + const variable_map&); + + extern "C" LIBBUILD2_SYMEXPORT module_functions + build2_test_load (); + } +} + +#endif // LIBBUILD2_TEST_INIT_HXX diff --git a/libbuild2/test/module.hxx b/libbuild2/test/module.hxx new file mode 100644 index 0000000..584cb84 --- /dev/null +++ b/libbuild2/test/module.hxx @@ -0,0 +1,37 @@ +// file : libbuild2/test/module.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_TEST_MODULE_HXX +#define LIBBUILD2_TEST_MODULE_HXX + +#include +#include + +#include + +#include +#include + +namespace build2 +{ + namespace test + { + struct module: module_base, virtual common, default_rule, group_rule + { + const test::group_rule& + group_rule () const + { + return *this; + } + + explicit + module (common_data&& d) + : common (move (d)), + test::default_rule (move (d)), + test::group_rule (move (d)) {} + }; + } +} + +#endif // LIBBUILD2_TEST_MODULE_HXX diff --git a/libbuild2/test/operation.cxx b/libbuild2/test/operation.cxx new file mode 100644 index 0000000..3ff7702 --- /dev/null +++ b/libbuild2/test/operation.cxx @@ -0,0 +1,55 @@ +// file : libbuild2/test/operation.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace test + { + static operation_id + test_pre (const values& params, meta_operation_id mo, const location& l) + { + if (!params.empty ()) + fail (l) << "unexpected parameters for operation test"; + + // Run update as a pre-operation, unless we are disfiguring. + // + return mo != disfigure_id ? update_id : 0; + } + + const operation_info op_test { + test_id, + 0, + "test", + "test", + "testing", + "tested", + "has nothing to test", // We cannot "be tested". + execution_mode::first, + 1, + &test_pre, + nullptr + }; + + // Also the explicit update-for-test operation alias. + // + const operation_info op_update_for_test { + update_id, // Note: not update_for_test_id. + test_id, + op_update.name, + op_update.name_do, + op_update.name_doing, + op_update.name_did, + op_update.name_done, + op_update.mode, + op_update.concurrency, + op_update.pre, + op_update.post + }; + } +} diff --git a/libbuild2/test/operation.hxx b/libbuild2/test/operation.hxx new file mode 100644 index 0000000..8a9aed7 --- /dev/null +++ b/libbuild2/test/operation.hxx @@ -0,0 +1,22 @@ +// file : libbuild2/test/operation.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_TEST_OPERATION_HXX +#define LIBBUILD2_TEST_OPERATION_HXX + +#include +#include + +#include + +namespace build2 +{ + namespace test + { + extern const operation_info op_test; + extern const operation_info op_update_for_test; + } +} + +#endif // LIBBUILD2_TEST_OPERATION_HXX diff --git a/libbuild2/test/rule.cxx b/libbuild2/test/rule.cxx new file mode 100644 index 0000000..a6796b4 --- /dev/null +++ b/libbuild2/test/rule.cxx @@ -0,0 +1,882 @@ +// file : libbuild2/test/rule.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace test + { + bool rule:: + match (action, target&, const string&) const + { + // We always match, even if this target is not testable (so that we can + // ignore it; see apply()). + // + return true; + } + + recipe rule:: + apply (action a, target& t) const + { + // Note that we are called both as the outer part during the update-for- + // test pre-operation and as the inner part during the test operation + // itself. + // + // In both cases we first determine if the target is testable and return + // noop if it's not. Otherwise, in the first case (update for test) we + // delegate to the normal update and in the second (test) -- perform the + // test. + // + // And to add a bit more complexity, we want to handle aliases slightly + // differently: we may not want to ignore their prerequisites if the + // alias is not testable since their prerequisites could be. + // + // Here is the state matrix: + // + // test'able | pass'able | neither + // | | + // update-for-test delegate (& pass) | pass | noop + // ---------------------------------------+-------------+--------- + // test test (& pass) | pass | noop + // + auto& pts (t.prerequisite_targets[a]); + + // Resolve group members. + // + if (!see_through || t.type ().see_through) + { + // Remember that we are called twice: first during update for test + // (pre-operation) and then during test. During the former, we rely on + // the normall update rule to resolve the group members. During the + // latter, there will be no rule to do this but the group will already + // have been resolved by the pre-operation. + // + // If the rule could not resolve the group, then we ignore it. + // + group_view gv (a.outer () + ? resolve_members (a, t) + : t.group_members (a)); + + if (gv.members != nullptr) + { + for (size_t i (0); i != gv.count; ++i) + { + if (const target* m = gv.members[i]) + pts.push_back (m); + } + + match_members (a, t, pts); + } + } + + // If we are passing-through, then match our prerequisites. + // + if (t.is_a () && pass (t)) + { + // For the test operation we have to implement our own search and + // match because we need to ignore prerequisites that are outside of + // our project. They can be from projects that don't use the test + // module (and thus won't have a suitable rule). Or they can be from + // no project at all (e.g., installed). Also, generally, not testing + // stuff that's not ours seems right. + // + match_prerequisites (a, t, t.root_scope ()); + } + + size_t pass_n (pts.size ()); // Number of pass-through prerequisites. + + // See if it's testable and if so, what kind. + // + bool test (false); + bool script (false); + + if (this->test (t)) + { + // We have two very different cases: testscript and simple test (plus + // it may not be a testable target at all). So as the first step + // determine which case this is. + // + // If we have any prerequisites of the testscript{} type, then this is + // the testscript case. + // + // If we can, go inside see-through groups. Normally groups won't be + // resolvable for this action but then normally they won't contain any + // testscripts either. In other words, if there is a group that + // contains testscripts as members then it will need to arrange for + // the members to be resolvable (e.g., by registering an appropriate + // rule for the test operation). + // + for (prerequisite_member p: + group_prerequisite_members (a, t, members_mode::maybe)) + { + if (include (a, t, p) != include_type::normal) // Excluded/ad hoc. + continue; + + if (p.is_a ()) + { + if (!script) + { + script = true; + + // We treat this target as testable unless the test variable is + // explicitly set to false. + // + const name* n (cast_null (t[var_test])); + test = (n == nullptr || !n->simple () || n->value != "false"); + + if (!test) + break; + } + + // Collect testscripts after the pass-through prerequisites. + // + const target& pt (p.search (t)); + + // Note that for the test operation itself we don't match nor + // execute them relying on update to assign their paths. + // + // Causing update for test inputs/scripts is tricky: we cannot + // match for update-for-install because this same rule will match + // and since the target is not testable, it will return the noop + // recipe. + // + // So what we are going to do is directly match (and also execute; + // see below) a recipe for the inner update (who thought we could + // do that... but it seems we can). While at first it might feel + // iffy, it does make sense: the outer rule we would have matched + // would have simply delegated to the inner so we might as well + // take a shortcut. The only potential drawback of this approach + // is that we won't be able to provide any for-test customizations + // when updating test inputs/scripts. But such a need seems rather + // far fetched. + // + if (a.operation () == update_id) + match_inner (a, pt); + + pts.push_back (&pt); + } + } + + // If this is not a script, then determine if it is a simple test. + // Ignore testscript files themselves at the outset. + // + if (!script && !t.is_a ()) + { + // For the simple case whether this is a test is controlled by the + // test variable. Also, it feels redundant to specify, say, "test = + // true" and "test.stdout = test.out" -- the latter already says this + // is a test. + // + const name* n (cast_null (t[var_test])); + + // If the test variable is explicitly set to false then we treat + // it as not testable regardless of what other test.* variables + // or prerequisites we might have. + // + // Note that the test variable can be set to an "override" target + // (which means 'true' for our purposes). + // + if (n != nullptr && n->simple () && n->value == "false") + test = false; + else + { + // Look for test input/stdin/stdout prerequisites. The same group + // reasoning as in the testscript case above. + // + for (prerequisite_member p: + group_prerequisite_members (a, t, members_mode::maybe)) + { + const auto& vars (p.prerequisite.vars); + + if (vars.empty ()) // Common case. + continue; + + if (include (a, t, p) != include_type::normal) // Excluded/ad hoc. + continue; + + bool rt ( cast_false (vars[test_roundtrip])); + bool si (rt || cast_false (vars[test_stdin])); + bool so (rt || cast_false (vars[test_stdout])); + bool in ( cast_false (vars[test_input])); + + if (si || so || in) + { + // Verify it is file-based. + // + if (!p.is_a ()) + { + fail << "test." << (si ? "stdin" : so ? "stdout" : "input") + << " prerequisite " << p << " of target " << t + << " is not a file"; + } + + if (!test) + { + test = true; + + // First matching prerequisite. Establish the structure in + // pts: the first element (after pass_n) is stdin (can be + // NULL), the second is stdout (can be NULL), and everything + // after that (if any) is inputs. + // + pts.push_back (nullptr); // stdin + pts.push_back (nullptr); // stdout + } + + // Collect them after the pass-through prerequisites. + // + // Note that for the test operation itself we don't match nor + // execute them relying on update to assign their paths. + // + auto match = [a, &p, &t] () -> const target* + { + const target& pt (p.search (t)); + + // The same match_inner() rationale as for the testcript + // prerequisites above. + // + if (a.operation () == update_id) + match_inner (a, pt); + + return &pt; + }; + + if (si) + { + if (pts[pass_n] != nullptr) + fail << "multiple test.stdin prerequisites for target " + << t; + + pts[pass_n] = match (); + } + + if (so) + { + if (pts[pass_n + 1] != nullptr) + fail << "multiple test.stdout prerequisites for target " + << t; + + pts[pass_n + 1] = match (); + } + + if (in) + pts.push_back (match ()); + } + } + + if (!test) + test = (n != nullptr); // We have the test variable. + + if (!test) + test = t[test_options] || t[test_arguments]; + } + } + } + + // Neither testing nor passing-through. + // + if (!test && pass_n == 0) + return noop_recipe; + + // If we are only passing-through, then use the default recipe (which + // will execute all the matched prerequisites). + // + if (!test) + return default_recipe; + + // Being here means we are definitely testing and maybe passing-through. + // + if (a.operation () == update_id) + { + // For the update pre-operation match the inner rule (actual update). + // + match_inner (a, t); + + return [pass_n] (action a, const target& t) + { + return perform_update (a, t, pass_n); + }; + } + else + { + if (script) + { + return [pass_n, this] (action a, const target& t) + { + return perform_script (a, t, pass_n); + }; + } + else + { + return [pass_n, this] (action a, const target& t) + { + return perform_test (a, t, pass_n); + }; + } + } + } + + target_state rule:: + perform_update (action a, const target& t, size_t pass_n) + { + // First execute the inner recipe then execute prerequisites. + // + target_state ts (execute_inner (a, t)); + + if (pass_n != 0) + ts |= straight_execute_prerequisites (a, t, pass_n); + + ts |= straight_execute_prerequisites_inner (a, t, 0, pass_n); + + return ts; + } + + static script::scope_state + perform_script_impl (const target& t, + const testscript& ts, + const dir_path& wd, + const common& c) + { + using namespace script; + + scope_state r; + + try + { + build2::test::script::script s (t, ts, wd); + + { + parser p; + p.pre_parse (s); + + default_runner r (c); + p.execute (s, r); + } + + r = s.state; + } + catch (const failed&) + { + r = scope_state::failed; + } + + return r; + } + + target_state rule:: + perform_script (action a, const target& t, size_t pass_n) const + { + // First pass through. + // + if (pass_n != 0) + straight_execute_prerequisites (a, t, pass_n); + + // Figure out whether the testscript file is called 'testscript', in + // which case it should be the only one. + // + auto& pts (t.prerequisite_targets[a]); + size_t pts_n (pts.size ()); + + bool one; + { + optional o; + for (size_t i (pass_n); i != pts_n; ++i) + { + const testscript& ts (*pts[i]->is_a ()); + + bool r (ts.name == "testscript"); + + if ((r && o) || (!r && o && *o)) + fail << "both 'testscript' and other names specified for " << t; + + o = r; + } + + assert (o); // We should have a testscript or we wouldn't be here. + one = *o; + } + + // Calculate root working directory. It is in the out_base of the target + // and is called just test for dir{} targets and test- for + // other targets. + // + dir_path wd (t.out_dir ()); + + if (t.is_a ()) + wd /= "test"; + else + wd /= "test-" + t.name; + + // Are we backlinking the test working directory to src? (See + // backlink_*() in algorithm.cxx for details.) + // + const scope& bs (t.base_scope ()); + const scope& rs (*bs.root_scope ()); + const path& buildignore_file (rs.root_extra->buildignore_file); + + dir_path bl; + if (cast_false (rs.vars[var_forwarded])) + { + bl = bs.src_path () / wd.leaf (bs.out_path ()); + clean_backlink (bl, verb_never); + } + + // If this is a (potentially) multi-testscript test, then create (and + // later cleanup) the root directory. If this is just 'testscript', then + // the root directory is used directly as test's working directory and + // it's the runner's responsibility to create and clean it up. + // + // Note that we create the root directory containing the .buildignore + // file to make sure that it is ignored by name patterns (see the + // buildignore description for details). + // + // What should we do if the directory already exists? We used to fail + // which meant the user had to go and clean things up manually every + // time a test failed. This turned out to be really annoying. So now we + // issue a warning and clean it up automatically. The drawbacks of this + // approach are the potential loss of data from the previous failed test + // run and the possibility of deleting user-created files. + // + if (exists (static_cast (wd), false)) + fail << "working directory " << wd << " is a file/symlink"; + + if (exists (wd)) + { + if (before != output_before::clean) + { + bool fail (before == output_before::fail); + + (fail ? error : warn) << "working directory " << wd << " exists " + << (empty_buildignore (wd, buildignore_file) + ? "" + : "and is not empty ") + << "at the beginning of the test"; + + if (fail) + throw failed (); + } + + // Remove the directory itself not to confuse the runner which tries + // to detect when tests stomp on each others feet. + // + build2::rmdir_r (wd, true, 2); + } + + // Delay actually creating the directory in case all the tests are + // ignored (via config.test). + // + bool mk (!one); + + // Start asynchronous execution of the testscripts. + // + wait_guard wg; + + if (!dry_run) + wg = wait_guard (target::count_busy (), t[a].task_count); + + // Result vector. + // + using script::scope_state; + + vector res; + res.reserve (pts_n - pass_n); // Make sure there are no reallocations. + + for (size_t i (pass_n); i != pts_n; ++i) + { + const testscript& ts (*pts[i]->is_a ()); + + // If this is just the testscript, then its id path is empty (and it + // can only be ignored by ignoring the test target, which makes sense + // since it's the only testscript file). + // + if (one || test (t, path (ts.name))) + { + // Because the creation of the output directory is shared between us + // and the script implementation (plus the fact that we actually + // don't clean the existing one), we are going to ignore it for + // dry-run. + // + if (!dry_run) + { + if (mk) + { + mkdir_buildignore (wd, buildignore_file, 2); + mk = false; + } + } + + if (verb) + { + diag_record dr (text); + dr << "test " << ts; + + if (!t.is_a ()) + dr << ' ' << t; + } + + res.push_back (dry_run ? scope_state::passed : scope_state::unknown); + + if (!dry_run) + { + scope_state& r (res.back ()); + + if (!sched.async (target::count_busy (), + t[a].task_count, + [this] (const diag_frame* ds, + scope_state& r, + const target& t, + const testscript& ts, + const dir_path& wd) + { + diag_frame::stack_guard dsg (ds); + r = perform_script_impl (t, ts, wd, *this); + }, + diag_frame::stack (), + ref (r), + cref (t), + cref (ts), + cref (wd))) + { + // Executed synchronously. If failed and we were not asked to + // keep going, bail out. + // + if (r == scope_state::failed && !keep_going) + break; + } + } + } + } + + if (!dry_run) + wg.wait (); + + // Re-examine. + // + bool bad (false); + for (scope_state r: res) + { + switch (r) + { + case scope_state::passed: break; + case scope_state::failed: bad = true; break; + case scope_state::unknown: assert (false); + } + + if (bad) + break; + } + + // Cleanup. + // + if (!dry_run) + { + if (!bad && !one && !mk && after == output_after::clean) + { + if (!empty_buildignore (wd, buildignore_file)) + fail << "working directory " << wd << " is not empty at the " + << "end of the test"; + + rmdir_buildignore (wd, buildignore_file, 2); + } + } + + // Backlink if the working directory exists. + // + // If we dry-run then presumably all tests passed and we shouldn't + // have anything left unless we are keeping the output. + // + if (!bl.empty () && (dry_run ? after == output_after::keep : exists (wd))) + update_backlink (wd, bl, true /* changed */); + + if (bad) + throw failed (); + + return target_state::changed; + } + + // The format of args shall be: + // + // name1 arg arg ... nullptr + // name2 arg arg ... nullptr + // ... + // nameN arg arg ... nullptr nullptr + // + static bool + run_test (const target& t, + diag_record& dr, + char const** args, + process* prev = nullptr) + { + // Find the next process, if any. + // + char const** next (args); + for (next++; *next != nullptr; next++) ; + next++; + + // Redirect stdout to a pipe unless we are last. + // + int out (*next != nullptr ? -1 : 1); + bool pr; + process_exit pe; + + try + { + process p (prev == nullptr + ? process (args, 0, out) // First process. + : process (args, *prev, out)); // Next process. + + pr = *next == nullptr || run_test (t, dr, next, &p); + p.wait (); + + assert (p.exit); + pe = *p.exit; + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + bool wr (pe.normal () && pe.code () == 0); + + if (!wr) + { + if (pr) // First failure? + dr << fail << "test " << t << " failed"; // Multi test: test 1. + + dr << error; + print_process (dr, args); + dr << " " << pe; + } + + return pr && wr; + } + + target_state rule:: + perform_test (action a, const target& tt, size_t pass_n) const + { + // First pass through. + // + if (pass_n != 0) + straight_execute_prerequisites (a, tt, pass_n); + + // See if we have the test executable override. + // + path p; + { + // Note that the test variable's visibility is target. + // + lookup l (tt[var_test]); + + // Note that we have similar code for scripted tests. + // + const target* t (nullptr); + + if (l.defined ()) + { + const name* n (cast_null (l)); + + if (n == nullptr) + fail << "invalid test executable override: null value"; + else if (n->empty ()) + fail << "invalid test executable override: empty value"; + else if (n->simple ()) + { + // Ignore the special 'true' value. + // + if (n->value != "true") + p = path (n->value); + else + t = &tt; + } + else if (n->directory ()) + fail << "invalid test executable override: '" << *n << "'"; + else + { + // Must be a target name. + // + // @@ OUT: what if this is a @-qualified pair of names? + // + t = search_existing (*n, tt.base_scope ()); + + if (t == nullptr) + fail << "invalid test executable override: unknown target: '" + << *n << "'"; + } + } + else + // By default we set it to the test target's path. + // + t = &tt; + + if (t != nullptr) + { + if (auto* pt = t->is_a ()) + { + // Do some sanity checks: the target better be up-to-date with + // an assigned path. + // + p = pt->path (); + + if (p.empty ()) + fail << "target " << *pt << " specified in the test variable " + << "is out of date" << + info << "consider specifying it as a prerequisite of " << tt; + } + else + fail << "target " << *t << (t != &tt + ? " specified in the test variable " + : " requested to be tested ") + << "is not path-based"; + } + } + + // See apply() for the structure of prerequisite_targets in the presence + // of test.{input,stdin,stdout}. + // + auto& pts (tt.prerequisite_targets[a]); + size_t pts_n (pts.size ()); + + cstrings args; + + // Do we have stdin? + // + // We simulate stdin redirect (as ()); + const path& ip (it.path ()); + assert (!ip.empty ()); // Should have been assigned by update. + + cat = process (process_exit (0)); // Successfully exited. + + if (!dry_run) + { + try + { + cat.in_ofd = fdopen (ip, fdopen_mode::in); + } + catch (const io_error& e) + { + fail << "unable to open " << ip << ": " << e; + } + } + + // Purely for diagnostics. + // + args.push_back ("cat"); + args.push_back (ip.string ().c_str ()); + args.push_back (nullptr); + } + + // If dry-run, the target may not exist. + // + process_path pp (!dry_run + ? run_search (p, true /* init */) + : try_run_search (p, true)); + args.push_back (pp.empty () ? p.string ().c_str () : pp.recall_string ()); + + // Do we have options and/or arguments? + // + if (auto l = tt[test_options]) + append_options (args, cast (l)); + + if (auto l = tt[test_arguments]) + append_options (args, cast (l)); + + // Do we have inputs? + // + for (size_t i (pass_n + 2); i < pts_n; ++i) + { + const file& it (pts[i]->as ()); + const path& ip (it.path ()); + assert (!ip.empty ()); // Should have been assigned by update. + args.push_back (ip.string ().c_str ()); + } + + args.push_back (nullptr); + + // Do we have stdout? + // + path dp ("diff"); + process_path dpp; + if (pass_n != pts_n && pts[pass_n + 1] != nullptr) + { + const file& ot (pts[pass_n + 1]->as ()); + const path& op (ot.path ()); + assert (!op.empty ()); // Should have been assigned by update. + + dpp = run_search (dp, true); + + args.push_back (dpp.recall_string ()); + args.push_back ("-u"); + + // Note that MinGW-built diff utility (as of 3.3) fails trying to + // detect if stdin contains text or binary data. We will help it a bit + // to workaround the issue. + // +#ifdef _WIN32 + args.push_back ("--text"); +#endif + + // Ignore Windows newline fluff if that's what we are running on. + // + if (cast (tt[test_target]).class_ == "windows") + args.push_back ("--strip-trailing-cr"); + + args.push_back (op.string ().c_str ()); + args.push_back ("-"); + args.push_back (nullptr); + } + + args.push_back (nullptr); // Second. + + if (verb >= 2) + print_process (args); + else if (verb) + text << "test " << tt; + + if (!dry_run) + { + diag_record dr; + if (!run_test (tt, + dr, + args.data () + (sin ? 3 : 0), // Skip cat. + sin ? &cat : nullptr)) + { + dr << info << "test command line: "; + print_process (dr, args); + dr << endf; // return + } + } + + return target_state::changed; + } + } +} diff --git a/libbuild2/test/rule.hxx b/libbuild2/test/rule.hxx new file mode 100644 index 0000000..7837074 --- /dev/null +++ b/libbuild2/test/rule.hxx @@ -0,0 +1,67 @@ +// file : libbuild2/test/rule.hxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_TEST_RULE_HXX +#define LIBBUILD2_TEST_RULE_HXX + +#include +#include + +#include +#include + +#include + +namespace build2 +{ + namespace test + { + class rule: public build2::rule, protected virtual common + { + public: + virtual bool + match (action, target&, const string&) const override; + + virtual recipe + apply (action, target&) const override; + + static target_state + perform_update (action, const target&, size_t); + + target_state + perform_test (action, const target&, size_t) const; + + target_state + perform_script (action, const target&, size_t) const; + + rule (common_data&& d, bool see_through_only) + : common (move (d)), see_through (see_through_only) {} + + bool see_through; + }; + + class default_rule: public rule + { + public: + explicit + default_rule (common_data&& d) + : common (move (d)), + rule (move (d), true /* see_through_only */) {} + }; + + // To be used for non-see-through groups that should exhibit the see- + // through behavior for install (see lib{} in the bin module for an + // example). + // + class group_rule: public rule + { + public: + explicit + group_rule (common_data&& d) + : common (move (d)), rule (move (d), false /* see_through_only */) {} + }; + } +} + +#endif // LIBBUILD2_TEST_RULE_HXX diff --git a/libbuild2/test/script/builtin.cxx b/libbuild2/test/script/builtin.cxx new file mode 100644 index 0000000..ab57d4f --- /dev/null +++ b/libbuild2/test/script/builtin.cxx @@ -0,0 +1,1979 @@ +// file : libbuild2/test/script/builtin.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include +#include +#include +#include // strtoull() + +#include +#include // use default operator<< implementation +#include // fdopen_mode, fdstream_mode +#include + +#include // sched + +#include + +// Strictly speaking a builtin which reads/writes from/to standard streams +// must be asynchronous so that the caller can communicate with it through +// pipes without being blocked on I/O operations. However, as an optimization, +// we allow builtins that only print diagnostics to STDERR to be synchronous +// assuming that their output will always fit the pipe buffer. Synchronous +// builtins must not read from STDIN and write to STDOUT. Later we may relax +// this rule to allow a "short" output for such builtins. +// +using namespace std; +using namespace butl; + +namespace build2 +{ + namespace test + { + namespace script + { + using builtin_impl = uint8_t (scope&, + const strings& args, + auto_fd in, auto_fd out, auto_fd err); + + // Operation failed, diagnostics has already been issued. + // + struct failed {}; + + // Accumulate an error message, print it atomically in dtor to the + // provided stream and throw failed afterwards if requested. Prefixes + // the message with the builtin name. + // + // Move constructible-only, not assignable (based to diag_record). + // + class error_record + { + public: + template + friend const error_record& + operator<< (const error_record& r, const T& x) + { + r.ss_ << x; + return r; + } + + error_record (ostream& o, bool fail, const char* name) + : os_ (o), fail_ (fail), empty_ (false) + { + ss_ << name << ": "; + } + + // Older versions of libstdc++ don't have the ostringstream move + // support. Luckily, GCC doesn't seem to be actually needing move due + // to copy/move elision. + // +#ifdef __GLIBCXX__ + error_record (error_record&&); +#else + error_record (error_record&& r) + : os_ (r.os_), + ss_ (move (r.ss_)), + fail_ (r.fail_), + empty_ (r.empty_) + { + r.empty_ = true; + } +#endif + + ~error_record () noexcept (false) + { + if (!empty_) + { + // The output stream can be in a bad state (for example as a + // result of unsuccessful attempt to report a previous error), so + // we check it. + // + if (os_.good ()) + { + ss_.put ('\n'); + os_ << ss_.str (); + os_.flush (); + } + + if (fail_) + throw failed (); + } + } + + private: + ostream& os_; + mutable ostringstream ss_; + + bool fail_; + bool empty_; + }; + + // Parse and normalize a path. Also, unless it is already absolute, make + // the path absolute using the specified directory. Throw invalid_path + // if the path is empty, and on parsing and normalization failures. + // + static path + parse_path (string s, const dir_path& d) + { + path p (move (s)); + + if (p.empty ()) + throw invalid_path (""); + + if (p.relative ()) + p = d / move (p); + + p.normalize (); + return p; + } + + // Builtin commands functions. + // + + // cat ... + // + // Note that POSIX doesn't specify if after I/O operation failure the + // command should proceed with the rest of the arguments. The current + // implementation exits immediatelly in such a case. + // + // @@ Shouldn't we check that we don't print a nonempty regular file to + // itself, as that would merely exhaust the output device? POSIX + // allows (but not requires) such a check and some implementations do + // this. That would require to fstat() file descriptors and complicate + // the code a bit. Was able to reproduce on a big file (should be + // bigger than the stream buffer size) with the test + // 'cat file >+file'. + // + // Note: must be executed asynchronously. + // + static uint8_t + cat (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + auto error = [&cerr] (bool fail = true) + { + return error_record (cerr, fail, "cat"); + }; + + try + { + ifdstream cin (move (in), fdstream_mode::binary); + ofdstream cout (move (out), fdstream_mode::binary); + + // Copy input stream to STDOUT. + // + auto copy = [&cout] (istream& is) + { + if (is.peek () != ifdstream::traits_type::eof ()) + cout << is.rdbuf (); + + is.clear (istream::eofbit); // Sets eofbit. + }; + + // Path of a file being printed to STDOUT. An empty path represents + // STDIN. Used in diagnostics. + // + path p; + + try + { + // Print STDIN. + // + if (args.empty ()) + copy (cin); + + // Print files. + // + for (auto i (args.begin ()); i != args.end (); ++i) + { + if (*i == "-") + { + if (!cin.eof ()) + { + p.clear (); + copy (cin); + } + + continue; + } + + p = parse_path (*i, sp.wd_path); + + ifdstream is (p, ifdstream::binary); + copy (is); + is.close (); + } + } + catch (const io_error& e) + { + error_record d (error ()); + d << "unable to print "; + + if (p.empty ()) + d << "stdin"; + else + d << "'" << p << "'"; + + d << ": " << e; + } + + cin.close (); + cout.close (); + r = 0; + } + catch (const invalid_path& e) + { + error (false) << "invalid path '" << e.path << "'"; + } + // Can be thrown while creating/closing cin, cout or writing to cerr. + // + catch (const io_error& e) + { + error (false) << e; + } + catch (const failed&) + { + // Diagnostics has already been issued. + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + + // Make a copy of a file at the specified path, preserving permissions, + // and registering a cleanup for a newly created file. The file paths + // must be absolute. Fail if an exception is thrown by the underlying + // copy operation. + // + static void + cpfile (scope& sp, + const path& from, const path& to, + bool overwrite, + bool attrs, + bool cleanup, + const function& fail) + { + try + { + bool exists (file_exists (to)); + + cpflags f ( + overwrite + ? cpflags::overwrite_permissions | cpflags::overwrite_content + : cpflags::none); + + if (attrs) + f |= cpflags::overwrite_permissions | cpflags::copy_timestamps; + + cpfile (from, to, f); + + if (!exists && cleanup) + sp.clean ({cleanup_type::always, to}, true); + } + catch (const system_error& e) + { + fail () << "unable to copy file '" << from << "' to '" << to + << "': " << e; + } + } + + // Make a copy of a directory at the specified path, registering a + // cleanup for the created directory. The directory paths must be + // absolute. Fail if the destination directory already exists or + // an exception is thrown by the underlying copy operation. + // + static void + cpdir (scope& sp, + const dir_path& from, const dir_path& to, + bool attrs, + bool cleanup, + const function& fail) + { + try + { + if (try_mkdir (to) == mkdir_status::already_exists) + throw_generic_error (EEXIST); + + if (cleanup) + sp.clean ({cleanup_type::always, to}, true); + + for (const auto& de: dir_iterator (from, + false /* ignore_dangling */)) + { + path f (from / de.path ()); + path t (to / de.path ()); + + if (de.type () == entry_type::directory) + cpdir (sp, + path_cast (move (f)), + path_cast (move (t)), + attrs, + cleanup, + fail); + else + cpfile (sp, f, t, false /* overwrite */, attrs, cleanup, fail); + } + + // Note that it is essential to copy timestamps and permissions after + // the directory content is copied. + // + if (attrs) + { + path_permissions (to, path_permissions (from)); + dir_time (to, dir_time (from)); + } + } + catch (const system_error& e) + { + fail () << "unable to copy directory '" << from << "' to '" << to + << "': " << e; + } + } + + // cp [-p] [--no-cleanup] + // cp [-p] [--no-cleanup] -R|-r + // cp [-p] [--no-cleanup] ... / + // cp [-p] [--no-cleanup] -R|-r ... / + // + // Note: can be executed synchronously. + // + static uint8_t + cp (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + auto error = [&cerr] (bool fail = true) + { + return error_record (cerr, fail, "cp"); + }; + + try + { + in.close (); + out.close (); + + auto i (args.begin ()); + auto e (args.end ()); + + // Process options. + // + bool recursive (false); + bool attrs (false); + bool cleanup (true); + for (; i != e; ++i) + { + const string& o (*i); + + if (o == "-R" || o == "-r") + recursive = true; + else if (o == "-p") + attrs = true; + else if (o == "--no-cleanup") + cleanup = false; + else + { + if (o == "--") + ++i; + + break; + } + } + + // Copy files or directories. + // + if (i == e) + error () << "missing arguments"; + + const dir_path& wd (sp.wd_path); + + auto j (args.rbegin ()); + path dst (parse_path (*j++, wd)); + e = j.base (); + + if (i == e) + error () << "missing source path"; + + auto fail = [&error] () {return error (true);}; + + // If destination is not a directory path (no trailing separator) + // then make a copy of the filesystem entry at the specified path + // (the only source path is allowed in such a case). Otherwise copy + // the source filesystem entries into the destination directory. + // + if (!dst.to_directory ()) + { + path src (parse_path (*i++, wd)); + + // If there are multiple sources but no trailing separator for the + // destination, then, most likelly, it is missing. + // + if (i != e) + error () << "multiple source paths without trailing separator " + << "for destination directory"; + + if (!recursive) + // Synopsis 1: make a file copy at the specified path. + // + cpfile (sp, + src, + dst, + true /* overwrite */, + attrs, + cleanup, + fail); + else + // Synopsis 2: make a directory copy at the specified path. + // + cpdir (sp, + path_cast (src), path_cast (dst), + attrs, + cleanup, + fail); + } + else + { + for (; i != e; ++i) + { + path src (parse_path (*i, wd)); + + if (recursive && dir_exists (src)) + // Synopsis 4: copy a filesystem entry into the specified + // directory. Note that we handle only source directories here. + // Source files are handled below. + // + cpdir (sp, + path_cast (src), + path_cast (dst / src.leaf ()), + attrs, + cleanup, + fail); + else + // Synopsis 3: copy a file into the specified directory. Also, + // here we cover synopsis 4 for the source path being a file. + // + cpfile (sp, + src, + dst / src.leaf (), + true /* overwrite */, + attrs, + cleanup, + fail); + } + } + + r = 0; + } + catch (const invalid_path& e) + { + error (false) << "invalid path '" << e.path << "'"; + } + // Can be thrown while closing in, out or writing to cerr. + // + catch (const io_error& e) + { + error (false) << e; + } + catch (const failed&) + { + // Diagnostics has already been issued. + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + + // echo ... + // + // Note: must be executed asynchronously. + // + static uint8_t + echo (scope&, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + try + { + in.close (); + ofdstream cout (move (out)); + + for (auto b (args.begin ()), i (b), e (args.end ()); i != e; ++i) + cout << (i != b ? " " : "") << *i; + + cout << '\n'; + cout.close (); + r = 0; + } + catch (const std::exception& e) + { + cerr << "echo: " << e << endl; + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + + // false + // + // Failure to close the file descriptors is silently ignored. + // + // Note: can be executed synchronously. + // + static builtin + false_ (scope&, uint8_t& r, const strings&, auto_fd, auto_fd, auto_fd) + { + return builtin (r = 1); + } + + // true + // + // Failure to close the file descriptors is silently ignored. + // + // Note: can be executed synchronously. + // + static builtin + true_ (scope&, uint8_t& r, const strings&, auto_fd, auto_fd, auto_fd) + { + return builtin (r = 0); + } + + // Create a symlink to a file or directory at the specified path. The + // paths must be absolute. Fall back to creating a hardlink, if symlink + // creation is not supported for the link path. If hardlink creation is + // not supported either, then fall back to copies. If requested, created + // filesystem entries are registered for cleanup. Fail if the target + // filesystem entry doesn't exist or an exception is thrown by the + // underlying filesystem operation (specifically for an already existing + // filesystem entry at the link path). + // + // Note that supporting optional removal of an existing filesystem entry + // at the link path (the -f option) tends to get hairy. As soon as an + // existing and the resulting filesystem entries could be of different + // types, we would end up with canceling an old cleanup and registering + // the new one. Also removing non-empty directories doesn't look very + // natural, but would be required if we want the behavior on POSIX and + // Windows to be consistent. + // + static void + mksymlink (scope& sp, + const path& target, const path& link, + bool cleanup, + const function& fail) + { + // Determine the target type, fail if the target doesn't exist. + // + bool dir (false); + + try + { + pair pe (path_entry (target)); + + if (!pe.first) + fail () << "unable to create symlink to '" << target << "': " + << "no such file or directory"; + + dir = pe.second.type == entry_type::directory; + } + catch (const system_error& e) + { + fail () << "unable to stat '" << target << "': " << e; + } + + // First we try to create a symlink. If that fails (e.g., "Windows + // happens"), then we resort to hard links. If that doesn't work out + // either (e.g., not on the same filesystem), then we fall back to + // copies. So things are going to get a bit nested. + // + try + { + mksymlink (target, link, dir); + + if (cleanup) + sp.clean ({cleanup_type::always, link}, true); + } + catch (const system_error& e) + { + // Note that we are not guaranteed (here and below) that the + // system_error exception is of the generic category. + // + int c (e.code ().value ()); + if (!(e.code ().category () == generic_category () && + (c == ENOSYS || // Not implemented. + c == EPERM))) // Not supported by the filesystem(s). + fail () << "unable to create symlink '" << link << "' to '" + << target << "': " << e; + + try + { + mkhardlink (target, link, dir); + + if (cleanup) + sp.clean ({cleanup_type::always, link}, true); + } + catch (const system_error& e) + { + c = e.code ().value (); + if (!(e.code ().category () == generic_category () && + (c == ENOSYS || // Not implemented. + c == EPERM || // Not supported by the filesystem(s). + c == EXDEV))) // On different filesystems. + fail () << "unable to create hardlink '" << link << "' to '" + << target << "': " << e; + + if (dir) + cpdir (sp, + path_cast (target), path_cast (link), + false, + cleanup, + fail); + else + cpfile (sp, + target, + link, + false /* overwrite */, + true /* attrs */, + cleanup, + fail); + } + } + } + + // ln [--no-cleanup] -s + // ln [--no-cleanup] -s ... / + // + // Note: can be executed synchronously. + // + static uint8_t + ln (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + auto error = [&cerr] (bool fail = true) + { + return error_record (cerr, fail, "ln"); + }; + + try + { + in.close (); + out.close (); + + auto i (args.begin ()); + auto e (args.end ()); + + // Process options. + // + bool cleanup (true); + bool symlink (false); + + for (; i != e; ++i) + { + const string& o (*i); + + if (o == "--no-cleanup") + cleanup = false; + else if (o == "-s") + symlink = true; + else + { + if (o == "--") + ++i; + + break; + } + } + + if (!symlink) + error () << "missing -s option"; + + // Create file or directory symlinks. + // + if (i == e) + error () << "missing arguments"; + + const dir_path& wd (sp.wd_path); + + auto j (args.rbegin ()); + path link (parse_path (*j++, wd)); + e = j.base (); + + if (i == e) + error () << "missing target path"; + + auto fail = [&error] () {return error (true);}; + + // If link is not a directory path (no trailing separator), then + // create a symlink to the target path at the specified link path + // (the only target path is allowed in such a case). Otherwise create + // links to the target paths inside the specified directory. + // + if (!link.to_directory ()) + { + path target (parse_path (*i++, wd)); + + // If there are multiple targets but no trailing separator for the + // link, then, most likelly, it is missing. + // + if (i != e) + error () << "multiple target paths with non-directory link path"; + + // Synopsis 1: create a target path symlink at the specified path. + // + mksymlink (sp, target, link, cleanup, fail); + } + else + { + for (; i != e; ++i) + { + path target (parse_path (*i, wd)); + + // Synopsis 2: create a target path symlink in the specified + // directory. + // + mksymlink (sp, target, link / target.leaf (), cleanup, fail); + } + } + + r = 0; + } + catch (const invalid_path& e) + { + error (false) << "invalid path '" << e.path << "'"; + } + // Can be thrown while closing in, out or writing to cerr. + // + catch (const io_error& e) + { + error (false) << e; + } + catch (const failed&) + { + // Diagnostics has already been issued. + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + + // Create a directory if not exist and its parent directories if + // necessary. Throw system_error on failure. Register created + // directories for cleanup. The directory path must be absolute. + // + static void + mkdir_p (scope& sp, const dir_path& p, bool cleanup) + { + if (!dir_exists (p)) + { + if (!p.root ()) + mkdir_p (sp, p.directory (), cleanup); + + try_mkdir (p); // Returns success or throws. + + if (cleanup) + sp.clean ({cleanup_type::always, p}, true); + } + } + + // mkdir [--no-cleanup] [-p] ... + // + // Note that POSIX doesn't specify if after a directory creation failure + // the command should proceed with the rest of the arguments. The current + // implementation exits immediatelly in such a case. + // + // Note: can be executed synchronously. + // + static uint8_t + mkdir (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + auto error = [&cerr] (bool fail = true) + { + return error_record (cerr, fail, "mkdir"); + }; + + try + { + in.close (); + out.close (); + + auto i (args.begin ()); + auto e (args.end ()); + + // Process options. + // + bool parent (false); + bool cleanup (true); + for (; i != e; ++i) + { + const string& o (*i); + + if (o == "-p") + parent = true; + else if (o == "--no-cleanup") + cleanup = false; + else + { + if (*i == "--") + ++i; + + break; + } + } + + // Create directories. + // + if (i == e) + error () << "missing directory"; + + for (; i != e; ++i) + { + dir_path p (path_cast (parse_path (*i, sp.wd_path))); + + try + { + if (parent) + mkdir_p (sp, p, cleanup); + else if (try_mkdir (p) == mkdir_status::success) + { + if (cleanup) + sp.clean ({cleanup_type::always, p}, true); + } + else // == mkdir_status::already_exists + throw_generic_error (EEXIST); + } + catch (const system_error& e) + { + error () << "unable to create directory '" << p << "': " << e; + } + } + + r = 0; + } + catch (const invalid_path& e) + { + error (false) << "invalid path '" << e.path << "'"; + } + // Can be thrown while closing in, out or writing to cerr. + // + catch (const io_error& e) + { + error (false) << e; + } + catch (const failed&) + { + // Diagnostics has already been issued. + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + + // mv [--no-cleanup] [-f] + // mv [--no-cleanup] [-f] ... / + // + // Note: can be executed synchronously. + // + static uint8_t + mv (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + auto error = [&cerr] (bool fail = true) + { + return error_record (cerr, fail, "mv"); + }; + + try + { + in.close (); + out.close (); + + auto i (args.begin ()); + auto e (args.end ()); + + // Process options. + // + bool no_cleanup (false); + bool force (false); + for (; i != e; ++i) + { + const string& o (*i); + + if (o == "--no-cleanup") + no_cleanup = true; + else if (*i == "-f") + force = true; + else + { + if (o == "--") + ++i; + + break; + } + } + + // Move filesystem entries. + // + if (i == e) + error () << "missing arguments"; + + const dir_path& wd (sp.wd_path); + + auto j (args.rbegin ()); + path dst (parse_path (*j++, wd)); + e = j.base (); + + if (i == e) + error () << "missing source path"; + + auto mv = [no_cleanup, force, &wd, &sp, &error] (const path& from, + const path& to) + { + const dir_path& rwd (sp.root->wd_path); + + if (!from.sub (rwd) && !force) + error () << "'" << from << "' is out of working directory '" + << rwd << "'"; + + try + { + auto check_wd = [&wd, &error] (const path& p) + { + if (wd.sub (path_cast (p))) + error () << "'" << p << "' contains test working directory '" + << wd << "'"; + }; + + check_wd (from); + check_wd (to); + + bool exists (butl::entry_exists (to)); + + // Fail if the source and destination paths are the same. + // + // Note that for mventry() function (that is based on the POSIX + // rename() function) this is a noop. + // + if (exists && to == from) + error () << "unable to move entity '" << from << "' to itself"; + + // Rename/move the filesystem entry, replacing an existing one. + // + mventry (from, + to, + cpflags::overwrite_permissions | + cpflags::overwrite_content); + + // Unless suppressed, adjust the cleanups that are sub-paths of + // the source path. + // + if (!no_cleanup) + { + // "Move" the matching cleanup if the destination path doesn't + // exist and is a sub-path of the working directory. Otherwise + // just remove it. + // + // Note that it's not enough to just change the cleanup paths. + // We also need to make sure that these cleanups happen before + // the destination directory (or any of its parents) cleanup, + // that is potentially registered. To achieve that we can just + // relocate these cleanup entries to the end of the list, + // preserving their mutual order. Remember that cleanups in + // the list are executed in the reversed order. + // + bool mv_cleanups (!exists && to.sub (rwd)); + cleanups cs; + + // Remove the source path sub-path cleanups from the list, + // adjusting/caching them if required (see above). + // + for (auto i (sp.cleanups.begin ()); i != sp.cleanups.end (); ) + { + cleanup& c (*i); + path& p (c.path); + + if (p.sub (from)) + { + if (mv_cleanups) + { + // Note that we need to preserve the cleanup path + // trailing separator which indicates the removal + // method. Also note that leaf(), in particular, does + // that. + // + p = p != from + ? to / p.leaf (path_cast (from)) + : p.to_directory () + ? path_cast (to) + : to; + + cs.push_back (move (c)); + } + + i = sp.cleanups.erase (i); + } + else + ++i; + } + + // Re-insert the adjusted cleanups at the end of the list. + // + sp.cleanups.insert (sp.cleanups.end (), + make_move_iterator (cs.begin ()), + make_move_iterator (cs.end ())); + } + } + catch (const system_error& e) + { + error () << "unable to move entity '" << from << "' to '" << to + << "': " << e; + } + }; + + // If destination is not a directory path (no trailing separator) + // then move the filesystem entry to the specified path (the only + // source path is allowed in such a case). Otherwise move the source + // filesystem entries into the destination directory. + // + if (!dst.to_directory ()) + { + path src (parse_path (*i++, wd)); + + // If there are multiple sources but no trailing separator for the + // destination, then, most likelly, it is missing. + // + if (i != e) + error () << "multiple source paths without trailing separator " + << "for destination directory"; + + // Synopsis 1: move an entity to the specified path. + // + mv (src, dst); + } + else + { + // Synopsis 2: move entities into the specified directory. + // + for (; i != e; ++i) + { + path src (parse_path (*i, wd)); + mv (src, dst / src.leaf ()); + } + } + + r = 0; + } + catch (const invalid_path& e) + { + error (false) << "invalid path '" << e.path << "'"; + } + // Can be thrown while closing in, out or writing to cerr. + // + catch (const io_error& e) + { + error (false) << e; + } + catch (const failed&) + { + // Diagnostics has already been issued. + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + + // rm [-r] [-f] ... + // + // The implementation deviates from POSIX in a number of ways. It doesn't + // interact with a user and fails immediatelly if unable to process an + // argument. It doesn't check for dots containment in the path, and + // doesn't consider files and directory permissions in any way just + // trying to remove a filesystem entry. Always fails if empty path is + // specified. + // + // Note: can be executed synchronously. + // + static uint8_t + rm (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + auto error = [&cerr] (bool fail = true) + { + return error_record (cerr, fail, "rm"); + }; + + try + { + in.close (); + out.close (); + + auto i (args.begin ()); + auto e (args.end ()); + + // Process options. + // + bool dir (false); + bool force (false); + for (; i != e; ++i) + { + if (*i == "-r") + dir = true; + else if (*i == "-f") + force = true; + else + { + if (*i == "--") + ++i; + + break; + } + } + + // Remove entries. + // + if (i == e && !force) + error () << "missing file"; + + const dir_path& wd (sp.wd_path); + const dir_path& rwd (sp.root->wd_path); + + for (; i != e; ++i) + { + path p (parse_path (*i, wd)); + + if (!p.sub (rwd) && !force) + error () << "'" << p << "' is out of working directory '" << rwd + << "'"; + + try + { + dir_path d (path_cast (p)); + + if (dir_exists (d)) + { + if (!dir) + error () << "'" << p << "' is a directory"; + + if (wd.sub (d)) + error () << "'" << p << "' contains test working directory '" + << wd << "'"; + + // The call can result in rmdir_status::not_exist. That's not + // very likelly but there is also nothing bad about it. + // + try_rmdir_r (d); + } + else if (try_rmfile (p) == rmfile_status::not_exist && !force) + throw_generic_error (ENOENT); + } + catch (const system_error& e) + { + error () << "unable to remove '" << p << "': " << e; + } + } + + r = 0; + } + catch (const invalid_path& e) + { + error (false) << "invalid path '" << e.path << "'"; + } + // Can be thrown while closing in, out or writing to cerr. + // + catch (const io_error& e) + { + error (false) << e; + } + catch (const failed&) + { + // Diagnostics has already been issued. + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + + // rmdir [-f] ... + // + // Note: can be executed synchronously. + // + static uint8_t + rmdir (scope& sp, + const strings& args, + auto_fd in, auto_fd out, auto_fd err) noexcept + try + { + uint8_t r (1); + ofdstream cerr (move (err)); + + auto error = [&cerr] (bool fail = true) + { + return error_record (cerr, fail, "rmdir"); + }; + + try + { + in.close (); + out.close (); + + auto i (args.begin ()); + auto e (args.end ()); + + // Process options. + // + bool force (false); + for (; i != e; ++i) + { + if (*i == "-f") + force = true; + else + { + if (*i == "--") + ++i; + + break; + } + } + + // Remove directories. + // + if (i == e && !force) + error () << "missing directory"; + + const dir_path& wd (sp.wd_path); + const dir_path& rwd (sp.root->wd_path); + + for (; i != e; ++i) + { + dir_path p (path_cast (parse_path (*i, wd))); + + if (wd.sub (p)) + error () << "'" << p << "' contains test working directory '" + << wd << "'"; + + if (!p.sub (rwd) && !force) + error () << "'" << p << "' is out of working directory '" + << rwd << "'"; + + try + { + rmdir_status s (try_rmdir (p)); + + if (s == rmdir_status::not_empty) + throw_generic_error (ENOTEMPTY); + else if (s == rmdir_status::not_exist && !force) + throw_generic_error (ENOENT); + } + catch (const system_error& e) + { + error () << "unable to remove '" << p << "': " << e; + } + } + + r = 0; + } + catch (const invalid_path& e) + { + error (false) << "invalid path '" << e.path << "'"; + } + // Can be thrown while closing in, out or writing to cerr. + // + catch (const io_error& e) + { + error (false) << e; + } + catch (const failed&) + { + // Diagnostics has already been issued. + } + + cerr.close (); + return r; + } + catch (const std::exception&) + { + return 1; + } + + // sed [-n] [-i] -e