diff options
Diffstat (limited to 'libbuild2/build/script')
21 files changed, 5495 insertions, 424 deletions
diff --git a/libbuild2/build/script/builtin-options.cxx b/libbuild2/build/script/builtin-options.cxx new file mode 100644 index 0000000..dba3c59 --- /dev/null +++ b/libbuild2/build/script/builtin-options.cxx @@ -0,0 +1,606 @@ +// -*- C++ -*- +// +// This file was generated by CLI, a command line interface +// compiler for C++. +// + +// Begin prologue. +// +#include <libbuild2/types-parsers.hxx> +// +// End prologue. + +#include <libbuild2/build/script/builtin-options.hxx> + +#include <map> +#include <set> +#include <string> +#include <vector> +#include <utility> +#include <ostream> +#include <sstream> +#include <cstring> + +namespace build2 +{ + namespace build + { + namespace cli + { + template <typename X> + struct parser + { + static void + parse (X& x, bool& xs, scanner& s) + { + using namespace std; + + const char* o (s.next ()); + if (s.more ()) + { + string v (s.next ()); + istringstream is (v); + if (!(is >> x && is.peek () == istringstream::traits_type::eof ())) + throw invalid_value (o, v); + } + else + throw missing_value (o); + + xs = true; + } + }; + + template <> + struct parser<bool> + { + static void + parse (bool& x, bool& xs, scanner& s) + { + const char* o (s.next ()); + + if (s.more ()) + { + const char* v (s.next ()); + + if (std::strcmp (v, "1") == 0 || + std::strcmp (v, "true") == 0 || + std::strcmp (v, "TRUE") == 0 || + std::strcmp (v, "True") == 0) + x = true; + else if (std::strcmp (v, "0") == 0 || + std::strcmp (v, "false") == 0 || + std::strcmp (v, "FALSE") == 0 || + std::strcmp (v, "False") == 0) + x = false; + else + throw invalid_value (o, v); + } + else + throw missing_value (o); + + xs = true; + } + }; + + template <> + struct parser<std::string> + { + static void + parse (std::string& x, bool& xs, scanner& s) + { + const char* o (s.next ()); + + if (s.more ()) + x = s.next (); + else + throw missing_value (o); + + xs = true; + } + }; + + template <typename X> + struct parser<std::pair<X, std::size_t> > + { + static void + parse (std::pair<X, std::size_t>& x, bool& xs, scanner& s) + { + x.second = s.position (); + parser<X>::parse (x.first, xs, s); + } + }; + + template <typename X> + struct parser<std::vector<X> > + { + static void + parse (std::vector<X>& c, bool& xs, scanner& s) + { + X x; + bool dummy; + parser<X>::parse (x, dummy, s); + c.push_back (x); + xs = true; + } + }; + + template <typename X, typename C> + struct parser<std::set<X, C> > + { + static void + parse (std::set<X, C>& c, bool& xs, scanner& s) + { + X x; + bool dummy; + parser<X>::parse (x, dummy, s); + c.insert (x); + xs = true; + } + }; + + template <typename K, typename V, typename C> + struct parser<std::map<K, V, C> > + { + static void + parse (std::map<K, V, C>& m, bool& xs, scanner& s) + { + const char* o (s.next ()); + + if (s.more ()) + { + std::size_t pos (s.position ()); + std::string ov (s.next ()); + std::string::size_type p = ov.find ('='); + + K k = K (); + V v = V (); + std::string kstr (ov, 0, p); + std::string vstr (ov, (p != std::string::npos ? p + 1 : ov.size ())); + + int ac (2); + char* av[] = + { + const_cast<char*> (o), + 0 + }; + + bool dummy; + if (!kstr.empty ()) + { + av[1] = const_cast<char*> (kstr.c_str ()); + argv_scanner s (0, ac, av, false, pos); + parser<K>::parse (k, dummy, s); + } + + if (!vstr.empty ()) + { + av[1] = const_cast<char*> (vstr.c_str ()); + argv_scanner s (0, ac, av, false, pos); + parser<V>::parse (v, dummy, s); + } + + m[k] = v; + } + else + throw missing_value (o); + + xs = true; + } + }; + + template <typename K, typename V, typename C> + struct parser<std::multimap<K, V, C> > + { + static void + parse (std::multimap<K, V, C>& m, bool& xs, scanner& s) + { + const char* o (s.next ()); + + if (s.more ()) + { + std::size_t pos (s.position ()); + std::string ov (s.next ()); + std::string::size_type p = ov.find ('='); + + K k = K (); + V v = V (); + std::string kstr (ov, 0, p); + std::string vstr (ov, (p != std::string::npos ? p + 1 : ov.size ())); + + int ac (2); + char* av[] = + { + const_cast<char*> (o), + 0 + }; + + bool dummy; + if (!kstr.empty ()) + { + av[1] = const_cast<char*> (kstr.c_str ()); + argv_scanner s (0, ac, av, false, pos); + parser<K>::parse (k, dummy, s); + } + + if (!vstr.empty ()) + { + av[1] = const_cast<char*> (vstr.c_str ()); + argv_scanner s (0, ac, av, false, pos); + parser<V>::parse (v, dummy, s); + } + + m.insert (typename std::multimap<K, V, C>::value_type (k, v)); + } + else + throw missing_value (o); + + xs = true; + } + }; + + template <typename X, typename T, T X::*M> + void + thunk (X& x, scanner& s) + { + parser<T>::parse (x.*M, s); + } + + template <typename X, bool X::*M> + void + thunk (X& x, scanner& s) + { + s.next (); + x.*M = true; + } + + template <typename X, typename T, T X::*M, bool X::*S> + void + thunk (X& x, scanner& s) + { + parser<T>::parse (x.*M, x.*S, s); + } + } + } +} + +#include <map> + +namespace build2 +{ + namespace build + { + namespace script + { + // depdb_dyndep_options + // + + depdb_dyndep_options:: + depdb_dyndep_options () + : file_ (), + file_specified_ (false), + format_ (), + format_specified_ (false), + what_ (), + what_specified_ (false), + include_path_ (), + include_path_specified_ (false), + default_type_ (), + default_type_specified_ (false), + adhoc_ (), + cwd_ (), + cwd_specified_ (false), + drop_cycles_ (), + target_what_ (), + target_what_specified_ (false), + target_default_type_ (), + target_default_type_specified_ (false), + target_extension_type_ (), + target_extension_type_specified_ (false), + target_cwd_ (), + target_cwd_specified_ (false) + { + } + + bool depdb_dyndep_options:: + parse (int& argc, + char** argv, + bool erase, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + { + ::build2::build::cli::argv_scanner s (argc, argv, erase); + bool r = _parse (s, opt, arg); + return r; + } + + bool depdb_dyndep_options:: + parse (int start, + int& argc, + char** argv, + bool erase, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + { + ::build2::build::cli::argv_scanner s (start, argc, argv, erase); + bool r = _parse (s, opt, arg); + return r; + } + + bool depdb_dyndep_options:: + parse (int& argc, + char** argv, + int& end, + bool erase, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + { + ::build2::build::cli::argv_scanner s (argc, argv, erase); + bool r = _parse (s, opt, arg); + end = s.end (); + return r; + } + + bool depdb_dyndep_options:: + parse (int start, + int& argc, + char** argv, + int& end, + bool erase, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + { + ::build2::build::cli::argv_scanner s (start, argc, argv, erase); + bool r = _parse (s, opt, arg); + end = s.end (); + return r; + } + + bool depdb_dyndep_options:: + parse (::build2::build::cli::scanner& s, + ::build2::build::cli::unknown_mode opt, + ::build2::build::cli::unknown_mode arg) + { + bool r = _parse (s, opt, arg); + return r; + } + + typedef + std::map<std::string, void (*) (depdb_dyndep_options&, ::build2::build::cli::scanner&)> + _cli_depdb_dyndep_options_map; + + static _cli_depdb_dyndep_options_map _cli_depdb_dyndep_options_map_; + + struct _cli_depdb_dyndep_options_map_init + { + _cli_depdb_dyndep_options_map_init () + { + _cli_depdb_dyndep_options_map_["--file"] = + &::build2::build::cli::thunk< depdb_dyndep_options, path, &depdb_dyndep_options::file_, + &depdb_dyndep_options::file_specified_ >; + _cli_depdb_dyndep_options_map_["--format"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::format_, + &depdb_dyndep_options::format_specified_ >; + _cli_depdb_dyndep_options_map_["--what"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::what_, + &depdb_dyndep_options::what_specified_ >; + _cli_depdb_dyndep_options_map_["--include-path"] = + &::build2::build::cli::thunk< depdb_dyndep_options, dir_paths, &depdb_dyndep_options::include_path_, + &depdb_dyndep_options::include_path_specified_ >; + _cli_depdb_dyndep_options_map_["-I"] = + &::build2::build::cli::thunk< depdb_dyndep_options, dir_paths, &depdb_dyndep_options::include_path_, + &depdb_dyndep_options::include_path_specified_ >; + _cli_depdb_dyndep_options_map_["--default-type"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::default_type_, + &depdb_dyndep_options::default_type_specified_ >; + _cli_depdb_dyndep_options_map_["--adhoc"] = + &::build2::build::cli::thunk< depdb_dyndep_options, &depdb_dyndep_options::adhoc_ >; + _cli_depdb_dyndep_options_map_["--cwd"] = + &::build2::build::cli::thunk< depdb_dyndep_options, dir_path, &depdb_dyndep_options::cwd_, + &depdb_dyndep_options::cwd_specified_ >; + _cli_depdb_dyndep_options_map_["--drop-cycles"] = + &::build2::build::cli::thunk< depdb_dyndep_options, &depdb_dyndep_options::drop_cycles_ >; + _cli_depdb_dyndep_options_map_["--target-what"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_what_, + &depdb_dyndep_options::target_what_specified_ >; + _cli_depdb_dyndep_options_map_["--target-default-type"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_default_type_, + &depdb_dyndep_options::target_default_type_specified_ >; + _cli_depdb_dyndep_options_map_["--target-extension-type"] = + &::build2::build::cli::thunk< depdb_dyndep_options, map<string, string>, &depdb_dyndep_options::target_extension_type_, + &depdb_dyndep_options::target_extension_type_specified_ >; + _cli_depdb_dyndep_options_map_["--target-cwd"] = + &::build2::build::cli::thunk< depdb_dyndep_options, dir_path, &depdb_dyndep_options::target_cwd_, + &depdb_dyndep_options::target_cwd_specified_ >; + } + }; + + static _cli_depdb_dyndep_options_map_init _cli_depdb_dyndep_options_map_init_; + + bool depdb_dyndep_options:: + _parse (const char* o, ::build2::build::cli::scanner& s) + { + _cli_depdb_dyndep_options_map::const_iterator i (_cli_depdb_dyndep_options_map_.find (o)); + + if (i != _cli_depdb_dyndep_options_map_.end ()) + { + (*(i->second)) (*this, s); + return true; + } + + return false; + } + + bool depdb_dyndep_options:: + _parse (::build2::build::cli::scanner& s, + ::build2::build::cli::unknown_mode opt_mode, + ::build2::build::cli::unknown_mode arg_mode) + { + // Can't skip combined flags (--no-combined-flags). + // + assert (opt_mode != ::build2::build::cli::unknown_mode::skip); + + bool r = false; + bool opt = true; + + while (s.more ()) + { + const char* o = s.peek (); + + if (std::strcmp (o, "--") == 0) + { + opt = false; + s.skip (); + r = true; + continue; + } + + if (opt) + { + if (_parse (o, s)) + { + r = true; + continue; + } + + if (std::strncmp (o, "-", 1) == 0 && o[1] != '\0') + { + // Handle combined option values. + // + std::string co; + if (const char* v = std::strchr (o, '=')) + { + co.assign (o, 0, v - o); + ++v; + + int ac (2); + char* av[] = + { + const_cast<char*> (co.c_str ()), + const_cast<char*> (v) + }; + + ::build2::build::cli::argv_scanner ns (0, ac, av); + + if (_parse (co.c_str (), ns)) + { + // Parsed the option but not its value? + // + if (ns.end () != 2) + throw ::build2::build::cli::invalid_value (co, v); + + s.next (); + r = true; + continue; + } + else + { + // Set the unknown option and fall through. + // + o = co.c_str (); + } + } + + // Handle combined flags. + // + char cf[3]; + { + const char* p = o + 1; + for (; *p != '\0'; ++p) + { + if (!((*p >= 'a' && *p <= 'z') || + (*p >= 'A' && *p <= 'Z') || + (*p >= '0' && *p <= '9'))) + break; + } + + if (*p == '\0') + { + for (p = o + 1; *p != '\0'; ++p) + { + std::strcpy (cf, "-"); + cf[1] = *p; + cf[2] = '\0'; + + int ac (1); + char* av[] = + { + cf + }; + + ::build2::build::cli::argv_scanner ns (0, ac, av); + + if (!_parse (cf, ns)) + break; + } + + if (*p == '\0') + { + // All handled. + // + s.next (); + r = true; + continue; + } + else + { + // Set the unknown option and fall through. + // + o = cf; + } + } + } + + switch (opt_mode) + { + case ::build2::build::cli::unknown_mode::skip: + { + s.skip (); + r = true; + continue; + } + case ::build2::build::cli::unknown_mode::stop: + { + break; + } + case ::build2::build::cli::unknown_mode::fail: + { + throw ::build2::build::cli::unknown_option (o); + } + } + + break; + } + } + + switch (arg_mode) + { + case ::build2::build::cli::unknown_mode::skip: + { + s.skip (); + r = true; + continue; + } + case ::build2::build::cli::unknown_mode::stop: + { + break; + } + case ::build2::build::cli::unknown_mode::fail: + { + throw ::build2::build::cli::unknown_argument (o); + } + } + + break; + } + + return r; + } + } + } +} + +// Begin epilogue. +// +// +// End epilogue. + diff --git a/libbuild2/build/script/builtin-options.hxx b/libbuild2/build/script/builtin-options.hxx new file mode 100644 index 0000000..a8c3440 --- /dev/null +++ b/libbuild2/build/script/builtin-options.hxx @@ -0,0 +1,284 @@ +// -*- C++ -*- +// +// This file was generated by CLI, a command line interface +// compiler for C++. +// + +#ifndef LIBBUILD2_BUILD_SCRIPT_BUILTIN_OPTIONS_HXX +#define LIBBUILD2_BUILD_SCRIPT_BUILTIN_OPTIONS_HXX + +// Begin prologue. +// +// +// End prologue. + +#include <libbuild2/common-options.hxx> + +namespace build2 +{ + namespace build + { + namespace script + { + class depdb_dyndep_options + { + public: + depdb_dyndep_options (); + + // Return true if anything has been parsed. + // + bool + parse (int& argc, + char** argv, + bool erase = false, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + bool + parse (int start, + int& argc, + char** argv, + bool erase = false, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + bool + parse (int& argc, + char** argv, + int& end, + bool erase = false, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + bool + parse (int start, + int& argc, + char** argv, + int& end, + bool erase = false, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + bool + parse (::build2::build::cli::scanner&, + ::build2::build::cli::unknown_mode option = ::build2::build::cli::unknown_mode::fail, + ::build2::build::cli::unknown_mode argument = ::build2::build::cli::unknown_mode::stop); + + // Option accessors and modifiers. + // + const path& + file () const; + + path& + file (); + + void + file (const path&); + + bool + file_specified () const; + + void + file_specified (bool); + + const string& + format () const; + + string& + format (); + + void + format (const string&); + + bool + format_specified () const; + + void + format_specified (bool); + + const string& + what () const; + + string& + what (); + + void + what (const string&); + + bool + what_specified () const; + + void + what_specified (bool); + + const dir_paths& + include_path () const; + + dir_paths& + include_path (); + + void + include_path (const dir_paths&); + + bool + include_path_specified () const; + + void + include_path_specified (bool); + + const string& + default_type () const; + + string& + default_type (); + + void + default_type (const string&); + + bool + default_type_specified () const; + + void + default_type_specified (bool); + + const bool& + adhoc () const; + + bool& + adhoc (); + + void + adhoc (const bool&); + + const dir_path& + cwd () const; + + dir_path& + cwd (); + + void + cwd (const dir_path&); + + bool + cwd_specified () const; + + void + cwd_specified (bool); + + const bool& + drop_cycles () const; + + bool& + drop_cycles (); + + void + drop_cycles (const bool&); + + const string& + target_what () const; + + string& + target_what (); + + void + target_what (const string&); + + bool + target_what_specified () const; + + void + target_what_specified (bool); + + const string& + target_default_type () const; + + string& + target_default_type (); + + void + target_default_type (const string&); + + bool + target_default_type_specified () const; + + void + target_default_type_specified (bool); + + const map<string, string>& + target_extension_type () const; + + map<string, string>& + target_extension_type (); + + void + target_extension_type (const map<string, string>&); + + bool + target_extension_type_specified () const; + + void + target_extension_type_specified (bool); + + const dir_path& + target_cwd () const; + + dir_path& + target_cwd (); + + void + target_cwd (const dir_path&); + + bool + target_cwd_specified () const; + + void + target_cwd_specified (bool); + + // Implementation details. + // + protected: + bool + _parse (const char*, ::build2::build::cli::scanner&); + + private: + bool + _parse (::build2::build::cli::scanner&, + ::build2::build::cli::unknown_mode option, + ::build2::build::cli::unknown_mode argument); + + public: + path file_; + bool file_specified_; + string format_; + bool format_specified_; + string what_; + bool what_specified_; + dir_paths include_path_; + bool include_path_specified_; + string default_type_; + bool default_type_specified_; + bool adhoc_; + dir_path cwd_; + bool cwd_specified_; + bool drop_cycles_; + string target_what_; + bool target_what_specified_; + string target_default_type_; + bool target_default_type_specified_; + map<string, string> target_extension_type_; + bool target_extension_type_specified_; + dir_path target_cwd_; + bool target_cwd_specified_; + }; + } + } +} + +#include <libbuild2/build/script/builtin-options.ixx> + +// Begin epilogue. +// +// +// End epilogue. + +#endif // LIBBUILD2_BUILD_SCRIPT_BUILTIN_OPTIONS_HXX diff --git a/libbuild2/build/script/builtin-options.ixx b/libbuild2/build/script/builtin-options.ixx new file mode 100644 index 0000000..20847c2 --- /dev/null +++ b/libbuild2/build/script/builtin-options.ixx @@ -0,0 +1,363 @@ +// -*- C++ -*- +// +// This file was generated by CLI, a command line interface +// compiler for C++. +// + +// Begin prologue. +// +// +// End prologue. + +namespace build2 +{ + namespace build + { + namespace script + { + // depdb_dyndep_options + // + + inline const path& depdb_dyndep_options:: + file () const + { + return this->file_; + } + + inline path& depdb_dyndep_options:: + file () + { + return this->file_; + } + + inline void depdb_dyndep_options:: + file (const path& x) + { + this->file_ = x; + } + + inline bool depdb_dyndep_options:: + file_specified () const + { + return this->file_specified_; + } + + inline void depdb_dyndep_options:: + file_specified (bool x) + { + this->file_specified_ = x; + } + + inline const string& depdb_dyndep_options:: + format () const + { + return this->format_; + } + + inline string& depdb_dyndep_options:: + format () + { + return this->format_; + } + + inline void depdb_dyndep_options:: + format (const string& x) + { + this->format_ = x; + } + + inline bool depdb_dyndep_options:: + format_specified () const + { + return this->format_specified_; + } + + inline void depdb_dyndep_options:: + format_specified (bool x) + { + this->format_specified_ = x; + } + + inline const string& depdb_dyndep_options:: + what () const + { + return this->what_; + } + + inline string& depdb_dyndep_options:: + what () + { + return this->what_; + } + + inline void depdb_dyndep_options:: + what (const string& x) + { + this->what_ = x; + } + + inline bool depdb_dyndep_options:: + what_specified () const + { + return this->what_specified_; + } + + inline void depdb_dyndep_options:: + what_specified (bool x) + { + this->what_specified_ = x; + } + + inline const dir_paths& depdb_dyndep_options:: + include_path () const + { + return this->include_path_; + } + + inline dir_paths& depdb_dyndep_options:: + include_path () + { + return this->include_path_; + } + + inline void depdb_dyndep_options:: + include_path (const dir_paths& x) + { + this->include_path_ = x; + } + + inline bool depdb_dyndep_options:: + include_path_specified () const + { + return this->include_path_specified_; + } + + inline void depdb_dyndep_options:: + include_path_specified (bool x) + { + this->include_path_specified_ = x; + } + + inline const string& depdb_dyndep_options:: + default_type () const + { + return this->default_type_; + } + + inline string& depdb_dyndep_options:: + default_type () + { + return this->default_type_; + } + + inline void depdb_dyndep_options:: + default_type (const string& x) + { + this->default_type_ = x; + } + + inline bool depdb_dyndep_options:: + default_type_specified () const + { + return this->default_type_specified_; + } + + inline void depdb_dyndep_options:: + default_type_specified (bool x) + { + this->default_type_specified_ = x; + } + + inline const bool& depdb_dyndep_options:: + adhoc () const + { + return this->adhoc_; + } + + inline bool& depdb_dyndep_options:: + adhoc () + { + return this->adhoc_; + } + + inline void depdb_dyndep_options:: + adhoc (const bool& x) + { + this->adhoc_ = x; + } + + inline const dir_path& depdb_dyndep_options:: + cwd () const + { + return this->cwd_; + } + + inline dir_path& depdb_dyndep_options:: + cwd () + { + return this->cwd_; + } + + inline void depdb_dyndep_options:: + cwd (const dir_path& x) + { + this->cwd_ = x; + } + + inline bool depdb_dyndep_options:: + cwd_specified () const + { + return this->cwd_specified_; + } + + inline void depdb_dyndep_options:: + cwd_specified (bool x) + { + this->cwd_specified_ = x; + } + + inline const bool& depdb_dyndep_options:: + drop_cycles () const + { + return this->drop_cycles_; + } + + inline bool& depdb_dyndep_options:: + drop_cycles () + { + return this->drop_cycles_; + } + + inline void depdb_dyndep_options:: + drop_cycles (const bool& x) + { + this->drop_cycles_ = x; + } + + inline const string& depdb_dyndep_options:: + target_what () const + { + return this->target_what_; + } + + inline string& depdb_dyndep_options:: + target_what () + { + return this->target_what_; + } + + inline void depdb_dyndep_options:: + target_what (const string& x) + { + this->target_what_ = x; + } + + inline bool depdb_dyndep_options:: + target_what_specified () const + { + return this->target_what_specified_; + } + + inline void depdb_dyndep_options:: + target_what_specified (bool x) + { + this->target_what_specified_ = x; + } + + inline const string& depdb_dyndep_options:: + target_default_type () const + { + return this->target_default_type_; + } + + inline string& depdb_dyndep_options:: + target_default_type () + { + return this->target_default_type_; + } + + inline void depdb_dyndep_options:: + target_default_type (const string& x) + { + this->target_default_type_ = x; + } + + inline bool depdb_dyndep_options:: + target_default_type_specified () const + { + return this->target_default_type_specified_; + } + + inline void depdb_dyndep_options:: + target_default_type_specified (bool x) + { + this->target_default_type_specified_ = x; + } + + inline const map<string, string>& depdb_dyndep_options:: + target_extension_type () const + { + return this->target_extension_type_; + } + + inline map<string, string>& depdb_dyndep_options:: + target_extension_type () + { + return this->target_extension_type_; + } + + inline void depdb_dyndep_options:: + target_extension_type (const map<string, string>& x) + { + this->target_extension_type_ = x; + } + + inline bool depdb_dyndep_options:: + target_extension_type_specified () const + { + return this->target_extension_type_specified_; + } + + inline void depdb_dyndep_options:: + target_extension_type_specified (bool x) + { + this->target_extension_type_specified_ = x; + } + + inline const dir_path& depdb_dyndep_options:: + target_cwd () const + { + return this->target_cwd_; + } + + inline dir_path& depdb_dyndep_options:: + target_cwd () + { + return this->target_cwd_; + } + + inline void depdb_dyndep_options:: + target_cwd (const dir_path& x) + { + this->target_cwd_ = x; + } + + inline bool depdb_dyndep_options:: + target_cwd_specified () const + { + return this->target_cwd_specified_; + } + + inline void depdb_dyndep_options:: + target_cwd_specified (bool x) + { + this->target_cwd_specified_ = x; + } + } + } +} + +// Begin epilogue. +// +// +// End epilogue. diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli new file mode 100644 index 0000000..5aea034 --- /dev/null +++ b/libbuild2/build/script/builtin.cli @@ -0,0 +1,128 @@ +// file : libbuild2/build/script/builtin.cli +// license : MIT; see accompanying LICENSE file + +include <libbuild2/common.cli>; + +// Note that options in this file are undocumented because we generate neither +// the usage printing code nor man pages. Instead, they are documented in the +// manual. +// +namespace build2 +{ + namespace build + { + namespace script + { + // Pseudo-builtin options. + // + class depdb_dyndep_options + { + // Note that --byproduct or --dyn-target, if any, must be the first + // option and is handled ad hoc. + // + // Similarly, --update-{include,exclude} are handled ad hoc and must + // be literals, similar to the -- separator. They specify prerequisite + // targets/patterns to include/exclude (from the static prerequisite + // set) for update during match (those excluded will be updated during + // execute). The order in which these options are specified is + // significant with the first target/pattern that matches determining + // the result. If only the --update-include options are specified, + // then only the explicitly included prerequisites will be updated. + // Otherwise, all prerequisites that are not explicitly excluded will + // be updated. If none of these options is specified, then all the + // static prerequisites are updated during match. Note also that these + // options do not apply to ad hoc prerequisites which are always + // updated during match. + // + // Note that in the future we may extend --cwd support to the non- + // byproduct mode where it will also have the `env --cwd` semantics + // (thus the matching name). Note that it will also be incompatible + // with support for generated files (and thus -I) at least in the make + // format where we use relative paths for non-existent files. + // + // Currently Supported dependency formats (--format) are `make` + // (default) and `lines`. + // + // The `make` format is the make dependency declaration in the + // `<target>...: [<prerequisite>...]` form. In the non-byproduct mode + // a relative prerequisite path is considered non-existent. + // + // The `lines` format lists targets and/or prerequisites one per line. + // If the --dyn-target option is specified then the target list is + // expected to come first separated from the prerequisites list with a + // blank line. If there are no prerequisites, then the blank line can + // be omitted. If the --dyn-target option is not specified, then all + // lines are treated as prerequisites and there should be no blank + // lines. In the non-byproduct mode a prerequisite line that starts + // with a leading space is considered a non-existent prerequisite. + // Currently only relative non-existent prerequisites are supported. + // Finally, in this mode, if the prerequisite is syntactically a + // directory (that is, it ends with a trailing directory separator), + // then it is added as fsdir{}. This can be used to handle situations + // where the dynamic targets are placed into subdirectories. + // + // Note on naming: whenever we (may) have two options, one for target + // and the other for prerequisite, we omit "prerequisite" as that's + // what we extract by default and most commonly. For example: + // + // --what --target-what + // --default-type --target-default-type + // + path --file; // Read from file rather than stdin. + + string --format; // Dependency format: `make` (default), + // or `lines`. + + // Dynamic dependency extraction options. + // + string --what; // Prerequisite kind, e.g., "header". + + dir_paths --include-path|-I; // Search paths for generated + // prerequisites. + + string --default-type; // Default prerequisite type to use if + // none could be derived from extension. + + bool --adhoc; // Treat dynamically discovered + // prerequisites as ad hoc (so they + // don't end up in $<; only in the + // normal mode). + + dir_path --cwd; // Builtin's working directory used + // to complete relative paths of + // prerequisites (only in --byproduct + // mode, lines format for existing + // paths). + + bool --drop-cycles; // Drop prerequisites that are also + // targets. Only use if you are sure + // such cycles are harmless, that is, + // the output is not affected by such + // prerequisites' content. + + // Dynamic target extraction options. + // + // This functionality is enabled with the --dyn-target option. Only + // the make format is supported, where the listed targets are added as + // ad hoc group members (unless already specified as static members). + // This functionality is not available in the byproduct mode. + // + string --target-what; // Target kind, e.g., "source". + + string --target-default-type; // Default target type to use if none + // could be derived from extension. + + map<string, string> // Extension to target type mapping in + --target-extension-type; // the <ext>=<type> form, for example, + // h=hxx. This mapping is considered + // before attempting to automatically + // map the extension and so can be used + // to resolve ambiguities. + + dir_path --target-cwd; // Builtin's working directory used to + // complete relative paths of targets. + + }; + } + } +} diff --git a/libbuild2/build/script/lexer+for-loop.test.testscript b/libbuild2/build/script/lexer+for-loop.test.testscript new file mode 100644 index 0000000..3f8e6b5 --- /dev/null +++ b/libbuild2/build/script/lexer+for-loop.test.testscript @@ -0,0 +1,188 @@ +# file : libbuild2/build/script/lexer+for-loop.test.testscript +# license : MIT; see accompanying LICENSE file + +test.arguments = for-loop + +: redirect +: +{ + : pass + : + $* <"cmd <| 1>|" >>EOO + 'cmd' + <| + '1' + >| + <newline> + EOO + + : null + : + $* <"cmd <- 1>-" >>EOO + 'cmd' + <- + '1' + >- + <newline> + EOO + + : trace + : + $* <"cmd 1>!" >>EOO + 'cmd' + '1' + >! + <newline> + EOO + + : merge + : + $* <"cmd 1>&2" >>EOO + 'cmd' + '1' + >& + '2' + <newline> + EOO + + : str + : + $* <"cmd <<<=a 1>>>?b" >>EOO + 'cmd' + <<<= + 'a' + '1' + >>>? + 'b' + <newline> + EOO + + : str-nn + : + $* <"cmd <<<=:a 1>>>?:b" >>EOO + 'cmd' + <<<=: + 'a' + '1' + >>>?: + 'b' + <newline> + EOO + + : str-nn-alias + : + $* <"cmd <<<:a 1>>>?:b" >>EOO + 'cmd' + <<<: + 'a' + '1' + >>>?: + 'b' + <newline> + EOO + + : doc + : + $* <"cmd <<EOI 1>>EOO" >>EOO + 'cmd' + << + 'EOI' + '1' + >> + 'EOO' + <newline> + EOO + + : doc-nn + : + $* <"cmd <<:EOI 1>>?:EOO" >>EOO + 'cmd' + <<: + 'EOI' + '1' + >>?: + 'EOO' + <newline> + EOO + + : file-cmp + : + $* <"cmd <=in >?out 2>?err" >>EOO + 'cmd' + <= + 'in' + >? + 'out' + '2' + >? + 'err' + <newline> + EOO + + : file-write + : + $* <"cmd >=out 2>+err" >>EOO + 'cmd' + >= + 'out' + '2' + >+ + 'err' + <newline> + EOO +} + +: cleanup +: +{ + : always + : + $* <"cmd &file" >>EOO + 'cmd' + & + 'file' + <newline> + EOO + + : maybe + : + $* <"cmd &?file" >>EOO + 'cmd' + &? + 'file' + <newline> + EOO + + : never + : + $* <"cmd &!file" >>EOO + 'cmd' + &! + 'file' + <newline> + EOO +} + +: for +: +{ + : form-1 + : + $* <"for x: a" >>EOO + 'for' + 'x' + : + 'a' + <newline> + EOO + + : form-3 + : + $* <"for <<<a x" >>EOO + 'for' + <<< + 'a' + 'x' + <newline> + EOO +} diff --git a/libbuild2/build/script/lexer.cxx b/libbuild2/build/script/lexer.cxx index d849ac9..e0d87fe 100644 --- a/libbuild2/build/script/lexer.cxx +++ b/libbuild2/build/script/lexer.cxx @@ -35,10 +35,7 @@ namespace build2 bool q (true); // quotes if (!esc) - { - assert (!state_.empty ()); - esc = state_.top ().escapes; - } + esc = current_state ().escapes; switch (m) { @@ -78,6 +75,19 @@ namespace build2 s2 = " "; break; } + case lexer_mode::for_loop: + { + // Leading tokens of the for-loop. Like command_line but + // recognizes colon as a separator and lsbrace like value. + // + // Note that while sensing the form of the for-loop (`for x:...` + // vs `for x <...`) we need to make sure that the pre-parsed token + // types are valid for the execution phase. + // + s1 = ":=!|&<> $(#\t\n"; + s2 = " == "; + break; + } default: { // Recognize special variable names ($>, $<, $~). @@ -94,7 +104,7 @@ namespace build2 } assert (ps == '\0'); - state_.push ( + mode_impl ( state {m, data, nullopt, false, false, ps, s, n, q, *esc, s1, s2}); } @@ -103,12 +113,13 @@ namespace build2 { token r; - switch (state_.top ().mode) + switch (mode ()) { case lexer_mode::command_line: case lexer_mode::first_token: case lexer_mode::second_token: case lexer_mode::variable_line: + case lexer_mode::for_loop: r = next_line (); break; default: return base_lexer::next (); @@ -128,7 +139,7 @@ namespace build2 xchar c (get ()); uint64_t ln (c.line), cn (c.column); - state st (state_.top ()); // Make copy (see first/second_token). + state st (current_state ()); // Make copy (see first/second_token). lexer_mode m (st.mode); auto make_token = [&sep, ln, cn] (type t) @@ -141,9 +152,10 @@ namespace build2 // if (st.lsbrace) { - assert (m == lexer_mode::variable_line); + assert (m == lexer_mode::variable_line || + m == lexer_mode::for_loop); - state_.top ().lsbrace = false; // Note: st is a copy. + current_state ().lsbrace = false; // Note: st is a copy. if (c == '[' && (!st.lsbrace_unsep || !sep)) return make_token (type::lsbrace); @@ -156,7 +168,7 @@ namespace build2 // we push any new mode (e.g., double quote). // if (m == lexer_mode::first_token || m == lexer_mode::second_token) - state_.pop (); + expire_mode (); // NOTE: remember to update mode() if adding new special characters. @@ -167,7 +179,7 @@ namespace build2 // Expire variable value mode at the end of the line. // if (m == lexer_mode::variable_line) - state_.pop (); + expire_mode (); sep = true; // Treat newline as always separated. return make_token (type::newline); @@ -179,11 +191,20 @@ namespace build2 case '(': return make_token (type::lparen); } + if (m == lexer_mode::for_loop) + { + switch (c) + { + case ':': return make_token (type::colon); + } + } + // Command line operator/separators. // if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { switch (c) { @@ -205,7 +226,8 @@ namespace build2 // if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { if (optional<token> t = next_cmd_op (c, sep)) return move (*t); diff --git a/libbuild2/build/script/lexer.hxx b/libbuild2/build/script/lexer.hxx index 646d3b9..3f51493 100644 --- a/libbuild2/build/script/lexer.hxx +++ b/libbuild2/build/script/lexer.hxx @@ -24,9 +24,10 @@ namespace build2 enum { command_line = base_type::value_next, - first_token, // Expires at the end of the token. - second_token, // Expires at the end of the token. - variable_line // Expires at the end of the line. + first_token, // Expires at the end of the token. + second_token, // Expires at the end of the token. + variable_line, // Expires at the end of the line. + for_loop // Used for sensing the for-loop leading tokens. }; lexer_mode () = default; @@ -67,6 +68,8 @@ namespace build2 static redirect_aliases_type redirect_aliases; private: + using build2::script::lexer::mode; // Getter. + token next_line (); }; diff --git a/libbuild2/build/script/lexer.test.cxx b/libbuild2/build/script/lexer.test.cxx index e496f94..d8733ba 100644 --- a/libbuild2/build/script/lexer.test.cxx +++ b/libbuild2/build/script/lexer.test.cxx @@ -35,6 +35,7 @@ namespace build2 else if (s == "second-token") m = lexer_mode::second_token; else if (s == "variable-line") m = lexer_mode::variable_line; else if (s == "variable") m = lexer_mode::variable; + else if (s == "for-loop") m = lexer_mode::for_loop; else assert (false); } diff --git a/libbuild2/build/script/parser+command-if.test.testscript b/libbuild2/build/script/parser+command-if.test.testscript index a18a885..8b19186 100644 --- a/libbuild2/build/script/parser+command-if.test.testscript +++ b/libbuild2/build/script/parser+command-if.test.testscript @@ -279,7 +279,7 @@ cmd end EOI - buildfile:12:1: error: 'end' without preceding 'if' + buildfile:12:1: error: 'end' without preceding 'if', 'for', or 'while' EOE : before diff --git a/libbuild2/build/script/parser+command-re-parse.test.testscript b/libbuild2/build/script/parser+command-re-parse.test.testscript index 56e05b5..3dbdc16 100644 --- a/libbuild2/build/script/parser+command-re-parse.test.testscript +++ b/libbuild2/build/script/parser+command-re-parse.test.testscript @@ -1,18 +1,14 @@ # file : libbuild2/build/script/parser+command-re-parse.test.testscript # license : MIT; see accompanying LICENSE file -# @@ TMP -# -#\ : double-quote : $* <<EOI >>EOO -x = [cmd_line] cmd \">-\" "'<-'" +x = [cmdline] cmd \">-\" "'<-'" $x EOI cmd '>-' '<-' EOO -#\ : literal-re-parse : diff --git a/libbuild2/build/script/parser+diag.test.testscript b/libbuild2/build/script/parser+diag.test.testscript index 30eb859..504c9a4 100644 --- a/libbuild2/build/script/parser+diag.test.testscript +++ b/libbuild2/build/script/parser+diag.test.testscript @@ -19,17 +19,99 @@ $* <<EOI >>EOO name: echo EOO -: diag +: name-operation : -$* <<EOI >>~%EOO% - echo abc - cat abc - diag copy >= $> - cp <- $> +$* <<EOI >>EOO + a = 'b' EOI - %diag: copy >= .+file\{driver\.\}% + name: update EOO +: preamble +: +{ + : disambiguate + : + $* <<EOI >>~%EOO% + echo abc | set v + cat abc | set v + diag copy >= $> + cp <- $> + EOI + echo abc | set v + cat abc | set v + %diag: copy >= .+file\{driver\.\}% + EOO + + : name + : + $* <<EOI >>EOO + n = foo + diag copy $n + cp $n $> + EOI + diag: copy foo + EOO + + : quoted + : + $* <<EOI >'diag: foo' + f = foo + diag "$f" + EOI + + : quoted-eval + : + $* <<EOI >'diag: foo' + f = foo + diag "($f)" + EOI + + : temp_dir + : + { + test.options += -t + + : no + : + $* <<EOI >false + f = foo + diag $f + f = $~/f + foo "$f" + EOI + + : no-depdb + : + $* <<EOI >false + f = $~/f + depdb hash "$f" + diag $f + f = $~/f + foo "$f" + EOI + + : yes + : + $* <<EOI >true + f = $~/f + diag $f + foo $f + EOI + + : yes-depdb + : + $* <<EOI >true + f = $~/f + depdb hash "$f" + f = $~/t + diag $f + f = $~/f + foo "$f" + EOI + } +} + : ambiguity : { @@ -67,16 +149,6 @@ $* <<EOI >>~%EOO% info: consider specifying it explicitly with the 'diag' recipe attribute info: or provide custom low-verbosity diagnostics with the 'diag' builtin EOE - - : none - : - $* <<EOI 2>>EOE != 0 - a = 'b' - EOI - buildfile:11:1: error: unable to deduce low-verbosity script diagnostics name - info: consider specifying it explicitly with the 'diag' recipe attribute - info: or provide custom low-verbosity diagnostics with the 'diag' builtin - EOE } : inside-if diff --git a/libbuild2/build/script/parser+expansion.test.testscript b/libbuild2/build/script/parser+expansion.test.testscript index 086ec8f..eb99ae2 100644 --- a/libbuild2/build/script/parser+expansion.test.testscript +++ b/libbuild2/build/script/parser+expansion.test.testscript @@ -24,19 +24,15 @@ EOI buildfile:12:5: info: while parsing string 'xy'a bc' EOE -# @@ TMP -# -#\ : invalid-redirect : $* <<EOI 2>>EOE != 0 -x = [cmd_line] "1>&a" +x = [cmdline] "1>&a" cmd $x EOI <string>:1:4: error: stdout merge redirect file descriptor must be 2 buildfile:12:5: info: while parsing string '1>&a' EOE -#\ : expansion-re-parse : diff --git a/libbuild2/build/script/parser+for.test.testscript b/libbuild2/build/script/parser+for.test.testscript new file mode 100644 index 0000000..847b253 --- /dev/null +++ b/libbuild2/build/script/parser+for.test.testscript @@ -0,0 +1,656 @@ +# file : libbuild2/build/script/parser+for.test.testscript +# license : MIT; see accompanying LICENSE file + +: form-1 +: +: for x: ... +: +{ + : for + : + { + : no-var + : + $* <<EOI 2>>EOE != 0 + for + cmd + end + EOI + buildfile:11:1: error: for: missing variable name + EOE + + : untyped + : + $* <<EOI >>EOO + for x: a b + cmd $x + end + EOI + cmd a + cmd b + EOO + + : null + : + $* <<EOI >:'' + for x: [null] + cmd $x + end + EOI + + : empty + : + $* <<EOI >:'' + for x: + cmd $x + end + EOI + + : expansion + : + $* <<EOI >>EOO + vs = a b + for x: $vs + cmd $x + end + EOI + cmd a + cmd b + EOO + + : typed-values + : + $* <<EOI >>~%EOO% + for x: [dir_paths] a b + cmd $x + end + EOI + %cmd (a/|'a\\')% + %cmd (b/|'b\\')% + EOO + + : typed-elem + : + $* <<EOI >>~%EOO% + for x [dir_path]: a b + cmd $x + end + EOI + %cmd (a/|'a\\')% + %cmd (b/|'b\\')% + EOO + + : typed-elem-value + : + $* <<EOI >>~%EOO% + for x [dir_path]: [strings] a b + cmd $x + end + EOI + %cmd (a/|'a\\')% + %cmd (b/|'b\\')% + EOO + + : defined-var + : + $* <<EOI >>EOO + x = x + + for x: a b + cmd $x + end + + cmd $x + EOI + cmd a + cmd b + cmd b + EOO + } + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + for x: a b + cmd + EOI + buildfile:13:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + for x: a b + elif true + cmd + end + end + EOI + buildfile:12:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + for x: a b + cmd1 $x # 1 + if ($x == "a") # 2 + cmd2 # 3 + for y: x y + cmd3 # 4 + end + else + cmd4 # 5 + end + cmd5 # 6 + end + cmd6 # 7 + EOI + cmd1 a # 1 i1 + ? true # 2 i1 + cmd2 # 3 i1 + cmd3 # 4 i1 i1 + cmd3 # 4 i1 i2 + cmd5 # 6 i1 + cmd1 b # 1 i2 + ? false # 2 i2 + cmd4 # 5 i2 + cmd5 # 6 i2 + cmd6 # 7 + EOO + } + + : contained + : + { + : eos + : + $* <<EOI 2>>EOE != 0 + for x: + EOI + buildfile:12:1: error: expected closing 'end' + EOE + } +} + +: form-2 +: +: ... | for x +: +{ + : for + : + { + : status + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x != 0 + cmd + end + EOI + buildfile:11:20: error: for-loop exit code cannot be checked + EOE + + : not-last + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x | echo x + cmd + end + EOI + buildfile:11:20: error: for-loop must be last command in a pipe + EOE + + : not-last-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x|echo x + cmd + end + EOI + buildfile:11:19: error: for-loop must be last command in a pipe + EOE + + : expression-after + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x && echo x + cmd + end + EOI + buildfile:11:20: error: command expression involving for-loop + EOE + + : expression-after-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x&&echo x + cmd + end + EOI + buildfile:11:19: error: command expression involving for-loop + EOE + + : expression-before + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && echo x | for x + cmd + end + EOI + buildfile:11:24: error: command expression involving for-loop + EOE + + : expression-before-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && echo x|for x + cmd + end + EOI + buildfile:11:22: error: command expression involving for-loop + EOE + + : cleanup + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x &f + cmd + end + EOI + buildfile:11:20: error: cleanup in for-loop + EOE + + : cleanup-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x&f + cmd + end + EOI + buildfile:11:19: error: cleanup in for-loop + EOE + + : stdout-redirect + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x >a + cmd + end + EOI + buildfile:11:20: error: output redirect in for-loop + EOE + + : stdout-redirect-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x>a + cmd + end + EOI + buildfile:11:19: error: output redirect in for-loop + EOE + + : stdin-redirect + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x <a + cmd + end + EOI + buildfile:11:20: error: stdin is both piped and redirected + EOE + + : no-var + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for + cmd + end + EOI + buildfile:11:1: error: for: missing variable name + EOE + + : untyped + : + $* <<EOI >>EOO + echo 'a b' | for -w x + cmd $x + end + EOI + echo 'a b' | for -w x + EOO + + : expansion + : + $* <<EOI >>EOO + vs = a b + echo $vs | for x + cmd $x + end + EOI + echo a b | for x + EOO + + : typed-elem + : + $* <<EOI >>EOO + echo 'a b' | for -w x [dir_path] + cmd $x + end + EOI + echo 'a b' | for -w x [dir_path] + EOO + } + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + cmd + EOI + buildfile:13:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + elif true + cmd + end + end + EOI + buildfile:12:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + echo 'a b' | for x # 1 + cmd1 $x # 2 + if ($x == "a") # 3 + cmd2 # 4 + echo x y | for y # 5 + cmd3 # 6 + end + else + cmd4 # 7 + end + cmd5 # 8 + end + cmd6 # 9 + EOI + echo 'a b' | for x # 1 + cmd6 # 9 + EOO + } +} + +: form-3 +: +: for x <... +: +{ + : for + : + { + : status + : + $* <<EOI 2>>EOE != 0 + for x <a != 0 + cmd + end + EOI + buildfile:11:10: error: for-loop exit code cannot be checked + EOE + + : not-last + : + $* <<EOI 2>>EOE != 0 + for x <a | echo x + cmd + end + EOI + buildfile:11:10: error: for-loop must be last command in a pipe + EOE + + : not-last-relex + : + $* <<EOI 2>>EOE != 0 + for <a x|echo x + cmd + end + EOI + buildfile:11:9: error: for-loop must be last command in a pipe + EOE + + : expression-after + : + $* <<EOI 2>>EOE != 0 + for x <a && echo x + cmd + end + EOI + buildfile:11:10: error: command expression involving for-loop + EOE + + : expression-after-relex + : + $* <<EOI 2>>EOE != 0 + for <a x&&echo x + cmd + end + EOI + buildfile:11:9: error: command expression involving for-loop + EOE + + : expression-before + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && for x <a + cmd + end + EOI + buildfile:11:15: error: command expression involving for-loop + EOE + + : cleanup + : + $* <<EOI 2>>EOE != 0 + for x <a &f + cmd + end + EOI + buildfile:11:10: error: cleanup in for-loop + EOE + + : cleanup-before-var + : + $* <<EOI 2>>EOE != 0 + for &f x <a + cmd + end + EOI + buildfile:11:5: error: cleanup in for-loop + EOE + + : cleanup-relex + : + $* <<EOI 2>>EOE != 0 + for <a x&f + cmd + end + EOI + buildfile:11:9: error: cleanup in for-loop + EOE + + : stdout-redirect + : + $* <<EOI 2>>EOE != 0 + for x >a + cmd + end + EOI + buildfile:11:7: error: output redirect in for-loop + EOE + + : stdout-redirect-before-var + : + $* <<EOI 2>>EOE != 0 + for >a x + cmd + end + EOI + buildfile:11:5: error: output redirect in for-loop + EOE + + : stdout-redirect-relex + : + $* <<EOI 2>>EOE != 0 + for x>a + cmd + end + EOI + buildfile:11:6: error: output redirect in for-loop + EOE + + : no-var + : + $* <<EOI 2>>EOE != 0 + for <a + cmd + end + EOI + buildfile:11:1: error: for: missing variable name + EOE + + : quoted-opt + : + $* <<EOI >>EOO + o = -w + for "$o" x <'a b' + cmd $x + end + for "($o)" x <'a b' + cmd $x + end + EOI + for -w x <'a b' + for -w x <'a b' + EOO + + : untyped + : + $* <<EOI >>EOO + for -w x <'a b' + cmd $x + end + EOI + for -w x <'a b' + EOO + + : expansion + : + $* <<EOI >>EOO + vs = a b + for x <$vs + cmd $x + end + EOI + for x b <a + EOO + + : typed-elem + : + $* <<EOI >>EOO + for -w x [dir_path] <'a b' + cmd $x + end + EOI + for -w x [dir_path] <'a b' + EOO + } + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + cmd + EOI + buildfile:13:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + elif true + cmd + end + end + EOI + buildfile:12:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + for -w x <'a b' # 1 + cmd1 $x # 2 + if ($x == "a") # 3 + cmd2 # 4 + for -w y <'x y' # 5 + cmd3 # 6 + end + else + cmd4 # 7 + end + cmd5 # 8 + end + cmd6 # 9 + EOI + for -w x <'a b' # 1 + cmd6 # 9 + EOO + } + + : contained + : + { + : eos + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + EOI + buildfile:12:1: error: expected closing 'end' + EOE + } +} diff --git a/libbuild2/build/script/parser+while.test.testscript b/libbuild2/build/script/parser+while.test.testscript new file mode 100644 index 0000000..5587291 --- /dev/null +++ b/libbuild2/build/script/parser+while.test.testscript @@ -0,0 +1,133 @@ +# file : libbuild2/build/script/parser+while.test.testscript +# license : MIT; see accompanying LICENSE file + +: while +: +{ + : true + : + $* <<EOI >>EOO + while ($v != "aa") + cmd "$v" + v = "$(v)a" + end + EOI + ? true + cmd '' + ? true + cmd a + ? false + EOO + + : false + : + $* <<EOI >>EOO + while ($v == "aa") + cmd "$v" + v = "$(v)a" + end + EOI + ? false + EOO + + : without-command + : + $* <<EOI 2>>EOE != 0 + while + cmd + end + EOI + buildfile:11:6: error: missing program + EOE +} + +: end +: +{ + : without-end + : + $* <<EOI 2>>EOE != 0 + while true + cmd + EOI + buildfile:13:1: error: expected closing 'end' + EOE +} + +: elif +: +{ + : without-if + : + $* <<EOI 2>>EOE != 0 + while false + elif true + cmd + end + end + EOI + buildfile:12:3: error: 'elif' without preceding 'if' + EOE +} + +: nested +: +{ + $* -l -r <<EOI >>EOO + while ($v != "aa") # 1 + cmd1 "$v" # 2 + if ($v == "a") # 3 + cmd2 # 4 + while ($v2 != "$v") # 5 + cmd3 # 6 + v2=$v + end + else + cmd4 # 7 + end + cmd5 # 8 + v = "$(v)a" + end + EOI + ? true # 1 i1 + cmd1 '' # 2 i1 + ? false # 3 i1 + cmd4 # 7 i1 + cmd5 # 8 i1 + ? true # 1 i2 + cmd1 a # 2 i2 + ? true # 3 i2 + cmd2 # 4 i2 + ? true # 5 i2 i1 + cmd3 # 6 i2 i1 + ? false # 5 i2 i2 + cmd5 # 8 i2 + ? false # 1 i3 + EOO +} + +: contained +: +{ + : eos + : + $* <<EOI 2>>EOE != 0 + while + EOI + buildfile:12:1: error: expected closing 'end' + EOE +} + +: var +: +$* <<EOI >>EOO +while ($v1 != "a") + v1 = "$(v1)a" + v2 = "$v1" +end +cmd $v1 +EOI +? true +? false +cmd a +EOO diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index 217fa11..3ecf23d 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -3,13 +3,25 @@ #include <libbuild2/build/script/parser.hxx> +#include <cstring> // strcmp() +#include <sstream> + #include <libbutl/builtin.hxx> +#include <libbutl/path-pattern.hxx> +#include <libbuild2/depdb.hxx> +#include <libbuild2/dyndep.hxx> #include <libbuild2/function.hxx> #include <libbuild2/algorithm.hxx> +#include <libbuild2/make-parser.hxx> + +#include <libbuild2/adhoc-rule-buildscript.hxx> + +#include <libbuild2/script/run.hxx> #include <libbuild2/build/script/lexer.hxx> #include <libbuild2/build/script/runner.hxx> +#include <libbuild2/build/script/builtin-options.hxx> using namespace std; using namespace butl; @@ -35,7 +47,7 @@ namespace build2 { path_ = &pn; - pre_parse_ = true; + top_pre_parse_ = pre_parse_ = true; lexer l (is, *path_, line, lexer_mode::command_line); set_lexer (&l); @@ -49,7 +61,7 @@ namespace build2 pbase_ = scope_->src_path_; - file_based_ = tt.is_a<file> (); + file_based_ = tt.is_a<file> () || tt.is_a<group> (); perform_update_ = find (as.begin (), as.end (), perform_update_id) != as.end (); @@ -81,15 +93,31 @@ namespace build2 info << "consider using 'depdb' builtin to track its result " << "changes"; - // Diagnose absent/ambigous script name. + // Diagnose computed variable exansions. + // + if (computed_var_) + fail (*computed_var_) + << "expansion of computed variable is only allowed in depdb " + << "preamble" << + info << "consider using 'depdb' builtin to track its value " + << "changes"; + + // Diagnose absent/ambiguous script name. But try to deduce an absent + // name from the script operation first. // { diag_record dr; - if (!diag_name_ && !diag_line_) + if (!diag_name_ && diag_preamble_.empty ()) { - dr << fail (s.start_loc) - << "unable to deduce low-verbosity script diagnostics name"; + if (as.size () == 1) + { + diag_name_ = make_pair (ctx->operation_table[as[0].operation ()], + location ()); + } + else + dr << fail (s.start_loc) + << "unable to deduce low-verbosity script diagnostics name"; } else if (diag_name2_) { @@ -115,16 +143,23 @@ namespace build2 // Save the script name or custom diagnostics line. // - assert (diag_name_.has_value () != diag_line_.has_value ()); + assert (diag_name_.has_value () == diag_preamble_.empty ()); if (diag_name_) s.diag_name = move (diag_name_->first); else - s.diag_line = move (diag_line_->first); + s.diag_preamble = move (diag_preamble_); // Save the custom dependency change tracking lines, if present. // s.depdb_clear = depdb_clear_.has_value (); + s.depdb_value = depdb_value_; + if (depdb_dyndep_) + { + s.depdb_dyndep = depdb_dyndep_->second; + s.depdb_dyndep_byproduct = depdb_dyndep_byproduct_; + s.depdb_dyndep_dyn_target = depdb_dyndep_dyn_target_; + } s.depdb_preamble = move (depdb_preamble_); return s; @@ -166,13 +201,27 @@ namespace build2 } } + // Parse a logical line, handling the flow control constructs + // recursively. + // + // If the flow control construct type is specified, then this line is + // assumed to belong to such a construct. + // void parser:: - pre_parse_line (token& t, type& tt, bool if_line) + pre_parse_line (token& t, type& tt, optional<line_type> fct) { + // enter: next token is peeked at (type in tt) + // leave: newline + + assert (!fct || + *fct == line_type::cmd_if || + *fct == line_type::cmd_while || + *fct == line_type::cmd_for_stream || + *fct == line_type::cmd_for_args); + // Determine the line type/start token. // - line_type lt ( - pre_parse_line_start (t, tt, lexer_mode::second_token)); + line_type lt (pre_parse_line_start (t, tt, lexer_mode::second_token)); line ln; @@ -205,22 +254,148 @@ namespace build2 break; } + // + // See pre_parse_line_start() for details. + // + case line_type::cmd_for_args: assert (false); break; + case line_type::cmd_for_stream: + { + // First we need to sense the next few tokens and detect which + // form of the loop we are dealing with, the first (for x: ...) + // or the third (x <...) one. Note that the second form (... | for + // x) is handled separately. + // + // If the next token doesn't look like a variable name, then this + // is the third form. Otherwise, if colon follows the variable + // name, potentially after the attributes, then this is the first + // form and the third form otherwise. + // + // Note that for the third form we will need to pass the 'for' + // token as a program name to the command expression parsing + // function since it will be gone from the token stream by that + // time. Thus, we save it. We also need to make sure the sensing + // always leaves the variable name token in t/tt. + // + // Note also that in this model it won't be possible to support + // options in the first form. + // + token pt (t); + assert (pt.type == type::word && pt.value == "for"); + + mode (lexer_mode::for_loop); + next (t, tt); + + // Note that we also consider special variable names (those that + // don't clash with the command line elements like redirects, etc) + // to later fail gracefully. + // + string& n (t.value); + + if (tt == type::word && t.qtype == quote_type::unquoted && + (n[0] == '_' || alpha (n[0]) || // Variable. + n == "~")) // Special variable. + { + // Detect patterns analogous to parse_variable_name() (so we + // diagnose `for x[string]: ...`). + // + if (n.find_first_of ("[*?") != string::npos) + fail (t) << "expected variable name instead of " << n; + + if (special_variable (n)) + fail (t) << "attempt to set '" << n << "' special variable"; + + // Parse out the element attributes, if present. + // + if (lexer_->peek_char ().first == '[') + { + // Save the variable name token before the attributes parsing + // and restore it afterwards. Also make sure that the token + // which follows the attributes stays in the stream. + // + token vt (move (t)); + next_with_attributes (t, tt); + + attributes_push (t, tt, + true /* standalone */, + false /* next_token */); + + t = move (vt); + tt = t.type; + } + + if (lexer_->peek_char ().first == ':') + lt = line_type::cmd_for_args; + } + + if (lt == line_type::cmd_for_stream) // for x <... + { + // At this point t/tt contains the variable name token. Now + // pre-parse the command expression in the command_line lexer + // mode starting from this position and also passing the 'for' + // token as a program name. + // + // Note that the fact that the potential attributes are already + // parsed doesn't affect the command expression pre-parsing. + // Also note that they will be available during the execution + // phase being replayed. + // + expire_mode (); // Expire the for-loop lexer mode. + + parse_command_expr_result r ( + parse_command_expr (t, tt, + lexer::redirect_aliases, + move (pt))); + + assert (r.for_loop); + + if (tt != type::newline) + fail (t) << "expected newline instead of " << t; + + parse_here_documents (t, tt, r); + } + else // for x: ... + { + next (t, tt); + + assert (tt == type::colon); + + expire_mode (); // Expire the for-loop lexer mode. + + // Parse the value similar to the var line type (see above). + // + mode (lexer_mode::variable_line); + parse_variable_line (t, tt); + + if (tt != type::newline) + fail (t) << "expected newline instead of " << t << " after for"; + } + + ln.var = nullptr; + ++level_; + break; + } case line_type::cmd_elif: case line_type::cmd_elifn: case line_type::cmd_else: - case line_type::cmd_end: { - if (!if_line) - { + if (!fct || *fct != line_type::cmd_if) fail (t) << lt << " without preceding 'if'"; - } + } + // Fall through. + case line_type::cmd_end: + { + if (!fct) + fail (t) << lt << " without preceding 'if', 'for', or 'while'"; } // Fall through. case line_type::cmd_if: case line_type::cmd_ifn: + case line_type::cmd_while: next (t, tt); // Skip to start of command. - if (lt == line_type::cmd_if || lt == line_type::cmd_ifn) + if (lt == line_type::cmd_if || + lt == line_type::cmd_ifn || + lt == line_type::cmd_while) ++level_; else if (lt == line_type::cmd_end) --level_; @@ -228,15 +403,24 @@ namespace build2 // Fall through. case line_type::cmd: { - pair<command_expr, here_docs> p; + parse_command_expr_result r; if (lt != line_type::cmd_else && lt != line_type::cmd_end) - p = parse_command_expr (t, tt, lexer::redirect_aliases); + r = parse_command_expr (t, tt, lexer::redirect_aliases); + + if (r.for_loop) + { + lt = line_type::cmd_for_stream; + ln.var = nullptr; + + ++level_; + } if (tt != type::newline) fail (t) << "expected newline instead of " << t; - parse_here_documents (t, tt, p); + parse_here_documents (t, tt, r); + break; } } @@ -254,22 +438,76 @@ namespace build2 *save_line_ = move (ln); } - if (lt == line_type::cmd_if || lt == line_type::cmd_ifn) + switch (lt) { - tt = peek (lexer_mode::first_token); + case line_type::cmd_if: + case line_type::cmd_ifn: + { + tt = peek (lexer_mode::first_token); - pre_parse_if_else (t, tt); + pre_parse_if_else (t, tt); + break; + } + case line_type::cmd_while: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: + { + tt = peek (lexer_mode::first_token); + + pre_parse_loop (t, tt, lt); + break; + } + default: break; } } + // Pre-parse the flow control construct block line. + // + void parser:: + pre_parse_block_line (token& t, type& tt, line_type bt) + { + // enter: peeked first token of the line (type in tt) + // leave: newline + + const location ll (get_location (peeked ())); + + if (tt == type::eos) + fail (ll) << "expected closing 'end'"; + + line_type fct; // Flow control type the block type relates to. + + switch (bt) + { + case line_type::cmd_if: + case line_type::cmd_ifn: + case line_type::cmd_elif: + case line_type::cmd_elifn: + case line_type::cmd_else: + { + fct = line_type::cmd_if; + break; + } + case line_type::cmd_while: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: + { + fct = bt; + break; + } + default: assert(false); + } + + pre_parse_line (t, tt, fct); + assert (tt == type::newline); + } + void parser:: pre_parse_if_else (token& t, type& tt) { // enter: peeked first token of next line (type in tt) // leave: newline - // Parse lines until we see closing 'end'. Nested if-else blocks are - // handled recursively. + // Parse lines until we see closing 'end'. // for (line_type bt (line_type::cmd_if); // Current block. ; @@ -277,25 +515,21 @@ namespace build2 { const location ll (get_location (peeked ())); - if (tt == type::eos) - fail (ll) << "expected closing 'end'"; - // Parse one line. Note that this one line can still be multiple - // lines in case of if-else. In this case we want to view it as - // cmd_if, not cmd_end. Thus remember the start position of the - // next logical line. + // lines in case of a flow control construct. In this case we want + // to view it as cmd_if, not cmd_end. Thus remember the start + // position of the next logical line. // size_t i (script_->body.size ()); - pre_parse_line (t, tt, true /* if_line */); - assert (tt == type::newline); + pre_parse_block_line (t, tt, bt); line_type lt (script_->body[i].type); // First take care of 'end'. // if (lt == line_type::cmd_end) - return; + break; // Check if-else block sequencing. // @@ -319,6 +553,29 @@ namespace build2 } } + void parser:: + pre_parse_loop (token& t, type& tt, line_type lt) + { + // enter: peeked first token of next line (type in tt) + // leave: newline + + assert (lt == line_type::cmd_while || + lt == line_type::cmd_for_stream || + lt == line_type::cmd_for_args); + + // Parse lines until we see closing 'end'. + // + for (;; tt = peek (lexer_mode::first_token)) + { + size_t i (script_->body.size ()); + + pre_parse_block_line (t, tt, lt); + + if (script_->body[i].type == line_type::cmd_end) + break; + } + } + command_expr parser:: parse_command_line (token& t, type& tt) { @@ -329,12 +586,12 @@ namespace build2 // assert (!pre_parse_); - pair<command_expr, here_docs> p ( + parse_command_expr_result pr ( parse_command_expr (t, tt, lexer::redirect_aliases)); assert (tt == type::newline); - parse_here_documents (t, tt, p); + parse_here_documents (t, tt, pr); assert (tt == type::newline); // @@ Note that currently running programs via a runner (e.g., see @@ -347,7 +604,7 @@ namespace build2 // passed to the environment constructor, similar to passing the // script deadline. // - return move (p.first); + return move (pr.expr); } // @@ -408,6 +665,12 @@ namespace build2 fail (l) << "'" << v << "' call via 'env' builtin"; }; + auto diag_loc = [this] () + { + assert (!diag_preamble_.empty ()); + return diag_preamble_.back ().tokens[0].location (); + }; + if (v == "diag") { verify (); @@ -424,24 +687,41 @@ namespace build2 } else // Custom diagnostics. { - assert (diag_line_); - fail (l) << "multiple 'diag' builtin calls" << - info (diag_line_->second) << "previous call is here"; + info (diag_loc ()) << "previous call is here"; } } - // Instruct the parser to save the diag builtin line separately - // from the script lines, when it is fully parsed. Note that it - // will be executed prior to the script body execution to obtain - // the custom diagnostics. + // Move the script body to the end of the diag preamble. // - diag_line_ = make_pair (line (), l); - save_line_ = &diag_line_->first; - diag_weight_ = 4; + // Note that we move into the preamble whatever is there and delay + // the check until the execution (see the depdb preamble + // collecting for the reasoning). + // + lines& ls (script_->body); + diag_preamble_.insert (diag_preamble_.end (), + make_move_iterator (ls.begin ()), + make_move_iterator (ls.end ())); + ls.clear (); + + // Also move the body_temp_dir flag, if it is true. + // + if (script_->body_temp_dir) + { + script_->diag_preamble_temp_dir = true; + script_->body_temp_dir = false; + } - diag_name_ = nullopt; - diag_name2_ = nullopt; + // Similar to the depdb preamble collection, instruct the parser + // to save the depdb builtin line separately from the script + // lines. + // + diag_preamble_.push_back (line ()); + save_line_ = &diag_preamble_.back (); + + diag_weight_ = 4; + diag_name_ = nullopt; + diag_name2_ = nullopt; // Note that the rest of the line contains the builtin argument to // be printed, thus we parse it in the value lexer mode. @@ -463,17 +743,16 @@ namespace build2 { if (a != perform_update_id) fail (l) << "'depdb' builtin cannot be used to " - << ctx.meta_operation_table[a.meta_operation ()].name - << ' ' << ctx.operation_table[a.operation ()]; + << ctx->meta_operation_table[a.meta_operation ()].name + << ' ' << ctx->operation_table[a.operation ()]; } if (!file_based_) - fail (l) << "'depdb' builtin can only be used for file-based " - << "targets"; + fail (l) << "'depdb' builtin can only be used for file- or " + << "file group-based targets"; - if (diag_line_) - fail (diag_line_->second) - << "'diag' builtin call before 'depdb' call" << + if (!diag_preamble_.empty ()) + fail (diag_loc ()) << "'diag' builtin call before 'depdb' call" << info (l) << "'depdb' call is here"; // Note that the rest of the line contains the builtin command @@ -487,7 +766,11 @@ namespace build2 next (t, tt); if (tt != type::word || - (v != "clear" && v != "hash" && v != "string" && v != "env")) + (v != "clear" && + v != "hash" && + v != "string" && + v != "env" && + v != "dyndep")) { fail (get_location (t)) << "expected 'depdb' builtin command instead of " << t; @@ -527,12 +810,49 @@ namespace build2 // the referenced variable list, since it won't be used. // depdb_clear_ = l; - save_line_ = nullptr; + save_line_ = nullptr; script_->vars.clear (); } else { + // Verify depdb-dyndep is last and detect the byproduct flavor. + // + if (v == "dyndep") + { + // Note that for now we do not allow multiple dyndep calls. + // But we may wan to relax this later (though alternating + // targets with prerequisites in depdb may be tricky -- maybe + // still only allow additional targets in the first call). + // + if (!depdb_dyndep_) + depdb_dyndep_ = make_pair (l, depdb_preamble_.size ()); + else + fail (l) << "multiple 'depdb dyndep' calls" << + info (depdb_dyndep_->first) << "previous call is here"; + + if (peek () == type::word) + { + const string& v (peeked ().value); + + // Note: --byproduct and --dyn-target are mutually + // exclusive. + // + if (v == "--byproduct") + depdb_dyndep_byproduct_ = true; + else if (v == "--dyn-target") + depdb_dyndep_dyn_target_ = true; + } + } + else + { + if (depdb_dyndep_) + fail (l) << "'depdb " << v << "' after 'depdb dyndep'" << + info (depdb_dyndep_->first) << "'depdb dyndep' call is here"; + } + + depdb_value_ = depdb_value_ || (v == "string" || v == "hash"); + // Move the script body to the end of the depdb preamble. // // Note that at this (pre-parsing) stage we cannot evaluate if @@ -557,10 +877,12 @@ namespace build2 script_->body_temp_dir = false; } - // Reset the impure function call info since it's valid for the - // depdb preamble. + // Reset the impure function call and computed variable + // expansion tracking since both are valid for the depdb + // preamble. // impure_func_ = nullopt; + computed_var_ = nullopt; // Instruct the parser to save the depdb builtin line separately // from the script lines, when it is fully parsed. Note that the @@ -612,10 +934,53 @@ namespace build2 // // This is also the reason why we add a diag frame. // + // The problem turned out to be worse than originally thought: we + // may call a function (for example, as part of if) with invalid + // arguments. And this could happen in the depdb preamble, which + // means we cannot fix this by moving the depdb builtin (which must + // come after the preamble). So let's peek at what's ahead and omit + // the expansion if it's anything iffy, namely, eval context or + // function call. + // + bool skip_diag (false); if (pre_parse_ && diag_weight_ != 4) { - pre_parse_ = false; // Make parse_names() perform expansions. - pre_parse_suspended_ = true; + // Based on the buildfile expansion parsing logic. + // + if (tt == type::lparen) // Evaluation context. + skip_diag = true; + else if (tt == type::dollar) + { + type ptt (peek (lexer_mode::variable)); + + if (!peeked ().separated) + { + if (ptt == type::lparen) + { + // While strictly speaking this can also be a function call, + // this is highly unusual and we will assume it's a variable + // expansion. + } + else if (ptt == type::word) + { + pair<char, bool> r (lexer_->peek_char ()); + + if (r.first == '(' && !r.second) // Function call. + skip_diag = true; + } + } + } + + if (!skip_diag) + { + // Sanity check: we should not be suspending the pre-parse mode + // turned on by the base parser. + // + assert (top_pre_parse_); + + pre_parse_ = false; // Make parse_names() perform expansions. + pre_parse_suspended_ = true; + } } auto df = make_diag_frame ( @@ -643,7 +1008,7 @@ namespace build2 pre_parse_ = true; } - if (pre_parse_ && diag_weight_ == 4) + if (pre_parse_ && (diag_weight_ == 4 || skip_diag)) return nullopt; } @@ -665,6 +1030,19 @@ namespace build2 return nullopt; } + // If this is a value of the special cmdline type, then only do + // certain tests below if the value is not quoted and doesn't contain + // any characters that would be consumed by re-lexing. + // + // This is somewhat of a hack but handling this properly would not + // only require unquoting but also keeping track of which special + // characters were quoted (and thus should be treated literally) and + // which were not (and thus should act as separators, etc). + // + bool qs (pr.type != nullptr && + pr.type->is_a<cmdline> () && + need_cmdline_relex (ns[0].value)); + // We have to handle process_path[_ex] and executable target. The // process_path[_ex] we may have to recognize syntactically because // of the loss of type, for example: @@ -698,10 +1076,14 @@ namespace build2 pp_vt = pr.type; ns.clear (); } - else if (ns[0].file ()) + else if (ns[0].file () && !qs) { // Find the end of the value. // + // Note that here we ignore the whole cmdline issue (see above) + // for the further values assuming that they are unquoted and + // don't contain any special characters. + // auto b (ns.begin ()); auto i (value_traits<process_path_ex>::find_end (ns)); @@ -768,40 +1150,45 @@ namespace build2 // else if (!ns[0].simple ()) { - if (const target* t = search_existing ( - ns[0], *scope_, ns[0].pair ? ns[1].dir : empty_dir_path)) + if (!qs) { - if (const auto* et = t->is_a<exe> ()) + // This could be a script from src so search like a prerequisite. + // + if (const target* t = search_existing ( + ns[0], *scope_, ns[0].pair ? ns[1].dir : empty_dir_path)) { - if (pre_parse_) + if (const auto* et = t->is_a<exe> ()) { - if (auto* n = et->lookup_metadata<string> ("name")) + if (pre_parse_) { - set_diag (*n, 3); - return nullopt; + if (auto* n = et->lookup_metadata<string> ("name")) + { + set_diag (*n, 3); + return nullopt; + } + // Fall through. } - // Fall through. - } - else - { - process_path pp (et->process_path ()); + else + { + process_path pp (et->process_path ()); - if (pp.empty ()) - fail (l) << "target " << *et << " is out of date" << - info << "consider specifying it as a prerequisite of " - << environment_->target; + if (pp.empty ()) + fail (l) << "target " << *et << " is out of date" << + info << "consider specifying it as a prerequisite of " + << environment_->target; - ns.erase (ns.begin (), ns.begin () + (ns[0].pair ? 2 : 1)); - return optional<process_path> (move (pp)); + ns.erase (ns.begin (), ns.begin () + (ns[0].pair ? 2 : 1)); + return optional<process_path> (move (pp)); + } } - } - if (pre_parse_) - { - diag_record dr (fail (l)); - dr << "unable to deduce low-verbosity script diagnostics name " - << "from target " << *t; - suggest_diag (dr); + if (pre_parse_) + { + diag_record dr (fail (l)); + dr << "unable to deduce low-verbosity script diagnostics name " + << "from target " << *t; + suggest_diag (dr); + } } } @@ -819,26 +1206,29 @@ namespace build2 { // If we are here, the name is simple and is not part of a pair. // - string& v (ns[0].value); + if (!qs) + { + string& v (ns[0].value); - // Try to interpret the name as a builtin. - // - const builtin_info* bi (builtins.find (v)); + // Try to interpret the name as a builtin. + // + const builtin_info* bi (builtins.find (v)); - if (bi != nullptr) - { - set_diag (move (v), bi->weight); - return nullopt; - } - // - // Try to interpret the name as a pseudo-builtin. - // - // Note that both of them has the zero weight and cannot be picked - // up as a script name. - // - else if (v == "set" || v == "exit") - { - return nullopt; + if (bi != nullptr) + { + set_diag (move (v), bi->weight); + return nullopt; + } + // + // Try to interpret the name as a pseudo-builtin. + // + // Note that both of them has the zero weight and cannot be picked + // up as a script name. + // + else if (v == "set" || v == "exit") + { + return nullopt; + } } diag_record dr (fail (l)); @@ -863,8 +1253,9 @@ namespace build2 // Note that we rely on "small function object" optimization here. // auto exec_cmd = [this] (token& t, build2::script::token_type& tt, - size_t li, + const iteration_index* ii, size_t li, bool single, + const function<command_function>& cf, const location& ll) { // We use the 0 index to signal that this is the only command. @@ -875,7 +1266,7 @@ namespace build2 command_expr ce ( parse_command_line (t, static_cast<token_type&> (tt))); - runner_->run (*environment_, ce, li, ll); + runner_->run (*environment_, ce, ii, li, cf, ll); }; exec_lines (s.body, exec_cmd); @@ -884,129 +1275,189 @@ namespace build2 runner_->leave (e, s.end_loc); } + // Return true if the specified expression executes the set builtin or + // is a for-loop. + // + static bool + valid_preamble_cmd (const command_expr& ce, + const function<command_function>& cf) + { + return find_if ( + ce.begin (), ce.end (), + [&cf] (const expr_term& et) + { + const process_path& p (et.pipe.back ().program); + return p.initial == nullptr && + (p.recall.string () == "set" || + (cf != nullptr && p.recall.string () == "for")); + }) != ce.end (); + } + void parser:: - execute_depdb_preamble (const scope& rs, const scope& bs, - environment& e, const script& s, runner& r, - depdb& dd) + exec_depdb_preamble (action a, const scope& bs, const target& t, + environment& e, const script& s, runner& r, + lines_iterator begin, lines_iterator end, + depdb& dd, + dynamic_targets* dyn_targets, + bool* update, + optional<timestamp> mt, + bool* deferred_failure, + dyndep_byproduct* byp) { - tracer trace ("execute_depdb_preamble"); + tracer trace ("exec_depdb_preamble"); // The only valid lines in the depdb preamble are the depdb builtin // itself as well as the variable assignments, including via the set // builtin. - pre_exec (rs, bs, e, &s, &r); + pre_exec (*bs.root_scope (), bs, e, &s, &r); // Let's "wrap up" the objects we operate upon into the single object // to rely on "small function object" optimization. // struct { + tracer& trace; + + action a; + const scope& bs; + const target& t; + environment& env; const script& scr; + depdb& dd; - tracer& trace; - } ctx {e, s, dd, trace}; - - auto exec_cmd = [&ctx, this] - (token& t, - build2::script::token_type& tt, - size_t li, - bool /* single */, - const location& ll) + dynamic_targets* dyn_targets; + bool* update; + bool* deferred_failure; + optional<timestamp> mt; + dyndep_byproduct* byp; + + } data { + trace, + a, bs, t, + e, s, + dd, dyn_targets, update, deferred_failure, mt, byp}; + + auto exec_cmd = [this, &data] (token& t, + build2::script::token_type& tt, + const iteration_index* ii, size_t li, + bool /* single */, + const function<command_function>& cf, + const location& ll) { + // Note that we never reset the line index to zero (as we do in + // execute_body()) assuming that there are some script body commands + // to follow. + // if (tt == type::word && t.value == "depdb") { - names ns (exec_special (t, tt)); + next (t, tt); // This should have been enforced during pre-parsing. // - assert (!ns.empty ()); // <cmd> ... <newline> + assert (tt == type::word); // <cmd> ... <newline> - const string& cmd (ns[0].value); + string cmd (move (t.value)); - if (cmd == "hash") + if (cmd == "dyndep") { - sha256 cs; - for (auto i (ns.begin () + 1); i != ns.end (); ++i) // Skip <cmd>. - to_checksum (cs, *i); - - if (ctx.dd.expect (cs.string ()) != nullptr) - l4 ([&] { - ctx.trace (ll) - << "'depdb hash' argument change forcing update of " - << ctx.env.target;}); + // Note: the cast is safe since the part where the target is + // modified is always executed in apply(). + // + exec_depdb_dyndep (t, tt, + li, ll, + data.a, data.bs, const_cast<target&> (data.t), + data.dd, + *data.dyn_targets, + *data.update, + *data.mt, + *data.deferred_failure, + data.byp); } - else if (cmd == "string") + else { - string s; - try - { - s = convert<string> ( - names (make_move_iterator (ns.begin () + 1), - make_move_iterator (ns.end ()))); - } - catch (const invalid_argument& e) - { - fail (ll) << "invalid 'depdb string' argument: " << e; - } + names ns (exec_special (t, tt, true /* skip <cmd> */)); - if (ctx.dd.expect (s) != nullptr) - l4 ([&] { - ctx.trace (ll) - << "'depdb string' argument change forcing update of " - << ctx.env.target;}); - } - else if (cmd == "env") - { - sha256 cs; - const char* pf ("invalid 'depdb env' argument: "); + string v; + const char* w (nullptr); + if (cmd == "hash") + { + sha256 cs; + for (const name& n: ns) + to_checksum (cs, n); - try + v = cs.string (); + w = "argument"; + } + else if (cmd == "string") { - // Skip <cmd>. - // - for (auto i (ns.begin () + 1); i != ns.end (); ++i) + try { - string vn (convert<string> (move (*i))); - build2::script::verify_environment_var_name (vn, pf, ll); - hash_environment (cs, vn); + v = convert<string> (move (ns)); } + catch (const invalid_argument& e) + { + fail (ll) << "invalid 'depdb string' argument: " << e; + } + + w = "argument"; } - catch (const invalid_argument& e) + else if (cmd == "env") { - fail (ll) << pf << e; + sha256 cs; + const char* pf ("invalid 'depdb env' argument: "); + + try + { + for (name& n: ns) + { + string vn (convert<string> (move (n))); + build2::script::verify_environment_var_name (vn, pf, ll); + hash_environment (cs, vn); + } + } + catch (const invalid_argument& e) + { + fail (ll) << pf << e; + } + + v = cs.string (); + w = "environment"; } + else + assert (false); - if (ctx.dd.expect (cs.string ()) != nullptr) + // Prefix the value with the type letter. This serves two + // purposes: + // + // 1. It makes sure the result is never a blank line. We use + // blank lines as anchors to skip directly to certain entries + // (e.g., dynamic targets). + // + // 2. It allows us to detect the beginning of prerequisites + // since an absolute path will be distinguishable from these + // entries (in the future we may want to add an explicit + // blank after such custom entries to make this easier). + // + v.insert (0, 1, ' '); + v.insert (0, 1, cmd[0]); // `h`, `s`, or `e` + + if (data.dd.expect (v) != nullptr) l4 ([&] { - ctx.trace (ll) - << "'depdb env' environment change forcing update of " - << ctx.env.target;}); + data.trace (ll) + << "'depdb " << cmd << "' " << w << " change forcing " + << "update of " << data.t;}); } - else - assert (false); } else { - // Note that we don't reset the line index to zero (as we do in - // execute_body()) assuming that there are some script body - // commands to follow. - // command_expr ce ( parse_command_line (t, static_cast<token_type&> (tt))); - // Verify that this expression executes the set builtin. - // - if (find_if (ce.begin (), ce.end (), - [] (const expr_term& et) - { - const process_path& p (et.pipe.back ().program); - return p.initial == nullptr && - p.recall.string () == "set"; - }) == ce.end ()) + if (!valid_preamble_cmd (ce, cf)) { - const replay_tokens& rt (ctx.scr.depdb_preamble.back ().tokens); + const replay_tokens& rt (data.scr.depdb_preamble.back ().tokens); assert (!rt.empty ()); fail (ll) << "disallowed command in depdb preamble" << @@ -1015,11 +1466,84 @@ namespace build2 info (rt[0].location ()) << "depdb preamble ends here"; } - runner_->run (*environment_, ce, li, ll); + runner_->run (*environment_, ce, ii, li, cf, ll); } }; - exec_lines (s.depdb_preamble, exec_cmd); + exec_lines (begin, end, exec_cmd); + } + + pair<names, location> parser:: + execute_diag_preamble (const scope& rs, const scope& bs, + environment& e, const script& s, runner& r, + bool diag, bool enter, bool leave) + { + tracer trace ("execute_diag_preamble"); + + assert (!s.diag_preamble.empty ()); + + const line& dl (s.diag_preamble.back ()); // Diag builtin line. + + pre_exec (rs, bs, e, &s, &r); + + if (enter) + runner_->enter (e, s.start_loc); + + // Perform the variable assignments. + // + auto exec_cmd = [&dl, this] (token& t, + build2::script::token_type& tt, + const iteration_index* ii, size_t li, + bool /* single */, + const function<command_function>& cf, + const location& ll) + { + // Note that we never reset the line index to zero (as we do in + // execute_body()) assuming that there are some script body commands + // to follow. + // + command_expr ce ( + parse_command_line (t, static_cast<token_type&> (tt))); + + if (!valid_preamble_cmd (ce, cf)) + { + const replay_tokens& rt (dl.tokens); + assert (!rt.empty ()); + + fail (ll) << "disallowed command in diag preamble" << + info << "only variable assignments are allowed in diag preamble" + << info (rt[0].location ()) << "diag preamble ends here"; + } + + runner_->run (*environment_, ce, ii, li, cf, ll); + }; + + exec_lines (s.diag_preamble.begin (), s.diag_preamble.end () - 1, + exec_cmd); + + // Execute the diag line, if requested. + // + names ns; + + if (diag) + { + // Copy the tokens and start playing. + // + replay_data (replay_tokens (dl.tokens)); + + token t; + build2::script::token_type tt; + next (t, tt); + + ns = exec_special (t, tt, true /* skip_first */); + + replay_stop (); + } + + if (leave) + runner_->leave (e, s.end_loc); + + return make_pair (ns, dl.tokens.front ().location ()); } void parser:: @@ -1028,7 +1552,7 @@ namespace build2 { path_ = nullptr; // Set by replays. - pre_parse_ = false; + top_pre_parse_ = pre_parse_ = false; set_lexer (nullptr); @@ -1051,7 +1575,7 @@ namespace build2 } void parser:: - exec_lines (const lines& lns, + exec_lines (lines_iterator begin, lines_iterator end, const function<exec_cmd_function>& exec_cmd) { // Note that we rely on "small function object" optimization for the @@ -1078,63 +1602,1786 @@ namespace build2 apply_value_attributes (&var, lhs, move (rhs), kind); }; - auto exec_if = [this] (token& t, build2::script::token_type& tt, - size_t li, - const location& ll) + auto exec_cond = [this] (token& t, build2::script::token_type& tt, + const iteration_index* ii, size_t li, + const location& ll) { command_expr ce ( parse_command_line (t, static_cast<token_type&> (tt))); - // Assume if-else always involves multiple commands. + // Assume a flow control construct always involves multiple + // commands. // - return runner_->run_if (*environment_, ce, li, ll); + return runner_->run_cond (*environment_, ce, ii, li, ll); }; - build2::script::parser::exec_lines (lns.begin (), lns.end (), - exec_set, exec_cmd, exec_if, - environment_->exec_line, - &environment_->var_pool); + auto exec_for = [this] (const variable& var, + value&& val, + const attributes& val_attrs, + const location&) + { + value& lhs (environment_->assign (var)); + + attributes_.push_back (val_attrs); + + apply_value_attributes (&var, lhs, move (val), type::assign); + }; + + build2::script::parser::exec_lines ( + begin, end, + exec_set, exec_cmd, exec_cond, exec_for, + nullptr /* iteration_index */, + environment_->exec_line, + &environment_->var_pool); } names parser:: - exec_special (token& t, build2::script::token_type& tt, - bool omit_builtin) + exec_special (token& t, build2::script::token_type& tt, bool skip_first) { - if (omit_builtin) + if (skip_first) { assert (tt != type::newline && tt != type::eos); - next (t, tt); } return tt != type::newline && tt != type::eos - ? parse_names (t, tt, pattern_mode::expand) + ? parse_names (t, tt, pattern_mode::ignore) : names (); } - names parser:: - execute_special (const scope& rs, const scope& bs, - environment& e, - const line& ln, - bool omit_builtin) + void parser:: + exec_depdb_dyndep (token& lt, build2::script::token_type& ltt, + size_t li, const location& ll, + action a, const scope& bs, target& t, + depdb& dd, + dynamic_targets& dyn_targets, + bool& update, + timestamp mt, + bool& deferred_failure, + dyndep_byproduct* byprod_result) { - pre_exec (rs, bs, e, nullptr /* script */, nullptr /* runner */); + tracer trace ("exec_depdb_dyndep"); + + context& ctx (t.ctx); + + depdb_dyndep_options ops; + bool prog (false); + bool byprod (false); + bool dyn_tgt (false); - // Copy the tokens and start playing. + // Prerequisite update filter (--update-*). // - replay_data (replay_tokens (ln.tokens)); + struct filter + { + location loc; + build2::name name; + bool include; + bool used = false; - token t; - build2::script::token_type tt; - next (t, tt); + union + { + const target_type* type; // For patterns. + const build2::target* target; // For non-patterns. + }; - names r (exec_special (t, tt, omit_builtin)); + filter (const location& l, + build2::name n, bool i, const target_type& tt) + : loc (l), name (move (n)), include (i), type (&tt) {} - replay_stop (); - return r; + filter (const location& l, + build2::name n, bool i, const build2::target& t) + : loc (l), name (move (n)), include (i), target (&t) {} + + const char* + option () const + { + return include ? "--update-include" : "--update-exclude"; + } + }; + + vector<filter> filters; + bool filter_default (false); // Note: incorrect if filter is empty. + + // Similar approach to parse_env_builtin(). + // + { + auto& t (lt); + auto& tt (ltt); + + next (t, tt); // Skip the 'dyndep' command. + + if (tt == type::word && ((byprod = (t.value == "--byproduct")) || + (dyn_tgt = (t.value == "--dyn-target")))) + next (t, tt); + + assert (byprod == (byprod_result != nullptr)); + + // Note that an option name and value can belong to different name + // chunks. That's why we parse the arguments in the chunking mode + // into the list up to the `--` separator and parse this list into + // options afterwards. Note that the `--` separator should be + // omitted if there is no program (i.e., additional dependency info + // is being read from one of the prerequisites). + // + strings args; + + for (names ns; tt != type::newline && tt != type::eos; ns.clear ()) + { + location l (get_location (t)); + + if (tt == type::word) + { + if (t.value == "--") + { + prog = true; + break; + } + + // See also the non-literal check in the options parsing below. + // + if ((t.value.compare (0, 16, "--update-include") == 0 || + t.value.compare (0, 16, "--update-exclude") == 0) && + (t.value[16] == '\0' || t.value[16] == '=')) + { + string o; + + if (t.value[16] == '\0') + { + o = t.value; + next (t, tt); + } + else + { + o.assign (t.value, 0, 16); + t.value.erase (0, 17); + + if (t.value.empty ()) // Think `--update-include=$yacc`. + { + next (t, tt); + + if (t.separated) // Think `--update-include= $yacc`. + fail (l) << "depdb dyndep: expected name after " << o; + } + } + + if (!start_names (tt)) + fail (l) << "depdb dyndep: expected name instead of " << t + << " after " << o; + + // The chunk may actually contain multiple (or zero) names + // (e.g., as a result of a variable expansion or {}-list). Oh, + // well, I guess it can be viewed as a feature (to compensate + // for the literal option names). + // + parse_names (t, tt, + ns, + pattern_mode::preserve, + true /* chunk */, + ("depdb dyndep " + o + " option value").c_str (), + nullptr); + + if (ns.empty ()) + continue; + + bool i (o[9] == 'i'); + + for (name& n: ns) + { + // @@ Maybe we will want to support out-qualified targets + // one day (but they should not be patterns). + // + if (n.pair) + fail (l) << "depdb dyndep: name pair in " << o << " value"; + + if (n.pattern) + { + if (*n.pattern != name::pattern_type::path) + fail (l) << "depdb dyndep: non-path pattern in " << o + << " value"; + + n.canonicalize (); + + // @@ TODO (here and below). + // + // The reasonable directory semantics for a pattern seems + // to be: + // + // - empty - any directory (the common case) + // - relative - complete with base scope and fall through + // - absolute - only match targets in subdirectories + // + // Plus things are complicated by the src/out split (feels + // like we should do this in terms of scopes). + // + // See also target type/pattern-specific vars (where the + // directory is used to open a scope) and ad hoc pattern + // rules (where we currently don't allow directories). + // + if (!n.dir.empty ()) + { + if (path_pattern (n.dir)) + fail (l) << "depdb dyndep: pattern in directory in " + << o << " value"; + + fail (l) << "depdb dyndep: directory in pattern " << o + << " value"; + } + + // Resolve target type. If none is specified, then it's + // file{}. + // + const target_type* tt (n.untyped () + ? &file::static_type + : bs.find_target_type (n.type)); + + if (tt == nullptr) + fail (l) << "depdb dyndep: unknown target type " + << n.type << " in " << o << " value"; + + filters.push_back (filter (l, move (n), i, *tt)); + } + else + { + const target* t (search_existing (n, bs)); + + if (t == nullptr) + fail (l) << "depdb dyndep: unknown target " << n + << " in " << o << " value"; + + filters.push_back (filter (l, move (n), i, *t)); + } + } + + // If we have --update-exclude, then the default is include. + // + if (!i) + filter_default = true; + + continue; + } + } + + if (!start_names (tt)) + fail (l) << "depdb dyndep: expected option or '--' separator " + << "instead of " << t; + + parse_names (t, tt, + ns, + pattern_mode::ignore, + true /* chunk */, + "depdb dyndep builtin argument", + nullptr); + + for (name& n: ns) + { + try + { + args.push_back (convert<string> (move (n))); + } + catch (const invalid_argument&) + { + diag_record dr (fail (l)); + dr << "depdb dyndep: invalid string value "; + to_stream (dr.os, n, quote_mode::normal); + } + } + } + + if (prog) + { + if (byprod) + fail (t) << "depdb dyndep: --byproduct cannot be used with " + << "program"; + + next (t, tt); // Skip '--'. + + if (tt == type::newline || tt == type::eos) + fail (t) << "depdb dyndep: expected program name instead of " + << t; + } + + // Parse the options. + // + // We would like to support both -I <dir> as well as -I<dir> forms + // for better compatibility. The latter requires manual parsing. + // + try + { + for (cli::vector_scanner scan (args); scan.more (); ) + { + if (ops.parse (scan, cli::unknown_mode::stop) && !scan.more ()) + break; + + const char* a (scan.peek ()); + + // Handle -I<dir> + // + if (a[0] == '-' && a[1] == 'I') + { + try + { + ops.include_path ().push_back (dir_path (a + 2)); + } + catch (const invalid_path&) + { + throw cli::invalid_value ("-I", a + 2); + } + + scan.next (); + continue; + } + + // Handle --byproduct and --dyn-target in the wrong place. + // + if (strcmp (a, "--byproduct") == 0) + { + fail (ll) << "depdb dyndep: " + << (dyn_tgt + ? "--byproduct specified with --dyn-target" + : "--byproduct must be first option"); + } + + if (strcmp (a, "--dyn-target") == 0) + { + fail (ll) << "depdb dyndep: " + << (byprod + ? "--dyn-target specified with --byproduct" + : "--dyn-target must be first option"); + } + + // Handle non-literal --update-*. + // + if ((strncmp (a, "--update-include", 16) == 0 || + strncmp (a, "--update-exclude", 16) == 0) && + (a[16] == '\0' || a[16] == '=')) + fail (ll) << "depdb dyndep: " << a << " must be literal"; + + // Handle unknown option. + // + if (a[0] == '-') + throw cli::unknown_option (a); + + // Handle unexpected argument. + // + fail (ll) << "depdb dyndep: unexpected argument '" << a << "'"; + } + } + catch (const cli::exception& e) + { + fail (ll) << "depdb dyndep: " << e; + } + } + + // --format + // + dyndep_format format (dyndep_format::make); + if (ops.format_specified ()) + { + const string& f (ops.format ()); + + if (f == "lines") format = dyndep_format::lines; + else if (f != "make") + fail (ll) << "depdb dyndep: invalid --format option value '" + << f << "'"; + } + + // Prerequisite-specific options. + // + + // --what + // + const char* what (ops.what_specified () + ? ops.what ().c_str () + : "file"); + + // --cwd + // + optional<dir_path> cwd; + if (ops.cwd_specified ()) + { + if (!byprod) + fail (ll) << "depdb dyndep: --cwd only valid in --byproduct mode"; + + cwd = move (ops.cwd ()); + + if (cwd->relative ()) + fail (ll) << "depdb dyndep: relative path specified with --cwd"; + } + + // --include + // + if (!ops.include_path ().empty ()) + { + if (byprod) + fail (ll) << "depdb dyndep: -I specified with --byproduct"; + } + + // --default-type + // + // Get the default prerequisite type falling back to file{} if not + // specified. + // + // The reason one would want to specify it is to make sure different + // rules "resolve" the same dynamic prerequisites to the same targets. + // For example, a rule that implements custom C compilation for some + // translation unit would want to make sure it resolves extracted + // system headers to h{} targets analogous to the c module's rule. + // + const target_type* def_pt (&file::static_type); + if (ops.default_type_specified ()) + { + const string& t (ops.default_type ()); + + def_pt = bs.find_target_type (t); + if (def_pt == nullptr) + fail (ll) << "depdb dyndep: unknown target type '" << t + << "' specified with --default-type"; + } + + // --adhoc + // + if (ops.adhoc ()) + { + if (byprod) + fail (ll) << "depdb dyndep: --adhoc specified with --byproduct"; + } + + // Target-specific options. + // + + // --target-what + // + const char* what_tgt ("file"); + if (ops.target_what_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-what specified without " + << "--dyn-target"; + + what_tgt = ops.target_what ().c_str (); + } + + // --target-cwd + // + optional<dir_path> cwd_tgt; + if (ops.target_cwd_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-cwd specified without " + << "--dyn-target"; + + cwd_tgt = move (ops.target_cwd ()); + + if (cwd_tgt->relative ()) + fail (ll) << "depdb dyndep: relative path specified with " + << "--target-cwd"; + } + + // --target-default-type + // + const target_type* def_tt (&file::static_type); + if (ops.target_default_type_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-default-type specified " + << "without --dyn-target"; + + const string& t (ops.target_default_type ()); + + def_tt = bs.find_target_type (t); + if (def_tt == nullptr) + fail (ll) << "depdb dyndep: unknown target type '" << t + << "' specified with --target-default-type"; + } + + map<string, const target_type*> map_tt; + if (ops.target_extension_type_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-extension-type specified " + << "without --dyn-target"; + + for (pair<const string, string>& p: ops.target_extension_type ()) + { + const target_type* tt (bs.find_target_type (p.second)); + if (tt == nullptr) + fail (ll) << "depdb dyndep: unknown target type '" << p.second + << "' specified with --target-extension-type"; + + map_tt[p.first] = tt; + } + } + + // --file (last since need --*cwd) + // + // Note that if --file is specified without a program, then we assume + // it is one of the static prerequisites. + // + optional<path> file; + if (ops.file_specified ()) + { + file = move (ops.file ()); + + if (file->relative ()) + { + if (!cwd && !cwd_tgt) + fail (ll) << "depdb dyndep: relative path specified with --file"; + + *file = (cwd ? *cwd : *cwd_tgt) / *file; + } + } + else if (!prog) + fail (ll) << "depdb dyndep: program or --file expected"; + + // Update prerequisite targets. + // + using dyndep = dyndep_rule; + + auto& pts (t.prerequisite_targets[a]); + + for (prerequisite_target& p: pts) + { + if (const target* pt = + (p.target != nullptr ? p.target : + p.adhoc () ? reinterpret_cast<target*> (p.data) + : nullptr)) + { + // Automatically skip update=unmatch that we could not unmatch. + // + // Note that we don't skip update=match here (unless filtered out) + // in order to incorporate the result into our out-of-date'ness. + // So there is a nuanced interaction between update=match and + // --update-*. + // + if ((p.include & adhoc_buildscript_rule::include_unmatch) != 0) + { + l6 ([&]{trace << "skipping unmatched " << *pt;}); + continue; + } + + // Apply the --update-* filter. + // + if (!p.adhoc () && !filters.empty ()) + { + // Compute and cache "effective" name that we will be pattern- + // matching (similar code to variable_type_map::find()). + // + auto ename = [pt, en = optional<string> ()] () mutable + -> const string& + { + if (!en) + { + en = string (); + pt->key ().effective_name (*en); + } + + return en->empty () ? pt->name : *en; + }; + + bool i (filter_default); + + for (filter& f: filters) + { + if (f.name.pattern) + { + const name& n (f.name); + +#if 0 + // Match directory if any. + // + if (!n.dir.empty ()) + { + // @@ TODO (here and above). + } +#endif + + // Match type. + // + if (!pt->is_a (*f.type)) + continue; + + // Match name. + // + if (n.value == "*" || butl::path_match (ename (), n.value)) + { + i = f.include; + break; + } + } + else + { + if (pt == f.target) + { + i = f.include; + f.used = true; + break; + } + } + } + + if (!i) + continue; + } + + update = dyndep::update ( + trace, a, *pt, update ? timestamp_unknown : mt) || update; + + // While implicit, it is for a static prerequisite, so marking it + // feels correct. + // + p.include |= prerequisite_target::include_udm; + + // Mark as updated (see execute_update_prerequisites() for + // details. + // + if (!p.adhoc ()) + p.data = 1; + } + } + + // Detect target filters that do not match anything. + // + for (const filter& f: filters) + { + if (!f.name.pattern && !f.used) + fail (f.loc) << "depdb dyndep: target " << f.name << " in " + << f.option () << " value does not match any " + << "prerequisites"; + } + + if (byprod) + { + *byprod_result = dyndep_byproduct { + ll, + format, + move (cwd), + move (*file), + ops.what_specified () ? move (ops.what ()) : string (what), + def_pt, + ops.drop_cycles ()}; + + return; + } + + const scope& rs (*bs.root_scope ()); + + group* g (t.is_a<group> ()); // If not group then file. + + // This code is based on the prior work in the cc module (specifically + // extract_headers()) where you can often find more detailed rationale + // for some of the steps performed. + + // Build the maps lazily, only if/when needed. + // + using prefix_map = dyndep::prefix_map; + using srcout_map = dyndep::srcout_map; + + function<dyndep::map_extension_func> map_ext ( + [] (const scope& bs, const string& n, const string& e) + { + // NOTE: another version in adhoc_buildscript_rule::apply(). + + // @@ TODO: allow specifying base target types. + // + // Feels like the only reason one would want to specify base types + // is to tighten things up (as opposed to making some setup work) + // since it essentially restricts the set of registered target + // types that we will consider. + // + // Note also that these would be this project's target types while + // the file could be from another project. + // + return dyndep::map_extension (bs, n, e, nullptr); + + // @@ TODO: should we return something as fallback (file{}, + // def_pt)? Note: not the same semantics as enter_file()'s + // fallback. Feels like it could conceivably be different + // (e.g., h{} for fallback and hxx{} for some "unmappable" gen + // header). It looks like the "best" way currently is to define + // a custom target types for it (see moc{} in libQt5Core). + // + // Note also that we should only do this if bs is in our + // project. + }); + + // Don't we want to insert a "local"/prefixless mapping in case the + // user did not specify any -I's? But then will also need src-out + // remapping. So it will be equivalent to -I$out_base -I$src_base? But + // then it's not hard to add explicitly... + // + function<dyndep::prefix_map_func> pfx_map; + + struct + { + tracer& trace; + const location& ll; + const depdb_dyndep_options& ops; + optional<prefix_map> map; + } pfx_data {trace, ll, ops, nullopt}; + + if (!ops.include_path ().empty ()) + { + pfx_map = [this, &pfx_data] (action, + const scope& bs, + const target& t) -> const prefix_map& + { + if (!pfx_data.map) + { + pfx_data.map = prefix_map (); + + const scope& rs (*bs.root_scope ()); + + for (dir_path d: pfx_data.ops.include_path ()) + { + if (d.relative ()) + fail (pfx_data.ll) << "depdb dyndep: relative include " + << "search path " << d; + + if (!d.normalized (false /* canonical dir seperators */)) + d.normalize (); + + // If we are not inside our project root, then ignore. + // + if (d.sub (rs.out_path ())) + dyndep::append_prefix ( + pfx_data.trace, *pfx_data.map, t, move (d)); + } + } + + return *pfx_data.map; + }; + } + + // Parse the remainder of the command line as a program (which can be + // a pipe). If file is absent, then we save the command's stdout to a + // pipe. Otherwise, assume the command writes to file and add it to + // the cleanups. + // + // Note that MSVC /showInclude sends its output to stderr (and so + // could do other broken tools). However, the user can always merge + // stderr to stdout (2>&1). + // + command_expr cmd; + srcout_map so_map; + + // Save/restore script cleanups. + // + struct cleanups + { + build2::script::cleanups ordinary; + paths special; + }; + optional<cleanups> script_cleanups; + + auto cleanups_guard = make_guard ( + [this, &script_cleanups] () + { + if (script_cleanups) + { + swap (environment_->cleanups, script_cleanups->ordinary); + swap (environment_->special_cleanups, script_cleanups->special); + } + }); + + auto init_run = [this, &ctx, + <, <t, &ll, + prog, &file, &ops, + &cmd, &so_map, &script_cleanups] () + { + // Populate the srcout map with the -I$out_base -I$src_base pairs. + // + { + dyndep::srcout_builder builder (ctx, so_map); + + for (dir_path d: ops.include_path ()) + builder.next (move (d)); + } + + if (prog) + { + script_cleanups = cleanups {}; + swap (environment_->cleanups, script_cleanups->ordinary); + swap (environment_->special_cleanups, script_cleanups->special); + + cmd = parse_command_line (lt, static_cast<token_type&> (ltt)); + + // If the output goes to stdout, then this should be a single + // pipeline without any logical operators (&& or ||). + // + if (!file && cmd.size () != 1) + fail (ll) << "depdb dyndep: command with stdout output cannot " + << "contain logical operators"; + + // Note that we may need to run this command multiple times. The + // two potential issues here are the re-registration of the + // clenups and re-use of the special files (stdin, stdout, etc; + // they include the line index in their names to avoid clashes + // between lines). + // + // Cleanups are not an issue, they will simply be replaced. And + // overriding the contents of the special files seems harmless and + // consistent with what would happen if the command redirects its + // output to a non-special file. + } + }; + + // Enter as a target, update, and add to the list of prerequisite + // targets a file. + // + size_t skip_count (0); + + auto add = [this, &trace, what, + a, &bs, &t, g, &pts, pts_n = pts.size (), + &ops, &map_ext, def_pt, &pfx_map, &so_map, + &dd, &skip_count] (path fp, + size_t* skip, + timestamp mt) -> optional<bool> + { + context& ctx (t.ctx); + + bool cache (skip == nullptr); + + // Handle fsdir{} prerequisite separately. + // + // Note: inspired by inject_fsdir(). + // + if (fp.to_directory ()) + { + if (!cache) + { + // Note: already absolute since cannot be non-existent. + // + fp.normalize (); + } + + const fsdir* dt (&search<fsdir> (t, + path_cast<dir_path> (fp), + dir_path (), + string (), nullptr, nullptr)); + + // Subset of code for file below. + // + if (!cache) + { + for (size_t i (0); i != pts_n; ++i) + { + const prerequisite_target& p (pts[i]); + + if (const target* pt = + (p.target != nullptr ? p.target : + p.adhoc () ? reinterpret_cast<target*> (p.data) : + nullptr)) + { + if (dt == pt) + return false; + } + } + + if (*skip != 0) + { + --(*skip); + return false; + } + } + + match_sync (a, *dt); + pts.push_back ( + prerequisite_target ( + nullptr, true /* adhoc */, reinterpret_cast<uintptr_t> (dt))); + + if (!cache) + dd.expect (fp.representation ()); + + skip_count++; + return false; + } + + // We can only defer the failure if we will be running the recipe + // body. + // + auto fail = [this, what, &ctx] (const auto& f) -> optional<bool> + { + bool df (!ctx.match_only && !ctx.dry_run_option); + + diag_record dr; + dr << error << what << ' ' << f << " not found and no rule to " + << "generate it"; + + if (df) + dr << info << "failure deferred to recipe body diagnostics"; + + if (verb < 4) + dr << info << "re-run with --verbose=4 for more information"; + + if (df) + return nullopt; + else + dr << endf; + }; + + if (const build2::file* ft = dyndep::enter_file ( + trace, what, + a, bs, t, + fp, cache, cache /* normalized */, + map_ext, *def_pt, pfx_map, so_map).first) + { + // We don't need to do these tests for the cached case since such + // prerequisites would have been skipped (and we won't get here if + // the target/prerequisite set changes since we hash them). + // + if (!cache) + { + // Skip if this is one of the static prerequisites provided it + // was updated. + // + for (size_t i (0); i != pts_n; ++i) + { + const prerequisite_target& p (pts[i]); + + if (const target* pt = + (p.target != nullptr ? p.target : + p.adhoc () ? reinterpret_cast<target*> (p.data) : + nullptr)) + { + if (ft == pt && (p.adhoc () || p.data == 1)) + return false; + } + } + + // Skip if this is one of the targets. + // + // Note that for dynamic targets this only works if we see the + // targets before prerequisites (like in the make dependency + // format). + // + if (ops.drop_cycles ()) + { + if (g != nullptr) + { + auto& ms (g->members); + if (find (ms.begin (), ms.end (), ft) != ms.end ()) + return false; + } + else + { + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (ft == m) + return false; + } + } + } + + // Skip until where we left off. + // + // Note that we used to do this outside of this lambda and + // before calling enter_file() but due to the above skips we can + // only do it here if we want to have a consistent view of the + // prerequisite lists between the cached and non-cached cases. + // + if (*skip != 0) + { + --(*skip); + return false; + } + } + + // Note: mark the injected prerequisite target as updated (see + // execute_update_prerequisites() for details). + // + if (optional<bool> u = dyndep::inject_file ( + trace, what, + a, t, + *ft, mt, + false /* fail */, + ops.adhoc () /* adhoc */)) + { + prerequisite_target& pt (pts.back ()); + + // Note: set the include_target flag for consistency (the + // updated_during_match() check does not apply since it's a + // dynamic prerequisite). + // + if (pt.adhoc ()) + { + pt.data = reinterpret_cast<uintptr_t> (pt.target); + pt.target = nullptr; + pt.include |= prerequisite_target::include_target; + } + else + pt.data = 1; // Already updated. + + if (!cache) + dd.expect (ft->path ()); // @@ Use fp (or verify match)? + + skip_count++; + return *u; + } + else if (cache) + { + dd.write (); // Invalidate this line. + return true; + } + else + return fail (*ft); + } + else + return fail (fp); + }; + + // If things go wrong (and they often do in this area), give the user + // a bit extra context. + // + auto df = make_diag_frame ( + [this, &ll, &t] (const diag_record& dr) + { + if (verb != 0) + dr << info (ll) << "while extracting dynamic dependencies for " + << t; + }); + + // While in the make format targets come before prerequisites, in + // depdb we store them after since any change to prerequisites can + // invalidate the set of targets. So we save them first and process + // later. + // + // Note also that we need to return them to the caller in case we are + // updating. + + // If nothing so far has invalidated the dependency database, then try + // the cached data before running the program. + // + bool cache (!update); + bool skip_blank (false); + + for (bool restart (true), first_run (true); restart; cache = false) + { + // Clear the state in case we are restarting. + // + if (dyn_tgt) + dyn_targets.clear (); + + restart = false; + + if (cache) + { + // If any, this is always the first run. + // + assert (skip_count == 0); + + // We should always end with a blank line after the list of + // dynamic prerequisites. + // + for (;;) + { + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + restart = true; + break; + } + + if (l->empty ()) // Done with prerequisites, nothing changed. + { + skip_blank = true; + break; + } + + if (optional<bool> r = add (path (move (*l)), nullptr, mt)) + { + restart = *r; + + if (restart) + { + update = true; + l6 ([&]{trace << "restarting (cache)";}); + break; + } + } + else + { + // Trigger rebuild and mark as expected to fail. + // + update = true; + deferred_failure = true; + return; + } + } + + if (!restart) // Nothing changed. + { + if (dyn_tgt) + { + // We should always end with a blank line after the list of + // dynamic targets. + // + for (;;) + { + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + restart = true; + break; + } + + if (l->empty ()) // Done with targets. + break; + + // Split into type and path (see below for background). + // + size_t p (l->find (' ')); + if (p == string::npos || // Invalid format. + p == 0 || // Empty type. + p + 1 == l->size ()) // Empty path. + { + dd.write (); // Invalidate this line. + restart = true; + break; + } + + string t (*l, 0, p); + l->erase (0, p + 1); + + dyn_targets.push_back ( + dynamic_target {move (t), path (move (*l))}); + } + } + + if (!restart) // Done, nothing changed. + break; // Break earliy to keep cache=true. + } + } + else + { + if (first_run) + { + init_run (); + first_run = false; + } + else + { + if (!prog) + fail (ll) << "generated " << what << " without program to retry"; + + // Drop dyndep cleanups accumulated on the previous run. + // + assert (script_cleanups); // Sanity check. + environment_->cleanups.clear (); + environment_->special_cleanups.clear (); + } + + // Save the timestamp just before we run the command. If we depend + // on any file that has been updated since, then we should assume + // we have "seen" the old copy and restart. + // + timestamp rmt (prog ? system_clock::now () : mt); + + // Run the command if any and reduce outputs to common istream. + // + // Note that the resulting stream should tolerate partial read. + // + // While reading the entire stdout into a string is not the most + // efficient way to do it, this does simplify things quite a bit, + // not least of which is not having to parse the output before + // knowing the program exist status. + // + istringstream iss; + if (prog) + { + // Note: depdb is disallowed inside flow control constructs. + // + if (!file) + { + function<command_function> cf ( + [&iss] + (build2::script::environment&, + const strings&, + auto_fd in, + pipe_command* pipe, + const optional<deadline>& dl, + const location& ll) + { + read (move (in), + false /* whitespace */, + false /* newline */, + true /* exact */, + [&iss] (string&& s) {iss.str (move (s));}, + pipe, + dl, + ll, + "depdb-dyndep"); + }); + + build2::script::run (*environment_, + cmd, + nullptr /* iteration_index */, li, + ll, + cf, false /* last_cmd */); + + iss.exceptions (istream::badbit); + } + else + { + build2::script::run ( + *environment_, cmd, nullptr /* iteration_index */, li, ll); + + // Note: make it a maybe-cleanup in case the command cleans it + // up itself. + // + environment_->clean ( + {build2::script::cleanup_type::maybe, *file}, + true /* implicit */); + } + } + + ifdstream ifs (ifdstream::badbit); + if (file) + try + { + ifs.open (*file); + } + catch (const io_error& e) + { + fail (ll) << "unable to open file " << *file << ": " << e; + } + + istream& is (file + ? static_cast<istream&> (ifs) + : static_cast<istream&> (iss)); + + const path_name& in (file + ? path_name (*file) + : path_name ("<stdin>")); + + location il (in, 1); + size_t skip (skip_count); + + // The way we parse things is format-specific. + // + // Note: similar code in + // adhoc_buildscript_rule::perform_update_file_dyndep_byproduct(). + // + switch (format) + { + case dyndep_format::make: + { + using make_state = make_parser; + using make_type = make_parser::type; + + make_parser make; + + for (string l; !restart; ++il.line) // Reuse the buffer. + { + if (eof (getline (is, l))) + { + if (make.state != make_state::end) + fail (il) << "incomplete make dependency declaration"; + + break; + } + + size_t pos (0); + do + { + pair<make_type, path> r; + { + auto df = make_diag_frame ( + [this, &l] (const diag_record& dr) + { + if (verb != 0) + dr << info << "while parsing make dependency " + << "declaration line '" << l << "'"; + }); + + r = make.next (l, pos, il); + } + + if (r.second.empty ()) + continue; + + // Skip targets unless requested to extract. + // + // BTW, if you are wondering why don't we extract targets + // by default, take GCC as an example, where things are + // quite messed up: by default it ignores -o and just + // takes the source file name and replaces the extension + // with a platform-appropriate object file extension. One + // can specify a custom target (or even multiple targets) + // with -MT or with -MQ (quoting). So in this case it's + // definitely easier for the user to ignore the targets + // and just specify everything in the buildfile. + // + if (r.first == make_type::target) + { + // NOTE: similar code below. + // + if (dyn_tgt) + { + path& f (r.second); + + if (f.relative ()) + { + if (!cwd_tgt) + fail (il) << "relative " << what_tgt + << " target path '" << f + << "' in make dependency declaration" << + info << "consider using --target-cwd to specify " + << "relative path base"; + + f = *cwd_tgt / f; + } + + // Note that unlike prerequisites, here we don't need + // normalize_external() since we expect the targets to + // be within this project. + // + try + { + f.normalize (); + } + catch (const invalid_path&) + { + fail (il) << "invalid " << what_tgt << " target " + << "path '" << f.string () << "'"; + } + + // The target must be within this project. + // + if (!f.sub (rs.out_path ())) + { + fail (il) << what_tgt << " target path " << f + << " must be inside project output " + << "directory " << rs.out_path (); + } + + // Note: type is resolved later. + // + dyn_targets.push_back ( + dynamic_target {string (), move (f)}); + } + + continue; + } + + // NOTE: similar code below. + // + if (optional<bool> u = add (move (r.second), &skip, rmt)) + { + restart = *u; + + if (restart) + { + update = true; + l6 ([&]{trace << "restarting";}); + break; + } + } + else + { + // Trigger recompilation, mark as expected to fail, and + // bail out. + // + update = true; + deferred_failure = true; + break; + } + } + while (pos != l.size ()); + + if (make.state == make_state::end || deferred_failure) + break; + } + + break; // case + } + case dyndep_format::lines: + { + bool tgt (dyn_tgt); // Reading targets or prerequisites. + + for (string l; !restart; ++il.line) // Reuse the buffer. + { + if (eof (getline (is, l))) + break; + + if (l.empty ()) + { + if (!tgt) + fail (il) << "blank line in prerequisites list"; + + tgt = false; // Targets/prerequisites separating blank. + continue; + } + + // See if this line start with space to indicate a non- + // existent prerequisite. This variable serves both as a + // flag and as a position of the beginning of the path. + // + size_t n (l.front () == ' ' ? 1 : 0); + + if (tgt) + { + // NOTE: similar code above. + // + path f; + try + { + // Non-existent target doesn't make sense. + // + if (n) + throw invalid_path (""); + + f = path (l); + + if (f.relative ()) + { + if (!cwd_tgt) + fail (il) << "relative " << what_tgt + << " target path '" << f + << "' in lines dependency declaration" << + info << "consider using --target-cwd to specify " + << "relative path base"; + + f = *cwd_tgt / f; + } + + // Note that unlike prerequisites, here we don't need + // normalize_external() since we expect the targets to + // be within this project. + // + f.normalize (); + } + catch (const invalid_path&) + { + fail (il) << "invalid " << what_tgt << " target path '" + << l << "'"; + } + + // The target must be within this project. + // + if (!f.sub (rs.out_path ())) + { + fail (il) << what_tgt << " target path " << f + << " must be inside project output directory " + << rs.out_path (); + } + + // Note: type is resolved later. + // + dyn_targets.push_back ( + dynamic_target {string (), move (f)}); + } + else + { + path f; + try + { + f = path (l.c_str () + n, l.size () - n); + + if (f.empty () || + (n && f.to_directory ())) // Non-existent fsdir{}. + throw invalid_path (""); + + if (f.relative ()) + { + if (!n) + { + if (!cwd) + fail (il) << "relative " << what + << " prerequisite path '" << f + << "' in lines dependency declaration" << + info << "consider using --cwd to specify " + << "relative path base"; + + f = *cwd / f; + } + } + else if (n) + { + // @@ TODO: non-existent absolute paths. + // + throw invalid_path (""); + } + } + catch (const invalid_path&) + { + fail (il) << "invalid " << what << " prerequisite path '" + << l << "'"; + } + + // NOTE: similar code above. + // + if (optional<bool> u = add (move (f), &skip, rmt)) + { + restart = *u; + + if (restart) + { + update = true; + l6 ([&]{trace << "restarting";}); + } + } + else + { + // Trigger recompilation, mark as expected to fail, and + // bail out. + // + update = true; + deferred_failure = true; + break; + } + } + } + + break; // case + } + } + + if (file) + ifs.close (); + + // Bail out early if we have deferred a failure. + // + if (deferred_failure) + return; + + // Clean after each depdb-dyndep execution. + // + if (prog) + clean (*environment_, ll); + } + } + + // Add the dynamic prerequisites terminating blank line if we are + // updating depdb and unless it's already there. + // + if (!cache && !skip_blank) + dd.expect (""); + + // Handle dynamic targets. + // + if (dyn_tgt) + { + if (g != nullptr && g->members_static == 0 && dyn_targets.empty ()) + fail (ll) << "group " << *g << " has no static or dynamic members"; + + // There is one more level (at least that we know of) to this rabbit + // hole: if the set of dynamic targets changes between clean and + // update and we do a `clean update` batch, then we will end up with + // old targets (as entered by clean from old depdb information) + // being present during update. So we need to clean them out. + // + // Optimize this for a first/single batch (common case) by noticing + // that there are only real targets to start with. + // + // Note that this doesn't affect explicit groups where we reset the + // members on each update (see adhoc_rule_buildscript::apply()). + // + optional<vector<const target*>> dts; + if (g == nullptr) + { + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (m->decl != target_decl::real) + dts = vector<const target*> (); + } + } + + struct map_ext_data + { + const char* what_tgt; + const map<string, const target_type*>& map_tt; + const path* f; // Updated on each iteration. + } d {what_tgt, map_tt, nullptr}; + + function<dyndep::map_extension_func> map_ext ( + [this, &d] (const scope& bs, const string& n, const string& e) + { + small_vector<const target_type*, 2> tts; + + // Check the custom mapping first. + // + auto i (d.map_tt.find (e)); + if (i != d.map_tt.end ()) + tts.push_back (i->second); + else + { + tts = dyndep::map_extension (bs, n, e, nullptr); + + // Issue custom diagnostics suggesting --target-extension-type. + // + if (tts.size () > 1) + { + diag_record dr (fail); + + dr << "mapping of " << d.what_tgt << " target path " << *d.f + << " to target type is ambiguous"; + + for (const target_type* tt: tts) + dr << info << "can be " << tt->name << "{}"; + + dr << info << "use --target-extension-type to provide custom " + << "mapping"; + } + } + + return tts; + }); + + function<dyndep::group_filter_func> filter; + if (g != nullptr) + { + // Skip static/duplicate members in explicit group. + // + filter = [] (mtime_target& g, const build2::file& m) + { + auto& ms (g.as<group> ().members); + return find (ms.begin (), ms.end (), &m) == ms.end (); + }; + } + + // Unlike for prerequisites, for targets we store in depdb both the + // resolved target type and path. The target type is used in clean + // (see adhoc_rule_buildscript::apply()) where we cannot easily get + // hold of all the dyndep options to map the path to target type. + // So the format of the target line is: + // + // <type> <path> + // + string l; // Reuse the buffer. + for (dynamic_target& dt: dyn_targets) + { + const path& f (dt.path); + + d.f = &f; // Current file being mapped. + + // Note that this logic should be consistent with what we have in + // adhoc_buildscript_rule::apply() for perform_clean. + // + const build2::file* ft (nullptr); + if (g != nullptr) + { + pair<const build2::file&, bool> r ( + dyndep::inject_group_member ( + what_tgt, + a, bs, *g, + f, // Can't move since need to return dyn_targets. + map_ext, *def_tt, filter)); + + // Note: no target_decl shenanigans since reset the members on + // each update. + // + if (!r.second) + { + dt.type.clear (); // Static indicator. + continue; + } + + ft = &r.first; + + // Note: we only currently support dynamic file members so it + // will be file if first. + // + g->members.push_back (ft); + } + else + { + pair<const build2::file&, bool> r ( + dyndep::inject_adhoc_group_member ( + what_tgt, + a, bs, t, + f, // Can't move since need to return dyn_targets. + map_ext, *def_tt)); + + // Note that we have to track the dynamic target even if it was + // already a member (think `b update && b clean update`). + // + if (!r.second && r.first.decl == target_decl::real) + { + dt.type.clear (); // Static indicator. + continue; + } + + ft = &r.first; + + if (dts) + dts->push_back (ft); + } + + const char* tn (ft->type ().name); + + if (dt.type.empty ()) + dt.type = tn; + else if (dt.type != tn) + { + // This can, for example, happen if the user changed the + // extension to target type mapping. Say swapped extension + // variable values of two target types. + // + fail << "mapping of " << what_tgt << " target path " << f + << " to target type has changed" << + info << "previously mapped to " << dt.type << "{}" << + info << "now mapped to " << tn << "{}" << + info << "perform from scratch rebuild of " << t; + } + + if (!cache) + { + l = dt.type; + l += ' '; + l += f.string (); + dd.expect (l); + } + } + + // Add the dynamic targets terminating blank line. + // + if (!cache) + dd.expect (""); + + // Clean out old dynamic targets (skip the primary member). + // + if (dts) + { + assert (g == nullptr); + + for (target* p (&t); p->adhoc_member != nullptr; ) + { + target* m (p->adhoc_member); + + if (m->decl != target_decl::real) + { + // While there could be quite a few dynamic targets (think + // something like Doxygen), this will hopefully be optimized + // down to a contiguous memory region scan for an integer and + // so should be fast. + // + if (find (dts->begin (), dts->end (), m) == dts->end ()) + { + p->adhoc_member = m->adhoc_member; // Drop m. + continue; + } + } + + p = m; + } + } + } + + // Reload $< and $> to make sure they contain the newly discovered + // prerequisites and targets. + // + if (update) + environment_->set_special_variables (a); } - // When add a special variable don't forget to update lexer::word(). + // When add a special variable don't forget to update lexer::word() and + // for-loop parsing in pre_parse_line(). // bool parser:: special_variable (const string& n) noexcept @@ -1143,15 +3390,24 @@ namespace build2 } lookup parser:: - lookup_variable (name&& qual, string&& name, const location& loc) + lookup_variable (names&& qual, string&& name, const location& loc) { // In the pre-parse mode collect the referenced variable names for the // script semantics change tracking. // + // Note that during pre-parse a computed (including qualified) name + // is signalled as an empty name. + // if (pre_parse_ || pre_parse_suspended_) { lookup r; + // Note that pre-parse can be switched on by the base parser even + // during execute. + // + if (!top_pre_parse_) + return r; + // Add the variable name skipping special variables and suppressing // duplicates, unless the default variables change tracking is // canceled with `depdb clear`. While at it, check if the script @@ -1167,10 +3423,8 @@ namespace build2 { if (pre_parse_suspended_) { - const variable* pvar (scope_->ctx.var_pool.find (name)); - - if (pvar != nullptr) - r = (*scope_)[*pvar]; + if (const variable* var = scope_->var_pool ().find (name)) + r = (*scope_)[*var]; } if (!depdb_clear_) @@ -1181,12 +3435,27 @@ namespace build2 vars.push_back (move (name)); } } + else + { + // What about pre_parse_suspended_? Don't think it makes sense to + // diagnose this since it can be indirect (that is, via an + // intermediate variable). + // + if (perform_update_ && file_based_ && !computed_var_) + computed_var_ = loc; + } return r; } if (!qual.empty ()) - fail (loc) << "qualified variable name"; + { + // Qualified variable is computed and we expect the user to track + // its changes manually. + // + return build2::script::parser::lookup_variable ( + move (qual), move (name), loc); + } lookup r (environment_->lookup (name)); @@ -1197,13 +3466,13 @@ namespace build2 // diag builtin argument change (which can be affected by such a // variable expansion) doesn't affect the script semantics and the // depdb argument is specifically used for the script semantics change - // tracking. We also omit this check it the depdb builtin is used in - // the script, assuming that such variables are tracked manually, if - // required. + // tracking. We also omit this check if the depdb "value" (string, + // hash) builtin is used in the script, assuming that such variables + // are tracked manually, if required. // if (script_ != nullptr && !script_->depdb_clear && - script_->depdb_preamble.empty ()) + !script_->depdb_value) { if (r.defined () && !r.belongs (*environment_)) { @@ -1221,9 +3490,12 @@ namespace build2 void parser:: lookup_function (string&& name, const location& loc) { - if (perform_update_ && file_based_ && !impure_func_) + // Note that pre-parse can be switched on by the base parser even + // during execute. + // + if (top_pre_parse_ && perform_update_ && file_based_ && !impure_func_) { - const function_overloads* f (ctx.functions.find (name)); + const function_overloads* f (ctx->functions.find (name)); if (f != nullptr && !f->pure) impure_func_ = make_pair (move (name), loc); diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx index b737a13..8f86b24 100644 --- a/libbuild2/build/script/parser.hxx +++ b/libbuild2/build/script/parser.hxx @@ -8,7 +8,6 @@ #include <libbuild2/forward.hxx> #include <libbuild2/utility.hxx> -#include <libbuild2/depdb.hxx> #include <libbuild2/diagnostics.hxx> #include <libbuild2/script/parser.hxx> @@ -29,7 +28,7 @@ namespace build2 // Pre-parse. Issue diagnostics and throw failed in case of an error. // public: - parser (context& c): build2::script::parser (c, false /* relex */) {} + parser (context& c): build2::script::parser (c) {} // Note that the returned script object references the passed path // name. @@ -66,11 +65,18 @@ namespace build2 pre_parse_script (); void - pre_parse_line (token&, token_type&, bool if_line = false); + pre_parse_line (token&, token_type&, + optional<line_type> flow_control_type = nullopt); + + void + pre_parse_block_line (token&, token_type&, line_type block_type); void pre_parse_if_else (token&, token_type&); + void + pre_parse_loop (token&, token_type&, line_type); + command_expr parse_command_line (token&, token_type&); @@ -82,31 +88,122 @@ namespace build2 // initialize/clean up the environment before/after the script // execution. // + // Note: having both root and base scopes for testing (where we pass + // global scope for both). + // void execute_body (const scope& root, const scope& base, environment&, const script&, runner&, bool enter = true, bool leave = true); - + // Execute the first or the second (dyndep) half of the depdb + // preamble. + // // Note that it's the caller's responsibility to make sure that the // runner's enter() function is called before the first preamble/body // command execution and leave() -- after the last command. // + // Note: target must be file or group. + // void - execute_depdb_preamble (const scope& root, const scope& base, - environment&, const script&, runner&, - depdb&); + execute_depdb_preamble (action a, const scope& base, const target& t, + environment& e, const script& s, runner& r, + depdb& dd) + { + auto b (s.depdb_preamble.begin ()); + exec_depdb_preamble ( + a, base, t, + e, s, r, + b, + (s.depdb_dyndep + ? b + *s.depdb_dyndep + : s.depdb_preamble.end ()), + dd); + } + + struct dynamic_target + { + string type; // Target type name (absent if static member). + build2::path path; + }; + + using dynamic_targets = vector<dynamic_target>; - - // Parse a special builtin line into names, performing the variable - // and pattern expansions. If omit_builtin is true, then omit the - // builtin name from the result. + void + execute_depdb_preamble_dyndep ( + action a, const scope& base, target& t, + environment& e, const script& s, runner& r, + depdb& dd, + dynamic_targets& dyn_targets, + bool& update, timestamp mt, bool& deferred_failure) + { + exec_depdb_preamble ( + a, base, t, + e, s, r, + s.depdb_preamble.begin () + *s.depdb_dyndep, + s.depdb_preamble.end (), + dd, &dyn_targets, &update, mt, &deferred_failure); + } + + // This version doesn't actually execute the depdb-dyndep builtin (but + // may execute some variable assignments) instead returning all the + // information (extracted from options) necessary to implement the + // depdb-dyndep --byproduct logic (which fits better into the rule + // implementation). + // + enum class dyndep_format {make, lines}; + + struct dyndep_byproduct + { + location_value location; + dyndep_format format; + optional<dir_path> cwd; + path file; + string what; + const target_type* default_type; + bool drop_cycles; + }; + + dyndep_byproduct + execute_depdb_preamble_dyndep_byproduct ( + action a, const scope& base, const target& t, + environment& e, const script& s, runner& r, + depdb& dd, bool& update, timestamp mt) + { + // Dummies. + // + // This is getting a bit ugly (we also don't really need to pass + // depdb here). One day we will find a better way... + // + dynamic_targets dyn_targets; + bool deferred_failure; + + dyndep_byproduct v; + exec_depdb_preamble ( + a, base, t, + e, s, r, + s.depdb_preamble.begin () + *s.depdb_dyndep, + s.depdb_preamble.end (), + dd, &dyn_targets, &update, mt, &deferred_failure, &v); + return v; + } + + // If the diag argument is true, then execute the preamble including + // the (trailing) diagnostics line and return the resulting names and + // its location (see exec_special() for the diagnostics line execution + // semantics). Otherwise, execute the preamble excluding the + // diagnostics line and return an empty names list and location. If + // requested, call the runner's enter() and leave() functions that + // initialize/clean up the environment before/after the preamble + // execution. // - names - execute_special (const scope& root, const scope& base, - environment&, - const line&, - bool omit_builtin = true); + // Note: having both root and base scopes for testing (where we pass + // global scope for both). + // + pair<names, location> + execute_diag_preamble (const scope& root, const scope& base, + environment&, const script&, runner&, + bool diag, bool enter, bool leave); protected: // Setup the parser for subsequent exec_*() function calls. @@ -115,12 +212,50 @@ namespace build2 pre_exec (const scope& root, const scope& base, environment&, const script*, runner*); + using lines_iterator = lines::const_iterator; + void - exec_lines (const lines&, const function<exec_cmd_function>&); + exec_lines (lines_iterator, lines_iterator, + const function<exec_cmd_function>&); + void + exec_lines (const lines& l, const function<exec_cmd_function>& c) + { + exec_lines (l.begin (), l.end (), c); + } + + // Parse a special builtin line into names, performing the variable + // and pattern expansions. Optionally, skip the first token (builtin + // name, etc). + // names - exec_special (token& t, build2::script::token_type& tt, - bool omit_builtin = true); + exec_special (token&, build2::script::token_type&, bool skip_first); + + // Note: target must be file or group. + // + void + exec_depdb_preamble (action, const scope& base, const target&, + environment&, const script&, runner&, + lines_iterator begin, lines_iterator end, + depdb&, + dynamic_targets* dyn_targets = nullptr, + bool* update = nullptr, + optional<timestamp> mt = nullopt, + bool* deferred_failure = nullptr, + dyndep_byproduct* = nullptr); + + // Note: target must be file or group. + // + void + exec_depdb_dyndep (token&, build2::script::token_type&, + size_t line_index, const location&, + action, const scope& base, target&, + depdb&, + dynamic_targets& dyn_targets, + bool& update, + timestamp, + bool& deferred_failure, + dyndep_byproduct*); // Helpers. // @@ -132,7 +267,7 @@ namespace build2 // protected: virtual lookup - lookup_variable (name&&, string&&, const location&) override; + lookup_variable (names&&, string&&, const location&) override; virtual void lookup_function (string&&, const location&) override; @@ -157,9 +292,9 @@ namespace build2 script* script_; const small_vector<action, 1>* actions_; // Non-NULL during pre-parse. - // True if this script is for file-based targets and performing update - // is one of the actions, respectively. Only set for the pre-parse - // mode. + // True if this script is for file- or file group-based targets and + // performing update is one of the actions, respectively. Only set for + // the pre-parse mode. // bool file_based_; bool perform_update_; @@ -189,18 +324,24 @@ namespace build2 // // If the diag builtin is encountered, then its whole line is saved // (including the leading 'diag' word) for later execution and the - // diagnostics weight is set to 4. + // diagnostics weight is set to 4. The preceding lines, which can only + // contain variable assignments (including via the set builtin, + // potentially inside the flow control constructs), are also saved. // // Any attempt to manually set the custom diagnostics twice (the diag // builtin after the script name or after another diag builtin) is // reported as ambiguity. // - // At the end of pre-parsing either diag_name_ or diag_line_ (but not - // both) are present. + // If no script name is deduced by the end of pre-parsing and the + // script is used for a single operation, then use this operation's + // name as a script name. + // + // At the end of pre-parsing either diag_name_ is present or + // diag_preamble_ is not empty (but not both). // optional<pair<string, location>> diag_name_; optional<pair<string, location>> diag_name2_; // Ambiguous script name. - optional<pair<line, location>> diag_line_; + lines diag_preamble_; uint8_t diag_weight_ = 0; // Custom dependency change tracking. @@ -219,8 +360,24 @@ namespace build2 // depdb env <var-names> - Track the environment variables change as a // hash. // - optional<location> depdb_clear_; // 'depdb clear' location if any. - lines depdb_preamble_; // Note: excludes 'depdb clear'. + // depdb dyndep ... - Extract dynamic dependency information. Can + // only be the last depdb builtin call in the + // preamble. Note that such dependencies don't + // end up in $<. We also don't cause clean of + // such dependencies (since there may be no .d + // file) -- they should also be listed as + // static prerequisites of some other target + // (e.g., lib{} for headers) or a custom clean + // recipe should be provided. + // + // + optional<location> depdb_clear_; // depdb-clear location. + bool depdb_value_ = false; // depdb-{string,hash} + optional<pair<location, size_t>> + depdb_dyndep_; // depdb-dyndep location/position. + bool depdb_dyndep_byproduct_ = false; // --byproduct + bool depdb_dyndep_dyn_target_ = false; // --dyn-target + lines depdb_preamble_; // Note: excluding depdb-clear. // If present, the first impure function called in the body of the // script that performs update of a file-based target. @@ -233,7 +390,18 @@ namespace build2 // optional<pair<string, location>> impure_func_; - // True during pre-parsing when the pre-parse mode is temporarily + // Similar to the impure function above but for a computed (e.g., + // target-qualified) variable expansion. In this case we don't have a + // name (it's computed). + // + optional<location> computed_var_; + + // True if we (rather than the base parser) turned on the pre-parse + // mode. + // + bool top_pre_parse_; + + // True during top-pre-parsing when the pre-parse mode is temporarily // suspended to perform expansion. // bool pre_parse_suspended_ = false; @@ -243,19 +411,19 @@ namespace build2 // Before the script line gets parsed, it is set to a temporary value // that will by default be appended to the script. However, // parse_program() can point it to a different location where the line - // should be saved instead (e.g., diag_line_, etc) or set it to NULL - // if the line is handled in an ad-hoc way and should be dropped - // (e.g., depdb_clear_, etc). + // should be saved instead (e.g., diag_preamble_ back, etc) or set it + // to NULL if the line is handled in an ad-hoc way and should be + // dropped (e.g., depdb_clear_, etc). // line* save_line_; - // The if-else nesting level (and in the future for other flow - // control constructs). + // The flow control constructs nesting level. // - // Maintained during pre-parsing and is incremented when the cmd_if or - // cmd_ifn lines are encountered, which in particular means that it is - // already incremented by the time the if-condition expression is - // pre-parsed. Decremented when the cmd_end line is encountered. + // Maintained during pre-parsing and is incremented when flow control + // construct condition lines are encountered, which in particular + // means that it is already incremented by the time the condition + // expression is pre-parsed. Decremented when the cmd_end line is + // encountered. // size_t level_ = 0; diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx index 4089efa..97eac22 100644 --- a/libbuild2/build/script/parser.test.cxx +++ b/libbuild2/build/script/parser.test.cxx @@ -29,35 +29,58 @@ namespace build2 class print_runner: public runner { public: - print_runner (bool line): line_ (line) {} + print_runner (bool line, bool iterations): + line_ (line), + iterations_ (iterations) {} virtual void enter (environment&, const location&) override {} virtual void - run (environment&, + run (environment& env, const command_expr& e, - size_t i, - const location&) override + const iteration_index* ii, size_t i, + const function<command_function>& cf, + const location& ll) override { + // If the functions is specified, then just execute it with an empty + // stdin so it can perform the housekeeping (stop replaying tokens, + // increment line index, etc). + // + if (cf != nullptr) + { + assert (e.size () == 1 && !e[0].pipe.empty ()); + + const command& c (e[0].pipe.back ()); + + // Must be enforced by the caller. + // + assert (!c.out && !c.err && !c.exit); + + cf (env, c.arguments, + fdopen_null (), nullptr /* pipe */, + nullopt /* deadline */, + ll); + } + cout << e; - if (line_) - cout << " # " << i; + if (line_ || iterations_) + print_line_info (ii, i); cout << endl; } virtual bool - run_if (environment&, - const command_expr& e, - size_t i, - const location&) override + run_cond (environment&, + const command_expr& e, + const iteration_index* ii, size_t i, + const location&) override { cout << "? " << e; - if (line_) - cout << " # " << i; + if (line_ || iterations_) + print_line_info (ii, i); cout << endl; @@ -68,16 +91,36 @@ namespace build2 leave (environment&, const location&) override {} private: + void + print_line_info (const iteration_index* ii, size_t i) const + { + cout << " #"; + + if (line_) + cout << ' ' << i; + + if (iterations_ && ii != nullptr) + { + string s; + for (const iteration_index* i (ii); i != nullptr; i = i->prev) + s.insert (0, " i" + to_string (i->index)); + + cout << s; + } + } + + private: bool line_; + bool iterations_; }; // Usages: // - // argv[0] [-l] + // argv[0] [-l] [-r] // argv[0] -b [-t] // argv[0] -d [-t] + // argv[0] -g [-t] [<diag-name>] // argv[0] -q - // argv[0] -g [<diag-name>] // // In the first form read the script from stdin and trace the script // body execution to stdout using the custom print runner. @@ -88,26 +131,33 @@ namespace build2 // In the third form read the script from stdin, parse it and dump the // depdb preamble lines to stdout. // - // In the forth form read the script from stdin, parse it and print - // line tokens quoting information to stdout. - // - // In the fifth form read the script from stdin, parse it and print the + // In the forth form read the script from stdin, parse it and print the // low-verbosity script diagnostics name or custom low-verbosity // diagnostics to stdout. If the script doesn't deduce any of them, then // print the diagnostics and exit with non-zero code. // + // In the fifth form read the script from stdin, parse it and print + // line tokens quoting information to stdout. + // // -l // Print the script line number for each executed expression. // + // -r + // Print the loop iteration numbers for each executed expression. + // // -b // Dump the parsed script body to stdout. // // -d // Dump the parsed script depdb preamble to stdout. // + // -g + // Dump the low-verbosity script diagnostics name or custom + // low-verbosity diagnostics to stdout. + // // -t - // Print true if the body (-b) or depdb preamble (-d) references the - // temporary directory and false otherwise. + // Print true if the body (-b), depdb preamble (-d), or diag preamble + // (-g) references the temporary directory and false otherwise. // // -q // Print the parsed script tokens quoting information to sdout. If a @@ -117,10 +167,6 @@ namespace build2 // <quoting> := 'S' | 'D' | 'M' // <completeness> := 'C' | 'P' // - // -g - // Dump the low-verbosity script diagnostics name or custom - // low-verbosity diagnostics to stdout. - // int main (int argc, char* argv[]) { @@ -131,11 +177,12 @@ namespace build2 run, body, depdb_preamble, - quoting, - diag + diag, + quoting } m (mode::run); bool print_line (false); + bool print_iterations (false); optional<string> diag_name; bool temp_dir (false); @@ -145,19 +192,23 @@ namespace build2 if (a == "-l") print_line = true; + else if (a == "-r") + print_iterations = true; else if (a == "-b") m = mode::body; else if (a == "-d") m = mode::depdb_preamble; + else if (a == "-g") + m = mode::diag; else if (a == "-t") { - assert (m == mode::body || m == mode::depdb_preamble); + assert (m == mode::body || + m == mode::depdb_preamble || + m == mode::diag); temp_dir = true; } else if (a == "-q") m = mode::quoting; - else if (a == "-g") - m = mode::diag; else { if (m == mode::diag) @@ -170,19 +221,20 @@ namespace build2 } } - assert (!print_line || m == mode::run); - assert (!diag_name || m == mode::diag); + assert (!print_line || m == mode::run || m == mode::diag); + assert (!print_iterations || m == mode::run || m == mode::diag); + assert (!diag_name || m == mode::diag); // Fake build system driver, default verbosity. // init_diag (1); - init (nullptr, argv[0]); + init (nullptr, argv[0], true); // Serial execution. // scheduler sched (1); global_mutexes mutexes (1); - file_cache fcache; + file_cache fcache (true); context ctx (sched, mutexes, fcache); try @@ -203,6 +255,8 @@ namespace build2 tt.path (path ("driver")); + const scope& bs (tt.base_scope ()); + small_vector<action, 1> acts {perform_update_id}; // Parse and run. @@ -210,7 +264,7 @@ namespace build2 parser p (ctx); path_name nm ("buildfile"); - script s (p.pre_parse (tt.base_scope (), tt.type (), acts, + script s (p.pre_parse (bs, tt.type (), acts, cin, nm, 11 /* line */, (m != mode::diag @@ -222,9 +276,29 @@ namespace build2 { case mode::run: { - environment e (perform_update_id, tt, s.body_temp_dir); - print_runner r (print_line); - p.execute_body (ctx.global_scope, ctx.global_scope, e, s, r); + environment e (perform_update_id, tt, bs, false /* temp_dir */); + print_runner r (print_line, print_iterations); + + bool exec_diag (!s.diag_preamble.empty ()); + + if (exec_diag) + { + if (s.diag_preamble_temp_dir) + e.set_temp_dir_variable (); + + p.execute_diag_preamble (ctx.global_scope, ctx.global_scope, + e, s, r, + false /* diag */, + true /* enter */, + false /* leave */); + } + + if (s.body_temp_dir && !s.diag_preamble_temp_dir) + e.set_temp_dir_variable (); + + p.execute_body (ctx.global_scope, ctx.global_scope, + e, s, r, + !exec_diag /* enter */); break; } case mode::diag: @@ -235,14 +309,26 @@ namespace build2 } else { - assert (s.diag_line); + if (!temp_dir) + { + environment e (perform_update_id, + tt, + bs, + s.diag_preamble_temp_dir); - environment e (perform_update_id, tt, false /* temp_dir */); + print_runner r (print_line, print_iterations); - cout << "diag: " << p.execute_special (ctx.global_scope, + names diag (p.execute_diag_preamble (ctx.global_scope, ctx.global_scope, - e, - *s.diag_line) << endl; + e, s, r, + true /* diag */, + true /* enter */, + true /* leave */).first); + + cout << "diag: " << diag << endl; + } + else + cout << (s.diag_preamble_temp_dir ? "true" : "false") << endl; } break; diff --git a/libbuild2/build/script/runner.cxx b/libbuild2/build/script/runner.cxx index 51139d4..5d9764b 100644 --- a/libbuild2/build/script/runner.cxx +++ b/libbuild2/build/script/runner.cxx @@ -28,12 +28,37 @@ namespace build2 // for (auto i (env.cleanups.begin ()); i != env.cleanups.end (); ) { - const target* m (&env.target); - for (; m != nullptr; m = m->adhoc_member) + const target* m (nullptr); + if (const group* g = env.target.is_a<group> ()) { - if (const path_target* pm = m->is_a<path_target> ()) - if (i->path == pm->path ()) - break; + for (const target* gm: g->members) + { + if (const path_target* pm = gm->is_a<path_target> ()) + { + if (i->path == pm->path ()) + { + m = gm; + break; + } + } + } + } + else if (const fsdir* fd = env.target.is_a<fsdir> ()) + { + // Compare ignoring the trailing directory separator. + // + if (path_traits::compare (i->path.string (), + fd->dir.string ()) == 0) + m = fd; + } + else + { + for (m = &env.target; m != nullptr; m = m->adhoc_member) + { + if (const path_target* pm = m->is_a<path_target> ()) + if (i->path == pm->path ()) + break; + } } if (m != nullptr) @@ -96,39 +121,43 @@ namespace build2 void default_runner:: run (environment& env, const command_expr& expr, - size_t li, + const iteration_index* ii, size_t li, + const function<command_function>& cf, const location& ll) { if (verb >= 3) text << ": " << expr; // Run the expression if we are not in the dry-run mode or if it - // executes the set or exit builtin and just print the expression - // otherwise at verbosity level 2 and up. + // executes the set or exit builtin or it is a for-loop. Otherwise, + // just print the expression otherwise at verbosity level 2 and up. // if (!env.context.dry_run || find_if (expr.begin (), expr.end (), - [] (const expr_term& et) + [&cf] (const expr_term& et) { const process_path& p (et.pipe.back ().program); return p.initial == nullptr && (p.recall.string () == "set" || - p.recall.string () == "exit"); + p.recall.string () == "exit" || + (cf != nullptr && + p.recall.string () == "for")); }) != expr.end ()) - build2::script::run (env, expr, li, ll); + build2::script::run (env, expr, ii, li, ll, cf); else if (verb >= 2) text << expr; } bool default_runner:: - run_if (environment& env, - const command_expr& expr, - size_t li, const location& ll) + run_cond (environment& env, + const command_expr& expr, + const iteration_index* ii, size_t li, + const location& ll) { if (verb >= 3) text << ": ?" << expr; - return build2::script::run_if (env, expr, li, ll); + return build2::script::run_cond (env, expr, ii, li, ll); } } } diff --git a/libbuild2/build/script/runner.hxx b/libbuild2/build/script/runner.hxx index 431c446..ec8a948 100644 --- a/libbuild2/build/script/runner.hxx +++ b/libbuild2/build/script/runner.hxx @@ -32,17 +32,21 @@ namespace build2 // Location is the start position of this command line in the script. // It can be used in diagnostics. // + // Optionally, execute the specified function instead of the last + // pipe command. + // virtual void run (environment&, const command_expr&, - size_t index, + const iteration_index*, size_t index, + const function<command_function>&, const location&) = 0; virtual bool - run_if (environment&, - const command_expr&, - size_t, - const location&) = 0; + run_cond (environment&, + const command_expr&, + const iteration_index*, size_t, + const location&) = 0; // Location is the script end location (for diagnostics, etc). // @@ -52,9 +56,9 @@ namespace build2 // Run command expressions. // - // In dry-run mode don't run the expressions unless they are if- - // conditions or execute the set or exit builtins, but prints them at - // verbosity level 2 and up. + // In dry-run mode don't run the expressions unless they are flow + // control construct conditions or execute the set or exit builtins, but + // print them at verbosity level 2 and up. // class default_runner: public runner { @@ -65,14 +69,15 @@ namespace build2 virtual void run (environment&, const command_expr&, - size_t, + const iteration_index*, size_t, + const function<command_function>&, const location&) override; virtual bool - run_if (environment&, - const command_expr&, - size_t, - const location&) override; + run_cond (environment&, + const command_expr&, + const iteration_index*, size_t, + const location&) override; virtual void leave (environment&, const location&) override; diff --git a/libbuild2/build/script/script.cxx b/libbuild2/build/script/script.cxx index f4f8da8..0d96cc3 100644 --- a/libbuild2/build/script/script.cxx +++ b/libbuild2/build/script/script.cxx @@ -7,6 +7,8 @@ #include <libbuild2/target.hxx> +#include <libbuild2/adhoc-rule-buildscript.hxx> // include_unmatch* + #include <libbuild2/script/timeout.hxx> #include <libbuild2/build/script/parser.hxx> @@ -28,54 +30,92 @@ namespace build2 environment:: environment (action a, const target_type& t, + const scope_type& s, bool temp, const optional<timestamp>& dl) : build2::script::environment ( t.ctx, - cast<target_triplet> (t.ctx.global_scope["build.host"]), + *t.ctx.build_host, dir_name_view (&work, &wd_name), temp_dir.path, false /* temp_dir_keep */, redirect (redirect_type::none), redirect (redirect_type::merge, 2), redirect (redirect_type::pass)), target (t), - vars (context, false /* global */), + scope (s), + vars (context, false /* shared */), // Note: managed. + var_ts (var_pool.insert (">")), + var_ps (var_pool.insert ("<")), script_deadline (to_deadline (dl, false /* success */)) { - // Set special variables. - // + set_special_variables (a); + + if (temp) + set_temp_dir_variable (); + } + + void environment:: + set_special_variables (action a) + { { // $> // + // What should it contain for an explicit group? While it may seem + // that just the members should be enough (and analogous to the ad + // hoc case), this won't let us get the group name for diagnostics. + // So the group name followed by all the members seems like the + // logical choice. + // names ns; - for (const target_type* m (&t); m != nullptr; m = m->adhoc_member) - m->as_name (ns); - assign (var_pool.insert (">")) = move (ns); + if (const group* g = target.is_a<group> ()) + { + g->as_name (ns); + for (const target_type* m: g->members) + m->as_name (ns); + } + else + { + for (const target_type* m (&target); + m != nullptr; + m = m->adhoc_member) + m->as_name (ns); + } + + assign (var_ts) = move (ns); } { // $< // - // Note that at this stage (after execute_prerequisites()) ad hoc - // prerequisites are no longer in prerequisite_targets which means - // they won't end up in $< either. While at first thought ad hoc - // prerequisites in ad hoc recipes don't seem to make much sense, - // they could be handy to exclude certain preresquisites from $< - // while still treating them as such. + // Note that ad hoc prerequisites don't end up in $<. While at first + // thought ad hoc prerequisites in ad hoc recipes don't seem to make + // much sense, they could be handy to exclude certain prerequisites + // from $< while still treating them as such, especially in rule. + // + // While initially we treated update=unmatch prerequisites as + // implicitly ad hoc, this turned out to be not quite correct, so + // now we add them unless they are explicitly marked ad hoc. // names ns; - for (const target_type* pt: t.prerequisite_targets[a]) + for (const prerequisite_target& p: target.prerequisite_targets[a]) { - if (pt != nullptr) + // See adhoc_buildscript_rule::execute_update_prerequisites(). + // + if (const target_type* pt = + p.target != nullptr ? (p.adhoc () ? nullptr : p.target) : + (p.include & adhoc_buildscript_rule::include_unmatch) != 0 && + (p.include & prerequisite_target::include_adhoc) == 0 && + (p.include & adhoc_buildscript_rule::include_unmatch_adhoc) == 0 + ? reinterpret_cast<target_type*> (p.data) + : nullptr) + { pt->as_name (ns); + } } - assign (var_pool.insert ("<")) = move (ns); + assign (var_ps) = move (ns); } - - if (temp) - set_temp_dir_variable (); } void environment:: @@ -146,7 +186,7 @@ namespace build2 } void environment:: - set_variable (string&& nm, + set_variable (string nm, names&& val, const string& attrs, const location& ll) @@ -225,7 +265,7 @@ namespace build2 // in parallel). Plus, if there is no such variable, then we cannot // possibly find any value. // - const variable* pvar (context.var_pool.find (n)); + const variable* pvar (scope.var_pool ().find (n)); if (pvar == nullptr) return lookup_type (); diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx index e11cb45..08f1bf4 100644 --- a/libbuild2/build/script/script.hxx +++ b/libbuild2/build/script/script.hxx @@ -20,14 +20,22 @@ namespace build2 namespace script { using build2::script::line; - using build2::script::lines; using build2::script::line_type; + using build2::script::lines; using build2::script::redirect; using build2::script::redirect_type; + using build2::script::command; using build2::script::expr_term; using build2::script::command_expr; + using build2::script::iteration_index; using build2::script::deadline; using build2::script::timeout; + using build2::script::pipe_command; + using build2::script::command_function; + + // Forward declarations. + // + class default_runner; // Notes: // @@ -40,13 +48,11 @@ namespace build2 class script { public: - using lines_type = build::script::lines; - // Note that the variables are not pre-entered into a pool during the // parsing phase, so the line variable pointers are NULL. // - lines_type body; - bool body_temp_dir = false; // True if the body references $~. + lines body; + bool body_temp_dir = false; // True if the body references $~. // Referenced ordinary (non-special) variables. // @@ -61,18 +67,24 @@ namespace build2 small_vector<string, 2> vars; // 2 for command and options. // Command name for low-verbosity diagnostics and custom low-verbosity - // diagnostics line. Note: cannot be both (see the script parser for + // diagnostics line, potentially preceded with the variable + // assignments. Note: cannot be both (see the script parser for // details). // optional<string> diag_name; - optional<line> diag_line; + lines diag_preamble; + bool diag_preamble_temp_dir = false; // True if refs $~. // The script's custom dependency change tracking lines (see the // script parser for details). // - bool depdb_clear; - lines_type depdb_preamble; - bool depdb_preamble_temp_dir = false; // True if references $~. + bool depdb_clear; + bool depdb_value; // String or hash. + optional<size_t> depdb_dyndep; // Pos of first dyndep. + bool depdb_dyndep_byproduct = false; // dyndep --byproduct + bool depdb_dyndep_dyn_target = false;// dyndep --dyn-target + lines depdb_preamble; // Note include vars. + bool depdb_preamble_temp_dir = false;// True if refs $~. location start_loc; location end_loc; @@ -81,24 +93,38 @@ namespace build2 class environment: public build2::script::environment { public: + using scope_type = build2::scope; using target_type = build2::target; environment (action, const target_type&, + const scope_type&, bool temp_dir, const optional<timestamp>& deadline = nullopt); + // (Re)set special $< and $> variables. + // + void + set_special_variables (action); + + // Create the temporary directory (if it doesn't exist yet) and set + // the $~ special variable to its path. + // + void + set_temp_dir_variable (); + environment (environment&&) = delete; environment (const environment&) = delete; environment& operator= (environment&&) = delete; environment& operator= (const environment&) = delete; public: - // Primary target this environment is for. + // Primary target this environment is for and its base scope; // const target_type& target; + const scope_type& scope; - // Script-local variable pool and map. + // Script-private variable pool and map. // // Note that it may be tempting to reuse the rule-specific variables // for this but they should not be modified during execution (i.e., @@ -111,6 +137,9 @@ namespace build2 variable_pool var_pool; variable_map vars; + const variable& var_ts; // $> + const variable& var_ps; // $< + // Temporary directory for the script run. // // Currently this directory is removed regardless of the script @@ -140,14 +169,8 @@ namespace build2 // size_t exec_line = 1; - // Create the temporary directory (if it doesn't exist yet) and set - // the $~ special variable to its path. - // - void - set_temp_dir_variable (); - virtual void - set_variable (string&& name, + set_variable (string name, names&&, const string& attrs, const location&) override; |