diff options
Diffstat (limited to 'libbuild2/build')
-rw-r--r-- | libbuild2/build/script/builtin-options.cxx | 72 | ||||
-rw-r--r-- | libbuild2/build/script/builtin-options.hxx | 68 | ||||
-rw-r--r-- | libbuild2/build/script/builtin-options.ixx | 120 | ||||
-rw-r--r-- | libbuild2/build/script/builtin.cli | 72 | ||||
-rw-r--r-- | libbuild2/build/script/lexer+for-loop.test.testscript | 188 | ||||
-rw-r--r-- | libbuild2/build/script/lexer.cxx | 48 | ||||
-rw-r--r-- | libbuild2/build/script/lexer.hxx | 9 | ||||
-rw-r--r-- | libbuild2/build/script/lexer.test.cxx | 1 | ||||
-rw-r--r-- | libbuild2/build/script/parser+diag.test.testscript | 106 | ||||
-rw-r--r-- | libbuild2/build/script/parser+for.test.testscript | 494 | ||||
-rw-r--r-- | libbuild2/build/script/parser.cxx | 1405 | ||||
-rw-r--r-- | libbuild2/build/script/parser.hxx | 103 | ||||
-rw-r--r-- | libbuild2/build/script/parser.test.cxx | 113 | ||||
-rw-r--r-- | libbuild2/build/script/runner.cxx | 48 | ||||
-rw-r--r-- | libbuild2/build/script/runner.hxx | 5 | ||||
-rw-r--r-- | libbuild2/build/script/script.cxx | 52 | ||||
-rw-r--r-- | libbuild2/build/script/script.hxx | 29 |
17 files changed, 2563 insertions, 370 deletions
diff --git a/libbuild2/build/script/builtin-options.cxx b/libbuild2/build/script/builtin-options.cxx index f7ba0e7..dba3c59 100644 --- a/libbuild2/build/script/builtin-options.cxx +++ b/libbuild2/build/script/builtin-options.cxx @@ -188,6 +188,56 @@ namespace build2 } }; + template <typename K, typename V, typename C> + struct parser<std::multimap<K, V, C> > + { + static void + parse (std::multimap<K, V, C>& m, bool& xs, scanner& s) + { + const char* o (s.next ()); + + if (s.more ()) + { + std::size_t pos (s.position ()); + std::string ov (s.next ()); + std::string::size_type p = ov.find ('='); + + K k = K (); + V v = V (); + std::string kstr (ov, 0, p); + std::string vstr (ov, (p != std::string::npos ? p + 1 : ov.size ())); + + int ac (2); + char* av[] = + { + const_cast<char*> (o), + 0 + }; + + bool dummy; + if (!kstr.empty ()) + { + av[1] = const_cast<char*> (kstr.c_str ()); + argv_scanner s (0, ac, av, false, pos); + parser<K>::parse (k, dummy, s); + } + + if (!vstr.empty ()) + { + av[1] = const_cast<char*> (vstr.c_str ()); + argv_scanner s (0, ac, av, false, pos); + parser<V>::parse (v, dummy, s); + } + + m.insert (typename std::multimap<K, V, C>::value_type (k, v)); + } + else + throw missing_value (o); + + xs = true; + } + }; + template <typename X, typename T, T X::*M> void thunk (X& x, scanner& s) @@ -239,7 +289,15 @@ namespace build2 adhoc_ (), cwd_ (), cwd_specified_ (false), - drop_cycles_ () + drop_cycles_ (), + target_what_ (), + target_what_specified_ (false), + target_default_type_ (), + target_default_type_specified_ (false), + target_extension_type_ (), + target_extension_type_specified_ (false), + target_cwd_ (), + target_cwd_specified_ (false) { } @@ -341,6 +399,18 @@ namespace build2 &depdb_dyndep_options::cwd_specified_ >; _cli_depdb_dyndep_options_map_["--drop-cycles"] = &::build2::build::cli::thunk< depdb_dyndep_options, &depdb_dyndep_options::drop_cycles_ >; + _cli_depdb_dyndep_options_map_["--target-what"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_what_, + &depdb_dyndep_options::target_what_specified_ >; + _cli_depdb_dyndep_options_map_["--target-default-type"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_default_type_, + &depdb_dyndep_options::target_default_type_specified_ >; + _cli_depdb_dyndep_options_map_["--target-extension-type"] = + &::build2::build::cli::thunk< depdb_dyndep_options, map<string, string>, &depdb_dyndep_options::target_extension_type_, + &depdb_dyndep_options::target_extension_type_specified_ >; + _cli_depdb_dyndep_options_map_["--target-cwd"] = + &::build2::build::cli::thunk< depdb_dyndep_options, dir_path, &depdb_dyndep_options::target_cwd_, + &depdb_dyndep_options::target_cwd_specified_ >; } }; diff --git a/libbuild2/build/script/builtin-options.hxx b/libbuild2/build/script/builtin-options.hxx index 590d3b2..a8c3440 100644 --- a/libbuild2/build/script/builtin-options.hxx +++ b/libbuild2/build/script/builtin-options.hxx @@ -174,6 +174,66 @@ namespace build2 void drop_cycles (const bool&); + const string& + target_what () const; + + string& + target_what (); + + void + target_what (const string&); + + bool + target_what_specified () const; + + void + target_what_specified (bool); + + const string& + target_default_type () const; + + string& + target_default_type (); + + void + target_default_type (const string&); + + bool + target_default_type_specified () const; + + void + target_default_type_specified (bool); + + const map<string, string>& + target_extension_type () const; + + map<string, string>& + target_extension_type (); + + void + target_extension_type (const map<string, string>&); + + bool + target_extension_type_specified () const; + + void + target_extension_type_specified (bool); + + const dir_path& + target_cwd () const; + + dir_path& + target_cwd (); + + void + target_cwd (const dir_path&); + + bool + target_cwd_specified () const; + + void + target_cwd_specified (bool); + // Implementation details. // protected: @@ -201,6 +261,14 @@ namespace build2 dir_path cwd_; bool cwd_specified_; bool drop_cycles_; + string target_what_; + bool target_what_specified_; + string target_default_type_; + bool target_default_type_specified_; + map<string, string> target_extension_type_; + bool target_extension_type_specified_; + dir_path target_cwd_; + bool target_cwd_specified_; }; } } diff --git a/libbuild2/build/script/builtin-options.ixx b/libbuild2/build/script/builtin-options.ixx index ea06a0f..20847c2 100644 --- a/libbuild2/build/script/builtin-options.ixx +++ b/libbuild2/build/script/builtin-options.ixx @@ -233,6 +233,126 @@ namespace build2 { this->drop_cycles_ = x; } + + inline const string& depdb_dyndep_options:: + target_what () const + { + return this->target_what_; + } + + inline string& depdb_dyndep_options:: + target_what () + { + return this->target_what_; + } + + inline void depdb_dyndep_options:: + target_what (const string& x) + { + this->target_what_ = x; + } + + inline bool depdb_dyndep_options:: + target_what_specified () const + { + return this->target_what_specified_; + } + + inline void depdb_dyndep_options:: + target_what_specified (bool x) + { + this->target_what_specified_ = x; + } + + inline const string& depdb_dyndep_options:: + target_default_type () const + { + return this->target_default_type_; + } + + inline string& depdb_dyndep_options:: + target_default_type () + { + return this->target_default_type_; + } + + inline void depdb_dyndep_options:: + target_default_type (const string& x) + { + this->target_default_type_ = x; + } + + inline bool depdb_dyndep_options:: + target_default_type_specified () const + { + return this->target_default_type_specified_; + } + + inline void depdb_dyndep_options:: + target_default_type_specified (bool x) + { + this->target_default_type_specified_ = x; + } + + inline const map<string, string>& depdb_dyndep_options:: + target_extension_type () const + { + return this->target_extension_type_; + } + + inline map<string, string>& depdb_dyndep_options:: + target_extension_type () + { + return this->target_extension_type_; + } + + inline void depdb_dyndep_options:: + target_extension_type (const map<string, string>& x) + { + this->target_extension_type_ = x; + } + + inline bool depdb_dyndep_options:: + target_extension_type_specified () const + { + return this->target_extension_type_specified_; + } + + inline void depdb_dyndep_options:: + target_extension_type_specified (bool x) + { + this->target_extension_type_specified_ = x; + } + + inline const dir_path& depdb_dyndep_options:: + target_cwd () const + { + return this->target_cwd_; + } + + inline dir_path& depdb_dyndep_options:: + target_cwd () + { + return this->target_cwd_; + } + + inline void depdb_dyndep_options:: + target_cwd (const dir_path& x) + { + this->target_cwd_ = x; + } + + inline bool depdb_dyndep_options:: + target_cwd_specified () const + { + return this->target_cwd_specified_; + } + + inline void depdb_dyndep_options:: + target_cwd_specified (bool x) + { + this->target_cwd_specified_ = x; + } } } } diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli index 7d0936f..5aea034 100644 --- a/libbuild2/build/script/builtin.cli +++ b/libbuild2/build/script/builtin.cli @@ -17,8 +17,8 @@ namespace build2 // class depdb_dyndep_options { - // Note that --byproduct, if any, must be the first option and is - // handled ad hoc, kind of as a sub-command. + // Note that --byproduct or --dyn-target, if any, must be the first + // option and is handled ad hoc. // // Similarly, --update-{include,exclude} are handled ad hoc and must // be literals, similar to the -- separator. They specify prerequisite @@ -40,23 +40,48 @@ namespace build2 // with support for generated files (and thus -I) at least in the make // format where we use relative paths for non-existent files. // + // Currently Supported dependency formats (--format) are `make` + // (default) and `lines`. + // + // The `make` format is the make dependency declaration in the + // `<target>...: [<prerequisite>...]` form. In the non-byproduct mode + // a relative prerequisite path is considered non-existent. + // + // The `lines` format lists targets and/or prerequisites one per line. + // If the --dyn-target option is specified then the target list is + // expected to come first separated from the prerequisites list with a + // blank line. If there are no prerequisites, then the blank line can + // be omitted. If the --dyn-target option is not specified, then all + // lines are treated as prerequisites and there should be no blank + // lines. In the non-byproduct mode a prerequisite line that starts + // with a leading space is considered a non-existent prerequisite. + // Currently only relative non-existent prerequisites are supported. + // Finally, in this mode, if the prerequisite is syntactically a + // directory (that is, it ends with a trailing directory separator), + // then it is added as fsdir{}. This can be used to handle situations + // where the dynamic targets are placed into subdirectories. + // // Note on naming: whenever we (may) have two options, one for target // and the other for prerequisite, we omit "prerequisite" as that's // what we extract by default and most commonly. For example: // - // --what --what-target - // --default-type --default-target-type + // --what --target-what + // --default-type --target-default-type // path --file; // Read from file rather than stdin. - string --format; // Dependency format: make (default). + string --format; // Dependency format: `make` (default), + // or `lines`. - string --what; // Dependency kind, e.g., "header". + // Dynamic dependency extraction options. + // + string --what; // Prerequisite kind, e.g., "header". - dir_paths --include-path|-I; // Search paths for generated files. + dir_paths --include-path|-I; // Search paths for generated + // prerequisites. - string --default-type; // Default prerequisite type to use - // if none could be derived from ext. + string --default-type; // Default prerequisite type to use if + // none could be derived from extension. bool --adhoc; // Treat dynamically discovered // prerequisites as ad hoc (so they @@ -64,14 +89,39 @@ namespace build2 // normal mode). dir_path --cwd; // Builtin's working directory used - // to complete relative paths (only - // in --byproduct mode). + // to complete relative paths of + // prerequisites (only in --byproduct + // mode, lines format for existing + // paths). bool --drop-cycles; // Drop prerequisites that are also // targets. Only use if you are sure // such cycles are harmless, that is, // the output is not affected by such // prerequisites' content. + + // Dynamic target extraction options. + // + // This functionality is enabled with the --dyn-target option. Only + // the make format is supported, where the listed targets are added as + // ad hoc group members (unless already specified as static members). + // This functionality is not available in the byproduct mode. + // + string --target-what; // Target kind, e.g., "source". + + string --target-default-type; // Default target type to use if none + // could be derived from extension. + + map<string, string> // Extension to target type mapping in + --target-extension-type; // the <ext>=<type> form, for example, + // h=hxx. This mapping is considered + // before attempting to automatically + // map the extension and so can be used + // to resolve ambiguities. + + dir_path --target-cwd; // Builtin's working directory used to + // complete relative paths of targets. + }; } } diff --git a/libbuild2/build/script/lexer+for-loop.test.testscript b/libbuild2/build/script/lexer+for-loop.test.testscript new file mode 100644 index 0000000..3f8e6b5 --- /dev/null +++ b/libbuild2/build/script/lexer+for-loop.test.testscript @@ -0,0 +1,188 @@ +# file : libbuild2/build/script/lexer+for-loop.test.testscript +# license : MIT; see accompanying LICENSE file + +test.arguments = for-loop + +: redirect +: +{ + : pass + : + $* <"cmd <| 1>|" >>EOO + 'cmd' + <| + '1' + >| + <newline> + EOO + + : null + : + $* <"cmd <- 1>-" >>EOO + 'cmd' + <- + '1' + >- + <newline> + EOO + + : trace + : + $* <"cmd 1>!" >>EOO + 'cmd' + '1' + >! + <newline> + EOO + + : merge + : + $* <"cmd 1>&2" >>EOO + 'cmd' + '1' + >& + '2' + <newline> + EOO + + : str + : + $* <"cmd <<<=a 1>>>?b" >>EOO + 'cmd' + <<<= + 'a' + '1' + >>>? + 'b' + <newline> + EOO + + : str-nn + : + $* <"cmd <<<=:a 1>>>?:b" >>EOO + 'cmd' + <<<=: + 'a' + '1' + >>>?: + 'b' + <newline> + EOO + + : str-nn-alias + : + $* <"cmd <<<:a 1>>>?:b" >>EOO + 'cmd' + <<<: + 'a' + '1' + >>>?: + 'b' + <newline> + EOO + + : doc + : + $* <"cmd <<EOI 1>>EOO" >>EOO + 'cmd' + << + 'EOI' + '1' + >> + 'EOO' + <newline> + EOO + + : doc-nn + : + $* <"cmd <<:EOI 1>>?:EOO" >>EOO + 'cmd' + <<: + 'EOI' + '1' + >>?: + 'EOO' + <newline> + EOO + + : file-cmp + : + $* <"cmd <=in >?out 2>?err" >>EOO + 'cmd' + <= + 'in' + >? + 'out' + '2' + >? + 'err' + <newline> + EOO + + : file-write + : + $* <"cmd >=out 2>+err" >>EOO + 'cmd' + >= + 'out' + '2' + >+ + 'err' + <newline> + EOO +} + +: cleanup +: +{ + : always + : + $* <"cmd &file" >>EOO + 'cmd' + & + 'file' + <newline> + EOO + + : maybe + : + $* <"cmd &?file" >>EOO + 'cmd' + &? + 'file' + <newline> + EOO + + : never + : + $* <"cmd &!file" >>EOO + 'cmd' + &! + 'file' + <newline> + EOO +} + +: for +: +{ + : form-1 + : + $* <"for x: a" >>EOO + 'for' + 'x' + : + 'a' + <newline> + EOO + + : form-3 + : + $* <"for <<<a x" >>EOO + 'for' + <<< + 'a' + 'x' + <newline> + EOO +} diff --git a/libbuild2/build/script/lexer.cxx b/libbuild2/build/script/lexer.cxx index d849ac9..e0d87fe 100644 --- a/libbuild2/build/script/lexer.cxx +++ b/libbuild2/build/script/lexer.cxx @@ -35,10 +35,7 @@ namespace build2 bool q (true); // quotes if (!esc) - { - assert (!state_.empty ()); - esc = state_.top ().escapes; - } + esc = current_state ().escapes; switch (m) { @@ -78,6 +75,19 @@ namespace build2 s2 = " "; break; } + case lexer_mode::for_loop: + { + // Leading tokens of the for-loop. Like command_line but + // recognizes colon as a separator and lsbrace like value. + // + // Note that while sensing the form of the for-loop (`for x:...` + // vs `for x <...`) we need to make sure that the pre-parsed token + // types are valid for the execution phase. + // + s1 = ":=!|&<> $(#\t\n"; + s2 = " == "; + break; + } default: { // Recognize special variable names ($>, $<, $~). @@ -94,7 +104,7 @@ namespace build2 } assert (ps == '\0'); - state_.push ( + mode_impl ( state {m, data, nullopt, false, false, ps, s, n, q, *esc, s1, s2}); } @@ -103,12 +113,13 @@ namespace build2 { token r; - switch (state_.top ().mode) + switch (mode ()) { case lexer_mode::command_line: case lexer_mode::first_token: case lexer_mode::second_token: case lexer_mode::variable_line: + case lexer_mode::for_loop: r = next_line (); break; default: return base_lexer::next (); @@ -128,7 +139,7 @@ namespace build2 xchar c (get ()); uint64_t ln (c.line), cn (c.column); - state st (state_.top ()); // Make copy (see first/second_token). + state st (current_state ()); // Make copy (see first/second_token). lexer_mode m (st.mode); auto make_token = [&sep, ln, cn] (type t) @@ -141,9 +152,10 @@ namespace build2 // if (st.lsbrace) { - assert (m == lexer_mode::variable_line); + assert (m == lexer_mode::variable_line || + m == lexer_mode::for_loop); - state_.top ().lsbrace = false; // Note: st is a copy. + current_state ().lsbrace = false; // Note: st is a copy. if (c == '[' && (!st.lsbrace_unsep || !sep)) return make_token (type::lsbrace); @@ -156,7 +168,7 @@ namespace build2 // we push any new mode (e.g., double quote). // if (m == lexer_mode::first_token || m == lexer_mode::second_token) - state_.pop (); + expire_mode (); // NOTE: remember to update mode() if adding new special characters. @@ -167,7 +179,7 @@ namespace build2 // Expire variable value mode at the end of the line. // if (m == lexer_mode::variable_line) - state_.pop (); + expire_mode (); sep = true; // Treat newline as always separated. return make_token (type::newline); @@ -179,11 +191,20 @@ namespace build2 case '(': return make_token (type::lparen); } + if (m == lexer_mode::for_loop) + { + switch (c) + { + case ':': return make_token (type::colon); + } + } + // Command line operator/separators. // if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { switch (c) { @@ -205,7 +226,8 @@ namespace build2 // if (m == lexer_mode::command_line || m == lexer_mode::first_token || - m == lexer_mode::second_token) + m == lexer_mode::second_token || + m == lexer_mode::for_loop) { if (optional<token> t = next_cmd_op (c, sep)) return move (*t); diff --git a/libbuild2/build/script/lexer.hxx b/libbuild2/build/script/lexer.hxx index 646d3b9..3f51493 100644 --- a/libbuild2/build/script/lexer.hxx +++ b/libbuild2/build/script/lexer.hxx @@ -24,9 +24,10 @@ namespace build2 enum { command_line = base_type::value_next, - first_token, // Expires at the end of the token. - second_token, // Expires at the end of the token. - variable_line // Expires at the end of the line. + first_token, // Expires at the end of the token. + second_token, // Expires at the end of the token. + variable_line, // Expires at the end of the line. + for_loop // Used for sensing the for-loop leading tokens. }; lexer_mode () = default; @@ -67,6 +68,8 @@ namespace build2 static redirect_aliases_type redirect_aliases; private: + using build2::script::lexer::mode; // Getter. + token next_line (); }; diff --git a/libbuild2/build/script/lexer.test.cxx b/libbuild2/build/script/lexer.test.cxx index e496f94..d8733ba 100644 --- a/libbuild2/build/script/lexer.test.cxx +++ b/libbuild2/build/script/lexer.test.cxx @@ -35,6 +35,7 @@ namespace build2 else if (s == "second-token") m = lexer_mode::second_token; else if (s == "variable-line") m = lexer_mode::variable_line; else if (s == "variable") m = lexer_mode::variable; + else if (s == "for-loop") m = lexer_mode::for_loop; else assert (false); } diff --git a/libbuild2/build/script/parser+diag.test.testscript b/libbuild2/build/script/parser+diag.test.testscript index 30eb859..504c9a4 100644 --- a/libbuild2/build/script/parser+diag.test.testscript +++ b/libbuild2/build/script/parser+diag.test.testscript @@ -19,17 +19,99 @@ $* <<EOI >>EOO name: echo EOO -: diag +: name-operation : -$* <<EOI >>~%EOO% - echo abc - cat abc - diag copy >= $> - cp <- $> +$* <<EOI >>EOO + a = 'b' EOI - %diag: copy >= .+file\{driver\.\}% + name: update EOO +: preamble +: +{ + : disambiguate + : + $* <<EOI >>~%EOO% + echo abc | set v + cat abc | set v + diag copy >= $> + cp <- $> + EOI + echo abc | set v + cat abc | set v + %diag: copy >= .+file\{driver\.\}% + EOO + + : name + : + $* <<EOI >>EOO + n = foo + diag copy $n + cp $n $> + EOI + diag: copy foo + EOO + + : quoted + : + $* <<EOI >'diag: foo' + f = foo + diag "$f" + EOI + + : quoted-eval + : + $* <<EOI >'diag: foo' + f = foo + diag "($f)" + EOI + + : temp_dir + : + { + test.options += -t + + : no + : + $* <<EOI >false + f = foo + diag $f + f = $~/f + foo "$f" + EOI + + : no-depdb + : + $* <<EOI >false + f = $~/f + depdb hash "$f" + diag $f + f = $~/f + foo "$f" + EOI + + : yes + : + $* <<EOI >true + f = $~/f + diag $f + foo $f + EOI + + : yes-depdb + : + $* <<EOI >true + f = $~/f + depdb hash "$f" + f = $~/t + diag $f + f = $~/f + foo "$f" + EOI + } +} + : ambiguity : { @@ -67,16 +149,6 @@ $* <<EOI >>~%EOO% info: consider specifying it explicitly with the 'diag' recipe attribute info: or provide custom low-verbosity diagnostics with the 'diag' builtin EOE - - : none - : - $* <<EOI 2>>EOE != 0 - a = 'b' - EOI - buildfile:11:1: error: unable to deduce low-verbosity script diagnostics name - info: consider specifying it explicitly with the 'diag' recipe attribute - info: or provide custom low-verbosity diagnostics with the 'diag' builtin - EOE } : inside-if diff --git a/libbuild2/build/script/parser+for.test.testscript b/libbuild2/build/script/parser+for.test.testscript index f4ebf3c..847b253 100644 --- a/libbuild2/build/script/parser+for.test.testscript +++ b/libbuild2/build/script/parser+for.test.testscript @@ -16,7 +16,7 @@ cmd end EOI - buildfile:11:4: error: expected variable name instead of <newline> + buildfile:11:1: error: for: missing variable name EOE : untyped @@ -58,7 +58,7 @@ cmd b EOO - : typed-value + : typed-values : $* <<EOI >>~%EOO% for x: [dir_paths] a b @@ -69,10 +69,10 @@ %cmd (b/|'b\\')% EOO - : typed-var + : typed-elem : $* <<EOI >>~%EOO% - for [dir_path] x: a b + for x [dir_path]: a b cmd $x end EOI @@ -80,17 +80,16 @@ %cmd (b/|'b\\')% EOO - : type-mismatch + : typed-elem-value : - $* <<EOI 2>>EOE != 0 - for [string] x: [dir_paths] a b + $* <<EOI >>~%EOO% + for x [dir_path]: [strings] a b cmd $x end EOI - error: type mismatch in variable x - info: value type is dir_path - info: variable type is string - EOE + %cmd (a/|'a\\')% + %cmd (b/|'b\\')% + EOO : defined-var : @@ -182,3 +181,476 @@ EOE } } + +: form-2 +: +: ... | for x +: +{ + : for + : + { + : status + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x != 0 + cmd + end + EOI + buildfile:11:20: error: for-loop exit code cannot be checked + EOE + + : not-last + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x | echo x + cmd + end + EOI + buildfile:11:20: error: for-loop must be last command in a pipe + EOE + + : not-last-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x|echo x + cmd + end + EOI + buildfile:11:19: error: for-loop must be last command in a pipe + EOE + + : expression-after + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x && echo x + cmd + end + EOI + buildfile:11:20: error: command expression involving for-loop + EOE + + : expression-after-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x&&echo x + cmd + end + EOI + buildfile:11:19: error: command expression involving for-loop + EOE + + : expression-before + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && echo x | for x + cmd + end + EOI + buildfile:11:24: error: command expression involving for-loop + EOE + + : expression-before-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && echo x|for x + cmd + end + EOI + buildfile:11:22: error: command expression involving for-loop + EOE + + : cleanup + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x &f + cmd + end + EOI + buildfile:11:20: error: cleanup in for-loop + EOE + + : cleanup-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x&f + cmd + end + EOI + buildfile:11:19: error: cleanup in for-loop + EOE + + : stdout-redirect + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x >a + cmd + end + EOI + buildfile:11:20: error: output redirect in for-loop + EOE + + : stdout-redirect-relex + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x>a + cmd + end + EOI + buildfile:11:19: error: output redirect in for-loop + EOE + + : stdin-redirect + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x <a + cmd + end + EOI + buildfile:11:20: error: stdin is both piped and redirected + EOE + + : no-var + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for + cmd + end + EOI + buildfile:11:1: error: for: missing variable name + EOE + + : untyped + : + $* <<EOI >>EOO + echo 'a b' | for -w x + cmd $x + end + EOI + echo 'a b' | for -w x + EOO + + : expansion + : + $* <<EOI >>EOO + vs = a b + echo $vs | for x + cmd $x + end + EOI + echo a b | for x + EOO + + : typed-elem + : + $* <<EOI >>EOO + echo 'a b' | for -w x [dir_path] + cmd $x + end + EOI + echo 'a b' | for -w x [dir_path] + EOO + } + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + cmd + EOI + buildfile:13:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + echo 'a b' | for x + elif true + cmd + end + end + EOI + buildfile:12:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + echo 'a b' | for x # 1 + cmd1 $x # 2 + if ($x == "a") # 3 + cmd2 # 4 + echo x y | for y # 5 + cmd3 # 6 + end + else + cmd4 # 7 + end + cmd5 # 8 + end + cmd6 # 9 + EOI + echo 'a b' | for x # 1 + cmd6 # 9 + EOO + } +} + +: form-3 +: +: for x <... +: +{ + : for + : + { + : status + : + $* <<EOI 2>>EOE != 0 + for x <a != 0 + cmd + end + EOI + buildfile:11:10: error: for-loop exit code cannot be checked + EOE + + : not-last + : + $* <<EOI 2>>EOE != 0 + for x <a | echo x + cmd + end + EOI + buildfile:11:10: error: for-loop must be last command in a pipe + EOE + + : not-last-relex + : + $* <<EOI 2>>EOE != 0 + for <a x|echo x + cmd + end + EOI + buildfile:11:9: error: for-loop must be last command in a pipe + EOE + + : expression-after + : + $* <<EOI 2>>EOE != 0 + for x <a && echo x + cmd + end + EOI + buildfile:11:10: error: command expression involving for-loop + EOE + + : expression-after-relex + : + $* <<EOI 2>>EOE != 0 + for <a x&&echo x + cmd + end + EOI + buildfile:11:9: error: command expression involving for-loop + EOE + + : expression-before + : + $* <<EOI 2>>EOE != 0 + echo 'a b' && for x <a + cmd + end + EOI + buildfile:11:15: error: command expression involving for-loop + EOE + + : cleanup + : + $* <<EOI 2>>EOE != 0 + for x <a &f + cmd + end + EOI + buildfile:11:10: error: cleanup in for-loop + EOE + + : cleanup-before-var + : + $* <<EOI 2>>EOE != 0 + for &f x <a + cmd + end + EOI + buildfile:11:5: error: cleanup in for-loop + EOE + + : cleanup-relex + : + $* <<EOI 2>>EOE != 0 + for <a x&f + cmd + end + EOI + buildfile:11:9: error: cleanup in for-loop + EOE + + : stdout-redirect + : + $* <<EOI 2>>EOE != 0 + for x >a + cmd + end + EOI + buildfile:11:7: error: output redirect in for-loop + EOE + + : stdout-redirect-before-var + : + $* <<EOI 2>>EOE != 0 + for >a x + cmd + end + EOI + buildfile:11:5: error: output redirect in for-loop + EOE + + : stdout-redirect-relex + : + $* <<EOI 2>>EOE != 0 + for x>a + cmd + end + EOI + buildfile:11:6: error: output redirect in for-loop + EOE + + : no-var + : + $* <<EOI 2>>EOE != 0 + for <a + cmd + end + EOI + buildfile:11:1: error: for: missing variable name + EOE + + : quoted-opt + : + $* <<EOI >>EOO + o = -w + for "$o" x <'a b' + cmd $x + end + for "($o)" x <'a b' + cmd $x + end + EOI + for -w x <'a b' + for -w x <'a b' + EOO + + : untyped + : + $* <<EOI >>EOO + for -w x <'a b' + cmd $x + end + EOI + for -w x <'a b' + EOO + + : expansion + : + $* <<EOI >>EOO + vs = a b + for x <$vs + cmd $x + end + EOI + for x b <a + EOO + + : typed-elem + : + $* <<EOI >>EOO + for -w x [dir_path] <'a b' + cmd $x + end + EOI + for -w x [dir_path] <'a b' + EOO + } + + : end + : + { + : without-end + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + cmd + EOI + buildfile:13:1: error: expected closing 'end' + EOE + } + + : elif + : + { + : without-if + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + elif true + cmd + end + end + EOI + buildfile:12:3: error: 'elif' without preceding 'if' + EOE + } + + : nested + : + { + $* -l -r <<EOI >>EOO + for -w x <'a b' # 1 + cmd1 $x # 2 + if ($x == "a") # 3 + cmd2 # 4 + for -w y <'x y' # 5 + cmd3 # 6 + end + else + cmd4 # 7 + end + cmd5 # 8 + end + cmd6 # 9 + EOI + for -w x <'a b' # 1 + cmd6 # 9 + EOO + } + + : contained + : + { + : eos + : + $* <<EOI 2>>EOE != 0 + for x <'a b' + EOI + buildfile:12:1: error: expected closing 'end' + EOE + } +} diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index cb0dbbb..3ecf23d 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -47,7 +47,7 @@ namespace build2 { path_ = &pn; - pre_parse_ = true; + top_pre_parse_ = pre_parse_ = true; lexer l (is, *path_, line, lexer_mode::command_line); set_lexer (&l); @@ -61,7 +61,7 @@ namespace build2 pbase_ = scope_->src_path_; - file_based_ = tt.is_a<file> (); + file_based_ = tt.is_a<file> () || tt.is_a<group> (); perform_update_ = find (as.begin (), as.end (), perform_update_id) != as.end (); @@ -102,15 +102,22 @@ namespace build2 info << "consider using 'depdb' builtin to track its value " << "changes"; - // Diagnose absent/ambigous script name. + // Diagnose absent/ambiguous script name. But try to deduce an absent + // name from the script operation first. // { diag_record dr; - if (!diag_name_ && !diag_line_) + if (!diag_name_ && diag_preamble_.empty ()) { - dr << fail (s.start_loc) - << "unable to deduce low-verbosity script diagnostics name"; + if (as.size () == 1) + { + diag_name_ = make_pair (ctx->operation_table[as[0].operation ()], + location ()); + } + else + dr << fail (s.start_loc) + << "unable to deduce low-verbosity script diagnostics name"; } else if (diag_name2_) { @@ -136,12 +143,12 @@ namespace build2 // Save the script name or custom diagnostics line. // - assert (diag_name_.has_value () != diag_line_.has_value ()); + assert (diag_name_.has_value () == diag_preamble_.empty ()); if (diag_name_) s.diag_name = move (diag_name_->first); else - s.diag_line = move (diag_line_->first); + s.diag_preamble = move (diag_preamble_); // Save the custom dependency change tracking lines, if present. // @@ -151,6 +158,7 @@ namespace build2 { s.depdb_dyndep = depdb_dyndep_->second; s.depdb_dyndep_byproduct = depdb_dyndep_byproduct_; + s.depdb_dyndep_dyn_target = depdb_dyndep_dyn_target_; } s.depdb_preamble = move (depdb_preamble_); @@ -205,15 +213,15 @@ namespace build2 // enter: next token is peeked at (type in tt) // leave: newline - assert (!fct || - *fct == line_type::cmd_if || - *fct == line_type::cmd_while || - *fct == line_type::cmd_for); + assert (!fct || + *fct == line_type::cmd_if || + *fct == line_type::cmd_while || + *fct == line_type::cmd_for_stream || + *fct == line_type::cmd_for_args); // Determine the line type/start token. // - line_type lt ( - pre_parse_line_start (t, tt, lexer_mode::second_token)); + line_type lt (pre_parse_line_start (t, tt, lexer_mode::second_token)); line ln; @@ -246,50 +254,124 @@ namespace build2 break; } - case line_type::cmd_for: + // + // See pre_parse_line_start() for details. + // + case line_type::cmd_for_args: assert (false); break; + case line_type::cmd_for_stream: { - // First take care of the variable name. + // First we need to sense the next few tokens and detect which + // form of the loop we are dealing with, the first (for x: ...) + // or the third (x <...) one. Note that the second form (... | for + // x) is handled separately. + // + // If the next token doesn't look like a variable name, then this + // is the third form. Otherwise, if colon follows the variable + // name, potentially after the attributes, then this is the first + // form and the third form otherwise. + // + // Note that for the third form we will need to pass the 'for' + // token as a program name to the command expression parsing + // function since it will be gone from the token stream by that + // time. Thus, we save it. We also need to make sure the sensing + // always leaves the variable name token in t/tt. + // + // Note also that in this model it won't be possible to support + // options in the first form. // - mode (lexer_mode::normal); + token pt (t); + assert (pt.type == type::word && pt.value == "for"); - next_with_attributes (t, tt); - attributes_push (t, tt); + mode (lexer_mode::for_loop); + next (t, tt); - if (tt != type::word || t.qtype != quote_type::unquoted) - fail (t) << "expected variable name instead of " << t; + // Note that we also consider special variable names (those that + // don't clash with the command line elements like redirects, etc) + // to later fail gracefully. + // + string& n (t.value); - const string& n (t.value); + if (tt == type::word && t.qtype == quote_type::unquoted && + (n[0] == '_' || alpha (n[0]) || // Variable. + n == "~")) // Special variable. + { + // Detect patterns analogous to parse_variable_name() (so we + // diagnose `for x[string]: ...`). + // + if (n.find_first_of ("[*?") != string::npos) + fail (t) << "expected variable name instead of " << n; - if (special_variable (n)) - fail (t) << "attempt to set '" << n << "' variable directly"; + if (special_variable (n)) + fail (t) << "attempt to set '" << n << "' special variable"; - // We don't pre-enter variables. - // - ln.var = nullptr; + // Parse out the element attributes, if present. + // + if (lexer_->peek_char ().first == '[') + { + // Save the variable name token before the attributes parsing + // and restore it afterwards. Also make sure that the token + // which follows the attributes stays in the stream. + // + token vt (move (t)); + next_with_attributes (t, tt); - next (t, tt); + attributes_push (t, tt, + true /* standalone */, + false /* next_token */); + + t = move (vt); + tt = t.type; + } + + if (lexer_->peek_char ().first == ':') + lt = line_type::cmd_for_args; + } - if (tt != type::colon) + if (lt == line_type::cmd_for_stream) // for x <... { - // @@ TMP We will need to fallback to parsing the 'for x <...' - // form instead. + // At this point t/tt contains the variable name token. Now + // pre-parse the command expression in the command_line lexer + // mode starting from this position and also passing the 'for' + // token as a program name. + // + // Note that the fact that the potential attributes are already + // parsed doesn't affect the command expression pre-parsing. + // Also note that they will be available during the execution + // phase being replayed. // - fail (t) << "expected ':' instead of " << t - << " after variable name"; + expire_mode (); // Expire the for-loop lexer mode. + + parse_command_expr_result r ( + parse_command_expr (t, tt, + lexer::redirect_aliases, + move (pt))); + + assert (r.for_loop); + + if (tt != type::newline) + fail (t) << "expected newline instead of " << t; + + parse_here_documents (t, tt, r); } + else // for x: ... + { + next (t, tt); - expire_mode (); // Expire the normal lexer mode. + assert (tt == type::colon); - // Parse the value similar to the var line type (see above). - // - mode (lexer_mode::variable_line); - parse_variable_line (t, tt); + expire_mode (); // Expire the for-loop lexer mode. - if (tt != type::newline) - fail (t) << "expected newline instead of " << t << " after for"; + // Parse the value similar to the var line type (see above). + // + mode (lexer_mode::variable_line); + parse_variable_line (t, tt); - ++level_; + if (tt != type::newline) + fail (t) << "expected newline instead of " << t << " after for"; + } + ln.var = nullptr; + ++level_; break; } case line_type::cmd_elif: @@ -321,15 +403,24 @@ namespace build2 // Fall through. case line_type::cmd: { - pair<command_expr, here_docs> p; + parse_command_expr_result r; if (lt != line_type::cmd_else && lt != line_type::cmd_end) - p = parse_command_expr (t, tt, lexer::redirect_aliases); + r = parse_command_expr (t, tt, lexer::redirect_aliases); + + if (r.for_loop) + { + lt = line_type::cmd_for_stream; + ln.var = nullptr; + + ++level_; + } if (tt != type::newline) fail (t) << "expected newline instead of " << t; - parse_here_documents (t, tt, p); + parse_here_documents (t, tt, r); + break; } } @@ -358,7 +449,8 @@ namespace build2 break; } case line_type::cmd_while: - case line_type::cmd_for: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: { tt = peek (lexer_mode::first_token); @@ -396,7 +488,8 @@ namespace build2 break; } case line_type::cmd_while: - case line_type::cmd_for: + case line_type::cmd_for_stream: + case line_type::cmd_for_args: { fct = bt; break; @@ -466,7 +559,9 @@ namespace build2 // enter: peeked first token of next line (type in tt) // leave: newline - assert (lt == line_type::cmd_while || lt == line_type::cmd_for); + assert (lt == line_type::cmd_while || + lt == line_type::cmd_for_stream || + lt == line_type::cmd_for_args); // Parse lines until we see closing 'end'. // @@ -491,12 +586,12 @@ namespace build2 // assert (!pre_parse_); - pair<command_expr, here_docs> p ( + parse_command_expr_result pr ( parse_command_expr (t, tt, lexer::redirect_aliases)); assert (tt == type::newline); - parse_here_documents (t, tt, p); + parse_here_documents (t, tt, pr); assert (tt == type::newline); // @@ Note that currently running programs via a runner (e.g., see @@ -509,7 +604,7 @@ namespace build2 // passed to the environment constructor, similar to passing the // script deadline. // - return move (p.first); + return move (pr.expr); } // @@ -570,6 +665,12 @@ namespace build2 fail (l) << "'" << v << "' call via 'env' builtin"; }; + auto diag_loc = [this] () + { + assert (!diag_preamble_.empty ()); + return diag_preamble_.back ().tokens[0].location (); + }; + if (v == "diag") { verify (); @@ -586,24 +687,41 @@ namespace build2 } else // Custom diagnostics. { - assert (diag_line_); - fail (l) << "multiple 'diag' builtin calls" << - info (diag_line_->second) << "previous call is here"; + info (diag_loc ()) << "previous call is here"; } } - // Instruct the parser to save the diag builtin line separately - // from the script lines, when it is fully parsed. Note that it - // will be executed prior to the script body execution to obtain - // the custom diagnostics. + // Move the script body to the end of the diag preamble. // - diag_line_ = make_pair (line (), l); - save_line_ = &diag_line_->first; - diag_weight_ = 4; + // Note that we move into the preamble whatever is there and delay + // the check until the execution (see the depdb preamble + // collecting for the reasoning). + // + lines& ls (script_->body); + diag_preamble_.insert (diag_preamble_.end (), + make_move_iterator (ls.begin ()), + make_move_iterator (ls.end ())); + ls.clear (); + + // Also move the body_temp_dir flag, if it is true. + // + if (script_->body_temp_dir) + { + script_->diag_preamble_temp_dir = true; + script_->body_temp_dir = false; + } + + // Similar to the depdb preamble collection, instruct the parser + // to save the depdb builtin line separately from the script + // lines. + // + diag_preamble_.push_back (line ()); + save_line_ = &diag_preamble_.back (); - diag_name_ = nullopt; - diag_name2_ = nullopt; + diag_weight_ = 4; + diag_name_ = nullopt; + diag_name2_ = nullopt; // Note that the rest of the line contains the builtin argument to // be printed, thus we parse it in the value lexer mode. @@ -630,12 +748,11 @@ namespace build2 } if (!file_based_) - fail (l) << "'depdb' builtin can only be used for file-based " - << "targets"; + fail (l) << "'depdb' builtin can only be used for file- or " + << "file group-based targets"; - if (diag_line_) - fail (diag_line_->second) - << "'diag' builtin call before 'depdb' call" << + if (!diag_preamble_.empty ()) + fail (diag_loc ()) << "'diag' builtin call before 'depdb' call" << info (l) << "'depdb' call is here"; // Note that the rest of the line contains the builtin command @@ -714,8 +831,18 @@ namespace build2 fail (l) << "multiple 'depdb dyndep' calls" << info (depdb_dyndep_->first) << "previous call is here"; - if (peek () == type::word && peeked ().value == "--byproduct") - depdb_dyndep_byproduct_ = true; + if (peek () == type::word) + { + const string& v (peeked ().value); + + // Note: --byproduct and --dyn-target are mutually + // exclusive. + // + if (v == "--byproduct") + depdb_dyndep_byproduct_ = true; + else if (v == "--dyn-target") + depdb_dyndep_dyn_target_ = true; + } } else { @@ -807,10 +934,53 @@ namespace build2 // // This is also the reason why we add a diag frame. // + // The problem turned out to be worse than originally thought: we + // may call a function (for example, as part of if) with invalid + // arguments. And this could happen in the depdb preamble, which + // means we cannot fix this by moving the depdb builtin (which must + // come after the preamble). So let's peek at what's ahead and omit + // the expansion if it's anything iffy, namely, eval context or + // function call. + // + bool skip_diag (false); if (pre_parse_ && diag_weight_ != 4) { - pre_parse_ = false; // Make parse_names() perform expansions. - pre_parse_suspended_ = true; + // Based on the buildfile expansion parsing logic. + // + if (tt == type::lparen) // Evaluation context. + skip_diag = true; + else if (tt == type::dollar) + { + type ptt (peek (lexer_mode::variable)); + + if (!peeked ().separated) + { + if (ptt == type::lparen) + { + // While strictly speaking this can also be a function call, + // this is highly unusual and we will assume it's a variable + // expansion. + } + else if (ptt == type::word) + { + pair<char, bool> r (lexer_->peek_char ()); + + if (r.first == '(' && !r.second) // Function call. + skip_diag = true; + } + } + } + + if (!skip_diag) + { + // Sanity check: we should not be suspending the pre-parse mode + // turned on by the base parser. + // + assert (top_pre_parse_); + + pre_parse_ = false; // Make parse_names() perform expansions. + pre_parse_suspended_ = true; + } } auto df = make_diag_frame ( @@ -838,7 +1008,7 @@ namespace build2 pre_parse_ = true; } - if (pre_parse_ && diag_weight_ == 4) + if (pre_parse_ && (diag_weight_ == 4 || skip_diag)) return nullopt; } @@ -982,6 +1152,8 @@ namespace build2 { if (!qs) { + // This could be a script from src so search like a prerequisite. + // if (const target* t = search_existing ( ns[0], *scope_, ns[0].pair ? ns[1].dir : empty_dir_path)) { @@ -1083,6 +1255,7 @@ namespace build2 auto exec_cmd = [this] (token& t, build2::script::token_type& tt, const iteration_index* ii, size_t li, bool single, + const function<command_function>& cf, const location& ll) { // We use the 0 index to signal that this is the only command. @@ -1093,7 +1266,7 @@ namespace build2 command_expr ce ( parse_command_line (t, static_cast<token_type&> (tt))); - runner_->run (*environment_, ce, ii, li, ll); + runner_->run (*environment_, ce, ii, li, cf, ll); }; exec_lines (s.body, exec_cmd); @@ -1102,11 +1275,30 @@ namespace build2 runner_->leave (e, s.end_loc); } + // Return true if the specified expression executes the set builtin or + // is a for-loop. + // + static bool + valid_preamble_cmd (const command_expr& ce, + const function<command_function>& cf) + { + return find_if ( + ce.begin (), ce.end (), + [&cf] (const expr_term& et) + { + const process_path& p (et.pipe.back ().program); + return p.initial == nullptr && + (p.recall.string () == "set" || + (cf != nullptr && p.recall.string () == "for")); + }) != ce.end (); + } + void parser:: - exec_depdb_preamble (action a, const scope& bs, const file& t, + exec_depdb_preamble (action a, const scope& bs, const target& t, environment& e, const script& s, runner& r, lines_iterator begin, lines_iterator end, depdb& dd, + dynamic_targets* dyn_targets, bool* update, optional<timestamp> mt, bool* deferred_failure, @@ -1129,23 +1321,29 @@ namespace build2 action a; const scope& bs; - const file& t; + const target& t; environment& env; const script& scr; depdb& dd; + dynamic_targets* dyn_targets; bool* update; bool* deferred_failure; optional<timestamp> mt; dyndep_byproduct* byp; - } data {trace, a, bs, t, e, s, dd, update, deferred_failure, mt, byp}; + } data { + trace, + a, bs, t, + e, s, + dd, dyn_targets, update, deferred_failure, mt, byp}; auto exec_cmd = [this, &data] (token& t, build2::script::token_type& tt, const iteration_index* ii, size_t li, bool /* single */, + const function<command_function>& cf, const location& ll) { // Note that we never reset the line index to zero (as we do in @@ -1169,8 +1367,9 @@ namespace build2 // exec_depdb_dyndep (t, tt, li, ll, - data.a, data.bs, const_cast<file&> (data.t), + data.a, data.bs, const_cast<target&> (data.t), data.dd, + *data.dyn_targets, *data.update, *data.mt, *data.deferred_failure, @@ -1180,35 +1379,29 @@ namespace build2 { names ns (exec_special (t, tt, true /* skip <cmd> */)); + string v; + const char* w (nullptr); if (cmd == "hash") { sha256 cs; for (const name& n: ns) to_checksum (cs, n); - if (data.dd.expect (cs.string ()) != nullptr) - l4 ([&] { - data.trace (ll) - << "'depdb hash' argument change forcing update of " - << data.t;}); + v = cs.string (); + w = "argument"; } else if (cmd == "string") { - string s; try { - s = convert<string> (move (ns)); + v = convert<string> (move (ns)); } catch (const invalid_argument& e) { fail (ll) << "invalid 'depdb string' argument: " << e; } - if (data.dd.expect (s) != nullptr) - l4 ([&] { - data.trace (ll) - << "'depdb string' argument change forcing update of " - << data.t;}); + w = "argument"; } else if (cmd == "env") { @@ -1229,14 +1422,32 @@ namespace build2 fail (ll) << pf << e; } - if (data.dd.expect (cs.string ()) != nullptr) - l4 ([&] { - data.trace (ll) - << "'depdb env' environment change forcing update of " - << data.t;}); + v = cs.string (); + w = "environment"; } else assert (false); + + // Prefix the value with the type letter. This serves two + // purposes: + // + // 1. It makes sure the result is never a blank line. We use + // blank lines as anchors to skip directly to certain entries + // (e.g., dynamic targets). + // + // 2. It allows us to detect the beginning of prerequisites + // since an absolute path will be distinguishable from these + // entries (in the future we may want to add an explicit + // blank after such custom entries to make this easier). + // + v.insert (0, 1, ' '); + v.insert (0, 1, cmd[0]); // `h`, `s`, or `e` + + if (data.dd.expect (v) != nullptr) + l4 ([&] { + data.trace (ll) + << "'depdb " << cmd << "' " << w << " change forcing " + << "update of " << data.t;}); } } else @@ -1244,15 +1455,7 @@ namespace build2 command_expr ce ( parse_command_line (t, static_cast<token_type&> (tt))); - // Verify that this expression executes the set builtin. - // - if (find_if (ce.begin (), ce.end (), - [] (const expr_term& et) - { - const process_path& p (et.pipe.back ().program); - return p.initial == nullptr && - p.recall.string () == "set"; - }) == ce.end ()) + if (!valid_preamble_cmd (ce, cf)) { const replay_tokens& rt (data.scr.depdb_preamble.back ().tokens); assert (!rt.empty ()); @@ -1263,20 +1466,93 @@ namespace build2 info (rt[0].location ()) << "depdb preamble ends here"; } - runner_->run (*environment_, ce, ii, li, ll); + runner_->run (*environment_, ce, ii, li, cf, ll); } }; exec_lines (begin, end, exec_cmd); } + pair<names, location> parser:: + execute_diag_preamble (const scope& rs, const scope& bs, + environment& e, const script& s, runner& r, + bool diag, bool enter, bool leave) + { + tracer trace ("execute_diag_preamble"); + + assert (!s.diag_preamble.empty ()); + + const line& dl (s.diag_preamble.back ()); // Diag builtin line. + + pre_exec (rs, bs, e, &s, &r); + + if (enter) + runner_->enter (e, s.start_loc); + + // Perform the variable assignments. + // + auto exec_cmd = [&dl, this] (token& t, + build2::script::token_type& tt, + const iteration_index* ii, size_t li, + bool /* single */, + const function<command_function>& cf, + const location& ll) + { + // Note that we never reset the line index to zero (as we do in + // execute_body()) assuming that there are some script body commands + // to follow. + // + command_expr ce ( + parse_command_line (t, static_cast<token_type&> (tt))); + + if (!valid_preamble_cmd (ce, cf)) + { + const replay_tokens& rt (dl.tokens); + assert (!rt.empty ()); + + fail (ll) << "disallowed command in diag preamble" << + info << "only variable assignments are allowed in diag preamble" + << info (rt[0].location ()) << "diag preamble ends here"; + } + + runner_->run (*environment_, ce, ii, li, cf, ll); + }; + + exec_lines (s.diag_preamble.begin (), s.diag_preamble.end () - 1, + exec_cmd); + + // Execute the diag line, if requested. + // + names ns; + + if (diag) + { + // Copy the tokens and start playing. + // + replay_data (replay_tokens (dl.tokens)); + + token t; + build2::script::token_type tt; + next (t, tt); + + ns = exec_special (t, tt, true /* skip_first */); + + replay_stop (); + } + + if (leave) + runner_->leave (e, s.end_loc); + + return make_pair (ns, dl.tokens.front ().location ()); + } + void parser:: pre_exec (const scope& rs, const scope& bs, environment& e, const script* s, runner* r) { path_ = nullptr; // Set by replays. - pre_parse_ = false; + top_pre_parse_ = pre_parse_ = false; set_lexer (nullptr); @@ -1305,19 +1581,25 @@ namespace build2 // Note that we rely on "small function object" optimization for the // exec_*() lambdas. // - auto exec_assign = [this] (const variable& var, - value&& val, - type kind, - const location& l) + auto exec_set = [this] (const variable& var, + token& t, build2::script::token_type& tt, + const location&) { + next (t, tt); + type kind (tt); // Assignment kind. + + mode (lexer_mode::variable_line); + value rhs (parse_variable_line (t, tt)); + + assert (tt == type::newline); + + // Assign. + // value& lhs (kind == type::assign ? environment_->assign (var) : environment_->append (var)); - if (kind == type::assign) - lhs = move (val); - else - append_value (&var, lhs, move (val), l); + apply_value_attributes (&var, lhs, move (rhs), kind); }; auto exec_cond = [this] (token& t, build2::script::token_type& tt, @@ -1333,9 +1615,21 @@ namespace build2 return runner_->run_cond (*environment_, ce, ii, li, ll); }; + auto exec_for = [this] (const variable& var, + value&& val, + const attributes& val_attrs, + const location&) + { + value& lhs (environment_->assign (var)); + + attributes_.push_back (val_attrs); + + apply_value_attributes (&var, lhs, move (val), type::assign); + }; + build2::script::parser::exec_lines ( begin, end, - exec_assign, exec_cmd, exec_cond, + exec_set, exec_cmd, exec_cond, exec_for, nullptr /* iteration_index */, environment_->exec_line, &environment_->var_pool); @@ -1355,33 +1649,12 @@ namespace build2 : names (); } - names parser:: - execute_special (const scope& rs, const scope& bs, - environment& e, - const line& ln, - bool omit_builtin) - { - pre_exec (rs, bs, e, nullptr /* script */, nullptr /* runner */); - - // Copy the tokens and start playing. - // - replay_data (replay_tokens (ln.tokens)); - - token t; - build2::script::token_type tt; - next (t, tt); - - names r (exec_special (t, tt, omit_builtin)); - - replay_stop (); - return r; - } - void parser:: exec_depdb_dyndep (token& lt, build2::script::token_type& ltt, size_t li, const location& ll, - action a, const scope& bs, file& t, + action a, const scope& bs, target& t, depdb& dd, + dynamic_targets& dyn_targets, bool& update, timestamp mt, bool& deferred_failure, @@ -1394,6 +1667,7 @@ namespace build2 depdb_dyndep_options ops; bool prog (false); bool byprod (false); + bool dyn_tgt (false); // Prerequisite update filter (--update-*). // @@ -1436,11 +1710,9 @@ namespace build2 next (t, tt); // Skip the 'dyndep' command. - if (tt == type::word && t.value == "--byproduct") - { - byprod = true; + if (tt == type::word && ((byprod = (t.value == "--byproduct")) || + (dyn_tgt = (t.value == "--dyn-target")))) next (t, tt); - } assert (byprod == (byprod_result != nullptr)); @@ -1659,10 +1931,23 @@ namespace build2 continue; } - // Handle --byproduct in the wrong place. + // Handle --byproduct and --dyn-target in the wrong place. // if (strcmp (a, "--byproduct") == 0) - fail (ll) << "depdb dyndep: --byproduct must be first option"; + { + fail (ll) << "depdb dyndep: " + << (dyn_tgt + ? "--byproduct specified with --dyn-target" + : "--byproduct must be first option"); + } + + if (strcmp (a, "--dyn-target") == 0) + { + fail (ll) << "depdb dyndep: " + << (byprod + ? "--dyn-target specified with --byproduct" + : "--dyn-target must be first option"); + } // Handle non-literal --update-*. // @@ -1687,29 +1972,31 @@ namespace build2 } } - // --what - // - const char* what (ops.what_specified () - ? ops.what ().c_str () - : "file"); - // --format // dyndep_format format (dyndep_format::make); - if (ops.format_specified ()) { const string& f (ops.format ()); - if (f != "make") + if (f == "lines") format = dyndep_format::lines; + else if (f != "make") fail (ll) << "depdb dyndep: invalid --format option value '" << f << "'"; } + // Prerequisite-specific options. + // + + // --what + // + const char* what (ops.what_specified () + ? ops.what ().c_str () + : "file"); + // --cwd // optional<dir_path> cwd; - if (ops.cwd_specified ()) { if (!byprod) @@ -1729,28 +2016,6 @@ namespace build2 fail (ll) << "depdb dyndep: -I specified with --byproduct"; } - // --file - // - // Note that if --file is specified without a program, then we assume - // it is one of the static prerequisites. - // - optional<path> file; - - if (ops.file_specified ()) - { - file = move (ops.file ()); - - if (file->relative ()) - { - if (!cwd) - fail (ll) << "depdb dyndep: relative path specified with --file"; - - *file = *cwd / *file; - } - } - else if (!prog) - fail (ll) << "depdb dyndep: program or --file expected"; - // --default-type // // Get the default prerequisite type falling back to file{} if not @@ -1762,7 +2027,7 @@ namespace build2 // translation unit would want to make sure it resolves extracted // system headers to h{} targets analogous to the c module's rule. // - const target_type* def_pt; + const target_type* def_pt (&file::static_type); if (ops.default_type_specified ()) { const string& t (ops.default_type ()); @@ -1770,10 +2035,8 @@ namespace build2 def_pt = bs.find_target_type (t); if (def_pt == nullptr) fail (ll) << "depdb dyndep: unknown target type '" << t - << "' specific with --default-type"; + << "' specified with --default-type"; } - else - def_pt = &file::static_type; // --adhoc // @@ -1783,6 +2046,93 @@ namespace build2 fail (ll) << "depdb dyndep: --adhoc specified with --byproduct"; } + // Target-specific options. + // + + // --target-what + // + const char* what_tgt ("file"); + if (ops.target_what_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-what specified without " + << "--dyn-target"; + + what_tgt = ops.target_what ().c_str (); + } + + // --target-cwd + // + optional<dir_path> cwd_tgt; + if (ops.target_cwd_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-cwd specified without " + << "--dyn-target"; + + cwd_tgt = move (ops.target_cwd ()); + + if (cwd_tgt->relative ()) + fail (ll) << "depdb dyndep: relative path specified with " + << "--target-cwd"; + } + + // --target-default-type + // + const target_type* def_tt (&file::static_type); + if (ops.target_default_type_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-default-type specified " + << "without --dyn-target"; + + const string& t (ops.target_default_type ()); + + def_tt = bs.find_target_type (t); + if (def_tt == nullptr) + fail (ll) << "depdb dyndep: unknown target type '" << t + << "' specified with --target-default-type"; + } + + map<string, const target_type*> map_tt; + if (ops.target_extension_type_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-extension-type specified " + << "without --dyn-target"; + + for (pair<const string, string>& p: ops.target_extension_type ()) + { + const target_type* tt (bs.find_target_type (p.second)); + if (tt == nullptr) + fail (ll) << "depdb dyndep: unknown target type '" << p.second + << "' specified with --target-extension-type"; + + map_tt[p.first] = tt; + } + } + + // --file (last since need --*cwd) + // + // Note that if --file is specified without a program, then we assume + // it is one of the static prerequisites. + // + optional<path> file; + if (ops.file_specified ()) + { + file = move (ops.file ()); + + if (file->relative ()) + { + if (!cwd && !cwd_tgt) + fail (ll) << "depdb dyndep: relative path specified with --file"; + + *file = (cwd ? *cwd : *cwd_tgt) / *file; + } + } + else if (!prog) + fail (ll) << "depdb dyndep: program or --file expected"; + // Update prerequisite targets. // using dyndep = dyndep_rule; @@ -1913,6 +2263,10 @@ namespace build2 return; } + const scope& rs (*bs.root_scope ()); + + group* g (t.is_a<group> ()); // If not group then file. + // This code is based on the prior work in the cc module (specifically // extract_headers()) where you can often find more detailed rationale // for some of the steps performed. @@ -2010,9 +2364,29 @@ namespace build2 command_expr cmd; srcout_map so_map; + // Save/restore script cleanups. + // + struct cleanups + { + build2::script::cleanups ordinary; + paths special; + }; + optional<cleanups> script_cleanups; + + auto cleanups_guard = make_guard ( + [this, &script_cleanups] () + { + if (script_cleanups) + { + swap (environment_->cleanups, script_cleanups->ordinary); + swap (environment_->special_cleanups, script_cleanups->special); + } + }); + auto init_run = [this, &ctx, <, <t, &ll, - prog, &file, &ops, &cmd, &so_map] () + prog, &file, &ops, + &cmd, &so_map, &script_cleanups] () { // Populate the srcout map with the -I$out_base -I$src_base pairs. // @@ -2025,6 +2399,10 @@ namespace build2 if (prog) { + script_cleanups = cleanups {}; + swap (environment_->cleanups, script_cleanups->ordinary); + swap (environment_->special_cleanups, script_cleanups->special); + cmd = parse_command_line (lt, static_cast<token_type&> (ltt)); // If the output goes to stdout, then this should be a single @@ -2040,15 +2418,10 @@ namespace build2 // they include the line index in their names to avoid clashes // between lines). // - // Cleanups are not an issue, they will simply replaced. And + // Cleanups are not an issue, they will simply be replaced. And // overriding the contents of the special files seems harmless and // consistent with what would happen if the command redirects its // output to a non-special file. - // - if (file) - environment_->clean ( - {build2::script::cleanup_type::always, *file}, - true /* implicit */); } }; @@ -2058,7 +2431,7 @@ namespace build2 size_t skip_count (0); auto add = [this, &trace, what, - a, &bs, &t, &pts, pts_n = pts.size (), + a, &bs, &t, g, &pts, pts_n = pts.size (), &ops, &map_ext, def_pt, &pfx_map, &so_map, &dd, &skip_count] (path fp, size_t* skip, @@ -2068,6 +2441,61 @@ namespace build2 bool cache (skip == nullptr); + // Handle fsdir{} prerequisite separately. + // + // Note: inspired by inject_fsdir(). + // + if (fp.to_directory ()) + { + if (!cache) + { + // Note: already absolute since cannot be non-existent. + // + fp.normalize (); + } + + const fsdir* dt (&search<fsdir> (t, + path_cast<dir_path> (fp), + dir_path (), + string (), nullptr, nullptr)); + + // Subset of code for file below. + // + if (!cache) + { + for (size_t i (0); i != pts_n; ++i) + { + const prerequisite_target& p (pts[i]); + + if (const target* pt = + (p.target != nullptr ? p.target : + p.adhoc () ? reinterpret_cast<target*> (p.data) : + nullptr)) + { + if (dt == pt) + return false; + } + } + + if (*skip != 0) + { + --(*skip); + return false; + } + } + + match_sync (a, *dt); + pts.push_back ( + prerequisite_target ( + nullptr, true /* adhoc */, reinterpret_cast<uintptr_t> (dt))); + + if (!cache) + dd.expect (fp.representation ()); + + skip_count++; + return false; + } + // We can only defer the failure if we will be running the recipe // body. // @@ -2122,13 +2550,26 @@ namespace build2 // Skip if this is one of the targets. // + // Note that for dynamic targets this only works if we see the + // targets before prerequisites (like in the make dependency + // format). + // if (ops.drop_cycles ()) { - for (const target* m (&t); m != nullptr; m = m->adhoc_member) + if (g != nullptr) { - if (ft == m) + auto& ms (g->members); + if (find (ms.begin (), ms.end (), ft) != ms.end ()) return false; } + else + { + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (ft == m) + return false; + } + } } // Skip until where we left off. @@ -2157,10 +2598,15 @@ namespace build2 { prerequisite_target& pt (pts.back ()); + // Note: set the include_target flag for consistency (the + // updated_during_match() check does not apply since it's a + // dynamic prerequisite). + // if (pt.adhoc ()) { pt.data = reinterpret_cast<uintptr_t> (pt.target); pt.target = nullptr; + pt.include |= prerequisite_target::include_target; } else pt.data = 1; // Already updated. @@ -2194,13 +2640,27 @@ namespace build2 << t; }); + // While in the make format targets come before prerequisites, in + // depdb we store them after since any change to prerequisites can + // invalidate the set of targets. So we save them first and process + // later. + // + // Note also that we need to return them to the caller in case we are + // updating. + // If nothing so far has invalidated the dependency database, then try // the cached data before running the program. // bool cache (!update); + bool skip_blank (false); for (bool restart (true), first_run (true); restart; cache = false) { + // Clear the state in case we are restarting. + // + if (dyn_tgt) + dyn_targets.clear (); + restart = false; if (cache) @@ -2209,7 +2669,8 @@ namespace build2 // assert (skip_count == 0); - // We should always end with a blank line. + // We should always end with a blank line after the list of + // dynamic prerequisites. // for (;;) { @@ -2223,8 +2684,11 @@ namespace build2 break; } - if (l->empty ()) // Done, nothing changed. - return; + if (l->empty ()) // Done with prerequisites, nothing changed. + { + skip_blank = true; + break; + } if (optional<bool> r = add (path (move (*l)), nullptr, mt)) { @@ -2246,6 +2710,52 @@ namespace build2 return; } } + + if (!restart) // Nothing changed. + { + if (dyn_tgt) + { + // We should always end with a blank line after the list of + // dynamic targets. + // + for (;;) + { + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + restart = true; + break; + } + + if (l->empty ()) // Done with targets. + break; + + // Split into type and path (see below for background). + // + size_t p (l->find (' ')); + if (p == string::npos || // Invalid format. + p == 0 || // Empty type. + p + 1 == l->size ()) // Empty path. + { + dd.write (); // Invalidate this line. + restart = true; + break; + } + + string t (*l, 0, p); + l->erase (0, p + 1); + + dyn_targets.push_back ( + dynamic_target {move (t), path (move (*l))}); + } + } + + if (!restart) // Done, nothing changed. + break; // Break earliy to keep cache=true. + } } else { @@ -2254,9 +2764,16 @@ namespace build2 init_run (); first_run = false; } - else if (!prog) + else { - fail (ll) << "generated " << what << " without program to retry"; + if (!prog) + fail (ll) << "generated " << what << " without program to retry"; + + // Drop dyndep cleanups accumulated on the previous run. + // + assert (script_cleanups); // Sanity check. + environment_->cleanups.clear (); + environment_->special_cleanups.clear (); } // Save the timestamp just before we run the command. If we depend @@ -2279,18 +2796,48 @@ namespace build2 { // Note: depdb is disallowed inside flow control constructs. // - string s; - build2::script::run (*environment_, - cmd, - nullptr /* iteration_index */, li, - ll, - !file ? &s : nullptr); - if (!file) { - iss.str (move (s)); + function<command_function> cf ( + [&iss] + (build2::script::environment&, + const strings&, + auto_fd in, + pipe_command* pipe, + const optional<deadline>& dl, + const location& ll) + { + read (move (in), + false /* whitespace */, + false /* newline */, + true /* exact */, + [&iss] (string&& s) {iss.str (move (s));}, + pipe, + dl, + ll, + "depdb-dyndep"); + }); + + build2::script::run (*environment_, + cmd, + nullptr /* iteration_index */, li, + ll, + cf, false /* last_cmd */); + iss.exceptions (istream::badbit); } + else + { + build2::script::run ( + *environment_, cmd, nullptr /* iteration_index */, li, ll); + + // Note: make it a maybe-cleanup in case the command cleans it + // up itself. + // + environment_->clean ( + {build2::script::cleanup_type::maybe, *file}, + true /* implicit */); + } } ifdstream ifs (ifdstream::badbit); @@ -2358,32 +2905,72 @@ namespace build2 if (r.second.empty ()) continue; - // @@ TODO: what should we do about targets? + // Skip targets unless requested to extract. // - // Note that if we take GCC as an example, things are + // BTW, if you are wondering why don't we extract targets + // by default, take GCC as an example, where things are // quite messed up: by default it ignores -o and just // takes the source file name and replaces the extension // with a platform-appropriate object file extension. One // can specify a custom target (or even multiple targets) - // with -MT or with -MQ (quoting). Though MinGW GCC still - // does not quote `:` with -MQ. So in this case it's + // with -MT or with -MQ (quoting). So in this case it's // definitely easier for the user to ignore the targets // and just specify everything in the buildfile. // - // On the other hand, other tools are likely to produce - // more sensible output (except perhaps for quoting). - // - // @@ Maybe in the lax mode we should only recognize `:` - // if it's separated on at least one side? - // - // Alternatively, we could detect Windows drives in - // paths and "handle" them (I believe this is what GNU - // make does). Maybe we should have three formats: - // make-lax, make, make-strict? - // if (r.first == make_type::target) + { + // NOTE: similar code below. + // + if (dyn_tgt) + { + path& f (r.second); + + if (f.relative ()) + { + if (!cwd_tgt) + fail (il) << "relative " << what_tgt + << " target path '" << f + << "' in make dependency declaration" << + info << "consider using --target-cwd to specify " + << "relative path base"; + + f = *cwd_tgt / f; + } + + // Note that unlike prerequisites, here we don't need + // normalize_external() since we expect the targets to + // be within this project. + // + try + { + f.normalize (); + } + catch (const invalid_path&) + { + fail (il) << "invalid " << what_tgt << " target " + << "path '" << f.string () << "'"; + } + + // The target must be within this project. + // + if (!f.sub (rs.out_path ())) + { + fail (il) << what_tgt << " target path " << f + << " must be inside project output " + << "directory " << rs.out_path (); + } + + // Note: type is resolved later. + // + dyn_targets.push_back ( + dynamic_target {string (), move (f)}); + } + continue; + } + // NOTE: similar code below. + // if (optional<bool> u = add (move (r.second), &skip, rmt)) { restart = *u; @@ -2411,20 +2998,380 @@ namespace build2 break; } - break; + break; // case + } + case dyndep_format::lines: + { + bool tgt (dyn_tgt); // Reading targets or prerequisites. + + for (string l; !restart; ++il.line) // Reuse the buffer. + { + if (eof (getline (is, l))) + break; + + if (l.empty ()) + { + if (!tgt) + fail (il) << "blank line in prerequisites list"; + + tgt = false; // Targets/prerequisites separating blank. + continue; + } + + // See if this line start with space to indicate a non- + // existent prerequisite. This variable serves both as a + // flag and as a position of the beginning of the path. + // + size_t n (l.front () == ' ' ? 1 : 0); + + if (tgt) + { + // NOTE: similar code above. + // + path f; + try + { + // Non-existent target doesn't make sense. + // + if (n) + throw invalid_path (""); + + f = path (l); + + if (f.relative ()) + { + if (!cwd_tgt) + fail (il) << "relative " << what_tgt + << " target path '" << f + << "' in lines dependency declaration" << + info << "consider using --target-cwd to specify " + << "relative path base"; + + f = *cwd_tgt / f; + } + + // Note that unlike prerequisites, here we don't need + // normalize_external() since we expect the targets to + // be within this project. + // + f.normalize (); + } + catch (const invalid_path&) + { + fail (il) << "invalid " << what_tgt << " target path '" + << l << "'"; + } + + // The target must be within this project. + // + if (!f.sub (rs.out_path ())) + { + fail (il) << what_tgt << " target path " << f + << " must be inside project output directory " + << rs.out_path (); + } + + // Note: type is resolved later. + // + dyn_targets.push_back ( + dynamic_target {string (), move (f)}); + } + else + { + path f; + try + { + f = path (l.c_str () + n, l.size () - n); + + if (f.empty () || + (n && f.to_directory ())) // Non-existent fsdir{}. + throw invalid_path (""); + + if (f.relative ()) + { + if (!n) + { + if (!cwd) + fail (il) << "relative " << what + << " prerequisite path '" << f + << "' in lines dependency declaration" << + info << "consider using --cwd to specify " + << "relative path base"; + + f = *cwd / f; + } + } + else if (n) + { + // @@ TODO: non-existent absolute paths. + // + throw invalid_path (""); + } + } + catch (const invalid_path&) + { + fail (il) << "invalid " << what << " prerequisite path '" + << l << "'"; + } + + // NOTE: similar code above. + // + if (optional<bool> u = add (move (f), &skip, rmt)) + { + restart = *u; + + if (restart) + { + update = true; + l6 ([&]{trace << "restarting";}); + } + } + else + { + // Trigger recompilation, mark as expected to fail, and + // bail out. + // + update = true; + deferred_failure = true; + break; + } + } + } + + break; // case } } + if (file) + ifs.close (); + // Bail out early if we have deferred a failure. // if (deferred_failure) return; + + // Clean after each depdb-dyndep execution. + // + if (prog) + clean (*environment_, ll); } } - // Add the terminating blank line (we are updating depdb). + // Add the dynamic prerequisites terminating blank line if we are + // updating depdb and unless it's already there. // - dd.expect (""); + if (!cache && !skip_blank) + dd.expect (""); + + // Handle dynamic targets. + // + if (dyn_tgt) + { + if (g != nullptr && g->members_static == 0 && dyn_targets.empty ()) + fail (ll) << "group " << *g << " has no static or dynamic members"; + + // There is one more level (at least that we know of) to this rabbit + // hole: if the set of dynamic targets changes between clean and + // update and we do a `clean update` batch, then we will end up with + // old targets (as entered by clean from old depdb information) + // being present during update. So we need to clean them out. + // + // Optimize this for a first/single batch (common case) by noticing + // that there are only real targets to start with. + // + // Note that this doesn't affect explicit groups where we reset the + // members on each update (see adhoc_rule_buildscript::apply()). + // + optional<vector<const target*>> dts; + if (g == nullptr) + { + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (m->decl != target_decl::real) + dts = vector<const target*> (); + } + } + + struct map_ext_data + { + const char* what_tgt; + const map<string, const target_type*>& map_tt; + const path* f; // Updated on each iteration. + } d {what_tgt, map_tt, nullptr}; + + function<dyndep::map_extension_func> map_ext ( + [this, &d] (const scope& bs, const string& n, const string& e) + { + small_vector<const target_type*, 2> tts; + + // Check the custom mapping first. + // + auto i (d.map_tt.find (e)); + if (i != d.map_tt.end ()) + tts.push_back (i->second); + else + { + tts = dyndep::map_extension (bs, n, e, nullptr); + + // Issue custom diagnostics suggesting --target-extension-type. + // + if (tts.size () > 1) + { + diag_record dr (fail); + + dr << "mapping of " << d.what_tgt << " target path " << *d.f + << " to target type is ambiguous"; + + for (const target_type* tt: tts) + dr << info << "can be " << tt->name << "{}"; + + dr << info << "use --target-extension-type to provide custom " + << "mapping"; + } + } + + return tts; + }); + + function<dyndep::group_filter_func> filter; + if (g != nullptr) + { + // Skip static/duplicate members in explicit group. + // + filter = [] (mtime_target& g, const build2::file& m) + { + auto& ms (g.as<group> ().members); + return find (ms.begin (), ms.end (), &m) == ms.end (); + }; + } + + // Unlike for prerequisites, for targets we store in depdb both the + // resolved target type and path. The target type is used in clean + // (see adhoc_rule_buildscript::apply()) where we cannot easily get + // hold of all the dyndep options to map the path to target type. + // So the format of the target line is: + // + // <type> <path> + // + string l; // Reuse the buffer. + for (dynamic_target& dt: dyn_targets) + { + const path& f (dt.path); + + d.f = &f; // Current file being mapped. + + // Note that this logic should be consistent with what we have in + // adhoc_buildscript_rule::apply() for perform_clean. + // + const build2::file* ft (nullptr); + if (g != nullptr) + { + pair<const build2::file&, bool> r ( + dyndep::inject_group_member ( + what_tgt, + a, bs, *g, + f, // Can't move since need to return dyn_targets. + map_ext, *def_tt, filter)); + + // Note: no target_decl shenanigans since reset the members on + // each update. + // + if (!r.second) + { + dt.type.clear (); // Static indicator. + continue; + } + + ft = &r.first; + + // Note: we only currently support dynamic file members so it + // will be file if first. + // + g->members.push_back (ft); + } + else + { + pair<const build2::file&, bool> r ( + dyndep::inject_adhoc_group_member ( + what_tgt, + a, bs, t, + f, // Can't move since need to return dyn_targets. + map_ext, *def_tt)); + + // Note that we have to track the dynamic target even if it was + // already a member (think `b update && b clean update`). + // + if (!r.second && r.first.decl == target_decl::real) + { + dt.type.clear (); // Static indicator. + continue; + } + + ft = &r.first; + + if (dts) + dts->push_back (ft); + } + + const char* tn (ft->type ().name); + + if (dt.type.empty ()) + dt.type = tn; + else if (dt.type != tn) + { + // This can, for example, happen if the user changed the + // extension to target type mapping. Say swapped extension + // variable values of two target types. + // + fail << "mapping of " << what_tgt << " target path " << f + << " to target type has changed" << + info << "previously mapped to " << dt.type << "{}" << + info << "now mapped to " << tn << "{}" << + info << "perform from scratch rebuild of " << t; + } + + if (!cache) + { + l = dt.type; + l += ' '; + l += f.string (); + dd.expect (l); + } + } + + // Add the dynamic targets terminating blank line. + // + if (!cache) + dd.expect (""); + + // Clean out old dynamic targets (skip the primary member). + // + if (dts) + { + assert (g == nullptr); + + for (target* p (&t); p->adhoc_member != nullptr; ) + { + target* m (p->adhoc_member); + + if (m->decl != target_decl::real) + { + // While there could be quite a few dynamic targets (think + // something like Doxygen), this will hopefully be optimized + // down to a contiguous memory region scan for an integer and + // so should be fast. + // + if (find (dts->begin (), dts->end (), m) == dts->end ()) + { + p->adhoc_member = m->adhoc_member; // Drop m. + continue; + } + } + + p = m; + } + } + } // Reload $< and $> to make sure they contain the newly discovered // prerequisites and targets. @@ -2433,7 +3380,8 @@ namespace build2 environment_->set_special_variables (a); } - // When add a special variable don't forget to update lexer::word(). + // When add a special variable don't forget to update lexer::word() and + // for-loop parsing in pre_parse_line(). // bool parser:: special_variable (const string& n) noexcept @@ -2454,6 +3402,12 @@ namespace build2 { lookup r; + // Note that pre-parse can be switched on by the base parser even + // during execute. + // + if (!top_pre_parse_) + return r; + // Add the variable name skipping special variables and suppressing // duplicates, unless the default variables change tracking is // canceled with `depdb clear`. While at it, check if the script @@ -2469,10 +3423,8 @@ namespace build2 { if (pre_parse_suspended_) { - const variable* pvar (scope_->ctx.var_pool.find (name)); - - if (pvar != nullptr) - r = (*scope_)[*pvar]; + if (const variable* var = scope_->var_pool ().find (name)) + r = (*scope_)[*var]; } if (!depdb_clear_) @@ -2538,7 +3490,10 @@ namespace build2 void parser:: lookup_function (string&& name, const location& loc) { - if (perform_update_ && file_based_ && !impure_func_) + // Note that pre-parse can be switched on by the base parser even + // during execute. + // + if (top_pre_parse_ && perform_update_ && file_based_ && !impure_func_) { const function_overloads* f (ctx->functions.find (name)); diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx index a81583c..8f86b24 100644 --- a/libbuild2/build/script/parser.hxx +++ b/libbuild2/build/script/parser.hxx @@ -96,7 +96,6 @@ namespace build2 environment&, const script&, runner&, bool enter = true, bool leave = true); - // Execute the first or the second (dyndep) half of the depdb // preamble. // @@ -104,8 +103,10 @@ namespace build2 // runner's enter() function is called before the first preamble/body // command execution and leave() -- after the last command. // + // Note: target must be file or group. + // void - execute_depdb_preamble (action a, const scope& base, const file& t, + execute_depdb_preamble (action a, const scope& base, const target& t, environment& e, const script& s, runner& r, depdb& dd) { @@ -120,18 +121,28 @@ namespace build2 dd); } + struct dynamic_target + { + string type; // Target type name (absent if static member). + build2::path path; + }; + + using dynamic_targets = vector<dynamic_target>; + void execute_depdb_preamble_dyndep ( - action a, const scope& base, file& t, + action a, const scope& base, target& t, environment& e, const script& s, runner& r, - depdb& dd, bool& update, timestamp mt, bool& deferred_failure) + depdb& dd, + dynamic_targets& dyn_targets, + bool& update, timestamp mt, bool& deferred_failure) { exec_depdb_preamble ( a, base, t, e, s, r, s.depdb_preamble.begin () + *s.depdb_dyndep, s.depdb_preamble.end (), - dd, &update, mt, &deferred_failure); + dd, &dyn_targets, &update, mt, &deferred_failure); } // This version doesn't actually execute the depdb-dyndep builtin (but @@ -140,7 +151,7 @@ namespace build2 // depdb-dyndep --byproduct logic (which fits better into the rule // implementation). // - enum class dyndep_format {make}; + enum class dyndep_format {make, lines}; struct dyndep_byproduct { @@ -155,14 +166,17 @@ namespace build2 dyndep_byproduct execute_depdb_preamble_dyndep_byproduct ( - action a, const scope& base, const file& t, + action a, const scope& base, const target& t, environment& e, const script& s, runner& r, depdb& dd, bool& update, timestamp mt) { + // Dummies. + // // This is getting a bit ugly (we also don't really need to pass // depdb here). One day we will find a better way... // - bool deferred_failure; // Dymmy. + dynamic_targets dyn_targets; + bool deferred_failure; dyndep_byproduct v; exec_depdb_preamble ( @@ -170,19 +184,26 @@ namespace build2 e, s, r, s.depdb_preamble.begin () + *s.depdb_dyndep, s.depdb_preamble.end (), - dd, &update, mt, &deferred_failure, &v); + dd, &dyn_targets, &update, mt, &deferred_failure, &v); return v; } - // Parse a special builtin line into names, performing the variable - // and pattern expansions. If omit_builtin is true, then omit the - // builtin name from the result. + // If the diag argument is true, then execute the preamble including + // the (trailing) diagnostics line and return the resulting names and + // its location (see exec_special() for the diagnostics line execution + // semantics). Otherwise, execute the preamble excluding the + // diagnostics line and return an empty names list and location. If + // requested, call the runner's enter() and leave() functions that + // initialize/clean up the environment before/after the preamble + // execution. // - names - execute_special (const scope& root, const scope& base, - environment&, - const line&, - bool omit_builtin = true); + // Note: having both root and base scopes for testing (where we pass + // global scope for both). + // + pair<names, location> + execute_diag_preamble (const scope& root, const scope& base, + environment&, const script&, runner&, + bool diag, bool enter, bool leave); protected: // Setup the parser for subsequent exec_*() function calls. @@ -203,24 +224,34 @@ namespace build2 exec_lines (l.begin (), l.end (), c); } + // Parse a special builtin line into names, performing the variable + // and pattern expansions. Optionally, skip the first token (builtin + // name, etc). + // names exec_special (token&, build2::script::token_type&, bool skip_first); + // Note: target must be file or group. + // void - exec_depdb_preamble (action, const scope& base, const file&, + exec_depdb_preamble (action, const scope& base, const target&, environment&, const script&, runner&, lines_iterator begin, lines_iterator end, depdb&, + dynamic_targets* dyn_targets = nullptr, bool* update = nullptr, optional<timestamp> mt = nullopt, bool* deferred_failure = nullptr, dyndep_byproduct* = nullptr); + // Note: target must be file or group. + // void exec_depdb_dyndep (token&, build2::script::token_type&, size_t line_index, const location&, - action, const scope& base, file&, + action, const scope& base, target&, depdb&, + dynamic_targets& dyn_targets, bool& update, timestamp, bool& deferred_failure, @@ -261,9 +292,9 @@ namespace build2 script* script_; const small_vector<action, 1>* actions_; // Non-NULL during pre-parse. - // True if this script is for file-based targets and performing update - // is one of the actions, respectively. Only set for the pre-parse - // mode. + // True if this script is for file- or file group-based targets and + // performing update is one of the actions, respectively. Only set for + // the pre-parse mode. // bool file_based_; bool perform_update_; @@ -293,18 +324,24 @@ namespace build2 // // If the diag builtin is encountered, then its whole line is saved // (including the leading 'diag' word) for later execution and the - // diagnostics weight is set to 4. + // diagnostics weight is set to 4. The preceding lines, which can only + // contain variable assignments (including via the set builtin, + // potentially inside the flow control constructs), are also saved. // // Any attempt to manually set the custom diagnostics twice (the diag // builtin after the script name or after another diag builtin) is // reported as ambiguity. // - // At the end of pre-parsing either diag_name_ or diag_line_ (but not - // both) are present. + // If no script name is deduced by the end of pre-parsing and the + // script is used for a single operation, then use this operation's + // name as a script name. + // + // At the end of pre-parsing either diag_name_ is present or + // diag_preamble_ is not empty (but not both). // optional<pair<string, location>> diag_name_; optional<pair<string, location>> diag_name2_; // Ambiguous script name. - optional<pair<line, location>> diag_line_; + lines diag_preamble_; uint8_t diag_weight_ = 0; // Custom dependency change tracking. @@ -339,6 +376,7 @@ namespace build2 optional<pair<location, size_t>> depdb_dyndep_; // depdb-dyndep location/position. bool depdb_dyndep_byproduct_ = false; // --byproduct + bool depdb_dyndep_dyn_target_ = false; // --dyn-target lines depdb_preamble_; // Note: excluding depdb-clear. // If present, the first impure function called in the body of the @@ -358,7 +396,12 @@ namespace build2 // optional<location> computed_var_; - // True during pre-parsing when the pre-parse mode is temporarily + // True if we (rather than the base parser) turned on the pre-parse + // mode. + // + bool top_pre_parse_; + + // True during top-pre-parsing when the pre-parse mode is temporarily // suspended to perform expansion. // bool pre_parse_suspended_ = false; @@ -368,9 +411,9 @@ namespace build2 // Before the script line gets parsed, it is set to a temporary value // that will by default be appended to the script. However, // parse_program() can point it to a different location where the line - // should be saved instead (e.g., diag_line_, etc) or set it to NULL - // if the line is handled in an ad-hoc way and should be dropped - // (e.g., depdb_clear_, etc). + // should be saved instead (e.g., diag_preamble_ back, etc) or set it + // to NULL if the line is handled in an ad-hoc way and should be + // dropped (e.g., depdb_clear_, etc). // line* save_line_; diff --git a/libbuild2/build/script/parser.test.cxx b/libbuild2/build/script/parser.test.cxx index f8c2696..97eac22 100644 --- a/libbuild2/build/script/parser.test.cxx +++ b/libbuild2/build/script/parser.test.cxx @@ -37,11 +37,32 @@ namespace build2 enter (environment&, const location&) override {} virtual void - run (environment&, + run (environment& env, const command_expr& e, const iteration_index* ii, size_t i, - const location&) override + const function<command_function>& cf, + const location& ll) override { + // If the functions is specified, then just execute it with an empty + // stdin so it can perform the housekeeping (stop replaying tokens, + // increment line index, etc). + // + if (cf != nullptr) + { + assert (e.size () == 1 && !e[0].pipe.empty ()); + + const command& c (e[0].pipe.back ()); + + // Must be enforced by the caller. + // + assert (!c.out && !c.err && !c.exit); + + cf (env, c.arguments, + fdopen_null (), nullptr /* pipe */, + nullopt /* deadline */, + ll); + } + cout << e; if (line_ || iterations_) @@ -98,8 +119,8 @@ namespace build2 // argv[0] [-l] [-r] // argv[0] -b [-t] // argv[0] -d [-t] + // argv[0] -g [-t] [<diag-name>] // argv[0] -q - // argv[0] -g [<diag-name>] // // In the first form read the script from stdin and trace the script // body execution to stdout using the custom print runner. @@ -110,14 +131,14 @@ namespace build2 // In the third form read the script from stdin, parse it and dump the // depdb preamble lines to stdout. // - // In the forth form read the script from stdin, parse it and print - // line tokens quoting information to stdout. - // - // In the fifth form read the script from stdin, parse it and print the + // In the forth form read the script from stdin, parse it and print the // low-verbosity script diagnostics name or custom low-verbosity // diagnostics to stdout. If the script doesn't deduce any of them, then // print the diagnostics and exit with non-zero code. // + // In the fifth form read the script from stdin, parse it and print + // line tokens quoting information to stdout. + // // -l // Print the script line number for each executed expression. // @@ -130,9 +151,13 @@ namespace build2 // -d // Dump the parsed script depdb preamble to stdout. // + // -g + // Dump the low-verbosity script diagnostics name or custom + // low-verbosity diagnostics to stdout. + // // -t - // Print true if the body (-b) or depdb preamble (-d) references the - // temporary directory and false otherwise. + // Print true if the body (-b), depdb preamble (-d), or diag preamble + // (-g) references the temporary directory and false otherwise. // // -q // Print the parsed script tokens quoting information to sdout. If a @@ -142,10 +167,6 @@ namespace build2 // <quoting> := 'S' | 'D' | 'M' // <completeness> := 'C' | 'P' // - // -g - // Dump the low-verbosity script diagnostics name or custom - // low-verbosity diagnostics to stdout. - // int main (int argc, char* argv[]) { @@ -156,8 +177,8 @@ namespace build2 run, body, depdb_preamble, - quoting, - diag + diag, + quoting } m (mode::run); bool print_line (false); @@ -177,15 +198,17 @@ namespace build2 m = mode::body; else if (a == "-d") m = mode::depdb_preamble; + else if (a == "-g") + m = mode::diag; else if (a == "-t") { - assert (m == mode::body || m == mode::depdb_preamble); + assert (m == mode::body || + m == mode::depdb_preamble || + m == mode::diag); temp_dir = true; } else if (a == "-q") m = mode::quoting; - else if (a == "-g") - m = mode::diag; else { if (m == mode::diag) @@ -198,8 +221,8 @@ namespace build2 } } - assert (!print_line || m == mode::run); - assert (!print_iterations || m == mode::run); + assert (!print_line || m == mode::run || m == mode::diag); + assert (!print_iterations || m == mode::run || m == mode::diag); assert (!diag_name || m == mode::diag); // Fake build system driver, default verbosity. @@ -232,6 +255,8 @@ namespace build2 tt.path (path ("driver")); + const scope& bs (tt.base_scope ()); + small_vector<action, 1> acts {perform_update_id}; // Parse and run. @@ -239,7 +264,7 @@ namespace build2 parser p (ctx); path_name nm ("buildfile"); - script s (p.pre_parse (tt.base_scope (), tt.type (), acts, + script s (p.pre_parse (bs, tt.type (), acts, cin, nm, 11 /* line */, (m != mode::diag @@ -251,9 +276,29 @@ namespace build2 { case mode::run: { - environment e (perform_update_id, tt, s.body_temp_dir); + environment e (perform_update_id, tt, bs, false /* temp_dir */); print_runner r (print_line, print_iterations); - p.execute_body (ctx.global_scope, ctx.global_scope, e, s, r); + + bool exec_diag (!s.diag_preamble.empty ()); + + if (exec_diag) + { + if (s.diag_preamble_temp_dir) + e.set_temp_dir_variable (); + + p.execute_diag_preamble (ctx.global_scope, ctx.global_scope, + e, s, r, + false /* diag */, + true /* enter */, + false /* leave */); + } + + if (s.body_temp_dir && !s.diag_preamble_temp_dir) + e.set_temp_dir_variable (); + + p.execute_body (ctx.global_scope, ctx.global_scope, + e, s, r, + !exec_diag /* enter */); break; } case mode::diag: @@ -264,14 +309,26 @@ namespace build2 } else { - assert (s.diag_line); + if (!temp_dir) + { + environment e (perform_update_id, + tt, + bs, + s.diag_preamble_temp_dir); - environment e (perform_update_id, tt, false /* temp_dir */); + print_runner r (print_line, print_iterations); - cout << "diag: " << p.execute_special (ctx.global_scope, + names diag (p.execute_diag_preamble (ctx.global_scope, ctx.global_scope, - e, - *s.diag_line) << endl; + e, s, r, + true /* diag */, + true /* enter */, + true /* leave */).first); + + cout << "diag: " << diag << endl; + } + else + cout << (s.diag_preamble_temp_dir ? "true" : "false") << endl; } break; diff --git a/libbuild2/build/script/runner.cxx b/libbuild2/build/script/runner.cxx index 157fc60..5d9764b 100644 --- a/libbuild2/build/script/runner.cxx +++ b/libbuild2/build/script/runner.cxx @@ -28,12 +28,37 @@ namespace build2 // for (auto i (env.cleanups.begin ()); i != env.cleanups.end (); ) { - const target* m (&env.target); - for (; m != nullptr; m = m->adhoc_member) + const target* m (nullptr); + if (const group* g = env.target.is_a<group> ()) { - if (const path_target* pm = m->is_a<path_target> ()) - if (i->path == pm->path ()) - break; + for (const target* gm: g->members) + { + if (const path_target* pm = gm->is_a<path_target> ()) + { + if (i->path == pm->path ()) + { + m = gm; + break; + } + } + } + } + else if (const fsdir* fd = env.target.is_a<fsdir> ()) + { + // Compare ignoring the trailing directory separator. + // + if (path_traits::compare (i->path.string (), + fd->dir.string ()) == 0) + m = fd; + } + else + { + for (m = &env.target; m != nullptr; m = m->adhoc_member) + { + if (const path_target* pm = m->is_a<path_target> ()) + if (i->path == pm->path ()) + break; + } } if (m != nullptr) @@ -97,25 +122,28 @@ namespace build2 run (environment& env, const command_expr& expr, const iteration_index* ii, size_t li, + const function<command_function>& cf, const location& ll) { if (verb >= 3) text << ": " << expr; // Run the expression if we are not in the dry-run mode or if it - // executes the set or exit builtin and just print the expression - // otherwise at verbosity level 2 and up. + // executes the set or exit builtin or it is a for-loop. Otherwise, + // just print the expression otherwise at verbosity level 2 and up. // if (!env.context.dry_run || find_if (expr.begin (), expr.end (), - [] (const expr_term& et) + [&cf] (const expr_term& et) { const process_path& p (et.pipe.back ().program); return p.initial == nullptr && (p.recall.string () == "set" || - p.recall.string () == "exit"); + p.recall.string () == "exit" || + (cf != nullptr && + p.recall.string () == "for")); }) != expr.end ()) - build2::script::run (env, expr, ii, li, ll); + build2::script::run (env, expr, ii, li, ll, cf); else if (verb >= 2) text << expr; } diff --git a/libbuild2/build/script/runner.hxx b/libbuild2/build/script/runner.hxx index 0652396..ec8a948 100644 --- a/libbuild2/build/script/runner.hxx +++ b/libbuild2/build/script/runner.hxx @@ -32,10 +32,14 @@ namespace build2 // Location is the start position of this command line in the script. // It can be used in diagnostics. // + // Optionally, execute the specified function instead of the last + // pipe command. + // virtual void run (environment&, const command_expr&, const iteration_index*, size_t index, + const function<command_function>&, const location&) = 0; virtual bool @@ -66,6 +70,7 @@ namespace build2 run (environment&, const command_expr&, const iteration_index*, size_t, + const function<command_function>&, const location&) override; virtual bool diff --git a/libbuild2/build/script/script.cxx b/libbuild2/build/script/script.cxx index a557065..0d96cc3 100644 --- a/libbuild2/build/script/script.cxx +++ b/libbuild2/build/script/script.cxx @@ -7,6 +7,8 @@ #include <libbuild2/target.hxx> +#include <libbuild2/adhoc-rule-buildscript.hxx> // include_unmatch* + #include <libbuild2/script/timeout.hxx> #include <libbuild2/build/script/parser.hxx> @@ -28,6 +30,7 @@ namespace build2 environment:: environment (action a, const target_type& t, + const scope_type& s, bool temp, const optional<timestamp>& dl) : build2::script::environment ( @@ -39,7 +42,8 @@ namespace build2 redirect (redirect_type::merge, 2), redirect (redirect_type::pass)), target (t), - vars (context, false /* global */), + scope (s), + vars (context, false /* shared */), // Note: managed. var_ts (var_pool.insert (">")), var_ps (var_pool.insert ("<")), script_deadline (to_deadline (dl, false /* success */)) @@ -56,11 +60,27 @@ namespace build2 { // $> // + // What should it contain for an explicit group? While it may seem + // that just the members should be enough (and analogous to the ad + // hoc case), this won't let us get the group name for diagnostics. + // So the group name followed by all the members seems like the + // logical choice. + // names ns; - for (const target_type* m (&target); - m != nullptr; - m = m->adhoc_member) - m->as_name (ns); + + if (const group* g = target.is_a<group> ()) + { + g->as_name (ns); + for (const target_type* m: g->members) + m->as_name (ns); + } + else + { + for (const target_type* m (&target); + m != nullptr; + m = m->adhoc_member) + m->as_name (ns); + } assign (var_ts) = move (ns); } @@ -73,13 +93,25 @@ namespace build2 // much sense, they could be handy to exclude certain prerequisites // from $< while still treating them as such, especially in rule. // + // While initially we treated update=unmatch prerequisites as + // implicitly ad hoc, this turned out to be not quite correct, so + // now we add them unless they are explicitly marked ad hoc. + // names ns; - for (const prerequisite_target& pt: target.prerequisite_targets[a]) + for (const prerequisite_target& p: target.prerequisite_targets[a]) { // See adhoc_buildscript_rule::execute_update_prerequisites(). // - if (pt.target != nullptr && !pt.adhoc ()) - pt.target->as_name (ns); + if (const target_type* pt = + p.target != nullptr ? (p.adhoc () ? nullptr : p.target) : + (p.include & adhoc_buildscript_rule::include_unmatch) != 0 && + (p.include & prerequisite_target::include_adhoc) == 0 && + (p.include & adhoc_buildscript_rule::include_unmatch_adhoc) == 0 + ? reinterpret_cast<target_type*> (p.data) + : nullptr) + { + pt->as_name (ns); + } } assign (var_ps) = move (ns); @@ -154,7 +186,7 @@ namespace build2 } void environment:: - set_variable (string&& nm, + set_variable (string nm, names&& val, const string& attrs, const location& ll) @@ -233,7 +265,7 @@ namespace build2 // in parallel). Plus, if there is no such variable, then we cannot // possibly find any value. // - const variable* pvar (context.var_pool.find (n)); + const variable* pvar (scope.var_pool ().find (n)); if (pvar == nullptr) return lookup_type (); diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx index f8df204..08f1bf4 100644 --- a/libbuild2/build/script/script.hxx +++ b/libbuild2/build/script/script.hxx @@ -24,11 +24,14 @@ namespace build2 using build2::script::lines; using build2::script::redirect; using build2::script::redirect_type; + using build2::script::command; using build2::script::expr_term; using build2::script::command_expr; using build2::script::iteration_index; using build2::script::deadline; using build2::script::timeout; + using build2::script::pipe_command; + using build2::script::command_function; // Forward declarations. // @@ -45,13 +48,11 @@ namespace build2 class script { public: - using lines_type = build::script::lines; - // Note that the variables are not pre-entered into a pool during the // parsing phase, so the line variable pointers are NULL. // - lines_type body; - bool body_temp_dir = false; // True if the body references $~. + lines body; + bool body_temp_dir = false; // True if the body references $~. // Referenced ordinary (non-special) variables. // @@ -66,11 +67,13 @@ namespace build2 small_vector<string, 2> vars; // 2 for command and options. // Command name for low-verbosity diagnostics and custom low-verbosity - // diagnostics line. Note: cannot be both (see the script parser for + // diagnostics line, potentially preceded with the variable + // assignments. Note: cannot be both (see the script parser for // details). // optional<string> diag_name; - optional<line> diag_line; + lines diag_preamble; + bool diag_preamble_temp_dir = false; // True if refs $~. // The script's custom dependency change tracking lines (see the // script parser for details). @@ -79,8 +82,9 @@ namespace build2 bool depdb_value; // String or hash. optional<size_t> depdb_dyndep; // Pos of first dyndep. bool depdb_dyndep_byproduct = false; // dyndep --byproduct - lines_type depdb_preamble; // Note include vars. - bool depdb_preamble_temp_dir = false; // True if refs $~. + bool depdb_dyndep_dyn_target = false;// dyndep --dyn-target + lines depdb_preamble; // Note include vars. + bool depdb_preamble_temp_dir = false;// True if refs $~. location start_loc; location end_loc; @@ -89,10 +93,12 @@ namespace build2 class environment: public build2::script::environment { public: + using scope_type = build2::scope; using target_type = build2::target; environment (action, const target_type&, + const scope_type&, bool temp_dir, const optional<timestamp>& deadline = nullopt); @@ -113,11 +119,12 @@ namespace build2 environment& operator= (const environment&) = delete; public: - // Primary target this environment is for. + // Primary target this environment is for and its base scope; // const target_type& target; + const scope_type& scope; - // Script-local variable pool and map. + // Script-private variable pool and map. // // Note that it may be tempting to reuse the rule-specific variables // for this but they should not be modified during execution (i.e., @@ -163,7 +170,7 @@ namespace build2 size_t exec_line = 1; virtual void - set_variable (string&& name, + set_variable (string name, names&&, const string& attrs, const location&) override; |