From ea22643b2217921df74ea14df47d7c83987d5761 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Fri, 9 Dec 2016 17:29:27 +0200 Subject: Initial parallel scheduler implementation, use to run testscrips --- build2/test/script/lexer.cxx | 17 ++-- build2/test/script/parser.cxx | 187 ++++++++++++++---------------------------- build2/test/script/runner.cxx | 5 +- build2/test/script/script | 17 +++- build2/test/script/script.cxx | 67 +++++++++++++-- 5 files changed, 142 insertions(+), 151 deletions(-) (limited to 'build2/test') diff --git a/build2/test/script/lexer.cxx b/build2/test/script/lexer.cxx index 74aa02e..8bd3484 100644 --- a/build2/test/script/lexer.cxx +++ b/build2/test/script/lexer.cxx @@ -516,7 +516,7 @@ namespace build2 lexer_mode m (st.mode); // Customized implementation that handles special variable names ($*, - // $NN, $~, $@). + // $N, $~, $@). // if (m != lexer_mode::variable) return base_lexer::word (st, sep); @@ -526,23 +526,16 @@ namespace build2 if (c != '*' && c != '~' && c != '@' && !digit (c)) return base_lexer::word (st, sep); - uint64_t ln (c.line), cn (c.column); - string lexeme; - get (); - lexeme += c; - if (digit (c)) - { - for (; digit (c = peek ()); get ()) - lexeme += c; - } + if (digit (c) && digit (peek ())) + fail (c) << "multi-digit special variable name"; state_.pop (); // Expire the variable mode. - return token (move (lexeme), + return token (string (1, c), sep, quote_type::unquoted, false, - ln, cn); + c.line, c.column); } } } diff --git a/build2/test/script/parser.cxx b/build2/test/script/parser.cxx index 059cb93..874e0d7 100644 --- a/build2/test/script/parser.cxx +++ b/build2/test/script/parser.cxx @@ -4,6 +4,8 @@ #include +#include + #include #include @@ -17,17 +19,13 @@ namespace build2 { using type = token_type; - // Return true if the string contains only digit characters (used to - // detect the special $NN variables). + // Return true if the string contains only a single digit characters + // (used to detect the special $N variables). // static inline bool - digits (const string& s) + digit (const string& s) { - for (char c: s) - if (!digit (c)) - return false; - - return !s.empty (); + return s.size () == 1 && butl::digit (s[0]); } // @@ -403,18 +401,24 @@ namespace build2 // bool semi (false); + line ln; switch (lt) { case line_type::var: { // Check if we are trying to modify any of the special aliases - // ($*, $~, $N). + // ($*, $N, $~, $@). // - const string& n (t.value); + string& n (t.value); - if (n == "*" || n == "~" || digits (n)) + if (n == "*" || n == "~" || n == "@" || digit (n)) fail (t) << "attempt to set '" << n << "' variable directly"; + // Pre-enter the variables now while we are executing serially. + // Once parallel, it becomes a lot harder to do. + // + ln.var = &script_->var_pool.insert (move (n)); + next (t, tt); // Assignment kind. parse_variable_line (t, tt); @@ -496,7 +500,9 @@ namespace build2 if (ls == nullptr) ls = &ls_data; - ls->push_back (line {lt, replay_data ()}); + ln.type = lt; + ln.tokens = replay_data (); + ls->push_back (move (ln)); if (lt == line_type::cmd_if || lt == line_type::cmd_ifn) { @@ -1163,14 +1169,14 @@ namespace build2 if (p != string::npos && ++p != r.details.size ()) r.details.resize (p); + if (r.empty ()) + fail (loc) << "empty description"; + // Insert id into the id map if we have one. // if (!r.id.empty ()) insert_id (r.id, loc); - if (r.empty ()) - fail (loc) << "empty description"; - return r; } @@ -1211,6 +1217,11 @@ namespace build2 if (r.empty ()) fail (loc) << "empty description"; + // Insert id into the id map if we have one. + // + if (pre_parse_ && !r.id.empty ()) + insert_id (r.id, loc); + return r; } @@ -2318,6 +2329,8 @@ namespace build2 { exec_lines (g->setup_.begin (), g->setup_.end (), li, false); + scheduler::atomic_count task_count (0); + for (const unique_ptr& chain: g->scopes) { // Pick a scope from the if-else chain. @@ -2388,11 +2401,25 @@ namespace build2 // exec_scope_body (); // scope_ = os; // - parser p; - p.execute (*s, *script_, *runner_); + + // @@ Exceptions. + // + sched.async (task_count, + [] (scope& scp, script& scr, runner& r) + { + parser p; + p.execute (scp, scr, r); + }, + ref (*s), + ref (*script_), + ref (*runner_)); } } + sched.wait (task_count); + + //@@ Check if failed. + exec_lines (g->tdown_.begin (), g->tdown_.end (), li, false); } else @@ -2409,11 +2436,11 @@ namespace build2 for (; i != e; ++i) { - line& l (*i); - line_type lt (l.type); + line& ln (*i); + line_type lt (ln.type); assert (path_ == nullptr); - replay_data (move (l.tokens)); // Set the tokens and start playing. + replay_data (move (ln.tokens)); // Set the tokens and start playing. // We don't really need to change the mode since we already know // the line type. @@ -2441,32 +2468,22 @@ namespace build2 // Assign. // - const variable& var (script_->var_pool.insert (move (name))); + const variable& var (*ln.var); value& lhs (kind == type::assign ? scope_->assign (var) : scope_->append (var)); - // @@ Need to adjust to make strings the default type. - // apply_value_attributes (&var, lhs, move (rhs), kind); - // Handle the $*, $NN special aliases. - // - // The plan is as follows: here we detect modification of the - // source variables (test*), and (re)set $* to NULL on this - // scope (this is important to both invalidate any old values - // but also to "stake" the lookup position). This signals to - // the variable lookup function below that the $* and $NN - // values need to be recalculated from their sources. Note - // that we don't need to invalidate $NN since their lookup - // always checks $* first. + // If we changes any of the test.* values, then reset the $*, + // $N special aliases. // if (var.name == script_->test_var.name || var.name == script_->opts_var.name || var.name == script_->args_var.name) { - scope_->assign (script_->cmd_var) = nullptr; + scope_->reset_special (); } replay_stop (); @@ -2623,100 +2640,18 @@ namespace build2 // If we have no scope (happens when pre-parsing directives), then we // only look for buildfile variables. // - if (scope_ == nullptr) - return script_->find_in_buildfile (name); - - // @@ MT: will need RW mutex on var_pool. Or maybe if it's not there - // then it can't possibly be found? Still will be setting variables. - // - if (name != "*" && !digits (name)) - return scope_->find (script_->var_pool.insert (move (name))); - - // Handle the $*, $NN special aliases. - // - // See the exec_lines() for the overall plan. - // - // @@ MT: we are potentially changing outer scopes. Could force - // lookup before executing tests in each group scope. Poblem is - // we don't know which $NN vars will be looked up from inside. - // Could we collect all the variable names during the pre-parse - // stage? They could be computed. - // - // Or we could set all the non-NULL $NN (i.e., based on the number - // of elements in $*). - // - - // In both cases first thing we do is lookup $*. It should always be - // defined since we set it on the script's root scope. - // - lookup l (scope_->find (script_->cmd_var)); - assert (l.defined ()); - - // $* NULL value means it needs to be (re)calculated. - // - value& v (const_cast (*l)); - bool recalc (v.null); - - if (recalc) - { - strings s; - - auto append = [&s] (const strings& v) - { - s.insert (s.end (), v.begin (), v.end ()); - }; - - if (lookup l = scope_->find (script_->test_var)) - s.push_back (cast (l).string ()); - - if (lookup l = scope_->find (script_->opts_var)) - append (cast (l)); - - if (lookup l = scope_->find (script_->args_var)) - append (cast (l)); - - v = move (s); - } - - if (name == "*") - return l; - - // Use the string type for the $NN variables. - // - const variable& var (script_->var_pool.insert (move (name))); - - // We need to look for $NN in the same scope as where we found $*. - // - variable_map& vars (const_cast (*l.vars)); - - // If there is already a value and no need to recalculate it, then we - // are done. - // - if (!recalc && (l = vars[var]).defined ()) - return l; - - // Convert the variable name to index we can use on $*. + // Otherwise, every variable that is ever set in a script has been + // pre-entered during pre-parse. Which means that if one is not found + // in the script pool then it can only possibly be set in the + // buildfile. // - unsigned long i; - - try - { - i = stoul (var.name); - } - catch (const exception&) - { - fail (loc) << "invalid $* index " << var.name << endf; - } - - const strings& s (cast (v)); - value& nv (vars.assign (var)); - - if (i < s.size ()) - nv = s[i]; - else - nv = nullptr; + const variable* pvar (scope_ != nullptr + ? script_->var_pool.find (name) + : nullptr); - return lookup (nv, vars); + return pvar != nullptr + ? scope_->find (*pvar) + : script_->find_in_buildfile (name); } size_t parser:: diff --git a/build2/test/script/runner.cxx b/build2/test/script/runner.cxx index ee059f4..3c646c7 100644 --- a/build2/test/script/runner.cxx +++ b/build2/test/script/runner.cxx @@ -188,8 +188,9 @@ namespace build2 // mkdir (sp.wd_path, 2); else - // The working directory is cleaned up by the test rule prior the - // script execution. + // Scope working directory shall be empty (the script working + // directory is cleaned up by the test rule prior the script + // execution). // assert (empty (sp.wd_path)); diff --git a/build2/test/script/script b/build2/test/script/script index 1be33bb..4f39c58 100644 --- a/build2/test/script/script +++ b/build2/test/script/script @@ -48,6 +48,11 @@ namespace build2 { line_type type; replay_tokens tokens; + + union + { + const variable* var; // Pre-entered for line_type::var. + }; }; // Most of the time we will have just one line (test command). @@ -296,6 +301,11 @@ namespace build2 value& append (const variable&); + // Reset special $*, $N variables based on the test.* values. + // + void + reset_special (); + // Cleanup. // public: @@ -400,9 +410,10 @@ namespace build2 const variable& opts_var; // test.options const variable& args_var; // test.arguments - const variable& cmd_var; // $* - const variable& wd_var; // $~ - const variable& id_var; // $@ + const variable& wd_var; // $~ + const variable& id_var; // $@ + const variable& cmd_var; // $* + const variable* cmdN_var[10]; // $N }; class script: public script_base, public group diff --git a/build2/test/script/script.cxx b/build2/test/script/script.cxx index 8fb8115..f1e1bd4 100644 --- a/build2/test/script/script.cxx +++ b/build2/test/script/script.cxx @@ -397,9 +397,20 @@ namespace build2 opts_var (var_pool.insert ("test.options")), args_var (var_pool.insert ("test.arguments")), - cmd_var (var_pool.insert ("*")), wd_var (var_pool.insert ("~")), - id_var (var_pool.insert ("@")) {} + id_var (var_pool.insert ("@")), + cmd_var (var_pool.insert ("*")), + cmdN_var { + &var_pool.insert ("0"), + &var_pool.insert ("1"), + &var_pool.insert ("2"), + &var_pool.insert ("3"), + &var_pool.insert ("4"), + &var_pool.insert ("5"), + &var_pool.insert ("6"), + &var_pool.insert ("7"), + &var_pool.insert ("8"), + &var_pool.insert ("9")} {} // script // @@ -445,10 +456,9 @@ namespace build2 v = path (tt.dir.string ()); // Strip trailing slash. } - // Also add the NULL $* value that signals it needs to be recalculated - // on first access. + // Set the special $*, $N variables. // - assign (cmd_var) = nullptr; + reset_special (); } lookup scope:: @@ -474,7 +484,7 @@ namespace build2 { // Switch to the corresponding buildfile variable. Note that we don't // want to insert a new variable into the pool (we might be running - // concurrently). Plus, if there is no such variable, then we cannot + // in parallel). Plus, if there is no such variable, then we cannot // possibly find any value. // const variable* pvar (build2::var_pool.find (n)); @@ -522,13 +532,54 @@ namespace build2 value& r (assign (var)); // NULL. - //@@ I guess this is where we convert untyped value to strings? - // if (l.defined ()) r = *l; // Copy value (and type) from the outer scope. return r; } + + void scope:: + reset_special () + { + // First assemble the $* value. + // + strings s; + + auto append = [&s] (const strings& v) + { + s.insert (s.end (), v.begin (), v.end ()); + }; + + if (lookup l = find (root->test_var)) + s.push_back (cast (l).representation ()); + + if (lookup l = find (root->opts_var)) + append (cast (l)); + + if (lookup l = find (root->args_var)) + append (cast (l)); + + // Set the $N values if present. + // + for (size_t i (0); i <= 9; ++i) + { + value& v (assign (*root->cmdN_var[i])); + + if (i < s.size ()) + { + if (i == 0) + v = path (s[i]); + else + v = s[i]; + } + else + v = nullptr; // Clear any old values. + } + + // Set $*. + // + assign (root->cmd_var) = move (s); + } } } } -- cgit v1.1