// file : libbuild2/context.cxx -*- C++ -*- // copyright : Copyright (c) 2014-2019 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file #include #include #include // uncaught_exception[s]() #include #include #include #include #include // uncaught_exceptions // For command line variable parsing. // #include #include #include using namespace std; using namespace butl; namespace build2 { scheduler sched; run_phase phase; phase_mutex phase_mutex::instance; size_t load_generation; bool phase_mutex:: lock (run_phase p) { bool r; { mlock l (m_); bool u (lc_ == 0 && mc_ == 0 && ec_ == 0); // Unlocked. // Increment the counter. // condition_variable* v (nullptr); switch (p) { case run_phase::load: lc_++; v = &lv_; break; case run_phase::match: mc_++; v = &mv_; break; case run_phase::execute: ec_++; v = &ev_; break; } // If unlocked, switch directly to the new phase. Otherwise wait for the // phase switch. Note that in the unlocked case we don't need to notify // since there is nobody waiting (all counters are zero). // if (u) { phase = p; r = !fail_; } else if (phase != p) { sched.deactivate (); for (; phase != p; v->wait (l)) ; r = !fail_; l.unlock (); // Important: activate() can block. sched.activate (); } else r = !fail_; } // In case of load, acquire the exclusive access mutex. // if (p == run_phase::load) { lm_.lock (); r = !fail_; // Re-query. } return r; } void phase_mutex:: unlock (run_phase p) { // In case of load, release the exclusive access mutex. // if (p == run_phase::load) lm_.unlock (); { mlock l (m_); // Decrement the counter and see if this phase has become unlocked. // bool u (false); switch (p) { case run_phase::load: u = (--lc_ == 0); break; case run_phase::match: u = (--mc_ == 0); break; case run_phase::execute: u = (--ec_ == 0); break; } // If the phase is unlocked, pick a new phase and notify the waiters. // Note that we notify all load waiters so that they can all serialize // behind the second-level mutex. // if (u) { condition_variable* v; if (lc_ != 0) {phase = run_phase::load; v = &lv_;} else if (mc_ != 0) {phase = run_phase::match; v = &mv_;} else if (ec_ != 0) {phase = run_phase::execute; v = &ev_;} else {phase = run_phase::load; v = nullptr;} if (v != nullptr) { l.unlock (); v->notify_all (); } } } } bool phase_mutex:: relock (run_phase o, run_phase n) { // Pretty much a fused unlock/lock implementation except that we always // switch into the new phase. // assert (o != n); bool r; if (o == run_phase::load) lm_.unlock (); { mlock l (m_); bool u (false); switch (o) { case run_phase::load: u = (--lc_ == 0); break; case run_phase::match: u = (--mc_ == 0); break; case run_phase::execute: u = (--ec_ == 0); break; } // Set if will be waiting or notifying others. // condition_variable* v (nullptr); switch (n) { case run_phase::load: v = lc_++ != 0 || !u ? &lv_ : nullptr; break; case run_phase::match: v = mc_++ != 0 || !u ? &mv_ : nullptr; break; case run_phase::execute: v = ec_++ != 0 || !u ? &ev_ : nullptr; break; } if (u) { phase = n; r = !fail_; // Notify others that could be waiting for this phase. // if (v != nullptr) { l.unlock (); v->notify_all (); } } else // phase != n { sched.deactivate (); for (; phase != n; v->wait (l)) ; r = !fail_; l.unlock (); // Important: activate() can block. sched.activate (); } } if (n == run_phase::load) { lm_.lock (); r = !fail_; // Re-query. } return r; } // C++17 deprecated uncaught_exception() so use uncaught_exceptions() if // available. // static inline bool uncaught_exception () { #ifdef __cpp_lib_uncaught_exceptions return std::uncaught_exceptions () != 0; #else return std::uncaught_exception (); #endif } // phase_lock // static #ifdef __cpp_thread_local thread_local #else __thread #endif phase_lock* phase_lock_instance; phase_lock:: phase_lock (run_phase p) : p (p) { if (phase_lock* l = phase_lock_instance) assert (l->p == p); else { if (!phase_mutex::instance.lock (p)) { phase_mutex::instance.unlock (p); throw failed (); } phase_lock_instance = this; //text << this_thread::get_id () << " phase acquire " << p; } } phase_lock:: ~phase_lock () { if (phase_lock_instance == this) { phase_lock_instance = nullptr; phase_mutex::instance.unlock (p); //text << this_thread::get_id () << " phase release " << p; } } // phase_unlock // phase_unlock:: phase_unlock (bool u) : l (u ? phase_lock_instance : nullptr) { if (u) { phase_lock_instance = nullptr; phase_mutex::instance.unlock (l->p); //text << this_thread::get_id () << " phase unlock " << l->p; } } phase_unlock:: ~phase_unlock () noexcept (false) { if (l != nullptr) { bool r (phase_mutex::instance.lock (l->p)); phase_lock_instance = l; // Fail unless we are already failing. Note that we keep the phase // locked since there will be phase_lock down the stack to unlock it. // if (!r && !uncaught_exception ()) throw failed (); //text << this_thread::get_id () << " phase lock " << l->p; } } // phase_switch // phase_switch:: phase_switch (run_phase n) : o (phase), n (n) { if (!phase_mutex::instance.relock (o, n)) { phase_mutex::instance.relock (n, o); throw failed (); } phase_lock_instance->p = n; if (n == run_phase::load) // Note: load lock is exclusive. load_generation++; //text << this_thread::get_id () << " phase switch " << o << " " << n; } phase_switch:: ~phase_switch () noexcept (false) { // If we are coming off a failed load phase, mark the phase_mutex as // failed to terminate all other threads since the build state may no // longer be valid. // if (n == run_phase::load && uncaught_exception ()) { mlock l (phase_mutex::instance.m_); phase_mutex::instance.fail_ = true; } bool r (phase_mutex::instance.relock (n, o)); phase_lock_instance->p = o; // Similar logic to ~phase_unlock(). // if (!r && !uncaught_exception ()) throw failed (); //text << this_thread::get_id () << " phase restore " << n << " " << o; } const variable* var_src_root; const variable* var_out_root; const variable* var_src_base; const variable* var_out_base; const variable* var_forwarded; const variable* var_project; const variable* var_amalgamation; const variable* var_subprojects; const variable* var_version; const variable* var_project_url; const variable* var_project_summary; const variable* var_import_target; const variable* var_clean; const variable* var_backlink; const variable* var_include; const char var_extension[10] = "extension"; const variable* var_build_meta_operation; string current_mname; string current_oname; const meta_operation_info* current_mif; const operation_info* current_inner_oif; const operation_info* current_outer_oif; size_t current_on; execution_mode current_mode; bool current_diag_noise; atomic_count dependency_count; atomic_count target_count; atomic_count skip_count; bool keep_going = false; bool dry_run = false; void (*config_save_variable) (scope&, const variable&, uint64_t); const string& (*config_preprocess_create) (const variable_overrides&, values&, vector_view&, bool, const location&); variable_overrides reset (const strings& cmd_vars) { tracer trace ("reset"); // @@ Need to unload modules when we dynamically load them. // l6 ([&]{trace << "resetting build state";}); auto& vp (variable_pool::instance); auto& sm (scope_map::instance); variable_overrides vos; targets.clear (); sm.clear (); vp.clear (); // Reset meta/operation tables. Note that the order should match the id // constants in . // meta_operation_table.clear (); meta_operation_table.insert ("noop"); meta_operation_table.insert ("perform"); meta_operation_table.insert ("configure"); meta_operation_table.insert ("disfigure"); if (config_preprocess_create != nullptr) meta_operation_table.insert ( meta_operation_data ("create", config_preprocess_create)); meta_operation_table.insert ("dist"); meta_operation_table.insert ("info"); operation_table.clear (); operation_table.insert ("default"); operation_table.insert ("update"); operation_table.insert ("clean"); operation_table.insert ("test"); operation_table.insert ("update-for-test"); operation_table.insert ("install"); operation_table.insert ("uninstall"); operation_table.insert ("update-for-install"); // Create global scope. Note that the empty path is a prefix for any other // path. See the comment in for details. // auto make_global_scope = [] () -> scope& { auto i (scope_map::instance.insert (dir_path ())); scope& r (i->second); r.out_path_ = &i->first; global_scope = scope::global_ = &r; return r; }; scope& gs (make_global_scope ()); // Setup the global scope before parsing any variable overrides since they // may reference these things. // gs.assign ("build.work") = work; gs.assign ("build.home") = home; // Build system driver process path. // gs.assign ("build.path") = process_path (nullptr, // Will be filled by value assignment. path (argv0.recall_string ()), path (argv0.effect)); // Build system verbosity level. // gs.assign ("build.verbosity") = verb; // Build system version (similar to what we do in the version module // except here we don't include package epoch/revision). // { const standard_version& v (build_version); auto set = [&gs] (const char* var, auto val) { using T = decltype (val); gs.assign (variable_pool::instance.insert (var)) = move (val); }; set ("build.version", v.string_project ()); set ("build.version.number", v.version); set ("build.version.id", v.string_project_id ()); set ("build.version.major", uint64_t (v.major ())); set ("build.version.minor", uint64_t (v.minor ())); set ("build.version.patch", uint64_t (v.patch ())); optional a (v.alpha ()); optional b (v.beta ()); set ("build.version.alpha", a.has_value ()); set ("build.version.beta", b.has_value ()); set ("build.version.pre_release", v.pre_release ().has_value ()); set ("build.version.pre_release_string", v.string_pre_release ()); set ("build.version.pre_release_number", uint64_t (a ? *a : b ? *b : 0)); set ("build.version.snapshot", v.snapshot ()); // bool set ("build.version.snapshot_sn", v.snapshot_sn); // uint64 set ("build.version.snapshot_id", v.snapshot_id); // string set ("build.version.snapshot_string", v.string_snapshot ()); // Allow detection (for example, in tests) whether this is a staged // toolchain. // // Note that it is either staged or public, without queued, since we do // not re-package things during the queued-to-public transition. // set ("build.version.stage", LIBBUILD2_STAGE); } // Enter the host information. Rather than jumping through hoops like // config.guess, for now we are just going to use the compiler target we // were built with. While it is not as precise (for example, a binary // built for i686 might be running on x86_64), it is good enough of an // approximation/fallback since most of the time we are interested in just // the target class (e.g., linux, windows, macosx). // { // Did the user ask us to use config.guess? // string orig (config_guess ? run (3, *config_guess, [](string& l, bool) {return move (l);}) : BUILD2_HOST_TRIPLET); l5 ([&]{trace << "original host: '" << orig << "'";}); try { target_triplet t (orig); l5 ([&]{trace << "canonical host: '" << t.string () << "'; " << "class: " << t.class_;}); // Also enter as build.host.{cpu,vendor,system,version,class} for // convenience of access. // gs.assign ("build.host.cpu") = t.cpu; gs.assign ("build.host.vendor") = t.vendor; gs.assign ("build.host.system") = t.system; gs.assign ("build.host.version") = t.version; gs.assign ("build.host.class") = t.class_; gs.assign ("build.host") = move (t); } catch (const invalid_argument& e) { fail << "unable to parse build host '" << orig << "': " << e << info << "consider using the --config-guess option"; } } // Register builtin target types. // { target_type_map& t (gs.target_types); t.insert (); t.insert (); t.insert (); t.insert (); t.insert (); t.insert (); t.insert (); t.insert (); { auto& tt (t.insert ()); t.insert_file ("manifest", tt); } { auto& tt (t.insert ()); t.insert_file ("buildfile", tt); } } // Parse and enter the command line variables. We do it before entering // any other variables so that all the variables that are overriden are // marked as such first. Then, as we enter variables, we can verify that // the override is alowed. // for (size_t i (0); i != cmd_vars.size (); ++i) { const string& s (cmd_vars[i]); istringstream is (s); is.exceptions (istringstream::failbit | istringstream::badbit); // Similar to buildspec we do "effective escaping" and only for ['"\$(] // (basically what's necessary inside a double-quoted literal plus the // single quote). // lexer l (is, path (""), 1 /* line */, "\'\"\\$("); // At the buildfile level the scope-specific variable should be // separated from the directory with a whitespace, for example: // // ./ foo=$bar // // However, requiring this for command line variables would be too // inconvinient so we support both. // // We also have the optional visibility modifier as a first character of // the variable name: // // ! - global // % - project // / - scope // // The last one clashes a bit with the directory prefix: // // ./ /foo=bar // .//foo=bar // // But that's probably ok (the need for a scope-qualified override with // scope visibility should be pretty rare). Note also that to set the // value on the global scope we use !. // // And so the first token should be a word which can be either a // variable name (potentially with the directory qualification) or just // the directory, in which case it should be followed by another word // (unqualified variable name). // token t (l.next ()); optional dir; if (t.type == token_type::word) { string& v (t.value); size_t p (path::traits_type::rfind_separator (v)); if (p != string::npos && p != 0) // If first then visibility. { if (p == v.size () - 1) { // Separate directory. // dir = dir_path (move (v)); t = l.next (); // Target-specific overrides are not yet supported (and probably // never will be; the beast is already complex enough). // if (t.type == token_type::colon) fail << "'" << s << "' is a target-specific override" << info << "use double '--' to treat this argument as buildspec"; } else { // Combined directory. // // If double separator (visibility marker), then keep the first in // name. // if (p != 0 && path::traits_type::is_separator (v[p - 1])) --p; dir = dir_path (t.value, 0, p + 1); // Include the separator. t.value.erase (0, p + 1); // Erase the separator. } if (dir->relative ()) { // Handle the special relative to base scope case (.../). // auto i (dir->begin ()); if (*i == "...") dir = dir_path (++i, dir->end ()); // Note: can become empty. else dir->complete (); // Relative to CWD. } if (dir->absolute ()) dir->normalize (); } } token_type tt (l.next ().type); // The token should be the variable name followed by =, +=, or =+. // if (t.type != token_type::word || t.value.empty () || (tt != token_type::assign && tt != token_type::prepend && tt != token_type::append)) { fail << "expected variable assignment instead of '" << s << "'" << info << "use double '--' to treat this argument as buildspec"; } // Take care of the visibility. Note that here we rely on the fact that // none of these characters are lexer's name separators. // char c (t.value[0]); if (path::traits_type::is_separator (c)) c = '/'; // Normalize. string n (t.value, c == '!' || c == '%' || c == '/' ? 1 : 0); if (c == '!' && dir) fail << "scope-qualified global override of variable " << n; variable& var (const_cast ( vp.insert (n, true /* overridable */))); const variable* o; { variable_visibility v (c == '/' ? variable_visibility::scope : c == '%' ? variable_visibility::project : variable_visibility::normal); const char* k (tt == token_type::assign ? "__override" : tt == token_type::append ? "__suffix" : "__prefix"); unique_ptr p ( new variable { n + '.' + to_string (i + 1) + '.' + k, nullptr /* aliases */, nullptr /* type */, nullptr /* overrides */, v}); // Back link. // p->aliases = p.get (); if (var.overrides != nullptr) swap (p->aliases, const_cast (var.overrides.get ())->aliases); // Forward link. // p->overrides = move (var.overrides); var.overrides = move (p); o = var.overrides.get (); } // Currently we expand project overrides in the global scope to keep // things simple. Pass original variable for diagnostics. Use current // working directory as pattern base. // parser p; pair r (p.parse_variable_value (l, gs, &work, var)); if (r.second.type != token_type::eos) fail << "unexpected " << r.second << " in variable assignment " << "'" << s << "'"; // Make sure the value is not typed. // if (r.first.type != nullptr) fail << "typed override of variable " << n; // Global and absolute scope overrides we can enter directly. Project // and relative scope ones will be entered by the caller for each // amalgamation/project. // if (c == '!' || (dir && dir->absolute ())) { scope& s (c == '!' ? gs : sm.insert (*dir)->second); auto p (s.vars.insert (*o)); assert (p.second); // Variable name is unique. value& v (p.first); v = move (r.first); } else vos.push_back ( variable_override {var, *o, move (dir), move (r.first)}); } // Enter builtin variables and patterns. // // All config. variables are by default overridable. // vp.insert_pattern ("config.**", nullopt, true, nullopt, true, false); // file.cxx:import() (note that order is important; see insert_pattern()). // vp.insert_pattern ( "config.import.*", true, variable_visibility::normal, true); vp.insert_pattern ( "config.import.**", true, variable_visibility::normal, true); // module.cxx:load_module(). // { auto v_p (variable_visibility::project); vp.insert_pattern ("**.booted", false, v_p); vp.insert_pattern ("**.loaded", false, v_p); vp.insert_pattern ("**.configured", false, v_p); } { auto v_p (variable_visibility::project); auto v_t (variable_visibility::target); auto v_q (variable_visibility::prereq); var_src_root = &vp.insert ("src_root"); var_out_root = &vp.insert ("out_root"); var_src_base = &vp.insert ("src_base"); var_out_base = &vp.insert ("out_base"); var_forwarded = &vp.insert ("forwarded", v_p); // Note that subprojects is not typed since the value requires // pre-processing (see file.cxx). // var_project = &vp.insert ("project", v_p); var_amalgamation = &vp.insert ("amalgamation", v_p); var_subprojects = &vp.insert ("subprojects", v_p); var_version = &vp.insert ("version", v_p); var_project_url = &vp.insert ("project.url", v_p); var_project_summary = &vp.insert ("project.summary", v_p); var_import_target = &vp.insert ("import.target"); var_clean = &vp.insert ("clean", v_t); var_backlink = &vp.insert ("backlink", v_t); var_include = &vp.insert ("include", v_q); vp.insert (var_extension, v_t); // Backlink executables and (generated) documentation by default. // gs.target_vars[exe::static_type]["*"].assign (var_backlink) = "true"; gs.target_vars[doc::static_type]["*"].assign (var_backlink) = "true"; var_build_meta_operation = &vp.insert ("build.meta_operation"); } // Register builtin rules. // { rule_map& r (gs.rules); // Note: global scope! //@@ outer r.insert (perform_id, 0, "alias", alias_rule::instance); r.insert (perform_update_id, "fsdir", fsdir_rule::instance); r.insert (perform_clean_id, "fsdir", fsdir_rule::instance); r.insert (perform_update_id, "file", file_rule::instance); r.insert (perform_clean_id, "file", file_rule::instance); } return vos; } dir_path src_out (const dir_path& out, const scope& r) { assert (r.root ()); return src_out (out, r.out_path (), r.src_path ()); } dir_path out_src (const dir_path& src, const scope& r) { assert (r.root ()); return out_src (src, r.out_path (), r.src_path ()); } dir_path src_out (const dir_path& o, const dir_path& out_root, const dir_path& src_root) { assert (o.sub (out_root)); return src_root / o.leaf (out_root); } dir_path out_src (const dir_path& s, const dir_path& out_root, const dir_path& src_root) { assert (s.sub (src_root)); return out_root / s.leaf (src_root); } // diag_do(), etc. // string diag_do (const action&) { const meta_operation_info& m (*current_mif); const operation_info& io (*current_inner_oif); const operation_info* oo (current_outer_oif); string r; // perform(update(x)) -> "update x" // configure(update(x)) -> "configure updating x" // if (m.name_do.empty ()) r = io.name_do; else { r = m.name_do; if (io.name_doing[0] != '\0') { r += ' '; r += io.name_doing; } } if (oo != nullptr) { r += " (for "; r += oo->name; r += ')'; } return r; } void diag_do (ostream& os, const action& a, const target& t) { os << diag_do (a) << ' ' << t; } string diag_doing (const action&) { const meta_operation_info& m (*current_mif); const operation_info& io (*current_inner_oif); const operation_info* oo (current_outer_oif); string r; // perform(update(x)) -> "updating x" // configure(update(x)) -> "configuring updating x" // if (!m.name_doing.empty ()) r = m.name_doing; if (io.name_doing[0] != '\0') { if (!r.empty ()) r += ' '; r += io.name_doing; } if (oo != nullptr) { r += " (for "; r += oo->name; r += ')'; } return r; } void diag_doing (ostream& os, const action& a, const target& t) { os << diag_doing (a) << ' ' << t; } string diag_did (const action&) { const meta_operation_info& m (*current_mif); const operation_info& io (*current_inner_oif); const operation_info* oo (current_outer_oif); string r; // perform(update(x)) -> "updated x" // configure(update(x)) -> "configured updating x" // if (!m.name_did.empty ()) { r = m.name_did; if (io.name_doing[0] != '\0') { r += ' '; r += io.name_doing; } } else r += io.name_did; if (oo != nullptr) { r += " (for "; r += oo->name; r += ')'; } return r; } void diag_did (ostream& os, const action& a, const target& t) { os << diag_did (a) << ' ' << t; } void diag_done (ostream& os, const action&, const target& t) { const meta_operation_info& m (*current_mif); const operation_info& io (*current_inner_oif); const operation_info* oo (current_outer_oif); // perform(update(x)) -> "x is up to date" // configure(update(x)) -> "updating x is configured" // if (m.name_done.empty ()) { os << t; if (io.name_done[0] != '\0') os << ' ' << io.name_done; if (oo != nullptr) os << " (for " << oo->name << ')'; } else { if (io.name_doing[0] != '\0') os << io.name_doing << ' '; if (oo != nullptr) os << "(for " << oo->name << ") "; os << t << ' ' << m.name_done; } } }