diff options
Diffstat (limited to 'build2/b.cxx')
| -rw-r--r-- | build2/b.cxx | 1198 |
1 files changed, 528 insertions, 670 deletions
diff --git a/build2/b.cxx b/build2/b.cxx index bb09675..8decadf 100644 --- a/build2/b.cxx +++ b/build2/b.cxx @@ -1,27 +1,18 @@ // file : build2/b.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file -#ifndef _WIN32 -# include <signal.h> // signal() -#else -# include <libbutl/win32-utility.hxx> -#endif - -#ifdef __GLIBCXX__ -# include <locale> -#endif - -#include <limits> #include <sstream> -#include <cstring> // strcmp(), strchr() #include <typeinfo> #include <iostream> // cout #include <exception> // terminate(), set_terminate(), terminate_handler #include <libbutl/pager.hxx> -#include <libbutl/fdstream.hxx> // stderr_fd(), fdterm() -#include <libbutl/backtrace.hxx> // backtrace() -#include <libbutl/default-options.hxx> +#include <libbutl/fdstream.hxx> // stderr_fd(), fdterm() +#include <libbutl/backtrace.hxx> // backtrace() + +#ifndef BUILD2_BOOTSTRAP +# include <libbutl/json/serializer.hxx> +#endif #include <libbuild2/types.hxx> #include <libbuild2/utility.hxx> @@ -44,7 +35,8 @@ #include <libbuild2/parser.hxx> -#include <build2/b-options.hxx> +#include <libbuild2/b-options.hxx> +#include <libbuild2/b-cmdline.hxx> // Build system modules. // @@ -62,8 +54,7 @@ #ifndef BUILD2_BOOTSTRAP # include <libbuild2/bash/init.hxx> - -# include <build2/cli/init.hxx> +# include <libbuild2/cli/init.hxx> #endif using namespace butl; @@ -71,75 +62,161 @@ using namespace std; namespace build2 { - static options ops; - int main (int argc, char* argv[]); +#ifndef BUILD2_BOOTSTRAP // Structured result printer (--structured-result mode). // class result_printer { public: - result_printer (const action_targets& tgs): tgs_ (tgs) {} + result_printer (const b_options& ops, + const action_targets& tgs, + json::stream_serializer& js) + : ops_ (ops), tgs_ (tgs), json_serializer_ (js) {} + ~result_printer (); private: + void + print_lines (); + + void + print_json (); + + private: + const b_options& ops_; const action_targets& tgs_; + json::stream_serializer& json_serializer_; }; + void result_printer:: + print_lines () + { + for (const action_target& at: tgs_) + { + if (at.state == target_state::unknown) + continue; // Not a target/no result. + + const target& t (at.as<target> ()); + context& ctx (t.ctx); + + cout << at.state + << ' ' << ctx.current_mif->name + << ' ' << ctx.current_inner_oif->name; + + if (ctx.current_outer_oif != nullptr) + cout << '(' << ctx.current_outer_oif->name << ')'; + + // There are two ways one may wish to identify the target of the + // operation: as something specific but inherently non-portable (say, a + // filesystem path, for example c:\tmp\foo.exe) or as something regular + // that can be used to refer to a target in a portable way (for example, + // c:\tmp\exe{foo}; note that the directory part is still not portable). + // Which one should we use is a good question. Let's go with the + // portable one for now and see how it goes (we can always add a format + // variant, e.g., --structured-result=lines-path). Note also that the + // json format includes both. + + // Set the stream extension verbosity to 0 to suppress extension + // printing by default (this can still be overriden by the target type's + // print function as is the case for file{}, for example). And set the + // path verbosity to 1 to always print absolute. + // + stream_verbosity sv (stream_verb (cout)); + stream_verb (cout, stream_verbosity (1, 0)); + + cout << ' ' << t << endl; + + stream_verb (cout, sv); + } + } + + void result_printer:: + print_json () + { + json::stream_serializer& s (json_serializer_); + + for (const action_target& at: tgs_) + { + if (at.state == target_state::unknown) + continue; // Not a target/no result. + + const target& t (at.as<target> ()); + context& ctx (t.ctx); + + s.begin_object (); + + // Quoted target. + // + s.member_name ("target"); + dump_quoted_target_name (s, t); + + // Display target. + // + s.member_name ("display_target"); + dump_display_target_name (s, t); + + s.member ("target_type", t.type ().name, false /* check */); + + if (t.is_a<dir> ()) + s.member ("target_path", t.dir.string ()); + else if (const auto* pt = t.is_a<path_target> ()) + s.member ("target_path", pt->path ().string ()); + + s.member ("meta_operation", ctx.current_mif->name, false /* check */); + s.member ("operation", ctx.current_inner_oif->name, false /* check */); + + if (ctx.current_outer_oif != nullptr) + s.member ("outer_operation", + ctx.current_outer_oif->name, + false /* check */); + + s.member ("state", to_string (at.state), false /* check */); + + s.end_object (); + } + } + result_printer:: ~result_printer () { // Let's do some sanity checking even when we are not in the structred // output mode. // +#ifndef NDEBUG for (const action_target& at: tgs_) { switch (at.state) { - case target_state::unknown: continue; // Not a target/no result. + case target_state::unknown: case target_state::unchanged: case target_state::changed: case target_state::failed: break; // Valid states. default: assert (false); } + } +#endif - if (ops.structured_result ()) + if (ops_.structured_result_specified ()) + { + switch (ops_.structured_result ()) { - const target& t (at.as<target> ()); - context& ctx (t.ctx); - - cout << at.state - << ' ' << ctx.current_mif->name - << ' ' << ctx.current_inner_oif->name; - - if (ctx.current_outer_oif != nullptr) - cout << '(' << ctx.current_outer_oif->name << ')'; - - // There are two ways one may wish to identify the target of the - // operation: as something specific but inherently non-portable (say, - // a filesystem path, for example c:\tmp\foo.exe) or as something - // regular that can be used to refer to a target in a portable way - // (for example, c:\tmp\exe{foo}; note that the directory part is - // still not portable). Which one should we use is a good question. - // Let's go with the portable one for now and see how it goes (we - // can always add a format version, e.g., --structured-result=2). - - // Set the stream extension verbosity to 0 to suppress extension - // printing by default (this can still be overriden by the target - // type's print function as is the case for file{}, for example). - // And set the path verbosity to 1 to always print absolute. - // - stream_verbosity sv (stream_verb (cout)); - stream_verb (cout, stream_verbosity (1, 0)); - - cout << ' ' << t << endl; - - stream_verb (cout, sv); + case structured_result_format::lines: + { + print_lines (); + break; + } + case structured_result_format::json: + { + print_json (); + break; + } } } } +#endif } // Print backtrace if terminating due to an unhandled exception. Note that @@ -172,445 +249,21 @@ main (int argc, char* argv[]) tracer trace ("main"); - int r (0); - - // This is a little hack to make out baseutils for Windows work when called - // with absolute path. In a nutshell, MSYS2's exec*p() doesn't search in the - // parent's executable directory, only in PATH. And since we are running - // without a shell (that would read /etc/profile which sets PATH to some - // sensible values), we are only getting Win32 PATH values. And MSYS2 /bin - // is not one of them. So what we are going to do is add /bin at the end of - // PATH (which will be passed as is by the MSYS2 machinery). This will make - // MSYS2 search in /bin (where our baseutils live). And for everyone else - // this should be harmless since it is not a valid Win32 path. - // -#ifdef _WIN32 - { - string mp; - if (optional<string> p = getenv ("PATH")) - { - mp = move (*p); - mp += ';'; - } - mp += "/bin"; - - setenv ("PATH", mp); - } -#endif - - // A data race happens in the libstdc++ (as of GCC 7.2) implementation of - // the ctype<char>::narrow() function (bug #77704). The issue is easily - // triggered by the testscript runner that indirectly (via regex) uses - // ctype<char> facet of the global locale (and can potentially be triggered - // by other locale- aware code). We work around this by pre-initializing the - // global locale facet internal cache. - // -#ifdef __GLIBCXX__ - { - const ctype<char>& ct (use_facet<ctype<char>> (locale ())); - - for (size_t i (0); i != 256; ++i) - ct.narrow (static_cast<char> (i), '\0'); - } -#endif - - // On POSIX ignore SIGPIPE which is signaled to a pipe-writing process if - // the pipe reading end is closed. Note that by default this signal - // terminates a process. Also note that there is no way to disable this - // behavior on a file descriptor basis or for the write() function call. - // -#ifndef _WIN32 - if (signal (SIGPIPE, SIG_IGN) == SIG_ERR) - fail << "unable to ignore broken pipe (SIGPIPE) signal: " - << system_error (errno, generic_category ()); // Sanitize. -#endif + init_process (); + int r (0); + b_options ops; scheduler sched; - // Parse the command line. + // Statistics. // + size_t phase_switch_contention (0); + try { - // Note that the diagnostics verbosity level can only be calculated after - // default options are loaded and merged (see below). Thus, until then we - // refer to the verbosity level specified on the command line. - // - auto verbosity = [] () - { - uint16_t v ( - ops.verbose_specified () - ? ops.verbose () - : ops.V () ? 3 : ops.v () ? 2 : ops.quiet () || ops.silent () ? 0 : 1); - - if (ops.silent () && v != 0) - fail << "specified with -v, -V, or --verbose verbosity level " << v - << " is incompatible with --silent"; - - return v; - }; - - // We want to be able to specify options, vars, and buildspecs in any - // order (it is really handy to just add -v at the end of the command - // line). + // Parse the command line. // - strings cmd_vars; - string args; - try - { - // Command line arguments starting position. - // - // We want the positions of the command line arguments to be after the - // default options files. Normally that would be achieved by passing the - // last position of the previous scanner to the next. The problem is - // that we parse the command line arguments first (for good reasons). - // Also the default options files parsing machinery needs the maximum - // number of arguments to be specified and assigns the positions below - // this value (see load_default_options() for details). So we are going - // to "reserve" the first half of the size_t value range for the default - // options positions and the second half for the command line arguments - // positions. - // - size_t args_pos (numeric_limits<size_t>::max () / 2); - cl::argv_file_scanner scan (argc, argv, "--options-file", args_pos); - - size_t argn (0); // Argument count. - bool shortcut (false); // True if the shortcut syntax is used. - - for (bool opt (true), var (true); scan.more (); ) - { - if (opt) - { - // Parse the next chunk of options until we reach an argument (or - // eos). - // - if (ops.parse (scan) && !scan.more ()) - break; - - // If we see first "--", then we are done parsing options. - // - if (strcmp (scan.peek (), "--") == 0) - { - scan.next (); - opt = false; - continue; - } - - // Fall through. - } - - const char* s (scan.next ()); - - // See if this is a command line variable. What if someone needs to - // pass a buildspec that contains '='? One way to support this would - // be to quote such a buildspec (e.g., "'/tmp/foo=bar/'"). Or invent - // another separator. Or use a second "--". Actually, let's just do - // the second "--". - // - if (var) - { - // If we see second "--", then we are also done parsing variables. - // - if (strcmp (s, "--") == 0) - { - var = false; - continue; - } - - if (const char* p = strchr (s, '=')) // Covers =, +=, and =+. - { - // Diagnose the empty variable name situation. Note that we don't - // allow "partially broken down" assignments (as in foo =bar) - // since foo= bar would be ambigous. - // - if (p == s || (p == s + 1 && *s == '+')) - fail << "missing variable name in '" << s << "'"; - - cmd_vars.push_back (s); - continue; - } - - // Handle the "broken down" variable assignments (i.e., foo = bar - // instead of foo=bar). - // - if (scan.more ()) - { - const char* a (scan.peek ()); - - if (strcmp (a, "=" ) == 0 || - strcmp (a, "+=") == 0 || - strcmp (a, "=+") == 0) - { - string v (s); - v += a; - - scan.next (); - - if (scan.more ()) - v += scan.next (); - - cmd_vars.push_back (move (v)); - continue; - } - } - - // Fall through. - } - - // Merge all the individual buildspec arguments into a single string. - // We use newlines to separate arguments so that line numbers in - // diagnostics signify argument numbers. Clever, huh? - // - if (argn != 0) - args += '\n'; - - args += s; - - // See if we are using the shortcut syntax. - // - if (argn == 0 && args.back () == ':') - { - args.back () = '('; - shortcut = true; - } - - argn++; - } - - // Add the closing parenthesis unless there wasn't anything in between - // in which case pop the opening one. - // - if (shortcut) - { - if (argn == 1) - args.pop_back (); - else - args += ')'; - } - - // Get/set an environment variable tracing the operation. - // - auto get_env = [&verbosity, &trace] (const char* nm) - { - optional<string> r (getenv (nm)); - - if (verbosity () >= 5) - { - if (r) - trace << nm << ": '" << *r << "'"; - else - trace << nm << ": <NULL>"; - } - - return r; - }; - - auto set_env = [&verbosity, &trace] (const char* nm, const string& vl) - { - try - { - if (verbosity () >= 5) - trace << "setting " << nm << "='" << vl << "'"; - - setenv (nm, vl); - } - catch (const system_error& e) - { - // The variable value can potentially be long/multi-line, so let's - // print it last. - // - fail << "unable to set environment variable " << nm << ": " << e << - info << "value: '" << vl << "'"; - } - }; - - // If the BUILD2_VAR_OVR environment variable is present, then parse its - // value as a newline-separated global variable overrides and prepend - // them to the overrides specified on the command line. - // - // Note that this means global overrides may not contain a newline. - - // Verify that the string is a valid global override. Uses the file name - // and the options flag for diagnostics only. - // - auto verify_glb_ovr = [] (const string& v, const path_name& fn, bool opt) - { - size_t p (v.find ('=', 1)); - if (p == string::npos || v[0] != '!') - { - diag_record dr (fail (fn)); - dr << "expected " << (opt ? "option or " : "") << "global " - << "variable override instead of '" << v << "'"; - - if (p != string::npos) - dr << info << "prefix variable assignment with '!'"; - } - - if (p == 1 || (p == 2 && v[1] == '+')) // '!=' or '!+=' ? - fail (fn) << "missing variable name in '" << v << "'"; - }; - - optional<string> env_ovr (get_env ("BUILD2_VAR_OVR")); - if (env_ovr) - { - path_name fn ("<BUILD2_VAR_OVR>"); - - auto i (cmd_vars.begin ()); - for (size_t b (0), e (0); next_word (*env_ovr, b, e, '\n', '\r'); ) - { - // Extract the override from the current line, stripping the leading - // and trailing spaces. - // - string s (*env_ovr, b, e - b); - trim (s); - - // Verify and save the override, unless the line is empty. - // - if (!s.empty ()) - { - verify_glb_ovr (s, fn, false /* opt */); - i = cmd_vars.insert (i, move (s)) + 1; - } - } - } - - // Load the default options files, unless --no-default-options is - // specified on the command line or the BUILD2_DEF_OPT environment - // variable is set to a value other than 'true' or '1'. - // - // If loaded, prepend the default global overrides to the variables - // specified on the command line, unless BUILD2_VAR_OVR is set in which - // case just ignore them. - // - optional<string> env_def (get_env ("BUILD2_DEF_OPT")); - - // False if --no-default-options is specified on the command line. Note - // that we cache the flag since it can be overridden by a default - // options file. - // - bool cmd_def (!ops.no_default_options ()); - - if (cmd_def && (!env_def || *env_def == "true" || *env_def == "1")) - try - { - optional<dir_path> extra; - if (ops.default_options_specified ()) - extra = ops.default_options (); - - // Load default options files. - // - default_options<options> def_ops ( - load_default_options<options, - cl::argv_file_scanner, - cl::unknown_mode> ( - nullopt /* sys_dir */, - path::home_directory (), // The home variable is not assigned yet. - extra, - default_options_files {{path ("b.options")}, - nullopt /* start */}, - [&trace, &verbosity] (const path& f, bool r, bool o) - { - if (verbosity () >= 3) - { - if (o) - trace << "treating " << f << " as " - << (r ? "remote" : "local"); - else - trace << "loading " << (r ? "remote " : "local ") << f; - } - }, - "--options-file", - args_pos, - 1024, - true /* args */)); - - // Merge the default and command line options. - // - ops = merge_default_options (def_ops, ops); - - // Merge the default and command line global overrides, unless - // BUILD2_VAR_OVR is already set (in which case we assume this has - // already been done). - // - // Note that the "broken down" variable assignments occupying a single - // line are naturally supported. - // - if (!env_ovr) - cmd_vars = - merge_default_arguments ( - def_ops, - cmd_vars, - [&verify_glb_ovr] (const default_options_entry<options>& e, - const strings&) - { - path_name fn (e.file); - - // Verify that all arguments are global overrides. - // - for (const string& a: e.arguments) - verify_glb_ovr (a, fn, true /* opt */); - }); - } - catch (const invalid_argument& e) - { - fail << "unable to load default options files: " << e; - } - catch (const pair<path, system_error>& e) - { - fail << "unable to load default options files: " << e.first << ": " - << e.second; - } - catch (const system_error& e) - { - fail << "unable to obtain home directory: " << e; - } - - // Verify and save the global overrides present in cmd_vars (default, - // from the command line, etc), if any, into the BUILD2_VAR_OVR - // environment variable. - // - if (!cmd_vars.empty ()) - { - string ovr; - for (const string& v: cmd_vars) - { - if (v[0] == '!') - { - if (v.find_first_of ("\n\r") != string::npos) - fail << "newline in global variable override '" << v << "'"; - - if (!ovr.empty ()) - ovr += '\n'; - - ovr += v; - } - } - - // Optimize for the common case. - // - // Note: cmd_vars may contain non-global overrides. - // - if (!ovr.empty () && (!env_ovr || *env_ovr != ovr)) - set_env ("BUILD2_VAR_OVR", ovr); - } - - // Propagate disabling of the default options files to the potential - // nested invocations. - // - if (!cmd_def && (!env_def || *env_def != "0")) - set_env ("BUILD2_DEF_OPT", "0"); - - // Validate options. - // - if (ops.progress () && ops.no_progress ()) - fail << "both --progress and --no-progress specified"; - - if (ops.mtime_check () && ops.no_mtime_check ()) - fail << "both --mtime-check and --no-mtime-check specified"; - } - catch (const cl::exception& e) - { - fail << e; - } + b_cmdline cmdl (parse_b_cmdline (trace, argc, argv, ops)); // Handle --build2-metadata (see also buildfile). // @@ -660,10 +313,10 @@ main (int argc, char* argv[]) // Initialize the diagnostics state. // - init_diag (verbosity (), + init_diag (cmdl.verbosity, ops.silent (), - (ops.progress () ? optional<bool> (true) : - ops.no_progress () ? optional<bool> (false) : nullopt), + cmdl.progress, + cmdl.diag_color, ops.no_line (), ops.no_column (), fdterm (stderr_fd ())); @@ -694,36 +347,14 @@ main (int argc, char* argv[]) } } - // Initialize time conversion data that is used by localtime_r(). - // -#ifndef _WIN32 - tzset (); -#else - _tzset (); -#endif - // Initialize the global state. // init (&::terminate, argv[0], - (ops.mtime_check () ? optional<bool> (true) : - ops.no_mtime_check () ? optional<bool> (false) : nullopt), - (ops.config_sub_specified () - ? optional<path> (ops.config_sub ()) - : nullopt), - (ops.config_guess_specified () - ? optional<path> (ops.config_guess ()) - : nullopt)); - -#ifdef _WIN32 - // On Windows disable displaying error reporting dialog box for the - // current and child processes unless we are in the stop mode. Failed that - // we may have multiple dialog boxes popping up. - // - if (!ops.serial_stop ()) - SetErrorMode (SetErrorMode (0) | // Returns the current mode. - SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); -#endif + ops.serial_stop (), + cmdl.mtime_check, + cmdl.config_sub, + cmdl.config_guess); // Load builtin modules. // @@ -740,64 +371,20 @@ main (int argc, char* argv[]) load_builtin_module (&in::build2_in_load); #ifndef BUILD2_BOOTSTRAP - load_builtin_module (&cli::build2_cli_load); load_builtin_module (&bash::build2_bash_load); + load_builtin_module (&cli::build2_cli_load); #endif // Start up the scheduler and allocate lock shards. // - size_t jobs (0); - - if (ops.jobs_specified ()) - jobs = ops.jobs (); - else if (ops.serial_stop ()) - jobs = 1; - - if (jobs == 0) - jobs = scheduler::hardware_concurrency (); - - if (jobs == 0) - { - warn << "unable to determine the number of hardware threads" << - info << "falling back to serial execution" << - info << "use --jobs|-j to override"; - - jobs = 1; - } - - size_t max_jobs (0); - - if (ops.max_jobs_specified ()) - { - max_jobs = ops.max_jobs (); - - if (max_jobs != 0 && max_jobs < jobs) - fail << "invalid --max-jobs|-J value"; - } - - sched.startup (jobs, - 1, - max_jobs, - jobs * ops.queue_depth (), - (ops.max_stack_specified () - ? optional<size_t> (ops.max_stack () * 1024) - : nullopt)); + sched.startup (cmdl.jobs, + 1 /* init_active */, + cmdl.max_jobs, + cmdl.jobs * ops.queue_depth (), + cmdl.max_stack); global_mutexes mutexes (sched.shard_size ()); - - bool fcache_comp (true); - if (ops.file_cache_specified ()) - { - const string& v (ops.file_cache ()); - if (v == "noop" || v == "none") - fcache_comp = false; - else if (v == "sync-lz4") - fcache_comp = true; - else - fail << "invalid --file-cache value '" << v << "'"; - } - - file_cache fcache (fcache_comp); + file_cache fcache (cmdl.fcache_compress); // Trace some overall environment information. // @@ -809,7 +396,7 @@ main (int argc, char* argv[]) trace << "home: " << home; trace << "path: " << (p ? *p : "<NULL>"); trace << "type: " << (build_installed ? "installed" : "development"); - trace << "jobs: " << jobs; + trace << "jobs: " << cmdl.jobs; } // Set the build context before parsing the buildspec since it relies on @@ -817,17 +404,37 @@ main (int argc, char* argv[]) // below). // unique_ptr<context> pctx; - auto new_context = [&pctx, &sched, &mutexes, &fcache, &cmd_vars] + auto new_context = [&ops, &cmdl, + &sched, &mutexes, &fcache, + &phase_switch_contention, + &pctx] { - pctx = nullptr; // Free first. + if (pctx != nullptr) + { + phase_switch_contention += (pctx->phase_mutex.contention + + pctx->phase_mutex.contention_load); + pctx = nullptr; // Free first to reuse memory. + } + + optional<match_only_level> mo; + if (ops.load_only ()) mo = match_only_level::alias; + else if (ops.match_only ()) mo = match_only_level::all; + pctx.reset (new context (sched, mutexes, fcache, - ops.match_only (), + mo, ops.no_external_modules (), ops.dry_run (), + ops.no_diag_buffer (), !ops.serial_stop () /* keep_going */, - cmd_vars)); + cmdl.cmd_vars)); + + if (ops.trace_match_specified ()) + pctx->trace_match = &ops.trace_match (); + + if (ops.trace_execute_specified ()) + pctx->trace_execute = &ops.trace_execute (); }; new_context (); @@ -835,17 +442,18 @@ main (int argc, char* argv[]) // Parse the buildspec. // buildspec bspec; + path_name bspec_name ("<buildspec>"); try { - istringstream is (args); + istringstream is (cmdl.buildspec); is.exceptions (istringstream::failbit | istringstream::badbit); parser p (*pctx); - bspec = p.parse_buildspec (is, path_name ("<buildspec>")); + bspec = p.parse_buildspec (is, bspec_name); } catch (const io_error&) { - fail << "unable to parse buildspec '" << args << "'"; + fail << "unable to parse buildspec '" << cmdl.buildspec << "'"; } l5 ([&]{trace << "buildspec: " << bspec;}); @@ -853,18 +461,194 @@ main (int argc, char* argv[]) if (bspec.empty ()) bspec.push_back (metaopspec ()); // Default meta-operation. + // The reserve values were picked experimentally. They allow building a + // sample application that depends on Qt and Boost without causing a + // rehash. + // + // Note: omit reserving anything for the info meta-operation since it + // won't be loading the buildfiles and needs to be as fast as possible. + // + bool mo_info (bspec.size () == 1 && + bspec.front ().size () == 1 && + (bspec.front ().name == "info" || + (bspec.front ().name.empty () && + bspec.front ().front ().name == "info"))); + + if (!mo_info) + { + // Note: also adjust in bpkg if adjusting here. + // + pctx->reserve (context::reserves { + 30000 /* targets */, + 1100 /* variables */}); + } + + bool load_only (ops.load_only ()); + const path& buildfile (ops.buildfile_specified () ? ops.buildfile () : empty_path); bool dump_load (false); bool dump_match (false); - if (ops.dump_specified ()) + bool dump_match_pre (false); + bool dump_match_post (false); + for (const string& p: ops.dump ()) { - dump_load = ops.dump ().find ("load") != ops.dump ().end (); - dump_match = ops.dump ().find ("match") != ops.dump ().end (); + if (p == "load") dump_load = true; + else if (p == "match") dump_match = true; + else if (p == "match-pre") dump_match_pre = true; + else if (p == "match-post") dump_match_post = true; + else fail << "unknown phase '" << p << "' specified with --dump"; } + dump_format dump_fmt (dump_format::buildfile); + if (ops.dump_format_specified ()) + { + const string& f (ops.dump_format ()); + + if (f == "json-v0.1") + { +#ifdef BUILD2_BOOTSTRAP + fail << "json dump not supported in bootstrap build system"; +#endif + dump_fmt = dump_format::json; + } + else if (f != "buildfile") + { + diag_record dr (fail); + + dr << "unsupported format '" << f << "' specified with --dump-format"; + + if (f.compare (0, 4, "json") == 0) + dr << info << "supported json format version is json-v0.1"; + } + } + + auto dump = [&trace, &ops, dump_fmt] (context& ctx, optional<action> a) + { + const dir_paths& scopes (ops.dump_scope ()); + const vector<pair<name, optional<name>>>& targets (ops.dump_target ()); + + if (scopes.empty () && targets.empty ()) + build2::dump (ctx, a, dump_fmt); + else + { + auto comp_norm = [] (dir_path& d, const char* what) + { + try + { + if (d.relative ()) + d.complete (); + + d.normalize (); + } + catch (const invalid_path& e) + { + fail << "invalid path '" << e.path << "' specified with " << what; + } + }; + + // If exact is false then return any outer scope that contains this + // directory except for the global scope. + // + auto find_scope = [&ctx, &comp_norm] (dir_path& d, + bool exact, + const char* what) -> const scope* + { + comp_norm (d, what); + + // This is always the output directory (specifically, see the target + // case below). + // + const scope& s (ctx.scopes.find_out (d)); + + return ((exact ? s.out_path () == d : s != ctx.global_scope) + ? &s + : nullptr); + }; + + // Dump scopes. + // + for (dir_path d: scopes) + { + const scope* s (find_scope (d, true, "--dump-scope")); + + if (s == nullptr) + l5 ([&]{trace << "unknown target scope " << d + << " specified with --dump-scope";}); + + build2::dump (s, a, dump_fmt); + } + + // Dump targets. + // + for (const pair<name, optional<name>>& p: targets) + { + const target* t (nullptr); + + // Find the innermost known scope that contains this target. This + // is where we are going to resolve its type. + // + dir_path d (p.second ? p.second->dir : p.first.dir); + + if (const scope* s = find_scope (d, false, "--dump-target")) + { + // Complete relative directories in names. + // + name n (p.first), o; + + if (p.second) + { + comp_norm (n.dir, "--dump-target"); + o.dir = move (d); + } + else + n.dir = move (d); + + // Similar logic to parser::enter_target::find_target() as used by + // the dump directive. Except here we treat unknown target type as + // unknown target. + // + auto r (s->find_target_type (n, location ())); + + if (r.first != nullptr) + { + t = ctx.targets.find (*r.first, // target type + n.dir, + o.dir, + n.value, + r.second, // extension + trace); + + if (t == nullptr) + l5 ([&] + { + // @@ TODO: default_extension? + // + target::combine_name (n.value, r.second, false); + names ns {move (n)}; + if (p.second) + ns.push_back (move (o)); + + trace << "unknown target " << ns + << " specified with --dump-target"; + }); + } + else + l5 ([&]{trace << "unknown target type '" << n.type << "' in " + << *s << " specified with --dump-target";}); + + } + else + l5 ([&]{trace << "unknown target scope " << d + << " specified with --dump-target";}); + + build2::dump (t, a, dump_fmt); + } + } + }; + // If not NULL, then lifted points to the operation that has been "lifted" // to the meta-operaion (see the logic below for details). Skip is the // position of the next operation. @@ -877,6 +661,20 @@ main (int argc, char* argv[]) // bool dirty (false); // Already (re)set for the first run. +#ifndef BUILD2_BOOTSTRAP + // Note that this constructor is cheap and so we rather call it always + // instead of resorting to dynamic allocations. + // + // Note also that we disable pretty-printing if there is also the JSON + // dump and thus we need to combine the two in the JSON Lines format. + // + json::stream_serializer js (cout, dump_fmt == dump_format::json ? 0 : 2); + + if (ops.structured_result_specified () && + ops.structured_result () == structured_result_format::json) + js.begin_array (); +#endif + for (auto mit (bspec.begin ()); mit != bspec.end (); ) { vector_view<opspec> opspecs; @@ -920,8 +718,7 @@ main (int argc, char* argv[]) context& ctx (*pctx); - const path p ("<buildspec>"); - const location l (p, 0, 0); //@@ TODO + const location l (bspec_name, 0, 0); //@@ TODO (also bpkg::pkg_configure()) meta_operation_id mid (0); // Not yet translated. const meta_operation_info* mif (nullptr); @@ -1059,12 +856,19 @@ main (int argc, char* argv[]) } } - if (out_base.relative ()) - out_base = work / out_base; + try + { + if (out_base.relative ()) + out_base = work / out_base; - // This directory came from the command line so actualize it. - // - out_base.normalize (true); + // This directory came from the command line so actualize it. + // + out_base.normalize (true); + } + catch (const invalid_path& e) + { + fail << "invalid out_base directory '" << e.path << "'"; + } // The order in which we determine the roots depends on whether // src_base was specified explicitly. @@ -1090,12 +894,19 @@ main (int argc, char* argv[]) if (!exists (src_base)) fail << "src_base directory " << src_base << " does not exist"; - if (src_base.relative ()) - src_base = work / src_base; + try + { + if (src_base.relative ()) + src_base = work / src_base; - // Also came from the command line, so actualize. - // - src_base.normalize (true); + // Also came from the command line, so actualize. + // + src_base.normalize (true); + } + catch (const invalid_path& e) + { + fail << "invalid src_base directory '" << e.path << "'"; + } // Make sure out_base is not a subdirectory of src_base. Who would // want to do that, you may ask. Well, you would be surprised... @@ -1250,8 +1061,13 @@ main (int argc, char* argv[]) // Now that we have src_root, load the src_root bootstrap file, // if there is one. // + // As an optimization, omit discovering subprojects for the info + // meta-operation if not needed. + // bootstrap_pre (rs, altn); - bootstrap_src (rs, altn); + bootstrap_src (rs, altn, + nullopt /* amalgamation */, + !mo_info || info_subprojects (mparams) /*subprojects*/); // If this is a simple project, then implicitly load the test and // install modules. @@ -1468,23 +1284,38 @@ main (int argc, char* argv[]) if (oif->outer_id != 0) outer_oif = lookup (oif->outer_id); + if (!oparams.empty ()) + { + // Operation parameters belong to outer operation, if any. + // + auto* i (outer_oif != nullptr ? outer_oif : oif); + + if (i->operation_pre == nullptr) + fail (l) << "unexpected parameters for operation " << i->name; + } + // Handle pre/post operations. // if (auto po = oif->pre_operation) { - if ((orig_pre_oid = po (ctx, oparams, mid, l)) != 0) + if ((orig_pre_oid = po ( + ctx, + outer_oif == nullptr ? oparams : values {}, + mid, + l)) != 0) { assert (orig_pre_oid != default_id); pre_oif = lookup (orig_pre_oid); pre_oid = pre_oif->id; // De-alias. } } - else if (!oparams.empty ()) - fail (l) << "unexpected parameters for operation " << oif->name; if (auto po = oif->post_operation) { - if ((orig_post_oid = po (ctx, oparams, mid)) != 0) + if ((orig_post_oid = po ( + ctx, + outer_oif == nullptr ? oparams : values {}, + mid)) != 0) { assert (orig_post_oid != default_id); post_oif = lookup (orig_post_oid); @@ -1530,6 +1361,9 @@ main (int argc, char* argv[]) // defined there (common with non-intrusive project conversions // where everything is built from a single root buildfile). // + // Note: we use find_plausible_buildfile() and not find_buildfile() + // to look in outer directories. + // optional<path> bf ( find_buildfile (src_base, src_base, altn, buildfile)); @@ -1580,62 +1414,7 @@ main (int argc, char* argv[]) // boundaries (specifically, amalgamation) are only known after // bootstrap. // - // The mildly tricky part here is to distinguish the situation where - // we are bootstrapping the same project multiple times. The first - // override that we set cannot already exist (because the override - // variable names are unique) so if it is already set, then it can - // only mean this project is already bootstrapped. - // - // This is further complicated by the project vs amalgamation logic - // (we may have already done the amalgamation but not the project). - // So we split it into two passes. - // - { - auto& sm (ctx.scopes.rw ()); - - for (const variable_override& o: ctx.var_overrides) - { - if (o.ovr.visibility != variable_visibility::global) - continue; - - // If we have a directory, enter the scope, similar to how we do - // it in the context ctor. - // - scope& s ( - o.dir - ? *sm.insert_out ((out_base / *o.dir).normalize ())->second.front () - : *rs.weak_scope ()); - - auto p (s.vars.insert (o.ovr)); - - if (!p.second) - break; - - value& v (p.first); - v = o.val; - } - - for (const variable_override& o: ctx.var_overrides) - { - // Ours is either project (%foo) or scope (/foo). - // - if (o.ovr.visibility == variable_visibility::global) - continue; - - scope& s ( - o.dir - ? *sm.insert_out ((out_base / *o.dir).normalize ())->second.front () - : rs); - - auto p (s.vars.insert (o.ovr)); - - if (!p.second) - break; - - value& v (p.first); - v = o.val; - } - } + ctx.enter_project_overrides (rs, out_base, ctx.var_overrides); ts.root_scope = &rs; ts.out_base = move (out_base); @@ -1650,6 +1429,9 @@ main (int argc, char* argv[]) break; } + if (load_only && (mid != perform_id || oid != update_id)) + fail << "--load-only requires perform(update) action"; + // Now load the buildfiles and search the targets. // action_targets tgs; @@ -1681,6 +1463,9 @@ main (int argc, char* argv[]) if (tt == nullptr) fail (l) << "unknown target type " << tn.type; + if (load_only && !tt->is_a<alias> ()) + fail << "--load-only requires alias target"; + if (mif->search != nullptr) { // If the directory is relative, assume it is relative to work @@ -1688,10 +1473,17 @@ main (int argc, char* argv[]) // dir_path& d (tn.dir); - if (d.relative ()) - d = work / d; + try + { + if (d.relative ()) + d = work / d; - d.normalize (true); // Actualize since came from command line. + d.normalize (true); // Actualize since came from command line. + } + catch (const invalid_path& e) + { + fail << "invalid target directory '" << e.path << "'"; + } if (ts.forwarded) d = rs.out_path () / d.leaf (rs.src_path ()); // Remap. @@ -1711,8 +1503,10 @@ main (int argc, char* argv[]) } } // target - if (dump_load) - dump (ctx); + // Delay until after match in the --load-only mode (see below). + // + if (dump_load && !load_only) + dump (ctx, nullopt /* action */); // Finally, match the rules and perform the operation. // @@ -1726,22 +1520,36 @@ main (int argc, char* argv[]) ctx.current_operation (*pre_oif, oif); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, oparams, false /* inner */, l); + + if (pre_oif->operation_pre != nullptr) + pre_oif->operation_pre (ctx, {}, true /* inner */, l); + action a (mid, pre_oid, oid); { - result_printer p (tgs); - uint16_t diag (ops.structured_result () ? 0 : 1); +#ifndef BUILD2_BOOTSTRAP + result_printer p (ops, tgs, js); +#endif + uint16_t diag (ops.structured_result_specified () ? 0 : 1); if (mif->match != nullptr) mif->match (mparams, a, tgs, diag, true /* progress */); - if (dump_match) + if (dump_match_pre) dump (ctx, a); if (mif->execute != nullptr && !ctx.match_only) mif->execute (mparams, a, tgs, diag, true /* progress */); } + if (pre_oif->operation_post != nullptr) + pre_oif->operation_post (ctx, {}, true /* inner */); + + if (oif->operation_post != nullptr) + oif->operation_post (ctx, oparams, false /* inner */); + if (mif->operation_post != nullptr) mif->operation_post (ctx, mparams, pre_oid); @@ -1753,11 +1561,22 @@ main (int argc, char* argv[]) ctx.current_operation (*oif, outer_oif); + if (outer_oif != nullptr && outer_oif->operation_pre != nullptr) + outer_oif->operation_pre (ctx, oparams, false /* inner */, l); + + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, + outer_oif == nullptr ? oparams : values {}, + true /* inner */, + l); + action a (mid, oid, oif->outer_id); { - result_printer p (tgs); - uint16_t diag (ops.structured_result () ? 0 : 2); +#ifndef BUILD2_BOOTSTRAP + result_printer p (ops, tgs, js); +#endif + uint16_t diag (ops.structured_result_specified () ? 0 : 2); if (mif->match != nullptr) mif->match (mparams, a, tgs, diag, true /* progress */); @@ -1769,6 +1588,14 @@ main (int argc, char* argv[]) mif->execute (mparams, a, tgs, diag, true /* progress */); } + if (oif->operation_post != nullptr) + oif->operation_post (ctx, + outer_oif == nullptr ? oparams : values {}, + true /* inner */); + + if (outer_oif != nullptr && outer_oif->operation_post != nullptr) + outer_oif->operation_post (ctx, oparams, false /* inner */); + if (post_oid != 0) { tgs.reset (); @@ -1781,22 +1608,36 @@ main (int argc, char* argv[]) ctx.current_operation (*post_oif, oif); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, oparams, false /* inner */, l); + + if (post_oif->operation_pre != nullptr) + post_oif->operation_pre (ctx, {}, true /* inner */, l); + action a (mid, post_oid, oid); { - result_printer p (tgs); - uint16_t diag (ops.structured_result () ? 0 : 1); +#ifndef BUILD2_BOOTSTRAP + result_printer p (ops, tgs, js); +#endif + uint16_t diag (ops.structured_result_specified () ? 0 : 1); if (mif->match != nullptr) mif->match (mparams, a, tgs, diag, true /* progress */); - if (dump_match) + if (dump_match_post) dump (ctx, a); if (mif->execute != nullptr && !ctx.match_only) mif->execute (mparams, a, tgs, diag, true /* progress */); } + if (post_oif->operation_post != nullptr) + post_oif->operation_post (ctx, {}, true /* inner */); + + if (oif->operation_post != nullptr) + oif->operation_post (ctx, oparams, false /* inner */); + if (mif->operation_post != nullptr) mif->operation_post (ctx, mparams, post_oid); @@ -1804,6 +1645,9 @@ main (int argc, char* argv[]) << ", id " << static_cast<uint16_t> (post_oid);}); } + if (dump_load && load_only) + dump (ctx, nullopt /* action */); + if (mif->operation_post != nullptr) mif->operation_post (ctx, mparams, oid); @@ -1823,6 +1667,18 @@ main (int argc, char* argv[]) if (lifted == nullptr && skip == 0) ++mit; } // meta-operation + +#ifndef BUILD2_BOOTSTRAP + if (ops.structured_result_specified () && + ops.structured_result () == structured_result_format::json) + { + js.end_array (); + cout << endl; + } +#endif + + phase_switch_contention += (pctx->phase_mutex.contention + + pctx->phase_mutex.contention_load); } catch (const failed&) { @@ -1844,16 +1700,18 @@ main (int argc, char* argv[]) { text << '\n' << "build statistics:" << "\n\n" - << " thread_max_active " << st.thread_max_active << '\n' - << " thread_max_total " << st.thread_max_total << '\n' - << " thread_helpers " << st.thread_helpers << '\n' - << " thread_max_waiting " << st.thread_max_waiting << '\n' + << " thread_max_active " << st.thread_max_active << '\n' + << " thread_max_total " << st.thread_max_total << '\n' + << " thread_helpers " << st.thread_helpers << '\n' + << " thread_max_waiting " << st.thread_max_waiting << '\n' + << '\n' + << " task_queue_depth " << st.task_queue_depth << '\n' + << " task_queue_full " << st.task_queue_full << '\n' << '\n' - << " task_queue_depth " << st.task_queue_depth << '\n' - << " task_queue_full " << st.task_queue_full << '\n' + << " wait_queue_slots " << st.wait_queue_slots << '\n' + << " wait_queue_collisions " << st.wait_queue_collisions << '\n' << '\n' - << " wait_queue_slots " << st.wait_queue_slots << '\n' - << " wait_queue_collisions " << st.wait_queue_collisions << '\n'; + << " phase_switch_contention " << phase_switch_contention << '\n'; } return r; |