diff options
| author | Boris Kolpackov <boris@codesynthesis.com> | 2020-04-27 09:49:45 +0200 |
|---|---|---|
| committer | Boris Kolpackov <boris@codesynthesis.com> | 2020-04-27 10:03:50 +0200 |
| commit | 9e5750ae2e3f837f80860aaab6b01e4d556213ed (patch) | |
| tree | d3b2e551e444c47b6ce0289969e78360161b6685 /libbuild2 | |
| parent | 028e10ba787a7dbb46e3fcba6f88f496b76cebc5 (diff) | |
Rework tool importation along with cli module
Specifically, now config.<tool> (like config.cli) is handled by the import
machinery (it is like a shorter alias for config.import.<tool>.<tool>.exe
that we already had). And the cli module now uses that instead of custom
logic.
This also adds support for uniform tool metadata extraction that is handled by
the import machinery. As a result, a tool that follows the "build2 way" can be
imported with metadata by the buildfile and/or corresponding module without
any tool-specific code or brittleness associated with parsing --version or
similar outputs. See the cli tool/module for details.
Finally, two new flavors of the import directive are now supported: import!
triggers immediate importation skipping any rule-specific logic while import?
is optional import (analogous to using?). Note that optional import is always
immediate. There is also the import-specific metadata attribute which can be
specified for these two import flavors in order to trigger metadata
importation. For example:
import? [metadata] cli = cli%exe{cli}
if ($cli != [null])
info "cli version $($cli:cli.version)"
Diffstat (limited to 'libbuild2')
31 files changed, 1380 insertions, 302 deletions
diff --git a/libbuild2/bin/init.cxx b/libbuild2/bin/init.cxx index ab4b686..34bfcd7 100644 --- a/libbuild2/bin/init.cxx +++ b/libbuild2/bin/init.cxx @@ -150,7 +150,7 @@ namespace build2 // We only support root loading (which means there can only be one). // - if (&rs != &bs) + if (rs != bs) fail (loc) << "bin.config module must be loaded in project root"; // Load bin.vars. diff --git a/libbuild2/c/init.cxx b/libbuild2/c/init.cxx index e6e28aa..5eea8de 100644 --- a/libbuild2/c/init.cxx +++ b/libbuild2/c/init.cxx @@ -142,7 +142,7 @@ namespace build2 // We only support root loading (which means there can only be one). // - if (&rs != &bs) + if (rs != bs) fail (loc) << "c.guess module must be loaded in project root"; // Load cc.core.vars so that we can cache all the cc.* variables. @@ -276,7 +276,7 @@ namespace build2 // We only support root loading (which means there can only be one). // - if (&rs != &bs) + if (rs != bs) fail (loc) << "c.config module must be loaded in project root"; // Load c.guess and share its module instance as ours. @@ -313,7 +313,7 @@ namespace build2 // We only support root loading (which means there can only be one). // - if (&rs != &bs) + if (rs != bs) fail (loc) << "c module must be loaded in project root"; // Load c.config. diff --git a/libbuild2/cc/common.cxx b/libbuild2/cc/common.cxx index 99de66e..cf6d546 100644 --- a/libbuild2/cc/common.cxx +++ b/libbuild2/cc/common.cxx @@ -465,20 +465,8 @@ namespace build2 { // This is import. // - name n (cn); - auto rp (s.find_target_type (n, location ())); // Note: changes name. - const target_type* tt (rp.first); - optional<string>& ext (rp.second); - - if (tt == nullptr) - fail << "unknown target type '" << n.type << "' in library " << n; - - // @@ OUT: for now we assume out is undetermined, just like in - // search (name, scope). - // - dir_path out; - - prerequisite_key pk {n.proj, {tt, &n.dir, &out, &n.value, ext}, &s}; + name n (cn), o; // Note: find_prerequisite_key() changes name. + prerequisite_key pk (s.find_prerequisite_key (n, o, location ())); xt = search_library_existing (a, sysd, usrd, pk); if (xt == nullptr) diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx index a8916cf..8b082cc 100644 --- a/libbuild2/cc/compile-rule.cxx +++ b/libbuild2/cc/compile-rule.cxx @@ -696,8 +696,8 @@ namespace build2 continue; // A dependency on a library is there so that we can get its - // *.export.poptions, modules, etc. This is the library - // meta-information protocol. See also append_lib_options(). + // *.export.poptions, modules, etc. This is the library metadata + // protocol. See also append_lib_options(). // if (pi == include_type::normal && (p.is_a<libx> () || @@ -4871,7 +4871,7 @@ namespace build2 // // For (direct) library prerequisites, check their prerequisite bmi{}s // (which should be searched and matched with module names discovered; - // see the library meta-information protocol for details). + // see the library metadata protocol for details). // // For our own bmi{} prerequisites, checking if each (better) matches // any of the imports. diff --git a/libbuild2/cc/init.cxx b/libbuild2/cc/init.cxx index 8d66376..2a0dbd2 100644 --- a/libbuild2/cc/init.cxx +++ b/libbuild2/cc/init.cxx @@ -399,7 +399,7 @@ namespace build2 // We only support root loading (which means there can only be one). // - if (&rs != &bs) + if (rs != bs) fail (loc) << m << " module must be loaded in project root"; // We want to order the loading to match what user specified on the diff --git a/libbuild2/cc/utility.hxx b/libbuild2/cc/utility.hxx index fa9f165..017765b 100644 --- a/libbuild2/cc/utility.hxx +++ b/libbuild2/cc/utility.hxx @@ -41,10 +41,10 @@ namespace build2 // // The reason we pass scope and not the target is because this function is // called not only for exe/lib but also for obj as part of the library - // meta-information protocol implementation. Normally the bin.*.lib values - // will be project-wide. With this scheme they can be customized on the - // per-directory basis but not per-target which means all exe/lib in the - // same directory have to have the same link order. + // metadata protocol implementation. Normally the bin.*.lib values will be + // project-wide. With this scheme they can be customized on the per- + // directory basis but not per-target which means all exe/lib in the same + // directory have to have the same link order. // lorder link_order (const scope& base, otype); diff --git a/libbuild2/config/utility.hxx b/libbuild2/config/utility.hxx index 614791f..fb3023b 100644 --- a/libbuild2/config/utility.hxx +++ b/libbuild2/config/utility.hxx @@ -311,6 +311,20 @@ namespace build2 // be used to "remember" that the module is left unconfigured in order to // avoid re-running the tests, etc. // + // @@ This functionality is WIP/unused and still has a number of issues: + // + // - This seems to be a subset of a bigger problem of caching discovered + // configuration results. In fact, what we do in the configured case, + // for example in the cc module (multiple path extraction runs, etc), is + // a lot more expensive. + // + // - The current semantics does not work well for the case where, say, the + // missing tool has appeared in PATH and can now be used via the default + // configuration. In fact, even reconfiguring will not help without a + // "nudge" (e.g., config.<tool>=<tool>). So maybe this value should be + // ignored during configuration? See the "Tool importation: unconfigured + // state" page for more notes. + // LIBBUILD2_SYMEXPORT bool unconfigured (scope& rs, const string& var); diff --git a/libbuild2/context.cxx b/libbuild2/context.cxx index 06cdc6d..0be0046 100644 --- a/libbuild2/context.cxx +++ b/libbuild2/context.cxx @@ -544,7 +544,10 @@ namespace build2 var_project_url = &vp.insert<string> ("project.url"); var_project_summary = &vp.insert<string> ("project.summary"); - var_import_target = &vp.insert<name> ("import.target"); + var_import_target = &vp.insert<name> ("import.target"); + var_import_metadata = &vp.insert<uint64_t> ("import.metadata"); + + var_export_metadata = &vp.insert<uint64_t> ("export.metadata", v_t); var_extension = &vp.insert<string> ("extension", v_t); var_clean = &vp.insert<bool> ("clean", v_t); diff --git a/libbuild2/context.hxx b/libbuild2/context.hxx index ea2c017..573b8d1 100644 --- a/libbuild2/context.hxx +++ b/libbuild2/context.hxx @@ -352,6 +352,11 @@ namespace build2 // const variable* var_import_build2; const variable* var_import_target; + const variable* var_import_metadata; + + // export.* + // + const variable* var_export_metadata; // [string] target visibility // diff --git a/libbuild2/cxx/init.cxx b/libbuild2/cxx/init.cxx index 1d8421c..265bbc0 100644 --- a/libbuild2/cxx/init.cxx +++ b/libbuild2/cxx/init.cxx @@ -382,7 +382,7 @@ namespace build2 // We only support root loading (which means there can only be one). // - if (&rs != &bs) + if (rs != bs) fail (loc) << "cxx.guess module must be loaded in project root"; // Load cc.core.vars so that we can cache all the cc.* variables. @@ -540,7 +540,7 @@ namespace build2 // We only support root loading (which means there can only be one). // - if (&rs != &bs) + if (rs != bs) fail (loc) << "cxx.config module must be loaded in project root"; // Load cxx.guess and share its module instance as ours. @@ -585,7 +585,7 @@ namespace build2 // We only support root loading (which means there can only be one). // - if (&rs != &bs) + if (rs != bs) fail (loc) << "cxx module must be loaded in project root"; // Load cxx.config. diff --git a/libbuild2/file.cxx b/libbuild2/file.cxx index 0bf3fd4..fb24160 100644 --- a/libbuild2/file.cxx +++ b/libbuild2/file.cxx @@ -4,19 +4,20 @@ #include <libbuild2/file.hxx> #include <iomanip> // left, setw() +#include <sstream> #include <libbuild2/scope.hxx> #include <libbuild2/target.hxx> #include <libbuild2/context.hxx> #include <libbuild2/filesystem.hxx> -#include <libbuild2/prerequisite.hxx> #include <libbuild2/diagnostics.hxx> +#include <libbuild2/prerequisite-key.hxx> #include <libbuild2/token.hxx> #include <libbuild2/lexer.hxx> #include <libbuild2/parser.hxx> -#include <libbuild2/config/utility.hxx> // save_variable() +#include <libbuild2/config/utility.hxx> // lookup_config() using namespace std; using namespace butl; @@ -25,6 +26,9 @@ namespace build2 { // Standard and alternative build file/directory naming schemes. // + + // build: + const dir_path std_build_dir ("build"); const dir_path std_root_dir (dir_path (std_build_dir) /= "root"); const dir_path std_bootstrap_dir (dir_path (std_build_dir) /= "bootstrap"); @@ -39,7 +43,7 @@ namespace build2 const path std_buildfile_file ("buildfile"); const path std_buildignore_file (".buildignore"); - // + // build2: const dir_path alt_build_dir ("build2"); const dir_path alt_root_dir (dir_path (alt_build_dir) /= "root"); @@ -171,7 +175,7 @@ namespace build2 try { l5 ([&]{trace << "sourcing " << fn;}); - p.parse_buildfile (l, root, base); + p.parse_buildfile (l, &root, base); } catch (const io_error& e) { @@ -1417,8 +1421,221 @@ namespace build2 return rs; } - pair<name, dir_path> - import_search (scope& ibase, name target, const location& loc, bool subp) + // Find or insert a target based on the file path. + // + static const target* + find_target (tracer& trace, context& ctx, + const target_type& tt, const path& p) + { + const target* t ( + ctx.targets.find (tt, + p.directory (), + dir_path (), + p.leaf ().base ().string (), + p.extension (), + trace)); + + if (t != nullptr) + { + if (const file* f = t->is_a<file> ()) + assert (f->path () == p); + } + + return t; + } + + static pair<target&, ulock> + insert_target (tracer& trace, context& ctx, + const target_type& tt, path p) + { + auto r ( + ctx.targets.insert_locked (tt, + p.directory (), + dir_path (), // No out (not in project). + p.leaf ().base ().string (), + p.extension (), // Always specified. + true /* implied */, + trace)); + + if (const file* f = r.first.is_a<file> ()) + f->path (move (p)); + + return r; + } + + // Extract metadata for an executable target by executing it with the + // --build2-metadata option. In case of an error, issue diagnostics and fail + // if opt is false and return nullopt if it's true. + // + // Note that loading of the metadata is split into two steps, extraction and + // parsing, because extraction also serves as validation that the executable + // is runnable, what we expected, etc. In other words, we sometimes do the + // extraction without parsing. In this light it would have been more + // efficient for extract to return the running process with a pipe rather + // than the extracted data. But this would complicate the code quite a bit + // plus we don't expect the data to be large, typically. + // + // Also note that we do not check the export.metadata here leaving it to + // the caller to do for both this case and export stub. + // + static optional<string> + extract_metadata (const process_path& pp, + const string& key, + bool opt, + const location& loc) + { + // Note: to ease handling (think patching third-party code) we will always + // specify the --build2-metadata option in this single-argument form. + // + const char* args[] {pp.recall_string (), "--build2-metadata=1", nullptr}; + + // @@ TODO This needs some more thinking/clarification. Specifically, what + // does it mean "x not found/not ours"? Is it just not found in PATH? + // That plus was not able to execute (e.g., some shared libraries + // missing)? That plus abnormal termination? That plus x that we found + // is something else? + // + // Specifically, at least on Linux, when a shared library is not found, + // it appears exec() issues the diagnostics and calls exit(127) (that + // is, exec() does not return). So this is a normal termination with a + // peculiar exit code. + // + // Overall, it feels like we should only silently ignore the "not + // found" and "not ours" cases since for all others the result is + // ambigous: it could be "ours" but just broken and the user expects + // us to use it but we silently ignored it. But then the same can be + // said about the "not ours" case: the user expected us to find "ours" + // but we didn't and silently ignored it. + // + try + { + // Note: not using run_*() functions since need to be able to suppress + // all errors, including inability to exec. + // + if (verb >= 3) + print_process (args); + + process pr (pp, + args, + -2 /* stdin to /dev/null */, + -1 /* stdout to pipe */, + opt ? -2 : 2 /* stderr to /dev/null or pass-through */); + + try + { + ifdstream is (move (pr.in_ofd), fdstream_mode::skip); + string r; + getline (is, r, '\0'); // Will fail if there is no data. + is.close (); // Detect errors. + + if (pr.wait ()) + { + // Check the signature line. It should be in the following form: + // + // # build2 buildfile <key> + // + // This makes sure we don't treat bogus output as metadata and also + // will allow us to support other formats (say, JSON) in the future. + // Note that we won't be able to add more options since trying them + // will be expensive. + // + if (r.compare (0, + 20 + key.size (), + "# build2 buildfile " + key + '\n') == 0) + return r; + + if (!opt) + error (loc) << "invalid metadata signature in " << args[0] + << " output"; + + goto fail; + } + + // Process error, fall through. + } + catch (const io_error& e) + { + // IO error (or process error), fall through. + } + + // Deal with process or IO error. + // + if (pr.wait ()) + { + if (!opt) + error (loc) << "unable to read metadata from " << args[0]; + } + else + { + // The child process presumably issued diagnostics but if it didn't, + // the result will be very confusing. So let's issue something + // generic for good measure. + // + if (!opt) + error (loc) << "unable to extract metadata from " << args[0]; + } + + goto fail; + } + catch (const process_error& e) + { + if (!opt) + error (loc) << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + goto fail; + } + + fail: + + if (opt) + return nullopt; + else + throw failed (); + } + + static void + parse_metadata (target& t, const string& md, const location& loc) + { + istringstream is (md); + path_name in ("<metadata>"); + + auto df = make_diag_frame ( + [&t, &loc] (const diag_record& dr) + { + dr << info (loc) << "while loading metadata for " << t; + }); + + parser p (t.ctx); + p.parse_buildfile (is, in, + nullptr /* root */, + t.base_scope ().rw (), // Load phase. + &t); + } + + // Return the processed target name as well as the project directory, if + // any. + // + // Absent project directory means nothing importable for this target was + // found (and the returned target name is the same as the original). Empty + // project directory means the target was found in an ad hoc manner, outside + // of any project (in which case it may still be qualified; see + // config.import.<proj>.<name>[.<type>]). + // + // Return empty name if an ad hoc import resulted in a NULL target (only + // allowed if optional is true). + // + pair<name, optional<dir_path>> + import_search (bool& new_value, + scope& ibase, + name tgt, + bool opt, + const optional<string>& meta, + bool subp, + const location& loc, + const char* what) { tracer trace ("import_search"); @@ -1426,24 +1643,24 @@ namespace build2 // short and sweet: we simply return it as empty-project-qualified and // let someone else (e.g., a rule) take a stab at it. // - if (target.unqualified ()) + if (tgt.unqualified ()) { - target.proj = project_name (); - return make_pair (move (target), dir_path ()); + tgt.proj = project_name (); + return make_pair (move (tgt), optional<dir_path> ()); } context& ctx (ibase.ctx); // Otherwise, get the project name and convert the target to unqualified. // - project_name proj (move (*target.proj)); - target.proj = nullopt; + project_name proj (move (*tgt.proj)); + tgt.proj = nullopt; scope& iroot (*ibase.root_scope ()); // Figure out the imported project's out_root. // - dir_path out_root; + optional<dir_path> out_root; // First try the config.import.* mechanism. The idea is that if the user // explicitly told us the project's location, then we should prefer that @@ -1452,95 +1669,208 @@ namespace build2 // auto& vp (iroot.var_pool ()); + using config::lookup_config; + for (;;) // Break-out loop. { - string n ("config.import." + proj.variable ()); + string projv (proj.variable ()); + string n ("config.import." + projv); - auto skip = [&target, &proj, &trace] () + // Skip import phase 1. + // + auto skip = [&tgt, &proj, &trace] () { - target.proj = move (proj); - l5 ([&]{trace << "skipping " << target;}); - return make_pair (move (target), dir_path ()); + tgt.proj = move (proj); + l5 ([&]{trace << "skipping " << tgt;}); + return make_pair (move (tgt), optional<dir_path> ()); }; - // config.import.<proj> + // Add hoc import. // - { - // Note: pattern-typed in context ctor as an overridable variable of - // type abs_dir_path (path auto-completion). - // - const variable& var (vp.insert (n)); - - if (auto l = iroot[var]) - { - out_root = cast<dir_path> (l); // Normalized and actualized. - - // Mark as part of config. - // - config::save_variable (iroot, var); - - // Empty config.import.* value means don't look in subprojects or - // amalgamations and go straight to the rule-specific import (e.g., - // to use system-installed). - // - if (out_root.empty ()) - return skip (); - - break; - } - } - // config.import.<proj>.<name>.<type> // config.import.<proj>.<name> // // For example: config.import.build2.b.exe=/opt/build2/bin/b // - if (!target.value.empty ()) + // If <type> is exe and <proj> and <name> are the same, then we also + // recognize the special config.<proj> (tool importation; we could + // also handle the case where <proj> is not the same as <name> via + // the config.<proj>.<name> variable). For backwards-compatibility + // reasons, it takes precedence over config.import. + // + // Note: see import phase 2 diagnostics if changing anything here. + // + // @@ How will this work for snake-case targets, say libs{build2-foo}? + // As well as for dot-separated target types, say, cli.cxx{}? + // + // @@ This duality has a nasty side-effect: if we have config.<proj> + // configured, then specifying config.<proj>.import has no effect + // (see also a note below on priority just among these options). + // + // Some ideas on how to resolve this include: using lookup depth, + // using override info, and using the "new value" status. All of + // these undoubtfully will complicate this logic (i.e., we will have + // to lookup all of them and then decide which one "wins"). + // + if (!tgt.value.empty ()) { - auto lookup = [&iroot, &vp, &loc] (string name) -> path + // Return NULL if not found and empty path if NULL. For executable + // targets (exe is true), also treat the special `false` value as + // NULL. + // + auto lookup = [&new_value, &iroot, opt, &loc, what] ( + const variable& var, bool exe) -> const path* { - // Note: pattern-typed in context ctor as an overridable variable of - // type path. - // - const variable& var (vp.insert (move (name))); + auto l (lookup_config (new_value, iroot, var)); - path r; - if (auto l = iroot[var]) + if (l.defined ()) { - r = cast<path> (l); + const path* p (cast_null<path> (l)); + + if (p != nullptr) + { + if (p->empty ()) + fail (loc) << "empty path in " << var; - if (r.empty ()) - fail (loc) << "empty path in " << var.name; + if (!exe || p->to_directory () || p->string () != "false") + return p; + } + + if (!opt) + fail (loc) << (p == nullptr ? "null" : "false") << " in " + << var << " for non-optional " << what; - config::save_variable (iroot, var); + return &empty_path; } - return r; + return nullptr; }; - // First try .<name>.<type>, then just .<name>. + // First try config.<proj>, then import.<name>.<type>, and finally + // just import.<name>. // - path p; - if (target.typed ()) - p = lookup (n + '.' + target.value + '.' + target.type); - - if (p.empty ()) - p = lookup (n + '.' + target.value); + // @@ What should we do if several of them are specified? For example, + // one is inherited from amalgamation while the other is specified + // on the project's root? We could pick the one with the least + // lookup depth. On the other hand, we expect people to stick with + // the config.<proj> notation for tools (since it's a lot easier to + // type) so let's not complicate things for the time being. + // + // Another alternative would be to see which one is new. + // + const path* p (nullptr); - if (!p.empty ()) + if (tgt.typed ()) { - // If the path is relative, then keep it project-qualified assuming - // import phase 2 knows what to do with it. Think: - // - // config.import.build2.b=b-boot + bool e (tgt.type == "exe"); + + // The config.import.* vars are pattern-typed in context ctor as an + // overridable variable of type path. The config.<proj> we have to + // type manually. // - if (p.relative ()) - target.proj = move (proj); + if (e && (projv == tgt.value || proj == tgt.value)) + p = lookup (vp.insert<path> ("config." + projv), e); + + if (p == nullptr) + p = lookup (vp.insert (n + '.' + tgt.value + '.' + tgt.type), e); + } + + if (p == nullptr) + p = lookup (vp.insert (n + '.' + tgt.value), false); + + if (p != nullptr) + { + if (p->empty ()) + tgt = name (); // NULL + else + { + tgt.dir = p->directory (); + tgt.value = p->leaf ().string (); + + // If the path is relative, then keep it project-qualified + // assuming import phase 2 knows what to do with it. Think: + // + // config.import.build2.b=b-boot + // + // @@ Maybe we should still complete it if it's not simple? After + // all, this is a path, do we want interpretations other than + // relative to CWD? Maybe we do, who knows. Doesn't seem to + // harm anything at the moment. + // + // Why not call import phase 2 directly here? Well, one good + // reason would be to allow for rule-specific import resolution. + // + if (p->relative ()) + tgt.proj = move (proj); + else + { + // Enter the target and assign its path (this will most commonly + // be some out of project file). + // + // @@ Should we check that the file actually exists (and cache + // the extracted timestamp)? Or just let things take their + // natural course? + // + name n (tgt); + auto r (ibase.find_target_type (n, loc)); - target.dir = p.directory (); - target.value = p.leaf ().string (); + if (r.first == nullptr) + fail (loc) << "unknown target type " << n.type << " in " << n; + + // Note: not using the extension extracted by find_target_type() + // to be consistent with import phase 2. + // + target& t (insert_target (trace, ctx, *r.first, *p).first); - return make_pair (move (target), dir_path ()); + // Load the metadata, similar to import phase 2. + // + if (meta) + { + if (exe* e = t.is_a<exe> ()) + { + if (!e->vars[ctx.var_export_metadata].defined ()) + { + parse_metadata (*e, + *extract_metadata (e->process_path (), + *meta, + false /* optional */, + loc), + loc); + } + } + } + } + } + + return make_pair (move (tgt), optional<dir_path> (dir_path ())); + } + } + + // Normal import. + // + // config.import.<proj> + // + // Note: see import phase 2 diagnostics if changing anything here. + // + { + // Note: pattern-typed in context ctor as an overridable variable of + // type abs_dir_path (path auto-completion). + // + auto l (lookup_config (new_value, iroot, vp.insert (n))); + + if (l.defined ()) + { + const dir_path* d (cast_null<dir_path> (l)); + + // Empty/NULL config.import.* value means don't look in subprojects + // or amalgamations and go straight to the rule-specific import + // (e.g., to use system-installed). + // + if (d == nullptr || d->empty ()) + return skip (); + + out_root = *d; // Normalized and actualized. + break; } } @@ -1560,7 +1890,7 @@ namespace build2 { out_root = cast<dir_path> (l); - if (out_root.empty ()) + if (out_root->empty ()) return skip (); break; @@ -1609,22 +1939,27 @@ namespace build2 // Add the qualification back to the target (import_load() will remove it // again). // - target.proj = move (proj); + tgt.proj = move (proj); - return make_pair (move (target), move (out_root)); + return make_pair (move (tgt), move (out_root)); } pair<names, const scope&> - import_load (context& ctx, pair<name, dir_path> x, const location& loc) + import_load (context& ctx, + pair<name, optional<dir_path>> x, + bool meta, + const location& loc) { tracer trace ("import_load"); - name target (move (x.first)); - dir_path out_root (move (x.second)); + assert (x.second); - assert (target.proj); - project_name proj (move (*target.proj)); - target.proj = nullopt; + name tgt (move (x.first)); + dir_path out_root (move (*x.second)); + + assert (tgt.proj); + project_name proj (move (*tgt.proj)); + tgt.proj = nullopt; // Bootstrap the imported root scope. This is pretty similar to what we do // in main() except that here we don't try to guess src_root. @@ -1746,15 +2081,27 @@ namespace build2 ts.assign (ctx.var_out_root) = move (out_root); ts.assign (ctx.var_src_root) = move (src_root); - // Also pass the target being imported in the import.target variable. + // Pass the target being imported in import.target. // { value& v (ts.assign (ctx.var_import_target)); - if (!target.empty ()) // Otherwise leave NULL. - v = target; // Can't move (need for diagnostics below). + if (!tgt.empty ()) // Otherwise leave NULL. + v = tgt; // Can't move (need for diagnostics below). } + // Pass the metadata compatibility version in import.metadata. + // + // This serves both as an indication that the metadata is required (can be + // useful, for example, in cases where it is expensive to calculate) as + // well as the maximum version we recognize. The exporter may return it in + // any version up to and including this maximum. And it may return it even + // if not requested (but only in version 1). The exporter should also set + // the returned version as the target-specific export.metadata variable. + // + if (meta) + ts.assign (ctx.var_import_metadata) = uint64_t (1); + // Load the export stub. Note that it is loaded in the context // of the importing project, not the imported one. The export // stub will normally switch to the imported root scope at some @@ -1778,8 +2125,8 @@ namespace build2 // If there were no export directive executed in an export stub, assume // the target is not exported. // - if (v.empty () && !target.empty ()) - fail (loc) << "target " << target << " is not exported by project " + if (v.empty () && !tgt.empty ()) + fail (loc) << "target " << tgt << " is not exported by project " << proj; return pair<names, const scope&> (move (v), *root); @@ -1790,32 +2137,104 @@ namespace build2 } } - names - import (scope& base, name target, const location& loc) + pair<names, import_kind> + import (scope& base, + name tgt, + bool ph2, + bool opt, + bool metadata, + const location& loc) { tracer trace ("import"); - l5 ([&]{trace << target << " from " << base;}); + l5 ([&]{trace << tgt << " from " << base;}); + + assert ((!opt || ph2) && (!metadata || ph2)); + + context& ctx (base.ctx); + assert (ctx.phase == run_phase::load); + + // If metadata is requested, delegate to import_direct() which will lookup + // the target and verify the metadata was loaded. + // + if (metadata) + { + pair<const target*, import_kind> r ( + import_direct (base, move (tgt), ph2, opt, metadata, loc)); + + return make_pair (r.first != nullptr ? r.first->as_name () : names {}, + r.second); + } + + // Save the original target name as metadata key. + // + auto meta (metadata ? optional<string> (tgt.value) : nullopt); - pair<name, dir_path> r (import_search (base, move (target), loc)); + pair<name, optional<dir_path>> r ( + import_search (base, move (tgt), opt, meta, true /* subpproj */, loc)); - // If we couldn't find the project, return to let someone else (e.g., a - // rule) take a stab at it. + // If there is no project, we are either done or go straight to phase 2. // - if (r.second.empty ()) + if (!r.second || r.second->empty ()) { - l5 ([&]{trace << "postponing " << r.first;}); - return names {move (r.first)}; + names ns; + + if (r.first.empty ()) + { + assert (opt); // NULL + } + else + { + ns.push_back (move (r.first)); + + // If the target is still qualified, it is either phase 2 now or we + // return it as is to let someone else (e.g., a rule, import phase 2) + // take a stab at it later. + // + if (ns.back ().qualified ()) + { + if (ph2) + { + // This is tricky: we only want the optional semantics for the + // fallback case. + // + if (const target* t = import (ctx, + base.find_prerequisite_key (ns, loc), + opt && !r.second /* optional */, + meta, + false /* existing */, + loc)) + ns = t->as_name (); + else + ns.clear (); // NULL + } + else + l5 ([&]{trace << "postponing " << r.first;}); + } + } + + return make_pair ( + move (ns), + r.second.has_value () ? import_kind::adhoc : import_kind::fallback); } - return import_load (base.ctx, move (r), loc).first; + return make_pair ( + import_load (base.ctx, move (r), metadata, loc).first, + import_kind::normal); } const target* - import (context& ctx, const prerequisite_key& pk, bool existing) + import (context& ctx, + const prerequisite_key& pk, + bool opt, + const optional<string>& meta, + bool exist, + const location& loc) { tracer trace ("import"); + assert (!meta || !exist); + assert (pk.proj); const project_name& proj (*pk.proj); @@ -1826,7 +2245,7 @@ namespace build2 // Try to find the executable in PATH (or CWD if relative). // - if (tt.is_a<exe> ()) + for (; tt.is_a<exe> (); ) // Breakout loop. { path n (*tk.dir); n /= *tk.name; @@ -1836,63 +2255,218 @@ namespace build2 n += *tk.ext; } - // Only search in PATH (or CWD). + // Only search in PATH (or CWD if not simple). // - process_path pp (process::try_path_search (n, true, dir_path (), true)); + process_path pp ( + process::try_path_search (n, + false /* init */, + dir_path () /* fallback */, + true /* path_only */)); + if (pp.empty ()) + break; - if (!pp.empty ()) - { - path& p (pp.effect); - assert (!p.empty ()); // We searched for a simple name. - - const exe* t ( - !existing - ? &ctx.targets.insert<exe> (tt, - p.directory (), - dir_path (), // No out (not in project). - p.leaf ().base ().string (), - p.extension (), // Always specified. - trace) - : ctx.targets.find<exe> (tt, - p.directory (), - dir_path (), - p.leaf ().base ().string (), - p.extension (), - trace)); - - if (t != nullptr) - { - if (!existing) - t->path (move (p)); - else - assert (t->path () == p); + const path& p (pp.effect); + assert (!p.empty ()); // We searched for a relative path. + if (exist) // Note: then meta is false. + { + if (const target* t = find_target (trace, ctx, tt, p)) return t; + + break; + } + + // Try hard to avoid re-extracting the metadata (think of a tool that is + // used by multiple projects in an amalgamation). + // + optional<string> md; + optional<const target*> t; + if (meta) + { + t = find_target (trace, ctx, tt, p); + + if (*t != nullptr && (*t)->vars[ctx.var_export_metadata].defined ()) + return *t; // We've got all we need. + + if (!(md = extract_metadata (pp, *meta, opt, loc))) + break; + } + + if (!t || *t == nullptr) + { + pair<target&, ulock> r (insert_target (trace, ctx, tt, p)); + t = &r.first; + + // Cache the process path if we've created the target (it's possible + // that the same target will be imported via different paths, e.g., as + // a simple name via PATH search and as an absolute path in which case + // the first import will determine the path). + // + if (r.second.owns_lock ()) + { + r.first.as<exe> ().process_path (move (pp)); + r.second.unlock (); } } + + // Save the metadata. Note that this happens during the load phase and + // so MT-safe. + // + if (meta) + parse_metadata ((*t)->rw (), *md, loc); + + return *t; } - if (existing) + if (opt || exist) return nullptr; - // @@ We no longer have location. This is especially bad for the - // empty case, i.e., where do I need to specify the project - // name)? Looks like the only way to do this is to keep location - // in name and then in prerequisite. Perhaps one day... - // diag_record dr; - dr << fail << "unable to import target " << pk; + dr << fail (loc) << "unable to import target " << pk; if (proj.empty ()) dr << info << "consider adding its installation location" << info << "or explicitly specify its project name"; else - dr << info << "use config.import." << proj.variable () - << " command line variable to specify its project out_root"; + { + string projv (proj.variable ()); + + // Suggest normal import. + // + dr << info << "use config.import." << projv << " configuration variable " + << "to specify its project out_root"; + + // Suggest ad hoc import. + // + string v (tt.is_a<exe> () && (projv == *tk.name || proj == *tk.name) + ? "config." + projv + : "config.import." + projv + '.' + *tk.name + '.' + tt.name); + + dr << info << "or use " << v << " configuration variable to specify its " + << "path"; + } dr << endf; } + pair<const target*, import_kind> + import_direct (bool& new_value, + scope& base, + name tgt, + bool ph2, + bool opt, + bool metadata, + const location& loc, + const char* what) + { + // This is like normal import() except we return the target rather than + // its name. + // + tracer trace ("import_direct"); + + l5 ([&]{trace << tgt << " from " << base << " for " << what;}); + + assert ((!opt || ph2) && (!metadata || ph2)); + + context& ctx (base.ctx); + assert (ctx.phase == run_phase::load); + + auto meta (metadata ? optional<string> (tgt.value) : nullopt); + + names ns; + import_kind k; + const target* t (nullptr); + + pair<name, optional<dir_path>> r ( + import_search (new_value, + base, + move (tgt), + opt, + meta, + true /* subpproj */, + loc, + what)); + + // If there is no project, we are either done or go straight to phase 2. + // + if (!r.second || r.second->empty ()) + { + k = r.second.has_value () ? import_kind::adhoc : import_kind::fallback; + + if (r.first.empty ()) + { + assert (opt); + return make_pair (t, k); // NULL + } + else if (r.first.qualified ()) + { + if (ph2) + { + names ns {move (r.first)}; + + // This is tricky: we only want the optional semantics for the + // fallback case. + // + t = import (ctx, + base.find_prerequisite_key (ns, loc), + opt && !r.second, + meta, + false /* existing */, + loc); + } + + if (t == nullptr) + return make_pair (t, k); // NULL + + // Otherwise fall through. + } + else + ns.push_back (move (r.first)); // And fall through. + } + else + { + k = import_kind::normal; + ns = import_load (base.ctx, move (r), metadata, loc).first; + } + + if (t == nullptr) + { + // Similar logic to perform's search(). + // + target_key tk (base.find_target_key (ns, loc)); + t = ctx.targets.find (tk, trace); + if (t == nullptr) + fail (loc) << "unknown imported target " << tk; + } + + if (meta) + { + if (auto* v = cast_null<uint64_t> (t->vars[ctx.var_export_metadata])) + { + if (*v != 1) + fail (loc) << "unexpected metadata version " << *v + << " in imported target " << *t; + } + else + fail (loc) << "no metadata for imported target " << *t; + } + + return make_pair (t, k); + } + + ostream& + operator<< (ostream& o, const pair<const exe*, import_kind>& p) + { + assert (p.first != nullptr); + + if (p.second == import_kind::normal) + o << *p.first; + else + o << p.first->process_path (); + + return o; + } + void create_project (const dir_path& d, const optional<dir_path>& amal, diff --git a/libbuild2/file.hxx b/libbuild2/file.hxx index b44efb6..231ea73 100644 --- a/libbuild2/file.hxx +++ b/libbuild2/file.hxx @@ -11,6 +11,7 @@ #include <libbuild2/utility.hxx> #include <libbuild2/scope.hxx> +#include <libbuild2/target.hxx> #include <libbuild2/variable.hxx> // list_value #include <libbuild2/export.hxx> @@ -241,37 +242,190 @@ namespace build2 // that (or failed to find anything usable), it calls the standard // prerequisite search() function which sees this is a project-qualified // prerequisite and goes straight to the second phase of import. Here, - // currently, we simply fail but in the future this will be the place where - // we can call custom "last resort" import hooks. For example, we can hook a - // package manager that will say, "Hey, dude, I see you are trying to import - // foo and I see there is a package foo available in repository bar. Wanna, - // like, download and use it or something?" + // currently, we only have special handling of exe{} targets (search in + // PATH) simply failing for the rest. But in the future this coud be the + // place where we could call custom "last resort" import hooks. For example, + // we can hook a package manager that will say, "Hey, dude, I see you are + // trying to import foo and I see there is a package foo available in + // repository bar. Wanna, like, download and use it or something?" Though + // the latest thoughts indicate this is probably a bad idea (implicitness, + // complexity, etc). + // + // More specifically, we have the following kinds of import (tried in this + // order): + // + // ad hoc + // + // The target is imported by specifying its path directly with + // config.import.<proj>.<name>[.<type>]. For example, this can be + // used to import an installed target. + // + // + // normal + // + // The target is imported from a project that was either specified with + // config.import.<proj> or was found via the subproject search. This also + // loads the target's dependency information. + // + // + // rule-specific + // + // The target was imported in a rule-specific manner (e.g., a library was + // found in the compiler's search paths). + // + // + // fallback/default + // + // The target was found by the second phase of import (e.g., an executable + // was found in PATH). + + // Import phase 1. Return the imported target(s) as well as the kind of + // import that was performed with `fallback` indicating it was not found. + // + // If second is `fallback`, then first contains the original, project- + // qualified target. If second is `adhoc`, first may still contain a + // project-qualified target (which may or may not be the same as the + // original; see the config.import.<proj>.<name>[.<type>] logic for details) + // in which case it should still be passed to import phase 2. + // + // If phase2 is true then the phase 2 is performed right away (we call it + // immediate import). Note that if optional is true, phase2 must be true as + // well (and thus there is no rule-specific logic for optional imports). In + // case of optional, empty names value is retuned if nothing was found. + // + // If metadata is true, then load the target metadata. In this case phase2 + // must be true as well. // // Note also that we return names rather than a single name: while normally // it will be a single target name, it can be an out-qualified pair (if // someone wants to return a source target) but it can also be a non-target // since we don't restrict what users can import/export. // - LIBBUILD2_SYMEXPORT names - import (scope& base, name, const location&); - - LIBBUILD2_SYMEXPORT pair<name, dir_path> - import_search (scope& base, name, const location&, bool subproj = true); + // Finally, note that import is (and should be kept) idempotent or, more + // precisely, "accumulatively idempotent" in that additional steps may be + // performed (phase 2, loading of the metadata) unless already done. + // + enum class import_kind {adhoc, normal, fallback}; - LIBBUILD2_SYMEXPORT pair<names, const scope&> - import_load (context&, pair<name, dir_path>, const location&); + LIBBUILD2_SYMEXPORT pair<names, import_kind> + import (scope& base, + name, + bool phase2, + bool optional, + bool metadata, + const location&); + // Import phase 2. + // const target& import (context&, const prerequisite_key&); - // As above but only imports as an already existing target. Unlike the above - // version, this one can be called during the execute phase. + // As above but import the target "here and now" without waiting for phase 2 + // (and thus omitting any rule-specific logic). This version of import is, + // for example, used by build system modules to perform an implicit import + // of the corresponding tool. + // + // If phase2 is false, then the second phase's fallback/default logic is + // only invoked if the import was ad hoc (i.e., a relative path was + // specified via config.import.<proj>.<name>[.<type>]) with NULL returned + // otherwise. + // + // If phase2 is true and optional is true, then NULL is returned instead of + // failing if phase 2 could not find anything. + // + // If metadata is true, then load the target metadata. In this case phase2 + // must be true as well. + // + // The what argument specifies what triggered the import (for example, + // "module load") and is used in diagnostics. + // + // This function also returns the kind of import that was performed. + // + pair<const target*, import_kind> + import_direct (scope& base, + name, + bool phase2, + bool optional, + bool metadata, + const location&, + const char* what = "import"); + + // As above but also return (in new_value) an indication of whether this + // import is based on a new config.* value. See config::lookup_config() for + // details. Note that a phase 2 fallback/default logic is not considered new + // (though this can be easily adjusted based on import kind). + // + LIBBUILD2_SYMEXPORT pair<const target*, import_kind> + import_direct (bool& new_value, + scope& base, + name, + bool phase2, + bool optional, + bool metadata, + const location&, + const char* what = "import"); + + + template <typename T> + pair<const T*, import_kind> + import_direct (scope&, + name, bool, bool, bool, + const location&, const char* = "import"); + + template <typename T> + pair<const T*, import_kind> + import_direct (bool&, + scope&, + name, + bool, bool, bool, + const location&, const char* = "import"); + + // Print import_direct<exe>() result either as a target for a normal import + // or as a process path for ad hoc and fallback imports. Normally used in + // build system modules to print the configuration report. + // + LIBBUILD2_SYMEXPORT ostream& + operator<< (ostream&, const pair<const exe*, import_kind>&); + + // As import phase 2 but only imports as an already existing target. But + // unlike it, this function can be called during the execute phase. // // Note: similar to search_existing(). // const target* import_existing (context&, const prerequisite_key&); + // Lower-level components of phase 1 (see implementation for details). + // + pair<name, optional<dir_path>> + import_search (scope& base, + name, + bool optional_, + const optional<string>& metadata, // False or metadata key. + bool subprojects, + const location&, + const char* what = "import"); + + // As above but also return (in new_value) an indication of whether this + // import is based on a new config.* value. See config::lookup_config() + // for details. + // + LIBBUILD2_SYMEXPORT pair<name, optional<dir_path>> + import_search (bool& new_value, + scope& base, + name, + bool optional_, + const optional<string>& metadata, + bool subprojects, + const location&, + const char* what = "import"); + + LIBBUILD2_SYMEXPORT pair<names, const scope&> + import_load (context&, + pair<name, optional<dir_path>>, + bool metadata, + const location&); + // Create a build system project in the specified directory. // LIBBUILD2_SYMEXPORT void diff --git a/libbuild2/file.ixx b/libbuild2/file.ixx index e940eb3..7c09d3d 100644 --- a/libbuild2/file.ixx +++ b/libbuild2/file.ixx @@ -11,20 +11,76 @@ namespace build2 return source_once (root, base, bf, base); } + inline pair<name, optional<dir_path>> + import_search (scope& base, + name tgt, + bool opt, const optional<string>& md, bool sp, + const location& loc, const char* w) + { + bool dummy (false); + return import_search (dummy, base, move (tgt), opt, md, sp, loc, w); + } + LIBBUILD2_SYMEXPORT const target* - import (context&, const prerequisite_key&, bool existing); + import (context&, + const prerequisite_key&, + bool optional_, + const optional<string>& metadata, // False or metadata key. + bool existing, + const location&); inline const target& import (context& ctx, const prerequisite_key& pk) { assert (ctx.phase == run_phase::match); - return *import (ctx, pk, false); + + // @@ We no longer have location. This is especially bad for the empty + // project case (i.e., where do I need to specify the project name)? + // Looks like the only way to do this is to keep location in name and + // then in prerequisite. Perhaps one day... + // + return *import (ctx, pk, false, nullopt, false, location ()); + } + + inline pair<const target*, import_kind> + import_direct (scope& base, + name tgt, + bool ph2, bool opt, bool md, + const location& loc, const char* w) + { + bool dummy (false); + return import_direct (dummy, base, move (tgt), ph2, opt, md, loc, w); + } + + template <typename T> + inline pair<const T*, import_kind> + import_direct (scope& base, + name tgt, + bool ph2, bool opt, bool md, + const location& loc, const char* w) + { + auto r (import_direct (base, move (tgt), ph2, opt, md, loc, w)); + return make_pair (r.first != nullptr ? &r.first->as<const T> () : nullptr, + r.second); + } + + template <typename T> + inline pair<const T*, import_kind> + import_direct (bool& nv, + scope& base, + name tgt, + bool ph2, bool opt, bool md, + const location& loc, const char* w) + { + auto r (import_direct (nv, base, move (tgt), ph2, opt, md, loc, w)); + return make_pair (r.first != nullptr ? &r.first->as<const T> () : nullptr, + r.second); } inline const target* import_existing (context& ctx, const prerequisite_key& pk) { assert (ctx.phase == run_phase::match || ctx.phase == run_phase::execute); - return import (ctx, pk, true); + return import (ctx, pk, false, nullopt, true, location ()); } } diff --git a/libbuild2/forward.hxx b/libbuild2/forward.hxx index b96aa31..1679b4a 100644 --- a/libbuild2/forward.hxx +++ b/libbuild2/forward.hxx @@ -55,10 +55,13 @@ namespace build2 class include_type; struct prerequisite_member; + // <libbuild2/prerequisite-key.hxx> + // + class prerequisite_key; + // <libbuild2/prerequisite.hxx> // class prerequisite; - class prerequisite_key; // <libbuild2/rule.hxx> // diff --git a/libbuild2/function.test.cxx b/libbuild2/function.test.cxx index e059619..514ac1e 100644 --- a/libbuild2/function.test.cxx +++ b/libbuild2/function.test.cxx @@ -124,7 +124,7 @@ namespace build2 scope& s (ctx.global_scope.rw ()); parser p (ctx); - p.parse_buildfile (cin, path_name ("buildfile"), s, s); + p.parse_buildfile (cin, path_name ("buildfile"), &s, s); } catch (const failed&) { diff --git a/libbuild2/module.cxx b/libbuild2/module.cxx index 28d2468..3abb102 100644 --- a/libbuild2/module.cxx +++ b/libbuild2/module.cxx @@ -147,18 +147,32 @@ namespace build2 // (remember, if it's top-level, then it must be in an isolated // configuration). // - pair<name, dir_path> ir ( + pair<name, optional<dir_path>> ir ( import_search (bs, name (proj, dir_path (), "libs", "build2-" + mod), - loc, - nested /* subprojects */)); + opt, + nullopt /* metadata */, + nested /* subprojects */, + loc)); - if (!ir.second.empty ()) + if (ir.first.empty ()) { + assert (opt); + return nullptr; + } + + if (ir.second) + { + // What if a module is specified with config.import.<mod>.<lib>.libs? + // Note that this could still be a project-qualified target. + // + if (ir.second->empty ()) + fail (loc) << "direct module target importation not yet supported"; + // We found the module as a target in a project. Now we need to update // the target (which will also give us the shared library path). // - l5 ([&]{trace << "found " << ir.first << " in " << ir.second;}); + l5 ([&]{trace << "found " << ir.first << " in " << *ir.second;}); // Create the build context if necessary. // @@ -215,7 +229,8 @@ namespace build2 // Load the imported project in the module context. // - pair<names, const scope&> lr (import_load (ctx, move (ir), loc)); + pair<names, const scope&> lr ( + import_load (ctx, move (ir), false /* metadata */, loc)); l5 ([&]{trace << "loaded " << lr.first;}); @@ -638,7 +653,7 @@ namespace build2 // Note that it would have been nice to keep these inline but we need the // definition of scope for the variable lookup. // - bool + const shared_ptr<module>* load_module (scope& rs, scope& bs, const string& name, @@ -646,8 +661,18 @@ namespace build2 bool opt, const variable_map& hints) { - return cast_false<bool> (bs[name + ".loaded"]) || - init_module (rs, bs, name, loc, opt, hints); + if (cast_false<bool> (bs[name + ".loaded"])) + { + if (cast_false<bool> (bs[name + ".configured"])) + return &rs.root_extra->modules.find (name)->second.module; + } + else + { + if (module_state* ms = init_module (rs, bs, name, loc, opt, hints)) + return &ms->module; + } + + return nullptr; } const shared_ptr<module>& @@ -657,6 +682,9 @@ namespace build2 const location& loc, const variable_map& hints) { + //@@ TODO: shouldn't we also check for configured? What if the previous + // attempt to load it was optional? + return cast_false<bool> (bs[name + ".loaded"]) ? rs.root_extra->modules.find (name)->second.module : init_module (rs, bs, name, loc, false /* optional */, hints)->module; diff --git a/libbuild2/module.hxx b/libbuild2/module.hxx index f5c726e..b2b42e4 100644 --- a/libbuild2/module.hxx +++ b/libbuild2/module.hxx @@ -170,9 +170,18 @@ namespace build2 // A wrapper over init_module() to use from other modules that incorporates // the <name>.loaded variable check (use init_module() directly to sidestep - // this check). + // this check). Return a pointer to the pointer to the module instance if it + // was both successfully loaded and configured and NULL otherwise (so can be + // used as bool). // - LIBBUILD2_SYMEXPORT bool + // Note also that NULL can be returned even of optional is false which + // happens if the requested module has already been loaded but failed to + // configure. The function could have issued diagnostics but the caller can + // normally provide additional information. + // + // Note: for non-optional modules use the versions below. + // + LIBBUILD2_SYMEXPORT const shared_ptr<module>* load_module (scope& root, scope& base, const string& name, diff --git a/libbuild2/name.hxx b/libbuild2/name.hxx index f623110..d0e8d85 100644 --- a/libbuild2/name.hxx +++ b/libbuild2/name.hxx @@ -50,6 +50,10 @@ namespace build2 name (dir_path d, string t, string v) : dir (move (d)), type (move (t)), value (move (v)) {} + name (string p, dir_path d, string t, string v) + : proj (project_name (move (p))), dir (move (d)), type (move (t)), + value (move (v)) {} + name (optional<project_name> p, dir_path d, string t, string v) : proj (move (p)), dir (move (d)), type (move (t)), value (move (v)) {} diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index cd6fe5e..a5c34d9 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -222,37 +222,53 @@ namespace build2 } void parser:: - parse_buildfile (istream& is, const path_name& in, scope& root, scope& base) + parse_buildfile (istream& is, + const path_name& in, + scope* root, + scope& base, + target* tgt, + prerequisite* prq) { lexer l (is, in); - parse_buildfile (l, root, base); + parse_buildfile (l, root, base, tgt, prq); } void parser:: - parse_buildfile (lexer& l, scope& root, scope& base) + parse_buildfile (lexer& l, + scope* root, + scope& base, + target* tgt, + prerequisite* prq) { path_ = &l.name (); lexer_ = &l; - root_ = &root; + root_ = root; scope_ = &base; - target_ = nullptr; - prerequisite_ = nullptr; + target_ = tgt; + prerequisite_ = prq; pbase_ = scope_->src_path_; - enter_buildfile (*path_); // Needs scope_. + if (path_->path != nullptr) + enter_buildfile (*path_->path); // Note: needs scope_. token t; type tt; next (t, tt); - parse_clause (t, tt); + if (target_ != nullptr || prerequisite_ != nullptr) + { + parse_variable_block (t, tt); + } + else + { + parse_clause (t, tt); + process_default_target (t); + } if (tt != type::eos) fail (t) << "unexpected " << t; - - process_default_target (t); } token parser:: @@ -385,7 +401,9 @@ namespace build2 { f = &parser::parse_run; } - else if (n == "import") + else if (n == "import" || + n == "import?" || + n == "import!") { f = &parser::parse_import; } @@ -892,7 +910,7 @@ namespace build2 // NULL, then this is a target type/pattern-specific block. // // enter: first token of first line in the block (normal lexer mode) - // leave: rcbrace + // leave: rcbrace or eos // // This is a more restricted variant of parse_clause() that only allows // variable assignments. @@ -1098,6 +1116,8 @@ namespace build2 // name n (tt != type::colon ? move (pn) : pn); + // See also scope::find_prerequisite_key(). + // auto rp (scope_->find_target_type (n, ploc)); const target_type* tt (rp.first); optional<string>& e (rp.second); @@ -1210,7 +1230,7 @@ namespace build2 { next (t, tt); // First token inside the block. - parse_variable_block (t, tt, nullptr, string ()); + parse_variable_block (t, tt); if (tt != type::rcbrace) fail (t) << "expected '}' instead of " << t; @@ -1318,7 +1338,7 @@ namespace build2 { next (t, tt); // First token inside the block. - parse_variable_block (t, tt, nullptr, string ()); + parse_variable_block (t, tt); if (tt != type::rcbrace) fail (t) << "expected '}' instead of " << t; @@ -1333,18 +1353,14 @@ namespace build2 } void parser:: - source (istream& is, - const path_name& in, - const location& loc, - bool enter, - bool deft) + source (istream& is, const path_name& in, const location& loc, bool deft) { tracer trace ("parser::source", &path_); l5 ([&]{trace (loc) << "entering " << in;}); - if (enter) - enter_buildfile (in); + if (in.path != nullptr) + enter_buildfile (*in.path); const path_name* op (path_); path_ = ∈ @@ -1417,7 +1433,6 @@ namespace build2 source (ifs, path_name (p), get_location (t), - true /* enter */, false /* default_target */); } catch (const io_error& e) @@ -1558,8 +1573,7 @@ namespace build2 source (ifs, path_name (p), get_location (t), - true /* enter */, - true /* default_target */); + true /* default_target */); } catch (const io_error& e) { @@ -1642,7 +1656,6 @@ namespace build2 source (is, path_name ("<stdout>"), l, - false /* enter */, false /* default_target */); } @@ -1689,13 +1702,10 @@ namespace build2 // the config[.**].<project>.** pattern where <project> is the innermost // named project. // - // Note that we currently don't allow just the config.<project> name even - // though this is used quite liberally in build system modules. Allowing - // this will complicate the logic (and documentation) a bit and there are - // no obvious use-cases. On the other hand, for tools that could be used - // during the build (say yacc), such a variable would most likely be used - // to specify its location (say config.yacc) . So let's "reserve" it for - // now. + // Note that we also allow just the config.<project> name which can be + // used by tools (such as source code generators) that use themselves in + // their own build. This is a bit of an advanced/experimental setup so + // we leave this undocumented for now. // // What should we do if there is no named project? We used to fail but // there are valid cases where this can happen, for example, a standalone @@ -1788,7 +1798,7 @@ namespace build2 name.compare (0, 7, "config.") != 0) { if (!as.empty ()) - fail (t) << "unexpected attributes for report-only variable"; + fail (as.loc) << "unexpected attributes for report-only variable"; attributes_pop (); @@ -1822,9 +1832,15 @@ namespace build2 if (!proj.empty ()) { - if (name.find ('.' + proj + '.') == string::npos) + size_t p (name.find ('.' + proj)); + + if (p == string::npos || + ((p += proj.size () + 1) != name.size () && // config.<proj> + name[p] != '.')) // config.<proj>. + { dr << fail (t) << "configuration variable '" << name << "' does not include project name"; + } } if (!dr.empty ()) @@ -1937,8 +1953,11 @@ namespace build2 // General import format: // - // import [<var>=](<project>|<project>/<target>])+ + // import[?!] [<attrs>] [<var>=](<project>|<project>%<target>])+ // + bool opt (t.value.back () == '?'); + bool ph2 (opt || t.value.back () == '!'); + type atype; // Assignment type. value* val (nullptr); const variable* var (nullptr); @@ -1950,13 +1969,38 @@ namespace build2 // switch to the value mode, get the first token, and then re-parse it // manually looking for =/=+/+=. // + // Note that if we ever wanted to support value attributes, that would be + // non-trivial. + // mode (lexer_mode::value, '@'); next_with_attributes (t, tt); - // Get variable attributes, if any (note that here we will go into a - // nested value mode with a different pair character). + // Get variable (or value) attributes, if any, and deal with the special + // metadata attribute. Since currently it can only appear in the import + // directive, we handle it in an ad hoc manner. // - auto at (attributes_push (t, tt)); + attributes_push (t, tt); + attributes& as (attributes_top ()); + + bool meta (false); + for (auto i (as.begin ()); i != as.end (); ) + { + if (i->name == "metadata") + { + if (!ph2) + fail (as.loc) << "loading metadata requires immediate import" << + info << "consider using the import! directive instead"; + + meta = true; + } + else + { + ++i; + continue; + } + + i = as.erase (i); + } const location vloc (get_location (t)); @@ -1964,8 +2008,9 @@ namespace build2 { // Split the token into the variable name and value at position (p) of // '=', taking into account leading/trailing '+'. The variable name is - // returned while the token is set to value. If the resulting token - // value is empty, get the next token. Also set assignment type (at). + // returned while the token is set to the value part. If the resulting + // token value is empty, get the next token. Also set assignment type + // (at). // auto split = [&atype, &t, &tt, this] (size_t p) -> string { @@ -2042,10 +2087,10 @@ namespace build2 } else { - if (at.first) - fail (at.second) << "attributes without variable"; - else - attributes_pop (); + if (!as.empty ()) + fail (as.loc) << "attributes without variable"; + + attributes_pop (); } // The rest should be a list of projects and/or targets. Parse them as @@ -2064,19 +2109,27 @@ namespace build2 // import() will check the name, if required. // - names r (import (*scope_, move (n), l)); + names r (import (*scope_, move (n), ph2, opt, meta, l).first); if (val != nullptr) { - if (atype == type::assign) + if (r.empty ()) // Optional not found. { - val->assign (move (r), var); - atype = type::append; // Append subsequent values. + if (atype == type::assign) + *val = nullptr; } - else if (atype == type::prepend) - val->prepend (move (r), var); else - val->append (move (r), var); + { + if (atype == type::assign) + val->assign (move (r), var); + else if (atype == type::prepend) + val->prepend (move (r), var); + else + val->append (move (r), var); + } + + if (atype == type::assign) + atype = type::append; // Append subsequent values. } } @@ -6170,11 +6223,10 @@ namespace build2 } void parser:: - enter_buildfile (const path_name& pn) + enter_buildfile (const path& p) { tracer trace ("parser::enter_buildfile", &path_); - const path& p (pn.path != nullptr ? *pn.path : path ()); dir_path d (p.directory ()); // Empty for a path name with the NULL path. // Figure out if we need out. @@ -6190,8 +6242,8 @@ namespace build2 ctx.targets.insert<buildfile> ( move (d), move (out), - pn.name ? *pn.name : p.leaf ().base ().string (), - p.extension (), // Always specified. + p.leaf ().base ().string (), + p.extension (), // Always specified. trace); } diff --git a/libbuild2/parser.hxx b/libbuild2/parser.hxx index 6552114..c55e14f 100644 --- a/libbuild2/parser.hxx +++ b/libbuild2/parser.hxx @@ -33,11 +33,17 @@ namespace build2 void parse_buildfile (istream&, const path_name&, - scope& root, - scope& base); + scope* root, + scope& base, + target* = nullptr, + prerequisite* = nullptr); void - parse_buildfile (lexer&, scope& root, scope& base); + parse_buildfile (lexer&, + scope* root, + scope& base, + target* = nullptr, + prerequisite* = nullptr); buildspec parse_buildspec (istream&, const path_name&); @@ -51,7 +57,7 @@ namespace build2 names parse_export_stub (istream& is, const path_name& name, scope& r, scope& b) { - parse_buildfile (is, name, r, b); + parse_buildfile (is, name, &r, b); return move (export_value); } @@ -99,7 +105,9 @@ namespace build2 parse_clause (token&, token_type&, bool one = false); void - parse_variable_block (token&, token_type&, const target_type*, string); + parse_variable_block (token&, token_type&, + const target_type* = nullptr, + string = string ()); // Ad hoc target names inside < ... >. // @@ -270,14 +278,14 @@ namespace build2 attributes& attributes_top () {return attributes_.back ();} - // Source a stream optionnaly entering it as a buildfile and performing - // the default target processing. + // Source a stream optionnaly performing the default target processing. + // If the specified path name has a real path, then also enter it as a + // buildfile. // void source (istream&, const path_name&, const location&, - bool enter, bool default_target); // The what argument is used in diagnostics (e.g., "expected <what> @@ -494,7 +502,7 @@ namespace build2 // Enter buildfile as a target. // void - enter_buildfile (const path_name&); + enter_buildfile (const path&); // Lexer. // diff --git a/libbuild2/prerequisite-key.hxx b/libbuild2/prerequisite-key.hxx new file mode 100644 index 0000000..12b94a8 --- /dev/null +++ b/libbuild2/prerequisite-key.hxx @@ -0,0 +1,44 @@ +// file : libbuild2/prerequisite-key.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_PREREQUISITE_KEY_HXX +#define LIBBUILD2_PREREQUISITE_KEY_HXX + +#include <libbuild2/types.hxx> +#include <libbuild2/forward.hxx> +#include <libbuild2/utility.hxx> + +#include <libbuild2/target-key.hxx> +#include <libbuild2/target-type.hxx> + +#include <libbuild2/export.hxx> + +namespace build2 +{ + // Light-weight (by being shallow-pointing) prerequisite key, similar + // to (and based on) target key. + // + // Note that unlike prerequisite, the key is not (necessarily) owned by a + // target. So for the key we instead have the base scope of the target that + // (would) own it. Note that we assume keys to be ephemeral enough for the + // base scope to remain unchanged. + // + class prerequisite_key + { + public: + using scope_type = build2::scope; + + const optional<project_name>& proj; + target_key tk; // The .dir and .out members can be relative. + const scope_type* scope; // Can be NULL if tk.dir is absolute. + + template <typename T> + bool is_a () const {return tk.is_a<T> ();} + bool is_a (const target_type& tt) const {return tk.is_a (tt);} + }; + + LIBBUILD2_SYMEXPORT ostream& + operator<< (ostream&, const prerequisite_key&); +} + +#endif // LIBBUILD2_PREREQUISITE_KEY_HXX diff --git a/libbuild2/prerequisite.hxx b/libbuild2/prerequisite.hxx index 9448fbc..476ed9d 100644 --- a/libbuild2/prerequisite.hxx +++ b/libbuild2/prerequisite.hxx @@ -13,36 +13,12 @@ #include <libbuild2/variable.hxx> #include <libbuild2/target-key.hxx> #include <libbuild2/diagnostics.hxx> +#include <libbuild2/prerequisite-key.hxx> #include <libbuild2/export.hxx> namespace build2 { - // Light-weight (by being shallow-pointing) prerequisite key, similar - // to (and based on) target key. - // - // Note that unlike prerequisite, the key is not (necessarily) owned by a - // target. So for the key we instead have the base scope of the target that - // (would) own it. Note that we assume keys to be ephemeral enough for the - // base scope to remain unchanged. - // - class prerequisite_key - { - public: - using scope_type = build2::scope; - - const optional<project_name>& proj; - target_key tk; // The .dir and .out members can be relative. - const scope_type* scope; // Can be NULL if tk.dir is absolute. - - template <typename T> - bool is_a () const {return tk.is_a<T> ();} - bool is_a (const target_type& tt) const {return tk.is_a (tt);} - }; - - LIBBUILD2_SYMEXPORT ostream& - operator<< (ostream&, const prerequisite_key&); - // Note that every data member except for the target is immutable (const). // class LIBBUILD2_SYMEXPORT prerequisite diff --git a/libbuild2/scope.cxx b/libbuild2/scope.cxx index 842b839..92fc12c 100644 --- a/libbuild2/scope.cxx +++ b/libbuild2/scope.cxx @@ -738,7 +738,7 @@ namespace build2 auto r (find_target_type (n, loc)); if (r.first == nullptr) - fail (loc) << "unknown target type " << n.type; + fail (loc) << "unknown target type " << n.type << " in " << n; bool src (n.pair); // If out-qualified, then it is from src. if (src) @@ -784,20 +784,54 @@ namespace build2 if (n == (ns[0].pair ? 2 : 1)) { name dummy; - target_key r (find_target_key (ns[0], n == 1 ? dummy : ns[1], loc)); - - return target_key { - r.type, - r.dir, - n == 1 ? &empty_dir_path : r.out, - r.name, - move (r.ext)}; + return find_target_key (ns[0], n == 1 ? dummy : ns[1], loc); } } fail (loc) << "invalid target name: " << ns << endf; } + pair<const target_type&, optional<string>> scope:: + find_prerequisite_type (name& n, name& o, const location& loc) const + { + auto r (find_target_type (n, loc)); + + if (r.first == nullptr) + fail (loc) << "unknown target type " << n.type << " in " << n; + + if (n.pair) // If out-qualified, then it is from src. + { + assert (n.pair == '@'); + + if (!o.directory ()) + fail (loc) << "expected directory after '@'"; + } + + if (!n.dir.empty ()) + n.dir.normalize (false, true); // Current dir collapses to an empty one. + + if (!o.dir.empty ()) + o.dir.normalize (false, true); // Ditto. + + return pair<const target_type&, optional<string>> ( + *r.first, move (r.second)); + } + + prerequisite_key scope:: + find_prerequisite_key (names& ns, const location& loc) const + { + if (size_t n = ns.size ()) + { + if (n == (ns[0].pair ? 2 : 1)) + { + name dummy; + return find_prerequisite_key (ns[0], n == 1 ? dummy : ns[1], loc); + } + } + + fail (loc) << "invalid prerequisite name: " << ns << endf; + } + static target* derived_tt_factory (context& c, const target_type& t, dir_path d, dir_path o, string n) diff --git a/libbuild2/scope.hxx b/libbuild2/scope.hxx index e1cdf78..e7c2db7 100644 --- a/libbuild2/scope.hxx +++ b/libbuild2/scope.hxx @@ -19,6 +19,7 @@ #include <libbuild2/target-key.hxx> #include <libbuild2/target-type.hxx> #include <libbuild2/target-state.hxx> +#include <libbuild2/prerequisite-key.hxx> #include <libbuild2/export.hxx> @@ -330,6 +331,20 @@ namespace build2 target_key find_target_key (names&, const location&) const; + // Similar to the find_target_type() but does not complete relative + // directories. + // + pair<const target_type&, optional<string>> + find_prerequisite_type (name&, name&, const location&) const; + + // As above, but return a prerequisite key. + // + prerequisite_key + find_prerequisite_key (name&, name&, const location&) const; + + prerequisite_key + find_prerequisite_key (names&, const location&) const; + // Dynamically derive a new target type from an existing one. Return the // reference to the target type and an indicator of whether it was // actually created. @@ -497,6 +512,12 @@ namespace build2 // NULL means no strong amalgamtion. }; + inline bool + operator== (const scope& x, const scope& y) { return &x == &y; } + + inline bool + operator!= (const scope& x, const scope& y) { return !(x == y); } + inline ostream& operator<< (ostream& os, const scope& s) { diff --git a/libbuild2/scope.ixx b/libbuild2/scope.ixx index f13aa70..9aecd48 100644 --- a/libbuild2/scope.ixx +++ b/libbuild2/scope.ixx @@ -55,7 +55,28 @@ namespace build2 find_target_key (name& n, name& o, const location& loc) const { auto p (find_target_type (n, o, loc)); - return target_key {&p.first, &n.dir, &o.dir, &n.value, move (p.second)}; + return target_key { + &p.first, + &n.dir, + o.dir.empty () ? &empty_dir_path : &o.dir, + &n.value, + move (p.second)}; + } + + inline prerequisite_key scope:: + find_prerequisite_key (name& n, name& o, const location& loc) const + { + auto p (find_prerequisite_type (n, o, loc)); + return prerequisite_key { + n.proj, + { + &p.first, + &n.dir, + o.dir.empty () ? &empty_dir_path : &o.dir, + &n.value, + move (p.second) + }, + this}; } inline dir_path diff --git a/libbuild2/search.cxx b/libbuild2/search.cxx index 0ff49ac..5887138 100644 --- a/libbuild2/search.cxx +++ b/libbuild2/search.cxx @@ -6,8 +6,8 @@ #include <libbuild2/scope.hxx> #include <libbuild2/target.hxx> #include <libbuild2/filesystem.hxx> // mtime() -#include <libbuild2/prerequisite.hxx> #include <libbuild2/diagnostics.hxx> +#include <libbuild2/prerequisite-key.hxx> using namespace std; using namespace butl; diff --git a/libbuild2/target-key.hxx b/libbuild2/target-key.hxx index bd9b8c7..0096d46 100644 --- a/libbuild2/target-key.hxx +++ b/libbuild2/target-key.hxx @@ -19,7 +19,7 @@ namespace build2 { // Light-weight (by being shallow-pointing) target key. // - class target_key + class LIBBUILD2_SYMEXPORT target_key { public: const target_type* const type; @@ -31,6 +31,11 @@ namespace build2 template <typename T> bool is_a () const {return type->is_a<T> ();} bool is_a (const target_type& tt) const {return type->is_a (tt);} + + // Return the target name or a pair of names if out-qualified. + // + names + as_name () const; }; inline bool diff --git a/libbuild2/target.cxx b/libbuild2/target.cxx index b49071a..d313b05 100644 --- a/libbuild2/target.cxx +++ b/libbuild2/target.cxx @@ -42,6 +42,27 @@ namespace build2 return false; } + // target_key + // + names target_key:: + as_name () const + { + names r; + + string v (*name); + target::combine_name (v, ext, false /* @@ TODO: what to do? */); + + r.push_back (build2::name (*dir, type->name, move (v))); + + if (!out->empty ()) + { + r.front ().pair = '@'; + r.push_back (build2::name (*out)); + } + + return r; + } + // target_state // static const char* const target_state_[] = diff --git a/libbuild2/target.hxx b/libbuild2/target.hxx index ea72925..6edfe5c 100644 --- a/libbuild2/target.hxx +++ b/libbuild2/target.hxx @@ -138,7 +138,7 @@ namespace build2 class LIBBUILD2_SYMEXPORT target { public: - // Context this scope belongs to. + // Context this target belongs to. // context& ctx; @@ -305,6 +305,12 @@ namespace build2 target_key key () const; + names + as_name () const + { + return key ().as_name (); + } + // Scoping. // public: @@ -764,6 +770,15 @@ namespace build2 virtual const target_type& dynamic_type () const = 0; static const target_type static_type; + // RW access. + // + target& + rw () const + { + assert (ctx.phase == run_phase::load); + return const_cast<target&> (*this); + } + public: // Split the name leaf into target name (in place) and extension // (returned). @@ -1529,8 +1544,15 @@ namespace build2 // // An empty path may signify special unknown/undetermined/unreal location // (for example, a binless library or an installed import library -- we - // know the DLL is there, just not exactly where). In this case you would - // also normally set its mtime. + // know the DLL is there, just not exactly where). In this case you could + // also set its mtime to timestamp_unreal (but don't have to, if a real + // timestamp can be derived, for example, the from the import library in + // the DLL case above). + // + // Note, however, that a target with timestamp_unreal does not have to + // have an empty path. One consequence of this arrangement (assigned path + // with unreal timestamp) is that the timestamp of such a target when used + // as a prerequisite won't affect the dependent's target out-of-date-ness. // // We used to return a pointer to properly distinguish between not set and // empty but that proved too tedious to work with. So now we return empty @@ -1705,9 +1727,44 @@ namespace build2 public: using file::file; + using process_path_type = build2::process_path; + + // Return the process path of this executable target. Normally it will be + // the absolute path returned by path() but can also be something custom + // if, for example, the target was found via a PATH search (see import for + // details). The idea is to use this path if we need to execute the target + // in which case, for the above example, we will see a short (recall) path + // instead of the absolute one in diagnostics. + // + process_path_type + process_path () const + { + // It's unfortunate we have to return by value but hopefully the + // compiler will see through it. Note also that returning empty + // process path if path is empty. + // + return process_path_.empty () + ? process_path_type (path ().string ().c_str (), + path_type (), + path_type ()) + : process_path_type (process_path_, false /* init */); + } + + // Note that setting the custom process path is not MT-safe and must be + // done while holding the insertion lock. + // + void + process_path (process_path_type p) + { + process_path_ = move (p); + } + public: static const target_type static_type; virtual const target_type& dynamic_type () const {return static_type;} + + private: + process_path_type process_path_; }; class LIBBUILD2_SYMEXPORT buildfile: public file diff --git a/libbuild2/target.txx b/libbuild2/target.txx index 0e4d9bf..b482d64 100644 --- a/libbuild2/target.txx +++ b/libbuild2/target.txx @@ -5,7 +5,6 @@ #include <libbuild2/scope.hxx> #include <libbuild2/diagnostics.hxx> -#include <libbuild2/prerequisite.hxx> namespace build2 { diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx index 3a5e708..9800d6c 100644 --- a/libbuild2/utility.hxx +++ b/libbuild2/utility.hxx @@ -579,7 +579,7 @@ namespace build2 void append_options (sha256&, T&, const char*); - // As above but from the strings value directly. + // As above but from the lookup directly. // LIBBUILD2_SYMEXPORT void append_options (cstrings&, const lookup&, const char* excl = nullptr); @@ -590,6 +590,8 @@ namespace build2 LIBBUILD2_SYMEXPORT void append_options (sha256&, const lookup&); + // As above but from the strings value directly. + // void append_options (cstrings&, const strings&, const char* excl = nullptr); |