diff options
Diffstat (limited to 'libbuild2/dist')
-rw-r--r-- | libbuild2/dist/init.cxx | 81 | ||||
-rw-r--r-- | libbuild2/dist/module.hxx | 13 | ||||
-rw-r--r-- | libbuild2/dist/operation.cxx | 533 | ||||
-rw-r--r-- | libbuild2/dist/rule.cxx | 88 | ||||
-rw-r--r-- | libbuild2/dist/rule.hxx | 20 | ||||
-rw-r--r-- | libbuild2/dist/types.hxx | 41 |
6 files changed, 576 insertions, 200 deletions
diff --git a/libbuild2/dist/init.cxx b/libbuild2/dist/init.cxx index 5e371bf..48a3e15 100644 --- a/libbuild2/dist/init.cxx +++ b/libbuild2/dist/init.cxx @@ -22,6 +22,7 @@ namespace build2 namespace dist { static const rule rule_; + static const file_rule file_rule_ (true /* check_type */); void boot (scope& rs, const location&, module_boot_extra& extra) @@ -33,7 +34,34 @@ namespace build2 // Enter module variables. Do it during boot in case they get assigned // in bootstrap.build (which is customary for, e.g., dist.package). // - auto& vp (rs.var_pool ()); + + // The dist flag or path. Normally it is a flag (true or false) but can + // also be used to remap the distribution location. + // + // In the latter case it specifies the "imaginary" source location which + // is used to derive the corresponding distribution local. This location + // can be specified as either a directory path (to remap with the same + // file name) or a file path (to remap with a different name). And the + // way we distinguish between the two is via the presence/absence of the + // trailing directory separator. If the path is relative, then it's + // treated relative to the target directory. Note that to make things + // less error prone, simple paths without any directory separators are + // not allowed (use ./<name> instead). + // + // Note that if multiple targets end up with the same source location, + // the behavior is undefined and no diagnostics is issued. + // + // Note also that such remapping has no effect in the bootstrap + // distribution mode. + // + // Note: project-private. + // + rs.var_pool ().insert<path> ("dist", variable_visibility::target); + + // The rest of the variables we enter are qualified so go straight for + // the public variable pool. + // + auto& vp (rs.var_pool (true /* public */)); // config.dist.archives is a list of archive extensions (e.g., zip, // tar.gz) that can be optionally prefixed with a directory. If it is @@ -72,27 +100,6 @@ namespace build2 vp.insert<paths> ("dist.archives"); vp.insert<paths> ("dist.checksums"); - // The dist flag or path. Normally it is a flag (true or false) but can - // also be used to remap the distribution location. - // - // In the latter case it specifies the "imaginary" source location which - // is used to derive the corresponding distribution local. This location - // can be specified as either a directory path (to remap with the same - // file name) or a file path (to remap with a different name). And the - // way we distinguish between the two is via the presence/absence of the - // trailing directory separator. If the path is relative, then it's - // treated relative to the target directory. Note that to make things - // less error prone, simple paths without any directory separators are - // not allowed (use ./<name> instead). - // - // Note that if multiple targets end up with the same source location, - // the behavior is undefined and no diagnostics is issued. - // - // Note also that such remapping has no effect in the bootstrap - // distribution mode. - // - vp.insert<path> ("dist", variable_visibility::target); - // Project's package name. Note: if set, must be in bootstrap.build. // auto& v_d_p (vp.insert<string> ("dist.package")); @@ -127,7 +134,7 @@ namespace build2 // bool s (specified_config (rs, "dist", {"bootstrap"})); - // dist.root + // config.dist.root // { value& v (rs.assign ("dist.root")); @@ -139,22 +146,24 @@ namespace build2 } } - // dist.cmd + // config.dist.cmd + // + // By default we use in-process code for creating directories and + // copying files (for performance, especially on Windows). But an + // external program (normally install) can be used if configured. // { - value& v (rs.assign<process_path> ("dist.cmd")); + value& v (rs.assign<process_path> ("dist.cmd")); // NULL if (s) { - if (lookup l = lookup_config (rs, - "config.dist.cmd", - path ("install"))) + if (lookup l = lookup_config (rs, "config.dist.cmd", nullptr)) v = run_search (cast<path> (l), true); } } - // dist.archives - // dist.checksums + // config.dist.archives + // config.dist.checksums // { value& a (rs.assign ("dist.archives")); @@ -177,7 +186,7 @@ namespace build2 } } - // dist.uncommitted + // config.dist.uncommitted // // Omit it from the configuration unless specified. // @@ -202,7 +211,7 @@ namespace build2 l5 ([&]{trace << "for " << rs;}); - auto& vp (rs.var_pool ()); + auto& vp (rs.var_pool (true /* public */)); // All qualified. // Register our wildcard rule. Do it explicitly for the alias to prevent // something like insert<target>(dist_id, test_id) taking precedence. @@ -214,10 +223,14 @@ namespace build2 // executables imported from /usr/bin, etc). We are registering it on // the global scope similar to builtin rules. // + // Note: use target instead of anything more specific (such as + // mtime_target) in order not to take precedence over the "dist" rule + // above. + // // See a similar rule in the config module. // - rs.global_scope ().insert_rule<mtime_target> ( - dist_id, 0, "dist.file", file_rule::instance); + rs.global_scope ().insert_rule<target> ( + dist_id, 0, "dist.file", file_rule_); // Configuration. // diff --git a/libbuild2/dist/module.hxx b/libbuild2/dist/module.hxx index 9c682d0..da97939 100644 --- a/libbuild2/dist/module.hxx +++ b/libbuild2/dist/module.hxx @@ -10,14 +10,17 @@ #include <libbuild2/module.hxx> #include <libbuild2/variable.hxx> +#include <libbuild2/dist/types.hxx> + #include <libbuild2/export.hxx> namespace build2 { namespace dist { - struct LIBBUILD2_SYMEXPORT module: build2::module + class LIBBUILD2_SYMEXPORT module: public build2::module { + public: static const string name; const variable& var_dist_package; @@ -38,6 +41,10 @@ namespace build2 adhoc.push_back (move (f)); } + // List of postponed prerequisites (see rule for details). + // + mutable postponed_prerequisites postponed; + // Distribution post-processing callbacks. // // Only the last component in the pattern may contain wildcards. If the @@ -69,8 +76,8 @@ namespace build2 // Implementation details. // - module (const variable& v_d_p) - : var_dist_package (v_d_p) {} + public: + module (const variable& v_d_p): var_dist_package (v_d_p) {} public: bool distributed = false; // True if this project is being distributed. diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx index 468f7bd..cfc90cf 100644 --- a/libbuild2/dist/operation.cxx +++ b/libbuild2/dist/operation.cxx @@ -6,6 +6,8 @@ #include <libbutl/sha1.hxx> #include <libbutl/sha256.hxx> +#include <libbutl/filesystem.hxx> // try_mkdir_p(), cpfile() + #include <libbuild2/file.hxx> #include <libbuild2/dump.hxx> #include <libbuild2/scope.hxx> @@ -15,6 +17,8 @@ #include <libbuild2/filesystem.hxx> #include <libbuild2/diagnostics.hxx> +#include <libbuild2/dist/types.hxx> +#include <libbuild2/dist/rule.hxx> #include <libbuild2/dist/module.hxx> using namespace std; @@ -27,14 +31,14 @@ namespace build2 // install -d <dir> // static void - install (const process_path&, const dir_path&); + install (const process_path*, context&, const dir_path&); // install <file> <dir>[/<name>] // // Return the destination file path. // static path - install (const process_path&, const file&, const dir_path&, const path&); + install (const process_path*, const file&, const dir_path&, const path&); // tar|zip ... <dir>/<pkg>.<ext> <pkg> // @@ -85,7 +89,7 @@ namespace build2 if (auto* m = rs.find_module<module> (module::name)) m->distributed = true; - load (vs, rs, bf, out_base, src_base, l); + perform_load (vs, rs, bf, out_base, src_base, l); } // Enter the specified source file as a target of type T. The path is @@ -106,9 +110,7 @@ namespace build2 // Figure out if we need out. // - dir_path out (rs.src_path () != rs.out_path () - ? out_src (d, rs) - : dir_path ()); + dir_path out (!rs.out_eq_src () ? out_src (d, rs) : dir_path ()); const T& t (rs.ctx.targets.insert<T> ( move (d), @@ -150,11 +152,11 @@ namespace build2 try { - for (const dir_entry& e: dir_iterator (d, false /* ignore_dangling */)) + for (const dir_entry& e: dir_iterator (d, dir_iterator::no_follow)) { const path& n (e.path ()); - if (n.string ()[0] != '.') + if (!n.empty () && n.string ().front () != '.') try { if (e.type () == entry_type::directory) // Can throw. @@ -236,8 +238,13 @@ namespace build2 fail << "unknown distribution package name" << info << "did you forget to set dist.package?"; + const module& mod (*rs.find_module<module> (module::name)); + const string& dist_package (cast<string> (l)); - const process_path& dist_cmd (cast<process_path> (rs.vars["dist.cmd"])); + const process_path* dist_cmd ( + cast_null<process_path> (rs.vars["dist.cmd"])); + + dir_path td (dist_root / dir_path (dist_package)); // We used to print 'dist <target>' at verbosity level 1 but that has // proven to be just noise. Though we still want to print something @@ -248,7 +255,7 @@ namespace build2 // (e.g., output directory creation) in all the operations below. // if (verb == 1) - text << "dist " << dist_package; + print_diag ("dist", src_root, td); // Get the list of files to distribute. // @@ -259,7 +266,7 @@ namespace build2 { l5 ([&]{trace << "load dist " << rs;}); - dist_var = ctx.var_pool.find ("dist"); + dist_var = rs.var_pool ().find ("dist"); // Match a rule for every operation supported by this project. Skip // default_id. @@ -267,63 +274,124 @@ namespace build2 // Note that we are not calling operation_pre/post() callbacks here // since the meta operation is dist and we know what we are doing. // - values params; path_name pn ("<dist>"); const location loc (pn); // Dummy location. + action_targets ts {tgt}; + + auto process_postponed = [&ctx, &mod] () { - action_targets ts {tgt}; + if (!mod.postponed.list.empty ()) + { + // Re-grab the phase lock similar to perform_match(). + // + phase_lock l (ctx, run_phase::match); - auto mog = make_guard ([&ctx] () {ctx.match_only = false;}); - ctx.match_only = true; + // Note that we don't need to bother with the mutex since we do + // all of this serially. But we can end up with new elements at + // the end. + // + // Strictly speaking, to handle this correctly we would need to do + // multiple passes over this list and only give up when we cannot + // make any progress since earlier entries that we cannot resolve + // could be "fixed" by later entries. But this feels far-fetched + // and so let's wait for a real example before complicating this. + // + for (auto i (mod.postponed.list.begin ()); + i != mod.postponed.list.end (); + ++i) + rule::match_postponed (*i); + } + }; + + auto mog = make_guard ([&ctx] () {ctx.match_only = nullopt;}); + ctx.match_only = match_only_level::all; - const operations& ops (rs.root_extra->operations); - for (operations::size_type id (default_id + 1); // Skip default_id. - id < ops.size (); - ++id) + const operations& ops (rs.root_extra->operations); + for (operations::size_type id (default_id + 1); // Skip default_id. + id < ops.size (); + ++id) + { + if (const operation_info* oif = ops[id]) { - if (const operation_info* oif = ops[id]) - { - // Skip aliases (e.g., update-for-install). In fact, one can - // argue the default update should be sufficient since it is - // assumed to update all prerequisites and we no longer support - // ad hoc stuff like test.input. Though here we are using the - // dist meta-operation, not perform. - // - if (oif->id != id) - continue; + // Skip aliases (e.g., update-for-install). In fact, one can argue + // the default update should be sufficient since it is assumed to + // update all prerequisites and we no longer support ad hoc stuff + // like test.input. Though here we are using the dist + // meta-operation, not perform. + // + if (oif->id != id) + continue; - // Use standard (perform) match. - // - if (auto pp = oif->pre_operation) + // Use standard (perform) match. + // + if (auto pp = oif->pre_operation) + { + if (operation_id pid = pp (ctx, {}, dist_id, loc)) { - if (operation_id pid = pp (ctx, params, dist_id, loc)) - { - const operation_info* poif (ops[pid]); - ctx.current_operation (*poif, oif, false /* diag_noise */); - action a (dist_id, poif->id, oif->id); - match (params, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - } + const operation_info* poif (ops[pid]); + ctx.current_operation (*poif, oif, false /* diag_noise */); + + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, false /* inner */, loc); + + if (poif->operation_pre != nullptr) + poif->operation_pre (ctx, {}, true /* inner */, loc); + + action a (dist_id, poif->id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (); + + if (poif->operation_post != nullptr) + poif->operation_post (ctx, {}, true /* inner */); + + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, false /* inner */); } + } + + ctx.current_operation (*oif, nullptr, false /* diag_noise */); + + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, true /* inner */, loc); - ctx.current_operation (*oif, nullptr, false /* diag_noise */); - action a (dist_id, oif->id); - match (params, a, ts, - 1 /* diag (failures only) */, - false /* progress */); + action a (dist_id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (); - if (auto po = oif->post_operation) + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, true /* inner */); + + if (auto po = oif->post_operation) + { + if (operation_id pid = po (ctx, {}, dist_id)) { - if (operation_id pid = po (ctx, params, dist_id)) - { - const operation_info* poif (ops[pid]); - ctx.current_operation (*poif, oif, false /* diag_noise */); - action a (dist_id, poif->id, oif->id); - match (params, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - } + const operation_info* poif (ops[pid]); + ctx.current_operation (*poif, oif, false /* diag_noise */); + + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, false /* inner */, loc); + + if (poif->operation_pre != nullptr) + poif->operation_pre (ctx, {}, true /* inner */, loc); + + action a (dist_id, poif->id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (); + + if (poif->operation_post != nullptr) + poif->operation_post (ctx, {}, true /* inner */); + + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, false /* inner */); } } } @@ -332,7 +400,7 @@ namespace build2 // Add ad hoc files and buildfiles that are not normally loaded as // part of the project, for example, the export stub. They will still // be ignored on the next step if the user explicitly marked them - // dist=false. + // with dist=false. // auto add_adhoc = [] (const scope& rs) { @@ -379,7 +447,7 @@ namespace build2 dir_path out_nroot (out_root / pd); const scope& nrs (ctx.scopes.find_out (out_nroot)); - if (nrs.out_path () != out_nroot) // This subproject not loaded. + if (nrs.out_path () != out_nroot) // This subproject is not loaded. continue; if (!nrs.src_path ().sub (src_root)) // Not a strong amalgamation. @@ -395,48 +463,96 @@ namespace build2 // Note that we are not showing progress here (e.g., "N targets to // distribute") since it will be useless (too fast). // - for (const auto& pt: ctx.targets) + auto see_through = [] (const target& t) { - file* ft (pt->is_a<file> ()); - - if (ft == nullptr) // Not a file. - continue; + return ((t.type ().flags & target_type::flag::see_through) == + target_type::flag::see_through); + }; - if (ft->dir.sub (src_root)) + auto collect = [&trace, &dist_var, + &src_root, &out_root] (const file& ft) + { + if (ft.dir.sub (src_root)) { // Include unless explicitly excluded. // - if (const path* v = cast_null<path> ((*ft)[dist_var])) + if (const path* v = cast_null<path> (ft[dist_var])) { if (v->string () == "false") { - l5 ([&]{trace << "excluding " << *ft;}); - continue; + l5 ([&]{trace << "excluding " << ft;}); + return false; } } - files.push_back (ft); + return true; } - else if (ft->dir.sub (out_root)) + else if (ft.dir.sub (out_root)) { // Exclude unless explicitly included. // - if (const path* v = cast_null<path> ((*ft)[dist_var])) + if (const path* v = cast_null<path> (ft[dist_var])) { if (v->string () != "false") { - l5 ([&]{trace << "including " << *ft;}); - files.push_back (ft); + l5 ([&]{trace << "including " << ft;}); + return true; } } + + return false; } + else + return false; // Out of project. + }; + + for (const auto& pt: ctx.targets) + { + // Collect see-through groups if they are marked with dist=true. + // + // Note that while it's possible that only their certain members are + // marked as such (e.g., via a pattern), we will still require + // dist=true on the group itself (and potentially dist=false on some + // of its members) for such cases because we don't want to update + // every see-through group only to discover that most of them don't + // have anything to distribute. + // + if (see_through (*pt)) + { + if (const path* v = cast_null<path> ((*pt)[dist_var])) + { + if (v->string () != "false") + { + l5 ([&]{trace << "including group " << *pt;}); + files.push_back (pt.get ()); + } + } + + continue; + } + + file* ft (pt->is_a<file> ()); + + if (ft == nullptr) // Not a file. + continue; + + // Skip member of see-through groups since after dist_* their list + // can be incomplete (or even bogus, e.g., the "representative + // sample"). Instead, we will collect them during perfrom_update + // below. + // + if (ft->group != nullptr && see_through (*ft->group)) + continue; + + if (collect (*ft)) + files.push_back (ft); } // Make sure what we need to distribute is up to date. // { if (mo_perform.meta_operation_pre != nullptr) - mo_perform.meta_operation_pre (ctx, params, loc); + mo_perform.meta_operation_pre (ctx, {}, loc); // This is a hack since according to the rules we need to completely // reset the state. We could have done that (i.e., saved target @@ -452,25 +568,75 @@ namespace build2 ctx.current_on = on + 1; if (mo_perform.operation_pre != nullptr) - mo_perform.operation_pre (ctx, params, update_id); + mo_perform.operation_pre (ctx, {}, update_id); ctx.current_operation (op_update, nullptr, false /* diag_noise */); + if (op_update.operation_pre != nullptr) + op_update.operation_pre (ctx, {}, true /* inner */, loc); + action a (perform_update_id); - mo_perform.match (params, a, files, + mo_perform.match ({}, a, files, 1 /* diag (failures only) */, prog /* progress */); - mo_perform.execute (params, a, files, + mo_perform.execute ({}, a, files, 1 /* diag (failures only) */, prog /* progress */); + // Replace see-through groups (which now should have their members + // resolved) with members. + // + for (auto i (files.begin ()); i != files.end (); ) + { + const target& t (i->as<target> ()); + if (see_through (t)) + { + group_view gv (t.group_members (a)); // Go directly. + + if (gv.members == nullptr) + fail << "unable to resolve see-through group " << t + << " members"; + + i = files.erase (i); // Drop the group itself. + + for (size_t j (0); j != gv.count; ++j) + { + if (const target* m = gv.members[j]) + { + if (const file* ft = m->is_a<file> ()) + { + // Note that a rule may only link-up its members to groups + // if/when matched (for example, the cli.cxx{} group). It + // feels harmless for us to do the linking here. + // + if (ft->group == nullptr) + const_cast<file*> (ft)->group = &t; + else + assert (ft->group == &t); // Sanity check. + + if (collect (*ft)) + { + i = files.insert (i, ft); // Insert instead of the group. + i++; // Stay after the group. + } + } + } + } + } + else + ++i; + } + + if (op_update.operation_post != nullptr) + op_update.operation_post (ctx, {}, true /* inner */); + if (mo_perform.operation_post != nullptr) - mo_perform.operation_post (ctx, params, update_id); + mo_perform.operation_post (ctx, {}, update_id); if (mo_perform.meta_operation_post != nullptr) - mo_perform.meta_operation_post (ctx, params); + mo_perform.meta_operation_post (ctx, {}); } } else @@ -496,26 +662,22 @@ namespace build2 // auto_project_env penv (rs); - dir_path td (dist_root / dir_path (dist_package)); - // Clean up the target directory. // if (rmdir_r (ctx, td, true, 2) == rmdir_status::not_empty) fail << "unable to clean target directory " << td; auto_rmdir rm_td (td); // Clean it up if things go bad. - install (dist_cmd, td); + install (dist_cmd, ctx, td); // Copy over all the files. Apply post-processing callbacks. // - const module& mod (*rs.find_module<module> (module::name)); - prog = prog && show_progress (1 /* max_verb */); size_t prog_percent (0); for (size_t i (0), n (files.size ()); i != n; ++i) { - const file& t (*files[i].as<target> ().is_a<file> ()); + const file& t (files[i].as<target> ().as<file> ()); // Only files. // Figure out where this file is inside the target directory. // @@ -571,7 +733,7 @@ namespace build2 dir_path d (td / dl); if (!exists (d)) - install (dist_cmd, d); + install (dist_cmd, ctx, d); path r (install (dist_cmd, t, d, rn)); @@ -727,66 +889,131 @@ namespace build2 // install -d <dir> // static void - install (const process_path& cmd, const dir_path& d) + install (const process_path* cmd, context& ctx, const dir_path& d) { - path reld (relative (d)); + path reld; + cstrings args; - cstrings args {cmd.recall_string (), "-d"}; + if (cmd != nullptr || verb >= 2) + { + reld = relative (d); - args.push_back ("-m"); - args.push_back ("755"); - args.push_back (reld.string ().c_str ()); - args.push_back (nullptr); + args.push_back (cmd != nullptr ? cmd->recall_string () : "install"); + args.push_back ("-d"); + args.push_back ("-m"); + args.push_back ("755"); + args.push_back (reld.string ().c_str ()); + args.push_back (nullptr); - if (verb >= 2) - print_process (args); + if (verb >= 2) + print_process (args); + } - run (cmd, args); + if (cmd != nullptr) + run (ctx, *cmd, args, 1 /* finish_verbosity */); + else + { + try + { + // Note that mode has no effect on Windows, which is probably for + // the best. + // + try_mkdir_p (d, 0755); + } + catch (const system_error& e) + { + fail << "unable to create directory " << d << ": " << e; + } + } } // install <file> <dir>[/<name>] // static path - install (const process_path& cmd, + install (const process_path* cmd, const file& t, const dir_path& d, const path& n) { - path reld (relative (d)); - path relf (relative (t.path ())); + const path& f (t.path ()); + path r (d / (n.empty () ? f.leaf () : n)); - if (!n.empty ()) - reld /= n.string (); + // Assume the file is executable if the owner has execute permission, + // in which case we make it executable for everyone. + // + bool exe ((path_perms (f) & permissions::xu) == permissions::xu); - cstrings args {cmd.recall_string ()}; + path relf, reld; + cstrings args; - // Preserve timestamps. This could becomes important if, for - // example, we have pre-generated sources. Note that the - // install-sh script doesn't support this option, while both - // Linux and BSD install's do. - // - args.push_back ("-p"); + if (cmd != nullptr || verb >= 2) + { + relf = relative (f); + reld = relative (d); - // Assume the file is executable if the owner has execute - // permission, in which case we make it executable for - // everyone. - // - args.push_back ("-m"); - args.push_back ( - (path_perms (t.path ()) & permissions::xu) == permissions::xu - ? "755" - : "644"); + if (!n.empty ()) // Leave as just directory if no custom name. + reld /= n; - args.push_back (relf.string ().c_str ()); - args.push_back (reld.string ().c_str ()); - args.push_back (nullptr); + args.push_back (cmd != nullptr ? cmd->recall_string () : "install"); - if (verb >= 2) - print_process (args); + // Preserve timestamps. This could becomes important if, for example, + // we have pre-generated sources. Note that the install-sh script + // doesn't support this option, while both Linux and BSD install's do. + // + args.push_back ("-p"); - run (cmd, args); + // Assume the file is executable if the owner has execute permission, + // in which case we make it executable for everyone. + // + args.push_back ("-m"); + args.push_back (exe ? "755" : "644"); + args.push_back (relf.string ().c_str ()); + args.push_back (reld.string ().c_str ()); + args.push_back (nullptr); - return d / (n.empty () ? relf.leaf () : n); + if (verb >= 2) + print_process (args); + } + + if (cmd != nullptr) + run (t.ctx, *cmd, args, 1 /* finish_verbosity */); + else + { + permissions perm (permissions::ru | permissions::wu | + permissions::rg | + permissions::ro); // 644 + if (exe) + perm |= permissions::xu | permissions::xg | permissions::xo; // 755 + + try + { + // Note that we don't pass cpflags::overwrite_content which means + // this will fail if the file already exists. Since we clean up the + // destination directory, this will detect cases where we have + // multiple source files with the same distribution destination. + // + cpfile (f, + r, + cpflags::overwrite_permissions | cpflags::copy_timestamps, + perm); + } + catch (const system_error& e) + { + if (e.code ().category () == generic_category () && + e.code ().value () == EEXIST) + { + // @@ TMP (added in 0.16.0). + // + warn << "multiple files are distributed as " << r << + info << "second file is " << f << + info << "this warning will become error in the future"; + } + else + fail << "unable to copy " << f << " to " << r << ": " << e; + } + } + + return r; } static path @@ -796,13 +1023,15 @@ namespace build2 const dir_path& dir, const string& e) { + // NOTE: similar code in bpkg (system-package-manager-archive.cxx). + path an (pkg + '.' + e); // Delete old archive for good measure. // path ap (dir / an); if (exists (ap, false)) - rmfile (ctx, ap); + rmfile (ctx, ap, 3 /* verbosity */); // Use zip for .zip archives. Also recognize and handle a few well-known // tar.xx cases (in case tar doesn't support -a or has other issues like @@ -818,7 +1047,7 @@ namespace build2 if (e == "zip") { - // On Windows we use libarchive's bsdtar (zip is an MSYS executabales). + // On Windows we use libarchive's bsdtar (zip is an MSYS executable). // // While not explicitly stated, the compression-level option works // for zip archives. @@ -932,19 +1161,20 @@ namespace build2 if (verb >= 2) print_process (args); else if (verb) - text << args[0] << ' ' << ap; + print_diag (args[0], dir / dir_path (pkg), ap); process apr; process cpr; - // Change the archiver's working directory to dist_root. + // Change the archiver's working directory to root. + // + // Note: this function is called during serial execution and so no + // diagnostics buffering is needed (here and below). // - apr = run_start (app, + apr = run_start (process_env (app, root), args, 0 /* stdin */, - (i != 0 ? -1 : 1) /* stdout */, - true /* error */, - root); + (i != 0 ? -1 : 1) /* stdout */); // Start the compressor if required. // @@ -956,10 +1186,17 @@ namespace build2 out_fd.get () /* stdout */); cpr.in_ofd.reset (); // Close the archiver's stdout on our side. - run_finish (args.data () + i, cpr); } - run_finish (args.data (), apr); + // Delay throwing until we diagnose both ends of the pipe. + // + if (!run_finish_code (args.data (), + apr, + 1 /* verbosity */, + false /* omit_normal */) || + !(i == 0 || run_finish_code (args.data () + i, cpr, 1, false))) + throw failed (); + out_rm.cancel (); return ap; @@ -978,7 +1215,7 @@ namespace build2 // path cp (dir / cn); if (exists (cp, false)) - rmfile (ctx, cp); + rmfile (ctx, cp, 3 /* verbosity */); auto_rmfile c_rm; // Note: must come first. auto_fd c_fd; @@ -1017,18 +1254,20 @@ namespace build2 if (verb >= 2) print_process (args); else if (verb) - text << args[0] << ' ' << cp; + print_diag (args[0], ap, cp); // Note that to only get the archive name (without the directory) in // the output we have to run from the archive's directory. // - process pr (run_start (pp, + // Note: this function is called during serial execution and so no + // diagnostics buffering is needed. + // + process pr (run_start (process_env (pp, ad /* cwd */), args, - 0 /* stdin */, - c_fd.get () /* stdout */, - true /* error */, - ad /* cwd */)); - run_finish (args, pr); + 0 /* stdin */, + c_fd.get () /* stdout */)); + + run_finish (args, pr, 1 /* verbosity */); } else { @@ -1048,7 +1287,7 @@ namespace build2 if (verb >= 2) text << "cat >" << cp; else if (verb) - text << e << "sum " << cp; + print_diag ((e + "sum").c_str (), ap, cp); string c; try @@ -1092,6 +1331,8 @@ namespace build2 // given the prescribed semantics of adhoc (match/execute but otherwise // ignore) is followed. // + // Note that we don't need to do anything for posthoc. + // if (i == include_type::excluded) { l5 ([&]{trace << "overriding exclusion of " << p;}); @@ -1116,11 +1357,11 @@ namespace build2 nullptr, // meta-operation pre &dist_operation_pre, &dist_load_load, - &search, // normal search - nullptr, // no match (see dist_execute()). + &perform_search, // normal search + nullptr, // no match (see dist_execute()). &dist_load_execute, - nullptr, // operation post - nullptr, // meta-operation post + nullptr, // operation post + nullptr, // meta-operation post &dist_include }; diff --git a/libbuild2/dist/rule.cxx b/libbuild2/dist/rule.cxx index ac3d440..c63f7f3 100644 --- a/libbuild2/dist/rule.cxx +++ b/libbuild2/dist/rule.cxx @@ -8,6 +8,9 @@ #include <libbuild2/algorithm.hxx> #include <libbuild2/diagnostics.hxx> +#include <libbuild2/dist/types.hxx> +#include <libbuild2/dist/module.hxx> + using namespace std; namespace build2 @@ -27,17 +30,34 @@ namespace build2 const dir_path& src_root (rs.src_path ()); const dir_path& out_root (rs.out_path ()); - // If we can, go inside see-through groups. + // Note that we don't go inside see-through groups since the members for + // dist_* may be incomplete (or even bogus, e.g., the "representative + // sample"). Instead, for see-through groups our plan is as follows: + // + // 1. Here we match them as groups (so that we still match all their + // prerequisites). + // + // 2. In dist_project() we collect them along with files after dist_* + // but before perform_update. Here we also skip files that are + // members of see-through groups (which we may still get). // - for (prerequisite_member pm: - group_prerequisite_members (a, t, members_mode::maybe)) + // 3. During perform_update we collect all the see-through group + // members, similar to files on step (2). + // + for (const prerequisite& p: group_prerequisites (t)) { // Note: no exclusion tests, we want all of them (and see also the - // dist_include() override). + // dist_include() override). But if we don't ignore post hoc ones + // here, we will end up with a cycle (they will still be handled + // by the post-pass). + // + lookup l; // Ignore any operation-specific values. + if (include (a, t, p, &l) == include_type::posthoc) + continue; // Skip prerequisites imported from other projects. // - if (pm.proj ()) + if (p.proj) continue; // We used to always search and match but that resulted in the @@ -56,18 +76,18 @@ namespace build2 // @@ Note that this is still an issue in a custom dist rule. // const target* pt (nullptr); - if (pm.is_a<file> ()) + if (p.is_a<file> ()) { - pt = pm.load (); + pt = p.target.load (); if (pt == nullptr) { - const prerequisite& p (pm.prerequisite); - // Search for an existing target or existing file in src. // + // Note: see also similar code in match_postponed() below. + // const prerequisite_key& k (p.key ()); - pt = k.tk.type->search (t, k); + pt = k.tk.type->search (t.ctx, &t, k); if (pt == nullptr) { @@ -79,15 +99,23 @@ namespace build2 !p.dir.sub (out_root)) continue; - fail << "prerequisite " << k << " is not existing source file " - << "nor known output target" << endf; + // This can be order-dependent: for example libs{} prerequisite + // may be unknown because we haven't matched the lib{} group + // yet. So we postpone this for later (see match_postponed()). + // + const module& mod (*rs.find_module<module> (module::name)); + + mlock l (mod.postponed.mutex); + mod.postponed.list.push_back ( + postponed_prerequisite {a, t, p, t.state[a].rule->first}); + continue; } search_custom (p, *pt); // Cache. } } else - pt = &pm.search (t); + pt = &search (t, p); // Don't match targets that are outside of our project. // @@ -97,5 +125,39 @@ namespace build2 return noop_recipe; // We will never be executed. } + + void rule:: + match_postponed (const postponed_prerequisite& pp) + { + action a (pp.action); + const target& t (pp.target); + const prerequisite& p (pp.prereq); + + const prerequisite_key& k (p.key ()); + const target* pt (k.tk.type->search (t.ctx, &t, k)); + + if (pt == nullptr) + { + // Note that we do loose the diag frame that we normally get when + // failing during match. So let's mention the target/rule manually. + // + fail << "prerequisite " << k << " is not existing source file nor " + << "known output target" << + info << "while applying rule " << pp.rule << " to " << diag_do (a, t); + } + + search_custom (p, *pt); // Cache. + + // It's theoretically possible that the target gets entered but nobody + // else depends on it but us. So we need to make sure it's matched + // (since it, in turns, can pull in other targets). Note that this could + // potentially add new postponed prerequisites to the list. + // + if (!pt->matched (a)) + { + if (pt->dir.sub (t.root_scope ().out_path ())) + match_direct_sync (a, *pt); + } + } } } diff --git a/libbuild2/dist/rule.hxx b/libbuild2/dist/rule.hxx index a864015..69ab3d9 100644 --- a/libbuild2/dist/rule.hxx +++ b/libbuild2/dist/rule.hxx @@ -11,6 +11,10 @@ #include <libbuild2/action.hxx> #include <libbuild2/target.hxx> +#include <libbuild2/dist/types.hxx> + +#include <libbuild2/export.hxx> + namespace build2 { namespace dist @@ -19,20 +23,28 @@ namespace build2 // // A custom rule (usually the same as perform_update) may be necessary to // establish group links (so that we see the dist variable set on a group) - // or to see through non-see-through groups (like lib{}; see the - // bin::lib_rule for an example). Note that in the latter case the rule - // should "see" all its members for the dist case. + // or to see through non-see-through groups (like lib{}, obj{}; see rule + // in the bin module for an example). Note that in the latter case the + // rule should "see" all its members for the dist case. // - class rule: public simple_rule + class LIBBUILD2_SYMEXPORT rule: public simple_rule { public: rule () {} + // Always matches (returns true). + // virtual bool match (action, target&) const override; + // Matches all the prerequisites (including from group) and returns + // noop_recipe (which will never be executed). + // virtual recipe apply (action, target&) const override; + + static void + match_postponed (const postponed_prerequisite&); }; } } diff --git a/libbuild2/dist/types.hxx b/libbuild2/dist/types.hxx new file mode 100644 index 0000000..b833951 --- /dev/null +++ b/libbuild2/dist/types.hxx @@ -0,0 +1,41 @@ +// file : libbuild2/dist/types.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_DIST_TYPES_HXX +#define LIBBUILD2_DIST_TYPES_HXX + +#include <libbuild2/types.hxx> +#include <libbuild2/forward.hxx> + +#include <libbuild2/prerequisite-key.hxx> + +namespace build2 +{ + namespace dist + { + // List of prerequisites that could not be searched to a target and were + // postponed for later re-search. This can happen, for example, because a + // prerequisite would resolve to a member of a group that hasn't been + // matched yet (for example, libs{} of lib{}). See rule::apply() for + // details. + // + // Note that we are using list instead of vector because new elements can + // be added at the end while we are iterating over the list. + // + struct postponed_prerequisite + { + build2::action action; + reference_wrapper<const build2::target> target; + reference_wrapper<const prerequisite> prereq; + string rule; + }; + + struct postponed_prerequisites + { + build2::mutex mutex; + build2::list<postponed_prerequisite> list; + }; + } +} + +#endif // LIBBUILD2_DIST_TYPES_HXX |