diff options
Diffstat (limited to 'libbuild2/dist/operation.cxx')
-rw-r--r-- | libbuild2/dist/operation.cxx | 752 |
1 files changed, 562 insertions, 190 deletions
diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx index edb8162..6dc830b 100644 --- a/libbuild2/dist/operation.cxx +++ b/libbuild2/dist/operation.cxx @@ -3,8 +3,10 @@ #include <libbuild2/dist/operation.hxx> -#include <libbutl/sha1.mxx> -#include <libbutl/sha256.mxx> +#include <libbutl/sha1.hxx> +#include <libbutl/sha256.hxx> + +#include <libbutl/filesystem.hxx> // try_mkdir_p(), cpfile() #include <libbuild2/file.hxx> #include <libbuild2/dump.hxx> @@ -15,6 +17,8 @@ #include <libbuild2/filesystem.hxx> #include <libbuild2/diagnostics.hxx> +#include <libbuild2/dist/types.hxx> +#include <libbuild2/dist/rule.hxx> #include <libbuild2/dist/module.hxx> using namespace std; @@ -27,14 +31,14 @@ namespace build2 // install -d <dir> // static void - install (const process_path& cmd, const dir_path&); + install (const process_path*, context&, const dir_path&); - // install <file> <dir> + // install <file> <dir>[/<name>] // // Return the destination file path. // static path - install (const process_path& cmd, const file&, const dir_path&); + install (const process_path*, const file&, const dir_path&, const path&); // tar|zip ... <dir>/<pkg>.<ext> <pkg> // @@ -56,7 +60,7 @@ namespace build2 const path& arc, const dir_path& dir, const string& ext); static operation_id - dist_operation_pre (const values&, operation_id o) + dist_operation_pre (context&, const values&, operation_id o) { if (o != default_id) fail << "explicit operation specified for dist meta-operation"; @@ -64,6 +68,30 @@ namespace build2 return o; } + static void + dist_load_load (const values& vs, + scope& rs, + const path& bf, + const dir_path& out_base, + const dir_path& src_base, + const location& l) + { + // @@ TMP: redo after release (do it here and not in execute, also add + // custom search and do the other half there). + // +#if 0 + if (rs.out_path () != out_base || rs.src_path () != src_base) + fail (l) << "dist meta-operation target must be project root directory"; +#endif + + // Mark this project as being distributed. + // + if (auto* m = rs.find_module<module> (module::name)) + m->distributed = true; + + perform_load (vs, rs, bf, out_base, src_base, l); + } + // Enter the specified source file as a target of type T. The path is // expected to be normalized and relative to src_root. If the third // argument is false, then first check if the file exists. If the fourth @@ -82,11 +110,9 @@ namespace build2 // Figure out if we need out. // - dir_path out (rs.src_path () != rs.out_path () - ? out_src (d, rs) - : dir_path ()); + dir_path out (!rs.out_eq_src () ? out_src (d, rs) : dir_path ()); - const T& t (rs.ctx.targets.insert<T> ( + const T& t (rs.ctx.targets.insert_implied<T> ( move (d), move (out), p.leaf ().base ().string (), @@ -105,24 +131,69 @@ namespace build2 // Recursively traverse an src_root subdirectory entering/collecting the // contained files and file symlinks as the file targets and skipping // entries that start with a dot. Follow directory symlinks (preserving - // their names) and fail on dangling symlinks. + // their names) and fail on dangling symlinks. Also detect directory + // symlink cycles. // + struct subdir + { + const subdir* prev; + const dir_path& dir; + }; + static void - add_subdir (const scope& rs, const dir_path& sd, action_targets& files) + add_subdir (const scope& rs, + const dir_path& sd, + action_targets& files, + const subdir* prev = nullptr) { dir_path d (rs.src_path () / sd); + const subdir next {prev, d}; + try { - for (const dir_entry& e: dir_iterator (d, false /* ignore_dangling */)) + for (const dir_entry& e: dir_iterator (d, dir_iterator::no_follow)) { const path& n (e.path ()); + const string& s (n.string ()); - if (n.string ()[0] != '.') + if (s.compare (0, 4, ".git") != 0 && + s != ".bdep" && + s != ".bpkg" && + s != ".build2") try { if (e.type () == entry_type::directory) // Can throw. - add_subdir (rs, sd / path_cast<dir_path> (n), files); + { + // If this is a symlink, check that it doesn't cause a cycle. + // + if (e.ltype () == entry_type::symlink) + { + // Note that the resulting path will be absolute and + // normalized. + // + dir_path ld (d / path_cast<dir_path> (n)); + dir_path td (path_cast<dir_path> (followsymlink (ld))); + + const subdir* s (&next); + for (; s != nullptr; s = s->prev) + { + if (s->dir == td) + { + if (verb) + warn << "directory cycle caused by symlink " << ld << + info << "symlink target " << td; + + break; + } + } + + if (s != nullptr) + break; + } + + add_subdir (rs, sd / path_cast<dir_path> (n), files, &next); + } else files.push_back (add_target<file> (rs, sd / n, true, true)); } @@ -171,8 +242,13 @@ namespace build2 fail << "unknown distribution package name" << info << "did you forget to set dist.package?"; + const module& mod (*rs.find_module<module> (module::name)); + const string& dist_package (cast<string> (l)); - const process_path& dist_cmd (cast<process_path> (rs.vars["dist.cmd"])); + const process_path* dist_cmd ( + cast_null<process_path> (rs.vars["dist.cmd"])); + + dir_path td (dist_root / dir_path (dist_package)); // We used to print 'dist <target>' at verbosity level 1 but that has // proven to be just noise. Though we still want to print something @@ -183,79 +259,143 @@ namespace build2 // (e.g., output directory creation) in all the operations below. // if (verb == 1) - text << "dist " << dist_package; + print_diag ("dist", src_root, td); // Get the list of files to distribute. // action_targets files; + const variable* dist_var (nullptr); if (tgt != nullptr) { l5 ([&]{trace << "load dist " << rs;}); + dist_var = rs.var_pool ().find ("dist"); + // Match a rule for every operation supported by this project. Skip // default_id. // // Note that we are not calling operation_pre/post() callbacks here // since the meta operation is dist and we know what we are doing. // - values params; path_name pn ("<dist>"); const location loc (pn); // Dummy location. + action_targets ts {tgt}; + + auto process_postponed = [&ctx, &mod] () { - action_targets ts {tgt}; + if (!mod.postponed.list.empty ()) + { + // Re-grab the phase lock similar to perform_match(). + // + phase_lock l (ctx, run_phase::match); + + // Note that we don't need to bother with the mutex since we do + // all of this serially. But we can end up with new elements at + // the end. + // + // Strictly speaking, to handle this correctly we would need to do + // multiple passes over this list and only give up when we cannot + // make any progress since earlier entries that we cannot resolve + // could be "fixed" by later entries. But this feels far-fetched + // and so let's wait for a real example before complicating this. + // + for (auto i (mod.postponed.list.begin ()); + i != mod.postponed.list.end (); + ++i) + rule::match_postponed (*i); + } + }; - auto mog = make_guard ([&ctx] () {ctx.match_only = false;}); - ctx.match_only = true; + auto mog = make_guard ([&ctx] () {ctx.match_only = nullopt;}); + ctx.match_only = match_only_level::all; - const operations& ops (rs.root_extra->operations); - for (operations::size_type id (default_id + 1); // Skip default_id. - id < ops.size (); - ++id) + const operations& ops (rs.root_extra->operations); + for (operations::size_type id (default_id + 1); // Skip default_id. + id < ops.size (); + ++id) + { + if (const operation_info* oif = ops[id]) { - if (const operation_info* oif = ops[id]) - { - // Skip aliases (e.g., update-for-install). In fact, one can - // argue the default update should be sufficient since it is - // assumed to update all prerequisites and we no longer support - // ad hoc stuff like test.input. Though here we are using the - // dist meta-operation, not perform. - // - if (oif->id != id) - continue; + // Skip aliases (e.g., update-for-install). In fact, one can argue + // the default update should be sufficient since it is assumed to + // update all prerequisites and we no longer support ad hoc stuff + // like test.input. Though here we are using the dist + // meta-operation, not perform. + // + if (oif->id != id) + continue; - // Use standard (perform) match. - // - if (oif->pre != nullptr) + // Use standard (perform) match. + // + if (auto pp = oif->pre_operation) + { + if (operation_id pid = pp (ctx, {}, dist_id, loc)) { - if (operation_id pid = oif->pre (params, dist_id, loc)) - { - const operation_info* poif (ops[pid]); - ctx.current_operation (*poif, oif, false /* diag_noise */); - action a (dist_id, poif->id, oif->id); - match (params, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - } + const operation_info* poif (ops[pid]); + ctx.current_operation (*poif, oif, false /* diag_noise */); + + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, false /* inner */, loc); + + if (poif->operation_pre != nullptr) + poif->operation_pre (ctx, {}, true /* inner */, loc); + + action a (dist_id, poif->id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (); + + if (poif->operation_post != nullptr) + poif->operation_post (ctx, {}, true /* inner */); + + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, false /* inner */); } + } + + ctx.current_operation (*oif, nullptr, false /* diag_noise */); - ctx.current_operation (*oif, nullptr, false /* diag_noise */); - action a (dist_id, oif->id); - match (params, a, ts, - 1 /* diag (failures only) */, - false /* progress */); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, true /* inner */, loc); - if (oif->post != nullptr) + action a (dist_id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (); + + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, true /* inner */); + + if (auto po = oif->post_operation) + { + if (operation_id pid = po (ctx, {}, dist_id)) { - if (operation_id pid = oif->post (params, dist_id)) - { - const operation_info* poif (ops[pid]); - ctx.current_operation (*poif, oif, false /* diag_noise */); - action a (dist_id, poif->id, oif->id); - match (params, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - } + const operation_info* poif (ops[pid]); + ctx.current_operation (*poif, oif, false /* diag_noise */); + + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, false /* inner */, loc); + + if (poif->operation_pre != nullptr) + poif->operation_pre (ctx, {}, true /* inner */, loc); + + action a (dist_id, poif->id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (); + + if (poif->operation_post != nullptr) + poif->operation_post (ctx, {}, true /* inner */); + + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, false /* inner */); } } } @@ -264,7 +404,7 @@ namespace build2 // Add ad hoc files and buildfiles that are not normally loaded as // part of the project, for example, the export stub. They will still // be ignored on the next step if the user explicitly marked them - // dist=false. + // with dist=false. // auto add_adhoc = [] (const scope& rs) { @@ -311,7 +451,7 @@ namespace build2 dir_path out_nroot (out_root / pd); const scope& nrs (ctx.scopes.find_out (out_nroot)); - if (nrs.out_path () != out_nroot) // This subproject not loaded. + if (nrs.out_path () != out_nroot) // This subproject is not loaded. continue; if (!nrs.src_path ().sub (src_root)) // Not a strong amalgamation. @@ -327,50 +467,96 @@ namespace build2 // Note that we are not showing progress here (e.g., "N targets to // distribute") since it will be useless (too fast). // - const variable& dist_var (ctx.var_pool["dist"]); - - for (const auto& pt: ctx.targets) + auto see_through = [] (const target& t) { - file* ft (pt->is_a<file> ()); - - if (ft == nullptr) // Not a file. - continue; + return ((t.type ().flags & target_type::flag::see_through) == + target_type::flag::see_through); + }; - if (ft->dir.sub (src_root)) + auto collect = [&trace, &dist_var, + &src_root, &out_root] (const file& ft) + { + if (ft.dir.sub (src_root)) { // Include unless explicitly excluded. // - auto l ((*ft)[dist_var]); - - if (l && !cast<bool> (l)) - l5 ([&]{trace << "excluding " << *ft;}); - else - files.push_back (ft); + if (const path* v = cast_null<path> (ft[dist_var])) + { + if (v->string () == "false") + { + l5 ([&]{trace << "excluding " << ft;}); + return false; + } + } - continue; + return true; } - - if (ft->dir.sub (out_root)) + else if (ft.dir.sub (out_root)) { // Exclude unless explicitly included. // - auto l ((*ft)[dist_var]); + if (const path* v = cast_null<path> (ft[dist_var])) + { + if (v->string () != "false") + { + l5 ([&]{trace << "including " << ft;}); + return true; + } + } - if (l && cast<bool> (l)) + return false; + } + else + return false; // Out of project. + }; + + for (const auto& pt: ctx.targets) + { + // Collect see-through groups if they are marked with dist=true. + // + // Note that while it's possible that only their certain members are + // marked as such (e.g., via a pattern), we will still require + // dist=true on the group itself (and potentially dist=false on some + // of its members) for such cases because we don't want to update + // every see-through group only to discover that most of them don't + // have anything to distribute. + // + if (see_through (*pt)) + { + if (const path* v = cast_null<path> ((*pt)[dist_var])) { - l5 ([&]{trace << "including " << *ft;}); - files.push_back (ft); + if (v->string () != "false") + { + l5 ([&]{trace << "including group " << *pt;}); + files.push_back (pt.get ()); + } } continue; } + + file* ft (pt->is_a<file> ()); + + if (ft == nullptr) // Not a file. + continue; + + // Skip member of see-through groups since after dist_* their list + // can be incomplete (or even bogus, e.g., the "representative + // sample"). Instead, we will collect them during perfrom_update + // below. + // + if (ft->group != nullptr && see_through (*ft->group)) + continue; + + if (collect (*ft)) + files.push_back (ft); } // Make sure what we need to distribute is up to date. // { if (mo_perform.meta_operation_pre != nullptr) - mo_perform.meta_operation_pre (params, loc); + mo_perform.meta_operation_pre (ctx, {}, loc); // This is a hack since according to the rules we need to completely // reset the state. We could have done that (i.e., saved target @@ -386,33 +572,83 @@ namespace build2 ctx.current_on = on + 1; if (mo_perform.operation_pre != nullptr) - mo_perform.operation_pre (params, update_id); + mo_perform.operation_pre (ctx, {}, update_id); ctx.current_operation (op_update, nullptr, false /* diag_noise */); + if (op_update.operation_pre != nullptr) + op_update.operation_pre (ctx, {}, true /* inner */, loc); + action a (perform_update_id); - mo_perform.match (params, a, files, + mo_perform.match ({}, a, files, 1 /* diag (failures only) */, prog /* progress */); - mo_perform.execute (params, a, files, + mo_perform.execute ({}, a, files, 1 /* diag (failures only) */, prog /* progress */); + // Replace see-through groups (which now should have their members + // resolved) with members. + // + for (auto i (files.begin ()); i != files.end (); ) + { + const target& t (i->as<target> ()); + if (see_through (t)) + { + group_view gv (t.group_members (a)); // Go directly. + + if (gv.members == nullptr) + fail << "unable to resolve see-through group " << t + << " members"; + + i = files.erase (i); // Drop the group itself. + + for (size_t j (0); j != gv.count; ++j) + { + if (const target* m = gv.members[j]) + { + if (const file* ft = m->is_a<file> ()) + { + // Note that a rule may only link-up its members to groups + // if/when matched (for example, the cli.cxx{} group). It + // feels harmless for us to do the linking here. + // + if (ft->group == nullptr) + const_cast<file*> (ft)->group = &t; + else + assert (ft->group == &t); // Sanity check. + + if (collect (*ft)) + { + i = files.insert (i, ft); // Insert instead of the group. + i++; // Stay after the group. + } + } + } + } + } + else + ++i; + } + + if (op_update.operation_post != nullptr) + op_update.operation_post (ctx, {}, true /* inner */); + if (mo_perform.operation_post != nullptr) - mo_perform.operation_post (params, update_id); + mo_perform.operation_post (ctx, {}, update_id); if (mo_perform.meta_operation_post != nullptr) - mo_perform.meta_operation_post (params); + mo_perform.meta_operation_post (ctx, {}); } } else { l5 ([&]{trace << "bootstrap dist " << rs;}); - // Recursively enter/collect file targets in src_root ignoring those - // that start with a dot. + // Recursively enter/collect file targets in src_root ignoring the + // following ones: .git*, .bdep, .bpkg, and .build2. // // Note that, in particular, we also collect the symlinks which point // outside src_root (think of third-party project packaging with the @@ -430,37 +666,80 @@ namespace build2 // auto_project_env penv (rs); - dir_path td (dist_root / dir_path (dist_package)); - // Clean up the target directory. // if (rmdir_r (ctx, td, true, 2) == rmdir_status::not_empty) fail << "unable to clean target directory " << td; auto_rmdir rm_td (td); // Clean it up if things go bad. - install (dist_cmd, td); + install (dist_cmd, ctx, td); // Copy over all the files. Apply post-processing callbacks. // - module& mod (*rs.find_module<module> (module::name)); - prog = prog && show_progress (1 /* max_verb */); size_t prog_percent (0); for (size_t i (0), n (files.size ()); i != n; ++i) { - const file& t (*files[i].as<target> ().is_a<file> ()); + const file& t (files[i].as<target> ().as<file> ()); // Only files. // Figure out where this file is inside the target directory. // - bool src (t.dir.sub (src_root)); - dir_path dl (src ? t.dir.leaf (src_root) : t.dir.leaf (out_root)); + // First see if the path has been remapped (unless bootstrap). + // + const path* rp (nullptr); + if (tgt != nullptr) + { + if ((rp = cast_null<path> (t[dist_var])) != nullptr) + { + if (rp->string () == "true") // Wouldn't be here if false. + rp = nullptr; + } + } + + bool src; + path rn; + dir_path dl; + if (rp == nullptr) + { + src = t.dir.sub (src_root); + dl = src ? t.dir.leaf (src_root) : t.dir.leaf (out_root); + } + else + { + // Sort the remapped path into name (if any) and directory, + // completing the latter if relative. + // + bool n (!rp->to_directory ()); + + if (n) + { + if (rp->simple ()) + { + fail << "expected true, false, of path in the dist variable " + << "value of target " << t << + info << "specify ./" << *rp << " to remap the name"; + } + + rn = rp->leaf (); + } + + dir_path rd (n ? rp->directory () : path_cast<dir_path> (*rp)); + + if (rd.relative ()) + rd = t.dir / rd; + + rd.normalize (); + + src = rd.sub (src_root); + dl = src ? rd.leaf (src_root) : rd.leaf (out_root); + } dir_path d (td / dl); if (!exists (d)) - install (dist_cmd, d); + install (dist_cmd, ctx, d); - path r (install (dist_cmd, t, d)); + path r (install (dist_cmd, t, d, rn)); // See if this file is in a subproject. // @@ -605,8 +884,8 @@ namespace build2 fail << "dist meta-operation target must be project root directory"; if (rs->out_eq_src ()) - fail << "in-tree distribution of target " << t << - info << "distribution requires out-of-tree build"; + fail << "in source distribution of target " << t << + info << "distribution requires out of source build"; dist_project (*rs, &t, prog); } @@ -614,60 +893,131 @@ namespace build2 // install -d <dir> // static void - install (const process_path& cmd, const dir_path& d) + install (const process_path* cmd, context& ctx, const dir_path& d) { - path reld (relative (d)); + path reld; + cstrings args; - cstrings args {cmd.recall_string (), "-d"}; + if (cmd != nullptr || verb >= 2) + { + reld = relative (d); - args.push_back ("-m"); - args.push_back ("755"); - args.push_back (reld.string ().c_str ()); - args.push_back (nullptr); + args.push_back (cmd != nullptr ? cmd->recall_string () : "install"); + args.push_back ("-d"); + args.push_back ("-m"); + args.push_back ("755"); + args.push_back (reld.string ().c_str ()); + args.push_back (nullptr); - if (verb >= 2) - print_process (args); + if (verb >= 2) + print_process (args); + } - run (cmd, args); + if (cmd != nullptr) + run (ctx, *cmd, args, 1 /* finish_verbosity */); + else + { + try + { + // Note that mode has no effect on Windows, which is probably for + // the best. + // + try_mkdir_p (d, 0755); + } + catch (const system_error& e) + { + fail << "unable to create directory " << d << ": " << e; + } + } } - // install <file> <dir> + // install <file> <dir>[/<name>] // static path - install (const process_path& cmd, const file& t, const dir_path& d) + install (const process_path* cmd, + const file& t, + const dir_path& d, + const path& n) { - dir_path reld (relative (d)); - path relf (relative (t.path ())); - - cstrings args {cmd.recall_string ()}; + const path& f (t.path ()); + path r (d / (n.empty () ? f.leaf () : n)); - // Preserve timestamps. This could becomes important if, for - // example, we have pre-generated sources. Note that the - // install-sh script doesn't support this option, while both - // Linux and BSD install's do. + // Assume the file is executable if the owner has execute permission, + // in which case we make it executable for everyone. // - args.push_back ("-p"); + bool exe ((path_perms (f) & permissions::xu) == permissions::xu); - // Assume the file is executable if the owner has execute - // permission, in which case we make it executable for - // everyone. - // - args.push_back ("-m"); - args.push_back ( - (path_perms (t.path ()) & permissions::xu) == permissions::xu - ? "755" - : "644"); + path relf, reld; + cstrings args; - args.push_back (relf.string ().c_str ()); - args.push_back (reld.string ().c_str ()); - args.push_back (nullptr); + if (cmd != nullptr || verb >= 2) + { + relf = relative (f); + reld = relative (d); - if (verb >= 2) - print_process (args); + if (!n.empty ()) // Leave as just directory if no custom name. + reld /= n; + + args.push_back (cmd != nullptr ? cmd->recall_string () : "install"); + + // Preserve timestamps. This could becomes important if, for example, + // we have pre-generated sources. Note that the install-sh script + // doesn't support this option, while both Linux and BSD install's do. + // + args.push_back ("-p"); + + // Assume the file is executable if the owner has execute permission, + // in which case we make it executable for everyone. + // + args.push_back ("-m"); + args.push_back (exe ? "755" : "644"); + args.push_back (relf.string ().c_str ()); + args.push_back (reld.string ().c_str ()); + args.push_back (nullptr); + + if (verb >= 2) + print_process (args); + } - run (cmd, args); + if (cmd != nullptr) + run (t.ctx, *cmd, args, 1 /* finish_verbosity */); + else + { + permissions perm (permissions::ru | permissions::wu | + permissions::rg | + permissions::ro); // 644 + if (exe) + perm |= permissions::xu | permissions::xg | permissions::xo; // 755 + + try + { + // Note that we don't pass cpflags::overwrite_content which means + // this will fail if the file already exists. Since we clean up the + // destination directory, this will detect cases where we have + // multiple source files with the same distribution destination. + // + cpfile (f, + r, + cpflags::overwrite_permissions | cpflags::copy_timestamps, + perm); + } + catch (const system_error& e) + { + if (e.code ().category () == generic_category () && + e.code ().value () == EEXIST) + { + // @@ TMP (added in 0.16.0). + // + warn << "multiple files are distributed as " << r << + info << "second file is " << f << + info << "this warning will become error in the future"; + } + else + fail << "unable to copy " << f << " to " << r << ": " << e; + } + } - return d / relf.leaf (); + return r; } static path @@ -677,13 +1027,15 @@ namespace build2 const dir_path& dir, const string& e) { + // NOTE: similar code in bpkg (system-package-manager-archive.cxx). + path an (pkg + '.' + e); // Delete old archive for good measure. // path ap (dir / an); if (exists (ap, false)) - rmfile (ctx, ap); + rmfile (ctx, ap, 3 /* verbosity */); // Use zip for .zip archives. Also recognize and handle a few well-known // tar.xx cases (in case tar doesn't support -a or has other issues like @@ -699,7 +1051,7 @@ namespace build2 if (e == "zip") { - // On Windows we use libarchive's bsdtar (zip is an MSYS executabales). + // On Windows we use libarchive's bsdtar (zip is an MSYS executable). // // While not explicitly stated, the compression-level option works // for zip archives. @@ -724,15 +1076,28 @@ namespace build2 // On Windows we use libarchive's bsdtar with auto-compression (tar // itself and quite a few compressors are MSYS executables). // + // OpenBSD tar does not support --format but it appear ustar is the + // default (while this is not said explicitly in tar(1), it is said in + // pax(1) and confirmed on the mailing list). Nor does it support -a, + // at least as of 7.1 but we will let this play out naturally, in case + // this support gets added. + // + // Note also that our long-term plan is to switch to libarchive in + // order to generate reproducible archives. + // const char* l (nullptr); // Compression level (option). #ifdef _WIN32 - const char* tar = "bsdtar"; + args = {"bsdtar", "--format", "ustar"}; if (e == "tar.gz") l = "--options=compression-level=9"; #else - const char* tar = "tar"; + args = {"tar" +#ifndef __OpenBSD__ + , "--format", "ustar" +#endif + }; // For gzip it's a good idea to use -9 by default. For bzip2, -9 is // the default. And for xz, -9 is not recommended as the default due @@ -750,13 +1115,10 @@ namespace build2 if (c != nullptr) { - args = {tar, - "--format", "ustar", - "-cf", "-", - pkg.c_str (), - nullptr}; - - i = args.size (); + args.push_back ("-cf"); + args.push_back ("-"); + args.push_back (pkg.c_str ()); + args.push_back (nullptr); i = args.size (); args.push_back (c); if (l != nullptr) args.push_back (l); @@ -777,20 +1139,13 @@ namespace build2 } else #endif - if (e == "tar") - args = {tar, - "--format", "ustar", - "-cf", ap.string ().c_str (), - pkg.c_str (), - nullptr}; - else { - args = {tar, - "--format", "ustar", - "-a"}; - - if (l != nullptr) - args.push_back (l); + if (e != "tar") + { + args.push_back ("-a"); + if (l != nullptr) + args.push_back (l); + } args.push_back ("-cf"); args.push_back (ap.string ().c_str ()); @@ -810,19 +1165,20 @@ namespace build2 if (verb >= 2) print_process (args); else if (verb) - text << args[0] << ' ' << ap; + print_diag (args[0], dir / dir_path (pkg), ap); process apr; process cpr; - // Change the archiver's working directory to dist_root. + // Change the archiver's working directory to root. + // + // Note: this function is called during serial execution and so no + // diagnostics buffering is needed (here and below). // - apr = run_start (app, + apr = run_start (process_env (app, root), args, 0 /* stdin */, - (i != 0 ? -1 : 1) /* stdout */, - true /* error */, - root); + (i != 0 ? -1 : 1) /* stdout */); // Start the compressor if required. // @@ -834,10 +1190,17 @@ namespace build2 out_fd.get () /* stdout */); cpr.in_ofd.reset (); // Close the archiver's stdout on our side. - run_finish (args.data () + i, cpr); } - run_finish (args.data (), apr); + // Delay throwing until we diagnose both ends of the pipe. + // + if (!run_finish_code (args.data (), + apr, + 1 /* verbosity */, + false /* omit_normal */) || + !(i == 0 || run_finish_code (args.data () + i, cpr, 1, false))) + throw failed (); + out_rm.cancel (); return ap; @@ -856,7 +1219,7 @@ namespace build2 // path cp (dir / cn); if (exists (cp, false)) - rmfile (ctx, cp); + rmfile (ctx, cp, 3 /* verbosity */); auto_rmfile c_rm; // Note: must come first. auto_fd c_fd; @@ -895,18 +1258,20 @@ namespace build2 if (verb >= 2) print_process (args); else if (verb) - text << args[0] << ' ' << cp; + print_diag (args[0], ap, cp); // Note that to only get the archive name (without the directory) in // the output we have to run from the archive's directory. // - process pr (run_start (pp, + // Note: this function is called during serial execution and so no + // diagnostics buffering is needed. + // + process pr (run_start (process_env (pp, ad /* cwd */), args, - 0 /* stdin */, - c_fd.get () /* stdout */, - true /* error */, - ad /* cwd */)); - run_finish (args, pr); + 0 /* stdin */, + c_fd.get () /* stdout */)); + + run_finish (args, pr, 1 /* verbosity */); } else { @@ -926,7 +1291,7 @@ namespace build2 if (verb >= 2) text << "cat >" << cp; else if (verb) - text << e << "sum " << cp; + print_diag ((e + "sum").c_str (), ap, cp); string c; try @@ -960,7 +1325,8 @@ namespace build2 dist_include (action, const target&, const prerequisite_member& p, - include_type i) + include_type i, + lookup& l) { tracer trace ("dist::dist_include"); @@ -969,12 +1335,18 @@ namespace build2 // given the prescribed semantics of adhoc (match/execute but otherwise // ignore) is followed. // + // Note that we don't need to do anything for posthoc. + // if (i == include_type::excluded) { l5 ([&]{trace << "overriding exclusion of " << p;}); i = include_type::adhoc; } + // Also clear any operation-specific overrides. + // + l = lookup (); + return i; } @@ -988,12 +1360,12 @@ namespace build2 true, // bootstrap_outer nullptr, // meta-operation pre &dist_operation_pre, - &load, // normal load - &search, // normal search - nullptr, // no match (see dist_execute()). + &dist_load_load, + &perform_search, // normal search + nullptr, // no match (see dist_execute()). &dist_load_execute, - nullptr, // operation post - nullptr, // meta-operation post + nullptr, // operation post + nullptr, // meta-operation post &dist_include }; @@ -1024,7 +1396,7 @@ namespace build2 init_config (rs); } - void + static void dist_bootstrap_search (const values&, const scope& rs, const scope&, |