From 40273d053e3024dc5c9acd063882a848358df4fa Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Sat, 4 Aug 2018 16:13:29 +0300 Subject: Add archive checksum files to build2-control branch in bdep-publish --- bdep/publish.cxx | 479 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 456 insertions(+), 23 deletions(-) (limited to 'bdep/publish.cxx') diff --git a/bdep/publish.cxx b/bdep/publish.cxx index bea883e..c27609a 100644 --- a/bdep/publish.cxx +++ b/bdep/publish.cxx @@ -6,8 +6,12 @@ #include // strtoul() +#include // fdterm() #include #include +#include + +#include #include #include @@ -297,7 +301,7 @@ namespace bdep v.push_back ("-s"); v.push_back ("-S"); // But show errors. } - else if (verb == 1) + else if (verb == 1 && fdterm (2)) v.push_back ("--progress-bar"); else if (verb > 3) v.push_back ("-v"); @@ -672,6 +676,8 @@ namespace bdep const dir_path& cfg, package_locations&& pkg_locs) { + using bpkg::package_manifest; + const url& repo (o.repository ()); optional ctrl; @@ -692,7 +698,8 @@ namespace bdep fail << "unable to obtain publisher's email" << info << "use --email to specify explicitly"; - // Collect package information (version, project, section). + // Collect package information (version, project, section, archive + // path/checksum, and manifest). // // @@ It would have been nice to publish them in the dependency order. // Perhaps we need something like bpkg-pkg-order (also would be needed @@ -707,6 +714,8 @@ namespace bdep path archive; string checksum; + + package_manifest manifest; }; vector pkgs; @@ -730,8 +739,13 @@ namespace bdep v.alpha () || v.major () == 0 ? "alpha" : v.beta () ? "beta" : "stable"); - pkgs.push_back ( - package {move (n), move (v), move (p), move (s), path (), string ()}); + pkgs.push_back (package {move (n), + move (v), + move (p), + move (s), + path () /* archive */, + string () /* checksum */, + package_manifest ()}); } // Print the plan and ask for confirmation. @@ -772,7 +786,8 @@ namespace bdep } // Prepare package archives and calculate their checksums. Also verify - // each archive with bpkg-pkg-verify for good measure. + // each archive with bpkg-pkg-verify and parse the package manifest it + // contains. // auto_rmdir dr_rm (tmp_dir ("publish")); const dir_path& dr (dr_rm.path); // dist.root @@ -802,9 +817,50 @@ namespace bdep if (!exists (c)) fail << "package distribution did not produce expected checksum " << c; - // Verify that archive name/content all match. + // Verify that archive name/content all match and while at it extract + // its manifest. // - run_bpkg (2 /* verbosity */, o, "pkg-verify", a); + process pr; + bool io (false); + try + { + fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate. + + pr = start_bpkg (2 /* verbosity */, + o, + pipe /* stdout */, + 2 /* stderr */, + "pkg-verify", + "--manifest", + a); + + pipe.out.close (); + ifdstream is (move (pipe.in), fdstream_mode::skip); + + manifest_parser mp (is, manifest_file.string ()); + p.manifest = package_manifest (mp); + is.close (); + } + // This exception is unlikely to be thrown as the package manifest is + // already validated by bpkg-pkg-verify. However, it's still possible if + // something is skew (e.g., different bpkg/bdep versions). + // + catch (const manifest_parsing& e) + { + finish_bpkg (o, pr); + + fail << "unable to parse package manifest in archive " << a << ": " + << e; + } + catch (const io_error&) + { + // Presumably the child process failed and issued diagnostics so let + // finish_bpkg() try to deal with that first. + // + io = true; + } + + finish_bpkg (o, pr, io); // Read the checksum. // @@ -825,6 +881,399 @@ namespace bdep p.archive = move (a); } + // Add the package archive "authorization" files to the build2-control + // branch. + // + // Their names are 16-character abbreviated checksums (a bit more than the + // standard 12 for security) and their content is the package manifest + // header (for the record). + // + // See if this is a VCS repository we recognize. + // + if (ctrl && git (prj)) + { + // Checkout the build2-control branch into a separate working tree not + // to interfere with the user's stuff. + // + auto_rmdir wd_rm (tmp_dir ("control")); + const dir_path& wd (wd_rm.path); + mk (wd); + + const dir_path submit_dir ("submit"); + dir_path sd (wd / submit_dir); + + // The 'git worktree add' command is quite verbose, printing something + // like: + // + // Preparing /tmp/hello/.bdep/tmp/control-14926-1 (identifier control-14926-1) + // HEAD is now at 3fd69a3 Publish hello/0.1.0-a.1 + // + // Note that there doesn't seem to be an option (yet) for suppressing + // this output. Also note that the first line is printed to stderr and + // the second one to stdout. So what we are going to do is redirect both + // stderr and stdout to /dev/null if the verbosity level is less than + // two and advise the user to re-run with -v on failure. + // + auto worktree_add = [&prj] (auto&&... args) + { + bool q (verb < 2); + auto_fd null (q ? fdnull () : auto_fd ()); + + process pr (start_git (0 /* stdin */, + q ? null.get () : 1 /* stdout */, + q ? null.get () : 2 /* stderr */, + prj, + "worktree", + "add", + forward (args)...)); + + if (pr.wait ()) + return; + + if (!q) + throw failed (); // Diagnostics already issued. + + assert (pr.exit); + + const process_exit& e (*pr.exit); + + if (e.normal ()) + fail << "unable to add worktree for build2-control branch" << + info << "re-run with -v for more information"; + else + fail << "git " << e; + }; + + auto worktree_prune = [&prj] () + { + run_git (prj, "worktree", "prune", verb > 2 ? "-v" : nullptr); + }; + + // Create the build2-control branch if it doesn't exist, from scratch if + // there is no remote-tracking branch and as a checkout with --track -b + // otherwise. If the local branch exists, then fast-forward it using the + // locally fetched data (no network). + // + // Note that we don't fetch in advance, so push conflicts are possible. + // The idea behind this is that it will be more efficient in most cases + // as the remote-tracking branch is likely to already be up-to-date due + // to the implicit branch fetches peformed by other operations like + // pull. In the rare conflict cases we will advise the user to run the + // fetch command and re-try. + // + bool local_exists (git_line (prj, + false /* ignore_error */, + "branch", + "--list", + "build2-control")); + + // @@ Should we allow using the remote name other than origin (here and + // everywhere) via the --remote option or smth? Maybe later. + // + bool remote_exists (git_line (prj, + false /* ignore_error */, + "branch", + "--list", + "--remote", + "origin/build2-control")); + + bool local_new (false); // The branch is created from scratch. + + // Note that the case where the local build2-control branch exists while + // the remote-tracking one doesn't is treated similar (but different) to + // the brand new branch: the upstream branch will be set up by the push + // operation but the local branch will not be deleted on the push + // failure. + // + if (!local_exists) + { + // Create the brand new local branch if the remote-tracking branch + // doesn't exist. + // + // The tricky part is to make sure that it doesn't inherit the current + // branch history. To accomplish this we will create an empty tree + // object, base the root (no parents) commit on it, and create the + // build2-control branch pointing to this commit. + // + if (!remote_exists) + { + // Create the empty tree object. + // + auto_fd null (fdnull ()); + fdpipe pipe (fdopen_pipe ()); + + process pr (start_git (null.get () /* stdin */, + pipe /* stdout */, + 2 /* stderr */, + prj, + "hash-object", + "-wt", "tree", + "--stdin")); + + optional tree (git_line (move (pr), move (pipe))); + + if (!tree) + fail << "unable to create git tree object for build2-control"; + + // Create the (empty) root commit. + // + optional commit (git_line (prj, + false, + "commit-tree", + "-m", "Start", + *tree)); + + if (!commit) + fail << "unable to create root git commit for build2-control"; + + // Create the branch. + // + // Note that we pass the empty oldvalue to make sure that the ref we + // are creating does not exist. It should be impossible but let's + // tighten things up a bit. + // + run_git (prj, + "update-ref", + "refs/heads/build2-control", + *commit, + "" /* oldvalue */); + + // Checkout the branch. Note that the upstream branch is not setup + // for it yet. This will be done by the push operation. + // + worktree_add (wd, "build2-control"); + + // Create the checksum files subdirectory. + // + mk (sd); + + local_new = true; + } + else + // Create the local branch, setting up the corresponding upstream + // branch. + // + worktree_add ("--track", + "-b", "build2-control", + wd, + "origin/build2-control"); + } + else + { + // Checkout the existing local branch. Note that we still need to + // fast-forward it (see below). + // + // Prune the build2-control worktree that could potentially stay from + // the interrupted previous publishing attempt. + // + worktree_prune (); + + worktree_add (wd, "build2-control"); + } + + // "Release" the checked out branch and delete the worktree, if exists. + // + // Note that until this is done the branch can not be checked out in any + // other worktree. + // + auto worktree_remove = [&prj, &wd, &wd_rm, &worktree_prune] () + { + if (exists (wd)) + { + // Note that we cannot (yet) use git-worktree-remove since it is not + // available in older versions. + // + rm_r (wd); + wd_rm.cancel (); + + worktree_prune (); + } + }; + + // Now, given that we successfully checked out the build2-control + // branch, add the authorization files for packages being published, + // commit, and push. Skip already existing files. Don't push if no files + // were added. + // + { + // Remove the control2-branch worktree on failure. Failed that we will + // get the 'build2-control is already checked out' error on the next + // publish attempt. + // + auto wg (make_exception_guard ([&worktree_remove] () + { + try + { + worktree_remove (); // Can throw failed. + } + catch (const failed&) + { + // We can't do much here and will let the user deal with the mess. + // Note that running 'git worktree prune' will likely be enough. + // Anyway, that's unlikely to happen. + } + })); + + // If the local branch existed from the beginning then fast-forward it + // over the remote-tracking branch. + // + // Note that fast-forwarding can potentially fail. That will mean the + // local branch has diverged from the remote one for some reason + // (e.g., inability to revert the commit, etc.). We again leave it to + // the use to deal with. + // + if (local_exists && remote_exists) + run_git (wd, + "merge", + verb < 2 ? "-q" : verb > 2 ? "-v" : nullptr, + "--ff-only", + "origin/build2-control"); + + // Create the authorization files and add them to the repository. + // + bool added (false); + + for (const package& p: pkgs) + { + // Use 16 characters of the sha256sum instead of 12 for extra + // security. + // + path ac (string (p.checksum, 0, 16)); + path mf (sd / ac); + + if (exists (mf)) + continue; + + try + { + ofdstream os (mf); + manifest_serializer s (os, mf.string ()); + p.manifest.serialize_header (s); + os.close (); + } + catch (const manifest_serialization&) + { + // This shouldn't happen as we just parsed the manifest. + // + assert (false); + } + catch (const io_error& e) + { + fail << "unable to write " << mf << ": " << e; + } + + run_git (wd, + "add", + verb > 2 ? "-v" : nullptr, + submit_dir / ac); + + added = true; + } + + // Commit and push. + // + // Note that we push even if we haven't committed anything in case we + // have added but haven't managed to push it on the previous run. + // + if (added) + { + // Format the commit message. + // + string m; + + auto pkg_str = [] (const package& p) + { + return p.name.string () + '/' + p.version.string (); + }; + + if (pkgs.size () == 1) + m = "Add " + pkg_str (pkgs[0]) + " publish authorization"; + else + { + m = "Add publish authorizations\n"; + + for (const package& p: pkgs) + { + m += '\n'; + m += pkg_str (p); + } + } + + run_git (wd, + "commit", + verb < 2 ? "-q" : verb > 2 ? "-v" : nullptr, + "-m", m); + } + + // If we fail to push the control branch, then revert the commit and + // advice the user to fetch the repository and re-try. + // + auto pg ( + make_exception_guard ( + [added, &prj, &wd, &worktree_remove, local_new] () + { + if (added) + try + { + // If the local build2-control branch was just created, then + // we need to drop not just the last commit but the whole + // branch (including it's root commit). Note that this is + // not an optimization. Imagine that the remote branch is + // not fetched yet and we just created the local one. If we + // leave this branch around after the failed push, then we + // will still be in trouble after the fetch since we won't + // be able to merge unrelated histories. + // + if (local_new) + { + worktree_remove (); // Release the branch before removal. + + run_git (prj, + "branch", + verb < 2 ? "-q" : nullptr, + "-D", + "build2-control"); + } + else + run_git (wd, + "reset", + verb < 2 ? "-q" : nullptr, + "--hard", + "HEAD^"); + + error << "unable to push build2-control branch" << + info << "run 'git fetch' and try again"; + } + catch (const failed&) + { + // We can't do much here and will leave the user to deal + // with the mess. Note that running 'git fetch' will not be + // enough as the local and remote branches are likely to + // have diverged. + } + })); + + if (verb) + text << "pushing build2-control"; + + // Note that we suppress the (too detailed) push command output if + // the verbosity level is 1. However, we still want to see the + // progress in this case. + // + run_git (wd, + "push", + + verb < 2 ? "-q" : verb > 3 ? "-v" : nullptr, + verb == 1 ? "--progress" : nullptr, + + !remote_exists + ? cstrings ({"--set-upstream", "origin", "build2-control"}) + : cstrings ()); + } + + worktree_remove (); + } + // Submit each package. // for (const package& p: pkgs) @@ -840,22 +1289,6 @@ namespace bdep if (verb) text << r.second << " (" << r.first << ")"; - - //@@ TODO [phase 2]: add checksum file to build2-control branch, commit - // and push (this will need some more discussion). - // - // - name (abbrev 12 char checksum) and subdir? - // - // - make the checksum file a manifest with basic info (name, version) - // - // - what if already exists (previous failed attempt)? Ignore? - // - // - make a separate checkout (in tmp facility) reusing the external - // .git/ dir? - // - // - should probably first fetch to avoid push conflicts. Or maybe - // fetch on push conflict (more efficient/robust)? - // } return 0; -- cgit v1.1