From 0d1d47c5e183adc61dc60f735a1fe2422ca6c864 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Wed, 14 Oct 2020 19:38:39 +0300 Subject: Rename project/package from bpkg-rep to bpkg-util --- INSTALL-PUBLISH | 24 +- NEWS | 2 +- README | 2 +- bpkg-rep/.gitignore | 2 - bpkg-rep/buildfile | 17 - bpkg-rep/manage.in | 957 -------------------------------------- bpkg-rep/package-archive.bash.in | 170 ------- bpkg-rep/publish.in | 329 ------------- bpkg-rep/utility.bash.in | 86 ---- bpkg-util/.gitignore | 2 + bpkg-util/buildfile | 17 + bpkg-util/manage.in | 957 ++++++++++++++++++++++++++++++++++++++ bpkg-util/package-archive.bash.in | 170 +++++++ bpkg-util/publish.in | 329 +++++++++++++ bpkg-util/utility.bash.in | 86 ++++ build/bootstrap.build | 2 +- build/export.build | 6 +- build/root.build | 2 +- manifest | 6 +- repositories.manifest | 2 +- tests/package-archive/buildfile | 2 +- tests/package-archive/driver.in | 4 +- tests/package-archive/testscript | 6 +- tests/publish/buildfile | 2 +- 24 files changed, 1591 insertions(+), 1591 deletions(-) delete mode 100644 bpkg-rep/.gitignore delete mode 100644 bpkg-rep/buildfile delete mode 100644 bpkg-rep/manage.in delete mode 100644 bpkg-rep/package-archive.bash.in delete mode 100644 bpkg-rep/publish.in delete mode 100644 bpkg-rep/utility.bash.in create mode 100644 bpkg-util/.gitignore create mode 100644 bpkg-util/buildfile create mode 100644 bpkg-util/manage.in create mode 100644 bpkg-util/package-archive.bash.in create mode 100644 bpkg-util/publish.in create mode 100644 bpkg-util/utility.bash.in diff --git a/INSTALL-PUBLISH b/INSTALL-PUBLISH index fea936c..aabbaf9 100644 --- a/INSTALL-PUBLISH +++ b/INSTALL-PUBLISH @@ -1,4 +1,4 @@ -This guide shows how to install and configure the bpkg-rep-publish script to +This guide shows how to install and configure the bpkg-util-publish script to update and publish an archive-based bpkg repository that is stored in a git repository. Normally the bpkg repository is signed and synchronized to a remote host. Both of these operations usually requires authentication so here @@ -6,14 +6,14 @@ we assume that you have arranged for the password-less repository signing and ssh authentication (for example, using unlocked private keys, ssh-agent and openssl-agent, etc). -See also the documentation at the beginning of bpkg-rep-publish for details +See also the documentation at the beginning of bpkg-util-publish for details on the script's operation. 1. Create 'bpub' User -This user will be used to run the bpkg-rep-publish script. We will also use -its home directory to build and install bpkg-rep package, clone the git +This user will be used to run the bpkg-util-publish script. We will also use +its home directory to build and install bpkg-util package, clone the git repository, create the logs directory, etc. We create this user with a disabled password so only root will be able to @@ -40,17 +40,17 @@ b) Unless you already have the build2 toolchain, install it by following instructions on https://build2.org/install.xhtml. -3. Build and Install bpkg-rep. +3. Build and Install bpkg-util. -$ mkdir bpkg-rep -$ cd bpkg-rep +$ mkdir bpkg-util +$ cd bpkg-util $ bpkg create \ cc \ config.bin.rpath=$HOME/install/lib \ config.install.root=$HOME/install -$ bpkg build bpkg-rep,bpkg@https://pkg.cppget.org/1/alpha +$ bpkg build bpkg-util,bpkg@https://pkg.cppget.org/1/alpha $ bpkg install -a $ cd .. # Back to bpub home. @@ -65,17 +65,17 @@ $ git clone https://git.example.org/foo.git $ mkdir foo.log Note that the bpkg repository contents are expected to be in the foo/1/ -subdirectory (see bpkg-rep-publish for details). +subdirectory (see bpkg-util-publish for details). Perform the initial publication to test the setup and make the subsequent synchronizations incremental. Here you may need to pass additionall -bpkg-rep-create(1) and rsync(1) options to the bpkg-rep-publish script. +bpkg-rep-create(1) and rsync(1) options to the bpkg-util-publish script. -$ install/bin/bpkg-rep-publish -d example.org:/var/bpkg/1/foo foo ... +$ install/bin/bpkg-util-publish -d example.org:/var/bpkg/1/foo foo ... Setup publishing as a cron job (every 5 minutes in this example): $ crontab -l MAILTO=publish@example.org PATH=/usr/local/bin:/bin:/usr/bin -0/5 * * * * $HOME/install/bin/bpkg-rep-publish -d example.org:/var/bpkg/1/foo --log-dir $HOME/foo.log --lock-timeout 600 $HOME/foo ... +0/5 * * * * $HOME/install/bin/bpkg-util-publish -d example.org:/var/bpkg/1/foo --log-dir $HOME/foo.log --lock-timeout 600 $HOME/foo ... diff --git a/NEWS b/NEWS index ae9de1a..cbad42a 100644 --- a/NEWS +++ b/NEWS @@ -1 +1 @@ -See https://git.build2.org/cgit/bpkg-rep/log/ for the change log. +See https://git.build2.org/cgit/bpkg-util/log/ for the change log. diff --git a/README b/README index e1eb555..a79e96e 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -This package contains bpkg repository management utilities. +This package contains extra build2 package management utilities. build2 is an open source, cross-platform toolchain for building and packaging C++ code. Its aim is a modern build system and dependency manager for the C++ diff --git a/bpkg-rep/.gitignore b/bpkg-rep/.gitignore deleted file mode 100644 index ec9e671..0000000 --- a/bpkg-rep/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -bpkg-rep-publish -bpkg-rep-manage diff --git a/bpkg-rep/buildfile b/bpkg-rep/buildfile deleted file mode 100644 index 3c66f8b..0000000 --- a/bpkg-rep/buildfile +++ /dev/null @@ -1,17 +0,0 @@ -# file : bpkg-rep/buildfile -# license : MIT; see accompanying LICENSE file - -import mods = libbutl.bash%bash{manifest-parser} - -./: exe{bpkg-rep-publish bpkg-rep-manage} bash{package-archive} - -exe{bpkg-rep-publish}: in{publish} bash{utility} -exe{bpkg-rep-manage}: in{manage} bash{utility package-archive} - -# Public modules. -# -bash{package-archive}: in{package-archive} $mods - -# Private modules. -# -bash{utility}: in{utility} diff --git a/bpkg-rep/manage.in b/bpkg-rep/manage.in deleted file mode 100644 index ceede3c..0000000 --- a/bpkg-rep/manage.in +++ /dev/null @@ -1,957 +0,0 @@ -#!/usr/bin/env bash - -# file : bpkg-rep/manage.in -# license : MIT; see accompanying LICENSE file - -# Interactively migrate newly-submitted packages from a source git repository -# to a destination git repository. -# -# Present the user with the list of commits that added the files currently in -# the source repository's working directory and ask the user to select from a -# menu the action to perform on a selection of these commits. -# -# As the files added by these commits are pending a move to the destination -# repository, these commits will be referred to as "pending" commits. -# -# Actions that can be performed on a selection of pending commits include -# moving them from the source repository to a single commit in the destination -# repository and dropping them from the source repository. -# -# The flow of this script, in broad strokes, is as follows: for each file in -# the source repository directory, find the hash of the commit that added it; -# these are the pending commits. Arrange the pending commits in chronological -# order. Display to the user the pending commits along with the files they -# added. Let the user select one or more pending commits and an action to be -# performed on them. Each successful action results in a commit to the source -# and/or destination repositories and leaves both repositories in a clean -# state. Once the action has been performed, redisplay the updated pending -# commit list and prompt for the next action. Pushing to the remote -# repositories, a supported operation, can be done at any time during the -# session. -# -# The directory into which the source and destination repositories have -# been checked out. If not specified, current directory is assumed. -# -usage="usage: $0 []" - -# Source/destination repository inside . Note: also used in commit -# messages. -# -src_repo_name=queue -dst_repo_name=public - -owd="$(pwd)" -trap "{ cd '$owd'; exit 1; }" ERR -set -o errtrace # Trap in functions. - -@import bpkg-rep/utility@ - -# Use the bpkg program from the script directory, if present. Otherwise, use -# just 'bpkg'. -# -bpkg_rep_bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" - -if [ ! -x "$bpkg_rep_bpkg" ]; then - bpkg_rep_bpkg=bpkg -fi - -@import bpkg-rep/package-archive@ - -# Set the working directory. -# -if [ $# -eq 0 ]; then - dir="$owd" -elif [ $# -eq 1 ]; then - dir="${1%/}" # with trailing slash removed. -else - error "$usage" -fi - -# The source and destination package repository directories. -# -# Note that, throughout this script, any path not explicitly prefixed with -# "$src_dir/" or "$dst_dir/" is relative to the root of the source or -# destination package repositories. -# -src_dir="$dir/$src_repo_name" -dst_dir="$dir/$dst_repo_name" - -if [ ! -d "$src_dir" ]; then - error "'$src_dir' does not exist or is not a directory" -fi - -if [ ! -d "$dst_dir" ]; then - error "'$dst_dir' does not exist or is not a directory" -fi - -# Check that both git repositories are clean. -# -if [ -n "$(git -C $src_dir status --porcelain)" ]; then - error "git repository in '$src_dir' is not clean" -fi - -if [ -n "$(git -C $dst_dir status --porcelain)" ]; then - error "git repository in '$dst_dir' is not clean" -fi - -# Use run() to show the user that git is the source of the diagnostics. -# "Already up to date", for example, is too vague. -# -run git -C "$src_dir" pull >&2 -run git -C "$dst_dir" pull >&2 - -# Load the source and destination repositories' submit configurations (section -# name/directory mappings and owners directory path). -# -# Each repository's settings are sourced into the temporary variables 'owners' -# and 'sections' and copied from there to source- and destination-specific -# variables. -# -declare owners -declare -A sections -source "$src_dir/submit.config.bash" - -src_owners="$owners" -declare -A src_sections -for s in "${!sections[@]}"; do - src_sections["$s"]="${sections[$s]}" -done - -owners= -sections=() -source "$dst_dir/submit.config.bash" - -dst_owners="$owners" -declare -A dst_sections -for s in "${!sections[@]}"; do - dst_sections["$s"]="${sections[$s]}" -done - -# Find all archive and owner manifest files in the source repository. -# -# Every file in a repository section directory except *.manifest is a package -# archive and every file in the owners directory is a project or package owner -# manifest. Therefore run find separately on each section directory and the -# owners directory to build a list containing only package-related files. -# -# Store the relative to the repository directory file paths in an array used -# to build the set of pending commits. -# -src_files=() -for s in "${src_sections[@]}"; do - while read f; do - src_files+=("${f#$src_dir/}") - done < <(find "$src_dir/$s" -type f -not -name "*.manifest") -done - -if [[ -n "$src_owners" && -d "$src_dir/$src_owners" ]]; then - while read f; do - src_files+=("${f#$src_dir/}") - done < <(find "$src_dir/$src_owners" -type f) -fi - -# Build the set of pending commit hashes ("pending set"). -# -# For each file in the source repository, find the most recent commit that -# added it and store its abbreviated hash (as key) inside the 'pending_set' -# associative array (note: unordered) and (as value) inside the 'file_commits' -# associative array. -# -# 'file_commits' maps from the file path to the last commit to add it to the -# repository. A file may have been added and removed by earlier commits and -# could thus be migrated with the wrong commit unless care is taken (see -# migrate() for an example). -# -declare -A pending_set -declare -A file_commits -for f in "${src_files[@]}"; do - - # -n 1: limit output to one commit (that is, the most recent) - # --diff-filter=A: only show commits that added files - # --pretty=format:%h: output only the abbreviated commit hash - # - h="$(git -C "$src_dir" log -n 1 --diff-filter=A --pretty=format:%h -- "$f")" - - # Note that the hash cannot be empty because, after our clean checks at the - # top, every file on disk must have been added by some commit (that is, - # there can be no untracked files). - # - pending_set["$h"]=true - file_commits["$f"]="$h" -done - -# Arrange the pending commits in the chronological order. -# -# Go through the most recent commits in the git log which added one or more -# files, skipping those not present in the pending set and keeping count to -# bail out as soon as we ordered all of them. -# -pending_seq=() -for (( i=0; i != ${#pending_set[@]}; )); do - read h # The abbreviated commit hash. - - # If this is a pending commit, prepend its hash to the ordered array. - # - if [ "${pending_set[$h]}" ]; then - pending_seq=("$h" "${pending_seq[@]}") - ((++i)) - fi -done < <(git -C "$src_dir" log --diff-filter=A --pretty=format:%h) - -if [ "${#pending_seq[@]}" -eq 0 ]; then - info "Good news, nothing to manage!" - exit 0 -fi - -# Return the list of files a commit added to the source repository. -# -function commit_files () # -{ - local h="$1" - - # git-diff-tree arguments: - # - # --diff-filter=A: select only files that were added. - # -z: don't munge file paths and separate output fields with - # NULs. - # -r: recurse into subtrees (directories). - # - git -C "$src_dir" diff-tree \ - --no-commit-id --name-only --diff-filter=A -z -r \ - "$h" -} - -# Extract the package name, version, and project from a package archive's -# manifest and print it to stdout in the ' ' form. If -# the manifest does not specify the project name, the package name is returned -# as the project name. -# -function extract_pkg_info () # -{ - local arc="$1" - - local r - r=($(bpkg_rep_pkg_verify_archive "$arc")) # - if [ ! -v r[2] ]; then - r[2]="${r[0]}" - fi - - # Verify that the archive parent directory name matches the project. - # - local p="${r[2]}" - if [ "$p" != "$(basename "$(dirname "$arc")")" ]; then - error "'$arc' archive directory name does not match package project '$p'" - fi - - echo -n "${r[@]}" -} - -# Migrate a package archive or ownership manifest file from the source -# repository to the destination repository. -# -# is the path of the source file, relative to the source respository -# directory. For example, '1/stable/foo/foo-1.2.3.tar.gz', -# 'owners/foo/project-owner.manifest', or -# 'owners/foo/foo/package-owner.manifest'. -# -# is the path of the destination directory, relative to the destination -# repository directory. For example, '1/testing/foo', 'ownership/foo', or -# 'ownership/foo/foo'. -# -# Note that the source and destination sections and owners directories may -# differ (as they do in these examples) which is why those components must -# be specified in both the source and destination paths. -# -# Move the file from the source repository directory to the destination -# repository directory, creating directories if required; stage the addition -# of the file to the destination repository; stage the removal of the file -# from the source repository. -# -function migrate_file () # -{ - local src="$1" - local dst="$2" - - mkdir -p "$dst_dir/$dst" - mv "$src_dir/$src" "$dst_dir/$dst/" - run git -C "$src_dir/" rm --quiet "$src" - run git -C "$dst_dir/" add "$dst/$(basename "$src")" -} - -# Migrate: -# -# 0. Assumptions: -# -# - All the packages in a bundle are migrating from/to the same sections -# (enforce source part). -# -# - All the packages are from the same project (enforce). -# -# 1. Move files: -# -# - Owners to owners directory. -# -# - Packages into corresponding sections: -# -# alpha -> alpha -# beta -> beta -# stable -> testing|stable -# -# Bonus: replace revisions. -# Bonus: offer to drop existing packages if moving to alpha or beta. -# -# 2. Come up with commit message for src and commit. -# -# "Migrate to $dst_repo_name/
" -# -# "remove /" -# "remove owners//*" -# -# 3. Come up with commit message for dst and commit. -# -# "Migrate from $src_repo_name/
" -# -# "add /" -# "replace / with " (if replacing) -# "add owners//*" -# -# 4. Commit. -# -# Note that when migrating we will need to confirm with git that each of a -# commit's added files were actually most recently added by that commit. For -# example (oldest commits first): -# -# commit 1: add foo.tar.gz, bar.tar.gz -# commit 2: del foo.tar.gz -# commit 3: add foo.tar.gz -# -# If the user chooses to migrate commit 1, only bar.tar.gz must be migrated, -# despite foo.tar.gz existing on disk. -# -# The commit bundle associative array is the set of selected pending -# commits. Its keys are the corresponding indexes of the 'pending_seq' array -# (but offset by +1 and formatted to match the displayed commit numbers). -# Note: the reason the commit bundle is an associative array is to prevent -# duplicates. -# -declare -A bundle - -# Migrate the selected commit bundle from the source repository to the -# destination repository. Set the global migrate_result variable to true if -# the migration has been successful, or issue appropriate diagnostics and set -# it to the empty string if any of the following is true: -# -# - The commit bundle is empty. -# -# - Files added by commits in the bundle are not from the same project (the -# "bundle project") or, in the case of archives, the same repository section -# (the "bundle section"). -# -# - The required section does not exist in the destination repository. -# -# - An identical package archive (same name and version) already exists in the -# destination repository. -# -# - Any file has an invalid path (for example, missing a valid project or -# section component). -# -# The migration process proceeds as follows: -# -# - Move files: all of the files in the selected commit bundle are moved from -# the source repository into the destination repository. -# -# Package archives may be removed and ownership manifests overwritten at the -# destination. Candidate files for replacement are selected as follows: -# -# - In the alpha and beta sections, any package archive files in the -# destination section directory belonging to the same package are -# considered for replacement, regardless of their versions. -# -# - In other sections, any package archives in the destination section -# directory with the same name and version but a different revision -# (currently whether lower or higher) are automatically replaced. -# -# - Project or package ownership manifests will be replaced (that is, simply -# overwritten) at the destination with any ownership manifests added by -# the commit bundle because their presence implies that ownership -# information has changed. -# -# Stage (but don't commit) the removal of the files from the source -# repository and their addition to the destination repository. -# -# - Make commits to the source and destination respositories with appropriate -# commit messages. -# -# If any part of the migration fails then all changes to the source and -# destination repositories are undone, leaving two clean repositories. -# -function migrate () -{ - migrate_result= - - if [ "${#bundle[@]}" -eq 0 ]; then - info "no commits selected" - return - fi - - # Check that every commit's added files are in the bundle section and/or - # bundle project before migrating any of them. Build the bundle's list of - # files as we go along, classifying them as package archives or ownership - # manifests based on their paths. - # - # The bundle section is derived from the first package archive encountered - # and the bundle project from the first package archive or owner manifest - # encountered. - # - # Note that the bundle traversal is unordered. - # - local src_sect= # Source section name. - local src_sect_dir= # Source section directory. - local proj= # The bundle (source) project. - local pkgs=() # The bundle's archive files. - local owns=() # The bundle's ownership manifests. - - local i - for i in "${!bundle[@]}"; do - local h="${pending_seq[i-1]}" # The current commit's abbreviated hash. - - # Check the files added by the current commit. - # - local f - while read -d '' f; do - if [ "${file_commits[$f]}" != "$h" ]; then - continue # This file was deleted by a subsequent commit. - fi - - # Derive the project and/or section names from the file path. - # - # The project name is taken directly from the file path. In the case of - # package archives, the section name is the key in the 'src_sections' - # associative array which maps to the section directory extracted from - # the file path. - # - local fproj= # Current file's project. - - if [[ -n "$src_owners" && ("$f" =~ ^"$src_owners"/([^/]+)/.+$) ]]; then - fproj="${BASH_REMATCH[1]}" - owns+=("$f") - elif [[ "$f" =~ ^(.+)/([^/]+)/[^/]+$ ]]; then # Package archive? - local fsect_dir="${BASH_REMATCH[1]}" - - fproj="${BASH_REMATCH[2]}" - pkgs+=("$f") - - # Find the archive section name associated with the extracted section - # directory in 'src_sections' (a value-to-key lookup). - # - local fsect= - - # The "*" key is a catch-all for unknown submitted section names and, - # if present, will share a value (section directory) with one of the - # known section names and therefore must be skipped. - # - # If there is no mapping in 'src_sections' to the extracted section - # directory then the file path is invalid. - # - local k - for k in "${!src_sections[@]}"; do - if [[ ("${src_sections[$k]%/}" == "$fsect_dir") && - ("$k" != "*") ]]; then - fsect="$k" # Current file's section name. - break - fi - done - - if [ -z "$fsect" ]; then - info "unable to find section name for file '$f'" - return - fi - - # Set the source section name and directory if unset; otherwise fail - # if the current file is not from the source section. - # - if [ -z "$src_sect" ]; then - src_sect="$fsect" - src_sect_dir="$fsect_dir" - elif [ "$fsect" != "$src_sect" ]; then - info "cannot include commit $i: '$f' is not in section $src_sect" - return - fi - else - info "unrecognized type of file '$f'" - return - fi - - # Set the bundle project if unset; otherwise fail if the current file is - # not from the bundle project. - # - # Note: $fproj cannot be empty here (see above). - # - if [ -z "$proj" ]; then - proj="$fproj" - elif [ "$fproj" != "$proj" ]; then - info "cannot include commit $i: '$f' is not in project $proj" - return - fi - done < <(commit_files "$h") - done - - # Finalize migration variables the values of which depend on whether the - # bundle contains at least one package archive or ownership manifests only. - # - # The source and destination commit messages are composed incrementally as - # the migration process proceeds. - # - local dst_sect # Destination section name. - local dst_sect_dir # Destination section directory. - local src_cmsg # Source commit message. - local dst_cmsg # Destination commit message. - - if [ ${#pkgs[@]} -ne 0 ]; then # Bundle contains package archive(s). - dst_sect="$src_sect" - - # If it exists, 'testing' overrides 'stable' at the destination. - # - if [[ ("$dst_sect" == "stable") && -v dst_sections["testing"] ]]; then - dst_sect="testing" - fi - - # Fail if the target section does not exist in the destination repository. - # - if [ ! -v dst_sections["$dst_sect"] ]; then - info "section '$dst_sect' does not exist in the destination repository" - return - fi - - dst_sect_dir="${dst_sections[$dst_sect]}" - - src_cmsg="Migrate $proj to $dst_repo_name/$dst_sect"$'\n\n' - dst_cmsg="Migrate $proj from $src_repo_name/$src_sect"$'\n\n' - else # Bundle consists only of ownership manifests. - - # The setup where the ownership authentication is disabled on the - # destination but enabled on source is probably obscure, but let's - # consider it possible since the submit-git handler allows such a setup. - # - if [ -n "$dst_owners" ]; then - src_cmsg="Migrate $proj ownership info to $dst_repo_name"$'\n\n' - dst_cmsg="Migrate $proj ownership info from $src_repo_name"$'\n\n' - else - src_cmsg="Remove $proj ownership info"$'\n\n' - dst_cmsg= # Nothing to commit. - fi - fi - - # Ensure that the source and destination repositories are clean if the - # migration of any file fails. - # - # Note that the source repository cannot have untracked files so we - # git-clean only the destination repository. - # - function cleanup () - { - info "migration failed; resetting and cleaning repositories" - - if ! run git -C "$src_dir" reset --hard || - ! run git -C "$dst_dir" reset --hard || - ! run git -C "$dst_dir" clean --force; then - info "failed to reset/clean repositories -- manual intervention required" - fi - } - trap cleanup EXIT - - # Migrate the bundle's package archive files. - # - for f in "${pkgs[@]}"; do - # Get the current package's name and version from its embedded manifest - # (we already have the source project in $proj). - # - local p - p=($(extract_pkg_info "$src_dir/$f")) - - local name="${p[0]}" - local src_version="${p[1]}" - - # Check for duplicate package in all sections. Use -.* - # without .tar.gz in case we want to support more archive types later. - # - # Note that, for example, foo-bar version 1.0 and foo version bar-1.0 have - # the same archive name foo-bar-1.0.tar.gz. - # - local s - for s in "${!dst_sections[@]}"; do - local p - IFS=$'\n' eval \ - 'p=($(bpkg_rep_pkg_find_archive "$name-$src_version.*" \ - "$dst_dir/${dst_sections[$s]}"))' - - if [ "${#p[@]}" -ne 0 ]; then - local n="${p[0]}" - local v="${p[1]}" - local a="${p[3]}" - - if [ "$n" == "$name" ]; then - error "duplicate of $name/$src_version at '$a'" - else - error "conflict of $name/$src_version with $n/$v at '$a'" - fi - fi - done - - # In the destination repository, find and remove package archive files - # which are other alpha/beta versions or revisions of the current source - # package. - # - local vpat # Version pattern. - case "$dst_sect" in - alpha|beta) vpat="*" ;; # All package versions. - *) vpat="$src_version*" ;; # All package version revisions. - esac - - # Packages in the destination repository to be considered for replacement. - # - local dst_files - - IFS=$'\n' eval \ - 'dst_files=($(bpkg_rep_pkg_find_archives "$name" \ - "$vpat" \ - "$dst_dir/$dst_sect_dir"))' - - # If true, the source package replaces one or more packages in the - # destination repository. - # - local repl= - - local dst_f - for dst_f in "${dst_files[@]}"; do - local p - p=($(extract_pkg_info "$dst_f")) - - local dst_version="${p[1]}" - local dst_project="${p[2]}" - - # Ask whether or not to drop the current destination package. - # - # Include the project names in the prompt if the destination package's - # project differs from that of the source package. - # - local src="$src_version" - local dst="$name/$dst_version" - if [ "$dst_project" != "$proj" ]; then - src+=" ($proj)" - dst+=" ($dst_project)" - fi - - while true; do - read -p "replace $dst with $src? [y/n]: " opt - - case "$opt" in - "y") - repl=true - dst_cmsg+=" replace $name/$dst_version with $src_version"$'\n' - run git -C "$dst_dir" rm --quiet "${dst_f#$dst_dir/}" - break - ;; - "n") - break - ;; - esac - done - done - - # Migrate the current package. - # - src_cmsg+=" remove $name/$src_version"$'\n' - if [ ! "$repl" ]; then - dst_cmsg+=" add $name/$src_version"$'\n' - fi - migrate_file "$f" "$dst_sect_dir/$proj" - done - - # Migrate the bundle's ownership manifests. - # - # If ownership authentication is disabled on the destination repository, - # only remove ownership manifests from the source repository (that is, don't - # migrate). - # - for f in "${owns[@]}"; do - src_cmsg+=" remove $(dirname $f)/*"$'\n' - - if [ -n "$dst_owners" ]; then - local dp=$(dirname "${f/$src_owners/$dst_owners}") # Destination path. - - # Let the commit message reflect whether this is a new ownership - # manifest or is replacing an existent one. - # - if [ ! -e "$dst_dir/$dp/$(basename "$f")" ]; then - dst_cmsg+=" add $dp/*"$'\n' - else - dst_cmsg+=" update $dp/*"$'\n' - fi - - migrate_file "$f" "$dp" - else - run git -C "$src_dir/" rm --quiet "$f" - fi - done - - # Commit the changes made to the source and destination repositories. - # - info - run git -C "$src_dir" commit -m "$src_cmsg" - - if [ -n "$dst_cmsg" ]; then - info - run git -C "$dst_dir" commit -m "$dst_cmsg" - fi - - info - - # Remove the migrated commits from the pending sequence and clear the - # bundle. - # - for i in "${!bundle[@]}"; do - unset pending_seq[i-1] - done - pending_seq=("${pending_seq[@]}") # Remove the gaps created by unset. - bundle=() - - migrate_result=true - - # All files have been migrated successfully so clear the EXIT trap. - # - trap EXIT - - # Pause to give the operator a chance to look at the commits before the list - # of remaining pending commits is displayed. - # - read -p "Press Enter to continue: " -} - -# Push local changes to the remote source and/or destination git repositories. -# -# Push to the destination repository first because thus the migrated files -# will be in both remote repositories until the completion of the subsequent -# push to the source repository (which may fail or take long). Although this -# is an inconsistent state, it is safe because other programs such as a -# submission handler will be able to detect the duplicates and therefore -# refuse to do anything. If, on the other hand, we pushed to the source first, -# the migrated files would not exist in either remote repository until the -# push to the destination repository completed. In this state the submission -# handler would, for example, accept a resubmission of the migrated packages or -# erroneously establish ownership for already owned project/package names. -# -function push () -{ - # Let's print additional diagnostics on git-push failure, to emphasize for - # the user which of the two repositories we have failed to push. - # - if ! run git -C "$dst_dir" push; then - error "push to $dst_repo_name failed" - fi - - if ! run git -C "$src_dir" push; then - error "push to $src_repo_name failed" - fi -} - -# Present the list of pending commits to the user, oldest first, marking files -# that were deleted by subsequent commits with `*`: -# -# 001 (deadbeef) Add libfoo/1.2.3 -# -# 1/testing/foo/libfoo-1.2.3.tar.gz -# owners/foo/project-owner.manifest -# owners/foo/libfoo/package-owner.manifest -# -# 002 (c00l0fff) Add bar/1.2.3 -# -# * 1/testing/bar/libbar-1.2.3.tar.gz -# 1/testing/bar/libbaz-1.2.3.tar.gz -# -# 003 (deadbabe) Add libbar/1.2.3+1 -# -# 1/testing/bar/libbar-1.2.3+1.tar.gz -# -# Note that files deleted by subsequent commits may still be in the -# repository. See migrate() for an example. -# -# Then prompt the user for the action (showing the current bundle): -# -# [001 002][,m,c,p,q,l,?]: -# -# - add commit to the commit bundle -# m - migrate the selected commit bundle -# c - clear the selected commit bundle -# p - push source and destination repositories -# l - print pending commits -# q - quit (prompting to push if any actions have been taken) -# ? - print this help -# -# The user interaction loop. -# -# In each iteration, present the list of pending commits, display the menu of -# actions, read the user's input, and perform the chosen action. -# -# True if any changes have been made to the source and/or destination git -# repositories (in which case the user will be asked whether or not to push -# before quitting). -# -need_push= - -while true; do - # Show the pending commits. - # - if [ "${#pending_seq[@]}" -eq 0 ]; then - info "no more pending commits" - fi - - for ((i=0; i != "${#pending_seq[@]}"; i++)); do - h="${pending_seq[$i]}" - - # Print commit number, hash, and subject. - # - # The commit number is left-padded with 0s to 3 digits. Prefix with a - # newline to separate the first commit from the git-pull output and the - # rest from the previous commit info block. - # - subj="$(git -C "$src_dir" log -n 1 --pretty=format:%s "$h")" - printf "\n%.3d (%s) %s\n\n" "$((i+1))" "$h" "$subj" >&2 - - # Print this commit's files. - # - # Fetch from the git repository the list of files added by the current - # commit. Print each file's path and, if it was deleted by a subsequent - # commit, mark with an asterisk. - # - # Note that 'file_commits' is populated above from the list of files - # currently in the source repository. Therefore, if git says a file was - # added by a commit but it is associated with a different commit hash in - # 'file_commits' it means the file was deleted and added back by later - # commits; and if there is no mapping for the file it means it was deleted - # but not added back (that is, it's no longer in the repository). So we - # mark the re-added file with an exclamation. - # - while read -d '' f; do - if [ "${file_commits[$f]}" == "$h" ]; then - info " $f" # File was last added by the current commit. - elif [ -v file_commits["$f"] ]; then - info " ! $f" # File was deleted and added back by subsequent commits. - else - info " * $f" # File was deleted but not added back. - fi - done < <(commit_files "$h") - done - - # Prompt the user for the action (showing the current bundle), get user - # input, and perform the selected action. - # - # Note that we could adapt the menu according to the current state (don't - # offer to migrate if the bundle array is empty, etc) but let's not - # complicate the logic. - # - # Breaking out of this loop prints the pending commit list again. - # - while true; do - # Sort commit bundle in ascending order. - # - # Expand the 'bundle' associative array's keys into a single word in which - # they are separated by spaces (the first member of IFS) using the - # ${!a[*]} syntax; replace each space with a newline before piping to - # 'sort', which is newline-based; finally collect sort's output into an - # array using the a=() syntax, which splits on newline (the last member of - # IFS) because neither space nor tab characters (the other members of IFS) - # can occur in the keys. - # - bundle_sorted=($(sed 's/ /\n/g' <<<"${!bundle[*]}" | sort -)) - - printf "\n" - read -p "[${bundle_sorted[*]}][,m,c,p,l,q,?]: " opt - - case "$opt" in - # Add commit to bundle. - # - [0-9]*) - if [[ ("$opt" -gt 0) && ("$opt " -le "${#pending_seq[@]}") ]]; then - printf -v opt "%.3d" "$opt" # Format as in pending commit list. - if [ ! -v bundle["$opt"] ]; then - bundle["$opt"]=true - info "commit $opt (${pending_seq[$opt-1]}) added to selected bundle" - else - info "commit $opt is already in the bundle" - fi - else - info "non-existent commit number $opt" - fi - ;; - # Migrate the commit bundle. - # - m) - migrate - if [ "$migrate_result" ]; then - need_push=true - break - fi - ;; - # Clear the commit bundle. - # - c) - bundle=() - break - ;; - # Push changes. - # - p) - push - need_push= - break - ;; - # Redraw the pending commit list. - # - l) - break - ;; - # Quit. - # - q) - if [ ! "$need_push" ]; then - exit 0 - fi - - while true; do - read -p "push changes? [y/n/(c)ancel]: " opt - - case "$opt" in - "c") - break # Print options menu again. - ;; - "y") - push - exit 0 - ;; - "n") - exit 0 - ;; - *) - continue - ;; - esac - done - ;; - # ? or invalid option: print menu. - # - *) - cat <<-EOF - - - add commit to the commit bundle - m - migrate the selected commit bundle - c - clear the selected commit bundle - p - push source and destination repositories - l - print pending commits - q - quit (prompting to push if any actions have been taken) - ? - print this help -EOF - ;; - esac - done -done diff --git a/bpkg-rep/package-archive.bash.in b/bpkg-rep/package-archive.bash.in deleted file mode 100644 index fd9497a..0000000 --- a/bpkg-rep/package-archive.bash.in +++ /dev/null @@ -1,170 +0,0 @@ -# file : bpkg-rep/package-archive.bash.in -# license : MIT; see accompanying LICENSE file - -# Utility functions useful for managing package archives. - -if [ "$bpkg_rep_package_archive" ]; then - return 0 -else - bpkg_rep_package_archive=true -fi - -@import libbutl/manifest-parser@ - -# We expect the user to set the bpkg_rep_bpkg variable to the bpkg program -# path. -# -if [ ! -v bpkg_rep_bpkg ]; then - echo "error: variable bpkg_rep_bpkg is not set" >&2 - exit 1 -fi - -# Extract the package information from a package archive and print it in the -# ' ' form, where the project field is empty if the -# project value is not specified in the manifest. -# -# Note that, in particular, it verifies that the archive file name matches the -# package name and version. -# -function bpkg_rep_pkg_verify_archive () # -{ - # We can't use the process substitution for input redirect here, since such - # a process failure is not trapped. Thus, read the manifest file into a - # variable and parse it afterwards, which is probably ok since package - # manifests are normally not too big. - # - # Note that alternatively we could use the process substitution for running - # bpkg, treat the name value absence as indication of a failure, and exit - # with non-zero status if that's the case. Feels a bit hackish though. - # - local m - m="$("$bpkg_rep_bpkg" pkg-verify --manifest "$1")" - - butl_manifest_parser_start <<<"$m" - - local name= - local version= - local project= - - local n v - while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do - case "$n" in - name) name="$v" ;; - version) version="$v" ;; - project) project="$v" ;; - esac - done - - butl_manifest_parser_finish - - echo -n "$name $version $project" -} - -# Search for package archives in a directory using the package name and -# version pattern and printing their paths newline-separated. If the version -# argument is '*', then print archives for all package versions. Otherwise if -# the version contains the trailing '*', then print archives for all revisions -# of the specified version and for the exact version otherwise. For example: -# -# bpkg_rep_pkg_find_archives foo '*' dir/ -# bpkg_rep_pkg_find_archives foo '1.0*' dir/ -# bpkg_rep_pkg_find_archives foo '1.0' dir/ -# -# Note that the resulting archive paths include the specified directory as a -# prefix. -# -# NOTE: this function can be called with overriden IFS. -# -function bpkg_rep_pkg_find_archives () # -{ - IFS=$' \t\n' bpkg_rep_pkg_find_archives_impl "$@" -} - -function bpkg_rep_pkg_find_archives_impl () -{ - local nam="$1" - local ver="$2" - local dir="$3" - - local r="" - - if [ -d "$dir" ]; then - local vr # Version with the revision stripped, if search for revisions. - local np # File name pattern for archives search. - - if [ "$ver" != "*" -a "${ver: -1}" == "*" ]; then # * - vr="$(sed -n -re 's%^(\+?[^+]+)(\+[0-9]+)?\*$%\1%p' <<<"$ver")" - np="$nam-$vr*.*" # foo-1.0*.*, etc. - else # * or - np="$nam-$ver.*" # foo-*.*, foo-1.0.*, etc. - fi - - # Go through the potentially matching archives (for example, for 'foo' - # '1.2.3+2*': foo-1.2.3.tar.gz, foo-1.2.3+1.tar.gz, foo-1.2.30.tar.gz, - # etc) and return those which package name and version match properly. - # - local f - while read f; do - local p - p=($(bpkg_rep_pkg_verify_archive "$f")) - - local n="${p[0]}" - local v="${p[1]}" - - if [[ "$n" == "$nam" && - ( "$ver" == "*" || \ - "$v" == "$ver" || \ - ( -n "$vr" && "$v" =~ ^"$vr"(\+[0-9]+)?$ )) ]]; then - - if [ -n "$r" ]; then - r="$r"$'\n'"$f" - else - r="$f" - fi - fi - done < <(find "$dir" -type f -name "$np") - fi - - if [ -n "$r" ]; then - echo -n "$r" - fi -} - -# Search for a package archive in a directory using a file name pattern. If -# the archive is found, then print the package information in the -# '\n\n\n' form, where the project field is -# empty if the project value is not specified in the manifest. -# -# Note that if there are multiple archives matching the pattern, then it is -# unspecified which one is picked. -# -# NOTE: this function can be called with overriden IFS. -# -function bpkg_rep_pkg_find_archive () # -{ - IFS=$' \t\n' bpkg_rep_pkg_find_archive_impl "$@" -} - -function bpkg_rep_pkg_find_archive_impl () -{ - local pat="$1" - local dir="$2" - - if [ -d "$dir" ]; then - local f - - # We could probably use -print -quit but this is not portable (NetBSD - # needs -exit instead of -quit). - # - f="$(find "$dir" -type f -name "$pat" | head -n 1)" - - if [ -n "$f" ]; then - - local p - p=($(bpkg_rep_pkg_verify_archive "$f")) - - printf "${p[0]}\n${p[1]}\n${p[2]}\n$f" - return - fi - fi -} diff --git a/bpkg-rep/publish.in b/bpkg-rep/publish.in deleted file mode 100644 index a7b3d1a..0000000 --- a/bpkg-rep/publish.in +++ /dev/null @@ -1,329 +0,0 @@ -#!/usr/bin/env bash - -# file : bpkg-rep/publish.in -# license : MIT; see accompanying LICENSE file - -# Update (bpkg-rep-create(1)) and publish (rsync(1)) an archive-based -# repository. -# -# Pull a pre-cloned (read-only) git repository with the contents of an -# archive-based bpkg repository. Bail out if nothing changed from the the -# previous run. Otherwise, regenerate the repository meta-data by running -# bpkg-rep-create(1) on each section of the repository and, optionally, -# synchronize it to one or more destinations with rsync. -# -# The repository contents are expected to be in the /1/ subdirectory. The -# script saves the last successfully published commit in the .publish -# file. -# -# --destination|-d : -# -# Remote host and directory to rsync the repository to. Note that the -# directory should include the 1/ component and any sub-directories that may -# follow. In other words, the rsync command will be in the form: -# -# rsync ... /1/ :/ -# -# See below for the actual rsync command including a brief explanation of -# options passed. -# -# Repeat this option to specify multiple destinations. In this case, the -# destinations are synced in the order specified with the first failure -# terminating the process (so if you have a "primary" destination and a -# "mirror", you probably want to specify the former first). -# -# --timeout -# -# Git and rsync operation timeout. Specifically, the operation will be -# aborted if there is no network activity for the specified time. Default is -# 60 seconds. Note that currently the git timeout is only supported for the -# http(s) transport. -# -# --lock-timeout -# -# The repository lock timeout. Fail if another instance of the script does -# not release the repository in the specified time. The default is 0 (do -# not wait). -# -# Note that you will most likely want to specify a non-zero timeout for cron -# jobs that may potentially overlap. -# -# --log-dir -# -# Directory to create the temporary log files in. If unspecified, the stderr -# is not redirected and no log is created by default. -# -# The log is dumped to stderr in case of an error or at the end of execution -# unless in the quiet mode and is then deleted. -# -# --quiet -# -# Run quiet. Specifically, don't dump the log to stderr on exit with zero -# status. -# -# --config -# -# The configuration file containing a bash fragment. Repeat this option to -# specify multiple configurations that will be sourced in the order -# specified. -# -# --bpkg -# -# The package manager program to be used for the repository update. This -# should be the path to the bpkg executable. -# -usage="usage: $0 [] [] [-- ]" - -trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. - -@import bpkg-rep/utility@ - -# The script own options. -# -repo_ver=1 -destinations=() -timeout=60 -lock_timeout=0 -log_dir= -quiet= -configurations=() -bpkg= - -while [ $# -gt 0 ]; do - case $1 in - --destination|-d) - shift - destinations+=("${1%/}") - shift || true - ;; - --timeout) - shift - timeout="$1" - shift || true - ;; - --lock-timeout) - shift - lock_timeout="$1" - shift || true - ;; - --log-dir) - shift - log_dir="${1%/}" - shift || true - ;; - --quiet) - shift - quiet=true - ;; - --config) - shift - configurations+=("$1") - shift || true - ;; - --bpkg) - shift - bpkg="$1" - shift || true - ;; - --) - shift - break - ;; - *) - break - ;; - esac -done - -# The repository directory. -# -repo_dir="${1%/}" -shift || true - -# bpkg-rep-create options. -# -rep_create_options=() - -while [ $# -gt 0 ]; do - case $1 in - --) - shift - break - ;; - *) - rep_create_options+=("$1") - shift - ;; - esac -done - -# rsync options. -# -rsync_options=() - -while [ $# -gt 0 ]; do - rsync_options+=("$1") - shift -done - -# Validate options and arguments. -# -if [ -z "$repo_dir" ]; then - error "$usage" -fi - -if [ ! -d "$repo_dir" ]; then - error "'$repo_dir' does not exist or is not a directory" -fi - -# If the log directory is specified then redirect stderr to the log file and -# setup the trap that dumps it on exit, if required. -# -if [ -n "$log_dir" ]; then - - if [ ! -d "$log_dir" ]; then - error "'$log_dir' does not exist or is not a directory" - fi - - # Create the log file. - # - log="$(mktemp "$log_dir/$(basename "$repo_dir").XXXXXXXXXX")" - - # Save the stderr file descriptor so we can dump the log into it on exit, if - # required. Then redirect it to the log file. - # - exec {stderr}>&2 - exec 2>>"$log" - - function exit_trap () - { - local status="$?" - - # Dump the log to stderr if exiting with non-zero status or verbose. - # - if [ $status -ne 0 -o ! "$quiet" ]; then - - # Keep the log if failed to dump for any reason. - # - if ! cat "$log" >&$stderr; then - return - fi - fi - - rm -f "$log" - } - - trap exit_trap EXIT -fi - -# Source the configurations. -# -for c in "${configurations[@]}"; do - source "$c" >&2 -done - -# Make sure the commit file is present. -# -published_commit="$repo_dir.publish" -touch "$published_commit" - -# Open the reading file descriptor and lock the repository. Fail if unable to -# lock before timeout. -# -exec {cfd}<"$published_commit" - -if ! flock -w "$lock_timeout" "$cfd"; then - info "another instance is already running" - exit 2 -fi - -# Pull the repository. -# -# Git doesn't support the connection timeout option. The options we use are -# just an approximation of the former, that, in particular, don't cover the -# connection establishing. To work around this problem, before running a git -# command that assumes the remote repository communication we manually check -# connectivity with the remote repository. -# -if ! remote_url="$(git -C "$repo_dir" config --get remote.origin.url)"; then - error "'$repo_dir' is not a git repository" -fi - -run check_git_connectivity "$remote_url" "$timeout" - -# Fail if no network activity happens during the time specified. -# -run git -c http.lowSpeedLimit=1 -c "http.lowSpeedTime=$timeout" \ --C "$repo_dir" pull -v >&2 - -# Match the HEAD commit id to the one stored in the file. If it matches, then -# nothing changed in the repository from the previous run and we can silently -# bail out. -# -commit="$(git -C "$repo_dir" rev-parse HEAD)" -pc="$(cat <&"$cfd")" - -if [ "$commit" == "$pc" ]; then - quiet=true - exit 0 -fi - -# If bpkg path is not specified, then use the bpkg program from the script -# directory, if present. Otherwise, use the 'bpkg' path. -# -if [ -z "$bpkg" ]; then - bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" - - if [ ! -x "$bpkg" ]; then - bpkg=bpkg - fi -fi - -# Find repository sections. -# -manifests="$(find "$repo_dir/$repo_ver" -type f -name repositories.manifest)" - -# Update the repository sections. -# -while read f; do - run "$bpkg" rep-create "${rep_create_options[@]}" "$(dirname "$f")" -done <<<"$manifests" - -# rsync (over ssh) the repository to the destinations. -# -# Approximate the data transfer timeout via the ServerAlive* ssh options, -# rounding the timeout up to the nearest multiple of ten. -# -# Note: must not contain spaces/use quoting (see rsync -e option). -# -n=$(($timeout > 0 ? ($timeout + 9) / 10 : 1)) -ssh_options=(-o ConnectTimeout=$timeout \ - -o ServerAliveInterval=10 \ - -o ServerAliveCountMax=$n) - -for d in "${destinations[@]}"; do - - # -r (recursive) - # -l (copy symlinks as symlinks) - # -t (preserve timestamps) - # -O (omit dir timestamps) - # - # -c (use checksum) - # -e (remote shell command) - # - # --chmod=ugo=rwX (give new files the destination-default permissions) - # --safe-links (ignore symlinks pointing outside the tree) - # --delay-updates (first upload all files on the side then move) - # --prune-empty-dirs (remove empty dirs) - # --delete-after (delete entries after the transfer) - # - # We also exclude hidden files (start with dot). - # - run rsync -v -rltO -c --chmod=ugo=rwX --safe-links --delay-updates \ ---exclude '.*' --prune-empty-dirs --delete-after -e "ssh ${ssh_options[*]}" \ -"${rsync_options[@]}" "$repo_dir/$repo_ver/" "$d/" >&2 - -done - -echo "$commit" >"$published_commit" diff --git a/bpkg-rep/utility.bash.in b/bpkg-rep/utility.bash.in deleted file mode 100644 index 7ac7c54..0000000 --- a/bpkg-rep/utility.bash.in +++ /dev/null @@ -1,86 +0,0 @@ -# file : bpkg-rep/utility.bash.in -# license : MIT; see accompanying LICENSE file - -# Utility functions useful for implementing bpkg repository utilities. - -if [ "$bpkg_rep_utility" ]; then - return 0 -else - bpkg_rep_utility=true -fi - -# Diagnostics. -# -function info () { echo "$*" 1>&2; } -function error () { info "$*"; exit 1; } - -# Trace a command line, quoting empty arguments as well as those that contain -# spaces. -# -function trace_cmd () # ... -{ - local s="+" - while [ $# -gt 0 ]; do - if [ -z "$1" -o -z "${1##* *}" ]; then - s="$s '$1'" - else - s="$s $1" - fi - - shift - done - - info "$s" -} - -# Trace the current function name and arguments. -# -function trace_func () # ... -{ - trace_cmd "${FUNCNAME[1]}" "$@" -} - -# Trace and run a command. -# -function run () # ... -{ - trace_cmd "$@" - "$@" -} - -# Return lower-case URL scheme or empty string if the argument doesn't look -# like a URL. -# -function url_scheme () # -{ - sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1" -} - -# Check that the git repository properly responds to the probing request -# before the timeout (in seconds). Noop for protocols other than HTTP(S). -# -function check_git_connectivity () # -{ - local url="$1" - local tmo="$2" - - local s - s="$(url_scheme "$url")" - - if [ "$s" == "http" -o "$s" == "https" ]; then - local u q - - u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part. - q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part. - - if [ -z "$q" ]; then - u="$u/info/refs?service=git-upload-pack" - else - u="$u/info/refs$q&service=git-upload-pack" - fi - - # Here we limit the time for the whole operation. - # - curl -S -s --max-time "$tmo" "$u" >/dev/null - fi -} diff --git a/bpkg-util/.gitignore b/bpkg-util/.gitignore new file mode 100644 index 0000000..1b4745f --- /dev/null +++ b/bpkg-util/.gitignore @@ -0,0 +1,2 @@ +bpkg-util-publish +bpkg-util-manage diff --git a/bpkg-util/buildfile b/bpkg-util/buildfile new file mode 100644 index 0000000..271ea36 --- /dev/null +++ b/bpkg-util/buildfile @@ -0,0 +1,17 @@ +# file : bpkg-util/buildfile +# license : MIT; see accompanying LICENSE file + +import mods = libbutl.bash%bash{manifest-parser} + +./: exe{bpkg-util-publish bpkg-util-manage} bash{package-archive} + +exe{bpkg-util-publish}: in{publish} bash{utility} +exe{bpkg-util-manage}: in{manage} bash{utility package-archive} + +# Public modules. +# +bash{package-archive}: in{package-archive} $mods + +# Private modules. +# +bash{utility}: in{utility} diff --git a/bpkg-util/manage.in b/bpkg-util/manage.in new file mode 100644 index 0000000..65ab378 --- /dev/null +++ b/bpkg-util/manage.in @@ -0,0 +1,957 @@ +#!/usr/bin/env bash + +# file : bpkg-util/manage.in +# license : MIT; see accompanying LICENSE file + +# Interactively migrate newly-submitted packages from a source git repository +# to a destination git repository. +# +# Present the user with the list of commits that added the files currently in +# the source repository's working directory and ask the user to select from a +# menu the action to perform on a selection of these commits. +# +# As the files added by these commits are pending a move to the destination +# repository, these commits will be referred to as "pending" commits. +# +# Actions that can be performed on a selection of pending commits include +# moving them from the source repository to a single commit in the destination +# repository and dropping them from the source repository. +# +# The flow of this script, in broad strokes, is as follows: for each file in +# the source repository directory, find the hash of the commit that added it; +# these are the pending commits. Arrange the pending commits in chronological +# order. Display to the user the pending commits along with the files they +# added. Let the user select one or more pending commits and an action to be +# performed on them. Each successful action results in a commit to the source +# and/or destination repositories and leaves both repositories in a clean +# state. Once the action has been performed, redisplay the updated pending +# commit list and prompt for the next action. Pushing to the remote +# repositories, a supported operation, can be done at any time during the +# session. +# +# The directory into which the source and destination repositories have +# been checked out. If not specified, current directory is assumed. +# +usage="usage: $0 []" + +# Source/destination repository inside . Note: also used in commit +# messages. +# +src_repo_name=queue +dst_repo_name=public + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +@import bpkg-util/utility@ + +# Use the bpkg program from the script directory, if present. Otherwise, use +# just 'bpkg'. +# +bpkg_util_bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" + +if [ ! -x "$bpkg_util_bpkg" ]; then + bpkg_util_bpkg=bpkg +fi + +@import bpkg-util/package-archive@ + +# Set the working directory. +# +if [ $# -eq 0 ]; then + dir="$owd" +elif [ $# -eq 1 ]; then + dir="${1%/}" # with trailing slash removed. +else + error "$usage" +fi + +# The source and destination package repository directories. +# +# Note that, throughout this script, any path not explicitly prefixed with +# "$src_dir/" or "$dst_dir/" is relative to the root of the source or +# destination package repositories. +# +src_dir="$dir/$src_repo_name" +dst_dir="$dir/$dst_repo_name" + +if [ ! -d "$src_dir" ]; then + error "'$src_dir' does not exist or is not a directory" +fi + +if [ ! -d "$dst_dir" ]; then + error "'$dst_dir' does not exist or is not a directory" +fi + +# Check that both git repositories are clean. +# +if [ -n "$(git -C $src_dir status --porcelain)" ]; then + error "git repository in '$src_dir' is not clean" +fi + +if [ -n "$(git -C $dst_dir status --porcelain)" ]; then + error "git repository in '$dst_dir' is not clean" +fi + +# Use run() to show the user that git is the source of the diagnostics. +# "Already up to date", for example, is too vague. +# +run git -C "$src_dir" pull >&2 +run git -C "$dst_dir" pull >&2 + +# Load the source and destination repositories' submit configurations (section +# name/directory mappings and owners directory path). +# +# Each repository's settings are sourced into the temporary variables 'owners' +# and 'sections' and copied from there to source- and destination-specific +# variables. +# +declare owners +declare -A sections +source "$src_dir/submit.config.bash" + +src_owners="$owners" +declare -A src_sections +for s in "${!sections[@]}"; do + src_sections["$s"]="${sections[$s]}" +done + +owners= +sections=() +source "$dst_dir/submit.config.bash" + +dst_owners="$owners" +declare -A dst_sections +for s in "${!sections[@]}"; do + dst_sections["$s"]="${sections[$s]}" +done + +# Find all archive and owner manifest files in the source repository. +# +# Every file in a repository section directory except *.manifest is a package +# archive and every file in the owners directory is a project or package owner +# manifest. Therefore run find separately on each section directory and the +# owners directory to build a list containing only package-related files. +# +# Store the relative to the repository directory file paths in an array used +# to build the set of pending commits. +# +src_files=() +for s in "${src_sections[@]}"; do + while read f; do + src_files+=("${f#$src_dir/}") + done < <(find "$src_dir/$s" -type f -not -name "*.manifest") +done + +if [[ -n "$src_owners" && -d "$src_dir/$src_owners" ]]; then + while read f; do + src_files+=("${f#$src_dir/}") + done < <(find "$src_dir/$src_owners" -type f) +fi + +# Build the set of pending commit hashes ("pending set"). +# +# For each file in the source repository, find the most recent commit that +# added it and store its abbreviated hash (as key) inside the 'pending_set' +# associative array (note: unordered) and (as value) inside the 'file_commits' +# associative array. +# +# 'file_commits' maps from the file path to the last commit to add it to the +# repository. A file may have been added and removed by earlier commits and +# could thus be migrated with the wrong commit unless care is taken (see +# migrate() for an example). +# +declare -A pending_set +declare -A file_commits +for f in "${src_files[@]}"; do + + # -n 1: limit output to one commit (that is, the most recent) + # --diff-filter=A: only show commits that added files + # --pretty=format:%h: output only the abbreviated commit hash + # + h="$(git -C "$src_dir" log -n 1 --diff-filter=A --pretty=format:%h -- "$f")" + + # Note that the hash cannot be empty because, after our clean checks at the + # top, every file on disk must have been added by some commit (that is, + # there can be no untracked files). + # + pending_set["$h"]=true + file_commits["$f"]="$h" +done + +# Arrange the pending commits in the chronological order. +# +# Go through the most recent commits in the git log which added one or more +# files, skipping those not present in the pending set and keeping count to +# bail out as soon as we ordered all of them. +# +pending_seq=() +for (( i=0; i != ${#pending_set[@]}; )); do + read h # The abbreviated commit hash. + + # If this is a pending commit, prepend its hash to the ordered array. + # + if [ "${pending_set[$h]}" ]; then + pending_seq=("$h" "${pending_seq[@]}") + ((++i)) + fi +done < <(git -C "$src_dir" log --diff-filter=A --pretty=format:%h) + +if [ "${#pending_seq[@]}" -eq 0 ]; then + info "Good news, nothing to manage!" + exit 0 +fi + +# Return the list of files a commit added to the source repository. +# +function commit_files () # +{ + local h="$1" + + # git-diff-tree arguments: + # + # --diff-filter=A: select only files that were added. + # -z: don't munge file paths and separate output fields with + # NULs. + # -r: recurse into subtrees (directories). + # + git -C "$src_dir" diff-tree \ + --no-commit-id --name-only --diff-filter=A -z -r \ + "$h" +} + +# Extract the package name, version, and project from a package archive's +# manifest and print it to stdout in the ' ' form. If +# the manifest does not specify the project name, the package name is returned +# as the project name. +# +function extract_pkg_info () # +{ + local arc="$1" + + local r + r=($(bpkg_util_pkg_verify_archive "$arc")) # + if [ ! -v r[2] ]; then + r[2]="${r[0]}" + fi + + # Verify that the archive parent directory name matches the project. + # + local p="${r[2]}" + if [ "$p" != "$(basename "$(dirname "$arc")")" ]; then + error "'$arc' archive directory name does not match package project '$p'" + fi + + echo -n "${r[@]}" +} + +# Migrate a package archive or ownership manifest file from the source +# repository to the destination repository. +# +# is the path of the source file, relative to the source respository +# directory. For example, '1/stable/foo/foo-1.2.3.tar.gz', +# 'owners/foo/project-owner.manifest', or +# 'owners/foo/foo/package-owner.manifest'. +# +# is the path of the destination directory, relative to the destination +# repository directory. For example, '1/testing/foo', 'ownership/foo', or +# 'ownership/foo/foo'. +# +# Note that the source and destination sections and owners directories may +# differ (as they do in these examples) which is why those components must +# be specified in both the source and destination paths. +# +# Move the file from the source repository directory to the destination +# repository directory, creating directories if required; stage the addition +# of the file to the destination repository; stage the removal of the file +# from the source repository. +# +function migrate_file () # +{ + local src="$1" + local dst="$2" + + mkdir -p "$dst_dir/$dst" + mv "$src_dir/$src" "$dst_dir/$dst/" + run git -C "$src_dir/" rm --quiet "$src" + run git -C "$dst_dir/" add "$dst/$(basename "$src")" +} + +# Migrate: +# +# 0. Assumptions: +# +# - All the packages in a bundle are migrating from/to the same sections +# (enforce source part). +# +# - All the packages are from the same project (enforce). +# +# 1. Move files: +# +# - Owners to owners directory. +# +# - Packages into corresponding sections: +# +# alpha -> alpha +# beta -> beta +# stable -> testing|stable +# +# Bonus: replace revisions. +# Bonus: offer to drop existing packages if moving to alpha or beta. +# +# 2. Come up with commit message for src and commit. +# +# "Migrate to $dst_repo_name/
" +# +# "remove /" +# "remove owners//*" +# +# 3. Come up with commit message for dst and commit. +# +# "Migrate from $src_repo_name/
" +# +# "add /" +# "replace / with " (if replacing) +# "add owners//*" +# +# 4. Commit. +# +# Note that when migrating we will need to confirm with git that each of a +# commit's added files were actually most recently added by that commit. For +# example (oldest commits first): +# +# commit 1: add foo.tar.gz, bar.tar.gz +# commit 2: del foo.tar.gz +# commit 3: add foo.tar.gz +# +# If the user chooses to migrate commit 1, only bar.tar.gz must be migrated, +# despite foo.tar.gz existing on disk. +# +# The commit bundle associative array is the set of selected pending +# commits. Its keys are the corresponding indexes of the 'pending_seq' array +# (but offset by +1 and formatted to match the displayed commit numbers). +# Note: the reason the commit bundle is an associative array is to prevent +# duplicates. +# +declare -A bundle + +# Migrate the selected commit bundle from the source repository to the +# destination repository. Set the global migrate_result variable to true if +# the migration has been successful, or issue appropriate diagnostics and set +# it to the empty string if any of the following is true: +# +# - The commit bundle is empty. +# +# - Files added by commits in the bundle are not from the same project (the +# "bundle project") or, in the case of archives, the same repository section +# (the "bundle section"). +# +# - The required section does not exist in the destination repository. +# +# - An identical package archive (same name and version) already exists in the +# destination repository. +# +# - Any file has an invalid path (for example, missing a valid project or +# section component). +# +# The migration process proceeds as follows: +# +# - Move files: all of the files in the selected commit bundle are moved from +# the source repository into the destination repository. +# +# Package archives may be removed and ownership manifests overwritten at the +# destination. Candidate files for replacement are selected as follows: +# +# - In the alpha and beta sections, any package archive files in the +# destination section directory belonging to the same package are +# considered for replacement, regardless of their versions. +# +# - In other sections, any package archives in the destination section +# directory with the same name and version but a different revision +# (currently whether lower or higher) are automatically replaced. +# +# - Project or package ownership manifests will be replaced (that is, simply +# overwritten) at the destination with any ownership manifests added by +# the commit bundle because their presence implies that ownership +# information has changed. +# +# Stage (but don't commit) the removal of the files from the source +# repository and their addition to the destination repository. +# +# - Make commits to the source and destination respositories with appropriate +# commit messages. +# +# If any part of the migration fails then all changes to the source and +# destination repositories are undone, leaving two clean repositories. +# +function migrate () +{ + migrate_result= + + if [ "${#bundle[@]}" -eq 0 ]; then + info "no commits selected" + return + fi + + # Check that every commit's added files are in the bundle section and/or + # bundle project before migrating any of them. Build the bundle's list of + # files as we go along, classifying them as package archives or ownership + # manifests based on their paths. + # + # The bundle section is derived from the first package archive encountered + # and the bundle project from the first package archive or owner manifest + # encountered. + # + # Note that the bundle traversal is unordered. + # + local src_sect= # Source section name. + local src_sect_dir= # Source section directory. + local proj= # The bundle (source) project. + local pkgs=() # The bundle's archive files. + local owns=() # The bundle's ownership manifests. + + local i + for i in "${!bundle[@]}"; do + local h="${pending_seq[i-1]}" # The current commit's abbreviated hash. + + # Check the files added by the current commit. + # + local f + while read -d '' f; do + if [ "${file_commits[$f]}" != "$h" ]; then + continue # This file was deleted by a subsequent commit. + fi + + # Derive the project and/or section names from the file path. + # + # The project name is taken directly from the file path. In the case of + # package archives, the section name is the key in the 'src_sections' + # associative array which maps to the section directory extracted from + # the file path. + # + local fproj= # Current file's project. + + if [[ -n "$src_owners" && ("$f" =~ ^"$src_owners"/([^/]+)/.+$) ]]; then + fproj="${BASH_REMATCH[1]}" + owns+=("$f") + elif [[ "$f" =~ ^(.+)/([^/]+)/[^/]+$ ]]; then # Package archive? + local fsect_dir="${BASH_REMATCH[1]}" + + fproj="${BASH_REMATCH[2]}" + pkgs+=("$f") + + # Find the archive section name associated with the extracted section + # directory in 'src_sections' (a value-to-key lookup). + # + local fsect= + + # The "*" key is a catch-all for unknown submitted section names and, + # if present, will share a value (section directory) with one of the + # known section names and therefore must be skipped. + # + # If there is no mapping in 'src_sections' to the extracted section + # directory then the file path is invalid. + # + local k + for k in "${!src_sections[@]}"; do + if [[ ("${src_sections[$k]%/}" == "$fsect_dir") && + ("$k" != "*") ]]; then + fsect="$k" # Current file's section name. + break + fi + done + + if [ -z "$fsect" ]; then + info "unable to find section name for file '$f'" + return + fi + + # Set the source section name and directory if unset; otherwise fail + # if the current file is not from the source section. + # + if [ -z "$src_sect" ]; then + src_sect="$fsect" + src_sect_dir="$fsect_dir" + elif [ "$fsect" != "$src_sect" ]; then + info "cannot include commit $i: '$f' is not in section $src_sect" + return + fi + else + info "unrecognized type of file '$f'" + return + fi + + # Set the bundle project if unset; otherwise fail if the current file is + # not from the bundle project. + # + # Note: $fproj cannot be empty here (see above). + # + if [ -z "$proj" ]; then + proj="$fproj" + elif [ "$fproj" != "$proj" ]; then + info "cannot include commit $i: '$f' is not in project $proj" + return + fi + done < <(commit_files "$h") + done + + # Finalize migration variables the values of which depend on whether the + # bundle contains at least one package archive or ownership manifests only. + # + # The source and destination commit messages are composed incrementally as + # the migration process proceeds. + # + local dst_sect # Destination section name. + local dst_sect_dir # Destination section directory. + local src_cmsg # Source commit message. + local dst_cmsg # Destination commit message. + + if [ ${#pkgs[@]} -ne 0 ]; then # Bundle contains package archive(s). + dst_sect="$src_sect" + + # If it exists, 'testing' overrides 'stable' at the destination. + # + if [[ ("$dst_sect" == "stable") && -v dst_sections["testing"] ]]; then + dst_sect="testing" + fi + + # Fail if the target section does not exist in the destination repository. + # + if [ ! -v dst_sections["$dst_sect"] ]; then + info "section '$dst_sect' does not exist in the destination repository" + return + fi + + dst_sect_dir="${dst_sections[$dst_sect]}" + + src_cmsg="Migrate $proj to $dst_repo_name/$dst_sect"$'\n\n' + dst_cmsg="Migrate $proj from $src_repo_name/$src_sect"$'\n\n' + else # Bundle consists only of ownership manifests. + + # The setup where the ownership authentication is disabled on the + # destination but enabled on source is probably obscure, but let's + # consider it possible since the submit-git handler allows such a setup. + # + if [ -n "$dst_owners" ]; then + src_cmsg="Migrate $proj ownership info to $dst_repo_name"$'\n\n' + dst_cmsg="Migrate $proj ownership info from $src_repo_name"$'\n\n' + else + src_cmsg="Remove $proj ownership info"$'\n\n' + dst_cmsg= # Nothing to commit. + fi + fi + + # Ensure that the source and destination repositories are clean if the + # migration of any file fails. + # + # Note that the source repository cannot have untracked files so we + # git-clean only the destination repository. + # + function cleanup () + { + info "migration failed; resetting and cleaning repositories" + + if ! run git -C "$src_dir" reset --hard || + ! run git -C "$dst_dir" reset --hard || + ! run git -C "$dst_dir" clean --force; then + info "failed to reset/clean repositories -- manual intervention required" + fi + } + trap cleanup EXIT + + # Migrate the bundle's package archive files. + # + for f in "${pkgs[@]}"; do + # Get the current package's name and version from its embedded manifest + # (we already have the source project in $proj). + # + local p + p=($(extract_pkg_info "$src_dir/$f")) + + local name="${p[0]}" + local src_version="${p[1]}" + + # Check for duplicate package in all sections. Use -.* + # without .tar.gz in case we want to support more archive types later. + # + # Note that, for example, foo-bar version 1.0 and foo version bar-1.0 have + # the same archive name foo-bar-1.0.tar.gz. + # + local s + for s in "${!dst_sections[@]}"; do + local p + IFS=$'\n' eval \ + 'p=($(bpkg_util_pkg_find_archive "$name-$src_version.*" \ + "$dst_dir/${dst_sections[$s]}"))' + + if [ "${#p[@]}" -ne 0 ]; then + local n="${p[0]}" + local v="${p[1]}" + local a="${p[3]}" + + if [ "$n" == "$name" ]; then + error "duplicate of $name/$src_version at '$a'" + else + error "conflict of $name/$src_version with $n/$v at '$a'" + fi + fi + done + + # In the destination repository, find and remove package archive files + # which are other alpha/beta versions or revisions of the current source + # package. + # + local vpat # Version pattern. + case "$dst_sect" in + alpha|beta) vpat="*" ;; # All package versions. + *) vpat="$src_version*" ;; # All package version revisions. + esac + + # Packages in the destination repository to be considered for replacement. + # + local dst_files + + IFS=$'\n' eval \ + 'dst_files=($(bpkg_util_pkg_find_archives "$name" \ + "$vpat" \ + "$dst_dir/$dst_sect_dir"))' + + # If true, the source package replaces one or more packages in the + # destination repository. + # + local repl= + + local dst_f + for dst_f in "${dst_files[@]}"; do + local p + p=($(extract_pkg_info "$dst_f")) + + local dst_version="${p[1]}" + local dst_project="${p[2]}" + + # Ask whether or not to drop the current destination package. + # + # Include the project names in the prompt if the destination package's + # project differs from that of the source package. + # + local src="$src_version" + local dst="$name/$dst_version" + if [ "$dst_project" != "$proj" ]; then + src+=" ($proj)" + dst+=" ($dst_project)" + fi + + while true; do + read -p "replace $dst with $src? [y/n]: " opt + + case "$opt" in + "y") + repl=true + dst_cmsg+=" replace $name/$dst_version with $src_version"$'\n' + run git -C "$dst_dir" rm --quiet "${dst_f#$dst_dir/}" + break + ;; + "n") + break + ;; + esac + done + done + + # Migrate the current package. + # + src_cmsg+=" remove $name/$src_version"$'\n' + if [ ! "$repl" ]; then + dst_cmsg+=" add $name/$src_version"$'\n' + fi + migrate_file "$f" "$dst_sect_dir/$proj" + done + + # Migrate the bundle's ownership manifests. + # + # If ownership authentication is disabled on the destination repository, + # only remove ownership manifests from the source repository (that is, don't + # migrate). + # + for f in "${owns[@]}"; do + src_cmsg+=" remove $(dirname $f)/*"$'\n' + + if [ -n "$dst_owners" ]; then + local dp=$(dirname "${f/$src_owners/$dst_owners}") # Destination path. + + # Let the commit message reflect whether this is a new ownership + # manifest or is replacing an existent one. + # + if [ ! -e "$dst_dir/$dp/$(basename "$f")" ]; then + dst_cmsg+=" add $dp/*"$'\n' + else + dst_cmsg+=" update $dp/*"$'\n' + fi + + migrate_file "$f" "$dp" + else + run git -C "$src_dir/" rm --quiet "$f" + fi + done + + # Commit the changes made to the source and destination repositories. + # + info + run git -C "$src_dir" commit -m "$src_cmsg" + + if [ -n "$dst_cmsg" ]; then + info + run git -C "$dst_dir" commit -m "$dst_cmsg" + fi + + info + + # Remove the migrated commits from the pending sequence and clear the + # bundle. + # + for i in "${!bundle[@]}"; do + unset pending_seq[i-1] + done + pending_seq=("${pending_seq[@]}") # Remove the gaps created by unset. + bundle=() + + migrate_result=true + + # All files have been migrated successfully so clear the EXIT trap. + # + trap EXIT + + # Pause to give the operator a chance to look at the commits before the list + # of remaining pending commits is displayed. + # + read -p "Press Enter to continue: " +} + +# Push local changes to the remote source and/or destination git repositories. +# +# Push to the destination repository first because thus the migrated files +# will be in both remote repositories until the completion of the subsequent +# push to the source repository (which may fail or take long). Although this +# is an inconsistent state, it is safe because other programs such as a +# submission handler will be able to detect the duplicates and therefore +# refuse to do anything. If, on the other hand, we pushed to the source first, +# the migrated files would not exist in either remote repository until the +# push to the destination repository completed. In this state the submission +# handler would, for example, accept a resubmission of the migrated packages or +# erroneously establish ownership for already owned project/package names. +# +function push () +{ + # Let's print additional diagnostics on git-push failure, to emphasize for + # the user which of the two repositories we have failed to push. + # + if ! run git -C "$dst_dir" push; then + error "push to $dst_repo_name failed" + fi + + if ! run git -C "$src_dir" push; then + error "push to $src_repo_name failed" + fi +} + +# Present the list of pending commits to the user, oldest first, marking files +# that were deleted by subsequent commits with `*`: +# +# 001 (deadbeef) Add libfoo/1.2.3 +# +# 1/testing/foo/libfoo-1.2.3.tar.gz +# owners/foo/project-owner.manifest +# owners/foo/libfoo/package-owner.manifest +# +# 002 (c00l0fff) Add bar/1.2.3 +# +# * 1/testing/bar/libbar-1.2.3.tar.gz +# 1/testing/bar/libbaz-1.2.3.tar.gz +# +# 003 (deadbabe) Add libbar/1.2.3+1 +# +# 1/testing/bar/libbar-1.2.3+1.tar.gz +# +# Note that files deleted by subsequent commits may still be in the +# repository. See migrate() for an example. +# +# Then prompt the user for the action (showing the current bundle): +# +# [001 002][,m,c,p,q,l,?]: +# +# - add commit to the commit bundle +# m - migrate the selected commit bundle +# c - clear the selected commit bundle +# p - push source and destination repositories +# l - print pending commits +# q - quit (prompting to push if any actions have been taken) +# ? - print this help +# +# The user interaction loop. +# +# In each iteration, present the list of pending commits, display the menu of +# actions, read the user's input, and perform the chosen action. +# +# True if any changes have been made to the source and/or destination git +# repositories (in which case the user will be asked whether or not to push +# before quitting). +# +need_push= + +while true; do + # Show the pending commits. + # + if [ "${#pending_seq[@]}" -eq 0 ]; then + info "no more pending commits" + fi + + for ((i=0; i != "${#pending_seq[@]}"; i++)); do + h="${pending_seq[$i]}" + + # Print commit number, hash, and subject. + # + # The commit number is left-padded with 0s to 3 digits. Prefix with a + # newline to separate the first commit from the git-pull output and the + # rest from the previous commit info block. + # + subj="$(git -C "$src_dir" log -n 1 --pretty=format:%s "$h")" + printf "\n%.3d (%s) %s\n\n" "$((i+1))" "$h" "$subj" >&2 + + # Print this commit's files. + # + # Fetch from the git repository the list of files added by the current + # commit. Print each file's path and, if it was deleted by a subsequent + # commit, mark with an asterisk. + # + # Note that 'file_commits' is populated above from the list of files + # currently in the source repository. Therefore, if git says a file was + # added by a commit but it is associated with a different commit hash in + # 'file_commits' it means the file was deleted and added back by later + # commits; and if there is no mapping for the file it means it was deleted + # but not added back (that is, it's no longer in the repository). So we + # mark the re-added file with an exclamation. + # + while read -d '' f; do + if [ "${file_commits[$f]}" == "$h" ]; then + info " $f" # File was last added by the current commit. + elif [ -v file_commits["$f"] ]; then + info " ! $f" # File was deleted and added back by subsequent commits. + else + info " * $f" # File was deleted but not added back. + fi + done < <(commit_files "$h") + done + + # Prompt the user for the action (showing the current bundle), get user + # input, and perform the selected action. + # + # Note that we could adapt the menu according to the current state (don't + # offer to migrate if the bundle array is empty, etc) but let's not + # complicate the logic. + # + # Breaking out of this loop prints the pending commit list again. + # + while true; do + # Sort commit bundle in ascending order. + # + # Expand the 'bundle' associative array's keys into a single word in which + # they are separated by spaces (the first member of IFS) using the + # ${!a[*]} syntax; replace each space with a newline before piping to + # 'sort', which is newline-based; finally collect sort's output into an + # array using the a=() syntax, which splits on newline (the last member of + # IFS) because neither space nor tab characters (the other members of IFS) + # can occur in the keys. + # + bundle_sorted=($(sed 's/ /\n/g' <<<"${!bundle[*]}" | sort -)) + + printf "\n" + read -p "[${bundle_sorted[*]}][,m,c,p,l,q,?]: " opt + + case "$opt" in + # Add commit to bundle. + # + [0-9]*) + if [[ ("$opt" -gt 0) && ("$opt " -le "${#pending_seq[@]}") ]]; then + printf -v opt "%.3d" "$opt" # Format as in pending commit list. + if [ ! -v bundle["$opt"] ]; then + bundle["$opt"]=true + info "commit $opt (${pending_seq[$opt-1]}) added to selected bundle" + else + info "commit $opt is already in the bundle" + fi + else + info "non-existent commit number $opt" + fi + ;; + # Migrate the commit bundle. + # + m) + migrate + if [ "$migrate_result" ]; then + need_push=true + break + fi + ;; + # Clear the commit bundle. + # + c) + bundle=() + break + ;; + # Push changes. + # + p) + push + need_push= + break + ;; + # Redraw the pending commit list. + # + l) + break + ;; + # Quit. + # + q) + if [ ! "$need_push" ]; then + exit 0 + fi + + while true; do + read -p "push changes? [y/n/(c)ancel]: " opt + + case "$opt" in + "c") + break # Print options menu again. + ;; + "y") + push + exit 0 + ;; + "n") + exit 0 + ;; + *) + continue + ;; + esac + done + ;; + # ? or invalid option: print menu. + # + *) + cat <<-EOF + + - add commit to the commit bundle + m - migrate the selected commit bundle + c - clear the selected commit bundle + p - push source and destination repositories + l - print pending commits + q - quit (prompting to push if any actions have been taken) + ? - print this help +EOF + ;; + esac + done +done diff --git a/bpkg-util/package-archive.bash.in b/bpkg-util/package-archive.bash.in new file mode 100644 index 0000000..9400ade --- /dev/null +++ b/bpkg-util/package-archive.bash.in @@ -0,0 +1,170 @@ +# file : bpkg-util/package-archive.bash.in +# license : MIT; see accompanying LICENSE file + +# Utility functions useful for managing package archives. + +if [ "$bpkg_util_package_archive" ]; then + return 0 +else + bpkg_util_package_archive=true +fi + +@import libbutl/manifest-parser@ + +# We expect the user to set the bpkg_util_bpkg variable to the bpkg program +# path. +# +if [ ! -v bpkg_util_bpkg ]; then + echo "error: variable bpkg_util_bpkg is not set" >&2 + exit 1 +fi + +# Extract the package information from a package archive and print it in the +# ' ' form, where the project field is empty if the +# project value is not specified in the manifest. +# +# Note that, in particular, it verifies that the archive file name matches the +# package name and version. +# +function bpkg_util_pkg_verify_archive () # +{ + # We can't use the process substitution for input redirect here, since such + # a process failure is not trapped. Thus, read the manifest file into a + # variable and parse it afterwards, which is probably ok since package + # manifests are normally not too big. + # + # Note that alternatively we could use the process substitution for running + # bpkg, treat the name value absence as indication of a failure, and exit + # with non-zero status if that's the case. Feels a bit hackish though. + # + local m + m="$("$bpkg_util_bpkg" pkg-verify --manifest "$1")" + + butl_manifest_parser_start <<<"$m" + + local name= + local version= + local project= + + local n v + while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do + case "$n" in + name) name="$v" ;; + version) version="$v" ;; + project) project="$v" ;; + esac + done + + butl_manifest_parser_finish + + echo -n "$name $version $project" +} + +# Search for package archives in a directory using the package name and +# version pattern and printing their paths newline-separated. If the version +# argument is '*', then print archives for all package versions. Otherwise if +# the version contains the trailing '*', then print archives for all revisions +# of the specified version and for the exact version otherwise. For example: +# +# bpkg_util_pkg_find_archives foo '*' dir/ +# bpkg_util_pkg_find_archives foo '1.0*' dir/ +# bpkg_util_pkg_find_archives foo '1.0' dir/ +# +# Note that the resulting archive paths include the specified directory as a +# prefix. +# +# NOTE: this function can be called with overriden IFS. +# +function bpkg_util_pkg_find_archives () # +{ + IFS=$' \t\n' bpkg_util_pkg_find_archives_impl "$@" +} + +function bpkg_util_pkg_find_archives_impl () +{ + local nam="$1" + local ver="$2" + local dir="$3" + + local r="" + + if [ -d "$dir" ]; then + local vr # Version with the revision stripped, if search for revisions. + local np # File name pattern for archives search. + + if [ "$ver" != "*" -a "${ver: -1}" == "*" ]; then # * + vr="$(sed -n -re 's%^(\+?[^+]+)(\+[0-9]+)?\*$%\1%p' <<<"$ver")" + np="$nam-$vr*.*" # foo-1.0*.*, etc. + else # * or + np="$nam-$ver.*" # foo-*.*, foo-1.0.*, etc. + fi + + # Go through the potentially matching archives (for example, for 'foo' + # '1.2.3+2*': foo-1.2.3.tar.gz, foo-1.2.3+1.tar.gz, foo-1.2.30.tar.gz, + # etc) and return those which package name and version match properly. + # + local f + while read f; do + local p + p=($(bpkg_util_pkg_verify_archive "$f")) + + local n="${p[0]}" + local v="${p[1]}" + + if [[ "$n" == "$nam" && + ( "$ver" == "*" || \ + "$v" == "$ver" || \ + ( -n "$vr" && "$v" =~ ^"$vr"(\+[0-9]+)?$ )) ]]; then + + if [ -n "$r" ]; then + r="$r"$'\n'"$f" + else + r="$f" + fi + fi + done < <(find "$dir" -type f -name "$np") + fi + + if [ -n "$r" ]; then + echo -n "$r" + fi +} + +# Search for a package archive in a directory using a file name pattern. If +# the archive is found, then print the package information in the +# '\n\n\n' form, where the project field is +# empty if the project value is not specified in the manifest. +# +# Note that if there are multiple archives matching the pattern, then it is +# unspecified which one is picked. +# +# NOTE: this function can be called with overriden IFS. +# +function bpkg_util_pkg_find_archive () # +{ + IFS=$' \t\n' bpkg_util_pkg_find_archive_impl "$@" +} + +function bpkg_util_pkg_find_archive_impl () +{ + local pat="$1" + local dir="$2" + + if [ -d "$dir" ]; then + local f + + # We could probably use -print -quit but this is not portable (NetBSD + # needs -exit instead of -quit). + # + f="$(find "$dir" -type f -name "$pat" | head -n 1)" + + if [ -n "$f" ]; then + + local p + p=($(bpkg_util_pkg_verify_archive "$f")) + + printf "${p[0]}\n${p[1]}\n${p[2]}\n$f" + return + fi + fi +} diff --git a/bpkg-util/publish.in b/bpkg-util/publish.in new file mode 100644 index 0000000..215c6a6 --- /dev/null +++ b/bpkg-util/publish.in @@ -0,0 +1,329 @@ +#!/usr/bin/env bash + +# file : bpkg-util/publish.in +# license : MIT; see accompanying LICENSE file + +# Update (bpkg-rep-create(1)) and publish (rsync(1)) an archive-based +# repository. +# +# Pull a pre-cloned (read-only) git repository with the contents of an +# archive-based bpkg repository. Bail out if nothing changed from the the +# previous run. Otherwise, regenerate the repository meta-data by running +# bpkg-rep-create(1) on each section of the repository and, optionally, +# synchronize it to one or more destinations with rsync. +# +# The repository contents are expected to be in the /1/ subdirectory. The +# script saves the last successfully published commit in the .publish +# file. +# +# --destination|-d : +# +# Remote host and directory to rsync the repository to. Note that the +# directory should include the 1/ component and any sub-directories that may +# follow. In other words, the rsync command will be in the form: +# +# rsync ... /1/ :/ +# +# See below for the actual rsync command including a brief explanation of +# options passed. +# +# Repeat this option to specify multiple destinations. In this case, the +# destinations are synced in the order specified with the first failure +# terminating the process (so if you have a "primary" destination and a +# "mirror", you probably want to specify the former first). +# +# --timeout +# +# Git and rsync operation timeout. Specifically, the operation will be +# aborted if there is no network activity for the specified time. Default is +# 60 seconds. Note that currently the git timeout is only supported for the +# http(s) transport. +# +# --lock-timeout +# +# The repository lock timeout. Fail if another instance of the script does +# not release the repository in the specified time. The default is 0 (do +# not wait). +# +# Note that you will most likely want to specify a non-zero timeout for cron +# jobs that may potentially overlap. +# +# --log-dir +# +# Directory to create the temporary log files in. If unspecified, the stderr +# is not redirected and no log is created by default. +# +# The log is dumped to stderr in case of an error or at the end of execution +# unless in the quiet mode and is then deleted. +# +# --quiet +# +# Run quiet. Specifically, don't dump the log to stderr on exit with zero +# status. +# +# --config +# +# The configuration file containing a bash fragment. Repeat this option to +# specify multiple configurations that will be sourced in the order +# specified. +# +# --bpkg +# +# The package manager program to be used for the repository update. This +# should be the path to the bpkg executable. +# +usage="usage: $0 [] [] [-- ]" + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +@import bpkg-util/utility@ + +# The script own options. +# +repo_ver=1 +destinations=() +timeout=60 +lock_timeout=0 +log_dir= +quiet= +configurations=() +bpkg= + +while [ $# -gt 0 ]; do + case $1 in + --destination|-d) + shift + destinations+=("${1%/}") + shift || true + ;; + --timeout) + shift + timeout="$1" + shift || true + ;; + --lock-timeout) + shift + lock_timeout="$1" + shift || true + ;; + --log-dir) + shift + log_dir="${1%/}" + shift || true + ;; + --quiet) + shift + quiet=true + ;; + --config) + shift + configurations+=("$1") + shift || true + ;; + --bpkg) + shift + bpkg="$1" + shift || true + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done + +# The repository directory. +# +repo_dir="${1%/}" +shift || true + +# bpkg-rep-create options. +# +rep_create_options=() + +while [ $# -gt 0 ]; do + case $1 in + --) + shift + break + ;; + *) + rep_create_options+=("$1") + shift + ;; + esac +done + +# rsync options. +# +rsync_options=() + +while [ $# -gt 0 ]; do + rsync_options+=("$1") + shift +done + +# Validate options and arguments. +# +if [ -z "$repo_dir" ]; then + error "$usage" +fi + +if [ ! -d "$repo_dir" ]; then + error "'$repo_dir' does not exist or is not a directory" +fi + +# If the log directory is specified then redirect stderr to the log file and +# setup the trap that dumps it on exit, if required. +# +if [ -n "$log_dir" ]; then + + if [ ! -d "$log_dir" ]; then + error "'$log_dir' does not exist or is not a directory" + fi + + # Create the log file. + # + log="$(mktemp "$log_dir/$(basename "$repo_dir").XXXXXXXXXX")" + + # Save the stderr file descriptor so we can dump the log into it on exit, if + # required. Then redirect it to the log file. + # + exec {stderr}>&2 + exec 2>>"$log" + + function exit_trap () + { + local status="$?" + + # Dump the log to stderr if exiting with non-zero status or verbose. + # + if [ $status -ne 0 -o ! "$quiet" ]; then + + # Keep the log if failed to dump for any reason. + # + if ! cat "$log" >&$stderr; then + return + fi + fi + + rm -f "$log" + } + + trap exit_trap EXIT +fi + +# Source the configurations. +# +for c in "${configurations[@]}"; do + source "$c" >&2 +done + +# Make sure the commit file is present. +# +published_commit="$repo_dir.publish" +touch "$published_commit" + +# Open the reading file descriptor and lock the repository. Fail if unable to +# lock before timeout. +# +exec {cfd}<"$published_commit" + +if ! flock -w "$lock_timeout" "$cfd"; then + info "another instance is already running" + exit 2 +fi + +# Pull the repository. +# +# Git doesn't support the connection timeout option. The options we use are +# just an approximation of the former, that, in particular, don't cover the +# connection establishing. To work around this problem, before running a git +# command that assumes the remote repository communication we manually check +# connectivity with the remote repository. +# +if ! remote_url="$(git -C "$repo_dir" config --get remote.origin.url)"; then + error "'$repo_dir' is not a git repository" +fi + +run check_git_connectivity "$remote_url" "$timeout" + +# Fail if no network activity happens during the time specified. +# +run git -c http.lowSpeedLimit=1 -c "http.lowSpeedTime=$timeout" \ +-C "$repo_dir" pull -v >&2 + +# Match the HEAD commit id to the one stored in the file. If it matches, then +# nothing changed in the repository from the previous run and we can silently +# bail out. +# +commit="$(git -C "$repo_dir" rev-parse HEAD)" +pc="$(cat <&"$cfd")" + +if [ "$commit" == "$pc" ]; then + quiet=true + exit 0 +fi + +# If bpkg path is not specified, then use the bpkg program from the script +# directory, if present. Otherwise, use the 'bpkg' path. +# +if [ -z "$bpkg" ]; then + bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" + + if [ ! -x "$bpkg" ]; then + bpkg=bpkg + fi +fi + +# Find repository sections. +# +manifests="$(find "$repo_dir/$repo_ver" -type f -name repositories.manifest)" + +# Update the repository sections. +# +while read f; do + run "$bpkg" rep-create "${rep_create_options[@]}" "$(dirname "$f")" +done <<<"$manifests" + +# rsync (over ssh) the repository to the destinations. +# +# Approximate the data transfer timeout via the ServerAlive* ssh options, +# rounding the timeout up to the nearest multiple of ten. +# +# Note: must not contain spaces/use quoting (see rsync -e option). +# +n=$(($timeout > 0 ? ($timeout + 9) / 10 : 1)) +ssh_options=(-o ConnectTimeout=$timeout \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=$n) + +for d in "${destinations[@]}"; do + + # -r (recursive) + # -l (copy symlinks as symlinks) + # -t (preserve timestamps) + # -O (omit dir timestamps) + # + # -c (use checksum) + # -e (remote shell command) + # + # --chmod=ugo=rwX (give new files the destination-default permissions) + # --safe-links (ignore symlinks pointing outside the tree) + # --delay-updates (first upload all files on the side then move) + # --prune-empty-dirs (remove empty dirs) + # --delete-after (delete entries after the transfer) + # + # We also exclude hidden files (start with dot). + # + run rsync -v -rltO -c --chmod=ugo=rwX --safe-links --delay-updates \ +--exclude '.*' --prune-empty-dirs --delete-after -e "ssh ${ssh_options[*]}" \ +"${rsync_options[@]}" "$repo_dir/$repo_ver/" "$d/" >&2 + +done + +echo "$commit" >"$published_commit" diff --git a/bpkg-util/utility.bash.in b/bpkg-util/utility.bash.in new file mode 100644 index 0000000..7201659 --- /dev/null +++ b/bpkg-util/utility.bash.in @@ -0,0 +1,86 @@ +# file : bpkg-util/utility.bash.in +# license : MIT; see accompanying LICENSE file + +# Utility functions useful for implementing package management utilities. + +if [ "$bpkg_util_utility" ]; then + return 0 +else + bpkg_util_utility=true +fi + +# Diagnostics. +# +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +# Trace a command line, quoting empty arguments as well as those that contain +# spaces. +# +function trace_cmd () # ... +{ + local s="+" + while [ $# -gt 0 ]; do + if [ -z "$1" -o -z "${1##* *}" ]; then + s="$s '$1'" + else + s="$s $1" + fi + + shift + done + + info "$s" +} + +# Trace the current function name and arguments. +# +function trace_func () # ... +{ + trace_cmd "${FUNCNAME[1]}" "$@" +} + +# Trace and run a command. +# +function run () # ... +{ + trace_cmd "$@" + "$@" +} + +# Return lower-case URL scheme or empty string if the argument doesn't look +# like a URL. +# +function url_scheme () # +{ + sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1" +} + +# Check that the git repository properly responds to the probing request +# before the timeout (in seconds). Noop for protocols other than HTTP(S). +# +function check_git_connectivity () # +{ + local url="$1" + local tmo="$2" + + local s + s="$(url_scheme "$url")" + + if [ "$s" == "http" -o "$s" == "https" ]; then + local u q + + u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part. + q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part. + + if [ -z "$q" ]; then + u="$u/info/refs?service=git-upload-pack" + else + u="$u/info/refs$q&service=git-upload-pack" + fi + + # Here we limit the time for the whole operation. + # + curl -S -s --max-time "$tmo" "$u" >/dev/null + fi +} diff --git a/build/bootstrap.build b/build/bootstrap.build index 557fee7..898aebc 100644 --- a/build/bootstrap.build +++ b/build/bootstrap.build @@ -1,7 +1,7 @@ # file : build/bootstrap.build # license : MIT; see accompanying LICENSE file -project = bpkg-rep +project = bpkg-util using version using config diff --git a/build/export.build b/build/export.build index 1ac1b48..2a15acc 100644 --- a/build/export.build +++ b/build/export.build @@ -3,12 +3,12 @@ $out_root/ { - include bpkg-rep/ + include bpkg-util/ } switch $import.target { - case exe{bpkg-rep-publish} + case exe{bpkg-util-publish} case bash{package-archive} - export $out_root/bpkg-rep/$import.target + export $out_root/bpkg-util/$import.target } diff --git a/build/root.build b/build/root.build index ae5e447..1cc51ca 100644 --- a/build/root.build +++ b/build/root.build @@ -5,4 +5,4 @@ # using bash -bpkg-rep/bash{*}: install.subdirs = true +bpkg-util/bash{*}: install.subdirs = true diff --git a/manifest b/manifest index 496e8bf..45ee2f6 100644 --- a/manifest +++ b/manifest @@ -1,15 +1,15 @@ : 1 -name: bpkg-rep +name: bpkg-util version: 0.14.0-a.0.z project: build2 -summary: bpkg repository management utilities +summary: extra build2 package management utilities license: MIT topics: package dependency management, build toolchain description-file: README changes-file: NEWS url: https://build2.org doc-url: https://build2.org/doc.xhtml -src-url: https://git.build2.org/cgit/bpkg-rep/tree/ +src-url: https://git.build2.org/cgit/bpkg-util/tree/ email: users@build2.org build-warning-email: builds@build2.org builds: all diff --git a/repositories.manifest b/repositories.manifest index 6fc6dd2..848e997 100644 --- a/repositories.manifest +++ b/repositories.manifest @@ -1,5 +1,5 @@ : 1 -summary: bpkg repository management utilities repository +summary: extra build2 package management utilities : role: prerequisite diff --git a/tests/package-archive/buildfile b/tests/package-archive/buildfile index a8ff42b..7702332 100644 --- a/tests/package-archive/buildfile +++ b/tests/package-archive/buildfile @@ -1,7 +1,7 @@ # file : tests/package-archive/buildfile # license : MIT; see accompanying LICENSE file -import mods = bpkg-rep%bash{package-archive} +import mods = bpkg-util%bash{package-archive} ./: exe{driver} file{*.tar.gz} diff --git a/tests/package-archive/driver.in b/tests/package-archive/driver.in index d760a1e..a32e571 100644 --- a/tests/package-archive/driver.in +++ b/tests/package-archive/driver.in @@ -5,12 +5,12 @@ # bpkg utility path. # -bpkg_rep_bpkg=bpkg +bpkg_util_bpkg=bpkg trap "{ exit 1; }" ERR set -o errtrace # Trap ERR in functions. -@import bpkg-rep/package-archive@ +@import bpkg-util/package-archive@ # Call the function passed on the command line. # diff --git a/tests/package-archive/testscript b/tests/package-archive/testscript index 6124d4a..0cc93b2 100644 --- a/tests/package-archive/testscript +++ b/tests/package-archive/testscript @@ -12,7 +12,7 @@ clone_arcs = \ : pkg-verify-archive : { - test.arguments += bpkg_rep_pkg_verify_archive + test.arguments += bpkg_util_pkg_verify_archive : non-existing-archive : @@ -28,7 +28,7 @@ clone_arcs = \ : pkg-find-archives : { - test.arguments += bpkg_rep_pkg_find_archives + test.arguments += bpkg_util_pkg_find_archives : none : @@ -106,7 +106,7 @@ clone_arcs = \ : pkg-find-archive : { - test.arguments += bpkg_rep_pkg_find_archive + test.arguments += bpkg_util_pkg_find_archive : non-existent : diff --git a/tests/publish/buildfile b/tests/publish/buildfile index 4cc1bd0..cdd601f 100644 --- a/tests/publish/buildfile +++ b/tests/publish/buildfile @@ -1,7 +1,7 @@ # file : tests/publish/buildfile # license : MIT; see accompanying LICENSE file -import publish = bpkg-rep%exe{bpkg-rep-publish} +import publish = bpkg-util%exe{bpkg-util-publish} ./: testscript $publish -- cgit v1.1