diff options
Diffstat (limited to 'bpkg-rep')
| -rw-r--r-- | bpkg-rep/.gitignore | 2 | ||||
| -rw-r--r-- | bpkg-rep/buildfile | 17 | ||||
| -rw-r--r-- | bpkg-rep/manage.in | 957 | ||||
| -rw-r--r-- | bpkg-rep/package-archive.bash.in | 170 | ||||
| -rw-r--r-- | bpkg-rep/publish.in | 329 | ||||
| -rw-r--r-- | bpkg-rep/utility.bash.in | 86 |
6 files changed, 0 insertions, 1561 deletions
diff --git a/bpkg-rep/.gitignore b/bpkg-rep/.gitignore deleted file mode 100644 index ec9e671..0000000 --- a/bpkg-rep/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -bpkg-rep-publish -bpkg-rep-manage diff --git a/bpkg-rep/buildfile b/bpkg-rep/buildfile deleted file mode 100644 index 3c66f8b..0000000 --- a/bpkg-rep/buildfile +++ /dev/null @@ -1,17 +0,0 @@ -# file : bpkg-rep/buildfile -# license : MIT; see accompanying LICENSE file - -import mods = libbutl.bash%bash{manifest-parser} - -./: exe{bpkg-rep-publish bpkg-rep-manage} bash{package-archive} - -exe{bpkg-rep-publish}: in{publish} bash{utility} -exe{bpkg-rep-manage}: in{manage} bash{utility package-archive} - -# Public modules. -# -bash{package-archive}: in{package-archive} $mods - -# Private modules. -# -bash{utility}: in{utility} diff --git a/bpkg-rep/manage.in b/bpkg-rep/manage.in deleted file mode 100644 index ceede3c..0000000 --- a/bpkg-rep/manage.in +++ /dev/null @@ -1,957 +0,0 @@ -#!/usr/bin/env bash - -# file : bpkg-rep/manage.in -# license : MIT; see accompanying LICENSE file - -# Interactively migrate newly-submitted packages from a source git repository -# to a destination git repository. -# -# Present the user with the list of commits that added the files currently in -# the source repository's working directory and ask the user to select from a -# menu the action to perform on a selection of these commits. -# -# As the files added by these commits are pending a move to the destination -# repository, these commits will be referred to as "pending" commits. -# -# Actions that can be performed on a selection of pending commits include -# moving them from the source repository to a single commit in the destination -# repository and dropping them from the source repository. -# -# The flow of this script, in broad strokes, is as follows: for each file in -# the source repository directory, find the hash of the commit that added it; -# these are the pending commits. Arrange the pending commits in chronological -# order. Display to the user the pending commits along with the files they -# added. Let the user select one or more pending commits and an action to be -# performed on them. Each successful action results in a commit to the source -# and/or destination repositories and leaves both repositories in a clean -# state. Once the action has been performed, redisplay the updated pending -# commit list and prompt for the next action. Pushing to the remote -# repositories, a supported operation, can be done at any time during the -# session. -# -# <dir> The directory into which the source and destination repositories have -# been checked out. If not specified, current directory is assumed. -# -usage="usage: $0 [<dir>]" - -# Source/destination repository inside <dir>. Note: also used in commit -# messages. -# -src_repo_name=queue -dst_repo_name=public - -owd="$(pwd)" -trap "{ cd '$owd'; exit 1; }" ERR -set -o errtrace # Trap in functions. - -@import bpkg-rep/utility@ - -# Use the bpkg program from the script directory, if present. Otherwise, use -# just 'bpkg'. -# -bpkg_rep_bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" - -if [ ! -x "$bpkg_rep_bpkg" ]; then - bpkg_rep_bpkg=bpkg -fi - -@import bpkg-rep/package-archive@ - -# Set the working directory. -# -if [ $# -eq 0 ]; then - dir="$owd" -elif [ $# -eq 1 ]; then - dir="${1%/}" # <dir> with trailing slash removed. -else - error "$usage" -fi - -# The source and destination package repository directories. -# -# Note that, throughout this script, any path not explicitly prefixed with -# "$src_dir/" or "$dst_dir/" is relative to the root of the source or -# destination package repositories. -# -src_dir="$dir/$src_repo_name" -dst_dir="$dir/$dst_repo_name" - -if [ ! -d "$src_dir" ]; then - error "'$src_dir' does not exist or is not a directory" -fi - -if [ ! -d "$dst_dir" ]; then - error "'$dst_dir' does not exist or is not a directory" -fi - -# Check that both git repositories are clean. -# -if [ -n "$(git -C $src_dir status --porcelain)" ]; then - error "git repository in '$src_dir' is not clean" -fi - -if [ -n "$(git -C $dst_dir status --porcelain)" ]; then - error "git repository in '$dst_dir' is not clean" -fi - -# Use run() to show the user that git is the source of the diagnostics. -# "Already up to date", for example, is too vague. -# -run git -C "$src_dir" pull >&2 -run git -C "$dst_dir" pull >&2 - -# Load the source and destination repositories' submit configurations (section -# name/directory mappings and owners directory path). -# -# Each repository's settings are sourced into the temporary variables 'owners' -# and 'sections' and copied from there to source- and destination-specific -# variables. -# -declare owners -declare -A sections -source "$src_dir/submit.config.bash" - -src_owners="$owners" -declare -A src_sections -for s in "${!sections[@]}"; do - src_sections["$s"]="${sections[$s]}" -done - -owners= -sections=() -source "$dst_dir/submit.config.bash" - -dst_owners="$owners" -declare -A dst_sections -for s in "${!sections[@]}"; do - dst_sections["$s"]="${sections[$s]}" -done - -# Find all archive and owner manifest files in the source repository. -# -# Every file in a repository section directory except *.manifest is a package -# archive and every file in the owners directory is a project or package owner -# manifest. Therefore run find separately on each section directory and the -# owners directory to build a list containing only package-related files. -# -# Store the relative to the repository directory file paths in an array used -# to build the set of pending commits. -# -src_files=() -for s in "${src_sections[@]}"; do - while read f; do - src_files+=("${f#$src_dir/}") - done < <(find "$src_dir/$s" -type f -not -name "*.manifest") -done - -if [[ -n "$src_owners" && -d "$src_dir/$src_owners" ]]; then - while read f; do - src_files+=("${f#$src_dir/}") - done < <(find "$src_dir/$src_owners" -type f) -fi - -# Build the set of pending commit hashes ("pending set"). -# -# For each file in the source repository, find the most recent commit that -# added it and store its abbreviated hash (as key) inside the 'pending_set' -# associative array (note: unordered) and (as value) inside the 'file_commits' -# associative array. -# -# 'file_commits' maps from the file path to the last commit to add it to the -# repository. A file may have been added and removed by earlier commits and -# could thus be migrated with the wrong commit unless care is taken (see -# migrate() for an example). -# -declare -A pending_set -declare -A file_commits -for f in "${src_files[@]}"; do - - # -n 1: limit output to one commit (that is, the most recent) - # --diff-filter=A: only show commits that added files - # --pretty=format:%h: output only the abbreviated commit hash - # - h="$(git -C "$src_dir" log -n 1 --diff-filter=A --pretty=format:%h -- "$f")" - - # Note that the hash cannot be empty because, after our clean checks at the - # top, every file on disk must have been added by some commit (that is, - # there can be no untracked files). - # - pending_set["$h"]=true - file_commits["$f"]="$h" -done - -# Arrange the pending commits in the chronological order. -# -# Go through the most recent commits in the git log which added one or more -# files, skipping those not present in the pending set and keeping count to -# bail out as soon as we ordered all of them. -# -pending_seq=() -for (( i=0; i != ${#pending_set[@]}; )); do - read h # The abbreviated commit hash. - - # If this is a pending commit, prepend its hash to the ordered array. - # - if [ "${pending_set[$h]}" ]; then - pending_seq=("$h" "${pending_seq[@]}") - ((++i)) - fi -done < <(git -C "$src_dir" log --diff-filter=A --pretty=format:%h) - -if [ "${#pending_seq[@]}" -eq 0 ]; then - info "Good news, nothing to manage!" - exit 0 -fi - -# Return the list of files a commit added to the source repository. -# -function commit_files () # <commit-hash> -{ - local h="$1" - - # git-diff-tree arguments: - # - # --diff-filter=A: select only files that were added. - # -z: don't munge file paths and separate output fields with - # NULs. - # -r: recurse into subtrees (directories). - # - git -C "$src_dir" diff-tree \ - --no-commit-id --name-only --diff-filter=A -z -r \ - "$h" -} - -# Extract the package name, version, and project from a package archive's -# manifest and print it to stdout in the '<name> <version> <project>' form. If -# the manifest does not specify the project name, the package name is returned -# as the project name. -# -function extract_pkg_info () # <archive> -{ - local arc="$1" - - local r - r=($(bpkg_rep_pkg_verify_archive "$arc")) # <name> <version> <project> - if [ ! -v r[2] ]; then - r[2]="${r[0]}" - fi - - # Verify that the archive parent directory name matches the project. - # - local p="${r[2]}" - if [ "$p" != "$(basename "$(dirname "$arc")")" ]; then - error "'$arc' archive directory name does not match package project '$p'" - fi - - echo -n "${r[@]}" -} - -# Migrate a package archive or ownership manifest file from the source -# repository to the destination repository. -# -# <src> is the path of the source file, relative to the source respository -# directory. For example, '1/stable/foo/foo-1.2.3.tar.gz', -# 'owners/foo/project-owner.manifest', or -# 'owners/foo/foo/package-owner.manifest'. -# -# <dst> is the path of the destination directory, relative to the destination -# repository directory. For example, '1/testing/foo', 'ownership/foo', or -# 'ownership/foo/foo'. -# -# Note that the source and destination sections and owners directories may -# differ (as they do in these examples) which is why those components must -# be specified in both the source and destination paths. -# -# Move the file from the source repository directory to the destination -# repository directory, creating directories if required; stage the addition -# of the file to the destination repository; stage the removal of the file -# from the source repository. -# -function migrate_file () # <src> <dst> -{ - local src="$1" - local dst="$2" - - mkdir -p "$dst_dir/$dst" - mv "$src_dir/$src" "$dst_dir/$dst/" - run git -C "$src_dir/" rm --quiet "$src" - run git -C "$dst_dir/" add "$dst/$(basename "$src")" -} - -# Migrate: -# -# 0. Assumptions: -# -# - All the packages in a bundle are migrating from/to the same sections -# (enforce source part). -# -# - All the packages are from the same project (enforce). -# -# 1. Move files: -# -# - Owners to owners directory. -# -# - Packages into corresponding sections: -# -# alpha -> alpha -# beta -> beta -# stable -> testing|stable -# -# Bonus: replace revisions. -# Bonus: offer to drop existing packages if moving to alpha or beta. -# -# 2. Come up with commit message for src and commit. -# -# "Migrate <project> to $dst_repo_name/<section>" -# -# "remove <package>/<version>" -# "remove owners/<project>/*" -# -# 3. Come up with commit message for dst and commit. -# -# "Migrate <project> from $src_repo_name/<section>" -# -# "add <package>/<version>" -# "replace <package>/<version> with <version>" (if replacing) -# "add owners/<project>/*" -# -# 4. Commit. -# -# Note that when migrating we will need to confirm with git that each of a -# commit's added files were actually most recently added by that commit. For -# example (oldest commits first): -# -# commit 1: add foo.tar.gz, bar.tar.gz -# commit 2: del foo.tar.gz -# commit 3: add foo.tar.gz -# -# If the user chooses to migrate commit 1, only bar.tar.gz must be migrated, -# despite foo.tar.gz existing on disk. -# -# The commit bundle associative array is the set of selected pending -# commits. Its keys are the corresponding indexes of the 'pending_seq' array -# (but offset by +1 and formatted to match the displayed commit numbers). -# Note: the reason the commit bundle is an associative array is to prevent -# duplicates. -# -declare -A bundle - -# Migrate the selected commit bundle from the source repository to the -# destination repository. Set the global migrate_result variable to true if -# the migration has been successful, or issue appropriate diagnostics and set -# it to the empty string if any of the following is true: -# -# - The commit bundle is empty. -# -# - Files added by commits in the bundle are not from the same project (the -# "bundle project") or, in the case of archives, the same repository section -# (the "bundle section"). -# -# - The required section does not exist in the destination repository. -# -# - An identical package archive (same name and version) already exists in the -# destination repository. -# -# - Any file has an invalid path (for example, missing a valid project or -# section component). -# -# The migration process proceeds as follows: -# -# - Move files: all of the files in the selected commit bundle are moved from -# the source repository into the destination repository. -# -# Package archives may be removed and ownership manifests overwritten at the -# destination. Candidate files for replacement are selected as follows: -# -# - In the alpha and beta sections, any package archive files in the -# destination section directory belonging to the same package are -# considered for replacement, regardless of their versions. -# -# - In other sections, any package archives in the destination section -# directory with the same name and version but a different revision -# (currently whether lower or higher) are automatically replaced. -# -# - Project or package ownership manifests will be replaced (that is, simply -# overwritten) at the destination with any ownership manifests added by -# the commit bundle because their presence implies that ownership -# information has changed. -# -# Stage (but don't commit) the removal of the files from the source -# repository and their addition to the destination repository. -# -# - Make commits to the source and destination respositories with appropriate -# commit messages. -# -# If any part of the migration fails then all changes to the source and -# destination repositories are undone, leaving two clean repositories. -# -function migrate () -{ - migrate_result= - - if [ "${#bundle[@]}" -eq 0 ]; then - info "no commits selected" - return - fi - - # Check that every commit's added files are in the bundle section and/or - # bundle project before migrating any of them. Build the bundle's list of - # files as we go along, classifying them as package archives or ownership - # manifests based on their paths. - # - # The bundle section is derived from the first package archive encountered - # and the bundle project from the first package archive or owner manifest - # encountered. - # - # Note that the bundle traversal is unordered. - # - local src_sect= # Source section name. - local src_sect_dir= # Source section directory. - local proj= # The bundle (source) project. - local pkgs=() # The bundle's archive files. - local owns=() # The bundle's ownership manifests. - - local i - for i in "${!bundle[@]}"; do - local h="${pending_seq[i-1]}" # The current commit's abbreviated hash. - - # Check the files added by the current commit. - # - local f - while read -d '' f; do - if [ "${file_commits[$f]}" != "$h" ]; then - continue # This file was deleted by a subsequent commit. - fi - - # Derive the project and/or section names from the file path. - # - # The project name is taken directly from the file path. In the case of - # package archives, the section name is the key in the 'src_sections' - # associative array which maps to the section directory extracted from - # the file path. - # - local fproj= # Current file's project. - - if [[ -n "$src_owners" && ("$f" =~ ^"$src_owners"/([^/]+)/.+$) ]]; then - fproj="${BASH_REMATCH[1]}" - owns+=("$f") - elif [[ "$f" =~ ^(.+)/([^/]+)/[^/]+$ ]]; then # Package archive? - local fsect_dir="${BASH_REMATCH[1]}" - - fproj="${BASH_REMATCH[2]}" - pkgs+=("$f") - - # Find the archive section name associated with the extracted section - # directory in 'src_sections' (a value-to-key lookup). - # - local fsect= - - # The "*" key is a catch-all for unknown submitted section names and, - # if present, will share a value (section directory) with one of the - # known section names and therefore must be skipped. - # - # If there is no mapping in 'src_sections' to the extracted section - # directory then the file path is invalid. - # - local k - for k in "${!src_sections[@]}"; do - if [[ ("${src_sections[$k]%/}" == "$fsect_dir") && - ("$k" != "*") ]]; then - fsect="$k" # Current file's section name. - break - fi - done - - if [ -z "$fsect" ]; then - info "unable to find section name for file '$f'" - return - fi - - # Set the source section name and directory if unset; otherwise fail - # if the current file is not from the source section. - # - if [ -z "$src_sect" ]; then - src_sect="$fsect" - src_sect_dir="$fsect_dir" - elif [ "$fsect" != "$src_sect" ]; then - info "cannot include commit $i: '$f' is not in section $src_sect" - return - fi - else - info "unrecognized type of file '$f'" - return - fi - - # Set the bundle project if unset; otherwise fail if the current file is - # not from the bundle project. - # - # Note: $fproj cannot be empty here (see above). - # - if [ -z "$proj" ]; then - proj="$fproj" - elif [ "$fproj" != "$proj" ]; then - info "cannot include commit $i: '$f' is not in project $proj" - return - fi - done < <(commit_files "$h") - done - - # Finalize migration variables the values of which depend on whether the - # bundle contains at least one package archive or ownership manifests only. - # - # The source and destination commit messages are composed incrementally as - # the migration process proceeds. - # - local dst_sect # Destination section name. - local dst_sect_dir # Destination section directory. - local src_cmsg # Source commit message. - local dst_cmsg # Destination commit message. - - if [ ${#pkgs[@]} -ne 0 ]; then # Bundle contains package archive(s). - dst_sect="$src_sect" - - # If it exists, 'testing' overrides 'stable' at the destination. - # - if [[ ("$dst_sect" == "stable") && -v dst_sections["testing"] ]]; then - dst_sect="testing" - fi - - # Fail if the target section does not exist in the destination repository. - # - if [ ! -v dst_sections["$dst_sect"] ]; then - info "section '$dst_sect' does not exist in the destination repository" - return - fi - - dst_sect_dir="${dst_sections[$dst_sect]}" - - src_cmsg="Migrate $proj to $dst_repo_name/$dst_sect"$'\n\n' - dst_cmsg="Migrate $proj from $src_repo_name/$src_sect"$'\n\n' - else # Bundle consists only of ownership manifests. - - # The setup where the ownership authentication is disabled on the - # destination but enabled on source is probably obscure, but let's - # consider it possible since the submit-git handler allows such a setup. - # - if [ -n "$dst_owners" ]; then - src_cmsg="Migrate $proj ownership info to $dst_repo_name"$'\n\n' - dst_cmsg="Migrate $proj ownership info from $src_repo_name"$'\n\n' - else - src_cmsg="Remove $proj ownership info"$'\n\n' - dst_cmsg= # Nothing to commit. - fi - fi - - # Ensure that the source and destination repositories are clean if the - # migration of any file fails. - # - # Note that the source repository cannot have untracked files so we - # git-clean only the destination repository. - # - function cleanup () - { - info "migration failed; resetting and cleaning repositories" - - if ! run git -C "$src_dir" reset --hard || - ! run git -C "$dst_dir" reset --hard || - ! run git -C "$dst_dir" clean --force; then - info "failed to reset/clean repositories -- manual intervention required" - fi - } - trap cleanup EXIT - - # Migrate the bundle's package archive files. - # - for f in "${pkgs[@]}"; do - # Get the current package's name and version from its embedded manifest - # (we already have the source project in $proj). - # - local p - p=($(extract_pkg_info "$src_dir/$f")) - - local name="${p[0]}" - local src_version="${p[1]}" - - # Check for duplicate package in all sections. Use <name>-<version>.* - # without .tar.gz in case we want to support more archive types later. - # - # Note that, for example, foo-bar version 1.0 and foo version bar-1.0 have - # the same archive name foo-bar-1.0.tar.gz. - # - local s - for s in "${!dst_sections[@]}"; do - local p - IFS=$'\n' eval \ - 'p=($(bpkg_rep_pkg_find_archive "$name-$src_version.*" \ - "$dst_dir/${dst_sections[$s]}"))' - - if [ "${#p[@]}" -ne 0 ]; then - local n="${p[0]}" - local v="${p[1]}" - local a="${p[3]}" - - if [ "$n" == "$name" ]; then - error "duplicate of $name/$src_version at '$a'" - else - error "conflict of $name/$src_version with $n/$v at '$a'" - fi - fi - done - - # In the destination repository, find and remove package archive files - # which are other alpha/beta versions or revisions of the current source - # package. - # - local vpat # Version pattern. - case "$dst_sect" in - alpha|beta) vpat="*" ;; # All package versions. - *) vpat="$src_version*" ;; # All package version revisions. - esac - - # Packages in the destination repository to be considered for replacement. - # - local dst_files - - IFS=$'\n' eval \ - 'dst_files=($(bpkg_rep_pkg_find_archives "$name" \ - "$vpat" \ - "$dst_dir/$dst_sect_dir"))' - - # If true, the source package replaces one or more packages in the - # destination repository. - # - local repl= - - local dst_f - for dst_f in "${dst_files[@]}"; do - local p - p=($(extract_pkg_info "$dst_f")) - - local dst_version="${p[1]}" - local dst_project="${p[2]}" - - # Ask whether or not to drop the current destination package. - # - # Include the project names in the prompt if the destination package's - # project differs from that of the source package. - # - local src="$src_version" - local dst="$name/$dst_version" - if [ "$dst_project" != "$proj" ]; then - src+=" ($proj)" - dst+=" ($dst_project)" - fi - - while true; do - read -p "replace $dst with $src? [y/n]: " opt - - case "$opt" in - "y") - repl=true - dst_cmsg+=" replace $name/$dst_version with $src_version"$'\n' - run git -C "$dst_dir" rm --quiet "${dst_f#$dst_dir/}" - break - ;; - "n") - break - ;; - esac - done - done - - # Migrate the current package. - # - src_cmsg+=" remove $name/$src_version"$'\n' - if [ ! "$repl" ]; then - dst_cmsg+=" add $name/$src_version"$'\n' - fi - migrate_file "$f" "$dst_sect_dir/$proj" - done - - # Migrate the bundle's ownership manifests. - # - # If ownership authentication is disabled on the destination repository, - # only remove ownership manifests from the source repository (that is, don't - # migrate). - # - for f in "${owns[@]}"; do - src_cmsg+=" remove $(dirname $f)/*"$'\n' - - if [ -n "$dst_owners" ]; then - local dp=$(dirname "${f/$src_owners/$dst_owners}") # Destination path. - - # Let the commit message reflect whether this is a new ownership - # manifest or is replacing an existent one. - # - if [ ! -e "$dst_dir/$dp/$(basename "$f")" ]; then - dst_cmsg+=" add $dp/*"$'\n' - else - dst_cmsg+=" update $dp/*"$'\n' - fi - - migrate_file "$f" "$dp" - else - run git -C "$src_dir/" rm --quiet "$f" - fi - done - - # Commit the changes made to the source and destination repositories. - # - info - run git -C "$src_dir" commit -m "$src_cmsg" - - if [ -n "$dst_cmsg" ]; then - info - run git -C "$dst_dir" commit -m "$dst_cmsg" - fi - - info - - # Remove the migrated commits from the pending sequence and clear the - # bundle. - # - for i in "${!bundle[@]}"; do - unset pending_seq[i-1] - done - pending_seq=("${pending_seq[@]}") # Remove the gaps created by unset. - bundle=() - - migrate_result=true - - # All files have been migrated successfully so clear the EXIT trap. - # - trap EXIT - - # Pause to give the operator a chance to look at the commits before the list - # of remaining pending commits is displayed. - # - read -p "Press Enter to continue: " -} - -# Push local changes to the remote source and/or destination git repositories. -# -# Push to the destination repository first because thus the migrated files -# will be in both remote repositories until the completion of the subsequent -# push to the source repository (which may fail or take long). Although this -# is an inconsistent state, it is safe because other programs such as a -# submission handler will be able to detect the duplicates and therefore -# refuse to do anything. If, on the other hand, we pushed to the source first, -# the migrated files would not exist in either remote repository until the -# push to the destination repository completed. In this state the submission -# handler would, for example, accept a resubmission of the migrated packages or -# erroneously establish ownership for already owned project/package names. -# -function push () -{ - # Let's print additional diagnostics on git-push failure, to emphasize for - # the user which of the two repositories we have failed to push. - # - if ! run git -C "$dst_dir" push; then - error "push to $dst_repo_name failed" - fi - - if ! run git -C "$src_dir" push; then - error "push to $src_repo_name failed" - fi -} - -# Present the list of pending commits to the user, oldest first, marking files -# that were deleted by subsequent commits with `*`: -# -# 001 (deadbeef) Add libfoo/1.2.3 -# -# 1/testing/foo/libfoo-1.2.3.tar.gz -# owners/foo/project-owner.manifest -# owners/foo/libfoo/package-owner.manifest -# -# 002 (c00l0fff) Add bar/1.2.3 -# -# * 1/testing/bar/libbar-1.2.3.tar.gz -# 1/testing/bar/libbaz-1.2.3.tar.gz -# -# 003 (deadbabe) Add libbar/1.2.3+1 -# -# 1/testing/bar/libbar-1.2.3+1.tar.gz -# -# Note that files deleted by subsequent commits may still be in the -# repository. See migrate() for an example. -# -# Then prompt the user for the action (showing the current bundle): -# -# [001 002][<N>,m,c,p,q,l,?]: -# -# <N> - add commit to the commit bundle -# m - migrate the selected commit bundle -# c - clear the selected commit bundle -# p - push source and destination repositories -# l - print pending commits -# q - quit (prompting to push if any actions have been taken) -# ? - print this help -# -# The user interaction loop. -# -# In each iteration, present the list of pending commits, display the menu of -# actions, read the user's input, and perform the chosen action. -# -# True if any changes have been made to the source and/or destination git -# repositories (in which case the user will be asked whether or not to push -# before quitting). -# -need_push= - -while true; do - # Show the pending commits. - # - if [ "${#pending_seq[@]}" -eq 0 ]; then - info "no more pending commits" - fi - - for ((i=0; i != "${#pending_seq[@]}"; i++)); do - h="${pending_seq[$i]}" - - # Print commit number, hash, and subject. - # - # The commit number is left-padded with 0s to 3 digits. Prefix with a - # newline to separate the first commit from the git-pull output and the - # rest from the previous commit info block. - # - subj="$(git -C "$src_dir" log -n 1 --pretty=format:%s "$h")" - printf "\n%.3d (%s) %s\n\n" "$((i+1))" "$h" "$subj" >&2 - - # Print this commit's files. - # - # Fetch from the git repository the list of files added by the current - # commit. Print each file's path and, if it was deleted by a subsequent - # commit, mark with an asterisk. - # - # Note that 'file_commits' is populated above from the list of files - # currently in the source repository. Therefore, if git says a file was - # added by a commit but it is associated with a different commit hash in - # 'file_commits' it means the file was deleted and added back by later - # commits; and if there is no mapping for the file it means it was deleted - # but not added back (that is, it's no longer in the repository). So we - # mark the re-added file with an exclamation. - # - while read -d '' f; do - if [ "${file_commits[$f]}" == "$h" ]; then - info " $f" # File was last added by the current commit. - elif [ -v file_commits["$f"] ]; then - info " ! $f" # File was deleted and added back by subsequent commits. - else - info " * $f" # File was deleted but not added back. - fi - done < <(commit_files "$h") - done - - # Prompt the user for the action (showing the current bundle), get user - # input, and perform the selected action. - # - # Note that we could adapt the menu according to the current state (don't - # offer to migrate if the bundle array is empty, etc) but let's not - # complicate the logic. - # - # Breaking out of this loop prints the pending commit list again. - # - while true; do - # Sort commit bundle in ascending order. - # - # Expand the 'bundle' associative array's keys into a single word in which - # they are separated by spaces (the first member of IFS) using the - # ${!a[*]} syntax; replace each space with a newline before piping to - # 'sort', which is newline-based; finally collect sort's output into an - # array using the a=() syntax, which splits on newline (the last member of - # IFS) because neither space nor tab characters (the other members of IFS) - # can occur in the keys. - # - bundle_sorted=($(sed 's/ /\n/g' <<<"${!bundle[*]}" | sort -)) - - printf "\n" - read -p "[${bundle_sorted[*]}][<N>,m,c,p,l,q,?]: " opt - - case "$opt" in - # Add commit to bundle. - # - [0-9]*) - if [[ ("$opt" -gt 0) && ("$opt " -le "${#pending_seq[@]}") ]]; then - printf -v opt "%.3d" "$opt" # Format as in pending commit list. - if [ ! -v bundle["$opt"] ]; then - bundle["$opt"]=true - info "commit $opt (${pending_seq[$opt-1]}) added to selected bundle" - else - info "commit $opt is already in the bundle" - fi - else - info "non-existent commit number $opt" - fi - ;; - # Migrate the commit bundle. - # - m) - migrate - if [ "$migrate_result" ]; then - need_push=true - break - fi - ;; - # Clear the commit bundle. - # - c) - bundle=() - break - ;; - # Push changes. - # - p) - push - need_push= - break - ;; - # Redraw the pending commit list. - # - l) - break - ;; - # Quit. - # - q) - if [ ! "$need_push" ]; then - exit 0 - fi - - while true; do - read -p "push changes? [y/n/(c)ancel]: " opt - - case "$opt" in - "c") - break # Print options menu again. - ;; - "y") - push - exit 0 - ;; - "n") - exit 0 - ;; - *) - continue - ;; - esac - done - ;; - # ? or invalid option: print menu. - # - *) - cat <<-EOF - - <N> - add commit to the commit bundle - m - migrate the selected commit bundle - c - clear the selected commit bundle - p - push source and destination repositories - l - print pending commits - q - quit (prompting to push if any actions have been taken) - ? - print this help -EOF - ;; - esac - done -done diff --git a/bpkg-rep/package-archive.bash.in b/bpkg-rep/package-archive.bash.in deleted file mode 100644 index fd9497a..0000000 --- a/bpkg-rep/package-archive.bash.in +++ /dev/null @@ -1,170 +0,0 @@ -# file : bpkg-rep/package-archive.bash.in -# license : MIT; see accompanying LICENSE file - -# Utility functions useful for managing package archives. - -if [ "$bpkg_rep_package_archive" ]; then - return 0 -else - bpkg_rep_package_archive=true -fi - -@import libbutl/manifest-parser@ - -# We expect the user to set the bpkg_rep_bpkg variable to the bpkg program -# path. -# -if [ ! -v bpkg_rep_bpkg ]; then - echo "error: variable bpkg_rep_bpkg is not set" >&2 - exit 1 -fi - -# Extract the package information from a package archive and print it in the -# '<name> <version> <project>' form, where the project field is empty if the -# project value is not specified in the manifest. -# -# Note that, in particular, it verifies that the archive file name matches the -# package name and version. -# -function bpkg_rep_pkg_verify_archive () # <path> -{ - # We can't use the process substitution for input redirect here, since such - # a process failure is not trapped. Thus, read the manifest file into a - # variable and parse it afterwards, which is probably ok since package - # manifests are normally not too big. - # - # Note that alternatively we could use the process substitution for running - # bpkg, treat the name value absence as indication of a failure, and exit - # with non-zero status if that's the case. Feels a bit hackish though. - # - local m - m="$("$bpkg_rep_bpkg" pkg-verify --manifest "$1")" - - butl_manifest_parser_start <<<"$m" - - local name= - local version= - local project= - - local n v - while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do - case "$n" in - name) name="$v" ;; - version) version="$v" ;; - project) project="$v" ;; - esac - done - - butl_manifest_parser_finish - - echo -n "$name $version $project" -} - -# Search for package archives in a directory using the package name and -# version pattern and printing their paths newline-separated. If the version -# argument is '*', then print archives for all package versions. Otherwise if -# the version contains the trailing '*', then print archives for all revisions -# of the specified version and for the exact version otherwise. For example: -# -# bpkg_rep_pkg_find_archives foo '*' dir/ -# bpkg_rep_pkg_find_archives foo '1.0*' dir/ -# bpkg_rep_pkg_find_archives foo '1.0' dir/ -# -# Note that the resulting archive paths include the specified directory as a -# prefix. -# -# NOTE: this function can be called with overriden IFS. -# -function bpkg_rep_pkg_find_archives () # <name> <version> <dir> -{ - IFS=$' \t\n' bpkg_rep_pkg_find_archives_impl "$@" -} - -function bpkg_rep_pkg_find_archives_impl () -{ - local nam="$1" - local ver="$2" - local dir="$3" - - local r="" - - if [ -d "$dir" ]; then - local vr # Version with the revision stripped, if search for revisions. - local np # File name pattern for archives search. - - if [ "$ver" != "*" -a "${ver: -1}" == "*" ]; then # <version>* - vr="$(sed -n -re 's%^(\+?[^+]+)(\+[0-9]+)?\*$%\1%p' <<<"$ver")" - np="$nam-$vr*.*" # foo-1.0*.*, etc. - else # * or <version> - np="$nam-$ver.*" # foo-*.*, foo-1.0.*, etc. - fi - - # Go through the potentially matching archives (for example, for 'foo' - # '1.2.3+2*': foo-1.2.3.tar.gz, foo-1.2.3+1.tar.gz, foo-1.2.30.tar.gz, - # etc) and return those which package name and version match properly. - # - local f - while read f; do - local p - p=($(bpkg_rep_pkg_verify_archive "$f")) - - local n="${p[0]}" - local v="${p[1]}" - - if [[ "$n" == "$nam" && - ( "$ver" == "*" || \ - "$v" == "$ver" || \ - ( -n "$vr" && "$v" =~ ^"$vr"(\+[0-9]+)?$ )) ]]; then - - if [ -n "$r" ]; then - r="$r"$'\n'"$f" - else - r="$f" - fi - fi - done < <(find "$dir" -type f -name "$np") - fi - - if [ -n "$r" ]; then - echo -n "$r" - fi -} - -# Search for a package archive in a directory using a file name pattern. If -# the archive is found, then print the package information in the -# '<name>\n<version>\n<project>\n<path>' form, where the project field is -# empty if the project value is not specified in the manifest. -# -# Note that if there are multiple archives matching the pattern, then it is -# unspecified which one is picked. -# -# NOTE: this function can be called with overriden IFS. -# -function bpkg_rep_pkg_find_archive () # <pattern> <dir> -{ - IFS=$' \t\n' bpkg_rep_pkg_find_archive_impl "$@" -} - -function bpkg_rep_pkg_find_archive_impl () -{ - local pat="$1" - local dir="$2" - - if [ -d "$dir" ]; then - local f - - # We could probably use -print -quit but this is not portable (NetBSD - # needs -exit instead of -quit). - # - f="$(find "$dir" -type f -name "$pat" | head -n 1)" - - if [ -n "$f" ]; then - - local p - p=($(bpkg_rep_pkg_verify_archive "$f")) - - printf "${p[0]}\n${p[1]}\n${p[2]}\n$f" - return - fi - fi -} diff --git a/bpkg-rep/publish.in b/bpkg-rep/publish.in deleted file mode 100644 index a7b3d1a..0000000 --- a/bpkg-rep/publish.in +++ /dev/null @@ -1,329 +0,0 @@ -#!/usr/bin/env bash - -# file : bpkg-rep/publish.in -# license : MIT; see accompanying LICENSE file - -# Update (bpkg-rep-create(1)) and publish (rsync(1)) an archive-based -# repository. -# -# Pull a pre-cloned (read-only) git repository with the contents of an -# archive-based bpkg repository. Bail out if nothing changed from the the -# previous run. Otherwise, regenerate the repository meta-data by running -# bpkg-rep-create(1) on each section of the repository and, optionally, -# synchronize it to one or more destinations with rsync. -# -# The repository contents are expected to be in the <dir>/1/ subdirectory. The -# script saves the last successfully published commit in the <dir>.publish -# file. -# -# --destination|-d <host>:<dir> -# -# Remote host and directory to rsync the repository to. Note that the -# directory should include the 1/ component and any sub-directories that may -# follow. In other words, the rsync command will be in the form: -# -# rsync ... <dir>/1/ <host>:<dir>/ -# -# See below for the actual rsync command including a brief explanation of -# options passed. -# -# Repeat this option to specify multiple destinations. In this case, the -# destinations are synced in the order specified with the first failure -# terminating the process (so if you have a "primary" destination and a -# "mirror", you probably want to specify the former first). -# -# --timeout <seconds> -# -# Git and rsync operation timeout. Specifically, the operation will be -# aborted if there is no network activity for the specified time. Default is -# 60 seconds. Note that currently the git timeout is only supported for the -# http(s) transport. -# -# --lock-timeout <seconds> -# -# The repository lock timeout. Fail if another instance of the script does -# not release the repository in the specified time. The default is 0 (do -# not wait). -# -# Note that you will most likely want to specify a non-zero timeout for cron -# jobs that may potentially overlap. -# -# --log-dir <dir> -# -# Directory to create the temporary log files in. If unspecified, the stderr -# is not redirected and no log is created by default. -# -# The log is dumped to stderr in case of an error or at the end of execution -# unless in the quiet mode and is then deleted. -# -# --quiet -# -# Run quiet. Specifically, don't dump the log to stderr on exit with zero -# status. -# -# --config <path> -# -# The configuration file containing a bash fragment. Repeat this option to -# specify multiple configurations that will be sourced in the order -# specified. -# -# --bpkg <path> -# -# The package manager program to be used for the repository update. This -# should be the path to the bpkg executable. -# -usage="usage: $0 [<options>] <dir> [<rep-create-options>] [-- <rsync-options>]" - -trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. - -@import bpkg-rep/utility@ - -# The script own options. -# -repo_ver=1 -destinations=() -timeout=60 -lock_timeout=0 -log_dir= -quiet= -configurations=() -bpkg= - -while [ $# -gt 0 ]; do - case $1 in - --destination|-d) - shift - destinations+=("${1%/}") - shift || true - ;; - --timeout) - shift - timeout="$1" - shift || true - ;; - --lock-timeout) - shift - lock_timeout="$1" - shift || true - ;; - --log-dir) - shift - log_dir="${1%/}" - shift || true - ;; - --quiet) - shift - quiet=true - ;; - --config) - shift - configurations+=("$1") - shift || true - ;; - --bpkg) - shift - bpkg="$1" - shift || true - ;; - --) - shift - break - ;; - *) - break - ;; - esac -done - -# The repository directory. -# -repo_dir="${1%/}" -shift || true - -# bpkg-rep-create options. -# -rep_create_options=() - -while [ $# -gt 0 ]; do - case $1 in - --) - shift - break - ;; - *) - rep_create_options+=("$1") - shift - ;; - esac -done - -# rsync options. -# -rsync_options=() - -while [ $# -gt 0 ]; do - rsync_options+=("$1") - shift -done - -# Validate options and arguments. -# -if [ -z "$repo_dir" ]; then - error "$usage" -fi - -if [ ! -d "$repo_dir" ]; then - error "'$repo_dir' does not exist or is not a directory" -fi - -# If the log directory is specified then redirect stderr to the log file and -# setup the trap that dumps it on exit, if required. -# -if [ -n "$log_dir" ]; then - - if [ ! -d "$log_dir" ]; then - error "'$log_dir' does not exist or is not a directory" - fi - - # Create the log file. - # - log="$(mktemp "$log_dir/$(basename "$repo_dir").XXXXXXXXXX")" - - # Save the stderr file descriptor so we can dump the log into it on exit, if - # required. Then redirect it to the log file. - # - exec {stderr}>&2 - exec 2>>"$log" - - function exit_trap () - { - local status="$?" - - # Dump the log to stderr if exiting with non-zero status or verbose. - # - if [ $status -ne 0 -o ! "$quiet" ]; then - - # Keep the log if failed to dump for any reason. - # - if ! cat "$log" >&$stderr; then - return - fi - fi - - rm -f "$log" - } - - trap exit_trap EXIT -fi - -# Source the configurations. -# -for c in "${configurations[@]}"; do - source "$c" >&2 -done - -# Make sure the commit file is present. -# -published_commit="$repo_dir.publish" -touch "$published_commit" - -# Open the reading file descriptor and lock the repository. Fail if unable to -# lock before timeout. -# -exec {cfd}<"$published_commit" - -if ! flock -w "$lock_timeout" "$cfd"; then - info "another instance is already running" - exit 2 -fi - -# Pull the repository. -# -# Git doesn't support the connection timeout option. The options we use are -# just an approximation of the former, that, in particular, don't cover the -# connection establishing. To work around this problem, before running a git -# command that assumes the remote repository communication we manually check -# connectivity with the remote repository. -# -if ! remote_url="$(git -C "$repo_dir" config --get remote.origin.url)"; then - error "'$repo_dir' is not a git repository" -fi - -run check_git_connectivity "$remote_url" "$timeout" - -# Fail if no network activity happens during the time specified. -# -run git -c http.lowSpeedLimit=1 -c "http.lowSpeedTime=$timeout" \ --C "$repo_dir" pull -v >&2 - -# Match the HEAD commit id to the one stored in the file. If it matches, then -# nothing changed in the repository from the previous run and we can silently -# bail out. -# -commit="$(git -C "$repo_dir" rev-parse HEAD)" -pc="$(cat <&"$cfd")" - -if [ "$commit" == "$pc" ]; then - quiet=true - exit 0 -fi - -# If bpkg path is not specified, then use the bpkg program from the script -# directory, if present. Otherwise, use the 'bpkg' path. -# -if [ -z "$bpkg" ]; then - bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" - - if [ ! -x "$bpkg" ]; then - bpkg=bpkg - fi -fi - -# Find repository sections. -# -manifests="$(find "$repo_dir/$repo_ver" -type f -name repositories.manifest)" - -# Update the repository sections. -# -while read f; do - run "$bpkg" rep-create "${rep_create_options[@]}" "$(dirname "$f")" -done <<<"$manifests" - -# rsync (over ssh) the repository to the destinations. -# -# Approximate the data transfer timeout via the ServerAlive* ssh options, -# rounding the timeout up to the nearest multiple of ten. -# -# Note: must not contain spaces/use quoting (see rsync -e option). -# -n=$(($timeout > 0 ? ($timeout + 9) / 10 : 1)) -ssh_options=(-o ConnectTimeout=$timeout \ - -o ServerAliveInterval=10 \ - -o ServerAliveCountMax=$n) - -for d in "${destinations[@]}"; do - - # -r (recursive) - # -l (copy symlinks as symlinks) - # -t (preserve timestamps) - # -O (omit dir timestamps) - # - # -c (use checksum) - # -e (remote shell command) - # - # --chmod=ugo=rwX (give new files the destination-default permissions) - # --safe-links (ignore symlinks pointing outside the tree) - # --delay-updates (first upload all files on the side then move) - # --prune-empty-dirs (remove empty dirs) - # --delete-after (delete entries after the transfer) - # - # We also exclude hidden files (start with dot). - # - run rsync -v -rltO -c --chmod=ugo=rwX --safe-links --delay-updates \ ---exclude '.*' --prune-empty-dirs --delete-after -e "ssh ${ssh_options[*]}" \ -"${rsync_options[@]}" "$repo_dir/$repo_ver/" "$d/" >&2 - -done - -echo "$commit" >"$published_commit" diff --git a/bpkg-rep/utility.bash.in b/bpkg-rep/utility.bash.in deleted file mode 100644 index 7ac7c54..0000000 --- a/bpkg-rep/utility.bash.in +++ /dev/null @@ -1,86 +0,0 @@ -# file : bpkg-rep/utility.bash.in -# license : MIT; see accompanying LICENSE file - -# Utility functions useful for implementing bpkg repository utilities. - -if [ "$bpkg_rep_utility" ]; then - return 0 -else - bpkg_rep_utility=true -fi - -# Diagnostics. -# -function info () { echo "$*" 1>&2; } -function error () { info "$*"; exit 1; } - -# Trace a command line, quoting empty arguments as well as those that contain -# spaces. -# -function trace_cmd () # <cmd> <arg>... -{ - local s="+" - while [ $# -gt 0 ]; do - if [ -z "$1" -o -z "${1##* *}" ]; then - s="$s '$1'" - else - s="$s $1" - fi - - shift - done - - info "$s" -} - -# Trace the current function name and arguments. -# -function trace_func () # <args>... -{ - trace_cmd "${FUNCNAME[1]}" "$@" -} - -# Trace and run a command. -# -function run () # <cmd> <arg>... -{ - trace_cmd "$@" - "$@" -} - -# Return lower-case URL scheme or empty string if the argument doesn't look -# like a URL. -# -function url_scheme () # <url> -{ - sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1" -} - -# Check that the git repository properly responds to the probing request -# before the timeout (in seconds). Noop for protocols other than HTTP(S). -# -function check_git_connectivity () # <repo-url> <timeout> -{ - local url="$1" - local tmo="$2" - - local s - s="$(url_scheme "$url")" - - if [ "$s" == "http" -o "$s" == "https" ]; then - local u q - - u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part. - q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part. - - if [ -z "$q" ]; then - u="$u/info/refs?service=git-upload-pack" - else - u="$u/info/refs$q&service=git-upload-pack" - fi - - # Here we limit the time for the whole operation. - # - curl -S -s --max-time "$tmo" "$u" >/dev/null - fi -} |