aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--LICENSE2
-rw-r--r--bpkg-util/manage.in1852
-rw-r--r--bpkg-util/package-archive.bash.in66
-rw-r--r--build/root.build7
-rw-r--r--manifest12
-rw-r--r--tests/package-archive/driver.in10
-rw-r--r--tests/package-archive/testscript20
7 files changed, 1404 insertions, 565 deletions
diff --git a/LICENSE b/LICENSE
index 2cf3738..da8df38 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
MIT License
-Copyright (c) 2014-2020 the build2 authors (see the AUTHORS file).
+Copyright (c) 2014-2023 the build2 authors (see the AUTHORS file).
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/bpkg-util/manage.in b/bpkg-util/manage.in
index 5fbcf40..39d4ee7 100644
--- a/bpkg-util/manage.in
+++ b/bpkg-util/manage.in
@@ -7,40 +7,56 @@
# license : MIT; see accompanying LICENSE file
# Interactively migrate packages and/or ownership manifests from a source git
-# repository to a destination git repository.
+# repository to a destination git repository or drop them from the source
+# repository.
#
-# Present the user with the list of commits that added the files currently in
+# The commit that most recently added a file or moved it to its current
+# location "owns" the file and the file "belongs to" the commit.
+#
+# Present the user with the list of commits that own the files currently in
# the source repository's working directory and ask the user to select from a
# menu the action to perform on a selection of these commits.
#
-# As the files added by these commits are pending a move to the destination
-# repository, these commits will be referred to as "pending" commits.
+# As these commits' files are pending removal or a move to the destination
+# repository they will be referred to as "pending" commits.
#
-# Actions that can be performed on a selection of pending commits include
-# moving them from the source repository to a single commit in the destination
-# repository and dropping them from the source repository.
+# Actions that can be performed on a selection of pending commits ("commit
+# operations") include moving them from the source repository to a single
+# commit in the destination repository and dropping them from the source
+# repository. A file that has been operated upon is called an "actioned file"
+# and a commit whose files have all been actioned an "actioned commit". It is
+# also possible to operate on only a subset of each selected commit's files
+# instead of all of them. These are called "file operations" and are selected
+# from the "commit splitting screen"; see below for more information.
#
# The flow of this script, in broad strokes, is as follows: for each file in
-# the source repository directory, find the hash of the commit that added it;
+# the source repository directory, find the hash of the commit it belongs to;
# these are the pending commits. Arrange the pending commits in chronological
# order. Display to the user the pending commits along with the files they
-# added. Let the user select one or more pending commits and an action to be
-# performed on them. Each successful action results in a commit to the source
-# and/or destination repositories and leaves both repositories in a clean
-# state. Once the action has been performed, redisplay the updated pending
-# commit list and prompt for the next action. Pushing to the remote
-# repositories, a supported operation, can be done at any time during the
-# session.
+# own. Let the user select one or more pending commits and an operation to be
+# performed on them. Each successful operation results in a commit to the
+# source and/or destination repositories and leaves both repositories in a
+# clean state. Once the operation has been performed, re-initialize the
+# pending commit list from the files in the source repository (that is, from
+# scratch), redisplay the updated pending commit list, and prompt for the next
+# action. Pushing to the remote repositories, a supported operation, can be
+# done at any time during the session. (Note that only in-memory structures
+# are updated after a file operation in the commit splitting screen -- the
+# pending commit list is only re-initialized (from disk) when the user returns
+# to the main screen.)
#
# Two modes of operation are supported. In source-management mode, package
-# archives -- from any section -- and/or ownership manifests are migrated from
-# the source repository to the destination repository. In
-# destination-management mode, package archives (only) are migrated from one
-# of the sections in the destination repository to its counterpart section
-# (also in the destination repository). These mode-specific source directories
-# are called "managed directories" and the files they contain "managed files",
-# excluding those with an invalid path (project component is missed, etc); any
-# other directory or file is "unmanaged".
+# archives -- from any section -- and/or ownership manifests are dropped from
+# the source repository or migrated to the destination repository. In
+# destination-management mode, starting from one of the sections in the
+# destination repository, package archives or ownership manifests are dropped,
+# or package archives (not ownership manifests) migrated to its counterpart
+# section (also in the destination repository). These mode-specific source
+# directories are called "managed directories" and the files they contain
+# "managed files". The "unmanaged files" include any file outside of a managed
+# directory, any ownership manifest not related (by package name or project
+# name) to a managed package archive, and any file inside a managed directory
+# with an invalid path (project component is missed, etc).
#
# The destination-management mode is actually just a name used to refer to a
# number of more specific, near-identical modes distinguished only by the
@@ -61,12 +77,27 @@
# --testing[=<filter>]
#
# Enter the testing-management mode: manage the testing->stable transitions
-# in the destination repository.
+# in, or drop commits and/or files from, the destination repository.
#
# --stable[=<filter>]
#
-# Enter the stable-management mode: manage the stable->legacy transitions
-# in the destination repository.
+# Enter the stable-management mode: manage the stable->legacy transitions in
+# the destination repository. Note that commits/files cannot be dropped in
+# this mode and, by extension, it does not support any operations on
+# ownership manifests.
+#
+# --alpha[=<filter>]
+#
+# Enter the alpha-management mode: drop commits and/or files from the alpha
+# section in the destination repository.
+#
+# --beta[=<filter>]
+#
+# Enter the beta-management mode: drop commits and/or files from the beta
+# section in the destination repository.
+#
+# If none of the above modes are specified, operate in the source-management
+# mode.
#
# If <filter> is specified, then it is used to narrow down the list of commits
# to only those that contain packages with project name or package name and
@@ -76,9 +107,6 @@
# --stable=libexpat-1.2.3
# --stable=libexpat-2.*
#
-# If neither --testing nor --stable are specified, operate in the
-# source-management mode.
-#
# Arguments:
#
# <dir> The directory into which the source and destination repositories have
@@ -86,15 +114,18 @@
#
usage="usage: $0 [<options>] [<dir>]"
-# Source/destination repository inside <dir>. Note: also used in commit
-# messages.
+# The names of the source and destination repositories inside <dir>. Note:
+# also used in commit messages.
#
src_repo_name=queue
dst_repo_name=public
owd="$(pwd)"
trap "{ cd '$owd'; exit 1; }" ERR
-set -o errtrace # Trap in functions and subshells.
+set -o errtrace # Trap in functions and subshells.
+set -o pipefail # Fail if any pipeline command fails.
+shopt -s lastpipe # Execute last pipeline command in the current shell.
+shopt -s nullglob # Expand no-match globs to nothing rather than themselves.
@import bpkg-util/utility@
@@ -103,7 +134,7 @@ set -o errtrace # Trap in functions and subshells.
#
bpkg_util_bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg"
-if [ ! -x "$bpkg_util_bpkg" ]; then
+if [[ ! -x "$bpkg_util_bpkg" ]]; then
bpkg_util_bpkg=bpkg
fi
@@ -120,7 +151,7 @@ mode="source"
#
filter="*"
-while [ "$#" -gt 0 ]; do
+while [[ "$#" -gt 0 ]]; do
case "$1" in
--testing=*)
filter="${1#--testing=}"
@@ -136,60 +167,87 @@ while [ "$#" -gt 0 ]; do
mode="stable"
shift
;;
- *) break ;;
+ --alpha=*)
+ filter="${1#--alpha=}"
+ ;&
+ --alpha)
+ mode="alpha"
+ shift
+ ;;
+ --beta=*)
+ filter="${1#--beta=}"
+ ;&
+ --beta)
+ mode="beta"
+ shift
+ ;;
+ # Catch invalid options.
+ #
+ -*)
+ error "unknown option: $1"
+ ;;
+ # <dir> or end of options.
+ #
+ *)
+ break
+ ;;
esac
done
# Set the working directory.
#
-if [ $# -eq 0 ]; then
+if [[ "$#" -eq 0 ]]; then
dir="$owd"
-elif [ $# -eq 1 ]; then
+elif [[ "$#" -eq 1 ]]; then
dir="${1%/}" # <dir> with trailing slash removed.
else
error "$usage"
fi
# If in one of the destination-management modes, set the source repository
-# name to that of the destination repository.
+# name to that of the destination repository. But first save the real source
+# repository directory because we sometimes need to operate on it even if in
+# destination-management mode.
#
-if [ "$mode" != "source" ]; then
+real_src_dir="$dir/$src_repo_name" # The real source repository directory.
+if [[ "$mode" != "source" ]]; then
src_repo_name="$dst_repo_name"
fi
# The source and destination package repository directories.
#
# Note that, throughout this script, any path not explicitly prefixed with
-# "$src_dir/" or "$dst_dir/" is relative to the root of the source or
-# destination package repositories.
+# "$src_dir/", "$dst_dir/", or "$real_src_dir/" is relative to the root of the
+# source or destination package repositories.
#
src_dir="$dir/$src_repo_name"
dst_dir="$dir/$dst_repo_name"
-if [ ! -d "$src_dir" ]; then
+if [[ ! -d "$src_dir" ]]; then
error "'$src_dir' does not exist or is not a directory"
fi
-if [ ! -d "$dst_dir" ]; then
+if [[ ! -d "$dst_dir" ]]; then
error "'$dst_dir' does not exist or is not a directory"
fi
# Check that both git repositories are clean.
#
-if [ -n "$(git -C $src_dir status --porcelain)" ]; then
+if [[ -n "$(git -C $src_dir status --porcelain)" ]]; then
error "git repository in '$src_dir' is not clean"
fi
-if [ -n "$(git -C $dst_dir status --porcelain)" ]; then
+if [[ -n "$(git -C $dst_dir status --porcelain)" ]]; then
error "git repository in '$dst_dir' is not clean"
fi
# Use run() to show the user that git is the source of the diagnostics.
# "Already up to date", for example, is too vague.
#
-if [ "$mode" == "source" ]; then
- run git -C "$src_dir" pull >&2
-fi
+# Always pull the real source repository because we sometimes need to operate
+# on it even if in destination-management mode.
+#
+run git -C "$real_src_dir" pull >&2
run git -C "$dst_dir" pull >&2
# Load the source and destination repositories' submit configurations (section
@@ -204,6 +262,9 @@ run git -C "$dst_dir" pull >&2
# dst_sections. Otherwise, in source-management mode, store all source and
# destination section directories.
#
+# Normalize section directory paths by removing trailing slashes if they
+# exist.
+#
declare owners
declare -A sections
source "$src_dir/submit.config.bash"
@@ -213,7 +274,7 @@ declare -A src_sections
for s in "${!sections[@]}"; do
if [[ ("$mode" == "source") || ("$s" == "$mode") ]]; then
- src_sections["$s"]="${sections[$s]}"
+ src_sections["$s"]="${sections[$s]%/}" # Remove trailing '/', if any.
fi
done
@@ -223,6 +284,9 @@ source "$dst_dir/submit.config.bash"
# Section counterparts.
#
+# Used only in destination-management mode. If a section has no counterpart
+# then migration from it is disabled.
+#
declare -A sect_cparts=(["testing"]="stable"
["stable"]="legacy")
@@ -230,153 +294,23 @@ dst_owners="$owners"
declare -A dst_sections
for s in "${!sections[@]}"; do
if [[ ("$mode" == "source") || ("$s" == "${sect_cparts[$mode]}") ]]; then
- dst_sections["$s"]="${sections[$s]}"
- fi
-done
-
-# Fail if in destination-management mode and the source and/or destination
-# section is not configured for the destination repository. (The equivalent
-# source-management mode checks can only be done later.)
-#
-if [[ ("$mode" != "source") &&
- (! -v src_sections["$mode"] ||
- ! -v dst_sections["${sect_cparts[$mode]}"]) ]]; then
- error "section '$mode' and/or '${sect_cparts[$mode]}' not configured \
-in the destination repository"
-fi
-
-# Find all package archives and, if in source-management mode, owner manifest
-# files in the source repository.
-#
-# Every file in a repository section directory except *.manifest is a package
-# archive and every file in the owners directory is a project or package owner
-# manifest. Therefore run find separately on each section directory and the
-# owners directory to build a list containing only package-related files.
-#
-# Store the relative to the repository directory file paths in an array used
-# to build the set of pending commits.
-#
-src_files=()
-for s in "${src_sections[@]}"; do
- while read f; do
- src_files+=("${f#$src_dir/}")
- done < <(find "$src_dir/$s" -type f -not -name "*.manifest")
-done
-
-if [[ ("$mode" == "source") &&
- -n "$src_owners" &&
- -d "$src_dir/$src_owners" ]]; then
- while read f; do
- src_files+=("${f#$src_dir/}")
- done < <(find "$src_dir/$src_owners" -type f)
-fi
-
-# Build the set of pending commit hashes ("pending set").
-#
-# For each file in the source repository, find the most recent commit that
-# added it or moved it to its current location and store its abbreviated hash
-# (as key) inside the 'pending_set' associative array (note: unordered) and
-# (as value) inside the 'file_commits' associative array.
-#
-# 'file_commits' maps from the file path to the last commit to add it to the
-# repository. A file may have been added and removed by earlier commits and
-# could thus be migrated with the wrong commit unless care is taken (see the
-# example in the migration notes below).
-#
-# If in destination-management mode, exclude from `pending_set` those commits
-# without any package archives that match the pattern in `filter`.
-#
-declare -A pending_set
-declare -A file_commits
-for f in "${src_files[@]}"; do
-
- # -n 1: limit output to one commit (that is, the most recent)
- # --diff-filter=A: only show commits that added the specified file or
- # moved it into its current location ('R' is useless
- # if a path is specified)
- # --pretty=format:%h: output only the abbreviated commit hash
- #
- h="$(git -C "$src_dir" log -n 1 --diff-filter=A --pretty=format:%h -- "$f")"
-
- # Note that the hash cannot be empty because, after our clean checks at the
- # top, every file on disk must have been added by some commit (that is,
- # there can be no untracked files).
-
- # Add the commit to the pending set unless the current file is filtered out.
- #
- # Note: `src_files` can contain only package archives if we're in
- # destination-management mode so there's no need to check the file type.
- #
- # Note: $filter must be unquoted to match as a wildcard pattern.
- #
- if [[ "$mode" == "source" ||
- ("$(basename "$(dirname "$f")")" == $filter) || # Project name?
- ("$(basename "$f")" == $filter.*) ]]; then # Filename (with ext)?
- pending_set["$h"]=true
+ dst_sections["$s"]="${sections[$s]%/}" # Remove trailing '/', if any.
fi
-
- file_commits["$f"]="$h"
done
-# Arrange the pending commits in the chronological order.
+# Fail if in destination-management mode and the source section is not
+# configured for the destination repository. (The equivalent source-management
+# mode checks can only be done later.)
#
-# Go through the most recent commits in the git log which added or
-# moved/renamed one or more files, skipping those not present in the pending
-# set and keeping count to bail out as soon as we ordered all of them.
+# Note that checking whether the non-existence of a counterpart section is an
+# error can only be done later because some operations support sections
+# without a counterpart (for example, dropping from the alpha or beta sections
+# in destination-management mode).
#
-pending_seq=()
-for (( i=0; i != "${#pending_set[@]}"; )); do
- read h # The abbreviated commit hash.
-
- # If this is a pending commit, prepend its hash to the ordered array.
- #
- if [ "${pending_set[$h]}" ]; then
- pending_seq=("$h" "${pending_seq[@]}")
- ((++i))
- fi
-
- # --diff-filter=AR: only show commits that added or renamed files
- #
-done < <(git -C "$src_dir" log --diff-filter=AR --pretty=format:%h)
-
-if [ "${#pending_seq[@]}" -eq 0 ]; then
- info "good news, nothing to manage"
- exit 0
+if [[ ("$mode" != "source") && ! -v "src_sections[$mode]" ]]; then
+ error "section '$mode' not configured in the destination repository"
fi
-# Clean the source and destination repositories by discarding uncommitted
-# changes and removing unstaged files. (Note that the source repository cannot
-# have untracked files so we git-clean only the destination repository.)
-#
-function cleanup ()
-{
- info "migration failed; resetting and cleaning repositories"
-
- if ([ "$mode" == "source" ] && ! run git -C "$src_dir" reset --hard) ||
- ! run git -C "$dst_dir" reset --hard ||
- ! run git -C "$dst_dir" clean --force; then
- info "failed to reset/clean repositories -- manual intervention required"
- fi
-}
-
-# Return the list of files a commit added to the source repository.
-#
-function commit_files () # <commit-hash>
-{
- local h="$1"
-
- # git-diff-tree arguments:
- #
- # --diff-filter=A: select only files that were added.
- # -z: don't munge file paths and separate output fields with
- # NULs.
- # -r: recurse into subtrees (directories).
- #
- git -C "$src_dir" diff-tree \
- --no-commit-id --name-only --diff-filter=A -z -r \
- "$h"
-}
-
# Print information about the path of a source repository file.
#
# The information includes its class (package archive, ownership manifest, or
@@ -386,12 +320,13 @@ function commit_files () # <commit-hash>
# <path> must be relative to the source repository directory (`src_dir`).
#
# If the path refers to a managed archive file in the source repository, then
-# print `archive <project> <section>`.
+# print `archive\n<project>\n<section-dir>\n`. The section directory will be
+# relative to the source repository directory.
#
# Otherwise, if the path refers to a managed ownership manifest file in the
-# source repository, then print `ownership <project>`.
+# source repository, then print `ownership\n<project>\n`.
#
-# Otherwise the file is unmanaged; print `unmanaged`.
+# Otherwise the file is unmanaged; print `unmanaged\n`.
#
# Note that the function doesn't validate the file path exhaustively and may
# classify improperly named file (invalid base name, etc) as an archive or
@@ -409,7 +344,9 @@ function src_path_info () # <path>
local s
for s in "${src_sections[@]}"; do
if [[ "$f" =~ ^"$s"/([^/]+)/[^/]+$ ]]; then
- echo -n "archive ${BASH_REMATCH[1]} $s"
+ echo "archive"
+ echo "${BASH_REMATCH[1]}"
+ echo "$s"
return
fi
done
@@ -420,35 +357,265 @@ function src_path_info () # <path>
# (the project directory, again).
#
if [[ -n "$src_owners" && ("$f" =~ ^"$src_owners"/([^/]+)/.+$) ]]; then
- echo -n "ownership ${BASH_REMATCH[1]}"
+ echo "ownership"
+ echo "${BASH_REMATCH[1]}"
else
- echo -n "unmanaged"
+ echo "unmanaged"
fi
}
# Extract the package name, version, and project from a package archive's
-# manifest and print it to stdout in the '<name> <version> <project>' form. If
-# the manifest does not specify the project name, the package name is returned
-# as the project name.
+# manifest and print it to stdout in the `<name>\n<version>\n<project>\n`
+# form. If the manifest does not specify the project name, the package name is
+# returned as the project name.
#
function extract_pkg_info () # <archive>
{
local arc="$1"
- local r
- r=($(bpkg_util_pkg_verify_archive "$arc")) # <name> <version> <project>
- if [ ! -v r[2] ]; then
+ local r # (<name> <version> <project>)
+ bpkg_util_pkg_verify_archive "$arc" | readarray -t r
+
+ if [[ -z "${r[2]}" ]]; then
r[2]="${r[0]}"
fi
# Verify that the archive parent directory name matches the project.
#
local p="${r[2]}"
- if [ "$p" != "$(basename "$(dirname "$arc")")" ]; then
+ if [[ "$p" != "$(basename "$(dirname "$arc")")" ]]; then
error "'$arc' archive directory name does not match package project '$p'"
fi
- echo -n "${r[@]}"
+ local e
+ for e in "${r[@]}"; do
+ echo "$e"
+ done
+}
+
+# Contains the hashes of the pending commits in chronological order.
+#
+pending_seq=()
+
+# Maps from the file path to the commit it belongs to. A file may have been
+# added and removed by earlier commits and could thus be migrated with the
+# wrong commit unless care is taken (see the example in the migration notes
+# below). Every file contained in this map exists on disk.
+#
+declare -A file_commits
+
+# Initialize global variables that depend on disk/repository state that is
+# modified by commit and file operations.
+#
+function init_globals ()
+{
+ pending_seq=()
+ file_commits=()
+
+ # Find all package archives and owner manifest files in the source
+ # repository.
+ #
+ # Every file in a repository section directory except *.manifest is a
+ # package archive and every file in the owners directory is a project or
+ # package owner manifest. Therefore run find separately on each section
+ # directory and the owners directory to build a list containing only
+ # package-related files.
+ #
+ # Store the relative to the repository directory file paths in an array,
+ # `src_files`, used to build the set of pending commits.
+ #
+ # Note that directories in `src_sections` may be deleted by certain
+ # operations (for example, if the last package is migrated out of a
+ # section).
+ #
+ local src_files=()
+
+ local s
+ for s in "${src_sections[@]}"; do
+ local d="$src_dir/$s"
+ if [[ -d "$d" ]]; then
+ local f
+ while read f; do
+ src_files+=("${f#$src_dir/}")
+ done < <(find "$d" -type f -not -name "*.manifest")
+ fi
+ done
+
+ # Don't load ownership manifests if in stable-management mode because it
+ # does not support any operations on them.
+ #
+ if [[ ("$mode" != "stable") &&
+ -n "$src_owners" && -d "$src_dir/$src_owners" ]]; then
+ local f
+ while read f; do
+ src_files+=("${f#$src_dir/}")
+ done < <(find "$src_dir/$src_owners" -type f)
+ fi
+
+ # Build the set of pending commit hashes ("pending set").
+ #
+ # For each file in the source repository, find the commit it belongs to and
+ # store its abbreviated hash (as value) inside the `file_commits`
+ # associative array (note: unordered). With some exceptions (see below),
+ # these files' commits will also be stored (as key) inside the `pending_set`
+ # associative array.
+ #
+ # If in destination-management mode:
+ #
+ # - Unless in stable-management mode (which does not operate on ownership
+ # manifests at all), do not add unmanaged ownership manifests' commits
+ # to `pending_set`. (Note that these commits could be added as a result
+ # of other files, and that, in source-management mode, all ownership
+ # manifests are managed.)
+ #
+ # The `managed_projs` and `managed_pkgnames` associative arrays store
+ # the project and package names, respectively, of every package archive
+ # in the managed section and are used to identify unmanaged ownership
+ # manifests (see below). (Both are empty in source-management and
+ # stable-management modes.)
+ #
+ # - Exclude from `pending_set` those commits without any package archives
+ # that match the pattern in `filter`.
+ #
+ # Every file in `src_files` is added to `file_commits` without exception
+ # (that is, regardless of whether or not its commit was added to
+ # `pending_set`) otherwise it could potentially get incorrectly attributed
+ # to an earlier, unfiltered commit (consider the edge case example under the
+ # general migration comments (above migrate_src()) with a filter of "bar.*")
+ # or caused to look like it's been deleted from disk when displayed to the
+ # user.
+ #
+ local -A pending_set=() # Hashes of the pending commits.
+ local -A managed_projs=() # Project names of all managed archives.
+ local -A managed_pkgnames=() # Package names of all managed archives.
+ local h
+ for f in "${src_files[@]}"; do
+
+ # -1: limit output to one commit (that is, the most
+ # recent)
+ # --diff-filter=A: only show commits that added the specified file or
+ # moved it into its current location ('R' is useless
+ # if a path is specified)
+ # --pretty=format:%h: output only the abbreviated commit hash
+ #
+ h="$(git -C "$src_dir" log -1 --diff-filter=A --pretty=format:%h -- "$f")"
+
+ # Note that the hash cannot be empty because, after our clean checks at
+ # the top, every file on disk must have been added by some commit (that
+ # is, there can be no untracked files).
+
+ file_commits["$f"]="$h"
+
+ # Don't add unmanaged ownership manifests to `pending_set` (by skipping
+ # them) if in any destination-management mode but stable-management (in
+ # which case there are none in `src_files`). An ownership manifest is
+ # unmanaged if its project or package name has not been seen (archives
+ # come before ownership manifests in `src_files`). To this end, also
+ # record the package and project names of every package archive.
+ #
+ if [[ ("$mode" != "source") && ("$mode" != "stable") ]]; then
+ local fi
+ src_path_info "$f" | readarray -t fi
+ local ftype="${fi[0]}" # File type.
+
+ case "$ftype" in
+ "archive")
+ # Record the package and project names of this package archive.
+ #
+ local p
+ extract_pkg_info "$src_dir/$f" | readarray -t p # (name ver proj)
+ managed_pkgnames["${p[0]}"]=true
+ managed_projs["${p[2]}"]=true
+ ;;
+ "ownership")
+ # Skip this ownership manifest if its package or project name has
+ # not been seen (which means it's unmanaged).
+ #
+ local k="$(basename $(dirname "$f"))"
+ if [[ (("$f" == */package-owner.manifest) &&
+ ! "${managed_pkgnames[$k]}") ||
+ (("$f" == */project-owner.manifest) &&
+ ! "${managed_projs[$k]}") ]]; then
+ continue
+ fi
+ ;;
+ esac
+ fi
+
+ # Add the commit to the pending set unless the current file is filtered
+ # out.
+ #
+ # Note: $filter must be unquoted to match as a wildcard pattern.
+ #
+ if [[ ("$mode" == "source") ||
+ ("$(basename "$(dirname "$f")")" == $filter) || # Project name?
+ ("$(basename "$f")" == $filter.*) ]]; then # Filename (with ext)?
+ pending_set["$h"]=true
+ fi
+ done
+
+ # Arrange the pending commits in the chronological order.
+ #
+ # Go through the most recent commits in the git log which added or
+ # moved/renamed one or more files, skipping those not present in the pending
+ # set and keeping count to bail out as soon as we ordered all of them.
+ #
+ local i
+ for (( i=0; i != "${#pending_set[@]}"; )); do
+ read h # The abbreviated commit hash.
+
+ # If this is a pending commit, prepend its hash to the ordered array.
+ #
+ if [[ "${pending_set[$h]}" ]]; then
+ pending_seq=("$h" "${pending_seq[@]}")
+ ((++i))
+ fi
+
+ # --diff-filter=AR: only show commits that added or renamed files
+ #
+ done < <(git -C "$src_dir" log --diff-filter=AR --pretty=format:%h)
+}
+
+# Clean the source and destination repositories by discarding uncommitted
+# changes and removing unstaged files. (Note that the source repository cannot
+# have untracked files so we git-clean only the destination repository.)
+#
+function cleanup ()
+{
+ info "migration failed; resetting and cleaning repositories"
+
+ if ([[ "$mode" == "source" ]] && ! run git -C "$src_dir" reset --hard) ||
+ ! run git -C "$dst_dir" reset --hard ||
+ ! run git -C "$dst_dir" clean --force; then
+ info "failed to reset/clean repositories -- manual intervention required"
+ fi
+}
+
+# Return the subject of a git commit in the source repository.
+#
+function commit_subject () # <commit-hash>
+{
+ local h="$1"
+ git -C "$src_dir" log -n 1 --pretty=format:%s "$h"
+}
+
+# Return the list of files a commit added to or moved within the source
+# repository.
+#
+function commit_files () # <commit-hash>
+{
+ local h="$1"
+
+ # git-diff-tree arguments:
+ #
+ # --diff-filter=A: select only files that were added.
+ # -z: don't munge file paths and separate output fields with
+ # NULs.
+ # -r: recurse into subtrees (directories).
+ #
+ git -C "$src_dir" diff-tree \
+ --no-commit-id --name-only --diff-filter=A -z -r \
+ "$h"
}
# Exit with an error if a package which is a duplicate of or is in conflict
@@ -472,15 +639,15 @@ function check_pkg_duplicate () # <pkg-name> <pkg-version>
# Use <name>-<version>.* without .tar.gz in case we want to support more
# archive types later.
#
- IFS=$'\n' eval \
- 'p=($(bpkg_util_pkg_find_archive "$name-$version.*" "$dst_dir/$sd"))'
+ bpkg_util_pkg_find_archive "$name-$version.*" "$dst_dir/$sd" | \
+ readarray -t p
- if [ "${#p[@]}" -ne 0 ]; then
+ if [[ "${#p[@]}" -ne 0 ]]; then
local a="${p[0]}"
local n="${p[1]}"
local v="${p[2]}"
- if [ "$n" == "$name" ]; then
+ if [[ "$n" == "$name" ]]; then
error "duplicate of $name/$version at '$a'"
else
error "conflict of $name/$version with $n/$v at '$a'"
@@ -509,10 +676,10 @@ function check_pkg_duplicate () # <pkg-name> <pkg-version>
# and remove or skip each candidate at the user's direction. Stage (but don't
# commit) the removals in the destination repository.
#
-# The versions of the removed packages are written to stdout, separated by
-# spaces. For example:
+# The versions of the removed packages are written to stdout, each on a
+# separate line. For example:
#
-# 1.2.3 1.3.0+1 1.3.0+2
+# 1.2.3\n1.3.0+1\n1.3.0+2\n
#
# Note that, currently, versions/revisions which are both lower and higher
# than <pkg-version> will be considered for replacement.
@@ -526,16 +693,14 @@ function remove_pkg_archives ()
local sver="$4"
local sproj="$5"
- local rv=() # Removed version numbers.
-
# Search for replacement candidates.
#
- local pkgs=() # Packages to be considered for replacement.
+ local pkgs= # Packages to be considered for replacement.
- IFS=$'\n' eval \
- 'pkgs=($(bpkg_util_pkg_find_archives "$name" \
- "$vpat" \
- "$dst_dir/${dst_sections[$dsect]}"))'
+ bpkg_util_pkg_find_archives "$name" \
+ "$vpat" \
+ "$dst_dir/${dst_sections[$dsect]}" | \
+ readarray -t pkgs
# For each replacement candidate, ask for confirmation and, depending on the
# answer, either remove it from the destination repository or leave it in
@@ -546,7 +711,7 @@ function remove_pkg_archives ()
# Get the destination archive's info from its embedded manifest.
#
local p
- p=($(extract_pkg_info "$f"))
+ extract_pkg_info "$f" | readarray -t p
local dver="${p[1]}" # Destination package version.
local dproj="${p[2]}" # Destination package project.
@@ -558,7 +723,7 @@ function remove_pkg_archives ()
#
local src="$sver"
local dst="$name/$dver"
- if [ "$dproj" != "$sproj" ]; then
+ if [[ "$dproj" != "$sproj" ]]; then
src+=" ($sproj)"
dst+=" ($dproj)"
fi
@@ -570,7 +735,7 @@ function remove_pkg_archives ()
case "$opt" in
y)
run git -C "$dst_dir" rm --quiet "${f#$dst_dir/}"
- rv+=("$dver")
+ echo "$dver"
break
;;
n)
@@ -579,34 +744,114 @@ function remove_pkg_archives ()
esac
done
done
+}
+
+# The commit bundle array is the set of selected pending commits. Its elements
+# are the corresponding indexes of the `pending_seq` array (but offset by +1).
+# The indexes are always kept in ascending and, therefore, chronological order
+# (because the commits in `pending_seq` are in chronological order).
+#
+bundle=()
+
+# Contains the managed file paths belonging to commits in the commit bundle.
+# Used as the starting point for operations such as the migration and dropping
+# of commits or files.
+#
+# These files must be grouped by commit and the commit groups ordered
+# chronologically, otherwise files could be processed in a different order
+# than they occur in the pending commit list (the `pending_seq` array). For
+# example, failing that, the commit groups/files in the commit splitting
+# screen or the file mentions in the commit messages might be in a different
+# order than they were in the main screen.
+#
+bundle_files=()
+
+# Collect the commit bundle's files into the global `bundle_files` array,
+# grouping them by commit. The commit groups are ordered chronologically to
+# match the ordering of the pending commit array, `pending_seq`.
+#
+# Include only files belonging to a commit in the bundle.
+#
+# Fail -- by leaving `bundle_files` empty -- if the commit bundle is empty or
+# any of the files are unmanaged.
+#
+function collect_bundle_files ()
+{
+ bundle_files=()
- echo -n "${rv[@]}"
+ local i
+ for i in "${bundle[@]}"; do
+ local h="${pending_seq[$i-1]}" # The current commit's abbreviated hash.
+
+ local f
+ while read -d '' f; do
+ # Fail (by clearing the `bundle_files` array) if the current file is
+ # unmanaged.
+ #
+ local fi
+ src_path_info "$f" | readarray -t fi
+ if [[ "${fi[0]}" == "unmanaged" ]]; then
+ info "cannot include commit $i: '$f' is unmanaged"
+ bundle_files=()
+ return
+ fi
+
+ # Add this file only if it belongs to the current commit.
+ #
+ if [[ "${file_commits[$f]}" == "$h" ]]; then
+ bundle_files+=("$f")
+ fi
+ done < <(commit_files "$h")
+ done
}
-# The commit bundle associative array is the set of selected pending
-# commits. Its keys are the corresponding indexes of the 'pending_seq' array
-# (but offset by +1). Note: the reason the commit bundle is an associative
-# array is to prevent duplicates.
+# Print "true" to stdout if <target> equals any of the subsequent
+# arguments/words.
#
-declare -A bundle
+function contains () # <target> <word0> <word1> ...
+{
+ local k="$1"
+ shift
+
+ local w
+ for w in "$@"; do
+ if [[ "$w" == "$k" ]]; then
+ echo -n "true"
+ return
+ fi
+ done
+}
-# After a successful migration, reset some global state and pause to give the
-# operator a chance to look at the commits before the list of remaining
-# pending commits is displayed.
+# Return the section name corresponding to the section directory specified by
+# <section-dir>.
+#
+# If the section name exists, print it to stdout; otherwise exit the script
+# with a diagnostic message and an error status code.
#
-function migrate_epilogue ()
+# The section name is the key which indexes <section-dir> in `src_sections`,
+# so this is a value-to-key lookup.
+#
+# Fail by exiting the script because a failed search can only result from a
+# programming error or a bad repository submit configuration (for example, a
+# "*" key with a unique value (see below)).
+#
+function src_section_name () # <section-dir>
{
- # Remove the hashes in the commit bundle from the pending sequence and clear
- # the commit bundle.
+ local sd="$1" # Section directory.
+
+ # The "*" key is a catch-all for unknown submitted section names and, if
+ # present, will share a value (section directory) with one of the known
+ # section names and therefore must be skipped.
#
- local i
- for i in "${!bundle[@]}"; do
- unset pending_seq[i-1]
+ local k
+ for k in "${!src_sections[@]}"; do
+ if [[ ("${src_sections[$k]}" == "$sd") && ("$k" != "*") ]]; then
+ echo -n "$k"
+ return
+ fi
done
- pending_seq=("${pending_seq[@]}") # Remove the gaps created by unset.
- bundle=()
- read -p "press Enter to continue"
+ error "no source section name found for directory '$sd'"
}
# Migrate (all management modes):
@@ -618,9 +863,9 @@ function migrate_epilogue ()
#
# Note the following edge case which applies to all management modes:
#
-# We will need to confirm with git that each of a commit's added files were
-# actually most recently added by that commit. For example (oldest commits
-# first):
+# We will need to confirm with git that each of a commit's files were
+# actually most recently added or moved by that commit. For example (oldest
+# commits first):
#
# commit 1: add foo.tar.gz, bar.tar.gz
# commit 2: del foo.tar.gz
@@ -632,18 +877,20 @@ function migrate_epilogue ()
# Source-management mode migration: migrate the selected commit bundle from
# the source repository to the destination repository.
#
-# If the migration succeeds, set the global migrate_result variable to
+# Takes as input the bundle files in the global `bundle_files` array which is
+# assumed to be non-empty and to contain only managed files (package archives
+# or ownership manifests) belonging to commits in the commit bundle.
+#
+# If the migration succeeds, set the global `operation_result` variable to
# true. Otherwise, in case of failure, issue appropriate diagnostics and set
-# migrate_result to the empty string before returning. In order to ensure that
-# both repositories are left clean if any of the files fail to be migrated, an
-# EXIT signal handler that discards all uncommitted changes is installed right
-# before the migration proper begins, and uninstalled when all files have been
-# migrated.
+# `operation_result` to the empty string before returning. In order to ensure
+# that both repositories are left clean if any of the files fail to be
+# migrated, an EXIT signal handler that discards all uncommitted changes is
+# installed right before the migration proper begins, and uninstalled when all
+# files have been migrated.
#
# The migration will fail if any of the following is true:
#
-# - The commit bundle is empty.
-#
# - Files in the commit bundle are not all from the same project (the "bundle
# project") or, in the case of archives, the same repository section (the
# "bundle section").
@@ -654,7 +901,9 @@ function migrate_epilogue ()
# conflicting archive in any of the sections in the destination repository
# (see check_pkg_duplicate()).
#
-# - Any file in the commit bundle is unmanaged.
+# - The migration would overwrite ownership info in the destination
+# repository. (Currently ownership is updated by modifying the files
+# directly.)
#
# The migration process proceeds as follows:
#
@@ -669,8 +918,8 @@ function migrate_epilogue ()
# beta -> beta
# stable -> testing|stable
#
-# Package archives may be removed and ownership manifests overwritten at the
-# destination. Candidate files for replacement are selected as follows:
+# Package archives may be removed at the destination. Candidate files for
+# replacement are selected as follows:
#
# - In the alpha and beta sections, any package archive files in the
# destination section directory belonging to the same package are
@@ -680,11 +929,6 @@ function migrate_epilogue ()
# directory with the same name and version but a different revision
# (currently whether lower or higher) are replaced.
#
-# - Project and package ownership manifests will be replaced (that is,
-# simply overwritten) at the destination with any ownership manifests
-# added by the commit bundle because their presence implies that ownership
-# information has changed.
-#
# Stage (but don't commit) the removal of the files from the source
# repository and their addition to the destination repository.
#
@@ -710,115 +954,58 @@ function migrate_epilogue ()
#
function migrate_src ()
{
- migrate_result=
-
- if [ "${#bundle[@]}" -eq 0 ]; then
- info "no commits selected"
- return
- fi
+ operation_result=
- # Check that every commit's files are managed packages or ownership
- # manifests in the bundle section and/or bundle project before migrating any
- # of them. Build the bundle's list of files as we go along, classifying them
- # as package archives or ownership manifests based on their paths.
+ # Check that all files in the bundle are in the bundle section and/or bundle
+ # project before migrating any of them. Classify each file as a package
+ # archive or ownership manifest as we go along, based on its path.
#
# The bundle section is derived from the first package archive encountered
- # and the bundle project from the first package archive or owner manifest
- # encountered.
- #
- # Note that the bundle traversal is unordered.
+ # and the bundle project from the first package archive or ownership
+ # manifest encountered.
#
local src_sect= # Source section name.
- local src_sect_dir= # Source section directory.
local proj= # The bundle (source) project.
- local pkgs=() # The bundle's archive files.
+ local pkgs=() # The bundle's package archives.
local owns=() # The bundle's ownership manifests.
- local i
- for i in "${!bundle[@]}"; do
- local h="${pending_seq[i-1]}" # The current commit's abbreviated hash.
-
- # Check the current commit's files.
- #
- local f
- while read -d '' f; do
- # Derive the project and/or section names from the file path.
- #
- local fproj= # Current file's project.
- local fi=($(src_path_info "$f"))
-
- if [ "${fi[0]}" == "ownership" ]; then
- if [ "${file_commits[$f]}" != "$h" ]; then
- continue # Ownership manifest was removed by a subsequent commit.
- fi
-
- fproj="${fi[1]}"
- owns+=("$f")
-
- elif [ "${fi[0]}" == "archive" ]; then
- if [ "${file_commits[$f]}" != "$h" ]; then
- continue # Archive was removed by a subsequent commit.
- fi
-
- fproj="${fi[1]}"
- local fsect_dir="${fi[2]}"
- pkgs+=("$f")
+ local f
+ for f in "${bundle_files[@]}"; do
+ local fi
+ src_path_info "$f" | readarray -t fi
+ local ftype="${fi[0]}" # Current file's type.
+ local fproj="${fi[1]}" # Current file's project.
- # Find, in `src_sections`, the archive section name associated with
- # the section directory extracted from the path (a value-to-key
- # lookup).
- #
- local fsect=
+ if [[ "$ftype" == "ownership" ]]; then
+ owns+=("$f")
- # The "*" key is a catch-all for unknown submitted section names and,
- # if present, will share a value (section directory) with one of the
- # known section names and therefore must be skipped.
- #
- local k
- for k in "${!src_sections[@]}"; do
- if [[ ("${src_sections[$k]%/}" == "$fsect_dir") &&
- ("$k" != "*") ]]; then
- fsect="$k" # Current file's section name.
- break
- fi
- done
+ elif [[ "$ftype" == "archive" ]]; then
+ pkgs+=("$f")
- if [ -z "$fsect" ]; then
- # The only way that fsect can be empty is due to a programming error
- # or if the "*" key has a unique value -- which would be a submit
- # config error. So it would probably be better to terminate the
- # script.
- #
- error "unable to find section name for file '$f'"
- fi
+ local fsect_dir="${fi[2]}" # Section dir from file path.
+ local fsect # File's src section name.
+ fsect="$(src_section_name "$fsect_dir")" # Exits script on failure.
- # Set the source section name and directory if unset; otherwise fail
- # if the current file is not from the source section.
- #
- if [ -z "$src_sect" ]; then
- src_sect="$fsect"
- src_sect_dir="$fsect_dir"
- elif [ "$fsect" != "$src_sect" ]; then
- info "cannot include commit $i: '$f' is not in section $src_sect"
- return
- fi
- else
- info "cannot include commit $i: '$f' is unmanaged"
- return
- fi
-
- # Set the bundle project if unset; otherwise fail if the current file is
- # not from the bundle project.
- #
- # Note: $fproj cannot be empty here (see above).
+ # Set the source section name if unset; otherwise fail if the current
+ # file is not from the source section.
#
- if [ -z "$proj" ]; then
- proj="$fproj"
- elif [ "$fproj" != "$proj" ]; then
- info "cannot include commit $i: '$f' is not in project $proj"
+ if [[ -z "$src_sect" ]]; then
+ src_sect="$fsect"
+ elif [[ "$fsect" != "$src_sect" ]]; then
+ info "'$f' is not in section $src_sect"
return
fi
- done < <(commit_files "$h")
+ fi
+
+ # Set the bundle project if unset; otherwise fail if the current file is
+ # not from the bundle project.
+ #
+ if [[ -z "$proj" ]]; then
+ proj="$fproj"
+ elif [[ "$fproj" != "$proj" ]]; then
+ info "'$f' is not in project $proj"
+ return
+ fi
done
# Finalize migration variables the values of which depend on whether the
@@ -832,7 +1019,7 @@ function migrate_src ()
local src_cmsg # Source commit message.
local dst_cmsg # Destination commit message.
- if [ "${#pkgs[@]}" -ne 0 ]; then # Bundle contains package archive(s).
+ if [[ "${#pkgs[@]}" -ne 0 ]]; then # Bundle contains package archive(s).
dst_sect="$src_sect"
# If it exists, 'testing' overrides 'stable' at the destination.
@@ -843,7 +1030,7 @@ function migrate_src ()
# Fail if the target section does not exist in the destination repository.
#
- if [ ! -v dst_sections["$dst_sect"] ]; then
+ if [[ ! -v "dst_sections[$dst_sect]" ]]; then
info "section '$dst_sect' does not exist in the destination repository"
return
fi
@@ -858,7 +1045,7 @@ function migrate_src ()
# destination but enabled on source is probably obscure, but let's
# consider it possible since the submit-git handler allows such a setup.
#
- if [ -n "$dst_owners" ]; then
+ if [[ -n "$dst_owners" ]]; then
src_cmsg="Migrate $proj ownership info to $dst_repo_name"$'\n\n'
dst_cmsg="Migrate $proj ownership info from $src_repo_name"$'\n\n'
else
@@ -882,8 +1069,8 @@ function migrate_src ()
# the destination repository directory.
#
# Note that the source and destination sections and owners directories may
- # differ (as they do in these examples) which is why those components must
- # be specified in both the source and destination paths.
+ # differ which is why those components must be specified in both the source
+ # and destination paths.
#
# Move the file from the source repository directory to the destination
# repository directory, creating directories if required; stage the addition
@@ -906,7 +1093,7 @@ function migrate_src ()
# (we already have the source project in $proj).
#
local p
- p=($(extract_pkg_info "$src_dir/$f"))
+ extract_pkg_info "$src_dir/$f" | readarray -t p
local name="${p[0]}"
local version="${p[1]}"
@@ -924,18 +1111,19 @@ function migrate_src ()
esac
local rv # Removed version numbers.
- rv=($(remove_pkg_archives "$name" "$vpat" \
- "$dst_sect" \
- "$version" "$proj"))
+ remove_pkg_archives "$name" "$vpat" \
+ "$dst_sect" \
+ "$version" "$proj" | readarray -t rv
# Update the commit messages and migrate the current package.
#
src_cmsg+=" remove $name/$version"$'\n'
- if [ "${#rv[@]}" -eq 0 ]; then
+ if [[ "${#rv[@]}" -eq 0 ]]; then
dst_cmsg+=" add $name/$version"$'\n'
else
- for ((i=0; i != "${#rv[@]}"; ++i)); do
- dst_cmsg+=" replace $name/${rv[i]} with $version"$'\n'
+ local v
+ for v in "${rv[@]}"; do
+ dst_cmsg+=" replace $name/$v with $version"$'\n'
done
fi
@@ -948,24 +1136,23 @@ function migrate_src ()
# only remove ownership manifests from the source repository (that is, don't
# migrate).
#
+ # Fail if an ownership manifest already exists in the destination
+ # repository.
+ #
for f in "${owns[@]}"; do
src_cmsg+=" remove $f"$'\n'
- if [ -n "$dst_owners" ]; then
+ if [[ -n "$dst_owners" ]]; then
local dp=$(dirname "${f/$src_owners/$dst_owners}") # Destination path.
+ local fn=$(basename "$f") # File name.
- # Let the commit message reflect whether this is a new ownership
- # manifest or is replacing an existent one.
- #
- local fn=$(basename "$f")
- if [ ! -e "$dst_dir/$dp/$fn" ]; then
- dst_cmsg+=" add $dp/$fn"$'\n'
- else
- dst_cmsg+=" update $dp/$fn"$'\n'
+ if [[ -f "$dst_dir/$dp/$fn" ]]; then
+ error "$f already exists at $dst_dir/$dp/$fn"
fi
+ dst_cmsg+=" add $dp/$fn"$'\n'
migrate_file "$f" "$dp"
- else
+ else # Ownership authentication disabled in the destination repository.
run git -C "$src_dir" rm --quiet "$f"
fi
done
@@ -975,7 +1162,7 @@ function migrate_src ()
info
run git -C "$src_dir" commit -m "$src_cmsg"
- if [ -n "$dst_cmsg" ]; then
+ if [[ -n "$dst_cmsg" ]]; then
info
run git -C "$dst_dir" commit -m "$dst_cmsg"
fi
@@ -985,15 +1172,21 @@ function migrate_src ()
# All files have been migrated successfully so set the result and clear the
# EXIT trap.
#
- migrate_result=true
+ operation_result=true
trap EXIT
- migrate_epilogue
+ read -p "press Enter to continue"
}
# Destination-management mode migration: migrate the package archives in the
# selected commit bundle from the source (managed) section to its counterpart
-# section.
+# section and skip ownership manifests (with a notification message).
+#
+# Takes as input the bundle files in the global `bundle_files` array which is
+# assumed to be non-empty and to contain only managed files belonging to
+# commits in the commit bundle.
+#
+# Assumes that the source and destination sections are valid.
#
# The general structure of this function is very similar to that of
# migrate_src() but most of the logic is simpler. Some noteworthy differences
@@ -1001,8 +1194,8 @@ function migrate_src ()
#
# - Only the destination repository is involved.
#
-# - All managed files in the commit bundle are package archives (ownership
-# manifests stay where they are and are therefore skipped/ignored).
+# - Ownership manifests cannot be migrated in destination-management mode and
+# are skipped.
#
# - All managed package archives in the commit bundle are known to be in the
# same section (because all files outside of the managed section are
@@ -1013,21 +1206,22 @@ function migrate_src ()
#
# The migration will fail if any of the following is true:
#
-# - The commit bundle is empty.
-#
-# - Package archives in the commit bundle are not all from the same project
-# (the "bundle project").
+# - Files in the commit bundle are not all from the same project (the "bundle
+# project").
#
# - There exists, for one or more package archives in the commit bundle, a
# duplicate or conflicting archive in the destination section (see
# check_pkg_duplicate()).
#
-# Note that the source and destination sections are assumed to be valid.
+# - Migration from the specified source section is not supported.
+#
+# - The file selection contained no package archives.
#
# The migration process proceeds as follows:
#
# - Move files: all of the package archives in the selected commit bundle are
-# moved from the source section to the destination section.
+# moved from the source section to the destination section. Ownership
+# manifests are skipped with a notification message.
#
# Any package archives in the destination section directory with the same
# name and version but a different revision (currently whether lower or
@@ -1045,65 +1239,65 @@ function migrate_src ()
#
function migrate_dst ()
{
- migrate_result=
+ operation_result=
+
+ local src_sect="$mode" # Source section.
+ local src_sect_dir="${src_sections[$src_sect]}" # Source section directory.
- if [ "${#bundle[@]}" -eq 0 ]; then
- info "no commits selected"
+ # Fail if the source section has no counterpart, in which case migration
+ # from it is not supported.
+ #
+ if [[ ! -v "sect_cparts[$src_sect]" ]]; then
+ info "migration from $src_sect not supported"
return
fi
- local src_sect="$mode" # Source section.
- local src_sect_dir="${src_sections[$src_sect]}" # Source section directory.
local dst_sect="${sect_cparts[$src_sect]}" # Destination section.
local dst_sect_dir="${dst_sections[$dst_sect]}" # Dst. section directory.
- # Check that every file in the commit bundle is a managed package archive
- # (ownership manifests are skipped) from the bundle project (taken from the
- # path of the first file encountered). Also build the bundle's list of files
- # as we go along.
+ # Check that every file is from the bundle project (taken from the path of
+ # the first file encountered) and skip ownership manifests.
#
- # Note that the bundle traversal is unordered.
- #
- local proj= # The bundle project.
- local pkgs=() # The bundle's files (package archives, all).
+ local proj= # The bundle project.
local i
- for i in "${!bundle[@]}"; do
- local h="${pending_seq[i-1]}" # The current commit's abbreviated hash.
-
- # Check the current commit's files.
+ for i in "${!bundle_files[@]}"; do
+ local f="${bundle_files[$i]}"
+ local fi
+ src_path_info "$f" | readarray -t fi
+ local ftype="${fi[0]}" # Current file's type.
+ local fproj="${fi[1]}" # Current file's project.
+
+ # Set the bundle project if unset; otherwise fail if the current file is
+ # not from the bundle project.
#
- local f
- while read -d '' f; do
- local fi=($(src_path_info "$f"))
-
- # Fail if this is an unmanaged file. Skip ownership manifests and
- # archives deleted by subsequent commits. (Note that ownership manifests
- # are not stored in `file_commits` in destination-management mode.)
- #
- if [ "${fi[0]}" == "unmanaged" ]; then
- info "cannot include commit $i: '$f' is unmanaged"
- return
- elif [ "${file_commits[$f]}" != "$h" ]; then
- continue # Ownership manifest or deleted package archive.
- fi
-
- local fproj="${fi[1]}" # Current file's project.
-
- # Set the bundle project if unset; otherwise fail if the current file is
- # not from the bundle project.
- #
- if [ -z "$proj" ]; then
- proj="$fproj"
- elif [ "$fproj" != "$proj" ]; then
- info "cannot include commit $i: '$f' is not in project $proj"
- return
- fi
+ if [[ -z "$proj" ]]; then
+ proj="$fproj"
+ elif [[ "$fproj" != "$proj" ]]; then
+ info "'$f' is not in project $proj"
+ return
+ fi
- pkgs+=("$f")
- done < <(commit_files "$h")
+ # Skip ownership manifests and remove them from `bundle_files` so that we
+ # need not worry about file type for the rest of this function.
+ #
+ # Note that, in commit-splitting mode, skipped ownership manifests will
+ # not be printed in the file list after the migration despite not having
+ # been actioned.
+ #
+ if [[ "$ftype" == "ownership" ]]; then
+ info "skipping '$f'"
+ unset "bundle_files[$i]"
+ fi
done
+ # Fail if there were no package archives in `bundle_files`.
+ #
+ if [[ "${#bundle_files[@]}" -eq 0 ]]; then
+ info "no package archives selected"
+ return
+ fi
+
# Migrate the bundle's files.
#
# Ensure that the source and destination repositories are clean if the
@@ -1111,14 +1305,16 @@ function migrate_dst ()
#
trap cleanup EXIT
- local cmsg= # The detailed part of the commit message.
+ # The commit message.
+ #
+ local cmsg="Migrate $proj from $src_sect to $dst_sect"$'\n\n'
- for f in "${pkgs[@]}"; do
+ for f in "${bundle_files[@]}"; do # Note: contains no ownership manifests.
# Get the current package's name and version from its embedded manifest
# (we already have the source project in $proj).
#
local p
- p=($(extract_pkg_info "$src_dir/$f"))
+ extract_pkg_info "$src_dir/$f" | readarray -t p
local name="${p[0]}"
local version="${p[1]}"
@@ -1130,18 +1326,19 @@ function migrate_dst ()
# Find and remove other revisions of the current package.
#
- local rv # Removed version numbers.
- rv=($(remove_pkg_archives "$name" "$version*" \
- "$dst_sect" \
- "$version" "$proj"))
+ local rv # Removed version numbers.
+ remove_pkg_archives "$name" "$version*" \
+ "$dst_sect" \
+ "$version" "$proj" | readarray -t rv
# Update the commit message.
#
- if [ "${#rv[@]}" -eq 0 ]; then
+ if [[ "${#rv[@]}" -eq 0 ]]; then
cmsg+=" move $name/$version"$'\n'
else
- for ((i=0; i != "${#rv[@]}"; ++i)); do
- cmsg+=" replace $name/${rv[i]} with $version"$'\n'
+ local v
+ for v in "${rv[@]}"; do
+ cmsg+=" replace $name/$v with $version"$'\n'
done
fi
@@ -1157,24 +1354,366 @@ function migrate_dst ()
# file is removed, so this does not need to be done in migrate_src().)
#
local d="$dst_dir/$src_sect_dir/$proj/"
- if [ -z "$(ls -A "$d")" ]; then
+ if [[ -z "$(ls -A "$d")" ]]; then
rmdir "$d"
fi
# Commit the staged changes.
#
info
- run git -C "$dst_dir" commit \
- -m "Migrate $proj from $src_sect to $dst_sect"$'\n\n'"$cmsg"
+ run git -C "$dst_dir" commit -m "$cmsg"
info
# All files have been migrated successfully so set the result and clear the
# EXIT trap.
#
- migrate_result=true
+ operation_result=true
+ trap EXIT
+
+ read -p "press Enter to continue"
+}
+
+# Check that ownership consistency would be maintained if the bundle's files
+# were dropped.
+#
+# Print "true" to stdout if all checks passed, and nothing in case of failure.
+#
+# Fail if a drop would leave, in the real source or destination repositories,
+# package archives without package or project ownership, or package ownership
+# manifests without project ownership.
+#
+# Note that in order to search for unselected packages and ownership
+# manifests, each repository's submit config is loaded again because the data
+# loaded during global init does not reflect the true state of the
+# repositories on disk if in destination-management mode.
+#
+function check_drop_ownership_consistency () # <proj>
+{
+ local proj="$1" # Project name.
+
+ # Print "true" to stdout if <repo-dir> is the managed repository's
+ # directory, and nothing otherwise.
+ #
+ function managed_repo () # <repo-dir>
+ {
+ local d="$1" # Repository directory.
+
+ if [[ (("$mode" == "source") && ("$d" == "$src_dir")) ||
+ (("$mode" != "source") && ("$d" == "$dst_dir")) ]]; then
+ echo -n "true"
+ fi
+ }
+
+ # Find the package names of all unselected package archives from <proj> in
+ # the real source and destination repositories.
+ #
+ local -A unsel_pkg_names=() # The names of unselected packages.
+
+ # Find unselected packages from <proj> in the repository directory specified
+ # by <repo-dir> and insert them into `unsel_pkg_names`.
+ #
+ # In deciding whether a file is unselected, only check `bundle_files` if
+ # <repo-dir> is the managed repository because if a file (erroneously)
+ # exists in both repositories and is selected in the managed repository it
+ # would be in `bundle_files` despite being unselected in the unmanaged one.
+ #
+ function find_unsel_pkg () # <repo-dir>
+ {
+ local rd="$1" # Repository directory.
+
+ local owners
+ local -A sections
+ source "$rd/submit.config.bash"
+
+ local s
+ for s in "${sections[@]}"; do
+ local pd="$rd/$s/$proj" # Project directory.
+ if [[ -d "$pd" ]]; then
+ local f
+ while read f; do
+ local frel="${f#$rd/}" # Path made relative to repo dir.
+ if [[ ! "$(managed_repo "$rd")" ||
+ ! "$(contains "$frel" "${bundle_files[@]}")" ]]; then
+ local p # (name ver proj)
+ bpkg_util_pkg_verify_archive "$f" | readarray -t p
+
+ unsel_pkg_names["${p[0]}"]=
+ fi
+ done < <(find "$pd" -type f -not -name "*.manifest")
+ fi
+ done
+ }
+
+ find_unsel_pkg "$real_src_dir"
+ find_unsel_pkg "$dst_dir"
+
+ # Find unselected package ownership from <proj> in the real source and
+ # destination repositories.
+ #
+ local unsel_pkg_owns= # True if there are unselected pkg ownership manifests.
+
+ # Find unselected package ownership manifest from <proj> in the repository
+ # directory specified by <repo-dir> and, if found, set `unsel_pkg_owns` to
+ # true. See find_unsel_pkg() above for a note about the role `bundle_files`
+ # plays.
+ #
+ function find_unsel_pkg_owns () # <repo-dir>
+ {
+ local rd="$1" # Repository directory.
+
+ local owners
+ local -A sections
+ source "$rd/submit.config.bash"
+
+ local pd="$rd/$owners/$proj" # Project directory.
+ if [[ -d "$pd" ]]; then
+ local f
+ while read f; do
+ f="${f#$rd/}" # Make path relative to repo dir.
+
+ if [[ ("$f" == */package-owner.manifest) &&
+ (! "$(managed_repo "$rd")" ||
+ ! "$(contains "$f" "${bundle_files[@]}")") ]]; then
+ unsel_pkg_owns=true
+ break
+ fi
+ done < <(find "$pd" -type f)
+ fi
+ }
+
+ find_unsel_pkg_owns "$real_src_dir"
+ find_unsel_pkg_owns "$dst_dir"
+
+ # Fail if any ownership info about to be dropped would leave any ownerless
+ # package archives or package ownership manifests.
+ #
+ local f
+ for f in "${bundle_files[@]}"; do
+ case "$f" in
+ */project-owner.manifest)
+ if [[ ("${#unsel_pkg_names[@]}" -ne 0) || "$unsel_pkg_owns" ]]; then
+ info "cannot drop project ownership info without \
+associated packages and/or package ownership"
+ return
+ fi
+ ;;
+ */package-owner.manifest)
+ local pname="$(basename $(dirname "$f"))"
+ if [[ -v "unsel_pkg_names[$pname]" ]]; then
+ info "cannot drop package ownership without associated packages"
+ return
+ fi
+ ;;
+ esac
+ done
+
+ echo -n "true"
+}
+
+# Drop the files in the selected commit bundle from the source repository.
+#
+# Takes as input the bundle files in the global `bundle_files` array which is
+# assumed to be non-empty and to contain only managed files (package archives
+# or ownership manifests) belonging to commits in the commit bundle.
+#
+# The mechanism by which success or failure is indicated is the same as for
+# migrate_src(). (In particular, the source repository will be left clean if
+# the removal of any of the files fails.)
+#
+# The operation will fail if any of the following is true:
+#
+# - Files in the commit bundle are not all from the same project (the "bundle
+# project") and section (the "bundle section").
+#
+# - Ownership info would be removed for any files remaining after the drop
+# (see check_drop_ownership_consistency()).
+#
+# - The mode is the stable-management, because stable, published packages
+# cannot be unpublished.
+#
+# The operation proceeds as follows:
+#
+# - Check that all files are from the same project and check the resulting
+# ownership consistency.
+#
+# - Get from the user the reason for dropping the files.
+#
+# - Remove each file from the source repository. Get user confirmation before
+# dropping ownership manifests, issuing a notification message for each one
+# skipped. Stage (but don't commit) each removal as the operation proceeds.
+#
+# - Make a commit to the source repository with an appropriate commit message.
+#
+# If dropping at least one package archive:
+#
+# "Drop <project> from <section> (<reason>)"
+#
+# " remove <package>/<version>"
+# " remove owners/<project>/project-owner.manifest"
+# " remove owners/<project>/<package>/package-owner.manifest"
+#
+# If dropping only ownership information:
+#
+# "Drop <project> ownership (<reason>)"
+#
+# " remove owners/<project>/project-owner.manifest"
+# " remove owners/<project>/<package>/package-owner.manifest"
+#
+function drop ()
+{
+ operation_result=
+
+ if [[ "$mode" == "stable" ]]; then
+ info "dropping files from $mode not supported"
+ return
+ fi
+
+ # Check that all files in the bundle are in the bundle project and that all
+ # package archives are in the bundle section before dropping any of
+ # them. Classify each file as a package archive or ownership manifest as we
+ # go along, based on its path.
+ #
+ # The bundle section is derived from the first package archive encountered
+ # and the bundle project from the first package archive or ownership
+ # manifest encountered.
+ #
+ # Get user confirmation to drop each ownership manifest, skipping those
+ # declined.
+ #
+ # Note that, in commit-splitting mode, skipped ownership manifests will not
+ # be printed in the file list after the migration despite not having been
+ # actioned.
+ #
+ local proj= # The bundle project.
+ local sect= # The bundle section name.
+ local pkgs=() # Package archives to be dropped.
+ local owns=() # Ownership manifests to be dropped.
+
+ local i
+ for i in "${!bundle_files[@]}"; do
+ local f="${bundle_files[$i]}"
+ local fi
+ src_path_info "$f" | readarray -t fi
+ local ftype="${fi[0]}" # Current file's type.
+ local fproj="${fi[1]}" # Current file's project.
+
+ if [[ "$ftype" == "ownership" ]]; then
+ # Ask whether or not this ownership manifest should be dropped. Add it
+ # to `owns` if the user confirmed or, if the user declined, skip it by
+ # not adding it to `owns` and by removing it from `bundle_files` (to
+ # prevent check_drop_ownership_consistency() from thinking it's still
+ # selected).
+ #
+ local opt=
+ while [[ ("$opt" != y) && ("$opt" != n) ]]; do
+ read -p "drop '$f'? [y/n]: " opt
+ done
+
+ if [[ "$opt" == y ]]; then
+ owns+=("$f")
+ else
+ info "skipping '$f'"
+ unset "bundle_files[$i]"
+ fi
+ elif [[ "$ftype" == "archive" ]]; then
+ pkgs+=("$f")
+
+ local fsect_dir="${fi[2]}" # Section dir from file path.
+ local fsect # File's section name.
+ fsect="$(src_section_name "$fsect_dir")" # Exits script on failure.
+
+ # Set the bundle section name if unset; otherwise fail if the current
+ # package archive is not from the bundle section.
+ #
+ if [[ -z "$sect" ]]; then
+ sect="$fsect"
+ elif [[ "$fsect" != "$sect" ]]; then
+ info "'$f' is not in section $sect"
+ return
+ fi
+ fi
+
+ # Set the bundle project if unset; otherwise fail if the current file is
+ # not from the bundle project.
+ #
+ if [[ -z "$proj" ]]; then
+ proj="$fproj"
+ elif [[ "$fproj" != "$proj" ]]; then
+ info "'$f' is not in project $proj"
+ return
+ fi
+ done
+
+ # Fail if there is nothing to drop or if the drop would remove ownership
+ # info from any remaining files.
+ #
+ # There would be nothing to drop if the bundle consisted only of ownership
+ # manifests and the user declined to drop all of them.
+ #
+ # Note that check_drop_ownership_consistency() prints diagnostics in case of
+ # failure.
+ #
+ if [[ (("${#pkgs[@]}" -eq 0) && ("${#owns[@]}" -eq 0)) ||
+ ! "$(check_drop_ownership_consistency "$proj")" ]]; then
+ return
+ fi
+
+ # Get the reason for dropping the files from the user.
+ #
+ local reason=
+ info
+ while [[ -z "$reason" ]]; do
+ read -p "reason for dropping: " reason
+ done
+
+ # Set the commit message subject depending on whether there is at least one
+ # package archive in the bundle.
+ #
+ local cmsg= # Commit message.
+ if [[ "${#pkgs[@]}" -ne 0 ]]; then # Bundle contains package archive(s).
+ cmsg="Drop $proj from $sect ($reason)"$'\n\n'
+ else # Bundle contains only ownership manifests.
+ cmsg="Drop $proj ownership ($reason)"$'\n\n'
+ fi
+
+ # Remove the bundle's files from the source repository and compose the
+ # commit message details in the process.
+ #
+ # Ensure that the source repository is clean if any of the removals fail.
+ #
+ trap cleanup EXIT
+
+ local f
+ for f in "${pkgs[@]}"; do
+ # Get the current package's name and version from its embedded manifest.
+ #
+ local p
+ extract_pkg_info "$src_dir/$f" | readarray -t p
+ local name="${p[0]}"
+ local version="${p[1]}"
+
+ cmsg+=" remove $name/$version"$'\n'
+ run git -C "$src_dir" rm "$f" # Automatically removes empty directories.
+ done
+
+ for f in "${owns[@]}"; do
+ cmsg+=" remove $f"$'\n'
+ run git -C "$src_dir" rm "$f" # Automatically removes empty directories.
+ done
+
+ # Commit the changes made to the source repository.
+ #
+ info
+ run git -C "$src_dir" commit -m "$cmsg"
+ info
+
+ # All files have been dropped successfully so set the result and clear the
+ # EXIT trap.
+ #
+ operation_result=true
trap EXIT
- migrate_epilogue
+ read -p "press Enter to continue"
}
# Push local changes to the remote source and/or destination git repositories.
@@ -1199,15 +1738,17 @@ function push ()
error "push to $dst_repo_name failed"
fi
- if [ "$mode" == "source" ] && ! run git -C "$src_dir" push; then
+ if [[ "$mode" == "source" ]] && ! run git -C "$src_dir" push; then
error "push to $src_repo_name failed"
fi
}
-# Present the list of pending commits to the user, oldest first, marking files
-# that were deleted by subsequent commits with `*` and, of those, the ones
-# that were then added back again with `!`. Files from unmanaged directories
-# are marked with `?`.
+# User interface:
+#
+# The main screen presents the list of pending commits to the user, oldest
+# first, with files that were deleted by subsequent commits marked with `*`
+# and, of those, the ones that were then added back again with `!`. Files from
+# unmanaged directories are marked with `?`.
#
# 1 (deadbeef) Add libfoo/1.2.3
#
@@ -1229,50 +1770,299 @@ function push ()
#
# Then prompt the user for the action (showing the current bundle):
#
-# [1 2][<N>,m,c,p,q,l,?]:
+# [1 2][<N>,m,d,s,c,p,q,l,?]:
#
# <N> - add commit to the commit bundle
# m - migrate the selected commit bundle
+# d - drop the selected commit bundle
+# s - split commit(s) (operate on selected files)
# c - clear the selected commit bundle
# p - push source and destination repositories
# l - list pending commits
-# q - quit (prompting to push if any actions have been taken)
+# q - quit (prompting to push if commits were made)
# ? - print this help
#
-# The user interaction loop.
+# The main screen operates at "commit granularity", in which each operation is
+# performed on entire commits (that is, on all of the files they own). At
+# "file granularity", on the other hand, operations are performed only on a
+# user-selected subset of the selected commits' files. To operate at file
+# granularity the user chooses, in the main menu, to split the selected
+# commits, thus switching to the "commit splitting screen". See
+# `split_commits()`, below.
#
-# In each iteration, present the list of pending commits, display the menu of
-# actions, read the user's input, and perform the chosen action.
+
+# Print the size of a file to stdout in the human-readable form (K, M, G).
#
+function file_size ()
+{
+ local f="$1"
+
+ # POSIX specifies the output of `ls -l` so this should be portable. -h turns
+ # on human-readable output and is present on Linux, FreeBSD, and Mac OS.
+ #
+ echo -n "$(ls -lh "$src_dir/$f" | cut -d ' ' -f 5)"
+}
+
+# Show the commit splitting screen: let the user select and operate on
+# specific files in the commit bundle (file granularity).
+#
+# Remove all but the selected files from `bundle_files` -- the global array
+# used as input to the commit and file operations -- before performing an
+# operation, and populate it with the unselected files afterwards.
+#
+# Assumes that the files in `bundle_files` are grouped by commit and that they
+# are all existent (that is, present in `file_commits`) and managed.
+#
+# This screen's layout is near identical to that of the main screen -- each
+# commit's files, grouped under the commit heading, with commit groups in
+# chronological order -- except that here the files are numbered instead of
+# the commits. The user is presented with a menu similar to the one in the
+# main screen including the selection of files by number and operations such
+# as `migrate` and `drop`.
+#
+# @@ Feature idea: invert selection. That is, if user wants to exclude rather
+# than include a file, can add it to the selection and then invert the
+# selection.
+#
+# Also: add 'push' and 'quit script' ('Q') options? Returning to the main
+# menu is a little annoying, but perhaps that's just because I've been
+# running this script so frequently.
+#
+function split_commits ()
+{
+ # The current file selection. Stores indexes into `bundle_files`.
+ #
+ local fsel=()
+
+ while true; do
+ if [[ "${#bundle_files[@]}" -ne 0 ]]; then
+ # Print the commits followed by their files. Because `bundle_files` is
+ # grouped by commit we know we are on a new commit when the current
+ # file's commit hash differs from that of its predecessor.
+ #
+ local h= # Hash of commit to which the current subset of files belong.
+ local i
+ for i in "${!bundle_files[@]}"; do
+ local f="${bundle_files[$i]}" # Current file.
+ local fh="${file_commits[$f]}" # Current file's commit hash.
+
+ if [[ "$fh" != "$h" ]]; then # New commit so first print its heading.
+ h="$fh"
+ printf "\n%s %s\n\n" "$fh" "$(commit_subject "$fh")" >&2
+ fi
+
+ # Get this file's size if it's a package archive.
+ #
+ local sz= # File size.
+ local fi
+ src_path_info "$f" | readarray -t fi
+ if [[ "${fi[0]}" == "archive" ]]; then
+ sz=" $(file_size "$f")"
+ fi
+
+ # Print the current file's number (left-padded to 3 columns), path,
+ # and size (if set).
+ #
+ printf "%3d %s%s\n" "$((i+1))" "$f" "$sz" >&2
+ done
+ else
+ info "no files in commit bundle"
+ fi
+
+ # Used to restore the unactioned files into `bundle_files` after an
+ # operation (see update_bundle_files(), below). We need to do it because
+ # our migrate functions operate on (global) bundle_files.
+ #
+ local unactioned_files=()
+
+ # Collect the selected files into `bundle_files` just before an
+ # operation. But first save its contents in `unactioned_files`.
+ #
+ # Note that after this function returns the indexes in `fsel` are valid
+ # for `unactioned_files` but not `bundle_files`.
+ #
+ function collect_selected_files ()
+ {
+ unactioned_files=("${bundle_files[@]}")
+
+ local r=()
+ local i
+ for i in "${fsel[@]}"; do
+ r+=("${bundle_files[$i-1]}")
+ done
+ bundle_files=("${r[@]}")
+ }
+
+ # Put the unactioned bundle files into `bundle_files` after an operation.
+ # If the operation succeeded the unactioned files are the unselected ones
+ # and if it failed it's the union of the selected and unselected files.
+ #
+ function update_bundle_files () # <operation-result>
+ {
+ local opres="$1"
+
+ if [[ "$opres" ]]; then
+ # Remove the selected/actioned files from `unactioned_files`.
+ #
+ local i
+ for i in "${fsel[@]}"; do
+ unset "unactioned_files[$i-1]"
+ done
+ fi
+
+ bundle_files=("${unactioned_files[@]}")
+ }
+
+ # Display the menu/prompt, build the file selection, and perform the
+ # requested actions. Breaking out of this loop prints the bundle's files
+ # again.
+ #
+ while true; do
+ # Print the file selection and menu/prompt. Sort the selected files into
+ # ascending order (for display and to maintain the ordering of
+ # `bundle_files`, which `fsel` is the source for).
+ #
+ fsel=($(sed 's/ /\n/g' <<<"${fsel[*]}" | sort -))
+ local opt
+ read -p $'\n'"[${fsel[*]}][<N>,m,d,c,l,q,?]: " opt
+
+ # Perform the selected action.
+ #
+ case "$opt" in
+ # Add file <N> to the selection.
+ #
+ [0-9]*)
+ if [[ ("$opt" =~ ^[1-9][0-9]*$) && -v "bundle_files[$opt-1]" ]]; then
+ if [[ ! "$(contains "$opt" "${fsel[@]}")" ]]; then
+ fsel+=("$opt")
+ info "file $opt added to selection"
+ else
+ info "file $opt already selected"
+ fi
+ else
+ info "non-existent file number $opt"
+ fi
+ ;;
+ # Migrate selected files.
+ #
+ m)
+ if [[ "${#fsel[@]}" -ne 0 ]]; then
+ collect_selected_files
+ if [[ "$mode" == "source" ]]; then
+ migrate_src
+ else
+ migrate_dst
+ fi
+
+ update_bundle_files "$operation_result"
+
+ if [[ "$operation_result" ]]; then
+ fsel=()
+ need_push=true
+ break
+ fi
+ else
+ info "no files selected"
+ fi
+ ;;
+ # Drop selected files.
+ #
+ d)
+ if [[ "${#fsel[@]}" -ne 0 ]]; then
+ collect_selected_files
+ drop
+
+ update_bundle_files "$operation_result"
+
+ if [[ "$operation_result" ]]; then
+ fsel=()
+ need_push=true
+ break
+ fi
+ else
+ info "no files selected"
+ fi
+ ;;
+ # Clear the file selection (and print the file list again).
+ #
+ c)
+ fsel=()
+ break
+ ;;
+ # Print the file list.
+ #
+ l) break ;;
+ # Quit (returning to the main screen).
+ #
+ q)
+ return
+ ;;
+ # ? or invalid option: print menu.
+ #
+ *)
+ cat <<EOF >&2
+
+ <N> - add file to selection
+ m - migrate the selected files
+ d - drop the selected files
+ c - clear the file selection
+ l - list files
+ q - quit (back to main menu)
+ ? - print this help
+EOF
+ ;;
+ esac
+ done
+ done
+}
+
# True if any changes have been made to the source and/or destination git
# repositories (in which case the user will be asked whether or not to push
# before quitting).
#
need_push=
+init=true # True if the global state needs to be re-initialized.
+
+# The main screen's user interaction loop.
+#
+# In each iteration, present the list of pending commits, display the menu of
+# actions, read the user's input, and perform the chosen action.
+#
while true; do
- # Show the pending commits.
+
+ # (Re)initialize the global state if necessary. The idea is that instead of
+ # performing complex house-keeping, after certain operations (e.g.,
+ # migration) we will just re-initialize the state from scratch.
#
- if [ "${#pending_seq[@]}" -eq 0 ]; then
- info "no more pending commits"
+ if [[ "$init" ]]; then
+ init_globals
+
+ if [[ "${#pending_seq[@]}" -eq 0 ]]; then
+ info "good news, nothing to manage"
+ if [[ ! "$need_push" ]]; then
+ exit 0
+ fi
+ fi
+
+ init=
+ bundle=()
fi
- for ((i=0; i != "${#pending_seq[@]}"; i++)); do
+ for i in "${!pending_seq[@]}"; do
h="${pending_seq[$i]}"
# Print commit number, hash, and subject.
#
- # The commit number is left-padded with 0s to 3 digits. Prefix with a
- # newline to separate the first commit from the git-pull output and the
- # rest from the previous commit info block.
+ # Prefix with a newline to separate the first commit from the git-pull
+ # output and the rest from the previous commit info block.
#
- subj="$(git -C "$src_dir" log -n 1 --pretty=format:%s "$h")"
- printf "\n%d (%s) %s\n\n" "$((i+1))" "$h" "$subj" >&2
+ printf "\n%d (%s) %s\n\n" "$((i+1))" "$h" "$(commit_subject "$h")" >&2
# Print this commit's files.
#
- # Fetch from the git repository the list of files added or moved by the
- # current commit and print each one's path.
+ # Fetch from the git repository the current commit's files and print each
+ # one's path.
#
# Mark files that cannot be migrated:
#
@@ -1280,7 +2070,7 @@ while true; do
# the commit.
#
# - Files associated with a different commit hash in 'file_commits' were
- # deleted and added back by subsequent commits. These files are marked
+ # deleted and added back by subsequent commits. These files are marked
# with '!' and will not prevent the migration of the commit.
#
# - Files with no association in 'file_commits' were deleted by a
@@ -1288,10 +2078,10 @@ while true; do
# '*' and will not prevent the migration of the commit.
#
while read -d '' f; do
- fi=($(src_path_info "$f"))
+ src_path_info "$f" | readarray -t fi
ftype="${fi[0]}"
- if [ "$ftype" == "unmanaged" ]; then
+ if [[ "$ftype" == "unmanaged" ]]; then
# File is unmanaged (and may or may not exist).
#
info "? $f"
@@ -1299,27 +2089,31 @@ while true; do
# File is managed.
# If this file is a package archive which exists (that is, it's in
- # `file_commits`), get its size in the human-readable form.
+ # `file_commits`), get its size.
#
sz= # File size.
- if [[ ("$ftype" == "archive") && -v file_commits["$f"] ]]; then
- # POSIX specifies the output of `ls -l` so this should be
- # portable. -h turns on human-readable output (K, M, G) and is
- # present on Linux, FreeBSD, and Mac OS.
- #
- sz="$(ls -lh "$src_dir/$f" | cut -d ' ' -f 5)"
+ if [[ ("$ftype" == "archive") && -v "file_commits[$f]" ]]; then
+ sz="$(file_size "$f")"
fi
- # Note that, in destination-management mode, there can be no ownership
- # manifests in `file_commits`.
- #
- if [ "${file_commits[$f]}" == "$h" ]; then
- info " $f $sz" # Last added or moved by the current commit.
- elif [ -v file_commits["$f"] ]; then
+ if [[ "${file_commits[$f]}" == "$h" ]]; then
+ # Last added or moved by the current commit.
+ #
+ # If a commit included some managed and some unmanaged ownership
+ # manifests then it will not have been filtered out at the top and
+ # the unmanaged ownerships will also be shown here. (For an example,
+ # see the first commit output by `bpkg-util-manage --alpha` on the
+ # real cppget.org repos.) I don't think it's worth checking here
+ # that ownerships are managed because those sorts of commits should
+ # be rare. This script also refuses to operate on packages from
+ # different projects or sections.
+ #
+ info " $f $sz"
+ elif [[ -v "file_commits[$f]" ]]; then
info "! $f $sz" # Deleted and added back by subsequent commits.
- elif [[ ("$mode" == "source") || ("$ftype" != "ownership") ]]; then
- # File was deleted and never added again and, if we're in
- # destination-management mode, is not an ownership manifest.
+ else
+ # File was deleted and never added again. Note that actioned files
+ # of partially actioned commits will also appear this way.
#
info "* $f"
fi
@@ -1327,6 +2121,8 @@ while true; do
done < <(commit_files "$h")
done
+ # The main screen's prompt loop.
+ #
# Prompt the user for the action (showing the current bundle), get user
# input, and perform the selected action.
#
@@ -1334,7 +2130,8 @@ while true; do
# offer to migrate if the bundle array is empty, etc) but let's not
# complicate the logic.
#
- # Breaking out of this loop prints the pending commit list again.
+ # Breaking out of this loop re-initializes the global state if `init` is
+ # true and reprints the pending commit list.
#
while true; do
# Sort commit bundle in ascending order.
@@ -1347,19 +2144,21 @@ while true; do
# IFS) because neither space nor tab characters (the other members of IFS)
# can occur in the keys.
#
- bundle_sorted=($(sed 's/ /\n/g' <<<"${!bundle[*]}" | sort -))
+ bundle=($(sed 's/ /\n/g' <<<"${bundle[*]}" | sort --numeric-sort -))
printf "\n"
- read -p "[${bundle_sorted[*]}][<N>,m,c,p,l,q,?]: " opt
+ read -p "[${bundle[*]}][<N>,m,d,s,c,p,l,q,?]: " opt
case "$opt" in
# Add commit to bundle.
#
[0-9]*)
- if [[ ("$opt" -gt 0) && ("$opt " -le "${#pending_seq[@]}") ]]; then
- if [ ! -v bundle["$opt"] ]; then
- bundle["$opt"]=true
- info "commit $opt (${pending_seq[$opt-1]}) added to selected bundle"
+ if [[ ("$opt" =~ ^[1-9][0-9]*$) && -v "pending_seq[$opt-1]" ]]; then
+ if [[ ! "$(contains "$opt" "${bundle[@]}")" ]]; then
+ bundle+=("$opt")
+ h="${pending_seq[$opt-1]}"
+ info "commit $opt ($h) \"$(commit_subject "$h")\" \
+added to selected bundle"
else
info "commit $opt is already in the bundle"
fi
@@ -1370,15 +2169,62 @@ while true; do
# Migrate the commit bundle.
#
m)
- if [ "$mode" == "source" ]; then
- migrate_src
+ if [[ "${#bundle[@]}" -ne 0 ]]; then
+ collect_bundle_files # Prints error if `bundle_files` left empty.
+
+ if [[ "${#bundle_files[@]}" -ne 0 ]]; then
+ if [[ "$mode" == "source" ]]; then
+ migrate_src
+ else
+ migrate_dst
+ fi
+ if [[ "$operation_result" ]]; then
+ need_push=true
+ init=true
+ break
+ fi
+ fi
else
- migrate_dst
+ info "no commits selected"
fi
-
- if [ "$migrate_result" ]; then
- need_push=true
- break
+ ;;
+ # Drop the commit bundle.
+ #
+ d)
+ if [[ "${#bundle[@]}" -ne 0 ]]; then
+ collect_bundle_files # Prints error if `bundle_files` left empty.
+
+ if [[ "${#bundle_files[@]}" -ne 0 ]]; then
+ drop
+ if [[ "$operation_result" ]]; then
+ need_push=true
+ init=true
+ break
+ fi
+ fi
+ else
+ info "no commits selected"
+ fi
+ ;;
+ # Show commit splitting screen (operate on a selection of the commit
+ # bundle's files).
+ #
+ s)
+ if [[ "${#bundle[@]}" -ne 0 ]]; then
+ collect_bundle_files # Prints error if `bundle_files` left empty.
+
+ if [[ "${#bundle_files[@]}" -ne 0 ]]; then
+ split_commits
+
+ # Note that the global state is re-initialized even if nothing was
+ # done in the commit-splitting screen but these should be the
+ # minority of cases.
+ #
+ init=true
+ break
+ fi
+ else
+ info "no commits selected"
fi
;;
# Clear the commit bundle.
@@ -1402,7 +2248,7 @@ while true; do
# Quit.
#
q)
- if [ ! "$need_push" ]; then
+ if [[ ! "$need_push" ]]; then
exit 0
fi
@@ -1429,14 +2275,16 @@ while true; do
# ? or invalid option: print menu.
#
*)
- cat <<-EOF
+ cat <<EOF
<N> - add commit to the commit bundle
m - migrate the selected commit bundle
+ d - drop the selected commit bundle
+ s - split commit(s) (operate on selected files)
c - clear the selected commit bundle
p - push source and destination repositories
l - list pending commits
- q - quit (prompting to push if any actions have been taken)
+ q - quit (prompting to push if commits were made)
? - print this help
EOF
;;
diff --git a/bpkg-util/package-archive.bash.in b/bpkg-util/package-archive.bash.in
index 899e3c4..ba73a15 100644
--- a/bpkg-util/package-archive.bash.in
+++ b/bpkg-util/package-archive.bash.in
@@ -9,7 +9,7 @@ else
bpkg_util_package_archive=true
fi
-@import libbutl/manifest-parser@
+@import libbutl.bash/manifest-parser@
# We expect the user to set the bpkg_util_bpkg variable to the bpkg program
# path.
@@ -19,9 +19,9 @@ if [ ! -v bpkg_util_bpkg ]; then
exit 1
fi
-# Extract the package information from a package archive and print it in the
-# '<name> <version> <project>' form, where the project field is empty if the
-# project value is not specified in the manifest.
+# Extract the package information from a package archive and print it to
+# stdout in the '<name>\n<version>\n<project>\n' form, where the project field
+# is empty if the project value is not specified in the manifest.
#
# Note that, in particular, it verifies that the archive file name matches the
# package name and version.
@@ -57,14 +57,17 @@ function bpkg_util_pkg_verify_archive () # <path>
butl_manifest_parser_finish
- echo -n "$name $version $project"
+ echo "$name"
+ echo "$version"
+ echo "$project"
}
# Search for package archives in a directory using the package name and
-# version pattern and printing their paths newline-separated. If the version
-# argument is '*', then print archives for all package versions. Otherwise if
-# the version contains the trailing '*', then print archives for all revisions
-# of the specified version and for the exact version otherwise. For example:
+# version pattern and printing their paths one per line to stdout. If the
+# version argument is '*', then print archives for all package versions.
+# Otherwise if the version contains the trailing '*', then print archives for
+# all revisions of the specified version and for the exact version otherwise.
+# For example:
#
# bpkg_util_pkg_find_archives foo '*' dir/
# bpkg_util_pkg_find_archives foo '1.0*' dir/
@@ -73,20 +76,14 @@ function bpkg_util_pkg_verify_archive () # <path>
# Note that the resulting archive paths include the specified directory as a
# prefix.
#
-# NOTE: this function can be called with overriden IFS.
-#
function bpkg_util_pkg_find_archives () # <name> <version> <dir>
{
- IFS=$' \t\n' bpkg_util_pkg_find_archives_impl "$@"
-}
-
-function bpkg_util_pkg_find_archives_impl ()
-{
local nam="$1"
local ver="$2"
local dir="$3"
- local r=""
+ local r=()
+ local f
if [ -d "$dir" ]; then
local vr # Version with the revision stripped, if search for revisions.
@@ -103,10 +100,9 @@ function bpkg_util_pkg_find_archives_impl ()
# '1.2.3+2*': foo-1.2.3.tar.gz, foo-1.2.3+1.tar.gz, foo-1.2.30.tar.gz,
# etc) and return those which package name and version match properly.
#
- local f
while read f; do
local p
- p=($(bpkg_util_pkg_verify_archive "$f"))
+ bpkg_util_pkg_verify_archive "$f" | readarray -t p
local n="${p[0]}"
local v="${p[1]}"
@@ -116,37 +112,26 @@ function bpkg_util_pkg_find_archives_impl ()
"$v" == "$ver" || \
( -n "$vr" && "$v" =~ ^"$vr"(\+[0-9]+)?$ )) ]]; then
- if [ -n "$r" ]; then
- r="$r"$'\n'"$f"
- else
- r="$f"
- fi
+ r+=("$f")
fi
done < <(find "$dir" -type f -name "$np")
fi
- if [ -n "$r" ]; then
- echo -n "$r"
- fi
+ for f in "${r[@]}"; do
+ echo "$f"
+ done
}
# Search for a package archive in a directory using a file name pattern. If
-# the archive is found, then print the package information in the
-# '<path>\n<name>\n<version>\n<project>' form, where the project field is
+# the archive is found, then print the package information to stdout in the
+# '<path>\n<name>\n<version>\n<project>\n' form, where the project field is
# empty if the project value is not specified in the manifest.
#
# Note that if there are multiple archives matching the pattern, then it is
# unspecified which one is picked.
#
-# NOTE: this function can be called with overriden IFS.
-#
function bpkg_util_pkg_find_archive () # <pattern> <dir>
{
- IFS=$' \t\n' bpkg_util_pkg_find_archive_impl "$@"
-}
-
-function bpkg_util_pkg_find_archive_impl ()
-{
local pat="$1"
local dir="$2"
@@ -156,14 +141,17 @@ function bpkg_util_pkg_find_archive_impl ()
# We could probably use -print -quit but this is not portable (NetBSD
# needs -exit instead of -quit).
#
- f="$(find "$dir" -type f -name "$pat" | head -n 1)"
+ f="$(find "$dir" -type f -name "$pat" | sed -n -e '1p')"
if [ -n "$f" ]; then
local p
- p=($(bpkg_util_pkg_verify_archive "$f"))
+ bpkg_util_pkg_verify_archive "$f" | readarray -t p
- printf "$f\n${p[0]}\n${p[1]}\n${p[2]}"
+ echo "$f"
+ echo "${p[0]}"
+ echo "${p[1]}"
+ echo "${p[2]}"
return
fi
fi
diff --git a/build/root.build b/build/root.build
index 1cc51ca..5bb99dc 100644
--- a/build/root.build
+++ b/build/root.build
@@ -1,8 +1,7 @@
# file : build/root.build
# license : MIT; see accompanying LICENSE file
-# Bash.
+# Work around issue with sanitizers messing up rpath (see issue #1219).
#
-using bash
-
-bpkg-util/bash{*}: install.subdirs = true
+if ($build.mode != 'skeleton')
+ using bash
diff --git a/manifest b/manifest
index 45ee2f6..8b33478 100644
--- a/manifest
+++ b/manifest
@@ -1,6 +1,6 @@
: 1
name: bpkg-util
-version: 0.14.0-a.0.z
+version: 0.17.0-a.0.z
project: build2
summary: extra build2 package management utilities
license: MIT
@@ -12,11 +12,11 @@ doc-url: https://build2.org/doc.xhtml
src-url: https://git.build2.org/cgit/bpkg-util/tree/
email: users@build2.org
build-warning-email: builds@build2.org
-builds: all
+builds: all : &host
builds: -windows ; Requires bash.
builds: -macos ; Requires bash >= 4.3.
requires: bash >= 4.3
-depends: * build2 >= 0.13.0
-depends: * bpkg >= 0.13.0
-depends: bpkg [0.14.0-a.0.1 0.14.0-a.1)
-depends: libbutl.bash [0.14.0-a.0.1 0.14.0-a.1)
+depends: * build2 >= 0.16.0-
+depends: * bpkg >= 0.16.0-
+depends: bpkg [0.17.0-a.0.1 0.17.0-a.1)
+depends: libbutl.bash [0.17.0-a.0.1 0.17.0-a.1)
diff --git a/tests/package-archive/driver.in b/tests/package-archive/driver.in
index a32e571..5adecf1 100644
--- a/tests/package-archive/driver.in
+++ b/tests/package-archive/driver.in
@@ -8,13 +8,13 @@
bpkg_util_bpkg=bpkg
trap "{ exit 1; }" ERR
-set -o errtrace # Trap ERR in functions.
+set -o errtrace # Trap in functions and subshells.
+set -o pipefail # Fail if any pipeline command fails.
+shopt -s lastpipe # Execute last pipeline command in the current shell.
+shopt -s nullglob # Expand no-match globs to nothing rather than themselves.
@import bpkg-util/package-archive@
# Call the function passed on the command line.
#
-# Note that we reset IFS to make sure that the function being tested is not
-# affected by its value set by the caller.
-#
-IFS= "$@"
+"$@"
diff --git a/tests/package-archive/testscript b/tests/package-archive/testscript
index b2eb7e3..75df25a 100644
--- a/tests/package-archive/testscript
+++ b/tests/package-archive/testscript
@@ -6,7 +6,7 @@
# find utility to fail with the 'no such file or directory' error. Thus, we
# clone the archives into the test working directories and search there.
#
-clone_arcs = \
+clone_arcs = [cmdline] \
cp $src_base/libhello-0.1.0.tar.gz $src_base/libhello-0.1.0+1.tar.gz ./
: pkg-verify-archive
@@ -22,7 +22,11 @@ clone_arcs = \
: success
:
- $* $src_base/libhello-0.1.0.tar.gz >:'libhello 0.1.0 hello'
+ $* $src_base/libhello-0.1.0.tar.gz >>EOO
+ libhello
+ 0.1.0
+ hello
+ EOO
}
: pkg-find-archives
@@ -43,7 +47,7 @@ clone_arcs = \
{
$clone_arcs;
- $* 'libhello' '*' $~ >>:/~"%EOO%"
+ $* 'libhello' '*' $~ >>/~"%EOO%"
%\(
$~/libhello-0.1.0.tar.gz
$~/libhello-0.1.0+1.tar.gz
@@ -59,7 +63,7 @@ clone_arcs = \
{
$clone_arcs;
- $* 'libhello' '0.1.0' $~ >:/"$~/libhello-0.1.0.tar.gz"
+ $* 'libhello' '0.1.0' $~ >/"$~/libhello-0.1.0.tar.gz"
}
: package-revision
@@ -67,7 +71,7 @@ clone_arcs = \
{
$clone_arcs;
- $* 'libhello' '0.1.0+1' $~ >:/"$~/libhello-0.1.0+1.tar.gz"
+ $* 'libhello' '0.1.0+1' $~ >/"$~/libhello-0.1.0+1.tar.gz"
}
: package-revisions1
@@ -75,7 +79,7 @@ clone_arcs = \
{
$clone_arcs;
- $* 'libhello' '0.1.0*' $~ >>:/~"%EOO%"
+ $* 'libhello' '0.1.0*' $~ >>/~"%EOO%"
%\(
$~/libhello-0.1.0.tar.gz
$~/libhello-0.1.0+1.tar.gz
@@ -91,7 +95,7 @@ clone_arcs = \
{
$clone_arcs;
- $* 'libhello' '0.1.0+2*' $~ >>:/~"%EOO%"
+ $* 'libhello' '0.1.0+2*' $~ >>/~"%EOO%"
%\(
$~/libhello-0.1.0.tar.gz
$~/libhello-0.1.0+1.tar.gz
@@ -117,7 +121,7 @@ clone_arcs = \
{
$clone_arcs;
- $* 'libhello-0.1.0.*' $~ >>:/"EOO"
+ $* 'libhello-0.1.0.*' $~ >>/"EOO"
$~/libhello-0.1.0.tar.gz
libhello
0.1.0