path: root/bpkg-util/manage.in

#!/usr/bin/env bash

# @@ TODO: Use `` instead of '' for single quotes in comments.
#

# file      : bpkg-util/manage.in
# license   : MIT; see accompanying LICENSE file

# Interactively migrate packages and/or ownership manifests from a source git
# repository to a destination git repository.
#
# Present the user with the list of commits that added the files currently in
# the source repository's working directory and ask the user to select from a
# menu the action to perform on a selection of these commits.
#
# As the files added by these commits are pending a move to the destination
# repository, these commits will be referred to as "pending" commits.
#
# Actions that can be performed on a selection of pending commits include
# moving them from the source repository to a single commit in the destination
# repository and dropping them from the source repository.
#
# The flow of this script, in broad strokes, is as follows: for each file in
# the source repository directory, find the hash of the commit that added it;
# these are the pending commits. Arrange the pending commits in chronological
# order. Display to the user the pending commits along with the files they
# added. Let the user select one or more pending commits and an action to be
# performed on them. Each successful action results in a commit to the source
# and/or destination repositories and leaves both repositories in a clean
# state. Once the action has been performed, redisplay the updated pending
# commit list and prompt for the next action. Pushing to the remote
# repositories, a supported operation, can be done at any time during the
# session.
#
# Two modes of operation are supported. In source-management mode, package
# archives -- from any section -- and/or ownership manifests are migrated from
# the source repository to the destination repository. In
# destination-management mode, package archives (only) are migrated from one
# of the sections in the destination repository to its counterpart section
# (also in the destination repository). These mode-specific source directories
# are called "managed directories" and the files they contain "managed files",
# excluding those with an invalid path (project component is missed, etc); any
# other directory or file is "unmanaged".
#
# The destination-management mode is actually just a name used to refer to a
# number of more specific, near-identical modes distinguished only by the
# source section being managed (and, by extension, the destination
# section). For example, in testing-management mode packages are migrated from
# the 'testing' section to its 'stable' counterpart and, in stable-management
# mode, packages are migrated from the 'stable' section to its 'legacy'
# counterpart.
#
# Note that in destination-management mode, the source and destination
# repositories both refer to the "real" destination repository and, therefore,
# that the distinction made between the two modes throughout this script is
# fairly shallow.
#
# Options:
#
# -t
# --testing[=<filter>]
#
#   Enter the testing-management mode: manage the testing->stable transitions
#   in the destination repository.
#
# --stable[=<filter>]
#
#   Enter the stable-management mode: manage the stable->legacy transitions
#   in the destination repository.
#
# If <filter> is specified, then it is used to narrow down the list of commits
# to only those that contain packages with project name or package name and
# version matching the filter as a wildcard pattern. For example:
#
#   --stable=expat
#   --stable=libexpat-1.2.3
#   --stable=libexpat-2.*
#
# If neither --testing nor --stable are specified, operate in the
# source-management mode.
#
# Arguments:
#
# <dir> The directory into which the source and destination repositories have
#       been checked out. If not specified, current directory is assumed.
#
usage="usage: $0 [<options>] [<dir>]"

# Source/destination repository inside <dir>. Note: also used in commit
# messages.
#
src_repo_name=queue
dst_repo_name=public

owd="$(pwd)"
trap "{ cd '$owd'; exit 1; }" ERR
set -o errtrace # Trap in functions and subshells.

@import bpkg-util/utility@

# Use the bpkg program from the script directory, if present. Otherwise, use
# just 'bpkg'.
#
bpkg_util_bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg"

if [ ! -x "$bpkg_util_bpkg" ]; then
  bpkg_util_bpkg=bpkg
fi

@import bpkg-util/package-archive@

# The mode of operation. If its value is "source", manage the source
# repository. Otherwise manage the destination repository with the value being
# the name of the source section to manage.
#
mode="source"

# Archive-filtering pattern used in destination-management mode. Match
# everything by default.
#
filter="*"

while [ "$#" -gt 0 ]; do
  case "$1" in
    --testing=*)
      filter="${1#--testing=}"
      ;&
    -t|--testing)
      mode="testing"
      shift
      ;;
    --stable=*)
      filter="${1#--stable=}"
      ;&
    --stable)
      mode="stable"
      shift
      ;;
    *) break ;;
  esac
done

# Set the working directory.
#
if [ $# -eq 0 ]; then
  dir="$owd"
elif [ $# -eq 1 ]; then
  dir="${1%/}" # <dir> with trailing slash removed.
else
  error "$usage"
fi

# If in one of the destination-management modes, set the source repository
# name to that of the destination repository.
#
if [ "$mode" != "source" ]; then
  src_repo_name="$dst_repo_name"
fi

# The source and destination package repository directories.
#
# Note that, throughout this script, any path not explicitly prefixed with
# "$src_dir/" or "$dst_dir/" is relative to the root of the source or
# destination package repositories.
#
src_dir="$dir/$src_repo_name"
dst_dir="$dir/$dst_repo_name"

if [ ! -d "$src_dir" ]; then
  error "'$src_dir' does not exist or is not a directory"
fi

if [ ! -d "$dst_dir" ]; then
  error "'$dst_dir' does not exist or is not a directory"
fi

# Check that both git repositories are clean.
#
if [ -n "$(git -C $src_dir status --porcelain)" ]; then
  error "git repository in '$src_dir' is not clean"
fi

if [ -n "$(git -C $dst_dir status --porcelain)" ]; then
  error "git repository in '$dst_dir' is not clean"
fi

# Use run() to show the user that git is the source of the diagnostics.
# "Already up to date", for example, is too vague.
#
if [ "$mode" == "source" ]; then
  run git -C "$src_dir" pull >&2
fi
run git -C "$dst_dir" pull >&2

# Load the source and destination repositories' submit configurations (section
# name/directory mappings and owners directory path).
#
# Each repository's settings are sourced into the temporary variables 'owners'
# and 'sections' and copied from there to source- and destination-specific
# variables.
#
# If in one of the destination-management modes, store only the directory for
# the section being managed in src_sections, and only its counterpart in
# dst_sections. Otherwise, in source-management mode, store all source and
# destination section directories.
#
declare owners
declare -A sections
source "$src_dir/submit.config.bash"

src_owners="$owners"
declare -A src_sections

for s in "${!sections[@]}"; do
  if [[ ("$mode" == "source") || ("$s" == "$mode") ]]; then
    src_sections["$s"]="${sections[$s]}"
  fi
done

owners=
sections=()
source "$dst_dir/submit.config.bash"

# Section counterparts.
#
declare -A sect_cparts=(["testing"]="stable"
                        ["stable"]="legacy")

dst_owners="$owners"
declare -A dst_sections
for s in "${!sections[@]}"; do
  if [[ ("$mode" == "source") || ("$s" == "${sect_cparts[$mode]}") ]]; then
    dst_sections["$s"]="${sections[$s]}"
  fi
done

# Fail if in destination-management mode and the source and/or destination
# section is not configured for the destination repository. (The equivalent
# source-management mode checks can only be done later.)
#
if [[ ("$mode" != "source") &&
      (! -v src_sections["$mode"] ||
       ! -v dst_sections["${sect_cparts[$mode]}"]) ]]; then
  error "section '$mode' and/or '${sect_cparts[$mode]}' not configured \
in the destination repository"
fi

# Find all package archives and, if in source-management mode, owner manifest
# files in the source repository.
#
# Every file in a repository section directory except *.manifest is a package
# archive and every file in the owners directory is a project or package owner
# manifest. Therefore run find separately on each section directory and the
# owners directory to build a list containing only package-related files.
#
# Store the relative to the repository directory file paths in an array used
# to build the set of pending commits.
#
src_files=()
for s in "${src_sections[@]}"; do
  while read f; do
    src_files+=("${f#$src_dir/}")
  done < <(find "$src_dir/$s" -type f -not -name "*.manifest")
done

if [[ ("$mode" == "source") &&
      -n "$src_owners"      &&
      -d "$src_dir/$src_owners" ]]; then
  while read f; do
    src_files+=("${f#$src_dir/}")
  done < <(find "$src_dir/$src_owners" -type f)
fi

# Build the set of pending commit hashes ("pending set").
#
# For each file in the source repository, find the most recent commit that
# added it or moved it to its current location and store its abbreviated hash
# (as key) inside the 'pending_set' associative array (note: unordered) and
# (as value) inside the 'file_commits' associative array.
#
# 'file_commits' maps from the file path to the last commit to add it to the
# repository. A file may have been added and removed by earlier commits and
# could thus be migrated with the wrong commit unless care is taken (see the
# example in the migration notes below).
#
# If in destination-management mode, exclude from `pending_set` those commits
# without any package archives that match the pattern in `filter`.
#
declare -A pending_set
declare -A file_commits
for f in "${src_files[@]}"; do

  # -n 1:                limit output to one commit (that is, the most recent)
  # --diff-filter=A:     only show commits that added the specified file or
  #                      moved it into its current location ('R' is useless
  #                      if a path is specified)
  # --pretty=format:%h:  output only the abbreviated commit hash
  #
  h="$(git -C "$src_dir" log -n 1 --diff-filter=A --pretty=format:%h -- "$f")"

  # Note that the hash cannot be empty because, after our clean checks at the
  # top, every file on disk must have been added by some commit (that is,
  # there can be no untracked files).

  # Add the commit to the pending set unless the current file is filtered out.
  #
  # Note: `src_files` can contain only package archives if we're in
  # destination-management mode so there's no need to check the file type.
  #
  # Note: $filter must be unquoted to match as a wildcard pattern.
  #
  if [[ "$mode" == "source" ||
        ("$(basename "$(dirname "$f")")" == $filter) || # Project name?
        ("$(basename "$f")" == $filter.*) ]]; then      # Filename (with ext)?
    pending_set["$h"]=true
  fi

  file_commits["$f"]="$h"
done

# Arrange the pending commits in the chronological order.
#
# Go through the most recent commits in the git log which added or
# moved/renamed one or more files, skipping those not present in the pending
# set and keeping count to bail out as soon as we ordered all of them.
#
pending_seq=()
for (( i=0; i != "${#pending_set[@]}"; )); do
  read h # The abbreviated commit hash.

  # If this is a pending commit, prepend its hash to the ordered array.
  #
  if [ "${pending_set[$h]}" ]; then
    pending_seq=("$h" "${pending_seq[@]}")
    ((++i))
  fi

  # --diff-filter=AR:     only show commits that added or renamed files
  #
done < <(git -C "$src_dir" log --diff-filter=AR --pretty=format:%h)

if [ "${#pending_seq[@]}" -eq 0 ]; then
  info "good news, nothing to manage"
  exit 0
fi

# Clean the source and destination repositories by discarding uncommitted
# changes and removing unstaged files. (Note that the source repository cannot
# have untracked files so we git-clean only the destination repository.)
#
function cleanup ()
{
  info "migration failed; resetting and cleaning repositories"

  if ([ "$mode" == "source" ] && ! run git -C "$src_dir" reset --hard) ||
     ! run git -C "$dst_dir" reset --hard                              ||
     ! run git -C "$dst_dir" clean --force; then
    info "failed to reset/clean repositories -- manual intervention required"
  fi
}

# Return the list of files a commit added to the source repository.
#
function commit_files () # <commit-hash>
{
  local h="$1"

  # git-diff-tree arguments:
  #
  # --diff-filter=A:  select only files that were added.
  # -z:               don't munge file paths and separate output fields with
  #                   NULs.
  # -r:               recurse into subtrees (directories).
  #
  git -C "$src_dir" diff-tree \
      --no-commit-id --name-only --diff-filter=A -z -r \
      "$h"
}

# Print information about the path of a source repository file.
#
# The information includes its class (package archive, ownership manifest, or
# unmanaged), the project directory, and in the case of package archives, the
# section directory.
#
# <path> must be relative to the source repository directory (`src_dir`).
#
# If the path refers to a managed archive file in the source repository, then
# print `archive <project> <section>`.
#
# Otherwise, if the path refers to a managed ownership manifest file in the
# source repository, then print `ownership <project>`.
#
# Otherwise the file is unmanaged; print `unmanaged`.
#
# Note that the function doesn't validate the file path exhaustively and may
# classify improperly named file (invalid base name, etc) as an archive or
# ownership.
#
function src_path_info () # <path>
{
  local f="$1"

  # Check whether the file is a package archive. Valid source repository
  # package paths start with one of the section directories in `src_sections`
  # which is followed by a single subdirectory (which would be the project
  # directory, but this is not checked).
  #
  local s
  for s in "${src_sections[@]}"; do
    if [[ "$f" =~ ^"$s"/([^/]+)/[^/]+$ ]]; then
      echo -n "archive ${BASH_REMATCH[1]} $s"
      return
    fi
  done

  # Not a managed archive path, so check whether it's an ownership
  # manifest. Valid source repository ownership manifest paths start with the
  # directory in `src_owners` and is followed by at least one subdirectory
  # (the project directory, again).
  #
  if [[ -n "$src_owners" && ("$f" =~ ^"$src_owners"/([^/]+)/.+$) ]]; then
    echo -n "ownership ${BASH_REMATCH[1]}"
  else
    echo -n "unmanaged"
  fi
}

# Extract the package name, version, and project from a package archive's
# manifest and print it to stdout in the '<name> <version> <project>' form. If
# the manifest does not specify the project name, the package name is returned
# as the project name.
#
function extract_pkg_info () # <archive>
{
  local arc="$1"

  local r
  r=($(bpkg_util_pkg_verify_archive "$arc")) # <name> <version> <project>
  if [ ! -v r[2] ]; then
    r[2]="${r[0]}"
  fi

  # Verify that the archive parent directory name matches the project.
  #
  local p="${r[2]}"
  if [ "$p" != "$(basename "$(dirname "$arc")")" ]; then
    error "'$arc' archive directory name does not match package project '$p'"
  fi

  echo -n "${r[@]}"
}

# Exit with an error if a package which is a duplicate of or is in conflict
# with the given package exists in any of the destination sections.
#
# Two packages are duplicates if they have the same name and version and, as a
# result, the same archive filename. If, on the other hand, they have
# different names and/or versions but the same archive filename, they are in
# conflict with one another. For example, foo-bar version 1.0 and foo version
# bar-1.0 have the same archive name foo-bar-1.0.tar.gz.
#
function check_pkg_duplicate () # <pkg-name> <pkg-version>
{
  local name="$1"
  local version="$2"

  local sd                      # Section directory.
  for sd in "${dst_sections[@]}"; do
    local p

    # Use <name>-<version>.* without .tar.gz in case we want to support more
    # archive types later.
    #
    IFS=$'\n' eval \
    'p=($(bpkg_util_pkg_find_archive "$name-$version.*" "$dst_dir/$sd"))'

    if [ "${#p[@]}" -ne 0 ]; then
      local a="${p[0]}"
      local n="${p[1]}"
      local v="${p[2]}"

      if [ "$n" == "$name" ]; then
        error "duplicate of $name/$version at '$a'"
      else
        error "conflict of $name/$version with $n/$v at '$a'"
      fi
    fi
  done
}

# Remove other versions and/or revisions of a package ("replacement
# candidates") in the destination section if the user so chooses. Return the
# removed packages' version numbers.
#
# The <name> argument is the name of the source (and the destination) package.
#
# The <version-pattern> argument must be either "*" to remove all other
# versions (including revisions), or "<version>*" to remove other revisions
# only.
#
# The <dst-sect> argument is the destination section name.
#
# The <src-version> and <src-proj> arguments are the version and project of
# the source package and are only used in the package removal confirmation
# prompt.
#
# Search <dst-sect> for replacement candidates according to <version-pattern>
# and remove or skip each candidate at the user's direction. Stage (but don't
# commit) the removals in the destination repository.
#
# The versions of the removed packages are written to stdout, separated by
# spaces. For example:
#
#   1.2.3 1.3.0+1 1.3.0+2
#
# Note that, currently, versions/revisions which are both lower and higher
# than <pkg-version> will be considered for replacement.
#
function remove_pkg_archives ()
# <name> <version-pattern> <dst-sect> <src-version> <src-proj>
{
  local name="$1"
  local vpat="$2"
  local dsect="$3"
  local sver="$4"
  local sproj="$5"

  local rv=()                   # Removed version numbers.

  # Search for replacement candidates.
  #
  local pkgs=()                 # Packages to be considered for replacement.

  IFS=$'\n' eval \
  'pkgs=($(bpkg_util_pkg_find_archives "$name" \
                                       "$vpat" \
                                       "$dst_dir/${dst_sections[$dsect]}"))'

  # For each replacement candidate, ask for confirmation and, depending on the
  # answer, either remove it from the destination repository or leave it in
  # place.
  #
  local f
  for f in "${pkgs[@]}"; do
    # Get the destination archive's info from its embedded manifest.
    #
    local p
    p=($(extract_pkg_info "$f"))

    local dver="${p[1]}"          # Destination package version.
    local dproj="${p[2]}"         # Destination package project.

    # Ask whether or not to drop the destination package. Include the project
    # names in the prompt if the destination package's project differs from
    # that of the source package (which is never the case in
    # destination-management mode).
    #
    local src="$sver"
    local dst="$name/$dver"
    if [ "$dproj" != "$sproj" ]; then
      src+=" ($sproj)"
      dst+=" ($dproj)"
    fi

    local opt
    while true; do
      read -p "replace $dst with $src in $dsect? [y/n]: " opt

      case "$opt" in
        y)
          run git -C "$dst_dir" rm --quiet "${f#$dst_dir/}"
          rv+=("$dver")
          break
          ;;
        n)
          break
          ;;
      esac
    done
  done

  echo -n "${rv[@]}"
}

# The commit bundle associative array is the set of selected pending
# commits. Its keys are the corresponding indexes of the 'pending_seq' array
# (but offset by +1). Note: the reason the commit bundle is an associative
# array is to prevent duplicates.
#
declare -A bundle

# After a successful migration, reset some global state and pause to give the
# operator a chance to look at the commits before the list of remaining
# pending commits is displayed.
#
function migrate_epilogue ()
{
  # Remove the hashes in the commit bundle from the pending sequence and clear
  # the commit bundle.
  #
  local i
  for i in "${!bundle[@]}"; do
    unset pending_seq[i-1]
  done
  pending_seq=("${pending_seq[@]}") # Remove the gaps created by unset.
  bundle=()

  read -p "press Enter to continue"
}

# Migrate (all management modes):
#
# Files belonging to one or more user-selected pending commits are moved from
# the source to the destination, with the move of each file staged in the
# source and/or destination repositories. The action taken on each file is
# also recorded in the commit message(s).
#
# Note the following edge case which applies to all management modes:
#
#   We will need to confirm with git that each of a commit's added files were
#   actually most recently added by that commit. For example (oldest commits
#   first):
#
#     commit 1: add foo.tar.gz, bar.tar.gz
#     commit 2: del foo.tar.gz
#     commit 3: add foo.tar.gz
#
#   If the user chooses to migrate commit 1, only bar.tar.gz must be migrated,
#   despite foo.tar.gz existing on disk.

# Source-management mode migration: migrate the selected commit bundle from
# the source repository to the destination repository.
#
# If the migration succeeds, set the global migrate_result variable to
# true. Otherwise, in case of failure, issue appropriate diagnostics and set
# migrate_result to the empty string before returning. In order to ensure that
# both repositories are left clean if any of the files fail to be migrated, an
# EXIT signal handler that discards all uncommitted changes is installed right
# before the migration proper begins, and uninstalled when all files have been
# migrated.
#
# The migration will fail if any of the following is true:
#
# - The commit bundle is empty.
#
# - Files in the commit bundle are not all from the same project (the "bundle
#   project") or, in the case of archives, the same repository section (the
#   "bundle section").
#
# - The required section does not exist in the destination repository.
#
# - There exists, for one or more files in the commit bundle, a duplicate or
#   conflicting archive in any of the sections in the destination repository
#   (see check_pkg_duplicate()).
#
# - Any file in the commit bundle is unmanaged.
#
# The migration process proceeds as follows:
#
# - Move files: all of the files in the selected commit bundle are moved from
#   the source repository into the destination repository:
#
#   - Owners to owners directory.
#
#   - Packages into corresponding sections:
#
#     alpha -> alpha
#     beta  -> beta
#     stable -> testing|stable
#
#   Package archives may be removed and ownership manifests overwritten at the
#   destination. Candidate files for replacement are selected as follows:
#
#   - In the alpha and beta sections, any package archive files in the
#     destination section directory belonging to the same package are
#     considered for replacement, regardless of their versions.
#
#   - In other sections, any package archives in the destination section
#     directory with the same name and version but a different revision
#     (currently whether lower or higher) are replaced.
#
#   - Project and package ownership manifests will be replaced (that is,
#     simply overwritten) at the destination with any ownership manifests
#     added by the commit bundle because their presence implies that ownership
#     information has changed.
#
#   Stage (but don't commit) the removal of the files from the source
#   repository and their addition to the destination repository.
#
# - Make commits to the source and destination respositories with appropriate
#   commit messages:
#
#    Source repository:
#
#    "Migrate <project> from $src_sect to $dst_repo_name/<section>"
#
#      "remove <package>/<version>"
#      "remove owners/<project>/project-owner.manifest"
#      "remove owners/<project>/<package>/package-owner.manifest"
#
#    Destination repository:
#
#    "Migrate <project> from $src_repo_name/<section> to $dst_sect"
#
#      "add <package>/<version>"
#      "replace <package>/<version> with <version>"     (if replacing)
#      "add owners/<project>/project-owner.manifest"
#      "add owners/<project>/<package>/package-owner.manifest"
#
function migrate_src ()
{
  migrate_result=

  if [ "${#bundle[@]}" -eq 0 ]; then
    info "no commits selected"
    return
  fi

  # Check that every commit's files are managed packages or ownership
  # manifests in the bundle section and/or bundle project before migrating any
  # of them. Build the bundle's list of files as we go along, classifying them
  # as package archives or ownership manifests based on their paths.
  #
  # The bundle section is derived from the first package archive encountered
  # and the bundle project from the first package archive or owner manifest
  # encountered.
  #
  # Note that the bundle traversal is unordered.
  #
  local src_sect=          # Source section name.
  local src_sect_dir=      # Source section directory.
  local proj=              # The bundle (source) project.
  local pkgs=()            # The bundle's archive files.
  local owns=()            # The bundle's ownership manifests.

  local i
  for i in "${!bundle[@]}"; do
    local h="${pending_seq[i-1]}" # The current commit's abbreviated hash.

    # Check the current commit's files.
    #
    local f
    while read -d '' f; do
      # Derive the project and/or section names from the file path.
      #
      local fproj= # Current file's project.
      local fi=($(src_path_info "$f"))

      if [ "${fi[0]}" == "ownership" ]; then
        if [ "${file_commits[$f]}" != "$h" ]; then
          continue  # Ownership manifest was removed by a subsequent commit.
        fi

        fproj="${fi[1]}"
        owns+=("$f")

      elif [ "${fi[0]}" == "archive" ]; then
        if [ "${file_commits[$f]}" != "$h" ]; then
          continue  # Archive was removed by a subsequent commit.
        fi

        fproj="${fi[1]}"
        local fsect_dir="${fi[2]}"
        pkgs+=("$f")

        # Find, in `src_sections`, the archive section name associated with
        # the section directory extracted from the path (a value-to-key
        # lookup).
        #
        local fsect=

        # The "*" key is a catch-all for unknown submitted section names and,
        # if present, will share a value (section directory) with one of the
        # known section names and therefore must be skipped.
        #
        local k
        for k in "${!src_sections[@]}"; do
          if [[ ("${src_sections[$k]%/}" == "$fsect_dir") &&
                ("$k" != "*") ]]; then
            fsect="$k"    # Current file's section name.
            break
          fi
        done

        if [ -z "$fsect" ]; then
          # The only way that fsect can be empty is due to a programming error
          # or if the "*" key has a unique value -- which would be a submit
          # config error. So it would probably be better to terminate the
          # script.
          #
          error "unable to find section name for file '$f'"
        fi

        # Set the source section name and directory if unset; otherwise fail
        # if the current file is not from the source section.
        #
        if [ -z "$src_sect" ]; then
          src_sect="$fsect"
          src_sect_dir="$fsect_dir"
        elif [ "$fsect" != "$src_sect" ]; then
          info "cannot include commit $i: '$f' is not in section $src_sect"
          return
        fi
      else
        info "cannot include commit $i: '$f' is unmanaged"
        return
      fi

      # Set the bundle project if unset; otherwise fail if the current file is
      # not from the bundle project.
      #
      # Note: $fproj cannot be empty here (see above).
      #
      if [ -z "$proj" ]; then
        proj="$fproj"
      elif [ "$fproj" != "$proj" ]; then
        info "cannot include commit $i: '$f' is not in project $proj"
        return
      fi
    done < <(commit_files "$h")
  done

  # Finalize migration variables the values of which depend on whether the
  # bundle contains at least one package archive or ownership manifests only.
  #
  # The source and destination commit messages are composed incrementally as
  # the migration process proceeds.
  #
  local dst_sect     # Destination section name.
  local dst_sect_dir # Destination section directory.
  local src_cmsg     # Source commit message.
  local dst_cmsg     # Destination commit message.

  if [ "${#pkgs[@]}" -ne 0 ]; then # Bundle contains package archive(s).
    dst_sect="$src_sect"

    # If it exists, 'testing' overrides 'stable' at the destination.
    #
    if [[ ("$dst_sect" == "stable") && -v dst_sections["testing"] ]]; then
      dst_sect="testing"
    fi

    # Fail if the target section does not exist in the destination repository.
    #
    if [ ! -v dst_sections["$dst_sect"] ]; then
      info "section '$dst_sect' does not exist in the destination repository"
      return
    fi

    dst_sect_dir="${dst_sections[$dst_sect]}"

    src_cmsg="Migrate $proj from $src_sect to $dst_repo_name/$dst_sect"$'\n\n'
    dst_cmsg="Migrate $proj from $src_repo_name/$src_sect to $dst_sect"$'\n\n'
  else                           # Bundle consists only of ownership manifests.

    # The setup where the ownership authentication is disabled on the
    # destination but enabled on source is probably obscure, but let's
    # consider it possible since the submit-git handler allows such a setup.
    #
    if [ -n "$dst_owners" ]; then
      src_cmsg="Migrate $proj ownership info to $dst_repo_name"$'\n\n'
      dst_cmsg="Migrate $proj ownership info from $src_repo_name"$'\n\n'
    else
      src_cmsg="Remove $proj ownership info"$'\n\n'
      dst_cmsg= # Nothing to commit.
    fi
  fi

  # Migrate the bundle's package archive files.
  #
  # Ensure that the source and destination repositories are clean if the
  # migration of any file fails.
  #
  trap cleanup EXIT

  # Migrate a package archive or ownership manifest file from the source
  # repository to the destination repository.
  #
  # <src> is the path of the source file, relative to the source repository
  # directory, and <dst> is the path of the destination directory, relative to
  # the destination repository directory.
  #
  # Note that the source and destination sections and owners directories may
  # differ (as they do in these examples) which is why those components must
  # be specified in both the source and destination paths.
  #
  # Move the file from the source repository directory to the destination
  # repository directory, creating directories if required; stage the addition
  # of the file to the destination repository; stage the removal of the file
  # from the source repository.
  #
  function migrate_file () # <src> <dst>
  {
    local src="$1"
    local dst="$2"

    mkdir -p "$dst_dir/$dst"
    mv "$src_dir/$src" "$dst_dir/$dst/"
    run git -C "$src_dir" rm --quiet "$src"
    run git -C "$dst_dir" add "$dst/$(basename "$src")"
  }

  for f in "${pkgs[@]}"; do
    # Get the current package's name and version from its embedded manifest
    # (we already have the source project in $proj).
    #
    local p
    p=($(extract_pkg_info "$src_dir/$f"))

    local name="${p[0]}"
    local version="${p[1]}"

    check_pkg_duplicate "$name" "$version"

    # In the destination repository, find and remove package archive files
    # which are other alpha/beta versions or revisions of the current source
    # package.
    #
    local vpat                  # Version pattern for replacement.
    case "$dst_sect" in
      alpha|beta) vpat="*"         ;; # All package versions.
      *)          vpat="$version*" ;; # All package version revisions.
    esac

    local rv                    # Removed version numbers.
    rv=($(remove_pkg_archives "$name" "$vpat" \
                              "$dst_sect" \
                              "$version" "$proj"))

    # Update the commit messages and migrate the current package.
    #
    src_cmsg+="  remove $name/$version"$'\n'
    if [ "${#rv[@]}" -eq 0 ]; then
      dst_cmsg+="  add $name/$version"$'\n'
    else
      for ((i=0; i != "${#rv[@]}"; ++i)); do
        dst_cmsg+="  replace $name/${rv[i]} with $version"$'\n'
      done
    fi

    migrate_file "$f" "$dst_sect_dir/$proj"
  done

  # Migrate the bundle's ownership manifests.
  #
  # If ownership authentication is disabled on the destination repository,
  # only remove ownership manifests from the source repository (that is, don't
  # migrate).
  #
  for f in "${owns[@]}"; do
    src_cmsg+="  remove $f"$'\n'

    if [ -n "$dst_owners" ]; then
      local dp=$(dirname "${f/$src_owners/$dst_owners}")  # Destination path.

      # Let the commit message reflect whether this is a new ownership
      # manifest or is replacing an existent one.
      #
      local fn=$(basename "$f")
      if [ ! -e "$dst_dir/$dp/$fn" ]; then
        dst_cmsg+="  add $dp/$fn"$'\n'
      else
        dst_cmsg+="  update $dp/$fn"$'\n'
      fi

      migrate_file "$f" "$dp"
    else
      run git -C "$src_dir" rm --quiet "$f"
    fi
  done

  # Commit the changes made to the source and destination repositories.
  #
  info
  run git -C "$src_dir" commit -m "$src_cmsg"

  if [ -n "$dst_cmsg" ]; then
    info
    run git -C "$dst_dir" commit -m "$dst_cmsg"
  fi

  info

  # All files have been migrated successfully so set the result and clear the
  # EXIT trap.
  #
  migrate_result=true
  trap EXIT

  migrate_epilogue
}

# Destination-management mode migration: migrate the package archives in the
# selected commit bundle from the source (managed) section to its counterpart
# section.
#
# The general structure of this function is very similar to that of
# migrate_src() but most of the logic is simpler. Some noteworthy differences
# include:
#
# - Only the destination repository is involved.
#
# - All managed files in the commit bundle are package archives (ownership
#   manifests stay where they are and are therefore skipped/ignored).
#
# - All managed package archives in the commit bundle are known to be in the
#   same section (because all files outside of the managed section are
#   ignored).
#
# The mechanism by which success or failure is indicated is the same as for
# migrate_src().
#
# The migration will fail if any of the following is true:
#
# - The commit bundle is empty.
#
# - Package archives in the commit bundle are not all from the same project
#   (the "bundle project").
#
# - There exists, for one or more package archives in the commit bundle, a
#   duplicate or conflicting archive in the destination section (see
#   check_pkg_duplicate()).
#
# Note that the source and destination sections are assumed to be valid.
#
# The migration process proceeds as follows:
#
# - Move files: all of the package archives in the selected commit bundle are
#   moved from the source section to the destination section.
#
#   Any package archives in the destination section directory with the same
#   name and version but a different revision (currently whether lower or
#   higher) will be replaced if the user so chooses.
#
#   Stage (but don't commit) each file move as the migration proceeds.
#
# - Make a commit to the destination repository with an appropriate commit
#   message:
#
#   "Migrate <project> from $src_sect to $dst_sect"
#
#     "move <package>/<version>"
#     "replace <package>/<version> with <version>"   (if replacing)
#
function migrate_dst ()
{
  migrate_result=

  if [ "${#bundle[@]}" -eq 0 ]; then
    info "no commits selected"
    return
  fi

  local src_sect="$mode"                          # Source section.
  local src_sect_dir="${src_sections[$src_sect]}" # Source section directory.
  local dst_sect="${sect_cparts[$src_sect]}"      # Destination section.
  local dst_sect_dir="${dst_sections[$dst_sect]}" # Dst. section directory.

  # Check that every file in the commit bundle is a managed package archive
  # (ownership manifests are skipped) from the bundle project (taken from the
  # path of the first file encountered). Also build the bundle's list of files
  # as we go along.
  #
  # Note that the bundle traversal is unordered.
  #
  local proj=   # The bundle project.
  local pkgs=() # The bundle's files (package archives, all).

  local i
  for i in "${!bundle[@]}"; do
    local h="${pending_seq[i-1]}" # The current commit's abbreviated hash.

    # Check the current commit's files.
    #
    local f
    while read -d '' f; do
      local fi=($(src_path_info "$f"))

      # Fail if this is an unmanaged file. Skip ownership manifests and
      # archives deleted by subsequent commits. (Note that ownership manifests
      # are not stored in `file_commits` in destination-management mode.)
      #
      if [ "${fi[0]}" == "unmanaged" ]; then
        info "cannot include commit $i: '$f' is unmanaged"
        return
      elif [ "${file_commits[$f]}" != "$h" ]; then
        continue              # Ownership manifest or deleted package archive.
      fi

      local fproj="${fi[1]}"    # Current file's project.

      # Set the bundle project if unset; otherwise fail if the current file is
      # not from the bundle project.
      #
      if [ -z "$proj" ]; then
        proj="$fproj"
      elif [ "$fproj" != "$proj" ]; then
        info "cannot include commit $i: '$f' is not in project $proj"
        return
      fi

      pkgs+=("$f")
    done < <(commit_files "$h")
  done

  # Migrate the bundle's files.
  #
  # Ensure that the source and destination repositories are clean if the
  # migration of any package archive fails.
  #
  trap cleanup EXIT

  local cmsg=                   # The detailed part of the commit message.

  for f in "${pkgs[@]}"; do
    # Get the current package's name and version from its embedded manifest
    # (we already have the source project in $proj).
    #
    local p
    p=($(extract_pkg_info "$src_dir/$f"))

    local name="${p[0]}"
    local version="${p[1]}"

    # Note that only the destination section is checked because that is the
    # only one loaded into dst_sections when in destination-management mode.
    #
    check_pkg_duplicate "$name" "$version"

    # Find and remove other revisions of the current package.
    #
    local rv                    # Removed version numbers.
    rv=($(remove_pkg_archives "$name" "$version*" \
                              "$dst_sect" \
                              "$version" "$proj"))

    # Update the commit message.
    #
    if [ "${#rv[@]}" -eq 0 ]; then
      cmsg+="  move $name/$version"$'\n'
    else
      for ((i=0; i != "${#rv[@]}"; ++i)); do
        cmsg+="  replace $name/${rv[i]} with $version"$'\n'
      done
    fi

    # Migrate the current package.
    #
    mkdir -p "$dst_dir/$dst_sect_dir/$proj"
    run git -C "$dst_dir" mv "$f" "$dst_sect_dir/$proj/"
  done

  # Remove the project directory from the source section if it is empty.
  #
  # (Unlike git-mv, git-rm automatically removes the directory when its last
  # file is removed, so this does not need to be done in migrate_src().)
  #
  local d="$dst_dir/$src_sect_dir/$proj/"
  if [ -z "$(ls -A "$d")" ]; then
    rmdir "$d"
  fi

  # Commit the staged changes.
  #
  info
  run git -C "$dst_dir" commit \
          -m "Migrate $proj from $src_sect to $dst_sect"$'\n\n'"$cmsg"
  info

  # All files have been migrated successfully so set the result and clear the
  # EXIT trap.
  #
  migrate_result=true
  trap EXIT

  migrate_epilogue
}

# Push local changes to the remote source and/or destination git repositories.
#
# Push to the destination repository first because thus the migrated files
# will be in both remote repositories until the completion of the subsequent
# push to the source repository (which may fail or take long). Although this
# is an inconsistent state, it is safe because other programs such as a
# submission handler will be able to detect the duplicates and therefore
# refuse to do anything. If, on the other hand, we pushed to the source first,
# the migrated files would not exist in either remote repository until the
# push to the destination repository completed. In this state the submission
# handler would, for example, accept a resubmission of the migrated packages or
# erroneously establish ownership for already owned project/package names.
#
function push ()
{
  # Let's print additional diagnostics on git-push failure, to emphasize for
  # the user which of the two repositories we have failed to push.
  #
  if ! run git -C "$dst_dir" push; then
    error "push to $dst_repo_name failed"
  fi

  if [ "$mode" == "source" ] && ! run git -C "$src_dir" push; then
    error "push to $src_repo_name failed"
  fi
}

# Present the list of pending commits to the user, oldest first, marking files
# that were deleted by subsequent commits with `*` and, of those, the ones
# that were then added back again with `!`. Files from unmanaged directories
# are marked with `?`.
#
# 1 (deadbeef) Add libfoo/1.2.3
#
#   1/testing/foo/libfoo-1.2.3.tar.gz
#   owners/foo/project-owner.manifest
#   owners/foo/libfoo/package-owner.manifest
#
# 2 (c00l0fff) Add bar/1.2.3
#
# * 1/testing/bar/libbar-1.2.3.tar.gz
#   1/testing/bar/libbaz-1.2.3.tar.gz
#
# 3 (deadbabe) Add libbar/1.2.3+1
#
#   1/testing/bar/libbar-1.2.3+1.tar.gz
#
# Note that files deleted by subsequent commits may still be in the
# repository. See the example in the above migration notes.
#
# Then prompt the user for the action (showing the current bundle):
#
# [1 2][<N>,m,c,p,q,l,?]:
#
# <N> - add commit to the commit bundle
#  m  - migrate the selected commit bundle
#  c  - clear the selected commit bundle
#  p  - push source and destination repositories
#  l  - list pending commits
#  q  - quit (prompting to push if any actions have been taken)
#  ?  - print this help
#
# The user interaction loop.
#
# In each iteration, present the list of pending commits, display the menu of
# actions, read the user's input, and perform the chosen action.
#
# True if any changes have been made to the source and/or destination git
# repositories (in which case the user will be asked whether or not to push
# before quitting).
#
need_push=

while true; do
  # Show the pending commits.
  #
  if [ "${#pending_seq[@]}" -eq 0 ]; then
    info "no more pending commits"
  fi

  for ((i=0; i != "${#pending_seq[@]}"; i++)); do
    h="${pending_seq[$i]}"

    # Print commit number, hash, and subject.
    #
    # The commit number is left-padded with 0s to 3 digits. Prefix with a
    # newline to separate the first commit from the git-pull output and the
    # rest from the previous commit info block.
    #
    subj="$(git -C "$src_dir" log -n 1 --pretty=format:%s "$h")"
    printf "\n%d (%s) %s\n\n" "$((i+1))" "$h" "$subj" >&2

    # Print this commit's files.
    #
    # Fetch from the git repository the list of files added or moved by the
    # current commit and print each one's path.
    #
    # Mark files that cannot be migrated:
    #
    # - Unmanaged files are marked with '?' and will prevent the migration of
    #   the commit.
    #
    # - Files associated with a different commit hash in 'file_commits' were
    #   deleted and added back by subsequent commits.  These files are marked
    #   with '!' and will not prevent the migration of the commit.
    #
    # - Files with no association in 'file_commits' were deleted by a
    #   subsequent commit and never added back. These files are marked with
    #   '*' and will not prevent the migration of the commit.
    #
    while read -d '' f; do
      fi=($(src_path_info "$f"))
      ftype="${fi[0]}"

      if [ "$ftype" == "unmanaged" ]; then
        # File is unmanaged (and may or may not exist).
        #
        info "? $f"
      else
        # File is managed.

        # If this file is a package archive which exists (that is, it's in
        # `file_commits`), get its size in the human-readable form.
        #
        sz=  # File size.
        if [[ ("$ftype" == "archive") && -v file_commits["$f"] ]]; then
          # POSIX specifies the output of `ls -l` so this should be
          # portable. -h turns on human-readable output (K, M, G) and is
          # present on Linux, FreeBSD, and Mac OS.
          #
          sz="$(ls -lh "$src_dir/$f" | cut -d ' ' -f 5)"
        fi

        # Note that, in destination-management mode, there can be no ownership
        # manifests in `file_commits`.
        #
        if [ "${file_commits[$f]}" == "$h" ]; then
          info "  $f  $sz"  # Last added or moved by the current commit.
        elif [ -v file_commits["$f"] ]; then
          info "! $f  $sz"  # Deleted and added back by subsequent commits.
        elif [[ ("$mode" == "source") || ("$ftype" != "ownership") ]]; then
          # File was deleted and never added again and, if we're in
          # destination-management mode, is not an ownership manifest.
          #
          info "* $f"
        fi
      fi
    done < <(commit_files "$h")
  done

  # Prompt the user for the action (showing the current bundle), get user
  # input, and perform the selected action.
  #
  # Note that we could adapt the menu according to the current state (don't
  # offer to migrate if the bundle array is empty, etc) but let's not
  # complicate the logic.
  #
  # Breaking out of this loop prints the pending commit list again.
  #
  while true; do
    # Sort commit bundle in ascending order.
    #
    # Expand the 'bundle' associative array's keys into a single word in which
    # they are separated by spaces (the first member of IFS) using the
    # ${!a[*]} syntax; replace each space with a newline before piping to
    # 'sort', which is newline-based; finally collect sort's output into an
    # array using the a=() syntax, which splits on newline (the last member of
    # IFS) because neither space nor tab characters (the other members of IFS)
    # can occur in the keys.
    #
    bundle_sorted=($(sed 's/ /\n/g' <<<"${!bundle[*]}" | sort -))

    printf "\n"
    read -p "[${bundle_sorted[*]}][<N>,m,c,p,l,q,?]: " opt

    case "$opt" in
      # Add commit to bundle.
      #
      [0-9]*)
        if [[ ("$opt" -gt 0) && ("$opt " -le "${#pending_seq[@]}") ]]; then
          if [ ! -v bundle["$opt"] ]; then
            bundle["$opt"]=true
            info "commit $opt (${pending_seq[$opt-1]}) added to selected bundle"
          else
            info "commit $opt is already in the bundle"
          fi
        else
          info "non-existent commit number $opt"
        fi
        ;;
      # Migrate the commit bundle.
      #
      m)
        if [ "$mode" == "source" ]; then
          migrate_src
        else
          migrate_dst
        fi

        if [ "$migrate_result" ]; then
          need_push=true
          break
        fi
        ;;
      # Clear the commit bundle.
      #
      c)
        bundle=()
        break
        ;;
      # Push changes.
      #
      p)
        push
        need_push=
        break
        ;;
      # Redraw the pending commit list.
      #
      l)
        break
        ;;
      # Quit.
      #
      q)
        if [ ! "$need_push" ]; then
          exit 0
        fi

        while true; do
          read -p "push changes? [y/n/(c)ancel]: " opt

          case "$opt" in
            c)
              break             # Print options menu again.
              ;;
            y)
              push
              exit 0
              ;;
            n)
              exit 0
              ;;
            *)
              continue
              ;;
          esac
        done
        ;;
      # ? or invalid option: print menu.
      #
      *)
        cat <<-EOF

 <N> - add commit to the commit bundle
  m  - migrate the selected commit bundle
  c  - clear the selected commit bundle
  p  - push source and destination repositories
  l  - list pending commits
  q  - quit (prompting to push if any actions have been taken)
  ?  - print this help
EOF
        ;;
    esac
  done
done