#!/usr/bin/env bash # @@ TODO: Use `` instead of '' for single quotes in comments. # # file : bpkg-util/manage.in # license : MIT; see accompanying LICENSE file # Interactively migrate packages and/or ownership manifests from a source git # repository to a destination git repository. # # The commit that most recently added a file or moved it to its current # location "owns" the file and the file "belongs to" the commit. # # Present the user with the list of commits that own the files currently in # the source repository's working directory and ask the user to select from a # menu the action to perform on a selection of these commits. # # As these commits' files are pending removal or a move to the destination # repository they will be referred to as "pending" commits. # # Actions that can be performed on a selection of pending commits ("commit # operations") include moving them from the source repository to a single # commit in the destination repository and dropping them from the source # repository. Note that, by implication, a file that has been operated upon no # longer exists in the source repository. A file that has been operated upon # is called an "actioned file" and a commit whose files have all been actioned # an "actioned commit". It is also possible to operate on only a subset of # each selected commit's files instead of all of them. These are called "file # operations" and are selected from the "commit splitting screen"; see below # for more information. # # The flow of this script, in broad strokes, is as follows: for each file in # the source repository directory, find the hash of the commit it belongs to; # these are the pending commits. Arrange the pending commits in chronological # order. Display to the user the pending commits along with the files they # own. Let the user select one or more pending commits and an operation to be # performed on them. Each successful operation results in a commit to the # source and/or destination repositories and leaves both repositories in a # clean state. Once the operation has been performed, re-initialize the # pending commit list from the files in the source repository (that is, from # scratch), redisplay the updated pending commit list, and prompt for the next # action. Pushing to the remote repositories, a supported operation, can be # done at any time during the session. (Note that only in-memory structures # are updated after a file operation in the commit splitting screen -- the # pending commit list is only re-initialized when the user returns to the main # screen.) # # Two modes of operation are supported. In source-management mode, package # archives -- from any section -- and/or ownership manifests are migrated from # the source repository to the destination repository. In # destination-management mode, package archives (only) are migrated from one # of the sections in the destination repository to its counterpart section # (also in the destination repository). These mode-specific source directories # are called "managed directories" and the files they contain "managed files", # excluding those with an invalid path (project component is missed, etc); any # other directory or file is "unmanaged". # # The destination-management mode is actually just a name used to refer to a # number of more specific, near-identical modes distinguished only by the # source section being managed (and, by extension, the destination # section). For example, in testing-management mode packages are migrated from # the 'testing' section to its 'stable' counterpart and, in stable-management # mode, packages are migrated from the 'stable' section to its 'legacy' # counterpart. # # Note that in destination-management mode, the source and destination # repositories both refer to the "real" destination repository and, therefore, # that the distinction made between the two modes throughout this script is # fairly shallow. # # Options: # # -t # --testing[=] # # Enter the testing-management mode: manage the testing->stable transitions # in the destination repository. # # --stable[=] # # Enter the stable-management mode: manage the stable->legacy transitions # in the destination repository. # # If is specified, then it is used to narrow down the list of commits # to only those that contain packages with project name or package name and # version matching the filter as a wildcard pattern. For example: # # --stable=expat # --stable=libexpat-1.2.3 # --stable=libexpat-2.* # # If neither --testing nor --stable are specified, operate in the # source-management mode. # # Arguments: # # The directory into which the source and destination repositories have # been checked out. If not specified, current directory is assumed. # usage="usage: $0 [] []" # Source/destination repository inside . Note: also used in commit # messages. # src_repo_name=queue dst_repo_name=public owd="$(pwd)" trap "{ cd '$owd'; exit 1; }" ERR set -o errtrace # Trap in functions and subshells. @import bpkg-util/utility@ # Use the bpkg program from the script directory, if present. Otherwise, use # just 'bpkg'. # bpkg_util_bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" if [ ! -x "$bpkg_util_bpkg" ]; then bpkg_util_bpkg=bpkg fi @import bpkg-util/package-archive@ # The mode of operation. If its value is "source", manage the source # repository. Otherwise manage the destination repository with the value being # the name of the source section to manage. # mode="source" # Archive-filtering pattern used in destination-management mode. Match # everything by default. # filter="*" while [ "$#" -gt 0 ]; do case "$1" in --testing=*) filter="${1#--testing=}" ;& -t|--testing) mode="testing" shift ;; --stable=*) filter="${1#--stable=}" ;& --stable) mode="stable" shift ;; *) break ;; esac done # Set the working directory. # if [ $# -eq 0 ]; then dir="$owd" elif [ $# -eq 1 ]; then dir="${1%/}" # with trailing slash removed. else error "$usage" fi # If in one of the destination-management modes, set the source repository # name to that of the destination repository. # if [ "$mode" != "source" ]; then src_repo_name="$dst_repo_name" fi # The source and destination package repository directories. # # Note that, throughout this script, any path not explicitly prefixed with # "$src_dir/" or "$dst_dir/" is relative to the root of the source or # destination package repositories. # src_dir="$dir/$src_repo_name" dst_dir="$dir/$dst_repo_name" if [ ! -d "$src_dir" ]; then error "'$src_dir' does not exist or is not a directory" fi if [ ! -d "$dst_dir" ]; then error "'$dst_dir' does not exist or is not a directory" fi # Check that both git repositories are clean. # if [ -n "$(git -C $src_dir status --porcelain)" ]; then error "git repository in '$src_dir' is not clean" fi if [ -n "$(git -C $dst_dir status --porcelain)" ]; then error "git repository in '$dst_dir' is not clean" fi # Use run() to show the user that git is the source of the diagnostics. # "Already up to date", for example, is too vague. # if [ "$mode" == "source" ]; then run git -C "$src_dir" pull >&2 fi run git -C "$dst_dir" pull >&2 # Load the source and destination repositories' submit configurations (section # name/directory mappings and owners directory path). # # Each repository's settings are sourced into the temporary variables 'owners' # and 'sections' and copied from there to source- and destination-specific # variables. # # If in one of the destination-management modes, store only the directory for # the section being managed in src_sections, and only its counterpart in # dst_sections. Otherwise, in source-management mode, store all source and # destination section directories. # declare owners declare -A sections source "$src_dir/submit.config.bash" src_owners="$owners" declare -A src_sections for s in "${!sections[@]}"; do if [[ ("$mode" == "source") || ("$s" == "$mode") ]]; then src_sections["$s"]="${sections[$s]}" fi done owners= sections=() source "$dst_dir/submit.config.bash" # Section counterparts. # declare -A sect_cparts=(["testing"]="stable" ["stable"]="legacy") dst_owners="$owners" declare -A dst_sections for s in "${!sections[@]}"; do if [[ ("$mode" == "source") || ("$s" == "${sect_cparts[$mode]}") ]]; then dst_sections["$s"]="${sections[$s]}" fi done # Fail if in destination-management mode and the source and/or destination # section is not configured for the destination repository. (The equivalent # source-management mode checks can only be done later.) # if [[ ("$mode" != "source") && (! -v src_sections["$mode"] || ! -v dst_sections["${sect_cparts[$mode]}"]) ]]; then error "section '$mode' and/or '${sect_cparts[$mode]}' not configured \ in the destination repository" fi # Contains the hashes of the pending commits in chronological order. # pending_seq=() # Maps from the file path to the commit it belongs to. A file may have been # added and removed by earlier commits and could thus be migrated with the # wrong commit unless care is taken (see the example in the migration notes # below). Every file contained in this map exists on disk. # declare -A file_commits # Initialize global variables that depend on disk/repository state that is # modified by commit and file operations. # function init_globals () { pending_seq=() file_commits=() # Find all package archives and, if in source-management mode, owner # manifest files in the source repository. # # Every file in a repository section directory except *.manifest is a # package archive and every file in the owners directory is a project or # package owner manifest. Therefore run find separately on each section # directory and the owners directory to build a list containing only # package-related files. # # Store the relative to the repository directory file paths in an array, # `src_files`, used to build the set of pending commits. # # Note that directories in `src_sections` may be deleted by certain # operations (for example, if the last package is migrated out of a # section). # local src_files=() local s for s in "${src_sections[@]}"; do if [ -d "$src_dir/$s" ]; then while read f; do src_files+=("${f#$src_dir/}") done < <(find "$src_dir/$s" -type f -not -name "*.manifest") fi done local f if [[ ("$mode" == "source") && -n "$src_owners" && -d "$src_dir/$src_owners" ]]; then while read f; do src_files+=("${f#$src_dir/}") done < <(find "$src_dir/$src_owners" -type f) fi # Build the set of pending commit hashes ("pending set"). # # For each file in the source repository, find the commit it belongs to and # store its abbreviated hash (as key) inside the `pending_set` associative # array (note: unordered) and (as value) inside the `file_commits` # associative array. # # If in destination-management mode, exclude from `pending_set` those # commits without any package archives that match the pattern in `filter`. # local -A pending_set=() local h for f in "${src_files[@]}"; do # -1: limit output to one commit (that is, the most # recent) # --diff-filter=A: only show commits that added the specified file or # moved it into its current location ('R' is useless # if a path is specified) # --pretty=format:%h: output only the abbreviated commit hash # h="$(git -C "$src_dir" log -1 --diff-filter=A --pretty=format:%h -- "$f")" # Note that the hash cannot be empty because, after our clean checks at # the top, every file on disk must have been added by some commit (that # is, there can be no untracked files). # Add the commit to the pending set unless the current file is filtered # out. # # Note: `src_files` can contain only package archives if we're in # destination-management mode so there's no need to check the file type. # # Note: $filter must be unquoted to match as a wildcard pattern. # if [[ "$mode" == "source" || ("$(basename "$(dirname "$f")")" == $filter) || # Project name? ("$(basename "$f")" == $filter.*) ]]; then # Filename (with ext)? pending_set["$h"]=true fi file_commits["$f"]="$h" done # Arrange the pending commits in the chronological order. # # Go through the most recent commits in the git log which added or # moved/renamed one or more files, skipping those not present in the pending # set and keeping count to bail out as soon as we ordered all of them. # local i for (( i=0; i != "${#pending_set[@]}"; )); do read h # The abbreviated commit hash. # If this is a pending commit, prepend its hash to the ordered array. # if [ "${pending_set[$h]}" ]; then pending_seq=("$h" "${pending_seq[@]}") ((++i)) fi # --diff-filter=AR: only show commits that added or renamed files # done < <(git -C "$src_dir" log --diff-filter=AR --pretty=format:%h) } # Clean the source and destination repositories by discarding uncommitted # changes and removing unstaged files. (Note that the source repository cannot # have untracked files so we git-clean only the destination repository.) # function cleanup () { info "migration failed; resetting and cleaning repositories" if ([ "$mode" == "source" ] && ! run git -C "$src_dir" reset --hard) || ! run git -C "$dst_dir" reset --hard || ! run git -C "$dst_dir" clean --force; then info "failed to reset/clean repositories -- manual intervention required" fi } # Return the subject of a git commit in the source repository. # function commit_subject () # { local h="$1" git -C "$src_dir" log -n 1 --pretty=format:%s "$h" } # Return the list of files a commit added to or moved within the source # repository. # function commit_files () # { local h="$1" # git-diff-tree arguments: # # --diff-filter=A: select only files that were added. # -z: don't munge file paths and separate output fields with # NULs. # -r: recurse into subtrees (directories). # git -C "$src_dir" diff-tree \ --no-commit-id --name-only --diff-filter=A -z -r \ "$h" } # Print information about the path of a source repository file. # # The information includes its class (package archive, ownership manifest, or # unmanaged), the project directory, and in the case of package archives, the # section directory. # # must be relative to the source repository directory (`src_dir`). # # If the path refers to a managed archive file in the source repository, then # print `archive
`. # # Otherwise, if the path refers to a managed ownership manifest file in the # source repository, then print `ownership `. # # Otherwise the file is unmanaged; print `unmanaged`. # # Note that the function doesn't validate the file path exhaustively and may # classify improperly named file (invalid base name, etc) as an archive or # ownership. # function src_path_info () # { local f="$1" # Check whether the file is a package archive. Valid source repository # package paths start with one of the section directories in `src_sections` # which is followed by a single subdirectory (which would be the project # directory, but this is not checked). # local s for s in "${src_sections[@]}"; do if [[ "$f" =~ ^"$s"/([^/]+)/[^/]+$ ]]; then echo -n "archive ${BASH_REMATCH[1]} $s" return fi done # Not a managed archive path, so check whether it's an ownership # manifest. Valid source repository ownership manifest paths start with the # directory in `src_owners` and is followed by at least one subdirectory # (the project directory, again). # if [[ -n "$src_owners" && ("$f" =~ ^"$src_owners"/([^/]+)/.+$) ]]; then echo -n "ownership ${BASH_REMATCH[1]}" else echo -n "unmanaged" fi } # Extract the package name, version, and project from a package archive's # manifest and print it to stdout in the ' ' form. If # the manifest does not specify the project name, the package name is returned # as the project name. # function extract_pkg_info () # { local arc="$1" local r r=($(bpkg_util_pkg_verify_archive "$arc")) # if [ ! -v r[2] ]; then r[2]="${r[0]}" fi # Verify that the archive parent directory name matches the project. # local p="${r[2]}" if [ "$p" != "$(basename "$(dirname "$arc")")" ]; then error "'$arc' archive directory name does not match package project '$p'" fi echo -n "${r[@]}" } # Exit with an error if a package which is a duplicate of or is in conflict # with the given package exists in any of the destination sections. # # Two packages are duplicates if they have the same name and version and, as a # result, the same archive filename. If, on the other hand, they have # different names and/or versions but the same archive filename, they are in # conflict with one another. For example, foo-bar version 1.0 and foo version # bar-1.0 have the same archive name foo-bar-1.0.tar.gz. # function check_pkg_duplicate () # { local name="$1" local version="$2" local sd # Section directory. for sd in "${dst_sections[@]}"; do local p # Use -.* without .tar.gz in case we want to support more # archive types later. # IFS=$'\n' eval \ 'p=($(bpkg_util_pkg_find_archive "$name-$version.*" "$dst_dir/$sd"))' if [ "${#p[@]}" -ne 0 ]; then local a="${p[0]}" local n="${p[1]}" local v="${p[2]}" if [ "$n" == "$name" ]; then error "duplicate of $name/$version at '$a'" else error "conflict of $name/$version with $n/$v at '$a'" fi fi done } # Remove other versions and/or revisions of a package ("replacement # candidates") in the destination section if the user so chooses. Return the # removed packages' version numbers. # # The argument is the name of the source (and the destination) package. # # The argument must be either "*" to remove all other # versions (including revisions), or "*" to remove other revisions # only. # # The argument is the destination section name. # # The and arguments are the version and project of # the source package and are only used in the package removal confirmation # prompt. # # Search for replacement candidates according to # and remove or skip each candidate at the user's direction. Stage (but don't # commit) the removals in the destination repository. # # The versions of the removed packages are written to stdout, separated by # spaces. For example: # # 1.2.3 1.3.0+1 1.3.0+2 # # Note that, currently, versions/revisions which are both lower and higher # than will be considered for replacement. # function remove_pkg_archives () # { local name="$1" local vpat="$2" local dsect="$3" local sver="$4" local sproj="$5" local rv=() # Removed version numbers. # Search for replacement candidates. # local pkgs=() # Packages to be considered for replacement. IFS=$'\n' eval \ 'pkgs=($(bpkg_util_pkg_find_archives "$name" \ "$vpat" \ "$dst_dir/${dst_sections[$dsect]}"))' # For each replacement candidate, ask for confirmation and, depending on the # answer, either remove it from the destination repository or leave it in # place. # local f for f in "${pkgs[@]}"; do # Get the destination archive's info from its embedded manifest. # local p p=($(extract_pkg_info "$f")) local dver="${p[1]}" # Destination package version. local dproj="${p[2]}" # Destination package project. # Ask whether or not to drop the destination package. Include the project # names in the prompt if the destination package's project differs from # that of the source package (which is never the case in # destination-management mode). # local src="$sver" local dst="$name/$dver" if [ "$dproj" != "$sproj" ]; then src+=" ($sproj)" dst+=" ($dproj)" fi local opt while true; do read -p "replace $dst with $src in $dsect? [y/n]: " opt case "$opt" in y) run git -C "$dst_dir" rm --quiet "${f#$dst_dir/}" rv+=("$dver") break ;; n) break ;; esac done done echo -n "${rv[@]}" } # The commit bundle array is the set of selected pending commits. Its elements # are the corresponding indexes of the `pending_seq` array (but offset by +1). # The indexes are always kept in ascending and, therefore, chronological order # (because the commits in `pending_seq` are in chronological order). # bundle=() # Contains the managed file paths belonging to commits in the commit # bundle. Used as the starting point for operations such as the migration and # dropping of commits or files. # # These files must be grouped by commit and the commit groups ordered # chronologically, otherwise files could be processed in a different order # than they occur in the pending commit list (the `pending_seq` array). For # example, failing that, the commit groups/files in the commit splitting # screen or the file mentions in the commit messages might be in a different # order than they were in the main screen. # bundle_files=() # Collect the commit bundle's files into the global `bundle_files` array, # grouping them by commit. The commit groups are ordered chronologically to # match the ordering of the pending commit array, `pending_seq`. # # Include only files belonging to a commit in the bundle. # # Fail -- by leaving `bundle_files` empty -- if the commit bundle is empty or # any of the files are unmanaged. # function collect_bundle_files () { bundle_files=() local i for i in "${bundle[@]}"; do local h="${pending_seq[$i-1]}" # The current commit's abbreviated hash. local f while read -d '' f; do # Fail (by clearing the `bundle_files` array) if the current file is # unmanaged. # local fi=($(src_path_info "$f")) if [ "${fi[0]}" == "unmanaged" ]; then info "cannot include commit $i: '$f' is unmanaged" bundle_files=() return fi # Add this file only if it belongs to the current commit. # if [ "${file_commits[$f]}" == "$h" ]; then bundle_files+=("$f") fi done < <(commit_files "$h") done } # Print "true" to stdout if equals any of the subsequent # arguments/words. # function contains () # ... { local k="$1" shift local w for w in "$@"; do if [ "$w" == "$k" ]; then echo -n "true" return fi done } # Migrate (all management modes): # # Files belonging to one or more user-selected pending commits are moved from # the source to the destination, with the move of each file staged in the # source and/or destination repositories. The action taken on each file is # also recorded in the commit message(s). # # Note the following edge case which applies to all management modes: # # We will need to confirm with git that each of a commit's files were # actually most recently added or moved by that commit. For example (oldest # commits first): # # commit 1: add foo.tar.gz, bar.tar.gz # commit 2: del foo.tar.gz # commit 3: add foo.tar.gz # # If the user chooses to migrate commit 1, only bar.tar.gz must be migrated, # despite foo.tar.gz existing on disk. # Source-management mode migration: migrate the selected commit bundle from # the source repository to the destination repository. # # Takes as input the bundle files in the global `bundle_files` array which is # assumed to be non-empty and to contain only managed files (package archives # or ownership manifests) belonging to commits in the commit bundle. # # If the migration succeeds, set the global migrate_result variable to # true. Otherwise, in case of failure, issue appropriate diagnostics and set # migrate_result to the empty string before returning. In order to ensure that # both repositories are left clean if any of the files fail to be migrated, an # EXIT signal handler that discards all uncommitted changes is installed right # before the migration proper begins, and uninstalled when all files have been # migrated. # # The migration will fail if any of the following is true: # # - Files in the commit bundle are not all from the same project (the "bundle # project") or, in the case of archives, the same repository section (the # "bundle section"). # # - The required section does not exist in the destination repository. # # - There exists, for one or more files in the commit bundle, a duplicate or # conflicting archive in any of the sections in the destination repository # (see check_pkg_duplicate()). # # The migration process proceeds as follows: # # - Move files: all of the files in the selected commit bundle are moved from # the source repository into the destination repository: # # - Owners to owners directory. # # - Packages into corresponding sections: # # alpha -> alpha # beta -> beta # stable -> testing|stable # # Package archives may be removed and ownership manifests overwritten at the # destination. Candidate files for replacement are selected as follows: # # - In the alpha and beta sections, any package archive files in the # destination section directory belonging to the same package are # considered for replacement, regardless of their versions. # # - In other sections, any package archives in the destination section # directory with the same name and version but a different revision # (currently whether lower or higher) are replaced. # # - Project and package ownership manifests will be replaced (that is, # simply overwritten) at the destination with any ownership manifests # added by the commit bundle because their presence implies that ownership # information has changed. # # Stage (but don't commit) the removal of the files from the source # repository and their addition to the destination repository. # # - Make commits to the source and destination respositories with appropriate # commit messages: # # Source repository: # # "Migrate from $src_sect to $dst_repo_name/
" # # "remove /" # "remove owners//project-owner.manifest" # "remove owners///package-owner.manifest" # # Destination repository: # # "Migrate from $src_repo_name/
to $dst_sect" # # "add /" # "replace / with " (if replacing) # "add owners//project-owner.manifest" # "add owners///package-owner.manifest" # function migrate_src () { migrate_result= # Check that all files in the bundle are in the bundle section and/or bundle # project before migrating any of them. Classify each file as a package # archive or ownership manifest as we go along, based on its path. # # The bundle section is derived from the first package archive encountered # and the bundle project from the first package archive or owner manifest # encountered. # local src_sect= # Source section name. local src_sect_dir= # Source section directory. local proj= # The bundle (source) project. local pkgs=() # The bundle's archive files. local owns=() # The bundle's ownership manifests. local f for f in "${bundle_files[@]}"; do local fi=($(src_path_info "$f")) local ftype="${fi[0]}" # Current file's type. local fproj="${fi[1]}" # Current file's project. if [ "$ftype" == "ownership" ]; then owns+=("$f") elif [ "$ftype" == "archive" ]; then local fsect_dir="${fi[2]}" pkgs+=("$f") # Find, in `src_sections`, the archive section name associated with the # section directory extracted from the path (a value-to-key lookup). # local fsect= # The "*" key is a catch-all for unknown submitted section names and, if # present, will share a value (section directory) with one of the known # section names and therefore must be skipped. # local k for k in "${!src_sections[@]}"; do if [[ ("${src_sections[$k]%/}" == "$fsect_dir") && ("$k" != "*") ]]; then fsect="$k" # Current file's section name. break fi done if [ -z "$fsect" ]; then # The only way that fsect can be empty is due to a programming error # or if the "*" key has a unique value -- which would be a submit # config error. So it would probably be better to terminate the # script. # error "unable to find section name for file '$f'" fi # Set the source section name and directory if unset; otherwise fail if # the current file is not from the source section. # if [ -z "$src_sect" ]; then src_sect="$fsect" src_sect_dir="$fsect_dir" elif [ "$fsect" != "$src_sect" ]; then info "'$f' is not in section $src_sect" return fi fi # Set the bundle project if unset; otherwise fail if the current file is # not from the bundle project. # # Note: $fproj cannot be empty here (see above). # if [ -z "$proj" ]; then proj="$fproj" elif [ "$fproj" != "$proj" ]; then info "'$f' is not in project $proj" return fi done # Finalize migration variables the values of which depend on whether the # bundle contains at least one package archive or ownership manifests only. # # The source and destination commit messages are composed incrementally as # the migration process proceeds. # local dst_sect # Destination section name. local dst_sect_dir # Destination section directory. local src_cmsg # Source commit message. local dst_cmsg # Destination commit message. if [ "${#pkgs[@]}" -ne 0 ]; then # Bundle contains package archive(s). dst_sect="$src_sect" # If it exists, 'testing' overrides 'stable' at the destination. # if [[ ("$dst_sect" == "stable") && -v dst_sections["testing"] ]]; then dst_sect="testing" fi # Fail if the target section does not exist in the destination repository. # if [ ! -v dst_sections["$dst_sect"] ]; then info "section '$dst_sect' does not exist in the destination repository" return fi dst_sect_dir="${dst_sections[$dst_sect]}" src_cmsg="Migrate $proj from $src_sect to $dst_repo_name/$dst_sect"$'\n\n' dst_cmsg="Migrate $proj from $src_repo_name/$src_sect to $dst_sect"$'\n\n' else # Bundle consists only of ownership manifests. # The setup where the ownership authentication is disabled on the # destination but enabled on source is probably obscure, but let's # consider it possible since the submit-git handler allows such a setup. # if [ -n "$dst_owners" ]; then src_cmsg="Migrate $proj ownership info to $dst_repo_name"$'\n\n' dst_cmsg="Migrate $proj ownership info from $src_repo_name"$'\n\n' else src_cmsg="Remove $proj ownership info"$'\n\n' dst_cmsg= # Nothing to commit. fi fi # Migrate the bundle's package archive files. # # Ensure that the source and destination repositories are clean if the # migration of any file fails. # trap cleanup EXIT # Migrate a package archive or ownership manifest file from the source # repository to the destination repository. # # is the path of the source file, relative to the source repository # directory, and is the path of the destination directory, relative to # the destination repository directory. # # Note that the source and destination sections and owners directories may # differ (as they do in these examples) which is why those components must # be specified in both the source and destination paths. # # Move the file from the source repository directory to the destination # repository directory, creating directories if required; stage the addition # of the file to the destination repository; stage the removal of the file # from the source repository. # function migrate_file () # { local src="$1" local dst="$2" mkdir -p "$dst_dir/$dst" mv "$src_dir/$src" "$dst_dir/$dst/" run git -C "$src_dir" rm --quiet "$src" run git -C "$dst_dir" add "$dst/$(basename "$src")" } for f in "${pkgs[@]}"; do # Get the current package's name and version from its embedded manifest # (we already have the source project in $proj). # local p p=($(extract_pkg_info "$src_dir/$f")) local name="${p[0]}" local version="${p[1]}" check_pkg_duplicate "$name" "$version" # In the destination repository, find and remove package archive files # which are other alpha/beta versions or revisions of the current source # package. # local vpat # Version pattern for replacement. case "$dst_sect" in alpha|beta) vpat="*" ;; # All package versions. *) vpat="$version*" ;; # All package version revisions. esac local rv # Removed version numbers. rv=($(remove_pkg_archives "$name" "$vpat" \ "$dst_sect" \ "$version" "$proj")) # Update the commit messages and migrate the current package. # src_cmsg+=" remove $name/$version"$'\n' if [ "${#rv[@]}" -eq 0 ]; then dst_cmsg+=" add $name/$version"$'\n' else for ((i=0; i != "${#rv[@]}"; ++i)); do dst_cmsg+=" replace $name/${rv[$i]} with $version"$'\n' done fi migrate_file "$f" "$dst_sect_dir/$proj" done # Migrate the bundle's ownership manifests. # # If ownership authentication is disabled on the destination repository, # only remove ownership manifests from the source repository (that is, don't # migrate). # for f in "${owns[@]}"; do src_cmsg+=" remove $f"$'\n' if [ -n "$dst_owners" ]; then local dp=$(dirname "${f/$src_owners/$dst_owners}") # Destination path. # Let the commit message reflect whether this is a new ownership # manifest or is replacing an existent one. # local fn=$(basename "$f") if [ ! -e "$dst_dir/$dp/$fn" ]; then dst_cmsg+=" add $dp/$fn"$'\n' else dst_cmsg+=" update $dp/$fn"$'\n' fi migrate_file "$f" "$dp" else run git -C "$src_dir" rm --quiet "$f" fi done # Commit the changes made to the source and destination repositories. # info run git -C "$src_dir" commit -m "$src_cmsg" if [ -n "$dst_cmsg" ]; then info run git -C "$dst_dir" commit -m "$dst_cmsg" fi info # All files have been migrated successfully so set the result and clear the # EXIT trap. # migrate_result=true trap EXIT read -p "press Enter to continue" } # Destination-management mode migration: migrate the package archives in the # selected commit bundle from the source (managed) section to its counterpart # section. # # Takes as input the bundle files in the global `bundle_files` array which is # assumed to be non-empty and to contain only managed package archives # belonging to commits in the commit bundle (that is, it contains no ownership # manifests (see below)). # # Assumes that the source and destination sections are valid. # # The general structure of this function is very similar to that of # migrate_src() but most of the logic is simpler. Some noteworthy differences # include: # # - Only the destination repository is involved. # # - All managed files in the commit bundle are package archives (ownership # manifests stay where they are and are therefore skipped/ignored). # # - All managed package archives in the commit bundle are known to be in the # same section (because all files outside of the managed section are # ignored). # # The mechanism by which success or failure is indicated is the same as for # migrate_src(). # # The migration will fail if any of the following is true: # # - Package archives in the commit bundle are not all from the same project # (the "bundle project"). # # - There exists, for one or more package archives in the commit bundle, a # duplicate or conflicting archive in the destination section (see # check_pkg_duplicate()). # # The migration process proceeds as follows: # # - Move files: all of the package archives in the selected commit bundle are # moved from the source section to the destination section. # # Any package archives in the destination section directory with the same # name and version but a different revision (currently whether lower or # higher) will be replaced if the user so chooses. # # Stage (but don't commit) each file move as the migration proceeds. # # - Make a commit to the destination repository with an appropriate commit # message: # # "Migrate from $src_sect to $dst_sect" # # "move /" # "replace / with " (if replacing) # function migrate_dst () { migrate_result= local src_sect="$mode" # Source section. local src_sect_dir="${src_sections[$src_sect]}" # Source section directory. local dst_sect="${sect_cparts[$src_sect]}" # Destination section. local dst_sect_dir="${dst_sections[$dst_sect]}" # Dst. section directory. # Check that every package archive is from the bundle project (taken from # the path of the first file encountered). # local proj= # The bundle project. local f for f in "${bundle_files[@]}"; do # Note: contains only package archives. local fi=($(src_path_info "$f")) local fproj="${fi[1]}" # Current file's project. # Set the bundle project if unset; otherwise fail if the current file is # not from the bundle project. # if [ -z "$proj" ]; then proj="$fproj" elif [ "$fproj" != "$proj" ]; then info "'$f' is not in project $proj" return fi done # Migrate the bundle's files. # # Ensure that the source and destination repositories are clean if the # migration of any package archive fails. # trap cleanup EXIT local cmsg= # The detailed part of the commit message. for f in "${bundle_files[@]}"; do # Get the current package's name and version from its embedded manifest # (we already have the source project in $proj). # local p p=($(extract_pkg_info "$src_dir/$f")) local name="${p[0]}" local version="${p[1]}" # Note that only the destination section is checked because that is the # only one loaded into dst_sections when in destination-management mode. # check_pkg_duplicate "$name" "$version" # Find and remove other revisions of the current package. # local rv # Removed version numbers. rv=($(remove_pkg_archives "$name" "$version*" \ "$dst_sect" \ "$version" "$proj")) # Update the commit message. # if [ "${#rv[@]}" -eq 0 ]; then cmsg+=" move $name/$version"$'\n' else for ((i=0; i != "${#rv[@]}"; ++i)); do cmsg+=" replace $name/${rv[$i]} with $version"$'\n' done fi # Migrate the current package. # mkdir -p "$dst_dir/$dst_sect_dir/$proj" run git -C "$dst_dir" mv "$f" "$dst_sect_dir/$proj/" done # Remove the project directory from the source section if it is empty. # # (Unlike git-mv, git-rm automatically removes the directory when its last # file is removed, so this does not need to be done in migrate_src().) # local d="$dst_dir/$src_sect_dir/$proj/" if [ -z "$(ls -A "$d")" ]; then rmdir "$d" fi # Commit the staged changes. # info run git -C "$dst_dir" commit \ -m "Migrate $proj from $src_sect to $dst_sect"$'\n\n'"$cmsg" info # All files have been migrated successfully so set the result and clear the # EXIT trap. # migrate_result=true trap EXIT read -p "press Enter to continue" } # Push local changes to the remote source and/or destination git repositories. # # Push to the destination repository first because thus the migrated files # will be in both remote repositories until the completion of the subsequent # push to the source repository (which may fail or take long). Although this # is an inconsistent state, it is safe because other programs such as a # submission handler will be able to detect the duplicates and therefore # refuse to do anything. If, on the other hand, we pushed to the source first, # the migrated files would not exist in either remote repository until the # push to the destination repository completed. In this state the submission # handler would, for example, accept a resubmission of the migrated packages or # erroneously establish ownership for already owned project/package names. # function push () { # Let's print additional diagnostics on git-push failure, to emphasize for # the user which of the two repositories we have failed to push. # if ! run git -C "$dst_dir" push; then error "push to $dst_repo_name failed" fi if [ "$mode" == "source" ] && ! run git -C "$src_dir" push; then error "push to $src_repo_name failed" fi } # User interface: # # The main screen presents the list of pending commits to the user, oldest # first, with files that were deleted by subsequent commits marked with `*` # and, of those, the ones that were then added back again with `!`. Files from # unmanaged directories are marked with `?`. # # 1 (deadbeef) Add libfoo/1.2.3 # # 1/testing/foo/libfoo-1.2.3.tar.gz # owners/foo/project-owner.manifest # owners/foo/libfoo/package-owner.manifest # # 2 (c00l0fff) Add bar/1.2.3 # # * 1/testing/bar/libbar-1.2.3.tar.gz # 1/testing/bar/libbaz-1.2.3.tar.gz # # 3 (deadbabe) Add libbar/1.2.3+1 # # 1/testing/bar/libbar-1.2.3+1.tar.gz # # Note that files deleted by subsequent commits may still be in the # repository. See the example in the above migration notes. # # Then prompt the user for the action (showing the current bundle): # # [1 2][,m,c,p,q,l,?]: # # - add commit to the commit bundle # m - migrate the selected commit bundle # c - clear the selected commit bundle # s - split commit(s) (operate on selected files) # p - push source and destination repositories # l - list pending commits # q - quit (prompting to push if commits were made) # ? - print this help # # The main screen operates at "commit granularity", in which each operation is # performed on entire commits (that is, on all of the files they own). At # "file granularity", on the other hand, operations are performed only on a # user-selected subset of the selected commits' files. To operate at file # granularity the user chooses, in the main menu, to split the selected # commits, thus switching to the "commit splitting screen". See # `split_commits()`, below. # # Print the size of a file to stdout in the human-readable form (K, M, G). # function file_size () { local f="$1" # POSIX specifies the output of `ls -l` so this should be portable. -h turns # on human-readable output and is present on Linux, FreeBSD, and Mac OS. # echo -n "$(ls -lh "$src_dir/$f" | cut -d ' ' -f 5)" } # Show the commit splitting screen: let the user select and operate on # specific files in the commit bundle (file granularity). # # Remove all but the selected files from `bundle_files` -- the global array # used as input to the commit and file operations -- before performing an # operation, and populate it with the unselected files afterwards. # # Assumes that the files in `bundle_files` are grouped by commit and that they # are all existent (that is, present in `file_commits`) and managed. # # This screen's layout is near identical to that of the main screen -- each # commit's files, grouped under the commit heading, with commit groups in # chronological order -- except that here the files are numbered instead of # the commits. The user is presented with a menu similar to the one in the # main screen including the selection of files by number and operations such # as `migrate` and `drop`. # # @@ Feature idea: invert selection. That is, if user wants to exclude rather # than include a file, can add it to the selection and then invert the # selection. # # Also: add 'push' and 'quit script' ('Q') options? Returning to the main # menu is a little annoying, but perhaps that's just because I've been # running this script so frequently. # function split_commits () { # The current file selection. Stores indexes into `bundle_files`. # local fsel=() while true; do if [ "${#bundle_files[@]}" -ne 0 ]; then # Print the commits followed by their files. Because `bundle_files` is # grouped by commit we know we are on a new commit when the current # file's commit hash differs from that of its predecessor. # local h= # Hash of commit to which the current subset of files belong. local i for i in "${!bundle_files[@]}"; do local f="${bundle_files[$i]}" # Current file. local fh="${file_commits[$f]}" # Current file's commit hash. if [ "$fh" != "$h" ]; then # New commit so first print its heading. h="$fh" printf "\n%s %s\n\n" "$fh" "$(commit_subject "$fh")" >&2 fi # Get this file's size if it's a package archive. # local sz= # File size. local fi=($(src_path_info "$f")) if [ "${fi[0]}" == "archive" ]; then sz=" $(file_size "$f")" fi # Print the current file's number (left-padded to 3 columns), path, # and size (if set). # printf "%3d %s%s\n" "$((i+1))" "$f" "$sz" >&2 done else info "no files in commit bundle" fi # Used to restore the unactioned files into `bundle_files` after an # operation (see update_bundle_files(), below). We need to do it because # our migrate functions operate on (global) bundle_files. # local unactioned_files=() # Collect the selected files into `bundle_files` just before an # operation. But first save its contents in `unactioned_files`. # # Note that after this function returns the indexes in `fsel` are valid # for `unactioned_files` but not `bundle_files`. # function collect_selected_files () { unactioned_files=("${bundle_files[@]}") local r=() local i for i in "${fsel[@]}"; do r+=("${bundle_files[$i-1]}") done bundle_files=("${r[@]}") } # Put the unactioned bundle files into `bundle_files` after an # operation. If the operation succeeded the unactioned files are the # unselected ones and if it failed it's the union of the selected and # unselected files. # function update_bundle_files () # { local opres="$1" if [ "$opres" ]; then # Remove the selected/actioned files from `unactioned_files`. # local i for i in "${fsel[@]}"; do unset unactioned_files[$i-1] done fi bundle_files=("${unactioned_files[@]}") } # Display the menu/prompt, build the file selection, and perform the # requested actions. Breaking out of this loop prints the bundle's files # again. # while true; do # Print the file selection and menu/prompt. Sort the selected files into # ascending order (for display and to maintain the ordering of # `bundle_files`, which `fsel` is the source for). # fsel=($(sed 's/ /\n/g' <<<"${fsel[*]}" | sort -)) local opt read -p $'\n'"[${fsel[*]}][,m,c,l,q,?]: " opt # Perform the selected action. # case "$opt" in # Add file to the selection. # [0-9]*) if [[ ("$opt" =~ ^[1-9][0-9]*$) && -v bundle_files[$opt-1] ]]; then if [ ! "$(contains "$opt" "${fsel[@]}")" ]; then fsel+=("$opt") info "file $opt added to selection" else info "file $opt already selected" fi else info "non-existent file number $opt" fi ;; # Migrate selected files. # m) if [ "${#fsel[@]}" -ne 0 ]; then collect_selected_files if [ "$mode" == "source" ]; then migrate_src else migrate_dst fi update_bundle_files "$migrate_result" if [ "$migrate_result" ]; then fsel=() need_push=true break fi else info "no files selected" fi ;; # Drop selected files. # d) # if [ "${#fsel[@]}" -ne 0 ]; then # collect_selected_files # drop # update_bundle_files "$migrate_result" # ... # else # info "no files selected" # fi # ;; # Clear the file selection (and print the file list again). # c) fsel=() break ;; # Print the file list. # l) break ;; # Quit (returning to the main screen). # q) return ;; # ? or invalid option: print menu. # *) cat <&2 - add file to selection m - migrate the selected files c - clear the file selection l - list files q - quit (back to main menu) ? - print this help EOF ;; esac done done } # True if any changes have been made to the source and/or destination git # repositories (in which case the user will be asked whether or not to push # before quitting). # need_push= init=true # True if the global state needs to be re-initialized. # The main screen's user interaction loop. # # In each iteration, present the list of pending commits, display the menu of # actions, read the user's input, and perform the chosen action. # while true; do # (Re)initialize the global state if necessary. The idea is that instead of # performing complex house-keeping, after certain operations (e.g., # migration) we will just re-initialize the state from scratch. # if [ "$init" ]; then init_globals if [ "${#pending_seq[@]}" -eq 0 ]; then info "good news, nothing to manage" if [ ! "$need_push" ]; then exit 0 fi fi init= bundle=() fi for i in "${!pending_seq[@]}"; do h="${pending_seq[$i]}" # Print commit number, hash, and subject. # # Prefix with a newline to separate the first commit from the git-pull # output and the rest from the previous commit info block. # printf "\n%d (%s) %s\n\n" "$((i+1))" "$h" "$(commit_subject "$h")" >&2 # Print this commit's files. # # Fetch from the git repository the current commit's files and print each # one's path. # # Mark files that cannot be migrated: # # - Unmanaged files are marked with '?' and will prevent the migration of # the commit. # # - Files associated with a different commit hash in 'file_commits' were # deleted and added back by subsequent commits. These files are marked # with '!' and will not prevent the migration of the commit. # # - Files with no association in 'file_commits' were deleted by a # subsequent commit and never added back. These files are marked with # '*' and will not prevent the migration of the commit. # while read -d '' f; do fi=($(src_path_info "$f")) ftype="${fi[0]}" if [ "$ftype" == "unmanaged" ]; then # File is unmanaged (and may or may not exist). # info "? $f" else # File is managed. # If this file is a package archive which exists (that is, it's in # `file_commits`), get its size. # sz= # File size. if [[ ("$ftype" == "archive") && -v file_commits["$f"] ]]; then sz="$(file_size "$f")" fi # Note that, in destination-management mode, there can be no ownership # manifests in `file_commits`. # if [ "${file_commits[$f]}" == "$h" ]; then info " $f $sz" # Last added or moved by the current commit. elif [ -v file_commits["$f"] ]; then info "! $f $sz" # Deleted and added back by subsequent commits. elif [[ ("$mode" == "source") || ("$ftype" != "ownership") ]]; then # File was deleted and never added again and, if we're in # destination-management mode, is not an ownership manifest. # # Note that actioned files of partially actioned commits will also # appear this way. # info "* $f" fi fi done < <(commit_files "$h") done # The main screen's prompt loop. # # Prompt the user for the action (showing the current bundle), get user # input, and perform the selected action. # # Note that we could adapt the menu according to the current state (don't # offer to migrate if the bundle array is empty, etc) but let's not # complicate the logic. # # Breaking out of this loop re-initializes the global state if `init` is # true and reprints the pending commit list. # while true; do # Sort commit bundle in ascending order. # # Expand the 'bundle' associative array's keys into a single word in which # they are separated by spaces (the first member of IFS) using the # ${!a[*]} syntax; replace each space with a newline before piping to # 'sort', which is newline-based; finally collect sort's output into an # array using the a=() syntax, which splits on newline (the last member of # IFS) because neither space nor tab characters (the other members of IFS) # can occur in the keys. # bundle=($(sed 's/ /\n/g' <<<"${bundle[*]}" | sort -)) printf "\n" read -p "[${bundle[*]}][,m,c,s,p,l,q,?]: " opt case "$opt" in # Add commit to bundle. # [0-9]*) if [[ ("$opt" =~ ^[1-9][0-9]*$) && -v pending_seq[$opt-1] ]]; then if [ ! "$(contains "$opt" "${bundle[@]}")" ]; then bundle+=("$opt") info "commit $opt (${pending_seq[$opt-1]}) \ added to selected bundle" else info "commit $opt is already in the bundle" fi else info "non-existent commit number $opt" fi ;; # Migrate the commit bundle. # m) if [ "${#bundle[@]}" -ne 0 ]; then collect_bundle_files # Prints error if `bundle_files` left empty. if [ "${#bundle_files[@]}" -ne 0 ]; then if [ "$mode" == "source" ]; then migrate_src else migrate_dst fi if [ "$migrate_result" ]; then need_push=true init=true break fi fi else info "no commits selected" fi ;; # Show commit splitting screen (operate on a selection of the commit # bundle's files). # s) if [ "${#bundle[@]}" -ne 0 ]; then collect_bundle_files # Prints error if `bundle_files` left empty. if [ "${#bundle_files[@]}" -ne 0 ]; then split_commits # Note that the global state is re-initialized even if nothing was # done in the commit-splitting screen but these should be the # minority of cases. # init=true break fi else info "no commits selected" fi ;; # Clear the commit bundle. # c) bundle=() break ;; # Push changes. # p) push need_push= break ;; # Redraw the pending commit list. # l) break ;; # Quit. # q) if [ ! "$need_push" ]; then exit 0 fi while true; do read -p "push changes? [y/n/(c)ancel]: " opt case "$opt" in c) break # Print options menu again. ;; y) push exit 0 ;; n) exit 0 ;; *) continue ;; esac done ;; # ? or invalid option: print menu. # *) cat < - add commit to the commit bundle m - migrate the selected commit bundle c - clear the selected commit bundle s - split commit(s) (operate on selected files) p - push source and destination repositories l - list pending commits q - quit (prompting to push if commits were made) ? - print this help EOF ;; esac done done