#!/usr/bin/env bash # @@ TODO: Use `` instead of '' for single quotes in comments. # # file : bpkg-util/manage.in # license : MIT; see accompanying LICENSE file # Interactively migrate packages and/or ownership manifests from a source git # repository to a destination git repository or drop them from the source # repository. # # The commit that most recently added a file or moved it to its current # location "owns" the file and the file "belongs to" the commit. # # Present the user with the list of commits that own the files currently in # the source repository's working directory and ask the user to select from a # menu the action to perform on a selection of these commits. # # As these commits' files are pending removal or a move to the destination # repository they will be referred to as "pending" commits. # # Actions that can be performed on a selection of pending commits ("commit # operations") include moving them from the source repository to a single # commit in the destination repository and dropping them from the source # repository. A file that has been operated upon is called an "actioned file" # and a commit whose files have all been actioned an "actioned commit". It is # also possible to operate on only a subset of each selected commit's files # instead of all of them. These are called "file operations" and are selected # from the "commit splitting screen"; see below for more information. # # The flow of this script, in broad strokes, is as follows: for each file in # the source repository directory, find the hash of the commit it belongs to; # these are the pending commits. Arrange the pending commits in chronological # order. Display to the user the pending commits along with the files they # own. Let the user select one or more pending commits and an operation to be # performed on them. Each successful operation results in a commit to the # source and/or destination repositories and leaves both repositories in a # clean state. Once the operation has been performed, re-initialize the # pending commit list from the files in the source repository (that is, from # scratch), redisplay the updated pending commit list, and prompt for the next # action. Pushing to the remote repositories, a supported operation, can be # done at any time during the session. (Note that only in-memory structures # are updated after a file operation in the commit splitting screen -- the # pending commit list is only re-initialized (from disk) when the user returns # to the main screen.) # # Two modes of operation are supported. In source-management mode, package # archives -- from any section -- and/or ownership manifests are dropped from # the source repository or migrated to the destination repository. In # destination-management mode, starting from one of the sections in the # destination repository, package archives or ownership manifests are dropped, # or package archives (not ownership manifests) migrated to its counterpart # section (also in the destination repository). These mode-specific source # directories are called "managed directories" and the files they contain # "managed files". The "unmanaged files" include any file outside of a managed # directory, any ownership manifest not related (by package name or project # name) to a managed package archive, and any file inside a managed directory # with an invalid path (project component is missed, etc). # # The destination-management mode is actually just a name used to refer to a # number of more specific, near-identical modes distinguished only by the # source section being managed (and, by extension, the destination # section). For example, in testing-management mode packages are migrated from # the 'testing' section to its 'stable' counterpart and, in stable-management # mode, packages are migrated from the 'stable' section to its 'legacy' # counterpart. # # Note that in destination-management mode, the source and destination # repositories both refer to the "real" destination repository and, therefore, # that the distinction made between the two modes throughout this script is # fairly shallow. # # Options: # # -t # --testing[=] # # Enter the testing-management mode: manage the testing->stable transitions # in, or drop commits and/or files from, the destination repository. # # --stable[=] # # Enter the stable-management mode: manage the stable->legacy transitions in # the destination repository. Note that commits/files cannot be dropped in # this mode and, by extension, it does not support any operations on # ownership manifests. # # --alpha[=] # # Enter the alpha-management mode: drop commits and/or files from the alpha # section in the destination repository. # # --beta[=] # # Enter the beta-management mode: drop commits and/or files from the beta # section in the destination repository. # # If none of the above modes are specified, operate in the source-management # mode. # # If is specified, then it is used to narrow down the list of commits # to only those that contain packages with project name or package name and # version matching the filter as a wildcard pattern. For example: # # --stable=expat # --stable=libexpat-1.2.3 # --stable=libexpat-2.* # # Arguments: # # The directory into which the source and destination repositories have # been checked out. If not specified, current directory is assumed. # usage="usage: $0 [] []" # The names of the source and destination repositories inside . Note: # also used in commit messages. # src_repo_name=queue dst_repo_name=public owd="$(pwd)" trap "{ cd '$owd'; exit 1; }" ERR set -o errtrace # Trap in functions and subshells. set -o pipefail # Fail if any pipeline command fails. shopt -s lastpipe # Execute last pipeline command in the current shell. shopt -s nullglob # Expand no-match globs to nothing rather than themselves. @import bpkg-util/utility@ # Use the bpkg program from the script directory, if present. Otherwise, use # just 'bpkg'. # bpkg_util_bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" if [[ ! -x "$bpkg_util_bpkg" ]]; then bpkg_util_bpkg=bpkg fi @import bpkg-util/package-archive@ # The mode of operation. If its value is "source", manage the source # repository. Otherwise manage the destination repository with the value being # the name of the source section to manage. # mode="source" # Archive-filtering pattern used in destination-management mode. Match # everything by default. # filter="*" while [[ "$#" -gt 0 ]]; do case "$1" in --testing=*) filter="${1#--testing=}" ;& -t|--testing) mode="testing" shift ;; --stable=*) filter="${1#--stable=}" ;& --stable) mode="stable" shift ;; --alpha=*) filter="${1#--alpha=}" ;& --alpha) mode="alpha" shift ;; --beta=*) filter="${1#--beta=}" ;& --beta) mode="beta" shift ;; # Catch invalid options. # -*) error "unknown option: $1" ;; # or end of options. # *) break ;; esac done # Set the working directory. # if [[ "$#" -eq 0 ]]; then dir="$owd" elif [[ "$#" -eq 1 ]]; then dir="${1%/}" # with trailing slash removed. else error "$usage" fi # If in one of the destination-management modes, set the source repository # name to that of the destination repository. But first save the real source # repository directory because we sometimes need to operate on it even if in # destination-management mode. # real_src_dir="$dir/$src_repo_name" # The real source repository directory. if [[ "$mode" != "source" ]]; then src_repo_name="$dst_repo_name" fi # The source and destination package repository directories. # # Note that, throughout this script, any path not explicitly prefixed with # "$src_dir/", "$dst_dir/", or "$real_src_dir/" is relative to the root of the # source or destination package repositories. # src_dir="$dir/$src_repo_name" dst_dir="$dir/$dst_repo_name" if [[ ! -d "$src_dir" ]]; then error "'$src_dir' does not exist or is not a directory" fi if [[ ! -d "$dst_dir" ]]; then error "'$dst_dir' does not exist or is not a directory" fi # Check that both git repositories are clean. # if [[ -n "$(git -C $src_dir status --porcelain)" ]]; then error "git repository in '$src_dir' is not clean" fi if [[ -n "$(git -C $dst_dir status --porcelain)" ]]; then error "git repository in '$dst_dir' is not clean" fi # Use run() to show the user that git is the source of the diagnostics. # "Already up to date", for example, is too vague. # # Always pull the real source repository because we sometimes need to operate # on it even if in destination-management mode. # run git -C "$real_src_dir" pull >&2 run git -C "$dst_dir" pull >&2 # Load the source and destination repositories' submit configurations (section # name/directory mappings and owners directory path). # # Each repository's settings are sourced into the temporary variables 'owners' # and 'sections' and copied from there to source- and destination-specific # variables. # # If in one of the destination-management modes, store only the directory for # the section being managed in src_sections, and only its counterpart in # dst_sections. Otherwise, in source-management mode, store all source and # destination section directories. # # Normalize section directory paths by removing trailing slashes if they # exist. # declare owners declare -A sections source "$src_dir/submit.config.bash" src_owners="$owners" declare -A src_sections for s in "${!sections[@]}"; do if [[ ("$mode" == "source") || ("$s" == "$mode") ]]; then src_sections["$s"]="${sections[$s]%/}" # Remove trailing '/', if any. fi done owners= sections=() source "$dst_dir/submit.config.bash" # Section counterparts. # # Used only in destination-management mode. If a section has no counterpart # then migration from it is disabled. # declare -A sect_cparts=(["testing"]="stable" ["stable"]="legacy") dst_owners="$owners" declare -A dst_sections for s in "${!sections[@]}"; do if [[ ("$mode" == "source") || ("$s" == "${sect_cparts[$mode]}") ]]; then dst_sections["$s"]="${sections[$s]%/}" # Remove trailing '/', if any. fi done # Fail if in destination-management mode and the source section is not # configured for the destination repository. (The equivalent source-management # mode checks can only be done later.) # # Note that checking whether the non-existence of a counterpart section is an # error can only be done later because some operations support sections # without a counterpart (for example, dropping from the alpha or beta sections # in destination-management mode). # if [[ ("$mode" != "source") && ! -v "src_sections[$mode]" ]]; then error "section '$mode' not configured in the destination repository" fi # Print information about the path of a source repository file. # # The information includes its class (package archive, ownership manifest, or # unmanaged), the project directory, and in the case of package archives, the # section directory. # # must be relative to the source repository directory (`src_dir`). # # If the path refers to a managed archive file in the source repository, then # print `archive\n\n\n`. The section directory will be # relative to the source repository directory. # # Otherwise, if the path refers to a managed ownership manifest file in the # source repository, then print `ownership\n\n`. # # Otherwise the file is unmanaged; print `unmanaged\n`. # # Note that the function doesn't validate the file path exhaustively and may # classify improperly named file (invalid base name, etc) as an archive or # ownership. # function src_path_info () # { local f="$1" # Check whether the file is a package archive. Valid source repository # package paths start with one of the section directories in `src_sections` # which is followed by a single subdirectory (which would be the project # directory, but this is not checked). # local s for s in "${src_sections[@]}"; do if [[ "$f" =~ ^"$s"/([^/]+)/[^/]+$ ]]; then echo "archive" echo "${BASH_REMATCH[1]}" echo "$s" return fi done # Not a managed archive path, so check whether it's an ownership # manifest. Valid source repository ownership manifest paths start with the # directory in `src_owners` and is followed by at least one subdirectory # (the project directory, again). # if [[ -n "$src_owners" && ("$f" =~ ^"$src_owners"/([^/]+)/.+$) ]]; then echo "ownership" echo "${BASH_REMATCH[1]}" else echo "unmanaged" fi } # Extract the package name, version, and project from a package archive's # manifest and print it to stdout in the `\n\n\n` # form. If the manifest does not specify the project name, the package name is # returned as the project name. # function extract_pkg_info () # { local arc="$1" local r # ( ) bpkg_util_pkg_verify_archive "$arc" | readarray -t r if [[ -z "${r[2]}" ]]; then r[2]="${r[0]}" fi # Verify that the archive parent directory name matches the project. # local p="${r[2]}" if [[ "$p" != "$(basename "$(dirname "$arc")")" ]]; then error "'$arc' archive directory name does not match package project '$p'" fi local e for e in "${r[@]}"; do echo "$e" done } # Contains the hashes of the pending commits in chronological order. # pending_seq=() # Maps from the file path to the commit it belongs to. A file may have been # added and removed by earlier commits and could thus be migrated with the # wrong commit unless care is taken (see the example in the migration notes # below). Every file contained in this map exists on disk. # declare -A file_commits # Initialize global variables that depend on disk/repository state that is # modified by commit and file operations. # function init_globals () { pending_seq=() file_commits=() # Find all package archives and owner manifest files in the source # repository. # # Every file in a repository section directory except *.manifest is a # package archive and every file in the owners directory is a project or # package owner manifest. Therefore run find separately on each section # directory and the owners directory to build a list containing only # package-related files. # # Store the relative to the repository directory file paths in an array, # `src_files`, used to build the set of pending commits. # # Note that directories in `src_sections` may be deleted by certain # operations (for example, if the last package is migrated out of a # section). # local src_files=() local s for s in "${src_sections[@]}"; do local d="$src_dir/$s" if [[ -d "$d" ]]; then local f while read f; do src_files+=("${f#$src_dir/}") done < <(find "$d" -type f -not -name "*.manifest") fi done # Don't load ownership manifests if in stable-management mode because it # does not support any operations on them. # if [[ ("$mode" != "stable") && -n "$src_owners" && -d "$src_dir/$src_owners" ]]; then local f while read f; do src_files+=("${f#$src_dir/}") done < <(find "$src_dir/$src_owners" -type f) fi # Build the set of pending commit hashes ("pending set"). # # For each file in the source repository, find the commit it belongs to and # store its abbreviated hash (as value) inside the `file_commits` # associative array (note: unordered). With some exceptions (see below), # these files' commits will also be stored (as key) inside the `pending_set` # associative array. # # If in destination-management mode: # # - Unless in stable-management mode (which does not operate on ownership # manifests at all), do not add unmanaged ownership manifests' commits # to `pending_set`. (Note that these commits could be added as a result # of other files, and that, in source-management mode, all ownership # manifests are managed.) # # The `managed_projs` and `managed_pkgnames` associative arrays store # the project and package names, respectively, of every package archive # in the managed section and are used to identify unmanaged ownership # manifests (see below). (Both are empty in source-management and # stable-management modes.) # # - Exclude from `pending_set` those commits without any package archives # that match the pattern in `filter`. # # Every file in `src_files` is added to `file_commits` without exception # (that is, regardless of whether or not its commit was added to # `pending_set`) otherwise it could potentially get incorrectly attributed # to an earlier, unfiltered commit (consider the edge case example under the # general migration comments (above migrate_src()) with a filter of "bar.*") # or caused to look like it's been deleted from disk when displayed to the # user. # local -A pending_set=() # Hashes of the pending commits. local -A managed_projs=() # Project names of all managed archives. local -A managed_pkgnames=() # Package names of all managed archives. local h for f in "${src_files[@]}"; do # -1: limit output to one commit (that is, the most # recent) # --diff-filter=A: only show commits that added the specified file or # moved it into its current location ('R' is useless # if a path is specified) # --pretty=format:%h: output only the abbreviated commit hash # h="$(git -C "$src_dir" log -1 --diff-filter=A --pretty=format:%h -- "$f")" # Note that the hash cannot be empty because, after our clean checks at # the top, every file on disk must have been added by some commit (that # is, there can be no untracked files). file_commits["$f"]="$h" # Don't add unmanaged ownership manifests to `pending_set` (by skipping # them) if in any destination-management mode but stable-management (in # which case there are none in `src_files`). An ownership manifest is # unmanaged if its project or package name has not been seen (archives # come before ownership manifests in `src_files`). To this end, also # record the package and project names of every package archive. # if [[ ("$mode" != "source") && ("$mode" != "stable") ]]; then local fi src_path_info "$f" | readarray -t fi local ftype="${fi[0]}" # File type. case "$ftype" in "archive") # Record the package and project names of this package archive. # local p extract_pkg_info "$src_dir/$f" | readarray -t p # (name ver proj) managed_pkgnames["${p[0]}"]=true managed_projs["${p[2]}"]=true ;; "ownership") # Skip this ownership manifest if its package or project name has # not been seen (which means it's unmanaged). # local k="$(basename $(dirname "$f"))" if [[ (("$f" == */package-owner.manifest) && ! "${managed_pkgnames[$k]}") || (("$f" == */project-owner.manifest) && ! "${managed_projs[$k]}") ]]; then continue fi ;; esac fi # Add the commit to the pending set unless the current file is filtered # out. # # Note: $filter must be unquoted to match as a wildcard pattern. # if [[ ("$mode" == "source") || ("$(basename "$(dirname "$f")")" == $filter) || # Project name? ("$(basename "$f")" == $filter.*) ]]; then # Filename (with ext)? pending_set["$h"]=true fi done # Arrange the pending commits in the chronological order. # # Go through the most recent commits in the git log which added or # moved/renamed one or more files, skipping those not present in the pending # set and keeping count to bail out as soon as we ordered all of them. # local i for (( i=0; i != "${#pending_set[@]}"; )); do read h # The abbreviated commit hash. # If this is a pending commit, prepend its hash to the ordered array. # if [[ "${pending_set[$h]}" ]]; then pending_seq=("$h" "${pending_seq[@]}") ((++i)) fi # --diff-filter=AR: only show commits that added or renamed files # done < <(git -C "$src_dir" log --diff-filter=AR --pretty=format:%h) } # Clean the source and destination repositories by discarding uncommitted # changes and removing unstaged files. (Note that the source repository cannot # have untracked files so we git-clean only the destination repository.) # function cleanup () { info "migration failed; resetting and cleaning repositories" if ([[ "$mode" == "source" ]] && ! run git -C "$src_dir" reset --hard) || ! run git -C "$dst_dir" reset --hard || ! run git -C "$dst_dir" clean --force; then info "failed to reset/clean repositories -- manual intervention required" fi } # Return the subject of a git commit in the source repository. # function commit_subject () # { local h="$1" git -C "$src_dir" log -n 1 --pretty=format:%s "$h" } # Return the list of files a commit added to or moved within the source # repository. # function commit_files () # { local h="$1" # git-diff-tree arguments: # # --diff-filter=A: select only files that were added. # -z: don't munge file paths and separate output fields with # NULs. # -r: recurse into subtrees (directories). # git -C "$src_dir" diff-tree \ --no-commit-id --name-only --diff-filter=A -z -r \ "$h" } # Exit with an error if a package which is a duplicate of or is in conflict # with the given package exists in any of the destination sections. # # Two packages are duplicates if they have the same name and version and, as a # result, the same archive filename. If, on the other hand, they have # different names and/or versions but the same archive filename, they are in # conflict with one another. For example, foo-bar version 1.0 and foo version # bar-1.0 have the same archive name foo-bar-1.0.tar.gz. # function check_pkg_duplicate () # { local name="$1" local version="$2" local sd # Section directory. for sd in "${dst_sections[@]}"; do local p # Use -.* without .tar.gz in case we want to support more # archive types later. # bpkg_util_pkg_find_archive "$name-$version.*" "$dst_dir/$sd" | \ readarray -t p if [[ "${#p[@]}" -ne 0 ]]; then local a="${p[0]}" local n="${p[1]}" local v="${p[2]}" if [[ "$n" == "$name" ]]; then error "duplicate of $name/$version at '$a'" else error "conflict of $name/$version with $n/$v at '$a'" fi fi done } # Remove other versions and/or revisions of a package ("replacement # candidates") in the destination section if the user so chooses. Return the # removed packages' version numbers. # # The argument is the name of the source (and the destination) package. # # The argument must be either "*" to remove all other # versions (including revisions), or "*" to remove other revisions # only. # # The argument is the destination section name. # # The and arguments are the version and project of # the source package and are only used in the package removal confirmation # prompt. # # Search for replacement candidates according to # and remove or skip each candidate at the user's direction. Stage (but don't # commit) the removals in the destination repository. # # The versions of the removed packages are written to stdout, each on a # separate line. For example: # # 1.2.3\n1.3.0+1\n1.3.0+2\n # # Note that, currently, versions/revisions which are both lower and higher # than will be considered for replacement. # function remove_pkg_archives () # { local name="$1" local vpat="$2" local dsect="$3" local sver="$4" local sproj="$5" # Search for replacement candidates. # local pkgs= # Packages to be considered for replacement. bpkg_util_pkg_find_archives "$name" \ "$vpat" \ "$dst_dir/${dst_sections[$dsect]}" | \ readarray -t pkgs # For each replacement candidate, ask for confirmation and, depending on the # answer, either remove it from the destination repository or leave it in # place. # local f for f in "${pkgs[@]}"; do # Get the destination archive's info from its embedded manifest. # local p extract_pkg_info "$f" | readarray -t p local dver="${p[1]}" # Destination package version. local dproj="${p[2]}" # Destination package project. # Ask whether or not to drop the destination package. Include the project # names in the prompt if the destination package's project differs from # that of the source package (which is never the case in # destination-management mode). # local src="$sver" local dst="$name/$dver" if [[ "$dproj" != "$sproj" ]]; then src+=" ($sproj)" dst+=" ($dproj)" fi local opt while true; do read -p "replace $dst with $src in $dsect? [y/n]: " opt case "$opt" in y) run git -C "$dst_dir" rm --quiet "${f#$dst_dir/}" echo "$dver" break ;; n) break ;; esac done done } # The commit bundle array is the set of selected pending commits. Its elements # are the corresponding indexes of the `pending_seq` array (but offset by +1). # The indexes are always kept in ascending and, therefore, chronological order # (because the commits in `pending_seq` are in chronological order). # bundle=() # Contains the managed file paths belonging to commits in the commit bundle. # Used as the starting point for operations such as the migration and dropping # of commits or files. # # These files must be grouped by commit and the commit groups ordered # chronologically, otherwise files could be processed in a different order # than they occur in the pending commit list (the `pending_seq` array). For # example, failing that, the commit groups/files in the commit splitting # screen or the file mentions in the commit messages might be in a different # order than they were in the main screen. # bundle_files=() # Collect the commit bundle's files into the global `bundle_files` array, # grouping them by commit. The commit groups are ordered chronologically to # match the ordering of the pending commit array, `pending_seq`. # # Include only files belonging to a commit in the bundle. # # Fail -- by leaving `bundle_files` empty -- if the commit bundle is empty or # any of the files are unmanaged. # function collect_bundle_files () { bundle_files=() local i for i in "${bundle[@]}"; do local h="${pending_seq[$i-1]}" # The current commit's abbreviated hash. local f while read -d '' f; do # Fail (by clearing the `bundle_files` array) if the current file is # unmanaged. # local fi src_path_info "$f" | readarray -t fi if [[ "${fi[0]}" == "unmanaged" ]]; then info "cannot include commit $i: '$f' is unmanaged" bundle_files=() return fi # Add this file only if it belongs to the current commit. # if [[ "${file_commits[$f]}" == "$h" ]]; then bundle_files+=("$f") fi done < <(commit_files "$h") done } # Print "true" to stdout if equals any of the subsequent # arguments/words. # function contains () # ... { local k="$1" shift local w for w in "$@"; do if [[ "$w" == "$k" ]]; then echo -n "true" return fi done } # Return the section name corresponding to the section directory specified by # . # # If the section name exists, print it to stdout; otherwise exit the script # with a diagnostic message and an error status code. # # The section name is the key which indexes in `src_sections`, # so this is a value-to-key lookup. # # Fail by exiting the script because a failed search can only result from a # programming error or a bad repository submit configuration (for example, a # "*" key with a unique value (see below)). # function src_section_name () # { local sd="$1" # Section directory. # The "*" key is a catch-all for unknown submitted section names and, if # present, will share a value (section directory) with one of the known # section names and therefore must be skipped. # local k for k in "${!src_sections[@]}"; do if [[ ("${src_sections[$k]}" == "$sd") && ("$k" != "*") ]]; then echo -n "$k" return fi done error "no source section name found for directory '$sd'" } # Migrate (all management modes): # # Files belonging to one or more user-selected pending commits are moved from # the source to the destination, with the move of each file staged in the # source and/or destination repositories. The action taken on each file is # also recorded in the commit message(s). # # Note the following edge case which applies to all management modes: # # We will need to confirm with git that each of a commit's files were # actually most recently added or moved by that commit. For example (oldest # commits first): # # commit 1: add foo.tar.gz, bar.tar.gz # commit 2: del foo.tar.gz # commit 3: add foo.tar.gz # # If the user chooses to migrate commit 1, only bar.tar.gz must be migrated, # despite foo.tar.gz existing on disk. # Source-management mode migration: migrate the selected commit bundle from # the source repository to the destination repository. # # Takes as input the bundle files in the global `bundle_files` array which is # assumed to be non-empty and to contain only managed files (package archives # or ownership manifests) belonging to commits in the commit bundle. # # If the migration succeeds, set the global `operation_result` variable to # true. Otherwise, in case of failure, issue appropriate diagnostics and set # `operation_result` to the empty string before returning. In order to ensure # that both repositories are left clean if any of the files fail to be # migrated, an EXIT signal handler that discards all uncommitted changes is # installed right before the migration proper begins, and uninstalled when all # files have been migrated. # # The migration will fail if any of the following is true: # # - Files in the commit bundle are not all from the same project (the "bundle # project") or, in the case of archives, the same repository section (the # "bundle section"). # # - The required section does not exist in the destination repository. # # - There exists, for one or more files in the commit bundle, a duplicate or # conflicting archive in any of the sections in the destination repository # (see check_pkg_duplicate()). # # - The migration would overwrite ownership info in the destination # repository. (Currently ownership is updated by modifying the files # directly.) # # The migration process proceeds as follows: # # - Move files: all of the files in the selected commit bundle are moved from # the source repository into the destination repository: # # - Owners to owners directory. # # - Packages into corresponding sections: # # alpha -> alpha # beta -> beta # stable -> testing|stable # # Package archives may be removed at the destination. Candidate files for # replacement are selected as follows: # # - In the alpha and beta sections, any package archive files in the # destination section directory belonging to the same package are # considered for replacement, regardless of their versions. # # - In other sections, any package archives in the destination section # directory with the same name and version but a different revision # (currently whether lower or higher) are replaced. # # Stage (but don't commit) the removal of the files from the source # repository and their addition to the destination repository. # # - Make commits to the source and destination respositories with appropriate # commit messages: # # Source repository: # # "Migrate from $src_sect to $dst_repo_name/
" # # "remove /" # "remove owners//project-owner.manifest" # "remove owners///package-owner.manifest" # # Destination repository: # # "Migrate from $src_repo_name/
to $dst_sect" # # "add /" # "replace / with " (if replacing) # "add owners//project-owner.manifest" # "add owners///package-owner.manifest" # function migrate_src () { operation_result= # Check that all files in the bundle are in the bundle section and/or bundle # project before migrating any of them. Classify each file as a package # archive or ownership manifest as we go along, based on its path. # # The bundle section is derived from the first package archive encountered # and the bundle project from the first package archive or ownership # manifest encountered. # local src_sect= # Source section name. local proj= # The bundle (source) project. local pkgs=() # The bundle's package archives. local owns=() # The bundle's ownership manifests. local f for f in "${bundle_files[@]}"; do local fi src_path_info "$f" | readarray -t fi local ftype="${fi[0]}" # Current file's type. local fproj="${fi[1]}" # Current file's project. if [[ "$ftype" == "ownership" ]]; then owns+=("$f") elif [[ "$ftype" == "archive" ]]; then pkgs+=("$f") local fsect_dir="${fi[2]}" # Section dir from file path. local fsect # File's src section name. fsect="$(src_section_name "$fsect_dir")" # Exits script on failure. # Set the source section name if unset; otherwise fail if the current # file is not from the source section. # if [[ -z "$src_sect" ]]; then src_sect="$fsect" elif [[ "$fsect" != "$src_sect" ]]; then info "'$f' is not in section $src_sect" return fi fi # Set the bundle project if unset; otherwise fail if the current file is # not from the bundle project. # if [[ -z "$proj" ]]; then proj="$fproj" elif [[ "$fproj" != "$proj" ]]; then info "'$f' is not in project $proj" return fi done # Finalize migration variables the values of which depend on whether the # bundle contains at least one package archive or ownership manifests only. # # The source and destination commit messages are composed incrementally as # the migration process proceeds. # local dst_sect # Destination section name. local dst_sect_dir # Destination section directory. local src_cmsg # Source commit message. local dst_cmsg # Destination commit message. if [[ "${#pkgs[@]}" -ne 0 ]]; then # Bundle contains package archive(s). dst_sect="$src_sect" # If it exists, 'testing' overrides 'stable' at the destination. # if [[ ("$dst_sect" == "stable") && -v dst_sections["testing"] ]]; then dst_sect="testing" fi # Fail if the target section does not exist in the destination repository. # if [[ ! -v "dst_sections[$dst_sect]" ]]; then info "section '$dst_sect' does not exist in the destination repository" return fi dst_sect_dir="${dst_sections[$dst_sect]}" src_cmsg="Migrate $proj from $src_sect to $dst_repo_name/$dst_sect"$'\n\n' dst_cmsg="Migrate $proj from $src_repo_name/$src_sect to $dst_sect"$'\n\n' else # Bundle consists only of ownership manifests. # The setup where the ownership authentication is disabled on the # destination but enabled on source is probably obscure, but let's # consider it possible since the submit-git handler allows such a setup. # if [[ -n "$dst_owners" ]]; then src_cmsg="Migrate $proj ownership info to $dst_repo_name"$'\n\n' dst_cmsg="Migrate $proj ownership info from $src_repo_name"$'\n\n' else src_cmsg="Remove $proj ownership info"$'\n\n' dst_cmsg= # Nothing to commit. fi fi # Migrate the bundle's package archive files. # # Ensure that the source and destination repositories are clean if the # migration of any file fails. # trap cleanup EXIT # Migrate a package archive or ownership manifest file from the source # repository to the destination repository. # # is the path of the source file, relative to the source repository # directory, and is the path of the destination directory, relative to # the destination repository directory. # # Note that the source and destination sections and owners directories may # differ which is why those components must be specified in both the source # and destination paths. # # Move the file from the source repository directory to the destination # repository directory, creating directories if required; stage the addition # of the file to the destination repository; stage the removal of the file # from the source repository. # function migrate_file () # { local src="$1" local dst="$2" mkdir -p "$dst_dir/$dst" mv "$src_dir/$src" "$dst_dir/$dst/" run git -C "$src_dir" rm --quiet "$src" run git -C "$dst_dir" add "$dst/$(basename "$src")" } for f in "${pkgs[@]}"; do # Get the current package's name and version from its embedded manifest # (we already have the source project in $proj). # local p extract_pkg_info "$src_dir/$f" | readarray -t p local name="${p[0]}" local version="${p[1]}" check_pkg_duplicate "$name" "$version" # In the destination repository, find and remove package archive files # which are other alpha/beta versions or revisions of the current source # package. # local vpat # Version pattern for replacement. case "$dst_sect" in alpha|beta) vpat="*" ;; # All package versions. *) vpat="$version*" ;; # All package version revisions. esac local rv # Removed version numbers. remove_pkg_archives "$name" "$vpat" \ "$dst_sect" \ "$version" "$proj" | readarray -t rv # Update the commit messages and migrate the current package. # src_cmsg+=" remove $name/$version"$'\n' if [[ "${#rv[@]}" -eq 0 ]]; then dst_cmsg+=" add $name/$version"$'\n' else local v for v in "${rv[@]}"; do dst_cmsg+=" replace $name/$v with $version"$'\n' done fi migrate_file "$f" "$dst_sect_dir/$proj" done # Migrate the bundle's ownership manifests. # # If ownership authentication is disabled on the destination repository, # only remove ownership manifests from the source repository (that is, don't # migrate). # # Fail if an ownership manifest already exists in the destination # repository. # for f in "${owns[@]}"; do src_cmsg+=" remove $f"$'\n' if [[ -n "$dst_owners" ]]; then local dp=$(dirname "${f/$src_owners/$dst_owners}") # Destination path. local fn=$(basename "$f") # File name. if [[ -f "$dst_dir/$dp/$fn" ]]; then error "$f already exists at $dst_dir/$dp/$fn" fi dst_cmsg+=" add $dp/$fn"$'\n' migrate_file "$f" "$dp" else # Ownership authentication disabled in the destination repository. run git -C "$src_dir" rm --quiet "$f" fi done # Commit the changes made to the source and destination repositories. # info run git -C "$src_dir" commit -m "$src_cmsg" if [[ -n "$dst_cmsg" ]]; then info run git -C "$dst_dir" commit -m "$dst_cmsg" fi info # All files have been migrated successfully so set the result and clear the # EXIT trap. # operation_result=true trap EXIT read -p "press Enter to continue" } # Destination-management mode migration: migrate the package archives in the # selected commit bundle from the source (managed) section to its counterpart # section and skip ownership manifests (with a notification message). # # Takes as input the bundle files in the global `bundle_files` array which is # assumed to be non-empty and to contain only managed files belonging to # commits in the commit bundle. # # Assumes that the source and destination sections are valid. # # The general structure of this function is very similar to that of # migrate_src() but most of the logic is simpler. Some noteworthy differences # include: # # - Only the destination repository is involved. # # - Ownership manifests cannot be migrated in destination-management mode and # are skipped. # # - All managed package archives in the commit bundle are known to be in the # same section (because all files outside of the managed section are # ignored). # # The mechanism by which success or failure is indicated is the same as for # migrate_src(). # # The migration will fail if any of the following is true: # # - Files in the commit bundle are not all from the same project (the "bundle # project"). # # - There exists, for one or more package archives in the commit bundle, a # duplicate or conflicting archive in the destination section (see # check_pkg_duplicate()). # # - Migration from the specified source section is not supported. # # - The file selection contained no package archives. # # The migration process proceeds as follows: # # - Move files: all of the package archives in the selected commit bundle are # moved from the source section to the destination section. Ownership # manifests are skipped with a notification message. # # Any package archives in the destination section directory with the same # name and version but a different revision (currently whether lower or # higher) will be replaced if the user so chooses. # # Stage (but don't commit) each file move as the migration proceeds. # # - Make a commit to the destination repository with an appropriate commit # message: # # "Migrate from $src_sect to $dst_sect" # # "move /" # "replace / with " (if replacing) # function migrate_dst () { operation_result= local src_sect="$mode" # Source section. local src_sect_dir="${src_sections[$src_sect]}" # Source section directory. # Fail if the source section has no counterpart, in which case migration # from it is not supported. # if [[ ! -v "sect_cparts[$src_sect]" ]]; then info "migration from $src_sect not supported" return fi local dst_sect="${sect_cparts[$src_sect]}" # Destination section. local dst_sect_dir="${dst_sections[$dst_sect]}" # Dst. section directory. # Check that every file is from the bundle project (taken from the path of # the first file encountered) and skip ownership manifests. # local proj= # The bundle project. local i for i in "${!bundle_files[@]}"; do local f="${bundle_files[$i]}" local fi src_path_info "$f" | readarray -t fi local ftype="${fi[0]}" # Current file's type. local fproj="${fi[1]}" # Current file's project. # Set the bundle project if unset; otherwise fail if the current file is # not from the bundle project. # if [[ -z "$proj" ]]; then proj="$fproj" elif [[ "$fproj" != "$proj" ]]; then info "'$f' is not in project $proj" return fi # Skip ownership manifests and remove them from `bundle_files` so that we # need not worry about file type for the rest of this function. # # Note that, in commit-splitting mode, skipped ownership manifests will # not be printed in the file list after the migration despite not having # been actioned. # if [[ "$ftype" == "ownership" ]]; then info "skipping '$f'" unset "bundle_files[$i]" fi done # Fail if there were no package archives in `bundle_files`. # if [[ "${#bundle_files[@]}" -eq 0 ]]; then info "no package archives selected" return fi # Migrate the bundle's files. # # Ensure that the source and destination repositories are clean if the # migration of any package archive fails. # trap cleanup EXIT # The commit message. # local cmsg="Migrate $proj from $src_sect to $dst_sect"$'\n\n' for f in "${bundle_files[@]}"; do # Note: contains no ownership manifests. # Get the current package's name and version from its embedded manifest # (we already have the source project in $proj). # local p extract_pkg_info "$src_dir/$f" | readarray -t p local name="${p[0]}" local version="${p[1]}" # Note that only the destination section is checked because that is the # only one loaded into dst_sections when in destination-management mode. # check_pkg_duplicate "$name" "$version" # Find and remove other revisions of the current package. # local rv # Removed version numbers. remove_pkg_archives "$name" "$version*" \ "$dst_sect" \ "$version" "$proj" | readarray -t rv # Update the commit message. # if [[ "${#rv[@]}" -eq 0 ]]; then cmsg+=" move $name/$version"$'\n' else local v for v in "${rv[@]}"; do cmsg+=" replace $name/$v with $version"$'\n' done fi # Migrate the current package. # mkdir -p "$dst_dir/$dst_sect_dir/$proj" run git -C "$dst_dir" mv "$f" "$dst_sect_dir/$proj/" done # Remove the project directory from the source section if it is empty. # # (Unlike git-mv, git-rm automatically removes the directory when its last # file is removed, so this does not need to be done in migrate_src().) # local d="$dst_dir/$src_sect_dir/$proj/" if [[ -z "$(ls -A "$d")" ]]; then rmdir "$d" fi # Commit the staged changes. # info run git -C "$dst_dir" commit -m "$cmsg" info # All files have been migrated successfully so set the result and clear the # EXIT trap. # operation_result=true trap EXIT read -p "press Enter to continue" } # Check that ownership consistency would be maintained if the bundle's files # were dropped. # # Print "true" to stdout if all checks passed, and nothing in case of failure. # # Fail if a drop would leave, in the real source or destination repositories, # package archives without package or project ownership, or package ownership # manifests without project ownership. # # Note that in order to search for unselected packages and ownership # manifests, each repository's submit config is loaded again because the data # loaded during global init does not reflect the true state of the # repositories on disk if in destination-management mode. # function check_drop_ownership_consistency () # { local proj="$1" # Project name. # Print "true" to stdout if is the managed repository's # directory, and nothing otherwise. # function managed_repo () # { local d="$1" # Repository directory. if [[ (("$mode" == "source") && ("$d" == "$src_dir")) || (("$mode" != "source") && ("$d" == "$dst_dir")) ]]; then echo -n "true" fi } # Find the package names of all unselected package archives from in # the real source and destination repositories. # local -A unsel_pkg_names=() # The names of unselected packages. # Find unselected packages from in the repository directory specified # by and insert them into `unsel_pkg_names`. # # In deciding whether a file is unselected, only check `bundle_files` if # is the managed repository because if a file (erroneously) # exists in both repositories and is selected in the managed repository it # would be in `bundle_files` despite being unselected in the unmanaged one. # function find_unsel_pkg () # { local rd="$1" # Repository directory. local owners local -A sections source "$rd/submit.config.bash" local s for s in "${sections[@]}"; do local pd="$rd/$s/$proj" # Project directory. if [[ -d "$pd" ]]; then local f while read f; do local frel="${f#$rd/}" # Path made relative to repo dir. if [[ ! "$(managed_repo "$rd")" || ! "$(contains "$frel" "${bundle_files[@]}")" ]]; then local p # (name ver proj) bpkg_util_pkg_verify_archive "$f" | readarray -t p unsel_pkg_names["${p[0]}"]= fi done < <(find "$pd" -type f -not -name "*.manifest") fi done } find_unsel_pkg "$real_src_dir" find_unsel_pkg "$dst_dir" # Find unselected package ownership from in the real source and # destination repositories. # local unsel_pkg_owns= # True if there are unselected pkg ownership manifests. # Find unselected package ownership manifest from in the repository # directory specified by and, if found, set `unsel_pkg_owns` to # true. See find_unsel_pkg() above for a note about the role `bundle_files` # plays. # function find_unsel_pkg_owns () # { local rd="$1" # Repository directory. local owners local -A sections source "$rd/submit.config.bash" local pd="$rd/$owners/$proj" # Project directory. if [[ -d "$pd" ]]; then local f while read f; do f="${f#$rd/}" # Make path relative to repo dir. if [[ ("$f" == */package-owner.manifest) && (! "$(managed_repo "$rd")" || ! "$(contains "$f" "${bundle_files[@]}")") ]]; then unsel_pkg_owns=true break fi done < <(find "$pd" -type f) fi } find_unsel_pkg_owns "$real_src_dir" find_unsel_pkg_owns "$dst_dir" # Fail if any ownership info about to be dropped would leave any ownerless # package archives or package ownership manifests. # local f for f in "${bundle_files[@]}"; do case "$f" in */project-owner.manifest) if [[ ("${#unsel_pkg_names[@]}" -ne 0) || "$unsel_pkg_owns" ]]; then info "cannot drop project ownership info without \ associated packages and/or package ownership" return fi ;; */package-owner.manifest) local pname="$(basename $(dirname "$f"))" if [[ -v "unsel_pkg_names[$pname]" ]]; then info "cannot drop package ownership without associated packages" return fi ;; esac done echo -n "true" } # Drop the files in the selected commit bundle from the source repository. # # Takes as input the bundle files in the global `bundle_files` array which is # assumed to be non-empty and to contain only managed files (package archives # or ownership manifests) belonging to commits in the commit bundle. # # The mechanism by which success or failure is indicated is the same as for # migrate_src(). (In particular, the source repository will be left clean if # the removal of any of the files fails.) # # The operation will fail if any of the following is true: # # - Files in the commit bundle are not all from the same project (the "bundle # project") and section (the "bundle section"). # # - Ownership info would be removed for any files remaining after the drop # (see check_drop_ownership_consistency()). # # - The mode is the stable-management, because stable, published packages # cannot be unpublished. # # The operation proceeds as follows: # # - Check that all files are from the same project and check the resulting # ownership consistency. # # - Get from the user the reason for dropping the files. # # - Remove each file from the source repository. Get user confirmation before # dropping ownership manifests, issuing a notification message for each one # skipped. Stage (but don't commit) each removal as the operation proceeds. # # - Make a commit to the source repository with an appropriate commit message. # # If dropping at least one package archive: # # "Drop from
()" # # " remove /" # " remove owners//project-owner.manifest" # " remove owners///package-owner.manifest" # # If dropping only ownership information: # # "Drop ownership ()" # # " remove owners//project-owner.manifest" # " remove owners///package-owner.manifest" # function drop () { operation_result= if [[ "$mode" == "stable" ]]; then info "dropping files from $mode not supported" return fi # Check that all files in the bundle are in the bundle project and that all # package archives are in the bundle section before dropping any of # them. Classify each file as a package archive or ownership manifest as we # go along, based on its path. # # The bundle section is derived from the first package archive encountered # and the bundle project from the first package archive or ownership # manifest encountered. # # Get user confirmation to drop each ownership manifest, skipping those # declined. # # Note that, in commit-splitting mode, skipped ownership manifests will not # be printed in the file list after the migration despite not having been # actioned. # local proj= # The bundle project. local sect= # The bundle section name. local pkgs=() # Package archives to be dropped. local owns=() # Ownership manifests to be dropped. local i for i in "${!bundle_files[@]}"; do local f="${bundle_files[$i]}" local fi src_path_info "$f" | readarray -t fi local ftype="${fi[0]}" # Current file's type. local fproj="${fi[1]}" # Current file's project. if [[ "$ftype" == "ownership" ]]; then # Ask whether or not this ownership manifest should be dropped. Add it # to `owns` if the user confirmed or, if the user declined, skip it by # not adding it to `owns` and by removing it from `bundle_files` (to # prevent check_drop_ownership_consistency() from thinking it's still # selected). # local opt= while [[ ("$opt" != y) && ("$opt" != n) ]]; do read -p "drop '$f'? [y/n]: " opt done if [[ "$opt" == y ]]; then owns+=("$f") else info "skipping '$f'" unset "bundle_files[$i]" fi elif [[ "$ftype" == "archive" ]]; then pkgs+=("$f") local fsect_dir="${fi[2]}" # Section dir from file path. local fsect # File's section name. fsect="$(src_section_name "$fsect_dir")" # Exits script on failure. # Set the bundle section name if unset; otherwise fail if the current # package archive is not from the bundle section. # if [[ -z "$sect" ]]; then sect="$fsect" elif [[ "$fsect" != "$sect" ]]; then info "'$f' is not in section $sect" return fi fi # Set the bundle project if unset; otherwise fail if the current file is # not from the bundle project. # if [[ -z "$proj" ]]; then proj="$fproj" elif [[ "$fproj" != "$proj" ]]; then info "'$f' is not in project $proj" return fi done # Fail if there is nothing to drop or if the drop would remove ownership # info from any remaining files. # # There would be nothing to drop if the bundle consisted only of ownership # manifests and the user declined to drop all of them. # # Note that check_drop_ownership_consistency() prints diagnostics in case of # failure. # if [[ (("${#pkgs[@]}" -eq 0) && ("${#owns[@]}" -eq 0)) || ! "$(check_drop_ownership_consistency "$proj")" ]]; then return fi # Get the reason for dropping the files from the user. # local reason= info while [[ -z "$reason" ]]; do read -p "reason for dropping: " reason done # Set the commit message subject depending on whether there is at least one # package archive in the bundle. # local cmsg= # Commit message. if [[ "${#pkgs[@]}" -ne 0 ]]; then # Bundle contains package archive(s). cmsg="Drop $proj from $sect ($reason)"$'\n\n' else # Bundle contains only ownership manifests. cmsg="Drop $proj ownership ($reason)"$'\n\n' fi # Remove the bundle's files from the source repository and compose the # commit message details in the process. # # Ensure that the source repository is clean if any of the removals fail. # trap cleanup EXIT local f for f in "${pkgs[@]}"; do # Get the current package's name and version from its embedded manifest. # local p extract_pkg_info "$src_dir/$f" | readarray -t p local name="${p[0]}" local version="${p[1]}" cmsg+=" remove $name/$version"$'\n' run git -C "$src_dir" rm "$f" # Automatically removes empty directories. done for f in "${owns[@]}"; do cmsg+=" remove $f"$'\n' run git -C "$src_dir" rm "$f" # Automatically removes empty directories. done # Commit the changes made to the source repository. # info run git -C "$src_dir" commit -m "$cmsg" info # All files have been dropped successfully so set the result and clear the # EXIT trap. # operation_result=true trap EXIT read -p "press Enter to continue" } # Push local changes to the remote source and/or destination git repositories. # # Push to the destination repository first because thus the migrated files # will be in both remote repositories until the completion of the subsequent # push to the source repository (which may fail or take long). Although this # is an inconsistent state, it is safe because other programs such as a # submission handler will be able to detect the duplicates and therefore # refuse to do anything. If, on the other hand, we pushed to the source first, # the migrated files would not exist in either remote repository until the # push to the destination repository completed. In this state the submission # handler would, for example, accept a resubmission of the migrated packages or # erroneously establish ownership for already owned project/package names. # function push () { # Let's print additional diagnostics on git-push failure, to emphasize for # the user which of the two repositories we have failed to push. # if ! run git -C "$dst_dir" push; then error "push to $dst_repo_name failed" fi if [[ "$mode" == "source" ]] && ! run git -C "$src_dir" push; then error "push to $src_repo_name failed" fi } # User interface: # # The main screen presents the list of pending commits to the user, oldest # first, with files that were deleted by subsequent commits marked with `*` # and, of those, the ones that were then added back again with `!`. Files from # unmanaged directories are marked with `?`. # # 1 (deadbeef) Add libfoo/1.2.3 # # 1/testing/foo/libfoo-1.2.3.tar.gz # owners/foo/project-owner.manifest # owners/foo/libfoo/package-owner.manifest # # 2 (c00l0fff) Add bar/1.2.3 # # * 1/testing/bar/libbar-1.2.3.tar.gz # 1/testing/bar/libbaz-1.2.3.tar.gz # # 3 (deadbabe) Add libbar/1.2.3+1 # # 1/testing/bar/libbar-1.2.3+1.tar.gz # # Note that files deleted by subsequent commits may still be in the # repository. See the example in the above migration notes. # # Then prompt the user for the action (showing the current bundle): # # [1 2][,m,d,s,c,p,q,l,?]: # # - add commit to the commit bundle # m - migrate the selected commit bundle # d - drop the selected commit bundle # s - split commit(s) (operate on selected files) # c - clear the selected commit bundle # p - push source and destination repositories # l - list pending commits # q - quit (prompting to push if commits were made) # ? - print this help # # The main screen operates at "commit granularity", in which each operation is # performed on entire commits (that is, on all of the files they own). At # "file granularity", on the other hand, operations are performed only on a # user-selected subset of the selected commits' files. To operate at file # granularity the user chooses, in the main menu, to split the selected # commits, thus switching to the "commit splitting screen". See # `split_commits()`, below. # # Print the size of a file to stdout in the human-readable form (K, M, G). # function file_size () { local f="$1" # POSIX specifies the output of `ls -l` so this should be portable. -h turns # on human-readable output and is present on Linux, FreeBSD, and Mac OS. # echo -n "$(ls -lh "$src_dir/$f" | cut -d ' ' -f 5)" } # Show the commit splitting screen: let the user select and operate on # specific files in the commit bundle (file granularity). # # Remove all but the selected files from `bundle_files` -- the global array # used as input to the commit and file operations -- before performing an # operation, and populate it with the unselected files afterwards. # # Assumes that the files in `bundle_files` are grouped by commit and that they # are all existent (that is, present in `file_commits`) and managed. # # This screen's layout is near identical to that of the main screen -- each # commit's files, grouped under the commit heading, with commit groups in # chronological order -- except that here the files are numbered instead of # the commits. The user is presented with a menu similar to the one in the # main screen including the selection of files by number and operations such # as `migrate` and `drop`. # # @@ Feature idea: invert selection. That is, if user wants to exclude rather # than include a file, can add it to the selection and then invert the # selection. # # Also: add 'push' and 'quit script' ('Q') options? Returning to the main # menu is a little annoying, but perhaps that's just because I've been # running this script so frequently. # function split_commits () { # The current file selection. Stores indexes into `bundle_files`. # local fsel=() while true; do if [[ "${#bundle_files[@]}" -ne 0 ]]; then # Print the commits followed by their files. Because `bundle_files` is # grouped by commit we know we are on a new commit when the current # file's commit hash differs from that of its predecessor. # local h= # Hash of commit to which the current subset of files belong. local i for i in "${!bundle_files[@]}"; do local f="${bundle_files[$i]}" # Current file. local fh="${file_commits[$f]}" # Current file's commit hash. if [[ "$fh" != "$h" ]]; then # New commit so first print its heading. h="$fh" printf "\n%s %s\n\n" "$fh" "$(commit_subject "$fh")" >&2 fi # Get this file's size if it's a package archive. # local sz= # File size. local fi src_path_info "$f" | readarray -t fi if [[ "${fi[0]}" == "archive" ]]; then sz=" $(file_size "$f")" fi # Print the current file's number (left-padded to 3 columns), path, # and size (if set). # printf "%3d %s%s\n" "$((i+1))" "$f" "$sz" >&2 done else info "no files in commit bundle" fi # Used to restore the unactioned files into `bundle_files` after an # operation (see update_bundle_files(), below). We need to do it because # our migrate functions operate on (global) bundle_files. # local unactioned_files=() # Collect the selected files into `bundle_files` just before an # operation. But first save its contents in `unactioned_files`. # # Note that after this function returns the indexes in `fsel` are valid # for `unactioned_files` but not `bundle_files`. # function collect_selected_files () { unactioned_files=("${bundle_files[@]}") local r=() local i for i in "${fsel[@]}"; do r+=("${bundle_files[$i-1]}") done bundle_files=("${r[@]}") } # Put the unactioned bundle files into `bundle_files` after an operation. # If the operation succeeded the unactioned files are the unselected ones # and if it failed it's the union of the selected and unselected files. # function update_bundle_files () # { local opres="$1" if [[ "$opres" ]]; then # Remove the selected/actioned files from `unactioned_files`. # local i for i in "${fsel[@]}"; do unset "unactioned_files[$i-1]" done fi bundle_files=("${unactioned_files[@]}") } # Display the menu/prompt, build the file selection, and perform the # requested actions. Breaking out of this loop prints the bundle's files # again. # while true; do # Print the file selection and menu/prompt. Sort the selected files into # ascending order (for display and to maintain the ordering of # `bundle_files`, which `fsel` is the source for). # fsel=($(sed 's/ /\n/g' <<<"${fsel[*]}" | sort -)) local opt read -p $'\n'"[${fsel[*]}][,m,d,c,l,q,?]: " opt # Perform the selected action. # case "$opt" in # Add file to the selection. # [0-9]*) if [[ ("$opt" =~ ^[1-9][0-9]*$) && -v "bundle_files[$opt-1]" ]]; then if [[ ! "$(contains "$opt" "${fsel[@]}")" ]]; then fsel+=("$opt") info "file $opt added to selection" else info "file $opt already selected" fi else info "non-existent file number $opt" fi ;; # Migrate selected files. # m) if [[ "${#fsel[@]}" -ne 0 ]]; then collect_selected_files if [[ "$mode" == "source" ]]; then migrate_src else migrate_dst fi update_bundle_files "$operation_result" if [[ "$operation_result" ]]; then fsel=() need_push=true break fi else info "no files selected" fi ;; # Drop selected files. # d) if [[ "${#fsel[@]}" -ne 0 ]]; then collect_selected_files drop update_bundle_files "$operation_result" if [[ "$operation_result" ]]; then fsel=() need_push=true break fi else info "no files selected" fi ;; # Clear the file selection (and print the file list again). # c) fsel=() break ;; # Print the file list. # l) break ;; # Quit (returning to the main screen). # q) return ;; # ? or invalid option: print menu. # *) cat <&2 - add file to selection m - migrate the selected files d - drop the selected files c - clear the file selection l - list files q - quit (back to main menu) ? - print this help EOF ;; esac done done } # True if any changes have been made to the source and/or destination git # repositories (in which case the user will be asked whether or not to push # before quitting). # need_push= init=true # True if the global state needs to be re-initialized. # The main screen's user interaction loop. # # In each iteration, present the list of pending commits, display the menu of # actions, read the user's input, and perform the chosen action. # while true; do # (Re)initialize the global state if necessary. The idea is that instead of # performing complex house-keeping, after certain operations (e.g., # migration) we will just re-initialize the state from scratch. # if [[ "$init" ]]; then init_globals if [[ "${#pending_seq[@]}" -eq 0 ]]; then info "good news, nothing to manage" if [[ ! "$need_push" ]]; then exit 0 fi fi init= bundle=() fi for i in "${!pending_seq[@]}"; do h="${pending_seq[$i]}" # Print commit number, hash, and subject. # # Prefix with a newline to separate the first commit from the git-pull # output and the rest from the previous commit info block. # printf "\n%d (%s) %s\n\n" "$((i+1))" "$h" "$(commit_subject "$h")" >&2 # Print this commit's files. # # Fetch from the git repository the current commit's files and print each # one's path. # # Mark files that cannot be migrated: # # - Unmanaged files are marked with '?' and will prevent the migration of # the commit. # # - Files associated with a different commit hash in 'file_commits' were # deleted and added back by subsequent commits. These files are marked # with '!' and will not prevent the migration of the commit. # # - Files with no association in 'file_commits' were deleted by a # subsequent commit and never added back. These files are marked with # '*' and will not prevent the migration of the commit. # while read -d '' f; do src_path_info "$f" | readarray -t fi ftype="${fi[0]}" if [[ "$ftype" == "unmanaged" ]]; then # File is unmanaged (and may or may not exist). # info "? $f" else # File is managed. # If this file is a package archive which exists (that is, it's in # `file_commits`), get its size. # sz= # File size. if [[ ("$ftype" == "archive") && -v "file_commits[$f]" ]]; then sz="$(file_size "$f")" fi if [[ "${file_commits[$f]}" == "$h" ]]; then # Last added or moved by the current commit. # # If a commit included some managed and some unmanaged ownership # manifests then it will not have been filtered out at the top and # the unmanaged ownerships will also be shown here. (For an example, # see the first commit output by `bpkg-util-manage --alpha` on the # real cppget.org repos.) I don't think it's worth checking here # that ownerships are managed because those sorts of commits should # be rare. This script also refuses to operate on packages from # different projects or sections. # info " $f $sz" elif [[ -v "file_commits[$f]" ]]; then info "! $f $sz" # Deleted and added back by subsequent commits. else # File was deleted and never added again. Note that actioned files # of partially actioned commits will also appear this way. # info "* $f" fi fi done < <(commit_files "$h") done # The main screen's prompt loop. # # Prompt the user for the action (showing the current bundle), get user # input, and perform the selected action. # # Note that we could adapt the menu according to the current state (don't # offer to migrate if the bundle array is empty, etc) but let's not # complicate the logic. # # Breaking out of this loop re-initializes the global state if `init` is # true and reprints the pending commit list. # while true; do # Sort commit bundle in ascending order. # # Expand the 'bundle' associative array's keys into a single word in which # they are separated by spaces (the first member of IFS) using the # ${!a[*]} syntax; replace each space with a newline before piping to # 'sort', which is newline-based; finally collect sort's output into an # array using the a=() syntax, which splits on newline (the last member of # IFS) because neither space nor tab characters (the other members of IFS) # can occur in the keys. # bundle=($(sed 's/ /\n/g' <<<"${bundle[*]}" | sort --numeric-sort -)) printf "\n" read -p "[${bundle[*]}][,m,d,s,c,p,l,q,?]: " opt case "$opt" in # Add commit to bundle. # [0-9]*) if [[ ("$opt" =~ ^[1-9][0-9]*$) && -v "pending_seq[$opt-1]" ]]; then if [[ ! "$(contains "$opt" "${bundle[@]}")" ]]; then bundle+=("$opt") h="${pending_seq[$opt-1]}" info "commit $opt ($h) \"$(commit_subject "$h")\" \ added to selected bundle" else info "commit $opt is already in the bundle" fi else info "non-existent commit number $opt" fi ;; # Migrate the commit bundle. # m) if [[ "${#bundle[@]}" -ne 0 ]]; then collect_bundle_files # Prints error if `bundle_files` left empty. if [[ "${#bundle_files[@]}" -ne 0 ]]; then if [[ "$mode" == "source" ]]; then migrate_src else migrate_dst fi if [[ "$operation_result" ]]; then need_push=true init=true break fi fi else info "no commits selected" fi ;; # Drop the commit bundle. # d) if [[ "${#bundle[@]}" -ne 0 ]]; then collect_bundle_files # Prints error if `bundle_files` left empty. if [[ "${#bundle_files[@]}" -ne 0 ]]; then drop if [[ "$operation_result" ]]; then need_push=true init=true break fi fi else info "no commits selected" fi ;; # Show commit splitting screen (operate on a selection of the commit # bundle's files). # s) if [[ "${#bundle[@]}" -ne 0 ]]; then collect_bundle_files # Prints error if `bundle_files` left empty. if [[ "${#bundle_files[@]}" -ne 0 ]]; then split_commits # Note that the global state is re-initialized even if nothing was # done in the commit-splitting screen but these should be the # minority of cases. # init=true break fi else info "no commits selected" fi ;; # Clear the commit bundle. # c) bundle=() break ;; # Push changes. # p) push need_push= break ;; # Redraw the pending commit list. # l) break ;; # Quit. # q) if [[ ! "$need_push" ]]; then exit 0 fi while true; do read -p "push changes? [y/n/(c)ancel]: " opt case "$opt" in c) break # Print options menu again. ;; y) push exit 0 ;; n) exit 0 ;; *) continue ;; esac done ;; # ? or invalid option: print menu. # *) cat < - add commit to the commit bundle m - migrate the selected commit bundle d - drop the selected commit bundle s - split commit(s) (operate on selected files) c - clear the selected commit bundle p - push source and destination repositories l - list pending commits q - quit (prompting to push if commits were made) ? - print this help EOF ;; esac done done