diff options
Diffstat (limited to 'brep/handler/submit')
| -rw-r--r-- | brep/handler/submit/submit-git.bash.in | 101 | ||||
| -rw-r--r-- | brep/handler/submit/submit-git.in | 36 | ||||
| -rw-r--r-- | brep/handler/submit/submit-pub.in | 58 | ||||
| -rw-r--r-- | brep/handler/submit/submit.bash.in | 56 |
4 files changed, 186 insertions, 65 deletions
diff --git a/brep/handler/submit/submit-git.bash.in b/brep/handler/submit/submit-git.bash.in index 9f25c28..cf7300d 100644 --- a/brep/handler/submit/submit-git.bash.in +++ b/brep/handler/submit/submit-git.bash.in @@ -59,6 +59,10 @@ function owners_dir () # <repo-dir> # Check if a repository already contains the package. Respond with the # 'duplicate submission' result manifest and exit if that's the case. # +# Also check if the repository contains newer revision of this package +# version. Respond with the 'newer revision is present' result manifest and +# exit if that's the case. +# function check_package_duplicate () # <name> <version> <repo-dir> { trace_func "$@" @@ -72,22 +76,54 @@ function check_package_duplicate () # <name> <version> <repo-dir> run source "$rep/submit.config.bash" - # Check for duplicate package in all sections. Use <name>-<version>.* - # without .tar.gz in case we want to support more archive types later. + local rev + rev="$(version_revision "$ver")" + + # Check for duplicate package and its newer revisions in all sections. Use + # <name>-<version>.* without .tar.gz in case we want to support more archive + # types later. # local s for s in "${!sections[@]}"; do local d="$rep/${sections[$s]}" - if [ -d "$d" ]; then - local f - f="$(run find "$d" -name "$nam-$ver.*")" + # Check for duplicate. + # + local p + run pkg_find_archive "$nam-$ver.*" "$d" | readarray -t p + + if [ "${#p[@]}" -ne 0 ]; then + local n="${p[1]}" + local v="${p[2]}" - if [ -n "$f" ]; then - trace "found: $f" + trace "found: $n/$v in ${p[0]}" + + if [ "$n" == "$nam" ]; then exit_with_manifest 422 "duplicate submission" + else + exit_with_manifest 422 "submission conflicts with $n/$v" fi fi + + # Check for newer revision. + # + local arcs + run pkg_find_archives "$nam" "$ver*" "$d" | readarray -t arcs + + local f + for f in "${arcs[@]}"; do + local p + pkg_verify_archive "$f" | readarray -t p + + local v="${p[1]}" + + local rv + rv="$(version_revision "$v")" + + if [ "$rv" -gt "$rev" ]; then + exit_with_manifest 422 "newer revision $nam/$v is present" + fi + done done } @@ -163,6 +199,7 @@ function auth_project () # <project> <control> <repo-dir> local r="unknown" local m="$d/$prj/project-owner.manifest" + local info= # If the project owner manifest exists then parse it and try to authenticate # the submitter as the project owner. @@ -175,16 +212,31 @@ function auth_project () # <project> <control> <repo-dir> local n v while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - if [[ "$n" == "control" && "$ctl" == "$v"* ]]; then - r="project" - break + if [[ "$n" == "control" ]]; then + if [[ "$ctl" == "$v"* ]]; then + r="project" + break + fi + + # If the control URLs don't match, then compare them case- + # insensitively, converting them to the lower case. If they match + # case-insensitively, then still fail the authentication but provide + # additional information in the manifest message value. + # + if [[ "${ctl,,}" == "${v,,}"* ]]; then + info=" + info: control repository URL differs only in character case + info: submitted URL: $ctl + info: project owner's URL: $v + info: consider using --control to specify exact URL" + fi fi done manifest_parser_finish if [ "$r" != "project" ]; then - exit_with_manifest 401 "project owner authentication failed" + exit_with_manifest 401 "project owner authentication failed$info" fi fi @@ -210,7 +262,8 @@ function auth_package () # <project> <package> <control> <repo-dir> local prj="$1" local pkg="$2" - local ctl="${3%.git}" # Strip the potential .git extension. + local ctl="${3%.git}" # For comparison strip the potential .git extension. + local ctl_orig="$3" # For diagnostics use the original URL. local rep="$4" local d @@ -227,6 +280,7 @@ function auth_package () # <project> <package> <control> <repo-dir> local r="unknown" local m="$d/$prj/$pkg/package-owner.manifest" + local info= # If the package owner manifest exists then parse it and try to authenticate # the submitter as the package owner. @@ -241,16 +295,31 @@ function auth_package () # <project> <package> <control> <repo-dir> # local n v while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - if [ "$n" == "control" -a "${v%.git}" == "$ctl" ]; then - r="package" - break + if [ "$n" == "control" ]; then + local u="${v%.git}" + + if [ "$u" == "$ctl" ]; then + r="package" + break + fi + + # If the control URLs don't match, then compare them case- + # insensitively (see auth_project() for details). + # + if [ "${u,,}" == "${ctl,,}" ]; then + info=" + info: control repository URL differs only in character case + info: submitted URL: $ctl_orig + info: package owner's URL: $v + info: consider using --control to specify exact URL" + fi fi done manifest_parser_finish if [ "$r" != "package" ]; then - exit_with_manifest 401 "package owner authentication failed" + exit_with_manifest 401 "package owner authentication failed$info" fi fi diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in index 54cd230..c882b84 100644 --- a/brep/handler/submit/submit-git.in +++ b/brep/handler/submit/submit-git.in @@ -186,8 +186,10 @@ git_timeout=10 ref_lock_timeout=30 trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. -set -o pipefail # Return the rightmost non-zero exit status in a pipeline. +set -o errtrace # Trap in functions and subshells. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in the current shell. +shopt -s nullglob # Expand no-match globs to nothing rather than themselves. @import brep/handler/handler@ @import brep/handler/submit/submit@ @@ -403,7 +405,7 @@ function git_add () # <repo-dir> <path>... local d="$1" shift - run git -C "$d" add $gvo "$@" >&2 + run git -C "$d" add --force $gvo "$@" >&2 } # For now we make 10 re-tries to add the package and push to target. Push can @@ -639,28 +641,10 @@ for i in {1..11}; do exit_with_manifest 400 "unrecognized section '$section'" fi - # Strips the version revision part, if present. - # - v="$(sed -n -re 's%^(\+?[^+]+)(\+[0-9]+)?$%\1%p' <<<"$version")" - - # Make sure the section directory exists before we run find in it. - # - d="$tgt_dir/$s/$project" - run mkdir -p "$d" # Create all the parent directories as well. - - # Go through the potentially matching archives (for example, for - # foo-1.2.3+2: foo-1.2.3.tar.gz, foo-1.2.3+1.tar.gz, foo-1.2.30.tar.gz, etc) - # and remove those that match exactly. - # - # Change CWD to the section directory to make sure that the found archive - # paths don't contain spaces. - # - fs=($(run cd "$tgt_dir/$s" && run find -name "$name-$v*")) + run pkg_find_archives "$name" "$version*" "$tgt_dir/$s" | readarray -t arcs - for f in "${fs[@]}"; do - if [[ "$f" =~ ^\./[^/]+/"$name-$v"(\+[0-9]+)?\.[^/]+$ ]]; then - run git -C "$tgt_dir" rm $gqo "$s/$f" >&2 - fi + for f in "${arcs[@]}"; do + run git -C "$tgt_dir" rm $gqo "${f#$tgt_dir/}" >&2 done # Finally, add the package archive to the target repository. @@ -669,8 +653,10 @@ for i in {1..11}; do # Make sure the project directory exists before we copy the archive into it. # Note that it was removed by git-rm if it became empty. # + d="$tgt_dir/$s/$project" + run mkdir -p "$d" # Create all the parent directories as well. + a="$d/$archive" - run mkdir -p "$d" # Create all the parent directories as well. run cp "$data_dir/$archive" "$a" git_add "$tgt_dir" "${a#$tgt_dir/}" diff --git a/brep/handler/submit/submit-pub.in b/brep/handler/submit/submit-pub.in index d262ae9..42d478d 100644 --- a/brep/handler/submit/submit-pub.in +++ b/brep/handler/submit/submit-pub.in @@ -12,7 +12,7 @@ # # Specifically, the handler performs the following steps: # -# - Lock the repository directory for the duraton of the package submission. +# - Lock the repository directory for the duration of the package submission. # # - Check for the package duplicate. # @@ -85,7 +85,10 @@ verbose= #true rep_lock_timeout=60 trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. +set -o errtrace # Trap in functions and subshells. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in the current shell. +shopt -s nullglob # Expand no-match globs to nothing rather than themselves. @import brep/handler/handler@ @import brep/handler/submit/submit@ @@ -254,6 +257,8 @@ else message_suffix=": $name/$version" fi +revision="$(version_revision "$version")" + # Open the reading file descriptor and lock the repository. Fail if unable to # lock before timeout. # @@ -294,8 +299,22 @@ trap exit_trap EXIT # Check for the package duplicate (in all projects). # -if [ -n "$(run find "$repo_old/1" -name "$archive")" ]; then - exit_with_manifest 422 "duplicate submission" +# Use <name>-<version>.* without .tar.gz in case we want to support more +# archive types later. +# +run pkg_find_archive "$name-$version.*" "$repo_old/1" | readarray -t p + +if [ "${#p[@]}" -ne 0 ]; then + n="${p[1]}" + v="${p[2]}" + + trace "found: $n/$v in ${p[0]}" + + if [ "$n" == "$name" ]; then + exit_with_manifest 422 "duplicate submission" + else + exit_with_manifest 422 "submission conflicts with $n/$v" + fi fi # Copy the current repository using hardlinks. @@ -310,27 +329,30 @@ fi run rsync -rtO --exclude 'packages.manifest' --link-dest="$repo_old" \ "$repo_old/" "$repo_new" -# Remove the package version revisions that may exist in the repository. +# Remove the package version revision archives that may exist in the +# repository. # -# Strips the version revision part, if present. +# But first check if the repository contains newer revision of this package +# version. Respond with the 'newer revision is present' result manifest and +# exit if that's the case. # -v="$(sed -n -re 's%^(\+?[^+]+)(\+[0-9]+)?$%\1%p' <<<"$version")" +run pkg_find_archives "$name" "$version*" "$repo_new/1" | readarray -t arcs -# Go through the potentially matching archives (for example, for foo-1.2.3+2: -# foo-1.2.3.tar.gz, foo-1.2.3+1.tar.gz, foo-1.2.30.tar.gz, etc) and remove -# those that match exactly. -# -# Change CWD to the section directory to make sure that the found archive -# paths don't contain spaces. -# -fs=($(run cd "$repo_new/1" && run find -name "$name-$v*")) +for f in "${arcs[@]}"; do + pkg_verify_archive "$f" | readarray -t p + + v="${p[1]}" + rv="$(version_revision "$v")" -for f in "${fs[@]}"; do - if [[ "$f" =~ ^\./[^/]+/"$name-$v"(\+[0-9]+)?\.[^/]+$ ]]; then - run rm "$repo_new/1/$f" >&2 + if [ "$rv" -gt "$revision" ]; then + exit_with_manifest 422 "newer revision $name/$v is present" fi done +for f in "${arcs[@]}"; do + run rm "$f" +done + # Copy the archive rather than moving it since we may need it for # troubleshooting. Note: the data and repository directories can be on # different filesystems and so hardlinking could fail. diff --git a/brep/handler/submit/submit.bash.in b/brep/handler/submit/submit.bash.in index 667bbc1..7826809 100644 --- a/brep/handler/submit/submit.bash.in +++ b/brep/handler/submit/submit.bash.in @@ -47,12 +47,29 @@ function extract_package_manifest () # <archive> <manifest> local arc="$1" local man="$2" - # Pass the --deep option to make sure that the *-file manifest values are - # resolvable, so rep-create will not fail due to this package down the road. - # Note that we also make sure that all the manifest values are known (see - # bpkg-pkg-verify for details). + # Pass the --deep option to make sure that the bootstrap buildfile is + # present and the *-file manifest values are resolvable, so rep-create will + # not fail due to this package down the road. Note that we also make sure + # that all the manifest values are known (see bpkg-pkg-verify for details). # - if ! run_silent bpkg pkg-verify --deep --manifest "$arc" >"$man"; then + local cmd=(bpkg pkg-verify --deep --manifest "$arc") + trace_cmd "${cmd[@]}" + + # Note that we used to just advise the user to run bpkg-pkg-verify locally + # for the details on the potential failure. That, however, may not always be + # helpful since the user can use a different version of the toolchain and so + # may observe a different behavior. Thus, we add the bpkg-pkg-verify error + # message to the response, turning it into an info. This way the user may + # potentially see the following bdep-publish diagnostics: + # + # error: package archive is not valid + # info: unable to satisfy constraint (build2 >= 0.17.0-) for package libhello-1.0.0.tar.gz + # info: available build2 version is 0.16.0 + # info: run bpkg pkg-verify for details + # info: reference: 308e155764c8 + # + local e + if ! e="$("${cmd[@]}" 2>&1 >"$man")"; then # Perform the sanity check to make sure that bpkg is runnable. # @@ -60,6 +77,33 @@ function extract_package_manifest () # <archive> <manifest> error "unable to run bpkg" fi - exit_with_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)" + # Note that bpkg-pkg-verify diagnostics may potentially contain the + # archive absolute path. Let's sanitize this diagnostics by stripping the + # archive directory path, if present. Also note that to use sed for that + # we first need to escape the special regex characters and slashes in the + # archive directory path (see sed's basic regular expressions for + # details). + # + local d="$(sed 's/[[\.*^$/]/\\&/g' <<<"$(dirname "$arc")/")" + + e="$(sed -e "s/$d//g" -e 's/^error:/ info:/' <<<"$e")" + e=$'package archive is not valid\n'"$e"$'\n info: run bpkg pkg-verify for details' + + exit_with_manifest 400 "$e" fi } + +# Extract the revision part from the package version. Return 0 if the version +# doesn't contain revision. +# +function version_revision () # version +{ + local r + r="$(sed -n -re 's%^(\+?[^+]+)(\+([0-9]+))?$%\3%p' <<<"$1")" + + if [ -z "$r" ]; then + r="0" + fi + + echo "$r" +} |