Always serialize reference submit result manifest value if available and restructure handlers dir

1 files changed, 0 insertions, 659 deletions

diff --git a/brep/submit/submit-git.in b/brep/submit/submit-git.in
deleted file mode 100644
index 0c90309..0000000
--- a/brep/submit/submit-git.in
+++ /dev/null
@@ -1,659 +0,0 @@
-#!/usr/bin/env bash
-
-# file      : brep/submit/submit-git.in
-# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
-# license   : MIT; see accompanying LICENSE file
-
-# Package submission handler with git repository storage.
-#
-# The overall idea behind this handler is to add the package archive into a
-# git repository. Another entity (for example, a human or a script) can then
-# pull this change and do something about it (for example, review it and/or
-# add it to an archive-based repository). In other words, git is used as a
-# kind of transport that is easy enough to access for both humans and scripts.
-#
-# The handler also implements the project/package name ownership verification
-# by performing the submitter authentication/authorization based on the
-# control repository mechanism described in bdep-publish(1). This
-# functionality is optional.
-#
-# The handler can operate with a single git repository, called "target", or
-# with two git repositories, in which case the first is the target and the
-# second is called "reference". The reference repository access is read-only
-# and it is only consulted for duplicate package suppression and name
-# ownership verification. The dual repository mode is normally used to
-# implement a two-stage queue/public setup where the package is first queued
-# for review and/or testing and then moved (for example, by a moderator) to a
-# public repository.
-#
-# The target repository argument (<tgt-repo>) should be a read-write git
-# repository URL. It is cloned (shallow) into the submission directory on
-# each submission.
-#
-# If specified, the reference repository argument (<ref-repo>) should be a
-# directory with a pre-cloned read-only reference repository. This directory
-# is shared between all instances of the handler. On each submission, the
-# handler will flock(1) this directory, git-pull, obtain the information it
-# needs, and release the lock.
-#
-# Both the target and, if specified, reference repositories should contain the
-# submit.config.bash repository configuration file in the root directory. The
-# configuration file is a bash fragment and is sourced by the handler script.
-# It provides the following information:
-#
-# - Mapping of section names to repository subdirectories in the 'sections'
-#   variable (declare -A sections; values are relative to the repository
-#   root).
-
-#   If there is no key for the submitted section name, then the entry with the
-#   special '*' key is used. If there is no such entry, then the submission is
-#   invalid. For example:
-#
-#     sections[alpha]=1/alpha
-#     sections[beta]=1/beta
-#     sections[stable]=1/testing
-#
-# - Optional owners subdirectory in the 'owners' variable (relative to the
-#   repository root). If not specified, then no ownership verification is
-#   performed. For example:
-#
-#     owners=owners
-#
-# If the ownership directory is specified, then the handler script maintains
-# the project/package name ownership information in this directory. It has the
-# following structure:
-#
-# <owners>/
-# |-- <project1>/
-# |   |-- <package1>/
-# |   |   `-- package-owner.manifest
-# |   |-- <package2>/
-# |   |   `-- package-owner.manifest
-# |   |-- ...
-# |   `-- project-owner.manifest
-# |-- <project2>/
-# |   `-- ...
-# `-- ...
-#
-# If the submitted project name is not yet known, then the handler script
-# creates a new project subdirectory and saves project-owner.manifest. The
-# project owner manifest contains the following values in the specified order:
-#
-#   name: <project-name>
-#   author-name: <name>
-#   author-email: <email>
-#   control: <url-prefix>
-#
-# The 'control' value is the control repository URL prefix and there can be
-# multiple such values in a single manifest. The handler script derives it
-# from the submitted control repository URL by removing the last path
-# component. So, for example, https://github.com/build2/libbutl.git becomes
-# https://github.com/build2/.
-#
-# If the submitted project name is already known, then the handler script
-# loads its project-owner.manifest and verifies that at least one of the
-# 'control' values is a prefix of the submitted control repository URL.
-#
-# Similarly, if the submitted package name is not yet known, then the handler
-# script creates a new package subdirectory and saves package-owner.manifest.
-# The package owner manifest contains the following values in the specified
-# order:
-#
-#   name: <package-name>
-#   author-name: <name>
-#   author-email: <email>
-#   control: <url>
-#
-# The 'control' value is the control repository URL and there can be multiple
-# such values in a single manifest.
-#
-# If the submission package is already known, then the handler script loads
-# its package-owner.manifest and verifies that at least one of the 'control'
-# values matches the submitted control repository URL.
-#
-# If all these ownership authentication tests pass, the handler script clones
-# (shallow) the build2-control branch of the control repository and verifies
-# that the submission authorization file is present (see bdep-publish(1) for
-# details).
-#
-# If the submission authorization test passes, then the handler script adds
-# the package archives to the target repository, commits this change, and
-# then pushes the commit to the remote.
-#
-# Notes:
-#
-# - It is possible that a submitted package name already exists in another
-#   project. In this case, such a submission is accepted only if the package
-#   already exists in the requested project. This allows the moderator to
-#   manually permit such multi-project packages (for example, to allow moving
-#   packages between projects).
-#
-# - There could be a race when moving package and ownership information from
-#   target to reference. To avoid it, the protocol for such a move is to first
-#   add, commit, and push to reference and then remove, commit, and push to
-#   target.
-#
-#   On the handler side, the script acts in the opposite order cloning the
-#   target prior pulling the reference in order not to get into the situation
-#   where it misses the ownership info that is not in the reference yet but no
-#   longer in the target. Note that if some move happens after the cloning,
-#   then the script will be unable to push the target modification and will
-#   re-try the whole authentication procedure from scratch.
-#
-# - Filesystem entries that exist or are created in the data directory:
-#
-#   <pkg>-<ver>.tar.gz   saved by brep (could be other archives in the future)
-#   request.manifest     created by brep
-#   package.manifest     extracted by the handler
-#   target/              cloned by the handler
-#   control/             cloned by the handler
-#   result.manifest      saved by brep
-#
-# Options:
-#
-# --committer-name <name>
-#
-#   Name to use for the target repository commits. "Submission Handler" if
-#   unspecified.
-#
-# --committer-email <email>
-#
-#   Email to use for the target repository commits. noreply@example.com if
-#   unspecified.
-#
-usage="usage: $0 [<options>] <tgt-repo> [<ref-repo>] <dir>"
-
-# Diagnostics.
-#
-verbose= #true
-
-# Git network operations timeout (seconds).
-#
-# Note that we don't cover protocols other than HTTP(S) since for them git
-# doesn't support any timeouts (though we may be able to cobble something
-# up for SSH).
-#
-git_timeout=10
-
-# The reference repository lock timeout (seconds).
-#
-ref_lock_timeout=30
-
-trap "{ exit 1; }" ERR
-set -o errtrace # Trap ERR in functions.
-
-@import brep/submit/submit@
-@import brep/submit/submit-git@
-
-# Parse the command line options.
-#
-committer_name="Submission Handler"
-committer_email="noreply@example.com"
-
-while [ $# -gt 0 ]; do
-  case $1 in
-    --committer-name)
-      shift
-      committer_name="$1"
-      shift
-      ;;
-    --committer-email)
-      shift
-      committer_email="$1"
-      shift
-      ;;
-    *)
-      break; # The end of options is encountered.
-      ;;
-  esac
-done
-
-if [ -z "$committer_name" -o -z "$committer_email" ]; then
-  error "$usage"
-fi
-
-# Parse the command line arguments.
-#
-if [ "$#" -lt 2 -o "$#" -gt 3 ]; then
-  error "$usage"
-fi
-
-# Target repository URL.
-#
-tgt_repo="$1"
-shift
-
-if [ -z "$tgt_repo" ]; then
-  error "$usage"
-fi
-
-# Reference repository directory.
-#
-# Note that the last argument is always the submission data directory.
-#
-ref_repo=
-
-if [ "$#" -gt 1 ]; then
-  ref_repo="$1"
-  shift
-
-  if [ -z "$ref_repo" ]; then
-    error "$usage"
-  fi
-
-  if [ ! -d "$ref_repo" ]; then
-    error "'$ref_repo' does not exist or is not a directory"
-  fi
-fi
-
-# Submission data directory.
-#
-data_dir="$1"
-shift
-
-if [ -z "$data_dir" ]; then
-  error "$usage"
-fi
-
-if [ ! -d "$data_dir" ]; then
-  error "'$data_dir' does not exist or is not a directory"
-fi
-
-reference="$(basename "$data_dir")"
-
-# Git verbosity options.
-#
-# Note that not all git commands support the -q/-v options. Also note that
-# these variable expansions should not be quoted.
-#
-if [ "$verbose" ]; then
-  gqo=
-  gvo="-v"
-else
-  gqo="-q"
-  gvo=
-fi
-
-# Git doesn't support the connection timeout option. The options we use are
-# just an approximation of the former, that, in particular, don't cover the
-# connection establishing. To work around this problem, before running a git
-# command that assumes the remote repository communication we manually check
-# connectivity with the remote repository.
-#
-git_http_timeout=("-c" "http.lowSpeedLimit=1" \
-		  "-c" "http.lowSpeedTime=$git_timeout")
-
-# Parse the submission request manifest and obtain the required values.
-#
-manifest_parser_start "$data_dir/request.manifest"
-
-archive=
-sha256sum=
-section=
-author_name=
-author_email=
-control=
-simulate=
-
-while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
-  case "$n" in
-    archive)      archive="$v"      ;;
-    sha256sum)    sha256sum="$v"    ;;
-    section)      section="$v"      ;;
-    author-name)  author_name="$v"  ;;
-    author-email) author_email="$v" ;;
-    control)      control="$v"      ;;
-    simulate)     simulate="$v"     ;;
-  esac
-done
-
-manifest_parser_finish
-
-if [ -z "$archive" ]; then
-  error "archive manifest value expected"
-fi
-
-if [ -z "$sha256sum" ]; then
-  error "sha256sum manifest value expected"
-fi
-
-if [ -n "$simulate" -a "$simulate" != "success" ]; then
-  exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
-fi
-
-# The author-* manifest values should both be present or absent.
-#
-if [ -z "$author_name" -a -n "$author_email" ]; then
-  exit_with_manifest 400 "author-name manifest value expected"
-fi
-
-if [ -z "$author_email" -a -n "$author_name" ]; then
-  exit_with_manifest 400 "author-email manifest value expected"
-fi
-
-# Note: checking for section, author-*, and control later.
-
-m="$data_dir/package.manifest"
-extract_package_manifest "$data_dir/$archive" "$m"
-
-# Parse the package manifest and obtain the package name and version.
-#
-manifest_parser_start "$m"
-
-name=
-version=
-project=
-
-while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
-  case "$n" in
-    name)    name="$v"    ;;
-    version) version="$v" ;;
-    project) project="$v" ;;
-  esac
-done
-
-manifest_parser_finish
-
-if [ -z "$name" ]; then
-  error "name manifest value expected"
-fi
-
-if [ -z "$version" ]; then
-  error "version manifest value expected"
-fi
-
-if [ -z "$project" ]; then
-  project="$name"
-fi
-
-function git_add () # <repo-dir> <path>...
-{
-  local d="$1"
-  shift
-
-  run git -C "$d" add $gvo "$@" >&2
-}
-
-function git_clone () # <repo-url> <repo-dir> <extra-options>...
-{
-  local url="$1"
-  shift
-
-  local dir="$1"
-  shift
-
-  check_connectivity "$url" "$git_timeout"
-  run git "${git_http_timeout[@]}" clone $gqo $gvo "$@" "$url" "$dir" >&2
-}
-
-# Dor now we make 10 re-tries to add the package and push to target. Push can
-# fail due to the target-to-reference information move race (see the above
-# notes for details) or because concurrent submissions. We may want to make it
-# configurable in the future.
-#
-pkg_added=
-
-for i in {1..11}; do
-
-  # Clone the target repository.
-  #
-  tgt_dir="$data_dir/target"
-  git_clone "$tgt_repo" "$tgt_dir" --single-branch --depth 1
-
-  check_package_duplicate "$name" "$version" "$tgt_dir"
-
-  # Check for duplicates and try to authenticate the package ownership using
-  # information in the reference repository, if specified.
-  #
-  if [ -n "$ref_repo" ]; then
-
-    remote_url=$(git -C "$ref_repo" config --get remote.origin.url)
-
-    # Open the reading file descriptor and lock the repository. Fail if unable
-    # to lock before timeout.
-    #
-    l="$ref_repo/submit.config.bash"
-    trace "+ exec {fd}<$l"
-    exec {fd}<"$l"
-
-    if ! run flock -w "$ref_lock_timeout" "$fd"; then
-      exit_with_manifest 503 "submission service temporarily unavailable"
-    fi
-
-    # Pull the reference repository.
-    #
-    check_connectivity "$remote_url" "$git_timeout"
-    run git "${git_http_timeout[@]}" -C "$ref_repo" pull $gqo $gvo >&2
-
-    # Check the package duplicate.
-    #
-    check_package_duplicate "$name" "$version" "$ref_repo"
-
-    # Authenticate the project ownership.
-    #
-    auth="$(auth_project "$project" "$control" "$ref_repo")"
-
-    # Try to authenticate the package ownership if the project ownership was
-    # authenticated successfully.
-    #
-    if [ "$auth" == "project" ]; then
-      a="$(auth_package "$project" "$name" "$control" "$ref_repo")"
-
-      # If the package is unknown to this project, we will try to authenticate
-      # the package name with the target repository later and so we keep the
-      # 'project' auth state.
-      #
-      if [ "$a" != "unknown" ]; then
-	auth="$a"
-      fi
-    fi
-
-    trace "reference auth: $auth"
-
-    if [ "${auth:0:1}" == ":" ]; then # Authentication error?
-      echo "$auth"
-      exit 0
-    fi
-
-    # If the package is not present in the specified project then we need to
-    # make sure it is also not present in any other project.
-    #
-    if [ "$auth" == "project" -o "$auth" == "unknown" ]; then
-      auth_package_unknown "$name" "$ref_repo"
-    fi
-
-    trace "+ exec {fd}<&-"
-    exec {fd}<&- # Close the file descriptor and unlock the repository.
-  else
-    auth="disabled"
-  fi
-
-  ref_auth="$auth"
-
-  # Now try to authenticate the package ownership using information in the
-  # target repository unless already authenticated with reference.
-  #
-  if [ "$auth" != "package" ]; then
-
-    # Don't authenticate the project ownership if this is already done with
-    # the reference repository.
-    #
-    if [ "$auth" != "project" ]; then
-      auth="$(auth_project "$project" "$control" "$tgt_dir")"
-    fi
-
-    # Try to authenticate the package ownership if the project ownership was
-    # authenticated successfully.
-    #
-    if [ "$auth" == "project" ]; then
-      auth="$(auth_package "$project" "$name" "$control" "$tgt_dir")"
-    fi
-
-    trace "target auth: $auth"
-
-    if [ "${auth:0:1}" == ":" ]; then # Authentication error?
-      echo "$auth"
-      exit 0
-    fi
-  fi
-
-  trace "resulting auth: $auth"
-
-  # Sanity check the auth variable value.
-  #
-  case "$auth" in
-    package)  ;;
-    unknown)  ;;
-    disabled) ;;
-
-    *) error "unexpected resulting auth '$auth'";;
-  esac
-
-  # Establish ownership of the package name unless already done.
-  #
-  if [ "$auth" == "unknown" ]; then
-
-    # Check that the necessary request manifest values are specified.
-    #
-    if [ -z "$author_name" ]; then
-      exit_with_manifest 400 "author-name manifest value expected"
-    fi
-
-    # Check that the package doesn't belong yet to some other project.
-    #
-    auth_package_unknown "$name" "$tgt_dir"
-
-    # First the project name.
-    #
-    # Note that owners_dir() shouldn't return an empty string at this stage.
-    #
-    d="$(owners_dir "$tgt_dir")/$project"
-
-    # Establish ownership of the project name unless already done. Note that
-    # it can only be owned by the submitter at this stage.
-    #
-    prj_man="$d/project-owner.manifest"
-
-    if [ ! -f "$prj_man" ]; then
-      run mkdir -p "$d" # Also creates the owners directory if not exist.
-
-      ctl="$(repository_base "$control")"
-
-      create_owner_manifest \
-	"$project" "$author_name" "$author_email" "$ctl" "$prj_man"
-
-      # Add the project owners manifest file to git repository using the path
-      # relative to the repository directory.
-      #
-      git_add "$tgt_dir" "${prj_man#$tgt_dir/}"
-    fi
-
-    # Now the package name.
-    #
-    d="$d/$name"
-    run mkdir "$d"
-
-    pkg_man="$d/package-owner.manifest"
-
-    create_owner_manifest \
-      "$name" "$author_name" "$author_email" "$control" "$pkg_man"
-
-    # Add the package owners manifest file using path relative to the
-    # repository directory.
-    #
-    git_add "$tgt_dir" "${pkg_man#$tgt_dir/}"
-
-    auth="package"
-  fi
-
-  # Respond with the 'unauthorized' manifest if we failed to authenticate the
-  # submitter as the package owner, unless both the reference and target
-  # repositories have the ownership authentication disabled. In the latter
-  # case no authorization is required.
-  #
-  if [ "$auth" != "disabled" -o "$ref_auth" != "disabled" ]; then
-
-    # Respond with the 'unauthorized' manifest if not the package owner.
-    #
-    if [ "$auth" != "package" ]; then
-      if [ "$auth" == "project" -o "$ref_auth" == "project" ]; then
-        exit_with_manifest 401 "package owner authentication failed"
-      else
-        exit_with_manifest 401 "project owner authentication failed"
-      fi
-    fi
-
-    # Authorize the submission.
-    #
-    ctl_dir="$data_dir/control"
-
-    git_clone "$control" "$ctl_dir" --single-branch --depth 1 \
- --branch "build2-control"
-
-    if [ ! -f "$ctl_dir/submit/${sha256sum:0:16}" ]; then
-      exit_with_manifest 401 "package publishing authorization failed"
-    fi
-  fi
-
-  # Add the package archive to the target repository.
-  #
-  s="$(section_dir "$section" "$tgt_dir")"
-
-  if [ -z "$s" ]; then
-    exit_with_manifest 400 "unrecognized section '$section'"
-  fi
-
-  d="$tgt_dir/$s/$project"
-  run mkdir -p "$d" # Create all the parent directories as well.
-
-  # We copy the archive rather than move it since we may need it for a re-try.
-  #
-  a="$d/$archive"
-  run cp "$data_dir/$archive" "$a"
-
-  git_add "$tgt_dir" "${a#$tgt_dir/}"
-
-  author=()
-  if [ -n "$author_name" ]; then
-    author=(--author="$author_name <$author_email>")
-  fi
-
-  run git -c "user.name=$committer_name" -c "user.email=$committer_email" \
--C "$tgt_dir" commit "${author[@]}" $gqo $gvo -F - <<EOF >&2
-Add $name/$version to $s/$project
-
-$(cat "$data_dir/request.manifest")
-EOF
-
-  check_connectivity "$tgt_repo" "$git_timeout"
-
-  # Try to push the target modifications, unless simulating. If this succeeds
-  # then we are done. Otherwise, drop the target directory and re-try the
-  # whole authentication/authorization procedure, unless we are out of
-  # attempts.
-  #
-  if [ -z "$simulate" ]; then
-    if run_silent git "${git_http_timeout[@]}" -C "$tgt_dir" push >&2; then
-      pkg_added=true
-      break
-    else
-      run rm -r -f "$tgt_dir" "$ctl_dir"
-    fi
-  fi
-done
-
-if [ ! "$pkg_added" ]; then
-  exit_with_manifest 503 "submission service temporarily unavailable"
-fi
-
-# Remove the no longer needed submission data directory.
-#
-run rm -r -f "$data_dir"
-
-if [ -n "$simulate" ]; then
-  trace "$name/$version submission is simulated"
-else
-  trace "$name/$version submission is queued"
-fi
-
-exit_with_manifest 200 "$name/$version submission is queued" "$reference"