From 7e0e141273032c7afc1a9129512aa42c672fcf5d Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Thu, 23 Aug 2018 17:36:06 +0300 Subject: Always serialize reference submit result manifest value if available and restructure handlers dir --- brep/handler/.gitignore | 1 + brep/handler/buildfile | 10 + brep/handler/handler.bash.in | 151 ++++++++ brep/handler/submit/.gitignore | 5 + brep/handler/submit/buildfile | 13 + brep/handler/submit/submit-dir.in | 108 ++++++ brep/handler/submit/submit-git.bash.in | 334 +++++++++++++++++ brep/handler/submit/submit-git.in | 660 +++++++++++++++++++++++++++++++++ brep/handler/submit/submit.bash.in | 60 +++ brep/submit/.gitignore | 5 - brep/submit/buildfile | 14 - brep/submit/submit-dir.in | 107 ------ brep/submit/submit-git.bash.in | 333 ----------------- brep/submit/submit-git.in | 659 -------------------------------- brep/submit/submit.bash.in | 197 ---------- mod/mod-submit.cxx | 38 +- tests/submit/buildfile | 2 +- tests/submit/submit-dir.test | 6 +- tests/submit/submit-git.test | 34 +- 19 files changed, 1392 insertions(+), 1345 deletions(-) create mode 100644 brep/handler/.gitignore create mode 100644 brep/handler/buildfile create mode 100644 brep/handler/handler.bash.in create mode 100644 brep/handler/submit/.gitignore create mode 100644 brep/handler/submit/buildfile create mode 100644 brep/handler/submit/submit-dir.in create mode 100644 brep/handler/submit/submit-git.bash.in create mode 100644 brep/handler/submit/submit-git.in create mode 100644 brep/handler/submit/submit.bash.in delete mode 100644 brep/submit/.gitignore delete mode 100644 brep/submit/buildfile delete mode 100644 brep/submit/submit-dir.in delete mode 100644 brep/submit/submit-git.bash.in delete mode 100644 brep/submit/submit-git.in delete mode 100644 brep/submit/submit.bash.in diff --git a/brep/handler/.gitignore b/brep/handler/.gitignore new file mode 100644 index 0000000..e3cf716 --- /dev/null +++ b/brep/handler/.gitignore @@ -0,0 +1 @@ +handler.bash diff --git a/brep/handler/buildfile b/brep/handler/buildfile new file mode 100644 index 0000000..5223b5c --- /dev/null +++ b/brep/handler/buildfile @@ -0,0 +1,10 @@ +# file : brep/handler/buildfile +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +import mods = libbutl.bash%bash{manifest-parser} +import mods += libbutl.bash%bash{manifest-serializer} + +./: bash{handler} submit/ + +bash{handler}: in{handler} $mods diff --git a/brep/handler/handler.bash.in b/brep/handler/handler.bash.in new file mode 100644 index 0000000..89e7e21 --- /dev/null +++ b/brep/handler/handler.bash.in @@ -0,0 +1,151 @@ +# file : brep/handler/handler.bash.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Utility functions useful for implementing request handlers. + +if [ "$brep_handler" ]; then + return 0 +else + brep_handler=true +fi + +@import libbutl/manifest-parser@ +@import libbutl/manifest-serializer@ + +# Diagnostics. +# +# We expect the user to set the verbose variable either to true or empty +# (false). +# +if [ ! -v verbose ]; then + echo "error: variable 'verbose' is not set" >&2 + exit 1 +fi + +# Normally the brep module's log record looks like this: +# +# [Mon Jul 23 17:48:46.945079 2018] [brep:error] [pid 123:tid 456] [brep::submit::init]: error description +# +# We will use the (almost) same format for our diagnostics (redirected to the +# Apache's error_log), so it can easily be attributed to the brep module. +# +info_self="$(basename $0)" + +if [ "$#" -gt 0 ]; then + # Last argument is the request data directory which leaf component normally + # identifies the posted entity. A handler may overwrite this value if that's + # not the case. + # + info_ref="$(basename "${!#/}")" +fi + +function info () # +{ + local severity="$1" + shift + + # Note: %N is Linux-specific. + # + local ts + if ! ts="$(date +"%a %b %d %H:%M:%S.%6N %Y")"; then + ts= + fi + + echo "[$ts] [brep:$severity] [ref $info_ref] [$info_self]: $*" 1>&2; +} + +function error () { info "error" "$*"; exit 1; } + +function trace () { if [ "$verbose" ]; then info "info" "$*"; fi } + +# Trace a command line, quoting empty arguments as well as those that contain +# spaces. +# +function trace_cmd () # ... +{ + if [ "$verbose" ]; then + local s="+" + while [ $# -gt 0 ]; do + if [ -z "$1" -o -z "${1##* *}" ]; then + s="$s '$1'" + else + s="$s $1" + fi + + shift + done + + info "info" "$s" + fi +} + +# Trace the current function name and arguments. +# +function trace_func () # ... +{ + trace_cmd "${FUNCNAME[1]}" "$@" +} + +# Trace and run a command. +# +function run () # ... +{ + trace_cmd "$@" + "$@" +} + +# Same as above but also redirect the command stderr to /dev/null, unless +# running in the verbose mode. +# +# Note that we don't redirect stdout, so it can still be captured. +# +function run_silent () # ... +{ + trace_cmd "$@" + + if [ "$verbose" ]; then + "$@" + else + "$@" 2>/dev/null + fi +} + +# Wrap libbutl manifest parsing/serializing functions to shorten names and to +# add tracing. +# +function manifest_parser_start () # [] +{ + trace_func "$@" + butl_manifest_parser_start "$@" + + manifest_parser_ofd="$butl_manifest_parser_ofd" +} + +function manifest_parser_finish () +{ + trace_func + butl_manifest_parser_finish +} + +function manifest_serializer_start () # [] +{ + trace_func "$@" + butl_manifest_serializer_start "$@" + + manifest_serializer_ifd="$butl_manifest_serializer_ifd" +} + +function manifest_serializer_finish () +{ + trace_func + butl_manifest_serializer_finish +} + +# Serialize one manifest name/value pair. +# +function manifest_serialize () # +{ +# trace "$1: $2" + printf "%s:%s\0" "$1" "$2" >&"$manifest_serializer_ifd" +} diff --git a/brep/handler/submit/.gitignore b/brep/handler/submit/.gitignore new file mode 100644 index 0000000..ef91424 --- /dev/null +++ b/brep/handler/submit/.gitignore @@ -0,0 +1,5 @@ +submit.bash +submit-git.bash + +brep-submit-dir +brep-submit-git diff --git a/brep/handler/submit/buildfile b/brep/handler/submit/buildfile new file mode 100644 index 0000000..b110a1d --- /dev/null +++ b/brep/handler/submit/buildfile @@ -0,0 +1,13 @@ +# file : brep/handler/submit/buildfile +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +./: exe{brep-submit-dir} exe{brep-submit-git} + +exe{brep-submit-dir}: in{submit-dir} bash{submit} ../bash{handler} + +exe{brep-submit-git}: in{submit-git} \ + bash{submit-git} bash{submit} ../bash{handler} + +bash{submit}: in{submit} ../bash{handler} +bash{submit-git}: in{submit-git} bash{submit} ../bash{handler} diff --git a/brep/handler/submit/submit-dir.in b/brep/handler/submit/submit-dir.in new file mode 100644 index 0000000..bda0bf8 --- /dev/null +++ b/brep/handler/submit/submit-dir.in @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +# file : brep/handler/submit/submit-dir.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Simple package submission handler with directory storage. +# +# Validate the package archive located in the specified submission directory +# extracting and parsing the package manifest (saved as package.manifest in +# the submission directory). Keep the submission directory unless simulating. +# Write the submission result manifest to stdout. +# +usage="usage: $0 " + +verbose= #true + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +@import brep/handler/handler@ +@import brep/handler/submit/submit@ + +if [ "$#" != 1 ]; then + error "$usage" +fi + +# Submission data directory (last and the only argument). +# +data_dir="${!#/}" + +if [ -z "$data_dir" ]; then + error "$usage" +fi + +if [ ! -d "$data_dir" ]; then + error "'$data_dir' does not exist or is not a directory" +fi + +reference="$(basename "$data_dir")" + +# Parse the submission request manifest and obtain the archive path as well +# as the simulate value. +# +manifest_parser_start "$data_dir/request.manifest" + +archive= +simulate= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + archive) archive="$v" ;; + simulate) simulate="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$archive" ]; then + error "archive manifest value expected" +fi + +if [ -n "$simulate" -a "$simulate" != "success" ]; then + exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" +fi + +m="$data_dir/package.manifest" +extract_package_manifest "$data_dir/$archive" "$m" + +# Parse the package manifest and obtain the package name, version, and +# project. +# +manifest_parser_start "$m" + +name= +version= +project= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + name) name="$v" ;; + version) version="$v" ;; + project) project="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$name" ]; then + error "name manifest value expected" +fi + +if [ -z "$version" ]; then + error "version manifest value expected" +fi + +if [ -z "$project" ]; then + project="$name" +fi + +if [ -n "$simulate" ]; then + rm -r "$data_dir" + trace "$name/$version submission is simulated" +else + trace "$name/$version submission is queued" +fi + +exit_with_manifest 200 "$name/$version submission is queued" diff --git a/brep/handler/submit/submit-git.bash.in b/brep/handler/submit/submit-git.bash.in new file mode 100644 index 0000000..2fd26b0 --- /dev/null +++ b/brep/handler/submit/submit-git.bash.in @@ -0,0 +1,334 @@ +# file : brep/handler/submit/submit-git.bash.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Utility functions for the submit-git handler. + +if [ "$brep_handler_submit_git" ]; then + return 0 +else + brep_handler_submit_git=true +fi + +@import brep/handler/handler@ +@import brep/handler/submit/submit@ + +# If the section is mapped to a directory in the repository configuration then +# return this directory path and empty string otherwise. +# +function section_dir () #
+{ + trace_func "$@" + + local sec="$1" + local rep="$2" + + local owners # Unused but is declared to avoid polluting the global space. + local -A sections + + run source "$rep/submit.config.bash" + + local r="${sections[$sec]}" + if [ -z "$r" ]; then + r="${sections['*']}" + fi + + echo "$r" +} + +# If the owners directory is set in the repository configuration then return +# this directory path prefixed with the repository directory path and the +# empty string otherwise. +# +function owners_dir () # +{ + local rep="$1" + + local owners + local -A sections # Is declared to avoid polluting the global space. + + run source "$rep/submit.config.bash" + + local r= + if [ -n "$owners" ]; then + r="$rep/$owners" + fi + + echo "$r" +} + +# Check if a repository already contains the package. Respond with the +# 'duplicate submission' result manifest and exit if that's the case. +# +function check_package_duplicate () # +{ + trace_func "$@" + + local nam="$1" + local ver="$2" + local rep="$3" + + local owners # Unused but is declared to avoid polluting the global space. + local -A sections + + run source "$rep/submit.config.bash" + + # Check for duplicate package in all sections. Use -.* + # without .tar.gz in case we want to support more archive types later. + # + local s + for s in "${!sections[@]}"; do + local d="$rep/${sections[$s]}" + + if [ -d "$d" ]; then + local f + f="$(run find "$d" -name "$nam-$ver.*")" + + if [ -n "$f" ]; then + trace "found: $f" + exit_with_manifest 422 "duplicate submission" + fi + fi + done +} + +# Serialize the project or package owner manifest (they have the same set of +# values) to the specified manifest file. +# +function create_owner_manifest () # +{ # + trace_func "$@" + + local nam="$1" + local anm="$2" + local aem="$3" + local ctl="$4" + local man="$5" + + if [ -f "$man" ]; then + error "'$man' already exists" + fi + + manifest_serializer_start "$man" + + manifest_serialize "" "1" # Start of manifest. + manifest_serialize "name" "$nam" + manifest_serialize "author-name" "$anm" + manifest_serialize "author-email" "$aem" + manifest_serialize "control" "$ctl" + + manifest_serializer_finish +} + +# Strip the query part and the leaf path component from the repository URL. +# The resulting URL contains the trailing slash. +# +function repository_base () # +{ + # First, strip the URL query part, then component. + # + sed -n \ +-e 's%^\([^?]*\).*$%\1%' \ +-e 's%^\(.*/\)[^/]\{1,\}/\{0,1\}$%\1%p' \ +<<<"$1" +} + +# Authenticate the project name owner. Make sure that the control manifest +# value is specified unless authentication is disabled. +# +# Possible return values: +# +# - 'project' if the project belongs to the submitter +# - 'unknown' if the project name is not yet known +# - 'disabled' if the owners directory is not configured +# - result manifest describing the authentication error +# +# Note that the authentication error result always starts with ':'. +# +function auth_project () # +{ + trace_func "$@" + + local prj="$1" + local ctl="$2" + local rep="$3" + + local d + d="$(owners_dir "$rep")" + + if [ -z "$d" ]; then + echo "disabled" + return + fi + + if [ -z "$ctl" ]; then + exit_with_manifest 400 "control manifest value expected" + fi + + local r="unknown" + local m="$d/$prj/project-owner.manifest" + + # If the project owner manifest exists then parse it and try to authenticate + # the submitter as the project owner. + # + if [ -f "$m" ]; then + + # Parse the project owner manifest. + # + manifest_parser_start "$m" + + local n v + while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + if [[ "$n" == "control" && "$ctl" == "$v"* ]]; then + r="project" + break + fi + done + + manifest_parser_finish + + if [ "$r" != "project" ]; then + exit_with_manifest 401 "project owner authentication failed" + fi + fi + + echo "$r" +} + +# Authenticate the package name owner. Make sure that the control manifest +# value is specified unless authentication is disabled. It is assumed that the +# project ownership is already authenticated (possibly by another repository). +# +# Possible return values: +# +# - 'package' if the package belongs to the submitter +# - 'unknown' if the package name is not taken in the project +# - 'disabled' if the owners directory is not configured +# - result manifest describing the authentication error +# +# Note that the authentication error result always starts with ':'. +# +function auth_package () # +{ + trace_func "$@" + + local prj="$1" + local pkg="$2" + local ctl="$3" + local rep="$4" + + local d + d="$(owners_dir "$rep")" + + if [ -z "$d" ]; then + echo "disabled" + return + fi + + if [ -z "$ctl" ]; then + exit_with_manifest 400 "control manifest value expected" + fi + + local r="unknown" + local m="$d/$prj/$pkg/package-owner.manifest" + + # If the package owner manifest exists then parse it and try to authenticate + # the submitter as the package owner. + # + if [ -f "$m" ]; then + + # Parse the package owner manifest. + # + manifest_parser_start "$m" + + local n v + while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + if [ "$n" == "control" -a "$v" == "$ctl" ]; then + r="package" + break + fi + done + + manifest_parser_finish + + if [ "$r" != "package" ]; then + exit_with_manifest 401 "package owner authentication failed" + fi + fi + + echo "$r" +} + +# Check that the package name is unknown to the repository. Owners directory +# is expected to be configured. +# +function auth_package_unknown () # +{ + trace_func "$@" + + local pkg="$1" + local rep="$2" + + local d + d="$(owners_dir "$rep")" + + # Sanity check that the owners directory configured for the repository. + # + if [ -z "$d" ]; then + error "no owners directory configured for '$rep'" + fi + + # While configured, the owners directory may not yet exist. + # + if [ -d "$d" ]; then + local f + f="$(run find "$d" -path "$d/*/$pkg/package-owner.manifest")" + + if [ -n "$f" ]; then + trace "found: $f" + exit_with_manifest 401 "package owner authentication failed" + fi + fi +} + +# Return lower-case URL scheme or empty string if the argument doesn't look +# like a URL. +# +function url_scheme () # +{ + sed -n -e 's%^\(.*\)://.*$%\L\1%p' <<<"$1" +} + +# Checks that the repository properly responds to the probing request before +# the timeout (in seconds). Noop for protocols other than HTTP(S). +# +function check_connectivity () # +{ + trace_func "$@" + + local url="$1" + local tmo="$2" + + local s + s="$(url_scheme "$url")" + + if [ "$s" == "http" -o "$s" == "https" ]; then + local u q + + u="$(sed -n -e 's%^\([^?]*\).*$%\1%p' <<<"$url")" # Strips query part. + q="$(sed -n -e 's%^[^?]*\(.*\)$%\1%p' <<<"$url")" # Query part. + + if [ -z "$q" ]; then + u="$u/info/refs?service=git-upload-pack" + else + u="$u/info/refs$q&service=git-upload-pack" + fi + + # This function is called on repositories other than ours (e.g., control) + # so we don't want a failure to be logged. + # + if ! run_silent curl -S -s --max-time "$tmo" "$u" >/dev/null; then + exit_with_manifest 503 "submission service temporarily unavailable" + fi + fi +} diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in new file mode 100644 index 0000000..47f9a1a --- /dev/null +++ b/brep/handler/submit/submit-git.in @@ -0,0 +1,660 @@ +#!/usr/bin/env bash + +# file : brep/handler/submit/submit-git.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Package submission handler with git repository storage. +# +# The overall idea behind this handler is to add the package archive into a +# git repository. Another entity (for example, a human or a script) can then +# pull this change and do something about it (for example, review it and/or +# add it to an archive-based repository). In other words, git is used as a +# kind of transport that is easy enough to access for both humans and scripts. +# +# The handler also implements the project/package name ownership verification +# by performing the submitter authentication/authorization based on the +# control repository mechanism described in bdep-publish(1). This +# functionality is optional. +# +# The handler can operate with a single git repository, called "target", or +# with two git repositories, in which case the first is the target and the +# second is called "reference". The reference repository access is read-only +# and it is only consulted for duplicate package suppression and name +# ownership verification. The dual repository mode is normally used to +# implement a two-stage queue/public setup where the package is first queued +# for review and/or testing and then moved (for example, by a moderator) to a +# public repository. +# +# The target repository argument () should be a read-write git +# repository URL. It is cloned (shallow) into the submission directory on +# each submission. +# +# If specified, the reference repository argument () should be a +# directory with a pre-cloned read-only reference repository. This directory +# is shared between all instances of the handler. On each submission, the +# handler will flock(1) this directory, git-pull, obtain the information it +# needs, and release the lock. +# +# Both the target and, if specified, reference repositories should contain the +# submit.config.bash repository configuration file in the root directory. The +# configuration file is a bash fragment and is sourced by the handler script. +# It provides the following information: +# +# - Mapping of section names to repository subdirectories in the 'sections' +# variable (declare -A sections; values are relative to the repository +# root). + +# If there is no key for the submitted section name, then the entry with the +# special '*' key is used. If there is no such entry, then the submission is +# invalid. For example: +# +# sections[alpha]=1/alpha +# sections[beta]=1/beta +# sections[stable]=1/testing +# +# - Optional owners subdirectory in the 'owners' variable (relative to the +# repository root). If not specified, then no ownership verification is +# performed. For example: +# +# owners=owners +# +# If the ownership directory is specified, then the handler script maintains +# the project/package name ownership information in this directory. It has the +# following structure: +# +# / +# |-- / +# | |-- / +# | | `-- package-owner.manifest +# | |-- / +# | | `-- package-owner.manifest +# | |-- ... +# | `-- project-owner.manifest +# |-- / +# | `-- ... +# `-- ... +# +# If the submitted project name is not yet known, then the handler script +# creates a new project subdirectory and saves project-owner.manifest. The +# project owner manifest contains the following values in the specified order: +# +# name: +# author-name: +# author-email: +# control: +# +# The 'control' value is the control repository URL prefix and there can be +# multiple such values in a single manifest. The handler script derives it +# from the submitted control repository URL by removing the last path +# component. So, for example, https://github.com/build2/libbutl.git becomes +# https://github.com/build2/. +# +# If the submitted project name is already known, then the handler script +# loads its project-owner.manifest and verifies that at least one of the +# 'control' values is a prefix of the submitted control repository URL. +# +# Similarly, if the submitted package name is not yet known, then the handler +# script creates a new package subdirectory and saves package-owner.manifest. +# The package owner manifest contains the following values in the specified +# order: +# +# name: +# author-name: +# author-email: +# control: +# +# The 'control' value is the control repository URL and there can be multiple +# such values in a single manifest. +# +# If the submission package is already known, then the handler script loads +# its package-owner.manifest and verifies that at least one of the 'control' +# values matches the submitted control repository URL. +# +# If all these ownership authentication tests pass, the handler script clones +# (shallow) the build2-control branch of the control repository and verifies +# that the submission authorization file is present (see bdep-publish(1) for +# details). +# +# If the submission authorization test passes, then the handler script adds +# the package archives to the target repository, commits this change, and +# then pushes the commit to the remote. +# +# Notes: +# +# - It is possible that a submitted package name already exists in another +# project. In this case, such a submission is accepted only if the package +# already exists in the requested project. This allows the moderator to +# manually permit such multi-project packages (for example, to allow moving +# packages between projects). +# +# - There could be a race when moving package and ownership information from +# target to reference. To avoid it, the protocol for such a move is to first +# add, commit, and push to reference and then remove, commit, and push to +# target. +# +# On the handler side, the script acts in the opposite order cloning the +# target prior pulling the reference in order not to get into the situation +# where it misses the ownership info that is not in the reference yet but no +# longer in the target. Note that if some move happens after the cloning, +# then the script will be unable to push the target modification and will +# re-try the whole authentication procedure from scratch. +# +# - Filesystem entries that exist or are created in the data directory: +# +# -.tar.gz saved by brep (could be other archives in the future) +# request.manifest created by brep +# package.manifest extracted by the handler +# target/ cloned by the handler +# control/ cloned by the handler +# result.manifest saved by brep +# +# Options: +# +# --committer-name +# +# Name to use for the target repository commits. "Submission Handler" if +# unspecified. +# +# --committer-email +# +# Email to use for the target repository commits. noreply@example.com if +# unspecified. +# +usage="usage: $0 [] [] " + +# Diagnostics. +# +verbose= #true + +# Git network operations timeout (seconds). +# +# Note that we don't cover protocols other than HTTP(S) since for them git +# doesn't support any timeouts (though we may be able to cobble something +# up for SSH). +# +git_timeout=10 + +# The reference repository lock timeout (seconds). +# +ref_lock_timeout=30 + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +@import brep/handler/handler@ +@import brep/handler/submit/submit@ +@import brep/handler/submit/submit-git@ + +# Parse the command line options. +# +committer_name="Submission Handler" +committer_email="noreply@example.com" + +while [ $# -gt 0 ]; do + case $1 in + --committer-name) + shift + committer_name="$1" + shift + ;; + --committer-email) + shift + committer_email="$1" + shift + ;; + *) + break; # The end of options is encountered. + ;; + esac +done + +if [ -z "$committer_name" -o -z "$committer_email" ]; then + error "$usage" +fi + +# Parse the command line arguments. +# +if [ "$#" -lt 2 -o "$#" -gt 3 ]; then + error "$usage" +fi + +# Target repository URL. +# +tgt_repo="$1" +shift + +if [ -z "$tgt_repo" ]; then + error "$usage" +fi + +# Reference repository directory. +# +# Note that the last argument is always the submission data directory. +# +ref_repo= + +if [ "$#" -gt 1 ]; then + ref_repo="$1" + shift + + if [ -z "$ref_repo" ]; then + error "$usage" + fi + + if [ ! -d "$ref_repo" ]; then + error "'$ref_repo' does not exist or is not a directory" + fi +fi + +# Submission data directory. +# +data_dir="$1" +shift + +if [ -z "$data_dir" ]; then + error "$usage" +fi + +if [ ! -d "$data_dir" ]; then + error "'$data_dir' does not exist or is not a directory" +fi + +reference="$(basename "$data_dir")" + +# Git verbosity options. +# +# Note that not all git commands support the -q/-v options. Also note that +# these variable expansions should not be quoted. +# +if [ "$verbose" ]; then + gqo= + gvo="-v" +else + gqo="-q" + gvo= +fi + +# Git doesn't support the connection timeout option. The options we use are +# just an approximation of the former, that, in particular, don't cover the +# connection establishing. To work around this problem, before running a git +# command that assumes the remote repository communication we manually check +# connectivity with the remote repository. +# +git_http_timeout=("-c" "http.lowSpeedLimit=1" \ + "-c" "http.lowSpeedTime=$git_timeout") + +# Parse the submission request manifest and obtain the required values. +# +manifest_parser_start "$data_dir/request.manifest" + +archive= +sha256sum= +section= +author_name= +author_email= +control= +simulate= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + archive) archive="$v" ;; + sha256sum) sha256sum="$v" ;; + section) section="$v" ;; + author-name) author_name="$v" ;; + author-email) author_email="$v" ;; + control) control="$v" ;; + simulate) simulate="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$archive" ]; then + error "archive manifest value expected" +fi + +if [ -z "$sha256sum" ]; then + error "sha256sum manifest value expected" +fi + +if [ -n "$simulate" -a "$simulate" != "success" ]; then + exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" +fi + +# The author-* manifest values should both be present or absent. +# +if [ -z "$author_name" -a -n "$author_email" ]; then + exit_with_manifest 400 "author-name manifest value expected" +fi + +if [ -z "$author_email" -a -n "$author_name" ]; then + exit_with_manifest 400 "author-email manifest value expected" +fi + +# Note: checking for section, author-*, and control later. + +m="$data_dir/package.manifest" +extract_package_manifest "$data_dir/$archive" "$m" + +# Parse the package manifest and obtain the package name and version. +# +manifest_parser_start "$m" + +name= +version= +project= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + name) name="$v" ;; + version) version="$v" ;; + project) project="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$name" ]; then + error "name manifest value expected" +fi + +if [ -z "$version" ]; then + error "version manifest value expected" +fi + +if [ -z "$project" ]; then + project="$name" +fi + +function git_add () # ... +{ + local d="$1" + shift + + run git -C "$d" add $gvo "$@" >&2 +} + +function git_clone () # ... +{ + local url="$1" + shift + + local dir="$1" + shift + + check_connectivity "$url" "$git_timeout" + run git "${git_http_timeout[@]}" clone $gqo $gvo "$@" "$url" "$dir" >&2 +} + +# Dor now we make 10 re-tries to add the package and push to target. Push can +# fail due to the target-to-reference information move race (see the above +# notes for details) or because concurrent submissions. We may want to make it +# configurable in the future. +# +pkg_added= + +for i in {1..11}; do + + # Clone the target repository. + # + tgt_dir="$data_dir/target" + git_clone "$tgt_repo" "$tgt_dir" --single-branch --depth 1 + + check_package_duplicate "$name" "$version" "$tgt_dir" + + # Check for duplicates and try to authenticate the package ownership using + # information in the reference repository, if specified. + # + if [ -n "$ref_repo" ]; then + + remote_url=$(git -C "$ref_repo" config --get remote.origin.url) + + # Open the reading file descriptor and lock the repository. Fail if unable + # to lock before timeout. + # + l="$ref_repo/submit.config.bash" + trace "+ exec {fd}<$l" + exec {fd}<"$l" + + if ! run flock -w "$ref_lock_timeout" "$fd"; then + exit_with_manifest 503 "submission service temporarily unavailable" + fi + + # Pull the reference repository. + # + check_connectivity "$remote_url" "$git_timeout" + run git "${git_http_timeout[@]}" -C "$ref_repo" pull $gqo $gvo >&2 + + # Check the package duplicate. + # + check_package_duplicate "$name" "$version" "$ref_repo" + + # Authenticate the project ownership. + # + auth="$(auth_project "$project" "$control" "$ref_repo")" + + # Try to authenticate the package ownership if the project ownership was + # authenticated successfully. + # + if [ "$auth" == "project" ]; then + a="$(auth_package "$project" "$name" "$control" "$ref_repo")" + + # If the package is unknown to this project, we will try to authenticate + # the package name with the target repository later and so we keep the + # 'project' auth state. + # + if [ "$a" != "unknown" ]; then + auth="$a" + fi + fi + + trace "reference auth: $auth" + + if [ "${auth:0:1}" == ":" ]; then # Authentication error? + echo "$auth" + exit 0 + fi + + # If the package is not present in the specified project then we need to + # make sure it is also not present in any other project. + # + if [ "$auth" == "project" -o "$auth" == "unknown" ]; then + auth_package_unknown "$name" "$ref_repo" + fi + + trace "+ exec {fd}<&-" + exec {fd}<&- # Close the file descriptor and unlock the repository. + else + auth="disabled" + fi + + ref_auth="$auth" + + # Now try to authenticate the package ownership using information in the + # target repository unless already authenticated with reference. + # + if [ "$auth" != "package" ]; then + + # Don't authenticate the project ownership if this is already done with + # the reference repository. + # + if [ "$auth" != "project" ]; then + auth="$(auth_project "$project" "$control" "$tgt_dir")" + fi + + # Try to authenticate the package ownership if the project ownership was + # authenticated successfully. + # + if [ "$auth" == "project" ]; then + auth="$(auth_package "$project" "$name" "$control" "$tgt_dir")" + fi + + trace "target auth: $auth" + + if [ "${auth:0:1}" == ":" ]; then # Authentication error? + echo "$auth" + exit 0 + fi + fi + + trace "resulting auth: $auth" + + # Sanity check the auth variable value. + # + case "$auth" in + package) ;; + unknown) ;; + disabled) ;; + + *) error "unexpected resulting auth '$auth'";; + esac + + # Establish ownership of the package name unless already done. + # + if [ "$auth" == "unknown" ]; then + + # Check that the necessary request manifest values are specified. + # + if [ -z "$author_name" ]; then + exit_with_manifest 400 "author-name manifest value expected" + fi + + # Check that the package doesn't belong yet to some other project. + # + auth_package_unknown "$name" "$tgt_dir" + + # First the project name. + # + # Note that owners_dir() shouldn't return an empty string at this stage. + # + d="$(owners_dir "$tgt_dir")/$project" + + # Establish ownership of the project name unless already done. Note that + # it can only be owned by the submitter at this stage. + # + prj_man="$d/project-owner.manifest" + + if [ ! -f "$prj_man" ]; then + run mkdir -p "$d" # Also creates the owners directory if not exist. + + ctl="$(repository_base "$control")" + + create_owner_manifest \ + "$project" "$author_name" "$author_email" "$ctl" "$prj_man" + + # Add the project owners manifest file to git repository using the path + # relative to the repository directory. + # + git_add "$tgt_dir" "${prj_man#$tgt_dir/}" + fi + + # Now the package name. + # + d="$d/$name" + run mkdir "$d" + + pkg_man="$d/package-owner.manifest" + + create_owner_manifest \ + "$name" "$author_name" "$author_email" "$control" "$pkg_man" + + # Add the package owners manifest file using path relative to the + # repository directory. + # + git_add "$tgt_dir" "${pkg_man#$tgt_dir/}" + + auth="package" + fi + + # Respond with the 'unauthorized' manifest if we failed to authenticate the + # submitter as the package owner, unless both the reference and target + # repositories have the ownership authentication disabled. In the latter + # case no authorization is required. + # + if [ "$auth" != "disabled" -o "$ref_auth" != "disabled" ]; then + + # Respond with the 'unauthorized' manifest if not the package owner. + # + if [ "$auth" != "package" ]; then + if [ "$auth" == "project" -o "$ref_auth" == "project" ]; then + exit_with_manifest 401 "package owner authentication failed" + else + exit_with_manifest 401 "project owner authentication failed" + fi + fi + + # Authorize the submission. + # + ctl_dir="$data_dir/control" + + git_clone "$control" "$ctl_dir" --single-branch --depth 1 \ + --branch "build2-control" + + if [ ! -f "$ctl_dir/submit/${sha256sum:0:16}" ]; then + exit_with_manifest 401 "package publishing authorization failed" + fi + fi + + # Add the package archive to the target repository. + # + s="$(section_dir "$section" "$tgt_dir")" + + if [ -z "$s" ]; then + exit_with_manifest 400 "unrecognized section '$section'" + fi + + d="$tgt_dir/$s/$project" + run mkdir -p "$d" # Create all the parent directories as well. + + # We copy the archive rather than move it since we may need it for a re-try. + # + a="$d/$archive" + run cp "$data_dir/$archive" "$a" + + git_add "$tgt_dir" "${a#$tgt_dir/}" + + author=() + if [ -n "$author_name" ]; then + author=(--author="$author_name <$author_email>") + fi + + run git -c "user.name=$committer_name" -c "user.email=$committer_email" \ +-C "$tgt_dir" commit "${author[@]}" $gqo $gvo -F - <&2 +Add $name/$version to $s/$project + +$(cat "$data_dir/request.manifest") +EOF + + check_connectivity "$tgt_repo" "$git_timeout" + + # Try to push the target modifications, unless simulating. If this succeeds + # then we are done. Otherwise, drop the target directory and re-try the + # whole authentication/authorization procedure, unless we are out of + # attempts. + # + if [ -z "$simulate" ]; then + if run_silent git "${git_http_timeout[@]}" -C "$tgt_dir" push >&2; then + pkg_added=true + break + else + run rm -r -f "$tgt_dir" "$ctl_dir" + fi + fi +done + +if [ ! "$pkg_added" ]; then + exit_with_manifest 503 "submission service temporarily unavailable" +fi + +# Remove the no longer needed submission data directory. +# +run rm -r -f "$data_dir" + +if [ -n "$simulate" ]; then + trace "$name/$version submission is simulated" +else + trace "$name/$version submission is queued" +fi + +exit_with_manifest 200 "$name/$version submission is queued" diff --git a/brep/handler/submit/submit.bash.in b/brep/handler/submit/submit.bash.in new file mode 100644 index 0000000..38a4e06 --- /dev/null +++ b/brep/handler/submit/submit.bash.in @@ -0,0 +1,60 @@ +# file : brep/handler/submit/submit.bash.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Utility functions useful for implementing package submission handlers. + +if [ "$brep_handler_submit" ]; then + return 0 +else + brep_handler_submit=true +fi + +@import brep/handler/handler@ + +# Serialize the package submission result manifest to stdout and exit the +# (sub-)shell with the zero status. +# +reference= # Should be assigned later by the handler, when becomes available. + +function exit_with_manifest () # +{ + trace_func "$@" + + local sts="$1" + local msg="$2" + + manifest_serializer_start + + manifest_serialize "" "1" # Start of manifest. + manifest_serialize "status" "$sts" + manifest_serialize "message" "$msg" + + if [ -n "$reference" ]; then + manifest_serialize "reference" "$reference" + elif [ "$sts" == "200" ]; then + error "no reference for code $sts" + fi + + manifest_serializer_finish + run exit 0 +} + +# Verify archive is a valid package and extract its manifest into +# file. +# +function extract_package_manifest () # +{ + local arc="$1" + local man="$2" + + if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then + # Perform the sanity check to make sure that bpkg is runnable. + # + if ! run bpkg --version >/dev/null; then + error "unable to run bpkg" + fi + + exit_with_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)" + fi +} diff --git a/brep/submit/.gitignore b/brep/submit/.gitignore deleted file mode 100644 index ef91424..0000000 --- a/brep/submit/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -submit.bash -submit-git.bash - -brep-submit-dir -brep-submit-git diff --git a/brep/submit/buildfile b/brep/submit/buildfile deleted file mode 100644 index 50f9615..0000000 --- a/brep/submit/buildfile +++ /dev/null @@ -1,14 +0,0 @@ -# file : brep/submit/buildfile -# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -# license : MIT; see accompanying LICENSE file - -import mods = libbutl.bash%bash{manifest-parser} -import mods += libbutl.bash%bash{manifest-serializer} - -./: exe{brep-submit-dir} exe{brep-submit-git} - -exe{brep-submit-dir}: in{submit-dir} bash{submit} -exe{brep-submit-git}: in{submit-git} bash{submit-git} bash{submit} - -bash{submit}: in{submit} $mods -bash{submit-git}: in{submit-git} bash{submit} diff --git a/brep/submit/submit-dir.in b/brep/submit/submit-dir.in deleted file mode 100644 index 4bcbe5f..0000000 --- a/brep/submit/submit-dir.in +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash - -# file : brep/submit/submit-dir.in -# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -# license : MIT; see accompanying LICENSE file - -# Simple package submission handler with directory storage. -# -# Validate the package archive located in the specified submission directory -# extracting and parsing the package manifest (saved as package.manifest in -# the submission directory). Keep the submission directory unless simulating. -# Write the submission result manifest to stdout. -# -usage="usage: $0 " - -verbose= #true - -trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. - -@import brep/submit/submit@ - -if [ "$#" != 1 ]; then - error "$usage" -fi - -# Submission data directory (last and the only argument). -# -data_dir="${!#/}" - -if [ -z "$data_dir" ]; then - error "$usage" -fi - -if [ ! -d "$data_dir" ]; then - error "'$data_dir' does not exist or is not a directory" -fi - -reference="$(basename "$data_dir")" - -# Parse the submission request manifest and obtain the archive path as well -# as the simulate value. -# -manifest_parser_start "$data_dir/request.manifest" - -archive= -simulate= - -while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - case "$n" in - archive) archive="$v" ;; - simulate) simulate="$v" ;; - esac -done - -manifest_parser_finish - -if [ -z "$archive" ]; then - error "archive manifest value expected" -fi - -if [ -n "$simulate" -a "$simulate" != "success" ]; then - exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" -fi - -m="$data_dir/package.manifest" -extract_package_manifest "$data_dir/$archive" "$m" - -# Parse the package manifest and obtain the package name, version, and -# project. -# -manifest_parser_start "$m" - -name= -version= -project= - -while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - case "$n" in - name) name="$v" ;; - version) version="$v" ;; - project) project="$v" ;; - esac -done - -manifest_parser_finish - -if [ -z "$name" ]; then - error "name manifest value expected" -fi - -if [ -z "$version" ]; then - error "version manifest value expected" -fi - -if [ -z "$project" ]; then - project="$name" -fi - -if [ -n "$simulate" ]; then - rm -r "$data_dir" - trace "$name/$version submission is simulated" -else - trace "$name/$version submission is queued" -fi - -exit_with_manifest 200 "$name/$version submission is queued" "$reference" diff --git a/brep/submit/submit-git.bash.in b/brep/submit/submit-git.bash.in deleted file mode 100644 index 06d9b7c..0000000 --- a/brep/submit/submit-git.bash.in +++ /dev/null @@ -1,333 +0,0 @@ -# file : brep/submit/submit-git.bash.in -# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -# license : MIT; see accompanying LICENSE file - -# Utility functions for the submit-git handler. - -if [ "$brep_submit_git" ]; then - return 0 -else - brep_submit_git=true -fi - -@import brep/submit/submit@ - -# If the section is mapped to a directory in the repository configuration then -# return this directory path and empty string otherwise. -# -function section_dir () #
-{ - trace_func "$@" - - local sec="$1" - local rep="$2" - - local owners # Unused but is declared to avoid polluting the global space. - local -A sections - - run source "$rep/submit.config.bash" - - local r="${sections[$sec]}" - if [ -z "$r" ]; then - r="${sections['*']}" - fi - - echo "$r" -} - -# If the owners directory is set in the repository configuration then return -# this directory path prefixed with the repository directory path and the -# empty string otherwise. -# -function owners_dir () # -{ - local rep="$1" - - local owners - local -A sections # Is declared to avoid polluting the global space. - - run source "$rep/submit.config.bash" - - local r= - if [ -n "$owners" ]; then - r="$rep/$owners" - fi - - echo "$r" -} - -# Check if a repository already contains the package. Respond with the -# 'duplicate submission' result manifest and exit if that's the case. -# -function check_package_duplicate () # -{ - trace_func "$@" - - local nam="$1" - local ver="$2" - local rep="$3" - - local owners # Unused but is declared to avoid polluting the global space. - local -A sections - - run source "$rep/submit.config.bash" - - # Check for duplicate package in all sections. Use -.* - # without .tar.gz in case we want to support more archive types later. - # - local s - for s in "${!sections[@]}"; do - local d="$rep/${sections[$s]}" - - if [ -d "$d" ]; then - local f - f="$(run find "$d" -name "$nam-$ver.*")" - - if [ -n "$f" ]; then - trace "found: $f" - exit_with_manifest 422 "duplicate submission" - fi - fi - done -} - -# Serialize the project or package owner manifest (they have the same set of -# values) to the specified manifest file. -# -function create_owner_manifest () # -{ # - trace_func "$@" - - local nam="$1" - local anm="$2" - local aem="$3" - local ctl="$4" - local man="$5" - - if [ -f "$man" ]; then - error "'$man' already exists" - fi - - manifest_serializer_start "$man" - - manifest_serialize "" "1" # Start of manifest. - manifest_serialize "name" "$nam" - manifest_serialize "author-name" "$anm" - manifest_serialize "author-email" "$aem" - manifest_serialize "control" "$ctl" - - manifest_serializer_finish -} - -# Strip the query part and the leaf path component from the repository URL. -# The resulting URL contains the trailing slash. -# -function repository_base () # -{ - # First, strip the URL query part, then component. - # - sed -n \ --e 's%^\([^?]*\).*$%\1%' \ --e 's%^\(.*/\)[^/]\{1,\}/\{0,1\}$%\1%p' \ -<<<"$1" -} - -# Authenticate the project name owner. Make sure that the control manifest -# value is specified unless authentication is disabled. -# -# Possible return values: -# -# - 'project' if the project belongs to the submitter -# - 'unknown' if the project name is not yet known -# - 'disabled' if the owners directory is not configured -# - result manifest describing the authentication error -# -# Note that the authentication error result always starts with ':'. -# -function auth_project () # -{ - trace_func "$@" - - local prj="$1" - local ctl="$2" - local rep="$3" - - local d - d="$(owners_dir "$rep")" - - if [ -z "$d" ]; then - echo "disabled" - return - fi - - if [ -z "$ctl" ]; then - exit_with_manifest 400 "control manifest value expected" - fi - - local r="unknown" - local m="$d/$prj/project-owner.manifest" - - # If the project owner manifest exists then parse it and try to authenticate - # the submitter as the project owner. - # - if [ -f "$m" ]; then - - # Parse the project owner manifest. - # - manifest_parser_start "$m" - - local n v - while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - if [[ "$n" == "control" && "$ctl" == "$v"* ]]; then - r="project" - break - fi - done - - manifest_parser_finish - - if [ "$r" != "project" ]; then - exit_with_manifest 401 "project owner authentication failed" - fi - fi - - echo "$r" -} - -# Authenticate the package name owner. Make sure that the control manifest -# value is specified unless authentication is disabled. It is assumed that the -# project ownership is already authenticated (possibly by another repository). -# -# Possible return values: -# -# - 'package' if the package belongs to the submitter -# - 'unknown' if the package name is not taken in the project -# - 'disabled' if the owners directory is not configured -# - result manifest describing the authentication error -# -# Note that the authentication error result always starts with ':'. -# -function auth_package () # -{ - trace_func "$@" - - local prj="$1" - local pkg="$2" - local ctl="$3" - local rep="$4" - - local d - d="$(owners_dir "$rep")" - - if [ -z "$d" ]; then - echo "disabled" - return - fi - - if [ -z "$ctl" ]; then - exit_with_manifest 400 "control manifest value expected" - fi - - local r="unknown" - local m="$d/$prj/$pkg/package-owner.manifest" - - # If the package owner manifest exists then parse it and try to authenticate - # the submitter as the package owner. - # - if [ -f "$m" ]; then - - # Parse the package owner manifest. - # - manifest_parser_start "$m" - - local n v - while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - if [ "$n" == "control" -a "$v" == "$ctl" ]; then - r="package" - break - fi - done - - manifest_parser_finish - - if [ "$r" != "package" ]; then - exit_with_manifest 401 "package owner authentication failed" - fi - fi - - echo "$r" -} - -# Check that the package name is unknown to the repository. Owners directory -# is expected to be configured. -# -function auth_package_unknown () # -{ - trace_func "$@" - - local pkg="$1" - local rep="$2" - - local d - d="$(owners_dir "$rep")" - - # Sanity check that the owners directory configured for the repository. - # - if [ -z "$d" ]; then - error "no owners directory configured for '$rep'" - fi - - # While configured, the owners directory may not yet exist. - # - if [ -d "$d" ]; then - local f - f="$(run find "$d" -path "$d/*/$pkg/package-owner.manifest")" - - if [ -n "$f" ]; then - trace "found: $f" - exit_with_manifest 401 "package owner authentication failed" - fi - fi -} - -# Return lower-case URL scheme or empty string if the argument doesn't look -# like a URL. -# -function url_scheme () # -{ - sed -n -e 's%^\(.*\)://.*$%\L\1%p' <<<"$1" -} - -# Checks that the repository properly responds to the probing request before -# the timeout (in seconds). Noop for protocols other than HTTP(S). -# -function check_connectivity () # -{ - trace_func "$@" - - local url="$1" - local tmo="$2" - - local s - s="$(url_scheme "$url")" - - if [ "$s" == "http" -o "$s" == "https" ]; then - local u q - - u="$(sed -n -e 's%^\([^?]*\).*$%\1%p' <<<"$url")" # Strips query part. - q="$(sed -n -e 's%^[^?]*\(.*\)$%\1%p' <<<"$url")" # Query part. - - if [ -z "$q" ]; then - u="$u/info/refs?service=git-upload-pack" - else - u="$u/info/refs$q&service=git-upload-pack" - fi - - # This function is called on repositories other than ours (e.g., control) - # so we don't want a failure to be logged. - # - if ! run_silent curl -S -s --max-time "$tmo" "$u" >/dev/null; then - exit_with_manifest 503 "submission service temporarily unavailable" - fi - fi -} diff --git a/brep/submit/submit-git.in b/brep/submit/submit-git.in deleted file mode 100644 index 0c90309..0000000 --- a/brep/submit/submit-git.in +++ /dev/null @@ -1,659 +0,0 @@ -#!/usr/bin/env bash - -# file : brep/submit/submit-git.in -# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -# license : MIT; see accompanying LICENSE file - -# Package submission handler with git repository storage. -# -# The overall idea behind this handler is to add the package archive into a -# git repository. Another entity (for example, a human or a script) can then -# pull this change and do something about it (for example, review it and/or -# add it to an archive-based repository). In other words, git is used as a -# kind of transport that is easy enough to access for both humans and scripts. -# -# The handler also implements the project/package name ownership verification -# by performing the submitter authentication/authorization based on the -# control repository mechanism described in bdep-publish(1). This -# functionality is optional. -# -# The handler can operate with a single git repository, called "target", or -# with two git repositories, in which case the first is the target and the -# second is called "reference". The reference repository access is read-only -# and it is only consulted for duplicate package suppression and name -# ownership verification. The dual repository mode is normally used to -# implement a two-stage queue/public setup where the package is first queued -# for review and/or testing and then moved (for example, by a moderator) to a -# public repository. -# -# The target repository argument () should be a read-write git -# repository URL. It is cloned (shallow) into the submission directory on -# each submission. -# -# If specified, the reference repository argument () should be a -# directory with a pre-cloned read-only reference repository. This directory -# is shared between all instances of the handler. On each submission, the -# handler will flock(1) this directory, git-pull, obtain the information it -# needs, and release the lock. -# -# Both the target and, if specified, reference repositories should contain the -# submit.config.bash repository configuration file in the root directory. The -# configuration file is a bash fragment and is sourced by the handler script. -# It provides the following information: -# -# - Mapping of section names to repository subdirectories in the 'sections' -# variable (declare -A sections; values are relative to the repository -# root). - -# If there is no key for the submitted section name, then the entry with the -# special '*' key is used. If there is no such entry, then the submission is -# invalid. For example: -# -# sections[alpha]=1/alpha -# sections[beta]=1/beta -# sections[stable]=1/testing -# -# - Optional owners subdirectory in the 'owners' variable (relative to the -# repository root). If not specified, then no ownership verification is -# performed. For example: -# -# owners=owners -# -# If the ownership directory is specified, then the handler script maintains -# the project/package name ownership information in this directory. It has the -# following structure: -# -# / -# |-- / -# | |-- / -# | | `-- package-owner.manifest -# | |-- / -# | | `-- package-owner.manifest -# | |-- ... -# | `-- project-owner.manifest -# |-- / -# | `-- ... -# `-- ... -# -# If the submitted project name is not yet known, then the handler script -# creates a new project subdirectory and saves project-owner.manifest. The -# project owner manifest contains the following values in the specified order: -# -# name: -# author-name: -# author-email: -# control: -# -# The 'control' value is the control repository URL prefix and there can be -# multiple such values in a single manifest. The handler script derives it -# from the submitted control repository URL by removing the last path -# component. So, for example, https://github.com/build2/libbutl.git becomes -# https://github.com/build2/. -# -# If the submitted project name is already known, then the handler script -# loads its project-owner.manifest and verifies that at least one of the -# 'control' values is a prefix of the submitted control repository URL. -# -# Similarly, if the submitted package name is not yet known, then the handler -# script creates a new package subdirectory and saves package-owner.manifest. -# The package owner manifest contains the following values in the specified -# order: -# -# name: -# author-name: -# author-email: -# control: -# -# The 'control' value is the control repository URL and there can be multiple -# such values in a single manifest. -# -# If the submission package is already known, then the handler script loads -# its package-owner.manifest and verifies that at least one of the 'control' -# values matches the submitted control repository URL. -# -# If all these ownership authentication tests pass, the handler script clones -# (shallow) the build2-control branch of the control repository and verifies -# that the submission authorization file is present (see bdep-publish(1) for -# details). -# -# If the submission authorization test passes, then the handler script adds -# the package archives to the target repository, commits this change, and -# then pushes the commit to the remote. -# -# Notes: -# -# - It is possible that a submitted package name already exists in another -# project. In this case, such a submission is accepted only if the package -# already exists in the requested project. This allows the moderator to -# manually permit such multi-project packages (for example, to allow moving -# packages between projects). -# -# - There could be a race when moving package and ownership information from -# target to reference. To avoid it, the protocol for such a move is to first -# add, commit, and push to reference and then remove, commit, and push to -# target. -# -# On the handler side, the script acts in the opposite order cloning the -# target prior pulling the reference in order not to get into the situation -# where it misses the ownership info that is not in the reference yet but no -# longer in the target. Note that if some move happens after the cloning, -# then the script will be unable to push the target modification and will -# re-try the whole authentication procedure from scratch. -# -# - Filesystem entries that exist or are created in the data directory: -# -# -.tar.gz saved by brep (could be other archives in the future) -# request.manifest created by brep -# package.manifest extracted by the handler -# target/ cloned by the handler -# control/ cloned by the handler -# result.manifest saved by brep -# -# Options: -# -# --committer-name -# -# Name to use for the target repository commits. "Submission Handler" if -# unspecified. -# -# --committer-email -# -# Email to use for the target repository commits. noreply@example.com if -# unspecified. -# -usage="usage: $0 [] [] " - -# Diagnostics. -# -verbose= #true - -# Git network operations timeout (seconds). -# -# Note that we don't cover protocols other than HTTP(S) since for them git -# doesn't support any timeouts (though we may be able to cobble something -# up for SSH). -# -git_timeout=10 - -# The reference repository lock timeout (seconds). -# -ref_lock_timeout=30 - -trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. - -@import brep/submit/submit@ -@import brep/submit/submit-git@ - -# Parse the command line options. -# -committer_name="Submission Handler" -committer_email="noreply@example.com" - -while [ $# -gt 0 ]; do - case $1 in - --committer-name) - shift - committer_name="$1" - shift - ;; - --committer-email) - shift - committer_email="$1" - shift - ;; - *) - break; # The end of options is encountered. - ;; - esac -done - -if [ -z "$committer_name" -o -z "$committer_email" ]; then - error "$usage" -fi - -# Parse the command line arguments. -# -if [ "$#" -lt 2 -o "$#" -gt 3 ]; then - error "$usage" -fi - -# Target repository URL. -# -tgt_repo="$1" -shift - -if [ -z "$tgt_repo" ]; then - error "$usage" -fi - -# Reference repository directory. -# -# Note that the last argument is always the submission data directory. -# -ref_repo= - -if [ "$#" -gt 1 ]; then - ref_repo="$1" - shift - - if [ -z "$ref_repo" ]; then - error "$usage" - fi - - if [ ! -d "$ref_repo" ]; then - error "'$ref_repo' does not exist or is not a directory" - fi -fi - -# Submission data directory. -# -data_dir="$1" -shift - -if [ -z "$data_dir" ]; then - error "$usage" -fi - -if [ ! -d "$data_dir" ]; then - error "'$data_dir' does not exist or is not a directory" -fi - -reference="$(basename "$data_dir")" - -# Git verbosity options. -# -# Note that not all git commands support the -q/-v options. Also note that -# these variable expansions should not be quoted. -# -if [ "$verbose" ]; then - gqo= - gvo="-v" -else - gqo="-q" - gvo= -fi - -# Git doesn't support the connection timeout option. The options we use are -# just an approximation of the former, that, in particular, don't cover the -# connection establishing. To work around this problem, before running a git -# command that assumes the remote repository communication we manually check -# connectivity with the remote repository. -# -git_http_timeout=("-c" "http.lowSpeedLimit=1" \ - "-c" "http.lowSpeedTime=$git_timeout") - -# Parse the submission request manifest and obtain the required values. -# -manifest_parser_start "$data_dir/request.manifest" - -archive= -sha256sum= -section= -author_name= -author_email= -control= -simulate= - -while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - case "$n" in - archive) archive="$v" ;; - sha256sum) sha256sum="$v" ;; - section) section="$v" ;; - author-name) author_name="$v" ;; - author-email) author_email="$v" ;; - control) control="$v" ;; - simulate) simulate="$v" ;; - esac -done - -manifest_parser_finish - -if [ -z "$archive" ]; then - error "archive manifest value expected" -fi - -if [ -z "$sha256sum" ]; then - error "sha256sum manifest value expected" -fi - -if [ -n "$simulate" -a "$simulate" != "success" ]; then - exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" -fi - -# The author-* manifest values should both be present or absent. -# -if [ -z "$author_name" -a -n "$author_email" ]; then - exit_with_manifest 400 "author-name manifest value expected" -fi - -if [ -z "$author_email" -a -n "$author_name" ]; then - exit_with_manifest 400 "author-email manifest value expected" -fi - -# Note: checking for section, author-*, and control later. - -m="$data_dir/package.manifest" -extract_package_manifest "$data_dir/$archive" "$m" - -# Parse the package manifest and obtain the package name and version. -# -manifest_parser_start "$m" - -name= -version= -project= - -while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - case "$n" in - name) name="$v" ;; - version) version="$v" ;; - project) project="$v" ;; - esac -done - -manifest_parser_finish - -if [ -z "$name" ]; then - error "name manifest value expected" -fi - -if [ -z "$version" ]; then - error "version manifest value expected" -fi - -if [ -z "$project" ]; then - project="$name" -fi - -function git_add () # ... -{ - local d="$1" - shift - - run git -C "$d" add $gvo "$@" >&2 -} - -function git_clone () # ... -{ - local url="$1" - shift - - local dir="$1" - shift - - check_connectivity "$url" "$git_timeout" - run git "${git_http_timeout[@]}" clone $gqo $gvo "$@" "$url" "$dir" >&2 -} - -# Dor now we make 10 re-tries to add the package and push to target. Push can -# fail due to the target-to-reference information move race (see the above -# notes for details) or because concurrent submissions. We may want to make it -# configurable in the future. -# -pkg_added= - -for i in {1..11}; do - - # Clone the target repository. - # - tgt_dir="$data_dir/target" - git_clone "$tgt_repo" "$tgt_dir" --single-branch --depth 1 - - check_package_duplicate "$name" "$version" "$tgt_dir" - - # Check for duplicates and try to authenticate the package ownership using - # information in the reference repository, if specified. - # - if [ -n "$ref_repo" ]; then - - remote_url=$(git -C "$ref_repo" config --get remote.origin.url) - - # Open the reading file descriptor and lock the repository. Fail if unable - # to lock before timeout. - # - l="$ref_repo/submit.config.bash" - trace "+ exec {fd}<$l" - exec {fd}<"$l" - - if ! run flock -w "$ref_lock_timeout" "$fd"; then - exit_with_manifest 503 "submission service temporarily unavailable" - fi - - # Pull the reference repository. - # - check_connectivity "$remote_url" "$git_timeout" - run git "${git_http_timeout[@]}" -C "$ref_repo" pull $gqo $gvo >&2 - - # Check the package duplicate. - # - check_package_duplicate "$name" "$version" "$ref_repo" - - # Authenticate the project ownership. - # - auth="$(auth_project "$project" "$control" "$ref_repo")" - - # Try to authenticate the package ownership if the project ownership was - # authenticated successfully. - # - if [ "$auth" == "project" ]; then - a="$(auth_package "$project" "$name" "$control" "$ref_repo")" - - # If the package is unknown to this project, we will try to authenticate - # the package name with the target repository later and so we keep the - # 'project' auth state. - # - if [ "$a" != "unknown" ]; then - auth="$a" - fi - fi - - trace "reference auth: $auth" - - if [ "${auth:0:1}" == ":" ]; then # Authentication error? - echo "$auth" - exit 0 - fi - - # If the package is not present in the specified project then we need to - # make sure it is also not present in any other project. - # - if [ "$auth" == "project" -o "$auth" == "unknown" ]; then - auth_package_unknown "$name" "$ref_repo" - fi - - trace "+ exec {fd}<&-" - exec {fd}<&- # Close the file descriptor and unlock the repository. - else - auth="disabled" - fi - - ref_auth="$auth" - - # Now try to authenticate the package ownership using information in the - # target repository unless already authenticated with reference. - # - if [ "$auth" != "package" ]; then - - # Don't authenticate the project ownership if this is already done with - # the reference repository. - # - if [ "$auth" != "project" ]; then - auth="$(auth_project "$project" "$control" "$tgt_dir")" - fi - - # Try to authenticate the package ownership if the project ownership was - # authenticated successfully. - # - if [ "$auth" == "project" ]; then - auth="$(auth_package "$project" "$name" "$control" "$tgt_dir")" - fi - - trace "target auth: $auth" - - if [ "${auth:0:1}" == ":" ]; then # Authentication error? - echo "$auth" - exit 0 - fi - fi - - trace "resulting auth: $auth" - - # Sanity check the auth variable value. - # - case "$auth" in - package) ;; - unknown) ;; - disabled) ;; - - *) error "unexpected resulting auth '$auth'";; - esac - - # Establish ownership of the package name unless already done. - # - if [ "$auth" == "unknown" ]; then - - # Check that the necessary request manifest values are specified. - # - if [ -z "$author_name" ]; then - exit_with_manifest 400 "author-name manifest value expected" - fi - - # Check that the package doesn't belong yet to some other project. - # - auth_package_unknown "$name" "$tgt_dir" - - # First the project name. - # - # Note that owners_dir() shouldn't return an empty string at this stage. - # - d="$(owners_dir "$tgt_dir")/$project" - - # Establish ownership of the project name unless already done. Note that - # it can only be owned by the submitter at this stage. - # - prj_man="$d/project-owner.manifest" - - if [ ! -f "$prj_man" ]; then - run mkdir -p "$d" # Also creates the owners directory if not exist. - - ctl="$(repository_base "$control")" - - create_owner_manifest \ - "$project" "$author_name" "$author_email" "$ctl" "$prj_man" - - # Add the project owners manifest file to git repository using the path - # relative to the repository directory. - # - git_add "$tgt_dir" "${prj_man#$tgt_dir/}" - fi - - # Now the package name. - # - d="$d/$name" - run mkdir "$d" - - pkg_man="$d/package-owner.manifest" - - create_owner_manifest \ - "$name" "$author_name" "$author_email" "$control" "$pkg_man" - - # Add the package owners manifest file using path relative to the - # repository directory. - # - git_add "$tgt_dir" "${pkg_man#$tgt_dir/}" - - auth="package" - fi - - # Respond with the 'unauthorized' manifest if we failed to authenticate the - # submitter as the package owner, unless both the reference and target - # repositories have the ownership authentication disabled. In the latter - # case no authorization is required. - # - if [ "$auth" != "disabled" -o "$ref_auth" != "disabled" ]; then - - # Respond with the 'unauthorized' manifest if not the package owner. - # - if [ "$auth" != "package" ]; then - if [ "$auth" == "project" -o "$ref_auth" == "project" ]; then - exit_with_manifest 401 "package owner authentication failed" - else - exit_with_manifest 401 "project owner authentication failed" - fi - fi - - # Authorize the submission. - # - ctl_dir="$data_dir/control" - - git_clone "$control" "$ctl_dir" --single-branch --depth 1 \ - --branch "build2-control" - - if [ ! -f "$ctl_dir/submit/${sha256sum:0:16}" ]; then - exit_with_manifest 401 "package publishing authorization failed" - fi - fi - - # Add the package archive to the target repository. - # - s="$(section_dir "$section" "$tgt_dir")" - - if [ -z "$s" ]; then - exit_with_manifest 400 "unrecognized section '$section'" - fi - - d="$tgt_dir/$s/$project" - run mkdir -p "$d" # Create all the parent directories as well. - - # We copy the archive rather than move it since we may need it for a re-try. - # - a="$d/$archive" - run cp "$data_dir/$archive" "$a" - - git_add "$tgt_dir" "${a#$tgt_dir/}" - - author=() - if [ -n "$author_name" ]; then - author=(--author="$author_name <$author_email>") - fi - - run git -c "user.name=$committer_name" -c "user.email=$committer_email" \ --C "$tgt_dir" commit "${author[@]}" $gqo $gvo -F - <&2 -Add $name/$version to $s/$project - -$(cat "$data_dir/request.manifest") -EOF - - check_connectivity "$tgt_repo" "$git_timeout" - - # Try to push the target modifications, unless simulating. If this succeeds - # then we are done. Otherwise, drop the target directory and re-try the - # whole authentication/authorization procedure, unless we are out of - # attempts. - # - if [ -z "$simulate" ]; then - if run_silent git "${git_http_timeout[@]}" -C "$tgt_dir" push >&2; then - pkg_added=true - break - else - run rm -r -f "$tgt_dir" "$ctl_dir" - fi - fi -done - -if [ ! "$pkg_added" ]; then - exit_with_manifest 503 "submission service temporarily unavailable" -fi - -# Remove the no longer needed submission data directory. -# -run rm -r -f "$data_dir" - -if [ -n "$simulate" ]; then - trace "$name/$version submission is simulated" -else - trace "$name/$version submission is queued" -fi - -exit_with_manifest 200 "$name/$version submission is queued" "$reference" diff --git a/brep/submit/submit.bash.in b/brep/submit/submit.bash.in deleted file mode 100644 index babf081..0000000 --- a/brep/submit/submit.bash.in +++ /dev/null @@ -1,197 +0,0 @@ -# file : brep/submit/submit.bash.in -# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd -# license : MIT; see accompanying LICENSE file - -# Utility functions useful for implementing submission handlers. - -if [ "$brep_submit" ]; then - return 0 -else - brep_submit=true -fi - -@import libbutl/manifest-parser@ -@import libbutl/manifest-serializer@ - -# Diagnostics. -# -# We expect the user to set the verbose variable either to true or empty -# (false). -# -if [ ! -v verbose ]; then - echo "error: variable 'verbose' is not set" >&2 - exit 1 -fi - -# Normally the brep module's log record looks like this: -# -# [Mon Jul 23 17:48:46.945079 2018] [brep:error] [pid 123:tid 456] [brep::submit::init]: error description -# -# We will use the (almost) same format for our diagnostics (redirected to the -# Apache's error_log), so it can easily be attributed to the brep module. -# -info_self="$(basename $0)" - -if [ "$#" -gt 0 ]; then - info_ref="$(basename "${!#/}")" # Last argument is the submission directory. -fi - -function info () # -{ - local severity="$1" - shift - - # Note: %N is Linux-specific. - # - local ts - if ! ts="$(date +"%a %b %d %H:%M:%S.%6N %Y")"; then - ts= - fi - - echo "[$ts] [brep:$severity] [ref $info_ref] [$info_self]: $*" 1>&2; -} - -function error () { info "error" "$*"; exit 1; } - -function trace () { if [ "$verbose" ]; then info "info" "$*"; fi } - -# Trace a command line, quoting empty arguments as well as those that contain -# spaces. -# -function trace_cmd () # ... -{ - if [ "$verbose" ]; then - local s="+" - while [ $# -gt 0 ]; do - if [ -z "$1" -o -z "${1##* *}" ]; then - s="$s '$1'" - else - s="$s $1" - fi - - shift - done - - info "info" "$s" - fi -} - -# Trace the current function name and arguments. -# -function trace_func () # ... -{ - trace_cmd "${FUNCNAME[1]}" "$@" -} - -# Trace and run a command. -# -function run () # ... -{ - trace_cmd "$@" - "$@" -} - -# Same as above but also redirect the command stderr to /dev/null, unless -# running in the verbose mode. -# -# Note that we don't redirect stdout, so it can still be captured. -# -function run_silent () # ... -{ - trace_cmd "$@" - - if [ "$verbose" ]; then - "$@" - else - "$@" 2>/dev/null - fi -} - -# Wrap libbutl manifest parsing/serializing functions to shorten names and to -# add tracing. -# -function manifest_parser_start () # [] -{ - trace_func "$@" - butl_manifest_parser_start "$@" - - manifest_parser_ofd="$butl_manifest_parser_ofd" -} - -function manifest_parser_finish () -{ - trace_func - butl_manifest_parser_finish -} - -function manifest_serializer_start () # [] -{ - trace_func "$@" - butl_manifest_serializer_start "$@" - - manifest_serializer_ifd="$butl_manifest_serializer_ifd" -} - -function manifest_serializer_finish () -{ - trace_func - butl_manifest_serializer_finish -} - -# Serialize one manifest name/value pair. -# -function manifest_serialize () # -{ -# trace "$1: $2" - printf "%s:%s\0" "$1" "$2" >&"$manifest_serializer_ifd" -} - -# Serialize the submission result manifest to stdout and exit the (sub-)shell -# with the zero status. -# -function exit_with_manifest () # [] -{ - trace_func "$@" - - local sts="$1" - local msg="$2" - local ref="$3" - - manifest_serializer_start - - manifest_serialize "" "1" # Start of manifest. - manifest_serialize "status" "$sts" - manifest_serialize "message" "$msg" - - if [ -n "$ref" ]; then - if [ "$sts" != "200" ]; then - error "reference for code $sts" - fi - - manifest_serialize "reference" "$ref" - elif [ "$sts" == "200" ]; then - error "no reference for code $sts" - fi - - manifest_serializer_finish - run exit 0 -} - -# Verify archive is a valid package and extract its manifest into -# file. -# -function extract_package_manifest () # -{ - local arc="$1" - local man="$2" - - if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then - # Perform the sanity check to make sure that bpkg is runnable. - # - if ! run bpkg --version >/dev/null; then - error "unable to run bpkg" - fi - - exit_with_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)" - fi -} diff --git a/mod/mod-submit.cxx b/mod/mod-submit.cxx index da698bc..61eeaf6 100644 --- a/mod/mod-submit.cxx +++ b/mod/mod-submit.cxx @@ -140,9 +140,9 @@ handle (request& rq, response& rs) // // return respond_error (); // Request is handled with an error. // - auto respond_manifest = [&rs] (status_code status, - const string& message, - const char* ref = nullptr) -> bool + string ref; // Will be set later. + auto respond_manifest = [&rs, &ref] (status_code status, + const string& message) -> bool { serializer s (rs.content (status, "text/manifest;charset=utf-8"), "response"); @@ -151,7 +151,7 @@ handle (request& rq, response& rs) s.next ("status", to_string (status)); s.next ("message", message); - if (ref != nullptr) + if (!ref.empty ()) s.next ("reference", ref); s.next ("", ""); // End of manifest. @@ -285,12 +285,16 @@ handle (request& rq, response& rs) return respond_manifest (400, "invalid parameter " + nv.name); } + // Note that from now on the result manifest will contain the reference + // value. + // + ref = string (sha256sum, 0, 12); + // Check for a duplicate submission. // // Respond with the unprocessable entity (422) code if a duplicate is found. // - string ac (sha256sum, 0, 12); - dir_path dd (options_->submit_data () / dir_path (ac)); + dir_path dd (options_->submit_data () / dir_path (ref)); if (dir_exists (dd) || simulate == "duplicate-archive") return respond_manifest (422, "duplicate submission"); @@ -306,7 +310,7 @@ handle (request& rq, response& rs) // using the abbreviated checksum can be helpful for troubleshooting. // td = dir_path (options_->submit_temp () / - dir_path (path::traits::temp_name (ac))); + dir_path (path::traits::temp_name (ref))); // It's highly unlikely but still possible that the temporary directory // already exists. This can only happen due to the unclean web server @@ -624,11 +628,11 @@ handle (request& rq, response& rs) dd)); pipe.out.close (); - auto kill = [&pr, &warn, &handler, &ac] () + auto kill = [&pr, &warn, &handler, &ref] () { // We may still end up well (see below), thus this is a warning. // - warn << "ref " << ac << ": process " << handler + warn << "ref " << ref << ": process " << handler << " execution timeout expired"; pr.kill (); @@ -716,7 +720,7 @@ handle (request& rq, response& rs) is.close (); - warn << "ref " << ac << ": process " << handler + warn << "ref " << ref << ": process " << handler << " stdout is not closed after termination (possibly " << "handler's child still running)"; } @@ -781,14 +785,14 @@ handle (request& rq, response& rs) if (*pr.exit) break; // Get out of the breakout loop. - error << "ref " << ac << ": process " << handler << " " << *pr.exit; + error << "ref " << ref << ": process " << handler << " " << *pr.exit; // Fall through. } catch (const io_error& e) { if (pr.wait ()) - error << "ref " << ac << ": unable to read handler's output: " << e; + error << "ref " << ref << ": unable to read handler's output: " << e; // Fall through. } @@ -861,7 +865,7 @@ handle (request& rq, response& rs) } catch (const parsing& e) { - error << "ref " << ac << ": unable to parse handler's output: " << e; + error << "ref " << ref << ": unable to parse handler's output: " << e; // It appears the handler had misbehaved, so let's stash the submission // directory for troubleshooting. @@ -885,7 +889,7 @@ handle (request& rq, response& rs) add ("", "1"); // Start of manifest. add ("status", "200"); add ("message", "submission is queued"); - add ("reference", ac); + add ("reference", ref); add ("", ""); // End of manifest. } @@ -895,7 +899,7 @@ handle (request& rq, response& rs) // serialization error log the error description and return false, on the // stream error pass through the io_error exception, otherwise return true. // - auto rsm = [&rvs, &error, &ac] (ostream& os) -> bool + auto rsm = [&rvs, &error, &ref] (ostream& os) -> bool { try { @@ -907,7 +911,7 @@ handle (request& rq, response& rs) } catch (const serialization& e) { - error << "ref " << ac << ": unable to serialize handler's output: " << e; + error << "ref " << ref << ": unable to serialize handler's output: " << e; return false; } }; @@ -971,7 +975,7 @@ handle (request& rq, response& rs) sendmail sm (print_args, 2 /* stderr */, options_->email (), - "new package submission " + a.string () + " (" + ac + ")", + "new package submission " + a.string () + " (" + ref + ")", {options_->submit_email ()}); // Write the submission request manifest. diff --git a/tests/submit/buildfile b/tests/submit/buildfile index 6606153..46e38ad 100644 --- a/tests/submit/buildfile +++ b/tests/submit/buildfile @@ -5,7 +5,7 @@ define common: file common{*}: extension = test -dir = ../../brep/submit/ +dir = ../../brep/handler/submit/ commons = data diff --git a/tests/submit/submit-dir.test b/tests/submit/submit-dir.test index 97f7edd..7fa7341 100644 --- a/tests/submit/submit-dir.test +++ b/tests/submit/submit-dir.test @@ -67,10 +67,11 @@ echo "junk" >=$checksum/libhello-0.1.0.tar.gz; - $* >>EOO + $* >>"EOO" : 1 status: 400 - message: archive is not a valid package (run bpkg pkg-verify for details) + message: archive is not a valid package \(run bpkg pkg-verify for details\) + reference: $checksum EOO } @@ -85,6 +86,7 @@ : 1 status: 400 message: unrecognized simulation outcome 'fly' + reference: $checksum EOO } } diff --git a/tests/submit/submit-git.test b/tests/submit/submit-git.test index a64e9a1..5bfa4d4 100644 --- a/tests/submit/submit-git.test +++ b/tests/submit/submit-git.test @@ -466,6 +466,7 @@ pkg_ctl="$prj_ctl/hello.git" : 1 status: 400 message: author-name manifest value expected + reference: $checksum EOO } @@ -482,6 +483,7 @@ pkg_ctl="$prj_ctl/hello.git" : 1 status: 400 message: author-email manifest value expected + reference: $checksum EOO } @@ -498,6 +500,7 @@ pkg_ctl="$prj_ctl/hello.git" : 1 status: 400 message: author-name manifest value expected + reference: $checksum EOO } @@ -519,10 +522,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C ref commit -m 'Add libhello-0.1.0.tar.gz'; $g -C ref push; - $* "$root_tgt_url" $~/ref $data_dir >>EOO + $* "$root_tgt_url" $~/ref $data_dir >>"EOO" : 1 status: 422 message: duplicate submission + reference: $checksum EOO } @@ -559,10 +563,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C ref commit -m 'Add ownership info'; $g -C ref push; - $* "$root_tgt_url" $~/ref $data_dir >>EOO + $* "$root_tgt_url" $~/ref $data_dir >>"EOO" : 1 status: 401 message: package owner authentication failed + reference: $checksum EOO } @@ -590,10 +595,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C ref commit -m 'Add project ownership info'; $g -C ref push; - $* "$root_tgt_url" $~/ref $data_dir >>EOO + $* "$root_tgt_url" $~/ref $data_dir >>"EOO" : 1 status: 401 message: project owner authentication failed + reference: $checksum EOO } @@ -630,10 +636,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C ref commit -m 'Add ownership info'; $g -C ref push; - $* "$root_tgt_url" $~/ref $data_dir >>EOO + $* "$root_tgt_url" $~/ref $data_dir >>"EOO" : 1 status: 401 message: package owner authentication failed + reference: $checksum EOO } @@ -655,10 +662,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C tgt commit -m 'Add libhello-0.1.0.tar.gz'; $g -C tgt push; - $* "file:///$~/tgt.git" $data_dir >>EOO + $* "file:///$~/tgt.git" $data_dir >>"EOO" : 1 status: 422 message: duplicate submission + reference: $checksum EOO } @@ -694,10 +702,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C tgt commit -m 'Add ownership info'; $g -C tgt push; - $* "file:///$~/tgt.git" $data_dir >>EOO + $* "file:///$~/tgt.git" $data_dir >>"EOO" : 1 status: 401 message: package owner authentication failed + reference: $checksum EOO } @@ -724,10 +733,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C tgt commit -am 'Disable ownership'; $g -C tgt push; - $* "file:///$~/tgt.git" $root_ref_dir $data_dir >>EOO + $* "file:///$~/tgt.git" $root_ref_dir $data_dir >>"EOO" : 1 status: 401 message: project owner authentication failed + reference: $checksum EOO } @@ -771,10 +781,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C tgt commit -am 'Disable ownership'; $g -C tgt push; - $* "file:///$~/tgt.git" ref $data_dir >>EOO + $* "file:///$~/tgt.git" ref $data_dir >>"EOO" : 1 status: 401 message: package owner authentication failed + reference: $checksum EOO } @@ -811,10 +822,11 @@ pkg_ctl="$prj_ctl/hello.git" $g -C tgt commit -m 'Add ownership info'; $g -C tgt push; - $* "file:///$~/tgt.git" $data_dir >>EOO + $* "file:///$~/tgt.git" $data_dir >>"EOO" : 1 status: 401 message: package owner authentication failed + reference: $checksum EOO } @@ -832,10 +844,11 @@ pkg_ctl="$prj_ctl/hello.git" $clone_root_tgt; $g clone tgt.git &tgt/***; - $* "file:///$~/tgt.git" $data_dir >>EOO + $* "file:///$~/tgt.git" $data_dir >>"EOO" : 1 status: 401 message: package publishing authorization failed + reference: $checksum EOO } @@ -851,6 +864,7 @@ pkg_ctl="$prj_ctl/hello.git" : 1 status: 400 message: unrecognized section 'delta' + reference: $checksum EOO } } -- cgit v1.1