From e3533fd4c2fc90969b77d2ddaccbda649dd74973 Mon Sep 17 00:00:00 2001
From: Karen Arutyunov <karen@codesynthesis.com>
Date: Thu, 9 Aug 2018 00:42:46 +0300
Subject: Implement submit-git

---
 brep/submit/.gitignore         |   4 +-
 brep/submit/buildfile          |   9 +-
 brep/submit/submit-dir.in      |  57 +++--
 brep/submit/submit-git.bash.in | 331 +++++++++++++++++++++++++++
 brep/submit/submit-git.in      | 498 +++++++++++++++++++++++++++++++++++++----
 brep/submit/submit.bash.in     | 149 +++++++++---
 6 files changed, 953 insertions(+), 95 deletions(-)
 create mode 100644 brep/submit/submit-git.bash.in

(limited to 'brep')

diff --git a/brep/submit/.gitignore b/brep/submit/.gitignore
index 8df3374..ef91424 100644
--- a/brep/submit/.gitignore
+++ b/brep/submit/.gitignore
@@ -1,3 +1,5 @@
+submit.bash
+submit-git.bash
+
 brep-submit-dir
 brep-submit-git
-
diff --git a/brep/submit/buildfile b/brep/submit/buildfile
index e3711f8..50f9615 100644
--- a/brep/submit/buildfile
+++ b/brep/submit/buildfile
@@ -5,5 +5,10 @@
 import mods  = libbutl.bash%bash{manifest-parser}
 import mods += libbutl.bash%bash{manifest-serializer}
 
-exe{brep-submit-dir}: in{submit-dir} bash{submit} $mods
-bash{submit}: in{submit} $mods # @@ Currently doesn't depend on manifest-parser.
+./: exe{brep-submit-dir} exe{brep-submit-git}
+
+exe{brep-submit-dir}: in{submit-dir} bash{submit}
+exe{brep-submit-git}: in{submit-git} bash{submit-git} bash{submit}
+
+bash{submit}:     in{submit} $mods
+bash{submit-git}: in{submit-git} bash{submit}
diff --git a/brep/submit/submit-dir.in b/brep/submit/submit-dir.in
index 31ae85d..4bcbe5f 100644
--- a/brep/submit/submit-dir.in
+++ b/brep/submit/submit-dir.in
@@ -13,70 +13,69 @@
 #
 usage="usage: $0 <dir>"
 
-verbose=true
+verbose= #true
 
 trap "{ exit 1; }" ERR
 set -o errtrace # Trap ERR in functions.
 
-@import libbutl/manifest-parser@
-@import libbutl/manifest-serializer@
-
 @import brep/submit/submit@
 
-# Submission data directory (last argument).
+if [ "$#" != 1 ]; then
+  error "$usage"
+fi
+
+# Submission data directory (last and the only argument).
 #
-dir="${!#/}"
+data_dir="${!#/}"
 
-if [ -z "$dir" ]; then
+if [ -z "$data_dir" ]; then
   error "$usage"
 fi
 
-if [ ! -d "$dir" ]; then
-  error "'$dir' does not exist or is not a directory"
+if [ ! -d "$data_dir" ]; then
+  error "'$data_dir' does not exist or is not a directory"
 fi
 
+reference="$(basename "$data_dir")"
+
 # Parse the submission request manifest and obtain the archive path as well
 # as the simulate value.
 #
-trace "parsing $dir/request.manifest"
-butl_manifest_parser_start "$dir/request.manifest"
+manifest_parser_start "$data_dir/request.manifest"
 
 archive=
 simulate=
 
-while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
   case "$n" in
-    archive)  archive="$v" ;;
+    archive)  archive="$v"  ;;
     simulate) simulate="$v" ;;
   esac
 done
 
-butl_manifest_parser_finish
+manifest_parser_finish
 
 if [ -z "$archive" ]; then
   error "archive manifest value expected"
 fi
 
 if [ -n "$simulate" -a "$simulate" != "success" ]; then
-  trace "unrecognized simulation outcome '$simulate'"
-  result_manifest 400 "unrecognized simulation outcome"
-  exit 0
+  exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
 fi
 
-manifest="$dir/package.manifest"
-
-extract_package_manifest "$dir/$archive" "$manifest"
+m="$data_dir/package.manifest"
+extract_package_manifest "$data_dir/$archive" "$m"
 
-# Parse the package manifest and obtain the package name and version.
+# Parse the package manifest and obtain the package name, version, and
+# project.
 #
-trace "parsing $manifest"
-butl_manifest_parser_start "$manifest"
+manifest_parser_start "$m"
 
 name=
 version=
 project=
 
-while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
   case "$n" in
     name)    name="$v"    ;;
     version) version="$v" ;;
@@ -84,14 +83,14 @@ while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do
   esac
 done
 
-butl_manifest_parser_finish
+manifest_parser_finish
 
 if [ -z "$name" ]; then
-  error "name manifest values expected"
+  error "name manifest value expected"
 fi
 
 if [ -z "$version" ]; then
-  error "version manifest values expected"
+  error "version manifest value expected"
 fi
 
 if [ -z "$project" ]; then
@@ -99,10 +98,10 @@ if [ -z "$project" ]; then
 fi
 
 if [ -n "$simulate" ]; then
-  rm -r "$dir"
+  rm -r "$data_dir"
   trace "$name/$version submission is simulated"
 else
   trace "$name/$version submission is queued"
 fi
 
-result_manifest 200 "$name/$version submission is queued" "$reference"
+exit_with_manifest 200 "$name/$version submission is queued" "$reference"
diff --git a/brep/submit/submit-git.bash.in b/brep/submit/submit-git.bash.in
new file mode 100644
index 0000000..8d5cc84
--- /dev/null
+++ b/brep/submit/submit-git.bash.in
@@ -0,0 +1,331 @@
+# file      : brep/submit/submit-git.bash.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Utility functions for the submit-git handler.
+
+if [ "$brep_submit_git" ]; then
+  return 0
+else
+  brep_submit_git=true
+fi
+
+@import brep/submit/submit@
+
+# If the section is mapped to a directory in the repository configuration then
+# return this directory path and empty string otherwise.
+#
+function section_dir () # <section> <repo-dir>
+{
+  trace_func "$@"
+
+  local sec="$1"
+  local rep="$2"
+
+  local owners # Unused but is declared to avoid polluting the global space.
+  local -A sections
+
+  run source "$rep/submit.config.bash"
+
+  local r="${sections[$sec]}"
+  if [ -z "$r" ]; then
+    r="${sections['*']}"
+  fi
+
+  echo "$r"
+}
+
+# If the owners directory is set in the repository configuration then return
+# this directory path prefixed with the repository directory path and the
+# empty string otherwise.
+#
+function owners_dir () # <repo-dir>
+{
+  local rep="$1"
+
+  local owners
+  local -A sections # Is declared to avoid polluting the global space.
+
+  run source "$rep/submit.config.bash"
+
+  local r=
+  if [ -n "$owners" ]; then
+    r="$rep/$owners"
+  fi
+
+  echo "$r"
+}
+
+# Check if a repository already contains the package. Respond with the
+# 'duplicate submission' result manifest and exit if that's the case.
+#
+function check_package_duplicate () # <name> <version> <repo-dir>
+{
+  trace_func "$@"
+
+  local nam="$1"
+  local ver="$2"
+  local rep="$3"
+
+  local owners # Unused but is declared to avoid polluting the global space.
+  local -A sections
+
+  run source "$rep/submit.config.bash"
+
+  # Check for duplicate package in all sections. Use <name>-<version>.*
+  # without .tar.gz in case we want to support more archive types later.
+  #
+  local s
+  for s in "${!sections[@]}"; do
+    local d="$rep/${sections[$s]}"
+
+    if [ -d "$d" ]; then
+      local f
+      f="$(run find "$d" -name "$nam-$ver.*")"
+
+      if [ -n "$f" ]; then
+	trace "found: $f"
+        exit_with_manifest 422 "duplicate submission"
+      fi
+    fi
+  done
+}
+
+# Serialize the project or package owner manifest (they have the same set of
+# values) to the specified manifest file.
+#
+function create_owner_manifest () # <name> <email> <control> <file>
+{
+  trace_func "$@"
+
+  local nam="$1"
+  local eml="$2"
+  local ctl="$3"
+  local man="$4"
+
+  if [ -f "$man" ]; then
+    error "'$man' already exists"
+  fi
+
+  manifest_serializer_start "$man"
+
+  manifest_serialize ""        "1"    # Start of manifest.
+  manifest_serialize "name"    "$nam"
+  manifest_serialize "email"   "$eml"
+  manifest_serialize "control" "$ctl"
+
+  manifest_serializer_finish
+}
+
+# Strip the query part and the leaf path component from the repository URL.
+#
+function repository_base () # <repo-url>
+{
+  # First, strip the URL query part, then component.
+  #
+  sed -n \
+-e 's%^\([^?]*\).*$%\1%' \
+-e 's%^\(.*\)/[^/]\{1,\}/\{0,1\}$%\1%p' \
+<<<"$1"
+}
+
+# Authenticate the project name owner. Make sure that the control manifest
+# value is specified unless authentication is disabled.
+#
+# Possible return values:
+#
+# - 'project'  if the project belongs to the submitter
+# - 'unknown'  if the project name is not yet known
+# - 'disabled' if the owners directory is not configured
+# - <manifest> result manifest describing the authentication error
+#
+# Note that the authentication error result always starts with ':'.
+#
+function auth_project () # <project> <control> <repo-dir>
+{
+  trace_func "$@"
+
+  local prj="$1"
+  local ctl="${2%/}"
+  local rep="$3"
+
+  local d
+  d="$(owners_dir "$rep")"
+
+  if [ -z "$d" ]; then
+    echo "disabled"
+    return
+  fi
+
+  if [ -z "$ctl" ]; then
+    exit_with_manifest 400 "control manifest value expected"
+  fi
+
+  local r="unknown"
+  local m="$d/$prj/project-owner.manifest"
+
+  # If the project owner manifest exists then parse it and try to authenticate
+  # the submitter as the project owner.
+  #
+  if [ -f "$m" ]; then
+    ctl="$(repository_base "$ctl")"
+
+    # Parse the project owner manifest.
+    #
+    manifest_parser_start "$m"
+
+    local n v
+    while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+      if [ "$n" == "control" -a "${v%/}" == "$ctl" ]; then
+        r="project"
+        break
+      fi
+    done
+
+    manifest_parser_finish
+
+    if [ "$r" != "project" ]; then
+      exit_with_manifest 401 "project owner authentication failed"
+    fi
+  fi
+
+  echo "$r"
+}
+
+# Authenticate the package name owner. Make sure that the control manifest
+# value is specified unless authentication is disabled. It is assumed that the
+# project ownership is already authenticated (possibly by another repository).
+#
+# Possible return values:
+#
+# - 'package'  if the package belongs to the submitter
+# - 'unknown'  if the package name is not taken in the project
+# - 'disabled' if the owners directory is not configured
+# - <manifest> result manifest describing the authentication error
+#
+# Note that the authentication error result always starts with ':'.
+#
+function auth_package () # <project> <package> <control> <repo-dir>
+{
+  trace_func "$@"
+
+  local prj="$1"
+  local pkg="$2"
+  local ctl="${3%/}"
+  local rep="$4"
+
+  local d
+  d="$(owners_dir "$rep")"
+
+  if [ -z "$d" ]; then
+    echo "disabled"
+    return
+  fi
+
+  if [ -z "$ctl" ]; then
+    exit_with_manifest 400 "control manifest value expected"
+  fi
+
+  local r="unknown"
+  local m="$d/$prj/$pkg/package-owner.manifest"
+
+  # If the package owner manifest exists then parse it and try to authenticate
+  # the submitter as the package owner.
+  #
+  if [ -f "$m" ]; then
+
+    # Parse the package owner manifest.
+    #
+    manifest_parser_start "$m"
+
+    local n v
+    while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+      if [ "$n" == "control" -a "${v%/}" == "$ctl" ]; then
+        r="package"
+        break
+      fi
+    done
+
+    manifest_parser_finish
+
+    if [ "$r" != "package" ]; then
+      exit_with_manifest 401 "package owner authentication failed"
+    fi
+  fi
+
+  echo "$r"
+}
+
+# Check that the package name is unknown to the repository. Owners directory
+# is expected to be configured.
+#
+function auth_package_unknown () # <package> <repo-dir>
+{
+  trace_func "$@"
+
+  local pkg="$1"
+  local rep="$2"
+
+  local d
+  d="$(owners_dir "$rep")"
+
+  # Sanity check that the owners directory configured for the repository.
+  #
+  if [ -z "$d" ]; then
+    error "no owners directory configured for '$rep'"
+  fi
+
+  # While configured, the owners directory may not yet exist.
+  #
+  if [ -d "$d" ]; then
+    local f
+    f="$(run find "$d" -path "$d/*/$pkg/package-owner.manifest")"
+
+    if [ -n "$f" ]; then
+      trace "found: $f"
+      exit_with_manifest 401 "package owner authentication failed"
+    fi
+  fi
+}
+
+# Return lower-case URL scheme or empty string if the argument doesn't look
+# like a URL.
+#
+function url_scheme () # <url>
+{
+  sed -n -e 's%^\(.*\)://.*$%\L\1%p' <<<"$1"
+}
+
+# Checks that the repository properly responds to the probing request before
+# the timeout (in seconds). Noop for protocols other than HTTP(S).
+#
+function check_connectivity () # <repo-url> <timeout>
+{
+  trace_func "$@"
+
+  local url="$1"
+  local tmo="$2"
+
+  local s
+  s="$(url_scheme "$url")"
+
+  if [ "$s" == "http" -o "$s" == "https" ]; then
+    local u q
+
+    u="$(sed -n -e 's%^\([^?]*\).*$%\1%p' <<<"$url")" # Strips query part.
+    q="$(sed -n -e 's%^[^?]*\(.*\)$%\1%p' <<<"$url")" # Query part.
+
+    if [ -z "$q" ]; then
+      u="$u/info/refs?service=git-upload-pack"
+    else
+      u="$u/info/refs$q&service=git-upload-pack"
+    fi
+
+    # This function is called on repositories other than ours (e.g., control)
+    # so we don't want a failure to be logged.
+    #
+    if ! run_silent curl -S -s --max-time "$tmo" "$u" >/dev/null; then
+      exit_with_manifest 503 "submission service temporarily unavailable"
+    fi
+  fi
+}
diff --git a/brep/submit/submit-git.in b/brep/submit/submit-git.in
index fbb69de..c27edab 100644
--- a/brep/submit/submit-git.in
+++ b/brep/submit/submit-git.in
@@ -14,8 +14,8 @@
 #
 # The handler also implements the project/package name ownership verification
 # by performing the submitter authentication/authorization based on the
-# control repository mechanism describe in bdep-publish(1). This functionality
-# is optional.
+# control repository mechanism described in bdep-publish(1). This
+# functionality is optional.
 #
 # The handler can operate with a single git repository, called "target", or
 # with two git repositories, in which case the first is the target and the
@@ -64,20 +64,20 @@
 # following structure:
 #
 # <owners>/
-# ├── <project1>/
-# │    ├── <package1>/
-# │    │    └── package-owner.manifest
-# │    ├── <package2>/
-# │    │    └── package-owner.manifest
-# │    ├── ...
-# │    └── project-owner.manifest
-# ├── <project2>/
-# │    └── ...
-# └──...
+# |-- <project1>/
+# |   |-- <package1>/
+# |   |   `-- package-owner.manifest
+# |   |-- <package2>/
+# |   |   `-- package-owner.manifest
+# |   |-- ...
+# |   `-- project-owner.manifest
+# |-- <project2>/
+# |   `-- ...
+# `-- ...
 #
 # If the submitted project name is not yet known, then the handler script
 # creates a new project subdirectory and saves project-owner.manifest. The
-# project owner manifest contain the following values in the specified order:
+# project owner manifest contains the following values in the specified order:
 #
 #   name:  <project-name>
 #   email: <submitter-email>
@@ -95,7 +95,7 @@
 #
 # Similarly, if the submitted package name is not yet known, then the handler
 # script creates a new package subdirectory and saves package-owner.manifest.
-# The package owner manifest contain the following values in the specified
+# The package owner manifest contains the following values in the specified
 # order:
 #
 #   name:  <package-name>
@@ -131,42 +131,464 @@
 #   add, commit, and push to reference and then remove, commit, and push to
 #   target.
 #
-#   On the handler side, before adding a package or new ownership for a
-#   project/package name, the script re-checks the reference repository for
-#   updated information.
+#   On the handler side, the script acts in the opposite order cloning the
+#   target prior pulling the reference in order not to get into the situation
+#   where it misses the ownership info that is not in the reference yet but no
+#   longer in the target. Note that if some move happens after the cloning,
+#   then the script will be unable to push the target modification and will
+#   re-try the whole authentication procedure from scratch.
+#
+# - Filesystem entries that exist or are created in the data directory:
+#
+#   <pkg>-<ver>.tar.gz   saved by brep (could be other archives in the future)
+#   request.manifest     created by brep
+#   package.manifest     extracted by the handler
+#   target/              cloned by the handler
+#   control/             cloned by the handler
+#   result.manifest      saved by brep
 #
 usage="usage: $0 <tgt-repo> [<ref-repo>] <dir>"
 
+# Diagnostics.
+#
+verbose= #true
+
+# Git network operations timeout (seconds).
+#
+# Note that we don't cover protocols other than HTTP(S) since for them git
+# doesn't support any timeouts (though we may be able to cobble something
+# up for SSH).
+#
+git_timeout=10
+
+# The reference repository lock timeout (seconds).
+#
+ref_lock_timeout=30
+
 trap "{ exit 1; }" ERR
 set -o errtrace # Trap ERR in functions.
 
-# Implementation notes:
+@import brep/submit/submit@
+@import brep/submit/submit-git@
+
+if [ "$#" -lt 2 -o "$#" -gt 3 ]; then
+  error "$usage"
+fi
+
+# Target repository URL.
 #
-# - Check for duplicate package archive in all the sections. Before auth. Use
-#   <name>-<version>.* instead of .tar.gz in case we support other formats
-#   later.
+tgt_repo="$1"
+shift
+
+if [ -z "$tgt_repo" ]; then
+  error "$usage"
+fi
+
+# Reference repository directory.
 #
-# - Push permission for target repo (add www-data to scm group)?
+# Note that the last argument is always the submission data directory.
 #
-# - Network errors/timeouts on git pull for ref repo? What is the error (try
-#   again)? I think also let's not assume target repo is local.
+ref_repo=
 
-# Workflow:
-#
-# 0. The same steps as submit-dir.
+if [ "$#" -gt 1 ]; then
+  ref_repo="$1"
+  shift
+
+  if [ -z "$ref_repo" ]; then
+    error "$usage"
+  fi
+
+  if [ ! -d "$ref_repo" ]; then
+    error "'$ref_repo' does not exist or is not a directory"
+  fi
+fi
+
+# Submission data directory.
 #
-# 1. If ref-repo specified, lock, pull, and check:
-#    - duplicate
-#    - auth (read-only)
+data_dir="$1"
+shift
+
+if [ -z "$data_dir" ]; then
+  error "$usage"
+fi
+
+if [ ! -d "$data_dir" ]; then
+  error "'$data_dir' does not exist or is not a directory"
+fi
+
+reference="$(basename "$data_dir")"
+
+# Git verbosity options.
+#
+# Note that not all git commands support the -q/-v options. Also note that
+# these variable expansions should not be quoted.
+#
+if [ "$verbose" ]; then
+  gqo=
+  gvo="-v"
+else
+  gqo="-q"
+  gvo=
+fi
+
+# Git doesn't support the connection timeout option. The options we use are
+# just an approximation of the former, that, in particular, don't cover the
+# connection establishing. To work around this problem, before running a git
+# command that assumes the remote repository communication we manually check
+# connectivity with the remote repository.
+#
+git_http_timeout=("-c" "http.lowSpeedLimit=1" \
+		  "-c" "http.lowSpeedTime=$git_timeout")
+
+# Parse the submission request manifest and obtain the required values.
 #
-# 2. Clone tgt-repo, check:
-#    - duplicate
-#    - auth (read-write)
-#      ? if fully auth'd by ref-repo, should we skip it here?
+manifest_parser_start "$data_dir/request.manifest"
+
+archive=
+sha256sum=
+section=
+email=
+control=
+simulate=
+
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+  case "$n" in
+    archive)   archive="$v"     ;;
+    sha256sum) sha256sum="$v"   ;;
+    section)   section="$v"     ;;
+    email)     email="$v"       ;;
+    control)   control="${v%/}" ;;
+    simulate)  simulate="$v"    ;;
+  esac
+done
+
+manifest_parser_finish
+
+if [ -z "$archive" ]; then
+  error "archive manifest value expected"
+fi
+
+if [ -z "$sha256sum" ]; then
+  error "sha256sum manifest value expected"
+fi
+
+if [ -n "$simulate" -a "$simulate" != "success" ]; then
+  exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
+fi
+
+# Note: checking for section, email, and control later.
+
+m="$data_dir/package.manifest"
+extract_package_manifest "$data_dir/$archive" "$m"
+
+# Parse the package manifest and obtain the package name and version.
 #
-# 3. Clone control branch and authorize.
+manifest_parser_start "$m"
+
+name=
+version=
+project=
+
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+  case "$n" in
+    name)    name="$v"    ;;
+    version) version="$v" ;;
+    project) project="$v" ;;
+  esac
+done
+
+manifest_parser_finish
+
+if [ -z "$name" ]; then
+  error "name manifest value expected"
+fi
+
+if [ -z "$version" ]; then
+  error "version manifest value expected"
+fi
+
+if [ -z "$project" ]; then
+  project="$name"
+fi
+
+function git_add () # <repo-dir> <path>...
+{
+  local d="$1"
+  shift
+
+  run git -C "$d" add $gvo "$@" >&2
+}
+
+function git_clone () # <repo-url> <repo-dir> <extra-options>...
+{
+  local url="$1"
+  shift
+
+  local dir="$1"
+  shift
+
+  check_connectivity "$url" "$git_timeout"
+  run git "${git_http_timeout[@]}" clone $gqo $gvo "$@" "$url" "$dir" >&2
+}
+
+# Dor now we make 10 re-tries to add the package and push to target. Push can
+# fail due to the target-to-reference information move race (see the above
+# notes for details) or because concurrent submissions. We may want to make it
+# configurable in the future.
 #
-# 4. Copy archive, commit and push. If push fails, remove clone and
-#    restart from 1 (yes, from ref-repo).
-#    - put submission manifest into commit message for record?
+pkg_added=
+
+for i in {1..11}; do
+
+  # Clone the target repository.
+  #
+  tgt_dir="$data_dir/target"
+  git_clone "$tgt_repo" "$tgt_dir" --single-branch --depth 1
+
+  check_package_duplicate "$name" "$version" "$tgt_dir"
+
+  # Check for duplicates and try to authenticate the package ownership using
+  # information in the reference repository, if specified.
+  #
+  if [ -n "$ref_repo" ]; then
+
+    remote_url=$(git -C "$ref_repo" config --get remote.origin.url)
+
+    # Open the reading file descriptor and lock the repository. Fail if unable
+    # to lock before timeout.
+    #
+    l="$ref_repo/submit.config.bash"
+    trace "+ exec {fd}<$l"
+    exec {fd}<"$l"
+
+    if ! run flock -w "$ref_lock_timeout" "$fd"; then
+      exit_with_manifest 503 "submission service temporarily unavailable"
+    fi
+
+    # Pull the reference repository.
+    #
+    check_connectivity "$remote_url" "$git_timeout"
+    run git "${git_http_timeout[@]}" -C "$ref_repo" pull $gqo $gvo >&2
+
+    # Check the package duplicate.
+    #
+    check_package_duplicate "$name" "$version" "$ref_repo"
+
+    # Authenticate the project ownership.
+    #
+    auth="$(auth_project "$project" "$control" "$ref_repo")"
+
+    # Try to authenticate the package ownership if the project ownership was
+    # authenticated successfully.
+    #
+    if [ "$auth" == "project" ]; then
+      a="$(auth_package "$project" "$name" "$control" "$ref_repo")"
+
+      # If the package is unknown to this project, we will try to authenticate
+      # the package name with the target repository later and so we keep the
+      # 'project' auth state.
+      #
+      if [ "$a" != "unknown" ]; then
+	auth="$a"
+      fi
+    fi
+
+    trace "reference auth: $auth"
+
+    if [ "${auth:0:1}" == ":" ]; then # Authentication error?
+      echo "$auth"
+      exit 0
+    fi
+
+    # If the package is not present in the specified project then we need to
+    # make sure it is also not present in any other project.
+    #
+    if [ "$auth" == "project" -o "$auth" == "unknown" ]; then
+      auth_package_unknown "$name" "$ref_repo"
+    fi
+
+    trace "+ exec {fd}<&-"
+    exec {fd}<&- # Close the file descriptor and unlock the repository.
+  else
+    auth="disabled"
+  fi
+
+  ref_auth="$auth"
+
+  # Now try to authenticate the package ownership using information in the
+  # target repository unless already authenticated with reference.
+  #
+  if [ "$auth" != "package" ]; then
+
+    # Don't authenticate the project ownership if this is already done with
+    # the reference repository.
+    #
+    if [ "$auth" != "project" ]; then
+      auth="$(auth_project "$project" "$control" "$tgt_dir")"
+    fi
+
+    # Try to authenticate the package ownership if the project ownership was
+    # authenticated successfully.
+    #
+    if [ "$auth" == "project" ]; then
+      auth="$(auth_package "$project" "$name" "$control" "$tgt_dir")"
+    fi
+
+    trace "target auth: $auth"
+
+    if [ "${auth:0:1}" == ":" ]; then # Authentication error?
+      echo "$auth"
+      exit 0
+    fi
+  fi
+
+  trace "resulting auth: $auth"
+
+  # Sanity check the auth variable value.
+  #
+  case "$auth" in
+    package)  ;;
+    unknown)  ;;
+    disabled) ;;
+
+    *) error "unexpected resulting auth '$auth'";;
+  esac
+
+  # Establish ownership of the package name unless already done.
+  #
+  if [ "$auth" == "unknown" ]; then
+
+    # Check that the necessary request manifest values are specified.
+    #
+    if [ -z "$email" ]; then
+      exit_with_manifest 400 "email manifest value expected"
+    fi
+
+    # Check that the package doesn't belong yet to some other project.
+    #
+    auth_package_unknown "$name" "$tgt_dir"
+
+    # First the project name.
+    #
+    # Note that owners_dir() shouldn't return an empty string at this stage.
+    #
+    d="$(owners_dir "$tgt_dir")/$project"
+
+    # Establish ownership of the project name unless already done. Note that
+    # it can only be owned by the submitter at this stage.
+    #
+    prj_man="$d/project-owner.manifest"
+
+    if [ ! -f "$prj_man" ]; then
+      run mkdir -p "$d" # Also creates the owners directory if not exist.
+
+      ctl="$(repository_base "$control")"
+      create_owner_manifest "$project" "$email" "$ctl" "$prj_man"
+
+      # Add the project owners manifest file to git repository using the path
+      # relative to the repository directory.
+      #
+      git_add "$tgt_dir" "${prj_man#$tgt_dir/}"
+    fi
+
+    # Now the package name.
+    #
+    d="$d/$name"
+    run mkdir "$d"
+
+    pkg_man="$d/package-owner.manifest"
+    create_owner_manifest "$name" "$email" "$control" "$pkg_man"
+
+    # Add the package owners manifest file using path relative to the
+    # repository directory.
+    #
+    git_add "$tgt_dir" "${pkg_man#$tgt_dir/}"
+
+    auth="package"
+  fi
+
+  # Respond with the 'unauthorized' manifest if we failed to authenticate the
+  # submitter as the package owner, unless both the reference and target
+  # repositories have the ownership authentication disabled. In the latter
+  # case no authorization is required.
+  #
+  if [ "$auth" != "disabled" -o "$ref_auth" != "disabled" ]; then
+
+    # Respond with the 'unauthorized' manifest if not the package owner.
+    #
+    if [ "$auth" != "package" ]; then
+      if [ "$auth" == "project" -o "$ref_auth" == "project" ]; then
+        exit_with_manifest 401 "package owner authentication failed"
+      else
+        exit_with_manifest 401 "project owner authentication failed"
+      fi
+    fi
+
+    # Authorize the submission.
+    #
+    ctl_dir="$data_dir/control"
+
+    git_clone "$control" "$ctl_dir" --single-branch --depth 1 \
+ --branch "build2-control"
+
+    if [ ! -f "$ctl_dir/submit/${sha256sum:0:16}" ]; then
+      exit_with_manifest 401 "package publishing authorization failed"
+    fi
+  fi
+
+  # Add the package archive to the target repository.
+  #
+  s="$(section_dir "$section" "$tgt_dir")"
+
+  if [ -z "$s" ]; then
+    exit_with_manifest 400 "unrecognized section '$section'"
+  fi
+
+  d="$tgt_dir/$s/$project"
+  run mkdir -p "$d" # Create all the parent directories as well.
+
+  # We copy the archive rather than move it since we may need it for a re-try.
+  #
+  a="$d/$archive"
+  run cp "$data_dir/$archive" "$a"
+
+  git_add "$tgt_dir" "${a#$tgt_dir/}"
+
+  run git -C "$tgt_dir" commit $gqo $gvo -F - <<EOF >&2
+Add $name/$version to $s/$project
+
+$(cat "$data_dir/request.manifest")
+EOF
+
+  check_connectivity "$tgt_repo" "$git_timeout"
+
+  # Try to push the target modifications, unless simulating. If this succeeds
+  # then we are done. Otherwise, drop the target directory and re-try the
+  # whole authentication/authorization procedure, unless we are out of
+  # attempts.
+  #
+  if [ -z "$simulate" ]; then
+    if run_silent git "${git_http_timeout[@]}" -C "$tgt_dir" push >&2; then
+      pkg_added=true
+      break
+    else
+      run rm -r -f "$tgt_dir" "$ctl_dir"
+    fi
+  fi
+done
+
+if [ ! "$pkg_added" ]; then
+  exit_with_manifest 503 "submission service temporarily unavailable"
+fi
+
+# Remove the no longer needed submission data directory.
 #
+run rm -r -f "$data_dir"
+
+if [ -n "$simulate" ]; then
+  trace "$name/$version submission is simulated"
+else
+  trace "$name/$version submission is queued"
+fi
+
+exit_with_manifest 200 "$name/$version submission is queued" "$reference"
diff --git a/brep/submit/submit.bash.in b/brep/submit/submit.bash.in
index 8315315..babf081 100644
--- a/brep/submit/submit.bash.in
+++ b/brep/submit/submit.bash.in
@@ -2,22 +2,24 @@
 # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
+# Utility functions useful for implementing submission handlers.
+
 if [ "$brep_submit" ]; then
   return 0
 else
   brep_submit=true
 fi
 
+@import libbutl/manifest-parser@
 @import libbutl/manifest-serializer@
 
 # Diagnostics.
 #
-
 # We expect the user to set the verbose variable either to true or empty
 # (false).
 #
 if [ ! -v verbose ]; then
-  echo "variable 'verbose' is not set" 2>&1
+  echo "error: variable 'verbose' is not set" >&2
   exit 1
 fi
 
@@ -26,10 +28,13 @@ fi
 # [Mon Jul 23 17:48:46.945079 2018] [brep:error] [pid 123:tid 456] [brep::submit::init]: error description
 #
 # We will use the (almost) same format for our diagnostics (redirected to the
-# Apache's error_log) so it can easily be attributed to the brep module.
+# Apache's error_log), so it can easily be attributed to the brep module.
 #
 info_self="$(basename $0)"
-info_ref="$(basename "${!#/}")" # Last argument is the submission directory.
+
+if [ "$#" -gt 0 ]; then
+  info_ref="$(basename "${!#/}")" # Last argument is the submission directory.
+fi
 
 function info () # <severity> <text>
 {
@@ -47,52 +52,146 @@ function info () # <severity> <text>
 }
 
 function error () { info "error" "$*"; exit 1; }
+
 function trace () { if [ "$verbose" ]; then info "info" "$*"; fi }
 
+# Trace a command line, quoting empty arguments as well as those that contain
+# spaces.
+#
+function trace_cmd () # <cmd> <arg>...
+{
+  if [ "$verbose" ]; then
+    local s="+"
+    while [ $# -gt 0 ]; do
+      if [ -z "$1" -o -z "${1##* *}" ]; then
+	s="$s '$1'"
+      else
+	s="$s $1"
+      fi
+
+      shift
+    done
+
+    info "info" "$s"
+  fi
+}
+
+# Trace the current function name and arguments.
+#
+function trace_func () # <args>...
+{
+  trace_cmd "${FUNCNAME[1]}" "$@"
+}
+
+# Trace and run a command.
+#
+function run () # <cmd> <arg>...
+{
+  trace_cmd "$@"
+  "$@"
+}
+
+# Same as above but also redirect the command stderr to /dev/null, unless
+# running in the verbose mode.
+#
+# Note that we don't redirect stdout, so it can still be captured.
+#
+function run_silent () # <cmd> <arg>...
+{
+  trace_cmd "$@"
+
+  if [ "$verbose" ]; then
+    "$@"
+  else
+    "$@" 2>/dev/null
+  fi
+}
+
+# Wrap libbutl manifest parsing/serializing functions to shorten names and to
+# add tracing.
+#
+function manifest_parser_start () # [<file>]
+{
+  trace_func "$@"
+  butl_manifest_parser_start "$@"
+
+  manifest_parser_ofd="$butl_manifest_parser_ofd"
+}
+
+function manifest_parser_finish ()
+{
+  trace_func
+  butl_manifest_parser_finish
+}
+
+function manifest_serializer_start () # [<file>]
+{
+  trace_func "$@"
+  butl_manifest_serializer_start "$@"
+
+  manifest_serializer_ifd="$butl_manifest_serializer_ifd"
+}
+
+function manifest_serializer_finish ()
+{
+  trace_func
+  butl_manifest_serializer_finish
+}
+
 # Serialize one manifest name/value pair.
 #
-function serialize () # <name> <value>
+function manifest_serialize () # <name> <value>
 {
-  printf "%s:%s\0" "$1" "$2" >&"$butl_manifest_serializer_ifd"
+#  trace "$1: $2"
+  printf "%s:%s\0" "$1" "$2" >&"$manifest_serializer_ifd"
 }
 
-# Serialize the submission result manifest to stdout.
+# Serialize the submission result manifest to stdout and exit the (sub-)shell
+# with the zero status.
 #
-function result_manifest () # <status> <message> [<reference>]
+function exit_with_manifest () # <status> <message> [<reference>]
 {
+  trace_func "$@"
+
   local sts="$1"
   local msg="$2"
   local ref="$3"
 
-  trace "serializing result manifest"
-  butl_manifest_serializer_start
+  manifest_serializer_start
 
-  serialize ""        "1"    # Start of manifest.
-  serialize "status"  "$sts"
-  serialize "message" "$msg"
+  manifest_serialize ""        "1"    # Start of manifest.
+  manifest_serialize "status"  "$sts"
+  manifest_serialize "message" "$msg"
 
   if [ -n "$ref" ]; then
-    serialize "reference" "$ref"
+    if [ "$sts" != "200" ]; then
+      error "reference for code $sts"
+    fi
+
+    manifest_serialize "reference" "$ref"
+  elif [ "$sts" == "200" ]; then
+    error "no reference for code $sts"
   fi
 
-  butl_manifest_serializer_finish
+  manifest_serializer_finish
+  run exit 0
 }
 
-# Verify the archive is a valid bpkg package and extract its manifest file.
+# Verify archive is a valid package and extract its manifest into
+# <manifest> file.
 #
 function extract_package_manifest () # <archive> <manifest>
 {
   local arc="$1"
   local man="$2"
 
-  # Should we remove the submission directory with an invalid package?
-  # Probably it's better to leave it for potential investigation. Note that we
-  # can always grep for such directories based on the result.manifest file
-  # they contain.
-  #
-  if ! bpkg pkg-verify --manifest "$arc" >"$man" 2>/dev/null; then
-    trace "$arc is not a valid package"
-    result_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)"
-    exit 0
+  if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then
+    # Perform the sanity check to make sure that bpkg is runnable.
+    #
+    if ! run bpkg --version >/dev/null; then
+      error "unable to run bpkg"
+    fi
+
+    exit_with_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)"
   fi
 }
-- 
cgit v1.1