aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2018-08-08 09:45:48 +0200
committerKaren Arutyunov <karen@codesynthesis.com>2018-08-17 23:26:34 +0300
commite3a8a11f7fdc64a560810cf021080c61c7d69dc5 (patch)
treea3b9ee0d93272917e054650357d92b71922bb588
parent95b154183516d6ba8bf9367c20fbadddee680391 (diff)
Document submit-git
-rw-r--r--brep/submit/.gitignore4
-rw-r--r--brep/submit/buildfile3
-rw-r--r--brep/submit/submit-dir.in21
-rw-r--r--brep/submit/submit-git.in164
-rw-r--r--brep/submit/submit.bash.in36
-rw-r--r--build/root.build4
6 files changed, 194 insertions, 38 deletions
diff --git a/brep/submit/.gitignore b/brep/submit/.gitignore
index b49e361..8df3374 100644
--- a/brep/submit/.gitignore
+++ b/brep/submit/.gitignore
@@ -1 +1,3 @@
-brep-submit
+brep-submit-dir
+brep-submit-git
+
diff --git a/brep/submit/buildfile b/brep/submit/buildfile
index f25b6e3..e3711f8 100644
--- a/brep/submit/buildfile
+++ b/brep/submit/buildfile
@@ -7,6 +7,3 @@ import mods += libbutl.bash%bash{manifest-serializer}
exe{brep-submit-dir}: in{submit-dir} bash{submit} $mods
bash{submit}: in{submit} $mods # @@ Currently doesn't depend on manifest-parser.
-
-#install.bin.subdirs = true
-#install.subdirs = true
diff --git a/brep/submit/submit-dir.in b/brep/submit/submit-dir.in
index ce8f134..31ae85d 100644
--- a/brep/submit/submit-dir.in
+++ b/brep/submit/submit-dir.in
@@ -4,16 +4,15 @@
# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-# Package submission handler example.
+# Simple package submission handler with directory storage.
#
# Validate the package archive located in the specified submission directory
-# extracting and parsing the package manifest. Remove the submission directory
-# if simulating. Write the submission result manifest to stdout.
+# extracting and parsing the package manifest (saved as package.manifest in
+# the submission directory). Keep the submission directory unless simulating.
+# Write the submission result manifest to stdout.
#
usage="usage: $0 <dir>"
-# Diagnostics.
-#
verbose=true
trap "{ exit 1; }" ERR
@@ -24,6 +23,18 @@ set -o errtrace # Trap ERR in functions.
@import brep/submit/submit@
+# Submission data directory (last argument).
+#
+dir="${!#/}"
+
+if [ -z "$dir" ]; then
+ error "$usage"
+fi
+
+if [ ! -d "$dir" ]; then
+ error "'$dir' does not exist or is not a directory"
+fi
+
# Parse the submission request manifest and obtain the archive path as well
# as the simulate value.
#
diff --git a/brep/submit/submit-git.in b/brep/submit/submit-git.in
index 0238c4e..fbb69de 100644
--- a/brep/submit/submit-git.in
+++ b/brep/submit/submit-git.in
@@ -4,11 +4,169 @@
# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-# Package submission handler.
+# Package submission handler with git repository storage.
#
-# @@ TODO doc/spec.
+# The overall idea behind this handler is to add the package archive into a
+# git repository. Another entity (for example, a human or a script) can then
+# pull this change and do something about it (for example, review it and/or
+# add it to an archive-based repository). In other words, git is used as a
+# kind of transport that is easy enough to access for both humans and scripts.
#
-usage="usage: $0 <dir>"
+# The handler also implements the project/package name ownership verification
+# by performing the submitter authentication/authorization based on the
+# control repository mechanism describe in bdep-publish(1). This functionality
+# is optional.
+#
+# The handler can operate with a single git repository, called "target", or
+# with two git repositories, in which case the first is the target and the
+# second is called "reference". The reference repository access is read-only
+# and it is only consulted for duplicate package suppression and name
+# ownership verification. The dual repository mode is normally used to
+# implement a two-stage queue/publish setup where the package is first queued
+# for review and/or testing and then moved (for example, by a moderator) to a
+# public repository.
+#
+# The target repository argument (<tgt-repo>) should be a read-write git
+# repository URL. It is cloned (shallow) into the submission directory on
+# each submission.
+#
+# If specified, the reference repository argument (<ref-repo>) should be a
+# directory with a pre-cloned read-only reference repository. This directory
+# is shared between all instances of the handler. On each submission, the
+# handler will flock(1) this directory, git-pull, obtain the information it
+# needs, and release the lock.
+#
+# Both the target and, if specified, reference repositories should contain the
+# submit.config.bash repository configuration file in the root directory. The
+# configuration file is a bash fragment and is sourced by the handler script.
+# It provides the following information:
+#
+# - Mapping of section names to repository subdirectories in the 'sections'
+# variable (declare -A sections; values are relative to the repository
+# root).
+
+# If there is no key for the submitted section name, then the entry with the
+# special '*' key is used. If there is no such entry, then the submission is
+# invalid. For example:
+#
+# sections[alpha]=1/alpha
+# sections[beta]=1/beta
+# sections[stable]=1/testing
+#
+# - Optional owners subdirectory in the 'owners' variable (relative to the
+# repository root). If not specified, then no ownership verification is
+# performed. For example:
+#
+# owners=owners
+#
+# If the ownership directory is specified, then the handler script maintains
+# the project/package name ownership information in this directory. It has the
+# following structure:
+#
+# <owners>/
+# ├── <project1>/
+# │ ├── <package1>/
+# │ │ └── package-owner.manifest
+# │ ├── <package2>/
+# │ │ └── package-owner.manifest
+# │ ├── ...
+# │ └── project-owner.manifest
+# ├── <project2>/
+# │ └── ...
+# └──...
+#
+# If the submitted project name is not yet known, then the handler script
+# creates a new project subdirectory and saves project-owner.manifest. The
+# project owner manifest contain the following values in the specified order:
+#
+# name: <project-name>
+# email: <submitter-email>
+# control: <url-prefix>
+#
+# The 'control' value is the control repository URL prefix and there can be
+# multiple such values in a single manifest. The handler script derives it
+# from the submitted control repository URL by removing the last path
+# component. So, for example, https://github.com/build2/libbutl.git becomes
+# https://github.com/build2/.
+#
+# If the submitted project name is already known, then the handler script
+# loads its project-owner.manifest and verifies that at least one of the
+# 'control' values is a prefix of the submitted control repository URL.
+#
+# Similarly, if the submitted package name is not yet known, then the handler
+# script creates a new package subdirectory and saves package-owner.manifest.
+# The package owner manifest contain the following values in the specified
+# order:
+#
+# name: <package-name>
+# email: <submitter-email>
+# control: <url>
+#
+# The 'control' value is the control repository URL and there can be multiple
+# such values in a single manifest.
+#
+# If the submission package is already known, then the handler script loads
+# its package-owner.manifest and verifies that at least one of the 'control'
+# values matches the submitted control repository URL.
+#
+# If all these ownership authentication tests pass, the handler script clones
+# (shallow) the build2-control branch of the control repository and verifies
+# that the submission authorization file is present (see bdep-publish(1) for
+# details).
+#
+# If the submission authorization test passes, then the handler script adds
+# the package archives to the target repository, commits this change, and
+# then pushes the commit to the remote.
+#
+# Notes:
+#
+# - It is possible that a submitted package name already exists in another
+# project. In this case, such a submission is accepted only if the package
+# already exists in the requested project. This allows the moderator to
+# manually permit such multi-project packages (for example, to allow moving
+# packages between projects).
+#
+# - There could be a race when moving package and ownership information from
+# target to reference. To avoid it, the protocol for such a move is to first
+# add, commit, and push to reference and then remove, commit, and push to
+# target.
+#
+# On the handler side, before adding a package or new ownership for a
+# project/package name, the script re-checks the reference repository for
+# updated information.
+#
+usage="usage: $0 <tgt-repo> [<ref-repo>] <dir>"
trap "{ exit 1; }" ERR
set -o errtrace # Trap ERR in functions.
+
+# Implementation notes:
+#
+# - Check for duplicate package archive in all the sections. Before auth. Use
+# <name>-<version>.* instead of .tar.gz in case we support other formats
+# later.
+#
+# - Push permission for target repo (add www-data to scm group)?
+#
+# - Network errors/timeouts on git pull for ref repo? What is the error (try
+# again)? I think also let's not assume target repo is local.
+
+# Workflow:
+#
+# 0. The same steps as submit-dir.
+#
+# 1. If ref-repo specified, lock, pull, and check:
+# - duplicate
+# - auth (read-only)
+#
+# 2. Clone tgt-repo, check:
+# - duplicate
+# - auth (read-write)
+# ? if fully auth'd by ref-repo, should we skip it here?
+#
+# 3. Clone control branch and authorize.
+#
+# 4. Copy archive, commit and push. If push fails, remove clone and
+# restart from 1 (yes, from ref-repo).
+# - put submission manifest into commit message for record?
+#
diff --git a/brep/submit/submit.bash.in b/brep/submit/submit.bash.in
index d6f0258..8315315 100644
--- a/brep/submit/submit.bash.in
+++ b/brep/submit/submit.bash.in
@@ -2,10 +2,6 @@
# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-# @@ Should we make this module besides defining utility functions to also
-# parse and remove from args the common arguments (directory)?
-#
-
if [ "$brep_submit" ]; then
return 0
else
@@ -16,11 +12,14 @@ fi
# Diagnostics.
#
-self="$(basename $0)"
-# Result reference (assigned later).
+# We expect the user to set the verbose variable either to true or empty
+# (false).
#
-reference=
+if [ ! -v verbose ]; then
+ echo "variable 'verbose' is not set" 2>&1
+ exit 1
+fi
# Normally the brep module's log record looks like this:
#
@@ -29,6 +28,9 @@ reference=
# We will use the (almost) same format for our diagnostics (redirected to the
# Apache's error_log) so it can easily be attributed to the brep module.
#
+info_self="$(basename $0)"
+info_ref="$(basename "${!#/}")" # Last argument is the submission directory.
+
function info () # <severity> <text>
{
local severity="$1"
@@ -41,30 +43,12 @@ function info () # <severity> <text>
ts=
fi
- echo "[$ts] [brep:$severity] [ref $reference] [$self]: $*" 1>&2;
+ echo "[$ts] [brep:$severity] [ref $info_ref] [$info_self]: $*" 1>&2;
}
function error () { info "error" "$*"; exit 1; }
function trace () { if [ "$verbose" ]; then info "info" "$*"; fi }
-# Submission data directory.
-#
-# @@ Doesn't comply with implementation assuming that directory comes before
-# submit-handler-arguments. Can we change the doc?
-#
-dir="${1%/}" # @@ Call it directory?
-shift
-
-if [ -z "$dir" ]; then
- error "$usage"
-fi
-
-if [ ! -d "$dir" ]; then
- error "'$dir' does not exist or is not a directory"
-fi
-
-reference="$(basename $dir)"
-
# Serialize one manifest name/value pair.
#
function serialize () # <name> <value>
diff --git a/build/root.build b/build/root.build
index 63e4c69..b928a3f 100644
--- a/build/root.build
+++ b/build/root.build
@@ -25,8 +25,12 @@ cxx.coptions += -Wno-unknown-pragmas
#
using? cli
+# Bash.
+#
using bash
+brep/bash{*}: install.subdirs = true
+
# All exe{} in tests/ are, well, tests. Also don't link whole archives
# by default there.
#