From e3a8a11f7fdc64a560810cf021080c61c7d69dc5 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 8 Aug 2018 09:45:48 +0200 Subject: Document submit-git --- brep/submit/.gitignore | 4 +- brep/submit/buildfile | 3 - brep/submit/submit-dir.in | 21 ++++-- brep/submit/submit-git.in | 164 ++++++++++++++++++++++++++++++++++++++++++++- brep/submit/submit.bash.in | 36 +++------- build/root.build | 4 ++ 6 files changed, 194 insertions(+), 38 deletions(-) diff --git a/brep/submit/.gitignore b/brep/submit/.gitignore index b49e361..8df3374 100644 --- a/brep/submit/.gitignore +++ b/brep/submit/.gitignore @@ -1 +1,3 @@ -brep-submit +brep-submit-dir +brep-submit-git + diff --git a/brep/submit/buildfile b/brep/submit/buildfile index f25b6e3..e3711f8 100644 --- a/brep/submit/buildfile +++ b/brep/submit/buildfile @@ -7,6 +7,3 @@ import mods += libbutl.bash%bash{manifest-serializer} exe{brep-submit-dir}: in{submit-dir} bash{submit} $mods bash{submit}: in{submit} $mods # @@ Currently doesn't depend on manifest-parser. - -#install.bin.subdirs = true -#install.subdirs = true diff --git a/brep/submit/submit-dir.in b/brep/submit/submit-dir.in index ce8f134..31ae85d 100644 --- a/brep/submit/submit-dir.in +++ b/brep/submit/submit-dir.in @@ -4,16 +4,15 @@ # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -# Package submission handler example. +# Simple package submission handler with directory storage. # # Validate the package archive located in the specified submission directory -# extracting and parsing the package manifest. Remove the submission directory -# if simulating. Write the submission result manifest to stdout. +# extracting and parsing the package manifest (saved as package.manifest in +# the submission directory). Keep the submission directory unless simulating. +# Write the submission result manifest to stdout. # usage="usage: $0 " -# Diagnostics. -# verbose=true trap "{ exit 1; }" ERR @@ -24,6 +23,18 @@ set -o errtrace # Trap ERR in functions. @import brep/submit/submit@ +# Submission data directory (last argument). +# +dir="${!#/}" + +if [ -z "$dir" ]; then + error "$usage" +fi + +if [ ! -d "$dir" ]; then + error "'$dir' does not exist or is not a directory" +fi + # Parse the submission request manifest and obtain the archive path as well # as the simulate value. # diff --git a/brep/submit/submit-git.in b/brep/submit/submit-git.in index 0238c4e..fbb69de 100644 --- a/brep/submit/submit-git.in +++ b/brep/submit/submit-git.in @@ -4,11 +4,169 @@ # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -# Package submission handler. +# Package submission handler with git repository storage. # -# @@ TODO doc/spec. +# The overall idea behind this handler is to add the package archive into a +# git repository. Another entity (for example, a human or a script) can then +# pull this change and do something about it (for example, review it and/or +# add it to an archive-based repository). In other words, git is used as a +# kind of transport that is easy enough to access for both humans and scripts. # -usage="usage: $0 " +# The handler also implements the project/package name ownership verification +# by performing the submitter authentication/authorization based on the +# control repository mechanism describe in bdep-publish(1). This functionality +# is optional. +# +# The handler can operate with a single git repository, called "target", or +# with two git repositories, in which case the first is the target and the +# second is called "reference". The reference repository access is read-only +# and it is only consulted for duplicate package suppression and name +# ownership verification. The dual repository mode is normally used to +# implement a two-stage queue/publish setup where the package is first queued +# for review and/or testing and then moved (for example, by a moderator) to a +# public repository. +# +# The target repository argument () should be a read-write git +# repository URL. It is cloned (shallow) into the submission directory on +# each submission. +# +# If specified, the reference repository argument () should be a +# directory with a pre-cloned read-only reference repository. This directory +# is shared between all instances of the handler. On each submission, the +# handler will flock(1) this directory, git-pull, obtain the information it +# needs, and release the lock. +# +# Both the target and, if specified, reference repositories should contain the +# submit.config.bash repository configuration file in the root directory. The +# configuration file is a bash fragment and is sourced by the handler script. +# It provides the following information: +# +# - Mapping of section names to repository subdirectories in the 'sections' +# variable (declare -A sections; values are relative to the repository +# root). + +# If there is no key for the submitted section name, then the entry with the +# special '*' key is used. If there is no such entry, then the submission is +# invalid. For example: +# +# sections[alpha]=1/alpha +# sections[beta]=1/beta +# sections[stable]=1/testing +# +# - Optional owners subdirectory in the 'owners' variable (relative to the +# repository root). If not specified, then no ownership verification is +# performed. For example: +# +# owners=owners +# +# If the ownership directory is specified, then the handler script maintains +# the project/package name ownership information in this directory. It has the +# following structure: +# +# / +# ├── / +# │ ├── / +# │ │ └── package-owner.manifest +# │ ├── / +# │ │ └── package-owner.manifest +# │ ├── ... +# │ └── project-owner.manifest +# ├── / +# │ └── ... +# └──... +# +# If the submitted project name is not yet known, then the handler script +# creates a new project subdirectory and saves project-owner.manifest. The +# project owner manifest contain the following values in the specified order: +# +# name: +# email: +# control: +# +# The 'control' value is the control repository URL prefix and there can be +# multiple such values in a single manifest. The handler script derives it +# from the submitted control repository URL by removing the last path +# component. So, for example, https://github.com/build2/libbutl.git becomes +# https://github.com/build2/. +# +# If the submitted project name is already known, then the handler script +# loads its project-owner.manifest and verifies that at least one of the +# 'control' values is a prefix of the submitted control repository URL. +# +# Similarly, if the submitted package name is not yet known, then the handler +# script creates a new package subdirectory and saves package-owner.manifest. +# The package owner manifest contain the following values in the specified +# order: +# +# name: +# email: +# control: +# +# The 'control' value is the control repository URL and there can be multiple +# such values in a single manifest. +# +# If the submission package is already known, then the handler script loads +# its package-owner.manifest and verifies that at least one of the 'control' +# values matches the submitted control repository URL. +# +# If all these ownership authentication tests pass, the handler script clones +# (shallow) the build2-control branch of the control repository and verifies +# that the submission authorization file is present (see bdep-publish(1) for +# details). +# +# If the submission authorization test passes, then the handler script adds +# the package archives to the target repository, commits this change, and +# then pushes the commit to the remote. +# +# Notes: +# +# - It is possible that a submitted package name already exists in another +# project. In this case, such a submission is accepted only if the package +# already exists in the requested project. This allows the moderator to +# manually permit such multi-project packages (for example, to allow moving +# packages between projects). +# +# - There could be a race when moving package and ownership information from +# target to reference. To avoid it, the protocol for such a move is to first +# add, commit, and push to reference and then remove, commit, and push to +# target. +# +# On the handler side, before adding a package or new ownership for a +# project/package name, the script re-checks the reference repository for +# updated information. +# +usage="usage: $0 [] " trap "{ exit 1; }" ERR set -o errtrace # Trap ERR in functions. + +# Implementation notes: +# +# - Check for duplicate package archive in all the sections. Before auth. Use +# -.* instead of .tar.gz in case we support other formats +# later. +# +# - Push permission for target repo (add www-data to scm group)? +# +# - Network errors/timeouts on git pull for ref repo? What is the error (try +# again)? I think also let's not assume target repo is local. + +# Workflow: +# +# 0. The same steps as submit-dir. +# +# 1. If ref-repo specified, lock, pull, and check: +# - duplicate +# - auth (read-only) +# +# 2. Clone tgt-repo, check: +# - duplicate +# - auth (read-write) +# ? if fully auth'd by ref-repo, should we skip it here? +# +# 3. Clone control branch and authorize. +# +# 4. Copy archive, commit and push. If push fails, remove clone and +# restart from 1 (yes, from ref-repo). +# - put submission manifest into commit message for record? +# diff --git a/brep/submit/submit.bash.in b/brep/submit/submit.bash.in index d6f0258..8315315 100644 --- a/brep/submit/submit.bash.in +++ b/brep/submit/submit.bash.in @@ -2,10 +2,6 @@ # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -# @@ Should we make this module besides defining utility functions to also -# parse and remove from args the common arguments (directory)? -# - if [ "$brep_submit" ]; then return 0 else @@ -16,11 +12,14 @@ fi # Diagnostics. # -self="$(basename $0)" -# Result reference (assigned later). +# We expect the user to set the verbose variable either to true or empty +# (false). # -reference= +if [ ! -v verbose ]; then + echo "variable 'verbose' is not set" 2>&1 + exit 1 +fi # Normally the brep module's log record looks like this: # @@ -29,6 +28,9 @@ reference= # We will use the (almost) same format for our diagnostics (redirected to the # Apache's error_log) so it can easily be attributed to the brep module. # +info_self="$(basename $0)" +info_ref="$(basename "${!#/}")" # Last argument is the submission directory. + function info () # { local severity="$1" @@ -41,30 +43,12 @@ function info () # ts= fi - echo "[$ts] [brep:$severity] [ref $reference] [$self]: $*" 1>&2; + echo "[$ts] [brep:$severity] [ref $info_ref] [$info_self]: $*" 1>&2; } function error () { info "error" "$*"; exit 1; } function trace () { if [ "$verbose" ]; then info "info" "$*"; fi } -# Submission data directory. -# -# @@ Doesn't comply with implementation assuming that directory comes before -# submit-handler-arguments. Can we change the doc? -# -dir="${1%/}" # @@ Call it directory? -shift - -if [ -z "$dir" ]; then - error "$usage" -fi - -if [ ! -d "$dir" ]; then - error "'$dir' does not exist or is not a directory" -fi - -reference="$(basename $dir)" - # Serialize one manifest name/value pair. # function serialize () # diff --git a/build/root.build b/build/root.build index 63e4c69..b928a3f 100644 --- a/build/root.build +++ b/build/root.build @@ -25,8 +25,12 @@ cxx.coptions += -Wno-unknown-pragmas # using? cli +# Bash. +# using bash +brep/bash{*}: install.subdirs = true + # All exe{} in tests/ are, well, tests. Also don't link whole archives # by default there. # -- cgit v1.1