aboutsummaryrefslogtreecommitdiff
path: root/brep
diff options
context:
space:
mode:
Diffstat (limited to 'brep')
-rw-r--r--brep/handler/ci/.gitignore1
-rw-r--r--brep/handler/ci/buildfile5
-rw-r--r--brep/handler/ci/ci-dir.in21
-rw-r--r--brep/handler/ci/ci-load.in186
-rw-r--r--brep/handler/ci/ci.bash.in24
-rw-r--r--brep/handler/submit/submit-dir.in2
-rw-r--r--brep/handler/submit/submit-git.bash.in11
-rw-r--r--brep/handler/submit/submit-git.in2
-rw-r--r--brep/handler/submit/submit.bash.in1
9 files changed, 237 insertions, 16 deletions
diff --git a/brep/handler/ci/.gitignore b/brep/handler/ci/.gitignore
index f31b542..8a7b2cf 100644
--- a/brep/handler/ci/.gitignore
+++ b/brep/handler/ci/.gitignore
@@ -1 +1,2 @@
brep-ci-dir
+brep-ci-load
diff --git a/brep/handler/ci/buildfile b/brep/handler/ci/buildfile
index c45a79b..ad57e33 100644
--- a/brep/handler/ci/buildfile
+++ b/brep/handler/ci/buildfile
@@ -2,10 +2,11 @@
# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-./: exe{brep-ci-dir}
+./: exe{brep-ci-dir} exe{brep-ci-load}
include ../
-exe{brep-ci-dir}: in{ci-dir} bash{ci} ../bash{handler}
+exe{brep-ci-dir}: in{ci-dir} bash{ci} ../bash{handler}
+exe{brep-ci-load}: in{ci-load} bash{ci} ../bash{handler}
bash{ci}: in{ci} ../bash{handler}
diff --git a/brep/handler/ci/ci-dir.in b/brep/handler/ci/ci-dir.in
index 6a4f0af..2915b25 100644
--- a/brep/handler/ci/ci-dir.in
+++ b/brep/handler/ci/ci-dir.in
@@ -6,13 +6,18 @@
# Simple package CI request handler with directory storage.
#
-# Keep the CI request directory unless simulating. Write the CI result
-# manifest to stdout.
+# Dump the repositories.manifest and packages.manifest files into the CI
+# request data directory as a sanity check. Keep the directory unless
+# simulating. Write the CI result manifest to stdout.
#
usage="usage: $0 <dir>"
verbose= #true
+# Repository information fetch timeout (seconds).
+#
+fetch_timeout=60
+
trap "{ exit 1; }" ERR
set -o errtrace # Trap ERR in functions.
@@ -80,14 +85,20 @@ fi
spec="$spec$repository"
+# Exit with the 'CI request is queued' response if simulating.
+#
+# Note that we can't assume a real repository URL is specified if simulating
+# so trying to query the repository info is not a good idea.
+#
if [ -n "$simulate" ]; then
- rm -r "$data_dir"
+ run rm -r "$data_dir"
trace "CI request for '$spec' is simulated"
else
+ dump_repository_manifests "$repository" "$data_dir" "$fetch_timeout"
trace "CI request for '$spec' is queued"
fi
-# The spec normally contains the full commit id and so feels too hairy for
-# including in the message.
+# The spec normally contains the full commit id and so feels too hairy to
+# include in the result manifest message.
#
exit_with_manifest 200 "CI request is queued"
diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in
new file mode 100644
index 0000000..98078b3
--- /dev/null
+++ b/brep/handler/ci/ci-load.in
@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+
+# file : brep/handler/ci/ci-load.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Package CI request handler that loads the packages into the brep database.
+#
+# --result-url <url>
+# Result URL base for the response. If specified, the handler will append the
+# brep tenant id to this value and include the resulting URL in the response
+# message.
+#
+# <loader-path>
+# Loader program (normally brep-load(1)).
+#
+# <loader-options>
+# Loader options (normally --db-*).
+#
+usage="usage: $0 [--result-url <url>] <loader-path> [<loader-options>] <dir>"
+
+verbose= #true
+
+# Repository information fetch timeout (seconds).
+#
+fetch_timeout=60
+
+trap "{ exit 1; }" ERR
+set -o errtrace # Trap ERR in functions.
+
+@import brep/handler/handler@
+@import brep/handler/ci/ci@
+
+# The handler's own options.
+#
+result_url=
+while [ $# -gt 0 ]; do
+ case $1 in
+ --result-url)
+ shift
+ result_url="${1%/}"
+ shift
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+# The loader path.
+#
+loader="$1"
+
+if [ -z "$loader" ]; then
+ error "$usage"
+fi
+
+shift
+
+# Assume that the remaining arguments except the last one are the loader
+# options.
+#
+loader_options=()
+while [ $# -gt 1 ]; do
+ loader_options+=("$1")
+ shift
+done
+
+# CI request data directory (last argument).
+#
+data_dir="$1"
+
+if [ -z "$data_dir" ]; then
+ error "$usage"
+fi
+
+if [ ! -d "$data_dir" ]; then
+ error "'$data_dir' does not exist or is not a directory"
+fi
+
+reference="$(basename "$data_dir")"
+
+# Parse the CI request manifest and obtain the repository URL, package names
+# with optional versions, as well as the simulate value.
+#
+manifest_parser_start "$data_dir/request.manifest"
+
+repository=
+packages=()
+simulate=
+
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+ case "$n" in
+ repository) repository="$v" ;;
+ package) packages+=("$v") ;;
+ simulate) simulate="$v" ;;
+ esac
+done
+
+manifest_parser_finish
+
+if [ -z "$repository" ]; then
+ error "repository manifest value expected"
+fi
+
+if [ -n "$simulate" -a "$simulate" != "success" ]; then
+ exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
+fi
+
+# Produce the bpkg-build(1)-like package spec for tracing.
+#
+# The spec normally contains the full commit id and so feels too hairy to
+# include in the result manifest message.
+#
+spec=
+for p in "${packages[@]}"; do
+ if [ -n "$spec" ]; then
+ spec="$spec,"
+ fi
+ spec="$spec$p"
+done
+
+if [ -n "$spec" ]; then
+ spec="$spec@"
+fi
+
+spec="$spec$repository"
+
+message_suffix=
+if [ -n "$result_url" ]; then
+ message_suffix=": $result_url"
+fi
+
+# Exit with the 'CI request is queued' response if simulating.
+#
+# Note that we can't assume a real repository URL is specified if simulating
+# so trying to query the repository info is not a good idea.
+#
+if [ -n "$simulate" ]; then
+ run rm -r "$data_dir"
+
+ trace "CI request for '$spec' is simulated$message_suffix"
+ exit_with_manifest 200 "CI request is queued$message_suffix"
+fi
+
+# Dump the repositories.manifest and packages.manifest files.
+#
+run mkdir "$data_dir/cache"
+dump_repository_manifests "$repository" "$data_dir/cache" "$fetch_timeout"
+
+# In most cases all the requested for CI packages belong to the same project,
+# which would be nice to use as the repository display name. However, that
+# would require parsing packages.manifest just to get this information. Which
+# feels like a bit of an overkill. So for now let's just use the leaf
+# component of the repository URL since it will be the same as the project
+# name in most (sane) cases.
+#
+# First, strip the URL query and fragment parts, then prefix, and, finally,
+# extension.
+#
+display_name="$(sed -r \
+-e 's%^([^?#]*).*$%\1%' \
+-e 's%^.*/([^/]+)/?$%\1%' \
+-e 's%(\.[^.]*)$%%' \
+<<<"$repository")"
+
+# Create the brep-load(1) loadtab file.
+#
+loadtab="$data_dir/loadtab"
+run echo "$repository $display_name cache:cache" >"$loadtab"
+
+# Load the repository into the brep package database.
+#
+# Note that for now we load all the packages the repository contains without
+# regard to the request manifest package values. Later, we could add filtering
+# of the packages.manifest file against the request manifest values. While at
+# it, we could also deduce the repository display name (see above). @@ TODO
+#
+run "$loader" "${loader_options[@]}" --force --shallow "$loadtab"
+
+# Remove the no longer needed CI request data directory.
+#
+run rm -r "$data_dir"
+
+trace "CI request for '$spec' is queued$message_suffix"
+exit_with_manifest 200 "CI request is queued$message_suffix"
diff --git a/brep/handler/ci/ci.bash.in b/brep/handler/ci/ci.bash.in
index 023e98e..d5af34b 100644
--- a/brep/handler/ci/ci.bash.in
+++ b/brep/handler/ci/ci.bash.in
@@ -39,3 +39,27 @@ function exit_with_manifest () # <status> <message>
manifest_serializer_finish
run exit 0
}
+
+# Dump the repositories.manifest and packages.manifest files into the
+# specified directory by running bpkg-rep-info command for the specified
+# repository and using the specified fetch timeout (in seconds).
+#
+function dump_repository_manifests () # <repo-url> <dir> <timeout>
+{
+ local url="$1"
+ local dir="$2"
+ local tmo="$3"
+
+ if ! run_silent bpkg rep-info --fetch-timeout "$tmo" --manifest \
+--repositories --repositories-file "$dir/repositories.manifest" \
+--packages --packages-file "$dir/packages.manifest" "$url"; then
+
+ # Perform the sanity check to make sure that bpkg is runnable.
+ #
+ if ! run bpkg --version >/dev/null; then
+ error "unable to run bpkg"
+ fi
+
+ exit_with_manifest 422 "unable to fetch repository information (run 'bpkg rep-info $url' for details)"
+ fi
+}
diff --git a/brep/handler/submit/submit-dir.in b/brep/handler/submit/submit-dir.in
index bda0bf8..685cf06 100644
--- a/brep/handler/submit/submit-dir.in
+++ b/brep/handler/submit/submit-dir.in
@@ -99,7 +99,7 @@ if [ -z "$project" ]; then
fi
if [ -n "$simulate" ]; then
- rm -r "$data_dir"
+ run rm -r "$data_dir"
trace "$name/$version submission is simulated"
else
trace "$name/$version submission is queued"
diff --git a/brep/handler/submit/submit-git.bash.in b/brep/handler/submit/submit-git.bash.in
index c77f18d..b1bb357 100644
--- a/brep/handler/submit/submit-git.bash.in
+++ b/brep/handler/submit/submit-git.bash.in
@@ -127,10 +127,7 @@ function repository_base () # <repo-url>
{
# First, strip the URL query part, then component.
#
- sed -n \
--e 's%^\([^?]*\).*$%\1%' \
--e 's%^\(.*/\)[^/]\{1,\}/\{0,1\}$%\1%p' \
-<<<"$1"
+ sed -n -r -e 's%^([^?]*).*$%\1%' -e 's%^(.*/)[^/]+/?$%\1%p' <<<"$1"
}
# Authenticate the project name owner. Make sure that the control manifest
@@ -296,7 +293,7 @@ function auth_package_unknown () # <package> <repo-dir>
#
function url_scheme () # <url>
{
- sed -n -e 's%^\(.*\)://.*$%\L\1%p' <<<"$1"
+ sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1"
}
# Check that the repository properly responds to the probing request before
@@ -320,8 +317,8 @@ function check_connectivity () # <repo-url> <timeout> <ours>
if [ "$s" == "http" -o "$s" == "https" ]; then
local u q
- u="$(sed -n -e 's%^\([^?]*\).*$%\1%p' <<<"$url")" # Strips query part.
- q="$(sed -n -e 's%^[^?]*\(.*\)$%\1%p' <<<"$url")" # Query part.
+ u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part.
+ q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part.
if [ -z "$q" ]; then
u="$u/info/refs?service=git-upload-pack"
diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in
index a403377..bc8ecf0 100644
--- a/brep/handler/submit/submit-git.in
+++ b/brep/handler/submit/submit-git.in
@@ -387,7 +387,7 @@ function git_add () # <repo-dir> <path>...
run git -C "$d" add $gvo "$@" >&2
}
-# Dor now we make 10 re-tries to add the package and push to target. Push can
+# For now we make 10 re-tries to add the package and push to target. Push can
# fail due to the target-to-reference information move race (see the above
# notes for details) or because concurrent submissions. We may want to make it
# configurable in the future.
diff --git a/brep/handler/submit/submit.bash.in b/brep/handler/submit/submit.bash.in
index 38a4e06..8ea510f 100644
--- a/brep/handler/submit/submit.bash.in
+++ b/brep/handler/submit/submit.bash.in
@@ -49,6 +49,7 @@ function extract_package_manifest () # <archive> <manifest>
local man="$2"
if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then
+
# Perform the sanity check to make sure that bpkg is runnable.
#
if ! run bpkg --version >/dev/null; then