aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2018-09-14 22:33:30 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2018-10-09 19:44:21 +0300
commite97c6b9b9c90a6f97019021d6bfcc73ed92580cc (patch)
tree3639a628d52b7b0ac6fff0eba5cf7e308921bf98
parent1367aa09951e0aa7491bc2a5bf7209b0b47be65e (diff)
Filter packages manifest against CI request manifest in ci-load
-rw-r--r--brep/handler/ci/ci-dir.in2
-rw-r--r--brep/handler/ci/ci-load.in208
-rw-r--r--brep/handler/handler.bash.in2
-rw-r--r--brep/handler/submit/submit-dir.in2
-rw-r--r--brep/handler/submit/submit-git.in2
-rw-r--r--tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest1
-rw-r--r--tests/ci/README29
-rw-r--r--tests/ci/ci-dir.testscript8
-rw-r--r--tests/ci/ci-load.testscript172
-rw-r--r--tests/ci/data.testscript9
-rw-r--r--tests/ci/hello.tar.gzbin0 -> 102400 bytes
-rw-r--r--tests/submit/README22
-rw-r--r--tests/submit/submit-dir.testscript8
-rw-r--r--tests/submit/submit-git.testscript8
14 files changed, 377 insertions, 96 deletions
diff --git a/brep/handler/ci/ci-dir.in b/brep/handler/ci/ci-dir.in
index 2915b25..ee4e88d 100644
--- a/brep/handler/ci/ci-dir.in
+++ b/brep/handler/ci/ci-dir.in
@@ -30,7 +30,7 @@ fi
# CI request data directory (last and the only argument).
#
-data_dir="${!#/}"
+data_dir="${!#%/}"
if [ -z "$data_dir" ]; then
error "$usage"
diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in
index ad96943..e1eaa9c 100644
--- a/brep/handler/ci/ci-load.in
+++ b/brep/handler/ci/ci-load.in
@@ -68,7 +68,7 @@ done
# CI request data directory (last argument).
#
-data_dir="$1"
+data_dir="${1%/}"
if [ -z "$data_dir" ]; then
error "$usage"
@@ -85,20 +85,50 @@ reference="$(basename "$data_dir")"
#
manifest_parser_start "$data_dir/request.manifest"
-repository=
-packages=()
simulate=
+repository=
+
+# Package map. We first enter packages from the request manifest as keys and
+# setting the values to true. Then we go through the repository package list
+# consulting this map and, if found, clearing the value to empty. Finally, we
+# go through the map looking for any "unhandled" packages (value is still
+# true).
+#
+# Note that keys can be in both <name> and <name>/<version> forms.
+#
+declare -A packages
+
+# While at it, produce the bpkg-build(1)-like package spec for tracing.
+#
+# The spec normally contains the full commit id and so feels too hairy to
+# include in the result manifest message.
+#
+spec=
while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
case "$n" in
- repository) repository="$v" ;;
- package) packages+=("$v") ;;
- simulate) simulate="$v" ;;
+ simulate) simulate="$v" ;;
+ repository) repository="$v" ;;
+
+ package)
+ packages["$v"]=true
+
+ if [ -n "$spec" ]; then
+ spec="$spec,"
+ fi
+ spec="$spec$v"
+ ;;
esac
done
manifest_parser_finish
+if [ -n "$spec" ]; then
+ spec="$spec@"
+fi
+
+spec="$spec$repository"
+
if [ -z "$repository" ]; then
error "repository manifest value expected"
fi
@@ -107,25 +137,6 @@ if [ -n "$simulate" -a "$simulate" != "success" ]; then
exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
fi
-# Produce the bpkg-build(1)-like package spec for tracing.
-#
-# The spec normally contains the full commit id and so feels too hairy to
-# include in the result manifest message.
-#
-spec=
-for p in "${packages[@]}"; do
- if [ -n "$spec" ]; then
- spec="$spec,"
- fi
- spec="$spec$p"
-done
-
-if [ -n "$spec" ]; then
- spec="$spec@"
-fi
-
-spec="$spec$repository"
-
message_suffix=
if [ -n "$result_url" ]; then
message_suffix=": $result_url/@$reference" # Append the tenant id.
@@ -145,37 +156,140 @@ fi
# Dump the repositories.manifest and packages.manifest files.
#
-run mkdir "$data_dir/cache"
-dump_repository_manifests "$repository" "$data_dir/cache" "$fetch_timeout"
+cache_dir="$data_dir/cache"
+run mkdir "$cache_dir"
+dump_repository_manifests "$repository" "$cache_dir" "$fetch_timeout"
+
+# Filter the packages manifest keeping only the packages listed in the request
+# manifest. Keep all the packages if the request specified no packages.
+#
+
+# Resulting manifest.
+#
+packages_manifest_names=()
+packages_manifest_values=()
-# In most cases all the requested for CI packages belong to the same project,
-# which would be nice to use as the repository display name. However, that
-# would require parsing packages.manifest just to get this information. Which
-# feels like a bit of an overkill. So for now let's just use the leaf
-# component of the repository URL since it will be the same as the project
-# name in most (sane) cases.
+# While at it, set the repository display name to the first package's project
+# name.
#
-# First, strip the URL query and fragment parts, then prefix, and, finally,
-# extension.
+display_name=
+
+manifest_parser_start "$cache_dir/packages.manifest"
+
+# The outer loop iterates over package manifests while the inner loop iterates
+# over manifest values in each such manifest.
+#
+# Note that the first manifest case is special in that we will see its version
+# value (empty name) first while for others -- last, as part of the previous
+# manifest. We try to deal with this irregularity by reducing the first case
+# (manifest_version is empty) to "as-if" it followed another manifest.
+#
+manifest_names=()
+manifest_values=()
+manifest_version=
+
+more=true
+while [ "$more" ]; do
+
+ if [ -n "$manifest_version" ]; then
+ manifest_names=("")
+ manifest_values=("$manifest_version")
+ fi
+
+ more=
+ project=
+
+ while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+ case "$n" in
+ "") # Start of (next) manifest.
+ more=true
+ manifest_version="$v"
+ break
+ ;;
+
+ name) name="$v" ;;
+ version) version="$v" ;;
+ project) project="$v" ;;
+ esac
+
+ manifest_names+=("$n")
+ manifest_values+=("$v")
+
+ done
+
+ # Reduce the first manifest case.
+ #
+ if [ ${#manifest_names[@]} -eq 0 ]; then
+ continue
+ fi
+
+ # Add or filter out the manifest, if present.
+ #
+ if [ ${#packages[@]} -ne 0 ]; then
+
+ if [[ -v packages["$name"] ]]; then
+ packages["$name"]=
+ packages["$name/$version"]= # Clear it either, as may also be present.
+ elif [[ -v packages["$name/$version"] ]]; then
+ packages["$name/$version"]=
+ else
+ continue # Skip.
+ fi
+
+ fi
+
+ packages_manifest_names+=("${manifest_names[@]}")
+ packages_manifest_values+=("${manifest_values[@]}")
+
+ if [ -z "$display_name" ]; then
+ if [ -n "$project" ]; then
+ display_name="$project"
+ else
+ display_name="$name"
+ fi
+ fi
+done
+
+manifest_parser_finish
+
+# Verify that all the listed in the request manifest packages are present in
+# the repository.
#
-display_name="$(sed -r \
--e 's%^([^?#]*).*$%\1%' \
--e 's%^.*/([^/]+)/?$%\1%' \
--e 's%(\.[^.]*)$%%' \
-<<<"$repository")"
+for p in "${!packages[@]}"; do
+ if [ "${packages[$p]}" ]; then
+ exit_with_manifest 422 "unknown package $p"
+ fi
+done
+
+# Verify that the repository is not empty. Failed that, the repository display
+# name wouldn't be set.
+#
+if [ -z "$display_name" ]; then
+ exit_with_manifest 422 "no packages in repository"
+fi
+
+# Stash the original packages manifest file for troubleshooting.
+#
+run mv "$cache_dir/packages.manifest" "$cache_dir/packages.manifest.orig"
+
+# Serialize the filtered packages manifest.
+#
+manifest_serializer_start "$cache_dir/packages.manifest"
+
+for ((i=0; i <= ${#packages_manifest_names[@]}; ++i)); do
+ manifest_serialize "${packages_manifest_names[$i]}" \
+ "${packages_manifest_values[$i]}"
+done
+
+manifest_serializer_finish
# Create the brep-load(1) loadtab file.
#
loadtab="$data_dir/loadtab"
run echo "$repository $display_name cache:cache" >"$loadtab"
-# Load the repository into the brep package database for the tenant identified
-# by the reference.
-#
-# Note that for now we load all the packages the repository contains without
-# regard to the request manifest package values. Later, we could add filtering
-# of the packages.manifest file against the request manifest values. While at
-# it, we could also deduce the repository display name (see above). @@ TODO
+# Load the requested repository packages into the brep package database for
+# the tenant identified by the reference.
#
run "$loader" "${loader_options[@]}" --force --shallow --tenant "$reference" \
"$loadtab"
diff --git a/brep/handler/handler.bash.in b/brep/handler/handler.bash.in
index 89e7e21..fa287a3 100644
--- a/brep/handler/handler.bash.in
+++ b/brep/handler/handler.bash.in
@@ -37,7 +37,7 @@ if [ "$#" -gt 0 ]; then
# identifies the posted entity. A handler may overwrite this value if that's
# not the case.
#
- info_ref="$(basename "${!#/}")"
+ info_ref="$(basename "${!#%/}")"
fi
function info () # <severity> <text>
diff --git a/brep/handler/submit/submit-dir.in b/brep/handler/submit/submit-dir.in
index 1f677e4..63c191c 100644
--- a/brep/handler/submit/submit-dir.in
+++ b/brep/handler/submit/submit-dir.in
@@ -27,7 +27,7 @@ fi
# Submission data directory (last and the only argument).
#
-data_dir="${!#/}"
+data_dir="${!#%/}"
if [ -z "$data_dir" ]; then
error "$usage"
diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in
index cc9927b..2835f7a 100644
--- a/brep/handler/submit/submit-git.in
+++ b/brep/handler/submit/submit-git.in
@@ -261,7 +261,7 @@ fi
# Submission data directory.
#
-data_dir="$1"
+data_dir="${1%/}"
shift
if [ -z "$data_dir" ]; then
diff --git a/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest b/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest
index 3ef553a..229e5ed 100644
--- a/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest
+++ b/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest
@@ -1,7 +1,6 @@
: 1
id: 8716f424-fd94-4def-9e2e-687203bbf4ad
repository: https://git.build2.org/hello/hello.git#master
-package: hello
timestamp: 2018-09-01T08:38:55Z
client-ip: fe80::56e1:adff:fe83:82f5
user-agent: curl/7.59.0
diff --git a/tests/ci/README b/tests/ci/README
index 14defe5..94a87b6 100644
--- a/tests/ci/README
+++ b/tests/ci/README
@@ -1,11 +1,34 @@
Prepare the test data with the following instructions.
+Create the git repository:
+
+$ mkdir hello.git
+$ git -C hello.git/ init --bare
+
+Create the project:
+
+$ bdep new -t empty hello
+$ BDEP_AUTHOR_EMAIL=user@example.org bdep new --package -t lib libhello -d hello
+$ BDEP_AUTHOR_EMAIL=user@example.org bdep new --package -t exe hello -d hello
+
+Edit hello/libhello/manifest and hello/hello/manifest files setting version to
+0.1.0.
+
+$ git -C hello remote add origin "$(pwd)/hello.git"
+$ git -C hello add '*'
+$ git -C hello commit -m "Create"
+$ git -C hello push --set-upstream origin master
+
+$ tar cf hello.tar.gz hello.git/
+
+Move the archive into tests/ci/ directory.
+
Locally run brep server configured to use ci-dir handler.
$ curl \
--form-string repository=https://git.build2.org/hello/hello.git#master \
---form-string package=hello \
http://localhost/pkg?ci
-Replace the submission data directory in brep/tests/ci/ with the one produced
-with the above command, removing all files it contains except request.manifest.
+Replace the CI request data directory in tests/ci/ with the one produced by
+the above command removing all entries it contains except request.manifest
+file.
diff --git a/tests/ci/ci-dir.testscript b/tests/ci/ci-dir.testscript
index 8388616..f6ff9d3 100644
--- a/tests/ci/ci-dir.testscript
+++ b/tests/ci/ci-dir.testscript
@@ -17,6 +17,14 @@
$* $~/dir 2>>~%EOE% != 0
%\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-dir\]: '.+dir' does not exist or is not a directory%
EOE
+
+ : slash-stripped
+ :
+ : Test that the trailing slash is stripped from the data directory path.
+ :
+ $* $~/dir/ 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-dir\]: '.+dir' does not exist or is not a directory%
+ EOE
}
: success
diff --git a/tests/ci/ci-load.testscript b/tests/ci/ci-load.testscript
index c574a32..ce8daf9 100644
--- a/tests/ci/ci-load.testscript
+++ b/tests/ci/ci-load.testscript
@@ -23,6 +23,14 @@
$* brep-load $~/dir 2>>~%EOE% != 0
%\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-load\]: '.+dir' does not exist or is not a directory%
EOE
+
+ : slash-stripped
+ :
+ : Test that the trailing slash is stripped from the data directory path.
+ :
+ $* brep-load $~/dir/ 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-load\]: '.+dir' does not exist or is not a directory%
+ EOE
}
: success
@@ -48,12 +56,13 @@
: for-real
:
{
- # Create the loader script that validates the arguments passed and the
- # files produced by the handler.
+ # Create the loader script that validates the arguments passed and prints
+ # the loadtab, repositories manifest, and packages manifest files to
+ # stderr.
#
loader=$~/brep-load
- cat <<"EOI" >=$loader;
+ +cat <<"EOI" >=$loader
#!/usr/bin/env bash
if [ "\$#" != 7 -o \
"\$1" != "--db-host=localhost" -o \
@@ -62,38 +71,139 @@
"\$4" != "--shallow" -o \
"\$5" != "--tenant" -o \
-z "\$6" -o \
- "\$7" != "$data_dir/loadtab" -o \
- ! -f "$data_dir/cache/repositories.manifest" -o \
- ! -f "$data_dir/cache/packages.manifest" ]; then
- exit 1
- fi
- if ! diff "$data_dir/loadtab" - <<<"https://git.build2.org/hello/hello.git#master hello cache:cache"; then
+ "\$7" != "$data_dir/loadtab" ]; then
exit 1
fi
+ cat \
+ "$data_dir/loadtab" \
+ "$data_dir/cache/repositories.manifest" \
+ "$data_dir/cache/packages.manifest" >&2
EOI
- chmod 755 $loader;
-
- $clone_root_data;
-
- $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO";
- : 1
- status: 200
- message: CI request is queued
- reference: $request_id
- EOO
-
- $clone_root_data;
-
- $* --result-url "http://example.com/" \
- $loader --db-host=localhost --db-port=8432 \
- $data_dir >>~"%EOO%"
- : 1
- status: 200
- %message: CI request is queued: http://example.com/@.+%
- %.
- reference: $request_id
- EOO
+ +chmod 755 $loader
+
+ : whole-repo
+ :
+ {
+ $clone_root_data;
+
+ $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO" 2>>~"%EOE%"
+ : 1
+ status: 200
+ message: CI request is queued
+ reference: $request_id
+ EOO
+ $rep hello cache:cache
+ : 1
+ summary: hello project repository
+ : 1
+ name: libhello
+ version: 0.1.0
+ project: hello
+ summary: hello library
+ license: TODO
+ url: https://example.org/hello
+ email: user@example.org
+ %depends: \\* build2 .+%
+ %depends: \\* bpkg .+%
+ location: libhello
+ %fragment: .+%
+ :
+ name: hello
+ version: 0.1.0
+ summary: hello executable
+ license: TODO
+ url: https://example.org/hello
+ email: user@example.org
+ %depends: \\* build2 .+%
+ %depends: \\* bpkg .+%
+ location: hello
+ %fragment: .+%
+ EOE
+ }
+
+ : package
+ :
+ {
+ $clone_root_data;
+
+ cat <<EOI >+$data_dir/request.manifest;
+ package: hello
+ EOI
+
+ $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO" 2>>~"%EOE%"
+ : 1
+ status: 200
+ message: CI request is queued
+ reference: $request_id
+ EOO
+ $rep hello cache:cache
+ : 1
+ summary: hello project repository
+ : 1
+ name: hello
+ version: 0.1.0
+ summary: hello executable
+ license: TODO
+ url: https://example.org/hello
+ email: user@example.org
+ %depends: \\* build2 .+%
+ %depends: \\* bpkg .+%
+ location: hello
+ %fragment: .+%
+ EOE
+ }
+
+ : package-version
+ :
+ {
+ $clone_root_data;
+
+ cat <<EOI >+$data_dir/request.manifest;
+ package: libhello/0.1.0
+ EOI
+
+ $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO" 2>>~"%EOE%"
+ : 1
+ status: 200
+ message: CI request is queued
+ reference: $request_id
+ EOO
+ $rep hello cache:cache
+ : 1
+ summary: hello project repository
+ : 1
+ name: libhello
+ version: 0.1.0
+ project: hello
+ summary: hello library
+ license: TODO
+ url: https://example.org/hello
+ email: user@example.org
+ %depends: \\* build2 .+%
+ %depends: \\* bpkg .+%
+ location: libhello
+ %fragment: .+%
+ EOE
+ }
+
+ : result-url
+ :
+ {
+ $clone_root_data;
+
+ $* --result-url "http://example.com/" \
+ $loader --db-host=localhost --db-port=8432 \
+ $data_dir >>~"%EOO%" 2>>~%EOE%
+ : 1
+ status: 200
+ %message: CI request is queued: http://example.com/@.+%
+ %.
+ reference: $request_id
+ EOO
+ %.*:.*%+
+ EOE
+ }
}
}
diff --git a/tests/ci/data.testscript b/tests/ci/data.testscript
index 711c7b1..662a14c 100644
--- a/tests/ci/data.testscript
+++ b/tests/ci/data.testscript
@@ -17,10 +17,19 @@ data_dir = $regex.replace($path_search('*/request.manifest', $src_base), \
request_id = "$data_dir"
+# Prepare the repository.
+#
+rep="file:$~/hello.git#master"
++tar -C $~ -xf $src_base/hello.tar.gz &hello.git/***
+
# Copy the original CI request data directory to the root scope.
#
+cp -r $src_base/$data_dir ./
+# Fix-up the repository value in the request manifest.
+#
++sed -i -e "s%^\(repository:\) .+\$%\\1 $rep%" $data_dir/request.manifest
+
root_data_dir = $~/$data_dir
# The most commonly used submission data directory cloning command that copies
diff --git a/tests/ci/hello.tar.gz b/tests/ci/hello.tar.gz
new file mode 100644
index 0000000..d264888
--- /dev/null
+++ b/tests/ci/hello.tar.gz
Binary files differ
diff --git a/tests/submit/README b/tests/submit/README
index 49ebd1e..1aecbd7 100644
--- a/tests/submit/README
+++ b/tests/submit/README
@@ -1,24 +1,26 @@
Prepare the test data with the following instructions.
-In an empty directory run:
+Create the git repository:
+
+$ mkdir hello.git
+$ git -C hello.git/ init --bare
+
+Create the project:
$ bdep new -t empty -C @cfg hello
-$ BDEP_EMAIL=user@example.org bdep new --package -t lib libhello -d hello
+$ BDEP_AUTHOR_EMAIL=user@example.org bdep new --package -t lib libhello -d hello
$ bdep init -d hello/libhello
Edit hello/libhello/manifest setting version to 0.1.0.
-$ mkdir hello.git
-$ git -C hello.git/ init --bare
-
$ git -C hello remote add origin "$(pwd)/hello.git"
$ git -C hello add '*'
$ git -C hello commit -m "Create"
$ git -C hello push --set-upstream origin master
-tar cf hello.tar.gz hello.git/
+$ tar cf hello.tar.gz hello.git/
-Move the archive into brep/tests/submit/ directory.
+Move the archive into tests/submit/ directory.
Locally run brep server configured to use submit-dir handler.
@@ -29,6 +31,6 @@ $ bdep publish \
--yes \
-d hello
-Replace the submission data directory in brep/tests/submit/ with the one
-produced with the above command, removing package.manifest and result.manifest
-files it contains.
+Replace the submission data directory in tests/submit/ with the one produced
+with the above command removing package.manifest and result.manifest files it
+contains.
diff --git a/tests/submit/submit-dir.testscript b/tests/submit/submit-dir.testscript
index 0b43aa7..8018d01 100644
--- a/tests/submit/submit-dir.testscript
+++ b/tests/submit/submit-dir.testscript
@@ -17,6 +17,14 @@
$* $~/dir 2>>~%EOE% != 0
%\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-dir\]: '.+dir' does not exist or is not a directory%
EOE
+
+ : slash-stripped
+ :
+ : Test that the trailing slash is stripped from the data directory path.
+ :
+ $* $~/dir/ 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-dir\]: '.+dir' does not exist or is not a directory%
+ EOE
}
: success
diff --git a/tests/submit/submit-git.testscript b/tests/submit/submit-git.testscript
index e99c1d5..a0d76f5 100644
--- a/tests/submit/submit-git.testscript
+++ b/tests/submit/submit-git.testscript
@@ -82,6 +82,14 @@ pkg_ctl="$prj_ctl/hello.git"
$* "$root_tgt_url" ref dir 2>>~%EOE% != 0
%\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: 'dir' does not exist or is not a directory%
EOE
+
+ : slash-stripped
+ :
+ : Test that the trailing slash is stripped from the data directory path.
+ :
+ $* "$root_tgt_url" dir/ 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: 'dir' does not exist or is not a directory%
+ EOE
}
: success