From b1888e516c9c9d750726318227bf69856ec91b8b Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Fri, 31 Aug 2018 21:41:53 +0300 Subject: Add ci-load --- brep/handler/ci/.gitignore | 1 + brep/handler/ci/buildfile | 5 +- brep/handler/ci/ci-dir.in | 21 ++- brep/handler/ci/ci-load.in | 186 +++++++++++++++++++++ brep/handler/ci/ci.bash.in | 24 +++ brep/handler/submit/submit-dir.in | 2 +- brep/handler/submit/submit-git.bash.in | 11 +- brep/handler/submit/submit-git.in | 2 +- brep/handler/submit/submit.bash.in | 1 + .../request.manifest | 7 + tests/ci/README | 11 ++ tests/ci/buildfile | 6 +- tests/ci/ci-dir.test | 19 ++- tests/ci/ci-load.test | 141 ++++++++++++++++ tests/ci/data.test | 32 ++-- tests/submit/0f6b1460b3ec/package.manifest | 10 -- tests/submit/0f6b1460b3ec/result.manifest | 4 - tests/submit/README | 12 +- tests/submit/submit-dir.test | 2 +- tests/submit/submit-git.test | 11 +- 20 files changed, 450 insertions(+), 58 deletions(-) create mode 100644 brep/handler/ci/ci-load.in create mode 100644 tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest create mode 100644 tests/ci/README create mode 100644 tests/ci/ci-load.test delete mode 100644 tests/submit/0f6b1460b3ec/package.manifest delete mode 100644 tests/submit/0f6b1460b3ec/result.manifest diff --git a/brep/handler/ci/.gitignore b/brep/handler/ci/.gitignore index f31b542..8a7b2cf 100644 --- a/brep/handler/ci/.gitignore +++ b/brep/handler/ci/.gitignore @@ -1 +1,2 @@ brep-ci-dir +brep-ci-load diff --git a/brep/handler/ci/buildfile b/brep/handler/ci/buildfile index c45a79b..ad57e33 100644 --- a/brep/handler/ci/buildfile +++ b/brep/handler/ci/buildfile @@ -2,10 +2,11 @@ # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -./: exe{brep-ci-dir} +./: exe{brep-ci-dir} exe{brep-ci-load} include ../ -exe{brep-ci-dir}: in{ci-dir} bash{ci} ../bash{handler} +exe{brep-ci-dir}: in{ci-dir} bash{ci} ../bash{handler} +exe{brep-ci-load}: in{ci-load} bash{ci} ../bash{handler} bash{ci}: in{ci} ../bash{handler} diff --git a/brep/handler/ci/ci-dir.in b/brep/handler/ci/ci-dir.in index 6a4f0af..2915b25 100644 --- a/brep/handler/ci/ci-dir.in +++ b/brep/handler/ci/ci-dir.in @@ -6,13 +6,18 @@ # Simple package CI request handler with directory storage. # -# Keep the CI request directory unless simulating. Write the CI result -# manifest to stdout. +# Dump the repositories.manifest and packages.manifest files into the CI +# request data directory as a sanity check. Keep the directory unless +# simulating. Write the CI result manifest to stdout. # usage="usage: $0 " verbose= #true +# Repository information fetch timeout (seconds). +# +fetch_timeout=60 + trap "{ exit 1; }" ERR set -o errtrace # Trap ERR in functions. @@ -80,14 +85,20 @@ fi spec="$spec$repository" +# Exit with the 'CI request is queued' response if simulating. +# +# Note that we can't assume a real repository URL is specified if simulating +# so trying to query the repository info is not a good idea. +# if [ -n "$simulate" ]; then - rm -r "$data_dir" + run rm -r "$data_dir" trace "CI request for '$spec' is simulated" else + dump_repository_manifests "$repository" "$data_dir" "$fetch_timeout" trace "CI request for '$spec' is queued" fi -# The spec normally contains the full commit id and so feels too hairy for -# including in the message. +# The spec normally contains the full commit id and so feels too hairy to +# include in the result manifest message. # exit_with_manifest 200 "CI request is queued" diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in new file mode 100644 index 0000000..98078b3 --- /dev/null +++ b/brep/handler/ci/ci-load.in @@ -0,0 +1,186 @@ +#!/usr/bin/env bash + +# file : brep/handler/ci/ci-load.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Package CI request handler that loads the packages into the brep database. +# +# --result-url +# Result URL base for the response. If specified, the handler will append the +# brep tenant id to this value and include the resulting URL in the response +# message. +# +# +# Loader program (normally brep-load(1)). +# +# +# Loader options (normally --db-*). +# +usage="usage: $0 [--result-url ] [] " + +verbose= #true + +# Repository information fetch timeout (seconds). +# +fetch_timeout=60 + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +@import brep/handler/handler@ +@import brep/handler/ci/ci@ + +# The handler's own options. +# +result_url= +while [ $# -gt 0 ]; do + case $1 in + --result-url) + shift + result_url="${1%/}" + shift + ;; + *) + break + ;; + esac +done + +# The loader path. +# +loader="$1" + +if [ -z "$loader" ]; then + error "$usage" +fi + +shift + +# Assume that the remaining arguments except the last one are the loader +# options. +# +loader_options=() +while [ $# -gt 1 ]; do + loader_options+=("$1") + shift +done + +# CI request data directory (last argument). +# +data_dir="$1" + +if [ -z "$data_dir" ]; then + error "$usage" +fi + +if [ ! -d "$data_dir" ]; then + error "'$data_dir' does not exist or is not a directory" +fi + +reference="$(basename "$data_dir")" + +# Parse the CI request manifest and obtain the repository URL, package names +# with optional versions, as well as the simulate value. +# +manifest_parser_start "$data_dir/request.manifest" + +repository= +packages=() +simulate= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + repository) repository="$v" ;; + package) packages+=("$v") ;; + simulate) simulate="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$repository" ]; then + error "repository manifest value expected" +fi + +if [ -n "$simulate" -a "$simulate" != "success" ]; then + exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" +fi + +# Produce the bpkg-build(1)-like package spec for tracing. +# +# The spec normally contains the full commit id and so feels too hairy to +# include in the result manifest message. +# +spec= +for p in "${packages[@]}"; do + if [ -n "$spec" ]; then + spec="$spec," + fi + spec="$spec$p" +done + +if [ -n "$spec" ]; then + spec="$spec@" +fi + +spec="$spec$repository" + +message_suffix= +if [ -n "$result_url" ]; then + message_suffix=": $result_url" +fi + +# Exit with the 'CI request is queued' response if simulating. +# +# Note that we can't assume a real repository URL is specified if simulating +# so trying to query the repository info is not a good idea. +# +if [ -n "$simulate" ]; then + run rm -r "$data_dir" + + trace "CI request for '$spec' is simulated$message_suffix" + exit_with_manifest 200 "CI request is queued$message_suffix" +fi + +# Dump the repositories.manifest and packages.manifest files. +# +run mkdir "$data_dir/cache" +dump_repository_manifests "$repository" "$data_dir/cache" "$fetch_timeout" + +# In most cases all the requested for CI packages belong to the same project, +# which would be nice to use as the repository display name. However, that +# would require parsing packages.manifest just to get this information. Which +# feels like a bit of an overkill. So for now let's just use the leaf +# component of the repository URL since it will be the same as the project +# name in most (sane) cases. +# +# First, strip the URL query and fragment parts, then prefix, and, finally, +# extension. +# +display_name="$(sed -r \ +-e 's%^([^?#]*).*$%\1%' \ +-e 's%^.*/([^/]+)/?$%\1%' \ +-e 's%(\.[^.]*)$%%' \ +<<<"$repository")" + +# Create the brep-load(1) loadtab file. +# +loadtab="$data_dir/loadtab" +run echo "$repository $display_name cache:cache" >"$loadtab" + +# Load the repository into the brep package database. +# +# Note that for now we load all the packages the repository contains without +# regard to the request manifest package values. Later, we could add filtering +# of the packages.manifest file against the request manifest values. While at +# it, we could also deduce the repository display name (see above). @@ TODO +# +run "$loader" "${loader_options[@]}" --force --shallow "$loadtab" + +# Remove the no longer needed CI request data directory. +# +run rm -r "$data_dir" + +trace "CI request for '$spec' is queued$message_suffix" +exit_with_manifest 200 "CI request is queued$message_suffix" diff --git a/brep/handler/ci/ci.bash.in b/brep/handler/ci/ci.bash.in index 023e98e..d5af34b 100644 --- a/brep/handler/ci/ci.bash.in +++ b/brep/handler/ci/ci.bash.in @@ -39,3 +39,27 @@ function exit_with_manifest () # manifest_serializer_finish run exit 0 } + +# Dump the repositories.manifest and packages.manifest files into the +# specified directory by running bpkg-rep-info command for the specified +# repository and using the specified fetch timeout (in seconds). +# +function dump_repository_manifests () # +{ + local url="$1" + local dir="$2" + local tmo="$3" + + if ! run_silent bpkg rep-info --fetch-timeout "$tmo" --manifest \ +--repositories --repositories-file "$dir/repositories.manifest" \ +--packages --packages-file "$dir/packages.manifest" "$url"; then + + # Perform the sanity check to make sure that bpkg is runnable. + # + if ! run bpkg --version >/dev/null; then + error "unable to run bpkg" + fi + + exit_with_manifest 422 "unable to fetch repository information (run 'bpkg rep-info $url' for details)" + fi +} diff --git a/brep/handler/submit/submit-dir.in b/brep/handler/submit/submit-dir.in index bda0bf8..685cf06 100644 --- a/brep/handler/submit/submit-dir.in +++ b/brep/handler/submit/submit-dir.in @@ -99,7 +99,7 @@ if [ -z "$project" ]; then fi if [ -n "$simulate" ]; then - rm -r "$data_dir" + run rm -r "$data_dir" trace "$name/$version submission is simulated" else trace "$name/$version submission is queued" diff --git a/brep/handler/submit/submit-git.bash.in b/brep/handler/submit/submit-git.bash.in index c77f18d..b1bb357 100644 --- a/brep/handler/submit/submit-git.bash.in +++ b/brep/handler/submit/submit-git.bash.in @@ -127,10 +127,7 @@ function repository_base () # { # First, strip the URL query part, then component. # - sed -n \ --e 's%^\([^?]*\).*$%\1%' \ --e 's%^\(.*/\)[^/]\{1,\}/\{0,1\}$%\1%p' \ -<<<"$1" + sed -n -r -e 's%^([^?]*).*$%\1%' -e 's%^(.*/)[^/]+/?$%\1%p' <<<"$1" } # Authenticate the project name owner. Make sure that the control manifest @@ -296,7 +293,7 @@ function auth_package_unknown () # # function url_scheme () # { - sed -n -e 's%^\(.*\)://.*$%\L\1%p' <<<"$1" + sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1" } # Check that the repository properly responds to the probing request before @@ -320,8 +317,8 @@ function check_connectivity () # if [ "$s" == "http" -o "$s" == "https" ]; then local u q - u="$(sed -n -e 's%^\([^?]*\).*$%\1%p' <<<"$url")" # Strips query part. - q="$(sed -n -e 's%^[^?]*\(.*\)$%\1%p' <<<"$url")" # Query part. + u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part. + q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part. if [ -z "$q" ]; then u="$u/info/refs?service=git-upload-pack" diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in index a403377..bc8ecf0 100644 --- a/brep/handler/submit/submit-git.in +++ b/brep/handler/submit/submit-git.in @@ -387,7 +387,7 @@ function git_add () # ... run git -C "$d" add $gvo "$@" >&2 } -# Dor now we make 10 re-tries to add the package and push to target. Push can +# For now we make 10 re-tries to add the package and push to target. Push can # fail due to the target-to-reference information move race (see the above # notes for details) or because concurrent submissions. We may want to make it # configurable in the future. diff --git a/brep/handler/submit/submit.bash.in b/brep/handler/submit/submit.bash.in index 38a4e06..8ea510f 100644 --- a/brep/handler/submit/submit.bash.in +++ b/brep/handler/submit/submit.bash.in @@ -49,6 +49,7 @@ function extract_package_manifest () # local man="$2" if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then + # Perform the sanity check to make sure that bpkg is runnable. # if ! run bpkg --version >/dev/null; then diff --git a/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest b/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest new file mode 100644 index 0000000..3ef553a --- /dev/null +++ b/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest @@ -0,0 +1,7 @@ +: 1 +id: 8716f424-fd94-4def-9e2e-687203bbf4ad +repository: https://git.build2.org/hello/hello.git#master +package: hello +timestamp: 2018-09-01T08:38:55Z +client-ip: fe80::56e1:adff:fe83:82f5 +user-agent: curl/7.59.0 diff --git a/tests/ci/README b/tests/ci/README new file mode 100644 index 0000000..14defe5 --- /dev/null +++ b/tests/ci/README @@ -0,0 +1,11 @@ +Prepare the test data with the following instructions. + +Locally run brep server configured to use ci-dir handler. + +$ curl \ +--form-string repository=https://git.build2.org/hello/hello.git#master \ +--form-string package=hello \ +http://localhost/pkg?ci + +Replace the submission data directory in brep/tests/ci/ with the one produced +with the above command, removing all files it contains except request.manifest. diff --git a/tests/ci/buildfile b/tests/ci/buildfile index b1c3146..1e085e0 100644 --- a/tests/ci/buildfile +++ b/tests/ci/buildfile @@ -11,6 +11,8 @@ include $dir commons = data -./: test{* -{$commons}} common{$commons} $dir/exe{brep-ci-dir} +./: test{* -{$commons}} common{$commons} \ + $dir/exe{brep-ci-dir} $dir/exe{brep-ci-load} -test{ci-dir}@./: test = $out_base/$dir/brep-ci-dir +test{ci-dir}@./: test = $out_base/$dir/brep-ci-dir +test{ci-load}@./: test = $out_base/$dir/brep-ci-load diff --git a/tests/ci/ci-dir.test b/tests/ci/ci-dir.test index 0e98f3a..11de2f0 100644 --- a/tests/ci/ci-dir.test +++ b/tests/ci/ci-dir.test @@ -12,7 +12,7 @@ %\[.+\] \[brep:error\] \[ref \] \[brep-ci-dir\]: usage: .+brep-ci-dir % EOE - : not-exist + : dir-not-exist : $* $~/dir 2>>~%EOE% != 0 %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-dir\]: '.+dir' does not exist or is not a directory% @@ -74,4 +74,21 @@ reference: $request_id EOO } + + : unable-fetch-repo + : + { + $clone_root_data_clean; + + sed -i -e "s%^\(repository:\) .+\$%\\1 http://example.com/repo.git%" \ + $data_dir/request.manifest; + + $* >>~"%EOO%" + : 1 + status: 422 + %message: unable to fetch repository information .+% + %.* + reference: $request_id + EOO + } } diff --git a/tests/ci/ci-load.test b/tests/ci/ci-load.test new file mode 100644 index 0000000..c4a4c62 --- /dev/null +++ b/tests/ci/ci-load.test @@ -0,0 +1,141 @@ +# file : tests/ci/ci-load.test +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +.include data.test + +: args +{ + : none + : + $* 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref \] \[brep-ci-load\]: usage: .+brep-ci-load \[--result-url \] \[\] % + EOE + + : no-dir + : + $* brep-load 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref brep-load\] \[brep-ci-load\]: usage: .+brep-ci-load \[--result-url \] \[\] % + EOE + + : dir-not-exist + : + $* brep-load $~/dir 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-load\]: '.+dir' does not exist or is not a directory% + EOE +} + +: success +: +{ + : simulate + : + { + $clone_root_data; + + echo 'simulate: success' >+$data_dir/request.manifest; + + $* ./brep-load $data_dir >>"EOO"; + : 1 + status: 200 + message: CI request is queued + reference: $request_id + EOO + + test -d $data_dir != 0 + } + + : for-real + : + { + # Create the loader script that validates the arguments passed and the + # files produced by the handler. + # + loader=$~/brep-load + + cat <<"EOI" >=$loader; + #!/usr/bin/env bash + if [ "\$#" != 5 -o \ + "\$1" != "--db-host=localhost" -o \ + "\$2" != "--db-port=8432" -o \ + "\$3" != "--force" -o \ + "\$4" != "--shallow" -o \ + "\$5" != "$data_dir/loadtab" -o \ + ! -f "$data_dir/cache/repositories.manifest" -o \ + ! -f "$data_dir/cache/packages.manifest" ]; then + exit 1 + fi + if ! diff "$data_dir/loadtab" - <<<"https://git.build2.org/hello/hello.git#master hello cache:cache"; then + exit 1 + fi + EOI + + chmod 755 $loader; + + $clone_root_data; + + $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO" + : 1 + status: 200 + message: CI request is queued + reference: $request_id + EOO + + $clone_root_data; + + $* --result-url "http://example.com/" \ + $loader --db-host=localhost --db-port=8432 \ + $data_dir >>"EOO" + : 1 + status: 200 + message: CI request is queued: http://example.com + reference: $request_id + EOO + } +} + +: failure +: +{ + test.arguments += ./brep-load $data_dir + + : bad-simulate + : + { + $clone_root_data_clean; + + echo 'simulate: fly' >+$data_dir/request.manifest; + + $* >>"EOO" + : 1 + status: 400 + message: unrecognized simulation outcome 'fly' + reference: $request_id + EOO + } + + : unable-fetch-repo + : + { + $clone_root_data_clean; + + sed -i -e "s%^\(repository:\) .+\$%\\1 http://example.com/repo.git%" \ + $data_dir/request.manifest; + + $* >>~"%EOO%" + : 1 + status: 422 + %message: unable to fetch repository information .+% + %.* + reference: $request_id + EOO + } + + : loader-not-exist + : + { + $clone_root_data_clean; + + $* 2>~'%.*brep-load.*%' != 0 + } +} diff --git a/tests/ci/data.test b/tests/ci/data.test index c30a691..19383b1 100644 --- a/tests/ci/data.test +++ b/tests/ci/data.test @@ -2,28 +2,24 @@ # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -# Pre-created CI request submission data directory that will be copied by -# subsequent tests and scope setup commands. The common approach will be that -# group scopes copy and modify the parent scope submission directory as -# required by the nested tests and scopes. Tests will also clone the parent -# scope submission data directory to optionally modify it, use and cleanup at -# the end. Note that configuration can not be shared between multiple -# submission handler processes. Also we need to make sure that submission data +# Pre-created CI request data directory that will be copied by subsequent +# tests and scope setup commands. The common approach will be that group +# scopes copy and modify the parent scope submission directory as required +# by the nested tests and scopes. Tests will also clone the parent scope +# submission data directory to optionally modify it, use and cleanup at the +# end. Note that configuration can not be shared between multiple submission +# handler processes. Also we need to make sure that submission data # directories are not cloned while being used by submission handler scripts. # -request_id = 4cfa00ec-8459-4f4f-9ef0-8883ddcc4f5b -data_dir = $request_id/ +data_dir = $regex.replace($path_search('*/request.manifest', $src_base), \ + '(.*)/.*', \ + '\1') -+mkdir $data_dir +request_id = "$data_dir" -+cat <<"EOI" >=$data_dir/request.manifest - : 1 - id: $request_id - repository: https://example.com/hello.git#master - package: foo - package: bar/1.0 - timestamp: 2018-08-24T18:08:01Z - EOI +# Copy the original CI request data directory to the root scope. +# ++cp -r $src_base/$data_dir ./ root_data_dir = $~/$data_dir diff --git a/tests/submit/0f6b1460b3ec/package.manifest b/tests/submit/0f6b1460b3ec/package.manifest deleted file mode 100644 index 6fc36e7..0000000 --- a/tests/submit/0f6b1460b3ec/package.manifest +++ /dev/null @@ -1,10 +0,0 @@ -: 1 -name: libhello -version: 0.1.0 -project: hello -summary: hello library -license: TODO -url: https://example.org/hello -email: user@example.org -depends: * build2 >= 0.8.0- -depends: * bpkg >= 0.8.0- diff --git a/tests/submit/0f6b1460b3ec/result.manifest b/tests/submit/0f6b1460b3ec/result.manifest deleted file mode 100644 index 93a3555..0000000 --- a/tests/submit/0f6b1460b3ec/result.manifest +++ /dev/null @@ -1,4 +0,0 @@ -: 1 -status: 200 -message: libhello/0.1.0 submission is queued -reference: 0f6b1460b3ec diff --git a/tests/submit/README b/tests/submit/README index 7286c83..49ebd1e 100644 --- a/tests/submit/README +++ b/tests/submit/README @@ -22,9 +22,13 @@ Move the archive into brep/tests/submit/ directory. Locally run brep server configured to use submit-dir handler. -$ bdep publish --control http://example.org/hello.git \ - --email user@example.org --repository http://localhost/pkg --yes \ - -d hello +$ bdep publish \ +--control http://example.org/hello.git \ +--email user@example.org \ +--repository http://localhost/pkg \ +--yes \ +-d hello Replace the submission data directory in brep/tests/submit/ with the one -produced with the above command. +produced with the above command, removing package.manifest and result.manifest +files it contains. diff --git a/tests/submit/submit-dir.test b/tests/submit/submit-dir.test index a6a7e3e..821898e 100644 --- a/tests/submit/submit-dir.test +++ b/tests/submit/submit-dir.test @@ -12,7 +12,7 @@ %\[.+\] \[brep:error\] \[ref \] \[brep-submit-dir\]: usage: .+brep-submit-dir % EOE - : not-exist + : dir-not-exist : $* $~/dir 2>>~%EOE% != 0 %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-dir\]: '.+dir' does not exist or is not a directory% diff --git a/tests/submit/submit-git.test b/tests/submit/submit-git.test index f96f79d..bedfd5f 100644 --- a/tests/submit/submit-git.test +++ b/tests/submit/submit-git.test @@ -66,15 +66,22 @@ pkg_ctl="$prj_ctl/hello.git" : dir-only : - $* $~/dir 2>>~%EOE% != 0 + $* dir 2>>~%EOE% != 0 %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: usage: .+brep-submit-git \[\] \[\] % EOE : ref-not-exist : - $* "$root_tgt_url" ref $~/dir 2>>~%EOE% != 0 + $* "$root_tgt_url" ref dir 2>>~%EOE% != 0 %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: 'ref' does not exist or is not a directory% EOE + + : dir-not-exist + : + mkdir ref; + $* "$root_tgt_url" ref dir 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: 'dir' does not exist or is not a directory% + EOE } : success -- cgit v1.1