#!/usr/bin/env bash # file : brep/handler/ci/ci-load.in # copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Package CI request handler that loads the packages into the brep database. # # --result-url # Result URL base for the response. If specified, the handler will append the # brep tenant id to this value and include the resulting URL in the response # message. # # # Loader program (normally brep-load(1)). # # # Loader options (normally --db-*). # usage="usage: $0 [--result-url ] [] " verbose= #true # Repository information fetch timeout (seconds). # fetch_timeout=60 trap "{ exit 1; }" ERR set -o errtrace # Trap ERR in functions. @import brep/handler/handler@ @import brep/handler/ci/ci@ # The handler's own options. # result_url= while [ $# -gt 0 ]; do case $1 in --result-url) shift result_url="${1%/}" shift ;; *) break ;; esac done # The loader path. # loader="$1" if [ -z "$loader" ]; then error "$usage" fi shift # Assume that the remaining arguments except the last one are the loader # options. # loader_options=() while [ $# -gt 1 ]; do loader_options+=("$1") shift done # CI request data directory (last argument). # data_dir="${1%/}" if [ -z "$data_dir" ]; then error "$usage" fi if [ ! -d "$data_dir" ]; then error "'$data_dir' does not exist or is not a directory" fi reference="$(basename "$data_dir")" # Parse the CI request manifest and obtain the repository URL, package names # with optional versions, as well as the simulate value. # manifest_parser_start "$data_dir/request.manifest" simulate= repository= # Package map. We first enter packages from the request manifest as keys and # setting the values to true. Then we go through the repository package list # consulting this map and, if found, clearing the value to empty. Finally, we # go through the map looking for any "unhandled" packages (value is still # true). # # Note that keys can be in both and / forms. # declare -A packages # While at it, produce the bpkg-build(1)-like package spec for tracing. # # The spec normally contains the full commit id and so feels too hairy to # include in the result manifest message. # spec= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in simulate) simulate="$v" ;; repository) repository="$v" ;; package) packages["$v"]=true if [ -n "$spec" ]; then spec="$spec," fi spec="$spec$v" ;; esac done manifest_parser_finish if [ -n "$spec" ]; then spec="$spec@" fi spec="$spec$repository" if [ -z "$repository" ]; then error "repository manifest value expected" fi if [ -n "$simulate" -a "$simulate" != "success" ]; then exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" fi message_suffix= if [ -n "$result_url" ]; then message_suffix=": $result_url/@$reference" # Append the tenant id. fi # Exit with the 'CI request is queued' response if simulating. # # Note that we can't assume a real repository URL is specified if simulating # so trying to query the repository info is not a good idea. # if [ -n "$simulate" ]; then run rm -r "$data_dir" trace "CI request for '$spec' is simulated$message_suffix" exit_with_manifest 200 "CI request is queued$message_suffix" fi # Dump the repositories.manifest and packages.manifest files. # cache_dir="$data_dir/cache" run mkdir "$cache_dir" dump_repository_manifests "$repository" "$cache_dir" "$fetch_timeout" # Filter the packages manifest keeping only the packages listed in the request # manifest. Keep all the packages if the request specified no packages. # # Resulting manifest. # packages_manifest_names=() packages_manifest_values=() # While at it, set the repository display name to the first package's project # name. # display_name= manifest_parser_start "$cache_dir/packages.manifest" # The outer loop iterates over package manifests while the inner loop iterates # over manifest values in each such manifest. # # Note that the first manifest case is special in that we will see its version # value (empty name) first while for others -- last, as part of the previous # manifest. We try to deal with this irregularity by reducing the first case # (manifest_version is empty) to "as-if" it followed another manifest. # manifest_names=() manifest_values=() manifest_version= more=true while [ "$more" ]; do if [ -n "$manifest_version" ]; then manifest_names=("") manifest_values=("$manifest_version") fi more= project= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in "") # Start of (next) manifest. more=true manifest_version="$v" break ;; name) name="$v" ;; version) version="$v" ;; project) project="$v" ;; esac manifest_names+=("$n") manifest_values+=("$v") done # Reduce the first manifest case. # if [ ${#manifest_names[@]} -eq 0 ]; then continue fi # Add or filter out the manifest, if present. # if [ ${#packages[@]} -ne 0 ]; then if [[ -v packages["$name"] ]]; then packages["$name"]= packages["$name/$version"]= # Clear it either, as may also be present. elif [[ -v packages["$name/$version"] ]]; then packages["$name/$version"]= else continue # Skip. fi fi packages_manifest_names+=("${manifest_names[@]}") packages_manifest_values+=("${manifest_values[@]}") if [ -z "$display_name" ]; then if [ -n "$project" ]; then display_name="$project" else display_name="$name" fi fi done manifest_parser_finish # Verify that all the listed in the request manifest packages are present in # the repository. # for p in "${!packages[@]}"; do if [ "${packages[$p]}" ]; then exit_with_manifest 422 "unknown package $p" fi done # Verify that the repository is not empty. Failed that, the repository display # name wouldn't be set. # if [ -z "$display_name" ]; then exit_with_manifest 422 "no packages in repository" fi # Stash the original packages manifest file for troubleshooting. # run mv "$cache_dir/packages.manifest" "$cache_dir/packages.manifest.orig" # Serialize the filtered packages manifest. # manifest_serializer_start "$cache_dir/packages.manifest" for ((i=0; i <= ${#packages_manifest_names[@]}; ++i)); do manifest_serialize "${packages_manifest_names[$i]}" \ "${packages_manifest_values[$i]}" done manifest_serializer_finish # Create the brep-load(1) loadtab file. # loadtab="$data_dir/loadtab" run echo "$repository $display_name cache:cache" >"$loadtab" # Load the requested repository packages into the brep package database for # the tenant identified by the reference. # run "$loader" "${loader_options[@]}" --force --shallow --tenant "$reference" \ "$loadtab" # Remove the no longer needed CI request data directory. # run rm -r "$data_dir" trace "CI request for '$spec' is queued$message_suffix" exit_with_manifest 200 "CI request is queued$message_suffix"