#!/usr/bin/env bash # file : brep/handler/ci/ci-load.in # license : MIT; see accompanying LICENSE file # Package CI request handler that loads the packages into the brep database. # # --result-url # Result URL base for the response. If specified, the handler will append the # brep tenant id to this value and include the resulting URL in the response # message. # # # Loader program (normally brep-load(1)). # # # Loader options (normally --db-*). # usage="usage: $0 [--result-url ] [] " verbose= #true # Repository information fetch timeout (seconds). # fetch_timeout=60 trap "{ exit 1; }" ERR set -o errtrace # Trap ERR in functions. set -o pipefail # Fail if any pipeline command fails. shopt -s lastpipe # Execute last pipeline command in the current shell. shopt -s nullglob # Expand no-match globs to nothing rather than themselves. @import brep/handler/handler@ @import brep/handler/ci/ci@ # The handler's own options. # result_url= while [[ "$#" -gt 0 ]]; do case $1 in --result-url) shift result_url="${1%/}" shift ;; *) break ;; esac done # The loader path. # loader="$1" if [[ -z "$loader" ]]; then error "$usage" fi shift # Assume that the remaining arguments except the last one are the loader # options. # loader_options=() while [[ "$#" -gt 1 ]]; do loader_options+=("$1") shift done # CI request data directory (last argument). # data_dir="${1%/}" if [[ -z "$data_dir" ]]; then error "$usage" fi if [[ ! -d "$data_dir" ]]; then error "'$data_dir' does not exist or is not a directory" fi reference="$(basename "$data_dir")" # Parse the CI request manifest and obtain the repository URL, package names # with optional versions, as well as the simulate value. # manifest_parser_start "$data_dir/request.manifest" repository= interactive= simulate= # Package map. We first enter packages from the request manifest as keys and # setting the values to true. Then we go through the repository package list # consulting this map and, if found, clearing the value to empty. Finally, we # go through the map looking for any "unhandled" packages (value is still # true). # # Note that keys can be in both and / forms. # declare -A packages # While at it, produce the bpkg-build(1)-like package spec for tracing. # # The spec normally contains the full commit id and so feels too hairy to # include in the result manifest message. # spec= # Third party service information which, if specified, needs to be associated # with the being created tenant. # service_id= service_type= service_data= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in repository) repository="$v" ;; interactive) interactive="$v" ;; simulate) simulate="$v" ;; package) packages["$v"]=true if [[ -n "$spec" ]]; then spec="$spec," fi spec="$spec$v" ;; service-id) service_id="$v" ;; service-type) service_type="$v" ;; service-data) service_data="$v" ;; esac done manifest_parser_finish if [[ -n "$spec" ]]; then spec="$spec@" fi spec="$spec$repository" if [[ -z "$repository" ]]; then error "repository manifest value expected" fi if [[ -n "$simulate" && "$simulate" != "success" ]]; then exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" fi # Use the generated reference if the tenant service id is not specified. # if [[ -n "$service_type" && -z "$service_id" ]]; then service_id="$reference" fi message_suffix= if [[ -n "$result_url" ]]; then message_suffix=": $result_url/@$reference" # Append the tenant id. fi # Exit with the 'CI request is queued' response if simulating. # # Note that we can't assume a real repository URL is specified if simulating # so trying to query the repository info is not a good idea. # if [[ -n "$simulate" ]]; then run rm -r "$data_dir" trace "CI request for '$spec' is simulated$message_suffix" exit_with_manifest 200 "CI request is queued$message_suffix" fi # Dump the repositories.manifest and packages.manifest files. # cache_dir="$data_dir/cache" run mkdir "$cache_dir" dump_repository_manifests "$repository" "$cache_dir" "$fetch_timeout" # Filter the packages manifest keeping only the packages listed in the request # manifest. Keep all the packages if the request specified no packages. # # Resulting manifest. # packages_manifest_names=() packages_manifest_values=() # While at it, set the repository display name to the first package's project # name. # display_name= manifest_parser_start "$cache_dir/packages.manifest" # The outer loop iterates over package manifests while the inner loop iterates # over manifest values in each such manifest. # # Note that the first manifest case is special in that we will see its version # value (empty name) first while for others -- last, as part of the previous # manifest. We try to deal with this irregularity by reducing the first case # (manifest_version is empty) to "as-if" it followed another manifest. # manifest_names=() manifest_values=() manifest_version= more=true while [[ "$more" ]]; do if [[ -n "$manifest_version" ]]; then manifest_names=("") manifest_values=("$manifest_version") fi more= project= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in "") # Start of (next) manifest. more=true manifest_version="$v" break ;; name) name="$v" ;; version) version="$v" ;; project) project="$v" ;; esac manifest_names+=("$n") manifest_values+=("$v") done # Reduce the first manifest case. # if [[ "${#manifest_names[@]}" -eq 0 ]]; then continue fi # Add or filter out the manifest, if present. # if [[ "${#packages[@]}" -ne 0 ]]; then if [[ -v "packages[$name]" ]]; then packages["$name"]= packages["$name/$version"]= # Clear it either, as may also be present. elif [[ -v "packages[$name/$version]" ]]; then packages["$name/$version"]= else continue # Skip. fi fi packages_manifest_names+=("${manifest_names[@]}") packages_manifest_values+=("${manifest_values[@]}") if [[ -z "$display_name" ]]; then if [[ -n "$project" ]]; then display_name="$project" else display_name="$name" fi fi done manifest_parser_finish # Verify that all the listed in the request manifest packages are present in # the repository. # for p in "${!packages[@]}"; do if [[ "${packages[$p]}" ]]; then exit_with_manifest 422 "unknown package $p" fi done # Verify that the repository is not empty. Failed that, the repository display # name wouldn't be set. # if [[ -z "$display_name" ]]; then exit_with_manifest 422 "no packages in repository" fi # Stash the original packages manifest file for troubleshooting. # run mv "$cache_dir/packages.manifest" "$cache_dir/packages.manifest.orig" # Serialize the filtered packages manifest. # manifest_serializer_start "$cache_dir/packages.manifest" for ((i=0; i != "${#packages_manifest_names[@]}"; ++i)); do manifest_serialize "${packages_manifest_names[$i]}" \ "${packages_manifest_values[$i]}" done manifest_serializer_finish # Create the brep-load(1) loadtab file. # loadtab="$data_dir/loadtab" run echo "$repository $display_name cache:cache" >"$loadtab" # Apply overrides, if uploaded. # if [[ -f "$data_dir/overrides.manifest" ]]; then loader_options+=(--overrides-file "$data_dir/overrides.manifest") fi # Load the requested repository packages into the brep package database for # the tenant identified by the reference. # loader_options+=(--force --shallow --tenant "$reference") # Build the packages interactively, if requested. # if [[ -n "$interactive" ]]; then loader_options+=(--interactive "$interactive") fi # Pass the tenant service information, if specified, to the loader. # if [[ -n "$service_id" ]]; then loader_options+=(--service-id "$service_id" --service-type "$service_type") if [[ -n "$service_data" ]]; then loader_options+=(--service-data "$service_data") fi fi run "$loader" "${loader_options[@]}" "$loadtab" # Remove the no longer needed CI request data directory. # run rm -r "$data_dir" trace "CI request for '$spec' is queued$message_suffix" exit_with_manifest 200 "CI request is queued$message_suffix"