aboutsummaryrefslogtreecommitdiff
path: root/brep/handler/ci/ci-load.in
blob: 915f9a6c6d7ab8621d1e213c5370923ac88c3caf (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
#!/usr/bin/env bash

# file      : brep/handler/ci/ci-load.in
# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
# license   : MIT; see accompanying LICENSE file

# Package CI request handler that loads the packages into the brep database.
#
# --result-url <url>
#  Result URL base for the response. If specified, the handler will append the
#  brep tenant id to this value and include the resulting URL in the response
#  message.
#
# <loader-path>
#  Loader program (normally brep-load(1)).
#
# <loader-options>
#  Loader options (normally --db-*).
#
usage="usage: $0 [--result-url <url>] <loader-path> [<loader-options>] <dir>"

verbose= #true

# Repository information fetch timeout (seconds).
#
fetch_timeout=60

trap "{ exit 1; }" ERR
set -o errtrace # Trap ERR in functions.

@import brep/handler/handler@
@import brep/handler/ci/ci@

# The handler's own options.
#
result_url=
while [ $# -gt 0 ]; do
  case $1 in
    --result-url)
      shift
      result_url="${1%/}"
      shift
      ;;
    *)
      break
      ;;
  esac
done

# The loader path.
#
loader="$1"

if [ -z "$loader" ]; then
  error "$usage"
fi

shift

# Assume that the remaining arguments except the last one are the loader
# options.
#
loader_options=()
while [ $# -gt 1 ]; do
  loader_options+=("$1")
  shift
done

# CI request data directory (last argument).
#
data_dir="${1%/}"

if [ -z "$data_dir" ]; then
  error "$usage"
fi

if [ ! -d "$data_dir" ]; then
  error "'$data_dir' does not exist or is not a directory"
fi

reference="$(basename "$data_dir")"

# Parse the CI request manifest and obtain the repository URL, package names
# with optional versions, as well as the simulate value.
#
manifest_parser_start "$data_dir/request.manifest"

simulate=
repository=

# Package map. We first enter packages from the request manifest as keys and
# setting the values to true. Then we go through the repository package list
# consulting this map and, if found, clearing the value to empty. Finally, we
# go through the map looking for any "unhandled" packages (value is still
# true).
#
# Note that keys can be in both <name> and <name>/<version> forms.
#
declare -A packages

# While at it, produce the bpkg-build(1)-like package spec for tracing.
#
# The spec normally contains the full commit id and so feels too hairy to
# include in the result manifest message.
#
spec=

while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
  case "$n" in
    simulate)   simulate="$v"   ;;
    repository) repository="$v" ;;

    package)
      packages["$v"]=true

      if [ -n "$spec" ]; then
        spec="$spec,"
      fi
      spec="$spec$v"
      ;;
  esac
done

manifest_parser_finish

if [ -n "$spec" ]; then
  spec="$spec@"
fi

spec="$spec$repository"

if [ -z "$repository" ]; then
  error "repository manifest value expected"
fi

if [ -n "$simulate" -a "$simulate" != "success" ]; then
  exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
fi

message_suffix=
if [ -n "$result_url" ]; then
  message_suffix=": $result_url/@$reference" # Append the tenant id.
fi

# Exit with the 'CI request is queued' response if simulating.
#
# Note that we can't assume a real repository URL is specified if simulating
# so trying to query the repository info is not a good idea.
#
if [ -n "$simulate" ]; then
  run rm -r "$data_dir"

  trace "CI request for '$spec' is simulated$message_suffix"
  exit_with_manifest 200 "CI request is queued$message_suffix"
fi

# Dump the repositories.manifest and packages.manifest files.
#
cache_dir="$data_dir/cache"
run mkdir "$cache_dir"
dump_repository_manifests "$repository" "$cache_dir" "$fetch_timeout"

# Filter the packages manifest keeping only the packages listed in the request
# manifest. Keep all the packages if the request specified no packages.
#

# Resulting manifest.
#
packages_manifest_names=()
packages_manifest_values=()

# While at it, set the repository display name to the first package's project
# name.
#
display_name=

manifest_parser_start "$cache_dir/packages.manifest"

# The outer loop iterates over package manifests while the inner loop iterates
# over manifest values in each such manifest.
#
# Note that the first manifest case is special in that we will see its version
# value (empty name) first while for others -- last, as part of the previous
# manifest. We try to deal with this irregularity by reducing the first case
# (manifest_version is empty) to "as-if" it followed another manifest.
#
manifest_names=()
manifest_values=()
manifest_version=

more=true
while [ "$more" ]; do

  if [ -n "$manifest_version" ]; then
    manifest_names=("")
    manifest_values=("$manifest_version")
  fi

  more=
  project=

  while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
    case "$n" in
      "") # Start of (next) manifest.
        more=true
        manifest_version="$v"
        break
        ;;

      name)    name="$v"    ;;
      version) version="$v" ;;
      project) project="$v" ;;
    esac

    manifest_names+=("$n")
    manifest_values+=("$v")

  done

  # Reduce the first manifest case.
  #
  if [ ${#manifest_names[@]} -eq 0 ]; then
    continue
  fi

  # Add or filter out the manifest, if present.
  #
  if [ ${#packages[@]} -ne 0 ]; then

    if [[ -v packages["$name"] ]]; then
      packages["$name"]=
      packages["$name/$version"]= # Clear it either, as may also be present.
    elif [[ -v packages["$name/$version"] ]]; then
      packages["$name/$version"]=
    else
      continue # Skip.
    fi

  fi

  packages_manifest_names+=("${manifest_names[@]}")
  packages_manifest_values+=("${manifest_values[@]}")

  if [ -z "$display_name" ]; then
    if [ -n "$project" ]; then
      display_name="$project"
    else
      display_name="$name"
    fi
  fi
done

manifest_parser_finish

# Verify that all the listed in the request manifest packages are present in
# the repository.
#
for p in "${!packages[@]}"; do
  if [ "${packages[$p]}" ]; then
    exit_with_manifest 422 "unknown package $p"
  fi
done

# Verify that the repository is not empty. Failed that, the repository display
# name wouldn't be set.
#
if [ -z "$display_name" ]; then
  exit_with_manifest 422 "no packages in repository"
fi

# Stash the original packages manifest file for troubleshooting.
#
run mv "$cache_dir/packages.manifest" "$cache_dir/packages.manifest.orig"

# Serialize the filtered packages manifest.
#
manifest_serializer_start "$cache_dir/packages.manifest"

for ((i=0; i <= ${#packages_manifest_names[@]}; ++i)); do
  manifest_serialize "${packages_manifest_names[$i]}" \
                     "${packages_manifest_values[$i]}"
done

manifest_serializer_finish

# Create the brep-load(1) loadtab file.
#
loadtab="$data_dir/loadtab"
run echo "$repository $display_name cache:cache" >"$loadtab"

# Apply overrides, if uploaded.
#
if [ -f "$data_dir/overrides.manifest" ]; then
  loader_options+=(--overrides-file "$data_dir/overrides.manifest")
fi

# Load the requested repository packages into the brep package database for
# the tenant identified by the reference.
#
loader_options+=(--force --shallow --tenant "$reference")

run "$loader" "${loader_options[@]}" "$loadtab"

# Remove the no longer needed CI request data directory.
#
run rm -r "$data_dir"

trace "CI request for '$spec' is queued$message_suffix"
exit_with_manifest 200 "CI request is queued$message_suffix"