diff options
Diffstat (limited to 'bpkg-util/publish.in')
| -rw-r--r-- | bpkg-util/publish.in | 329 |
1 files changed, 329 insertions, 0 deletions
diff --git a/bpkg-util/publish.in b/bpkg-util/publish.in new file mode 100644 index 0000000..215c6a6 --- /dev/null +++ b/bpkg-util/publish.in @@ -0,0 +1,329 @@ +#!/usr/bin/env bash + +# file : bpkg-util/publish.in +# license : MIT; see accompanying LICENSE file + +# Update (bpkg-rep-create(1)) and publish (rsync(1)) an archive-based +# repository. +# +# Pull a pre-cloned (read-only) git repository with the contents of an +# archive-based bpkg repository. Bail out if nothing changed from the the +# previous run. Otherwise, regenerate the repository meta-data by running +# bpkg-rep-create(1) on each section of the repository and, optionally, +# synchronize it to one or more destinations with rsync. +# +# The repository contents are expected to be in the <dir>/1/ subdirectory. The +# script saves the last successfully published commit in the <dir>.publish +# file. +# +# --destination|-d <host>:<dir> +# +# Remote host and directory to rsync the repository to. Note that the +# directory should include the 1/ component and any sub-directories that may +# follow. In other words, the rsync command will be in the form: +# +# rsync ... <dir>/1/ <host>:<dir>/ +# +# See below for the actual rsync command including a brief explanation of +# options passed. +# +# Repeat this option to specify multiple destinations. In this case, the +# destinations are synced in the order specified with the first failure +# terminating the process (so if you have a "primary" destination and a +# "mirror", you probably want to specify the former first). +# +# --timeout <seconds> +# +# Git and rsync operation timeout. Specifically, the operation will be +# aborted if there is no network activity for the specified time. Default is +# 60 seconds. Note that currently the git timeout is only supported for the +# http(s) transport. +# +# --lock-timeout <seconds> +# +# The repository lock timeout. Fail if another instance of the script does +# not release the repository in the specified time. The default is 0 (do +# not wait). +# +# Note that you will most likely want to specify a non-zero timeout for cron +# jobs that may potentially overlap. +# +# --log-dir <dir> +# +# Directory to create the temporary log files in. If unspecified, the stderr +# is not redirected and no log is created by default. +# +# The log is dumped to stderr in case of an error or at the end of execution +# unless in the quiet mode and is then deleted. +# +# --quiet +# +# Run quiet. Specifically, don't dump the log to stderr on exit with zero +# status. +# +# --config <path> +# +# The configuration file containing a bash fragment. Repeat this option to +# specify multiple configurations that will be sourced in the order +# specified. +# +# --bpkg <path> +# +# The package manager program to be used for the repository update. This +# should be the path to the bpkg executable. +# +usage="usage: $0 [<options>] <dir> [<rep-create-options>] [-- <rsync-options>]" + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +@import bpkg-util/utility@ + +# The script own options. +# +repo_ver=1 +destinations=() +timeout=60 +lock_timeout=0 +log_dir= +quiet= +configurations=() +bpkg= + +while [ $# -gt 0 ]; do + case $1 in + --destination|-d) + shift + destinations+=("${1%/}") + shift || true + ;; + --timeout) + shift + timeout="$1" + shift || true + ;; + --lock-timeout) + shift + lock_timeout="$1" + shift || true + ;; + --log-dir) + shift + log_dir="${1%/}" + shift || true + ;; + --quiet) + shift + quiet=true + ;; + --config) + shift + configurations+=("$1") + shift || true + ;; + --bpkg) + shift + bpkg="$1" + shift || true + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done + +# The repository directory. +# +repo_dir="${1%/}" +shift || true + +# bpkg-rep-create options. +# +rep_create_options=() + +while [ $# -gt 0 ]; do + case $1 in + --) + shift + break + ;; + *) + rep_create_options+=("$1") + shift + ;; + esac +done + +# rsync options. +# +rsync_options=() + +while [ $# -gt 0 ]; do + rsync_options+=("$1") + shift +done + +# Validate options and arguments. +# +if [ -z "$repo_dir" ]; then + error "$usage" +fi + +if [ ! -d "$repo_dir" ]; then + error "'$repo_dir' does not exist or is not a directory" +fi + +# If the log directory is specified then redirect stderr to the log file and +# setup the trap that dumps it on exit, if required. +# +if [ -n "$log_dir" ]; then + + if [ ! -d "$log_dir" ]; then + error "'$log_dir' does not exist or is not a directory" + fi + + # Create the log file. + # + log="$(mktemp "$log_dir/$(basename "$repo_dir").XXXXXXXXXX")" + + # Save the stderr file descriptor so we can dump the log into it on exit, if + # required. Then redirect it to the log file. + # + exec {stderr}>&2 + exec 2>>"$log" + + function exit_trap () + { + local status="$?" + + # Dump the log to stderr if exiting with non-zero status or verbose. + # + if [ $status -ne 0 -o ! "$quiet" ]; then + + # Keep the log if failed to dump for any reason. + # + if ! cat "$log" >&$stderr; then + return + fi + fi + + rm -f "$log" + } + + trap exit_trap EXIT +fi + +# Source the configurations. +# +for c in "${configurations[@]}"; do + source "$c" >&2 +done + +# Make sure the commit file is present. +# +published_commit="$repo_dir.publish" +touch "$published_commit" + +# Open the reading file descriptor and lock the repository. Fail if unable to +# lock before timeout. +# +exec {cfd}<"$published_commit" + +if ! flock -w "$lock_timeout" "$cfd"; then + info "another instance is already running" + exit 2 +fi + +# Pull the repository. +# +# Git doesn't support the connection timeout option. The options we use are +# just an approximation of the former, that, in particular, don't cover the +# connection establishing. To work around this problem, before running a git +# command that assumes the remote repository communication we manually check +# connectivity with the remote repository. +# +if ! remote_url="$(git -C "$repo_dir" config --get remote.origin.url)"; then + error "'$repo_dir' is not a git repository" +fi + +run check_git_connectivity "$remote_url" "$timeout" + +# Fail if no network activity happens during the time specified. +# +run git -c http.lowSpeedLimit=1 -c "http.lowSpeedTime=$timeout" \ +-C "$repo_dir" pull -v >&2 + +# Match the HEAD commit id to the one stored in the file. If it matches, then +# nothing changed in the repository from the previous run and we can silently +# bail out. +# +commit="$(git -C "$repo_dir" rev-parse HEAD)" +pc="$(cat <&"$cfd")" + +if [ "$commit" == "$pc" ]; then + quiet=true + exit 0 +fi + +# If bpkg path is not specified, then use the bpkg program from the script +# directory, if present. Otherwise, use the 'bpkg' path. +# +if [ -z "$bpkg" ]; then + bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" + + if [ ! -x "$bpkg" ]; then + bpkg=bpkg + fi +fi + +# Find repository sections. +# +manifests="$(find "$repo_dir/$repo_ver" -type f -name repositories.manifest)" + +# Update the repository sections. +# +while read f; do + run "$bpkg" rep-create "${rep_create_options[@]}" "$(dirname "$f")" +done <<<"$manifests" + +# rsync (over ssh) the repository to the destinations. +# +# Approximate the data transfer timeout via the ServerAlive* ssh options, +# rounding the timeout up to the nearest multiple of ten. +# +# Note: must not contain spaces/use quoting (see rsync -e option). +# +n=$(($timeout > 0 ? ($timeout + 9) / 10 : 1)) +ssh_options=(-o ConnectTimeout=$timeout \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=$n) + +for d in "${destinations[@]}"; do + + # -r (recursive) + # -l (copy symlinks as symlinks) + # -t (preserve timestamps) + # -O (omit dir timestamps) + # + # -c (use checksum) + # -e (remote shell command) + # + # --chmod=ugo=rwX (give new files the destination-default permissions) + # --safe-links (ignore symlinks pointing outside the tree) + # --delay-updates (first upload all files on the side then move) + # --prune-empty-dirs (remove empty dirs) + # --delete-after (delete entries after the transfer) + # + # We also exclude hidden files (start with dot). + # + run rsync -v -rltO -c --chmod=ugo=rwX --safe-links --delay-updates \ +--exclude '.*' --prune-empty-dirs --delete-after -e "ssh ${ssh_options[*]}" \ +"${rsync_options[@]}" "$repo_dir/$repo_ver/" "$d/" >&2 + +done + +echo "$commit" >"$published_commit" |