From fd4b8ec3d690fa7341159b0166b382dd43c6c967 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Thu, 18 Oct 2018 18:42:00 +0300 Subject: Add implementation --- INSTALL-PUBLISH | 1 + LICENSE | 20 +++ NEWS | 3 + README | 20 +++ bpkg-rep/.gitignore | 1 + bpkg-rep/buildfile | 9 ++ bpkg-rep/publish.in | 309 ++++++++++++++++++++++++++++++++++++++++++++ bpkg-rep/utility.bash.in | 87 +++++++++++++ build/.gitignore | 3 + build/bootstrap.build | 11 ++ build/export.build | 11 ++ build/root.build | 9 ++ buildfile | 10 ++ manifest | 21 +++ repositories.manifest | 2 + tests/.gitignore | 2 + tests/build/.gitignore | 3 + tests/build/bootstrap.build | 9 ++ tests/build/root.build | 7 + tests/buildfile | 7 + tests/publish.testscript | 9 ++ 21 files changed, 554 insertions(+) create mode 100644 INSTALL-PUBLISH create mode 100644 LICENSE create mode 100644 NEWS create mode 100644 README create mode 100644 bpkg-rep/.gitignore create mode 100644 bpkg-rep/buildfile create mode 100644 bpkg-rep/publish.in create mode 100644 bpkg-rep/utility.bash.in create mode 100644 build/.gitignore create mode 100644 build/bootstrap.build create mode 100644 build/export.build create mode 100644 build/root.build create mode 100644 buildfile create mode 100644 manifest create mode 100644 repositories.manifest create mode 100644 tests/.gitignore create mode 100644 tests/build/.gitignore create mode 100644 tests/build/bootstrap.build create mode 100644 tests/build/root.build create mode 100644 tests/buildfile create mode 100644 tests/publish.testscript diff --git a/INSTALL-PUBLISH b/INSTALL-PUBLISH new file mode 100644 index 0000000..a309a51 --- /dev/null +++ b/INSTALL-PUBLISH @@ -0,0 +1 @@ +@@ TODO diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c1332e6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2014-2018 Code Synthesis Ltd + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..32bf3ce --- /dev/null +++ b/NEWS @@ -0,0 +1,3 @@ +Version 0.9.0 + + * First public release. diff --git a/README b/README new file mode 100644 index 0000000..e1eb555 --- /dev/null +++ b/README @@ -0,0 +1,20 @@ +This package contains bpkg repository management utilities. + +build2 is an open source, cross-platform toolchain for building and packaging +C++ code. Its aim is a modern build system and dependency manager for the C++ +language that provide a consistent, out of the box interface across multiple +platforms and compilers. For more information see: + +https://build2.org/ + +See the NEWS file for the user-visible changes from the previous release. + +See the LICENSE file for the distribution conditions. + +See the INSTALL file for the prerequisites and installation instructions. + +See the doc/ directory for documentation. + +Send questions, bug reports, or any other feedback to the users@build2.org +mailing list. You can post without subscribing. See https://lists.build2.org +for searchable archives, posting guidelines, etc. diff --git a/bpkg-rep/.gitignore b/bpkg-rep/.gitignore new file mode 100644 index 0000000..39d8ec1 --- /dev/null +++ b/bpkg-rep/.gitignore @@ -0,0 +1 @@ +bpkg-rep-publish diff --git a/bpkg-rep/buildfile b/bpkg-rep/buildfile new file mode 100644 index 0000000..6f4a025 --- /dev/null +++ b/bpkg-rep/buildfile @@ -0,0 +1,9 @@ +# file : bpkg-rep/buildfile +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +./: exe{bpkg-rep-publish} + +exe{bpkg-rep-publish}: in{publish} bash{utility} + +bash{utility}: in{utility} diff --git a/bpkg-rep/publish.in b/bpkg-rep/publish.in new file mode 100644 index 0000000..e828692 --- /dev/null +++ b/bpkg-rep/publish.in @@ -0,0 +1,309 @@ +#!/usr/bin/env bash + +# file : bpkg-rep/publish.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Update (bpkg-rep-create(1)) and publish (rsync(1)) an archive-based +# repository. +# +# Pull a pre-cloned (read-only) git repository with the contents of an +# archive-based bpkg repository. Bail out if nothing changed from the the +# previous run. Otherwise, regenerate the repository meta-data by running +# bpkg-rep-create(1) on each section of the repository and, optionally, +# synchronize it to one or more destinations with rsync. +# +# The repository contents are expected to be in the /1/ subdirectory. The +# script saves the last successfully published commit in the .publish +# file. +# +# --destination|-d : +# +# Remote host and directory to rsync the repository to. Note that the +# trailing 1/ will be added automatically. In other words, the rsync command +# will be in the form: +# +# rsync ... /1/ :/1/ +# +# Repeat this option to specify multiple destinations. In this case, the +# destinations are synced in the order specified with the first failure +# terminating the process (so if you have a "primary" destination and a +# "mirror", you probably want to specify the former first). +# +# --timeout +# +# Git and rsync operation timeout. Specifically, the operation will be +# aborted if there is no network activity for the specified time. Default is +# 60 seconds. Note that currently the git timeout is only supported for the +# http(s) transport. +# +# --lock-timeout +# +# The repository lock timeout. Fail if another instance of the script does +# not release the repository in the specified time. The default is 0 (do +# not wait). +# +# Note that you will most likely want to specify a non-zero timeout for cron +# jobs that may potentially overlap. +# +# --log-dir +# +# Directory to create the temporary log files in. If unspecified, the stderr +# is not redirected and no log is created by default. +# +# The log is dumped to stderr in case of an error or at the end of execution +# unless in the quiet mode and is then deleted. +# +# --quiet +# +# Run quiet. Specifically, don't dump the log to stderr on exit with zero +# status. +# +# --bpkg +# +# The package manager program to be used for the repository update. This +# should be the path to the bpkg executable. +# +usage="usage: $0 [] [] [-- ]" + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +@import bpkg-rep/utility@ + +# The script own options. +# +repo_ver=1 +destinations=() +timeout=60 +lock_timeout=0 +log_dir= +quiet= +bpkg= + +while [ $# -gt 0 ]; do + case $1 in + --destination|-d) + shift + destinations+=("$1") + shift || true + ;; + --timeout) + shift + timeout="$1" + shift || true + ;; + --lock-timeout) + shift + lock_timeout="$1" + shift || true + ;; + --log-dir) + shift + log_dir="${1%/}" + shift || true + ;; + --quiet) + shift + quiet=true + ;; + --bpkg) + shift + bpkg="$1" + shift || true + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done + +# The repository directory. +# +repo_dir="${1%/}" +shift || true + +# bpkg-rep-create options. +# +rep_create_options=() + +while [ $# -gt 0 ]; do + case $1 in + --) + shift + break + ;; + *) + rep_create_options+=("$1") + shift + ;; + esac +done + +# rsync options. +# +rsync_options=() + +while [ $# -gt 0 ]; do + rsync_options+=("$1") + shift +done + +# Validate options and arguments. +# +if [ -z "$repo_dir" ]; then + error "$usage" +fi + +if [ ! -d "$repo_dir" ]; then + error "'$repo_dir' does not exist or is not a directory" +fi + +# If the log directory is specified then redirect stderr to the log file and +# setup the trap that dumps it on exit, if required. +# +if [ -n "$log_dir" ]; then + + if [ ! -d "$log_dir" ]; then + error "'$log_dir' does not exist or is not a directory" + fi + + # Create the log file. + # + log="$(mktemp "$log_dir/$(basename "$repo_dir").XXXXXXXXXX")" + + # Save the stderr file descriptor so we can dump the log into it on exit, if + # required. Then redirect it to the log file. + # + exec {stderr}>&2 + exec 2>>"$log" + + function exit_trap () + { + local status="$?" + + # Dump the log to stderr if exiting with non-zero status or verbose. + # + if [ $status -ne 0 -o ! "$quiet" ]; then + + # Keep the log if failed to dump for any reason. + # + if ! cat "$log" >&$stderr; then + return + fi + fi + + rm -f "$log" + } + + trap exit_trap EXIT +fi + +# Make sure the commit file is present. +# +published_commit="$repo_dir.publish" +touch "$published_commit" + +# Open the reading file descriptor and lock the repository. Fail if unable to +# lock before timeout. +# +exec {cfd}<"$published_commit" + +if ! flock -w "$lock_timeout" "$cfd"; then + info "another instance is already running" + exit 2 +fi + +# Pull the repository. +# +# Git doesn't support the connection timeout option. The options we use are +# just an approximation of the former, that, in particular, don't cover the +# connection establishing. To work around this problem, before running a git +# command that assumes the remote repository communication we manually check +# connectivity with the remote repository. +# +if ! remote_url="$(git -C "$repo_dir" config --get remote.origin.url)"; then + error "'$repo_dir' is not a git repository" +fi + +run check_git_connectivity "$remote_url" "$timeout" + +# Fail if no network activity happens during the time specified. +# +run git -c http.lowSpeedLimit=1 -c "http.lowSpeedTime=$timeout" \ +-C "$repo_dir" pull -v >&2 + +# Match the HEAD commit id to the one stored in the file. If it matches, then +# nothing changed in the repository from the previous run and we can silently +# bail out. +# +commit="$(git -C "$repo_dir" rev-parse HEAD)" +pc="$(cat <&"$cfd")" + +if [ "$commit" == "$pc" ]; then + quiet=true + exit 0 +fi + +# If bpkg path is not specified, then use the bpkg program from the script +# directory, if present. Otherwise, use the 'bpkg' path. +# +if [ -z "$bpkg" ]; then + bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg" + + if [ ! -x "$bpkg" ]; then + bpkg=bpkg + fi +fi + +# Find repository sections. +# +manifests="$(find "$repo_dir/$repo_ver" -type f -name repositories.manifest)" + +# Update the repository sections. +# +while read f; do + run "$bpkg" rep-create "${rep_create_options[@]}" "$(dirname "$f")" +done <<<"$manifests" + +# rsync (over ssh) the repository to the destinations. +# +# Approximate the data transfer timeout via the ServerAlive* ssh options, +# rounding the timeout up to the nearest multiple of ten. +# +# Note: must not contain spaces/use quoting (see rsync -e option). +# +n=$(($timeout > 0 ? ($timeout + 9) / 10 : 1)) +ssh_options=(-o ConnectTimeout=$timeout \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=$n) + +for d in "${destinations[@]}"; do + + # -r (recursive) + # -l (copy symlinks and symlinks) + # -p (preserve perms) + # -t (preserve timestamps) + # -O (omit dir timestamps) + # + # -c (use checksum) + # -e (remote shell command) + # + # --safe-links (ignore symlinks pointing outside the tree) + # --delay-updates (first upload all files on the side then move) + # --prune-empty-dirs (remove empty dirs) + # --delete-after (delete entries after the transfer) + # + # We also exclude hiddent files (start with dot). + # + run rsync -v -rlptO -c --safe-links --delay-updates --exclude '.*' \ +--prune-empty-dirs --delete-after -e "ssh ${ssh_options[*]}" \ +"${rsync_options[@]}" "$repo_dir/$repo_ver/" "$d/$repo_ver/" >&2 + +done + +echo "$commit" >"$published_commit" diff --git a/bpkg-rep/utility.bash.in b/bpkg-rep/utility.bash.in new file mode 100644 index 0000000..bcf25d8 --- /dev/null +++ b/bpkg-rep/utility.bash.in @@ -0,0 +1,87 @@ +# file : bpkg-rep/utility.bash.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Utility functions useful for implementing bpkg repository utilities. + +if [ "$bpkg_rep_utility" ]; then + return 0 +else + bpkg_rep_utility=true +fi + +# Diagnostics. +# +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +# Trace a command line, quoting empty arguments as well as those that contain +# spaces. +# +function trace_cmd () # ... +{ + local s="+" + while [ $# -gt 0 ]; do + if [ -z "$1" -o -z "${1##* *}" ]; then + s="$s '$1'" + else + s="$s $1" + fi + + shift + done + + info "$s" +} + +# Trace the current function name and arguments. +# +function trace_func () # ... +{ + trace_cmd "${FUNCNAME[1]}" "$@" +} + +# Trace and run a command. +# +function run () # ... +{ + trace_cmd "$@" + "$@" +} + +# Return lower-case URL scheme or empty string if the argument doesn't look +# like a URL. +# +function url_scheme () # +{ + sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1" +} + +# Check that the git repository properly responds to the probing request +# before the timeout (in seconds). Noop for protocols other than HTTP(S). +# +function check_git_connectivity () # +{ + local url="$1" + local tmo="$2" + + local s + s="$(url_scheme "$url")" + + if [ "$s" == "http" -o "$s" == "https" ]; then + local u q + + u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part. + q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part. + + if [ -z "$q" ]; then + u="$u/info/refs?service=git-upload-pack" + else + u="$u/info/refs$q&service=git-upload-pack" + fi + + # Here we limit the time for the whole operation. + # + curl -S -s --max-time "$tmo" "$u" >/dev/null + fi +} diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000..4a730a3 --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,3 @@ +config.build +root/ +bootstrap/ diff --git a/build/bootstrap.build b/build/bootstrap.build new file mode 100644 index 0000000..3685688 --- /dev/null +++ b/build/bootstrap.build @@ -0,0 +1,11 @@ +# file : build/bootstrap.build +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +project = bpkg-rep + +using version +using config +using dist +using test +using install diff --git a/build/export.build b/build/export.build new file mode 100644 index 0000000..8c676d6 --- /dev/null +++ b/build/export.build @@ -0,0 +1,11 @@ +# file : build/export.build +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +$out_root/ +{ + include bpkg-rep/ +} + +if ($import.target == exe{bpkg-rep-publish}) + export $out_root/bpkg-rep/exe{bpkg-rep-publish} diff --git a/build/root.build b/build/root.build new file mode 100644 index 0000000..0a2560f --- /dev/null +++ b/build/root.build @@ -0,0 +1,9 @@ +# file : build/root.build +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Bash. +# +using bash + +bpkg-rep/bash{*}: install.subdirs = true diff --git a/buildfile b/buildfile new file mode 100644 index 0000000..ad6e61a --- /dev/null +++ b/buildfile @@ -0,0 +1,10 @@ +# file : buildfile +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +./: {*/ -build/} doc{INSTALL-PUBLISH LICENSE NEWS README} manifest + +# Don't install tests or the INSTALL* files. +# +tests/: install = false +doc{INSTALL-PUBLISH}@./: install = false diff --git a/manifest b/manifest new file mode 100644 index 0000000..6ed76f1 --- /dev/null +++ b/manifest @@ -0,0 +1,21 @@ +: 1 +name: bpkg-rep +version: 0.9.0-a.0.z +project: build2 +summary: bpkg repository management utilities +license: MIT +tags: bpkg, repository, management, utility, publish +description-file: README +changes-file: NEWS +url: https://build2.org +doc-url: https://build2.org/doc.xhtml +src-url: https://git.build2.org/cgit/bpkg-rep/tree/ +email: users@build2.org +build-email: builds@build2.org +build-exclude: windows*; Requires bash +build-exclude: macos*; Requires bash >= 4.3 +build-include: * +requires: bash >= 4.3 +depends: * build2 >= 0.8.0- +depends: * bpkg >= 0.8.0- +depends: bpkg [0.9.0-a.0.1 0.9.0-a.1) diff --git a/repositories.manifest b/repositories.manifest new file mode 100644 index 0000000..d33e418 --- /dev/null +++ b/repositories.manifest @@ -0,0 +1,2 @@ +: 1 +summary: bpkg repository management utilities repository diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..35ec43f --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,2 @@ +test/ +test-*/ diff --git a/tests/build/.gitignore b/tests/build/.gitignore new file mode 100644 index 0000000..4a730a3 --- /dev/null +++ b/tests/build/.gitignore @@ -0,0 +1,3 @@ +config.build +root/ +bootstrap/ diff --git a/tests/build/bootstrap.build b/tests/build/bootstrap.build new file mode 100644 index 0000000..91bc3e9 --- /dev/null +++ b/tests/build/bootstrap.build @@ -0,0 +1,9 @@ +# file : tests/build/bootstrap.build +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +project = # Unnamed subproject. + +using config +using dist +using test diff --git a/tests/build/root.build b/tests/build/root.build new file mode 100644 index 0000000..1d7601b --- /dev/null +++ b/tests/build/root.build @@ -0,0 +1,7 @@ +# file : tests/build/root.build +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Setup the utilities that we are testing. +# +import publish = bpkg-rep%exe{bpkg-rep-publish} diff --git a/tests/buildfile b/tests/buildfile new file mode 100644 index 0000000..36c24d7 --- /dev/null +++ b/tests/buildfile @@ -0,0 +1,7 @@ +# file : tests/buildfile +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +./: testscript{*} $publish + +testscript{publish}@./: test = $publish diff --git a/tests/publish.testscript b/tests/publish.testscript new file mode 100644 index 0000000..95fd6f5 --- /dev/null +++ b/tests/publish.testscript @@ -0,0 +1,9 @@ +# file : tests/publish.testscript +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +: args +: +{ + $* 2>~"/usage: .+/" != 0 : no-args +} -- cgit v1.1