aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--INSTALL-PUBLISH1
-rw-r--r--LICENSE20
-rw-r--r--NEWS3
-rw-r--r--README20
-rw-r--r--bpkg-rep/.gitignore1
-rw-r--r--bpkg-rep/buildfile9
-rw-r--r--bpkg-rep/publish.in309
-rw-r--r--bpkg-rep/utility.bash.in87
-rw-r--r--build/.gitignore3
-rw-r--r--build/bootstrap.build11
-rw-r--r--build/export.build11
-rw-r--r--build/root.build9
-rw-r--r--buildfile10
-rw-r--r--manifest21
-rw-r--r--repositories.manifest2
-rw-r--r--tests/.gitignore2
-rw-r--r--tests/build/.gitignore3
-rw-r--r--tests/build/bootstrap.build9
-rw-r--r--tests/build/root.build7
-rw-r--r--tests/buildfile7
-rw-r--r--tests/publish.testscript9
21 files changed, 554 insertions, 0 deletions
diff --git a/INSTALL-PUBLISH b/INSTALL-PUBLISH
new file mode 100644
index 0000000..a309a51
--- /dev/null
+++ b/INSTALL-PUBLISH
@@ -0,0 +1 @@
+@@ TODO
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..c1332e6
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,20 @@
+Copyright (c) 2014-2018 Code Synthesis Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/NEWS b/NEWS
new file mode 100644
index 0000000..32bf3ce
--- /dev/null
+++ b/NEWS
@@ -0,0 +1,3 @@
+Version 0.9.0
+
+ * First public release.
diff --git a/README b/README
new file mode 100644
index 0000000..e1eb555
--- /dev/null
+++ b/README
@@ -0,0 +1,20 @@
+This package contains bpkg repository management utilities.
+
+build2 is an open source, cross-platform toolchain for building and packaging
+C++ code. Its aim is a modern build system and dependency manager for the C++
+language that provide a consistent, out of the box interface across multiple
+platforms and compilers. For more information see:
+
+https://build2.org/
+
+See the NEWS file for the user-visible changes from the previous release.
+
+See the LICENSE file for the distribution conditions.
+
+See the INSTALL file for the prerequisites and installation instructions.
+
+See the doc/ directory for documentation.
+
+Send questions, bug reports, or any other feedback to the users@build2.org
+mailing list. You can post without subscribing. See https://lists.build2.org
+for searchable archives, posting guidelines, etc.
diff --git a/bpkg-rep/.gitignore b/bpkg-rep/.gitignore
new file mode 100644
index 0000000..39d8ec1
--- /dev/null
+++ b/bpkg-rep/.gitignore
@@ -0,0 +1 @@
+bpkg-rep-publish
diff --git a/bpkg-rep/buildfile b/bpkg-rep/buildfile
new file mode 100644
index 0000000..6f4a025
--- /dev/null
+++ b/bpkg-rep/buildfile
@@ -0,0 +1,9 @@
+# file : bpkg-rep/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+./: exe{bpkg-rep-publish}
+
+exe{bpkg-rep-publish}: in{publish} bash{utility}
+
+bash{utility}: in{utility}
diff --git a/bpkg-rep/publish.in b/bpkg-rep/publish.in
new file mode 100644
index 0000000..e828692
--- /dev/null
+++ b/bpkg-rep/publish.in
@@ -0,0 +1,309 @@
+#!/usr/bin/env bash
+
+# file : bpkg-rep/publish.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Update (bpkg-rep-create(1)) and publish (rsync(1)) an archive-based
+# repository.
+#
+# Pull a pre-cloned (read-only) git repository with the contents of an
+# archive-based bpkg repository. Bail out if nothing changed from the the
+# previous run. Otherwise, regenerate the repository meta-data by running
+# bpkg-rep-create(1) on each section of the repository and, optionally,
+# synchronize it to one or more destinations with rsync.
+#
+# The repository contents are expected to be in the <dir>/1/ subdirectory. The
+# script saves the last successfully published commit in the <dir>.publish
+# file.
+#
+# --destination|-d <host>:<dir>
+#
+# Remote host and directory to rsync the repository to. Note that the
+# trailing 1/ will be added automatically. In other words, the rsync command
+# will be in the form:
+#
+# rsync ... <dir>/1/ <host>:<dir>/1/
+#
+# Repeat this option to specify multiple destinations. In this case, the
+# destinations are synced in the order specified with the first failure
+# terminating the process (so if you have a "primary" destination and a
+# "mirror", you probably want to specify the former first).
+#
+# --timeout <seconds>
+#
+# Git and rsync operation timeout. Specifically, the operation will be
+# aborted if there is no network activity for the specified time. Default is
+# 60 seconds. Note that currently the git timeout is only supported for the
+# http(s) transport.
+#
+# --lock-timeout <seconds>
+#
+# The repository lock timeout. Fail if another instance of the script does
+# not release the repository in the specified time. The default is 0 (do
+# not wait).
+#
+# Note that you will most likely want to specify a non-zero timeout for cron
+# jobs that may potentially overlap.
+#
+# --log-dir <dir>
+#
+# Directory to create the temporary log files in. If unspecified, the stderr
+# is not redirected and no log is created by default.
+#
+# The log is dumped to stderr in case of an error or at the end of execution
+# unless in the quiet mode and is then deleted.
+#
+# --quiet
+#
+# Run quiet. Specifically, don't dump the log to stderr on exit with zero
+# status.
+#
+# --bpkg <path>
+#
+# The package manager program to be used for the repository update. This
+# should be the path to the bpkg executable.
+#
+usage="usage: $0 [<options>] <dir> [<rep-create-options>] [-- <rsync-options>]"
+
+trap "{ exit 1; }" ERR
+set -o errtrace # Trap ERR in functions.
+
+@import bpkg-rep/utility@
+
+# The script own options.
+#
+repo_ver=1
+destinations=()
+timeout=60
+lock_timeout=0
+log_dir=
+quiet=
+bpkg=
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ --destination|-d)
+ shift
+ destinations+=("$1")
+ shift || true
+ ;;
+ --timeout)
+ shift
+ timeout="$1"
+ shift || true
+ ;;
+ --lock-timeout)
+ shift
+ lock_timeout="$1"
+ shift || true
+ ;;
+ --log-dir)
+ shift
+ log_dir="${1%/}"
+ shift || true
+ ;;
+ --quiet)
+ shift
+ quiet=true
+ ;;
+ --bpkg)
+ shift
+ bpkg="$1"
+ shift || true
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+# The repository directory.
+#
+repo_dir="${1%/}"
+shift || true
+
+# bpkg-rep-create options.
+#
+rep_create_options=()
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ --)
+ shift
+ break
+ ;;
+ *)
+ rep_create_options+=("$1")
+ shift
+ ;;
+ esac
+done
+
+# rsync options.
+#
+rsync_options=()
+
+while [ $# -gt 0 ]; do
+ rsync_options+=("$1")
+ shift
+done
+
+# Validate options and arguments.
+#
+if [ -z "$repo_dir" ]; then
+ error "$usage"
+fi
+
+if [ ! -d "$repo_dir" ]; then
+ error "'$repo_dir' does not exist or is not a directory"
+fi
+
+# If the log directory is specified then redirect stderr to the log file and
+# setup the trap that dumps it on exit, if required.
+#
+if [ -n "$log_dir" ]; then
+
+ if [ ! -d "$log_dir" ]; then
+ error "'$log_dir' does not exist or is not a directory"
+ fi
+
+ # Create the log file.
+ #
+ log="$(mktemp "$log_dir/$(basename "$repo_dir").XXXXXXXXXX")"
+
+ # Save the stderr file descriptor so we can dump the log into it on exit, if
+ # required. Then redirect it to the log file.
+ #
+ exec {stderr}>&2
+ exec 2>>"$log"
+
+ function exit_trap ()
+ {
+ local status="$?"
+
+ # Dump the log to stderr if exiting with non-zero status or verbose.
+ #
+ if [ $status -ne 0 -o ! "$quiet" ]; then
+
+ # Keep the log if failed to dump for any reason.
+ #
+ if ! cat "$log" >&$stderr; then
+ return
+ fi
+ fi
+
+ rm -f "$log"
+ }
+
+ trap exit_trap EXIT
+fi
+
+# Make sure the commit file is present.
+#
+published_commit="$repo_dir.publish"
+touch "$published_commit"
+
+# Open the reading file descriptor and lock the repository. Fail if unable to
+# lock before timeout.
+#
+exec {cfd}<"$published_commit"
+
+if ! flock -w "$lock_timeout" "$cfd"; then
+ info "another instance is already running"
+ exit 2
+fi
+
+# Pull the repository.
+#
+# Git doesn't support the connection timeout option. The options we use are
+# just an approximation of the former, that, in particular, don't cover the
+# connection establishing. To work around this problem, before running a git
+# command that assumes the remote repository communication we manually check
+# connectivity with the remote repository.
+#
+if ! remote_url="$(git -C "$repo_dir" config --get remote.origin.url)"; then
+ error "'$repo_dir' is not a git repository"
+fi
+
+run check_git_connectivity "$remote_url" "$timeout"
+
+# Fail if no network activity happens during the time specified.
+#
+run git -c http.lowSpeedLimit=1 -c "http.lowSpeedTime=$timeout" \
+-C "$repo_dir" pull -v >&2
+
+# Match the HEAD commit id to the one stored in the file. If it matches, then
+# nothing changed in the repository from the previous run and we can silently
+# bail out.
+#
+commit="$(git -C "$repo_dir" rev-parse HEAD)"
+pc="$(cat <&"$cfd")"
+
+if [ "$commit" == "$pc" ]; then
+ quiet=true
+ exit 0
+fi
+
+# If bpkg path is not specified, then use the bpkg program from the script
+# directory, if present. Otherwise, use the 'bpkg' path.
+#
+if [ -z "$bpkg" ]; then
+ bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg"
+
+ if [ ! -x "$bpkg" ]; then
+ bpkg=bpkg
+ fi
+fi
+
+# Find repository sections.
+#
+manifests="$(find "$repo_dir/$repo_ver" -type f -name repositories.manifest)"
+
+# Update the repository sections.
+#
+while read f; do
+ run "$bpkg" rep-create "${rep_create_options[@]}" "$(dirname "$f")"
+done <<<"$manifests"
+
+# rsync (over ssh) the repository to the destinations.
+#
+# Approximate the data transfer timeout via the ServerAlive* ssh options,
+# rounding the timeout up to the nearest multiple of ten.
+#
+# Note: must not contain spaces/use quoting (see rsync -e option).
+#
+n=$(($timeout > 0 ? ($timeout + 9) / 10 : 1))
+ssh_options=(-o ConnectTimeout=$timeout \
+ -o ServerAliveInterval=10 \
+ -o ServerAliveCountMax=$n)
+
+for d in "${destinations[@]}"; do
+
+ # -r (recursive)
+ # -l (copy symlinks and symlinks)
+ # -p (preserve perms)
+ # -t (preserve timestamps)
+ # -O (omit dir timestamps)
+ #
+ # -c (use checksum)
+ # -e (remote shell command)
+ #
+ # --safe-links (ignore symlinks pointing outside the tree)
+ # --delay-updates (first upload all files on the side then move)
+ # --prune-empty-dirs (remove empty dirs)
+ # --delete-after (delete entries after the transfer)
+ #
+ # We also exclude hiddent files (start with dot).
+ #
+ run rsync -v -rlptO -c --safe-links --delay-updates --exclude '.*' \
+--prune-empty-dirs --delete-after -e "ssh ${ssh_options[*]}" \
+"${rsync_options[@]}" "$repo_dir/$repo_ver/" "$d/$repo_ver/" >&2
+
+done
+
+echo "$commit" >"$published_commit"
diff --git a/bpkg-rep/utility.bash.in b/bpkg-rep/utility.bash.in
new file mode 100644
index 0000000..bcf25d8
--- /dev/null
+++ b/bpkg-rep/utility.bash.in
@@ -0,0 +1,87 @@
+# file : bpkg-rep/utility.bash.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Utility functions useful for implementing bpkg repository utilities.
+
+if [ "$bpkg_rep_utility" ]; then
+ return 0
+else
+ bpkg_rep_utility=true
+fi
+
+# Diagnostics.
+#
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+# Trace a command line, quoting empty arguments as well as those that contain
+# spaces.
+#
+function trace_cmd () # <cmd> <arg>...
+{
+ local s="+"
+ while [ $# -gt 0 ]; do
+ if [ -z "$1" -o -z "${1##* *}" ]; then
+ s="$s '$1'"
+ else
+ s="$s $1"
+ fi
+
+ shift
+ done
+
+ info "$s"
+}
+
+# Trace the current function name and arguments.
+#
+function trace_func () # <args>...
+{
+ trace_cmd "${FUNCNAME[1]}" "$@"
+}
+
+# Trace and run a command.
+#
+function run () # <cmd> <arg>...
+{
+ trace_cmd "$@"
+ "$@"
+}
+
+# Return lower-case URL scheme or empty string if the argument doesn't look
+# like a URL.
+#
+function url_scheme () # <url>
+{
+ sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1"
+}
+
+# Check that the git repository properly responds to the probing request
+# before the timeout (in seconds). Noop for protocols other than HTTP(S).
+#
+function check_git_connectivity () # <repo-url> <timeout>
+{
+ local url="$1"
+ local tmo="$2"
+
+ local s
+ s="$(url_scheme "$url")"
+
+ if [ "$s" == "http" -o "$s" == "https" ]; then
+ local u q
+
+ u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part.
+ q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part.
+
+ if [ -z "$q" ]; then
+ u="$u/info/refs?service=git-upload-pack"
+ else
+ u="$u/info/refs$q&service=git-upload-pack"
+ fi
+
+ # Here we limit the time for the whole operation.
+ #
+ curl -S -s --max-time "$tmo" "$u" >/dev/null
+ fi
+}
diff --git a/build/.gitignore b/build/.gitignore
new file mode 100644
index 0000000..4a730a3
--- /dev/null
+++ b/build/.gitignore
@@ -0,0 +1,3 @@
+config.build
+root/
+bootstrap/
diff --git a/build/bootstrap.build b/build/bootstrap.build
new file mode 100644
index 0000000..3685688
--- /dev/null
+++ b/build/bootstrap.build
@@ -0,0 +1,11 @@
+# file : build/bootstrap.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+project = bpkg-rep
+
+using version
+using config
+using dist
+using test
+using install
diff --git a/build/export.build b/build/export.build
new file mode 100644
index 0000000..8c676d6
--- /dev/null
+++ b/build/export.build
@@ -0,0 +1,11 @@
+# file : build/export.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+$out_root/
+{
+ include bpkg-rep/
+}
+
+if ($import.target == exe{bpkg-rep-publish})
+ export $out_root/bpkg-rep/exe{bpkg-rep-publish}
diff --git a/build/root.build b/build/root.build
new file mode 100644
index 0000000..0a2560f
--- /dev/null
+++ b/build/root.build
@@ -0,0 +1,9 @@
+# file : build/root.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Bash.
+#
+using bash
+
+bpkg-rep/bash{*}: install.subdirs = true
diff --git a/buildfile b/buildfile
new file mode 100644
index 0000000..ad6e61a
--- /dev/null
+++ b/buildfile
@@ -0,0 +1,10 @@
+# file : buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+./: {*/ -build/} doc{INSTALL-PUBLISH LICENSE NEWS README} manifest
+
+# Don't install tests or the INSTALL* files.
+#
+tests/: install = false
+doc{INSTALL-PUBLISH}@./: install = false
diff --git a/manifest b/manifest
new file mode 100644
index 0000000..6ed76f1
--- /dev/null
+++ b/manifest
@@ -0,0 +1,21 @@
+: 1
+name: bpkg-rep
+version: 0.9.0-a.0.z
+project: build2
+summary: bpkg repository management utilities
+license: MIT
+tags: bpkg, repository, management, utility, publish
+description-file: README
+changes-file: NEWS
+url: https://build2.org
+doc-url: https://build2.org/doc.xhtml
+src-url: https://git.build2.org/cgit/bpkg-rep/tree/
+email: users@build2.org
+build-email: builds@build2.org
+build-exclude: windows*; Requires bash
+build-exclude: macos*; Requires bash >= 4.3
+build-include: *
+requires: bash >= 4.3
+depends: * build2 >= 0.8.0-
+depends: * bpkg >= 0.8.0-
+depends: bpkg [0.9.0-a.0.1 0.9.0-a.1)
diff --git a/repositories.manifest b/repositories.manifest
new file mode 100644
index 0000000..d33e418
--- /dev/null
+++ b/repositories.manifest
@@ -0,0 +1,2 @@
+: 1
+summary: bpkg repository management utilities repository
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..35ec43f
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1,2 @@
+test/
+test-*/
diff --git a/tests/build/.gitignore b/tests/build/.gitignore
new file mode 100644
index 0000000..4a730a3
--- /dev/null
+++ b/tests/build/.gitignore
@@ -0,0 +1,3 @@
+config.build
+root/
+bootstrap/
diff --git a/tests/build/bootstrap.build b/tests/build/bootstrap.build
new file mode 100644
index 0000000..91bc3e9
--- /dev/null
+++ b/tests/build/bootstrap.build
@@ -0,0 +1,9 @@
+# file : tests/build/bootstrap.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+project = # Unnamed subproject.
+
+using config
+using dist
+using test
diff --git a/tests/build/root.build b/tests/build/root.build
new file mode 100644
index 0000000..1d7601b
--- /dev/null
+++ b/tests/build/root.build
@@ -0,0 +1,7 @@
+# file : tests/build/root.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Setup the utilities that we are testing.
+#
+import publish = bpkg-rep%exe{bpkg-rep-publish}
diff --git a/tests/buildfile b/tests/buildfile
new file mode 100644
index 0000000..36c24d7
--- /dev/null
+++ b/tests/buildfile
@@ -0,0 +1,7 @@
+# file : tests/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+./: testscript{*} $publish
+
+testscript{publish}@./: test = $publish
diff --git a/tests/publish.testscript b/tests/publish.testscript
new file mode 100644
index 0000000..95fd6f5
--- /dev/null
+++ b/tests/publish.testscript
@@ -0,0 +1,9 @@
+# file : tests/publish.testscript
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+: args
+:
+{
+ $* 2>~"/usage: .+/" != 0 : no-args
+}