Add implementation

21 files changed, 554 insertions, 0 deletions

diff --git a/INSTALL-PUBLISH b/INSTALL-PUBLISH
new file mode 100644
index 0000000..a309a51
--- /dev/null
+++ b/INSTALL-PUBLISH
@@ -0,0 +1 @@
+@@ TODO
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..c1332e6
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,20 @@
+Copyright (c) 2014-2018 Code Synthesis Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/NEWS b/NEWS
new file mode 100644
index 0000000..32bf3ce
--- /dev/null
+++ b/NEWS
@@ -0,0 +1,3 @@
+Version 0.9.0
+
+  * First public release.
diff --git a/README b/README
new file mode 100644
index 0000000..e1eb555
--- /dev/null
+++ b/README
@@ -0,0 +1,20 @@
+This package contains bpkg repository management utilities.
+
+build2 is an open source, cross-platform toolchain for building and packaging
+C++ code. Its aim is a modern build system and dependency manager for the C++
+language that provide a consistent, out of the box interface across multiple
+platforms and compilers. For more information see:
+
+https://build2.org/
+
+See the NEWS file for the user-visible changes from the previous release.
+
+See the LICENSE file for the distribution conditions.
+
+See the INSTALL file for the prerequisites and installation instructions.
+
+See the doc/ directory for documentation.
+
+Send questions, bug reports, or any other feedback to the users@build2.org
+mailing list. You can post without subscribing. See https://lists.build2.org
+for searchable archives, posting guidelines, etc.
diff --git a/bpkg-rep/.gitignore b/bpkg-rep/.gitignore
new file mode 100644
index 0000000..39d8ec1
--- /dev/null
+++ b/bpkg-rep/.gitignore
@@ -0,0 +1 @@
+bpkg-rep-publish
diff --git a/bpkg-rep/buildfile b/bpkg-rep/buildfile
new file mode 100644
index 0000000..6f4a025
--- /dev/null
+++ b/bpkg-rep/buildfile
@@ -0,0 +1,9 @@
+# file      : bpkg-rep/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: exe{bpkg-rep-publish}
+
+exe{bpkg-rep-publish}: in{publish} bash{utility}
+
+bash{utility}: in{utility}
diff --git a/bpkg-rep/publish.in b/bpkg-rep/publish.in
new file mode 100644
index 0000000..e828692
--- /dev/null
+++ b/bpkg-rep/publish.in
@@ -0,0 +1,309 @@
+#!/usr/bin/env bash
+
+# file      : bpkg-rep/publish.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Update (bpkg-rep-create(1)) and publish (rsync(1)) an archive-based
+# repository.
+#
+# Pull a pre-cloned (read-only) git repository with the contents of an
+# archive-based bpkg repository. Bail out if nothing changed from the the
+# previous run. Otherwise, regenerate the repository meta-data by running
+# bpkg-rep-create(1) on each section of the repository and, optionally,
+# synchronize it to one or more destinations with rsync.
+#
+# The repository contents are expected to be in the <dir>/1/ subdirectory. The
+# script saves the last successfully published commit in the <dir>.publish
+# file.
+#
+# --destination|-d <host>:<dir>
+#
+#   Remote host and directory to rsync the repository to. Note that the
+#   trailing 1/ will be added automatically. In other words, the rsync command
+#   will be in the form:
+#
+#   rsync ... <dir>/1/ <host>:<dir>/1/
+#
+#   Repeat this option to specify multiple destinations. In this case, the
+#   destinations are synced in the order specified with the first failure
+#   terminating the process (so if you have a "primary" destination and a
+#   "mirror", you probably want to specify the former first).
+#
+# --timeout <seconds>
+#
+#   Git and rsync operation timeout. Specifically, the operation will be
+#   aborted if there is no network activity for the specified time. Default is
+#   60 seconds. Note that currently the git timeout is only supported for the
+#   http(s) transport.
+#
+# --lock-timeout <seconds>
+#
+#   The repository lock timeout. Fail if another instance of the script does
+#   not release the repository in the specified time. The default is 0 (do
+#   not wait).
+#
+#   Note that you will most likely want to specify a non-zero timeout for cron
+#   jobs that may potentially overlap.
+#
+# --log-dir <dir>
+#
+#   Directory to create the temporary log files in. If unspecified, the stderr
+#   is not redirected and no log is created by default.
+#
+#   The log is dumped to stderr in case of an error or at the end of execution
+#   unless in the quiet mode and is then deleted.
+#
+# --quiet
+#
+#   Run quiet. Specifically, don't dump the log to stderr on exit with zero
+#   status.
+#
+# --bpkg <path>
+#
+#   The package manager program to be used for the repository update. This
+#   should be the path to the bpkg executable.
+#
+usage="usage: $0 [<options>] <dir> [<rep-create-options>] [-- <rsync-options>]"
+
+trap "{ exit 1; }" ERR
+set -o errtrace # Trap ERR in functions.
+
+@import bpkg-rep/utility@
+
+# The script own options.
+#
+repo_ver=1
+destinations=()
+timeout=60
+lock_timeout=0
+log_dir=
+quiet=
+bpkg=
+
+while [ $# -gt 0 ]; do
+  case $1 in
+    --destination|-d)
+      shift
+      destinations+=("$1")
+      shift || true
+      ;;
+    --timeout)
+      shift
+      timeout="$1"
+      shift || true
+      ;;
+    --lock-timeout)
+      shift
+      lock_timeout="$1"
+      shift || true
+      ;;
+    --log-dir)
+      shift
+      log_dir="${1%/}"
+      shift || true
+      ;;
+    --quiet)
+      shift
+      quiet=true
+      ;;
+    --bpkg)
+      shift
+      bpkg="$1"
+      shift || true
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+# The repository directory.
+#
+repo_dir="${1%/}"
+shift || true
+
+# bpkg-rep-create options.
+#
+rep_create_options=()
+
+while [ $# -gt 0 ]; do
+  case $1 in
+    --)
+      shift
+      break
+      ;;
+    *)
+      rep_create_options+=("$1")
+      shift
+      ;;
+  esac
+done
+
+# rsync options.
+#
+rsync_options=()
+
+while [ $# -gt 0 ]; do
+  rsync_options+=("$1")
+  shift
+done
+
+# Validate options and arguments.
+#
+if [ -z "$repo_dir" ]; then
+  error "$usage"
+fi
+
+if [ ! -d "$repo_dir" ]; then
+  error "'$repo_dir' does not exist or is not a directory"
+fi
+
+# If the log directory is specified then redirect stderr to the log file and
+# setup the trap that dumps it on exit, if required.
+#
+if [ -n "$log_dir" ]; then
+
+  if [ ! -d "$log_dir" ]; then
+    error "'$log_dir' does not exist or is not a directory"
+  fi
+
+  # Create the log file.
+  #
+  log="$(mktemp "$log_dir/$(basename "$repo_dir").XXXXXXXXXX")"
+
+  # Save the stderr file descriptor so we can dump the log into it on exit, if
+  # required. Then redirect it to the log file.
+  #
+  exec {stderr}>&2
+  exec 2>>"$log"
+
+  function exit_trap ()
+  {
+    local status="$?"
+
+    # Dump the log to stderr if exiting with non-zero status or verbose.
+    #
+    if [ $status -ne 0 -o ! "$quiet" ]; then
+
+      # Keep the log if failed to dump for any reason.
+      #
+      if ! cat "$log" >&$stderr; then
+        return
+      fi
+    fi
+
+    rm -f "$log"
+  }
+
+  trap exit_trap EXIT
+fi
+
+# Make sure the commit file is present.
+#
+published_commit="$repo_dir.publish"
+touch "$published_commit"
+
+# Open the reading file descriptor and lock the repository. Fail if unable to
+# lock before timeout.
+#
+exec {cfd}<"$published_commit"
+
+if ! flock -w "$lock_timeout" "$cfd"; then
+  info "another instance is already running"
+  exit 2
+fi
+
+# Pull the repository.
+#
+# Git doesn't support the connection timeout option. The options we use are
+# just an approximation of the former, that, in particular, don't cover the
+# connection establishing. To work around this problem, before running a git
+# command that assumes the remote repository communication we manually check
+# connectivity with the remote repository.
+#
+if ! remote_url="$(git -C "$repo_dir" config --get remote.origin.url)"; then
+  error "'$repo_dir' is not a git repository"
+fi
+
+run check_git_connectivity "$remote_url" "$timeout"
+
+# Fail if no network activity happens during the time specified.
+#
+run git -c http.lowSpeedLimit=1 -c "http.lowSpeedTime=$timeout" \
+-C "$repo_dir" pull -v >&2
+
+# Match the HEAD commit id to the one stored in the file. If it matches, then
+# nothing changed in the repository from the previous run and we can silently
+# bail out.
+#
+commit="$(git -C "$repo_dir" rev-parse HEAD)"
+pc="$(cat <&"$cfd")"
+
+if [ "$commit" == "$pc" ]; then
+  quiet=true
+  exit 0
+fi
+
+# If bpkg path is not specified, then use the bpkg program from the script
+# directory, if present. Otherwise, use the 'bpkg' path.
+#
+if [ -z "$bpkg" ]; then
+  bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg"
+
+  if [ ! -x "$bpkg" ]; then
+    bpkg=bpkg
+  fi
+fi
+
+# Find repository sections.
+#
+manifests="$(find "$repo_dir/$repo_ver" -type f -name repositories.manifest)"
+
+# Update the repository sections.
+#
+while read f; do
+  run "$bpkg" rep-create "${rep_create_options[@]}" "$(dirname "$f")"
+done <<<"$manifests"
+
+# rsync (over ssh) the repository to the destinations.
+#
+# Approximate the data transfer timeout via the ServerAlive* ssh options,
+# rounding the timeout up to the nearest multiple of ten.
+#
+# Note: must not contain spaces/use quoting (see rsync -e option).
+#
+n=$(($timeout > 0 ? ($timeout + 9) / 10 : 1))
+ssh_options=(-o ConnectTimeout=$timeout \
+	     -o ServerAliveInterval=10 \
+	     -o ServerAliveCountMax=$n)
+
+for d in "${destinations[@]}"; do
+
+  # -r (recursive)
+  # -l (copy symlinks and symlinks)
+  # -p (preserve perms)
+  # -t (preserve timestamps)
+  # -O (omit dir timestamps)
+  #
+  # -c (use checksum)
+  # -e (remote shell command)
+  #
+  # --safe-links       (ignore symlinks pointing outside the tree)
+  # --delay-updates    (first upload all files on the side then move)
+  # --prune-empty-dirs (remove empty dirs)
+  # --delete-after     (delete entries after the transfer)
+  #
+  # We also exclude hiddent files (start with dot).
+  #
+  run rsync -v -rlptO -c --safe-links --delay-updates --exclude '.*' \
+--prune-empty-dirs --delete-after -e "ssh ${ssh_options[*]}" \
+"${rsync_options[@]}" "$repo_dir/$repo_ver/" "$d/$repo_ver/" >&2
+
+done
+
+echo "$commit" >"$published_commit"
diff --git a/bpkg-rep/utility.bash.in b/bpkg-rep/utility.bash.in
new file mode 100644
index 0000000..bcf25d8
--- /dev/null
+++ b/bpkg-rep/utility.bash.in
@@ -0,0 +1,87 @@
+# file      : bpkg-rep/utility.bash.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Utility functions useful for implementing bpkg repository utilities.
+
+if [ "$bpkg_rep_utility" ]; then
+  return 0
+else
+  bpkg_rep_utility=true
+fi
+
+# Diagnostics.
+#
+function info  () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+# Trace a command line, quoting empty arguments as well as those that contain
+# spaces.
+#
+function trace_cmd () # <cmd> <arg>...
+{
+  local s="+"
+  while [ $# -gt 0 ]; do
+    if [ -z "$1" -o -z "${1##* *}" ]; then
+      s="$s '$1'"
+    else
+      s="$s $1"
+    fi
+
+    shift
+  done
+
+  info "$s"
+}
+
+# Trace the current function name and arguments.
+#
+function trace_func () # <args>...
+{
+  trace_cmd "${FUNCNAME[1]}" "$@"
+}
+
+# Trace and run a command.
+#
+function run () # <cmd> <arg>...
+{
+  trace_cmd "$@"
+  "$@"
+}
+
+# Return lower-case URL scheme or empty string if the argument doesn't look
+# like a URL.
+#
+function url_scheme () # <url>
+{
+  sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1"
+}
+
+# Check that the git repository properly responds to the probing request
+# before the timeout (in seconds). Noop for protocols other than HTTP(S).
+#
+function check_git_connectivity () # <repo-url> <timeout>
+{
+  local url="$1"
+  local tmo="$2"
+
+  local s
+  s="$(url_scheme "$url")"
+
+  if [ "$s" == "http" -o "$s" == "https" ]; then
+    local u q
+
+    u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part.
+    q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part.
+
+    if [ -z "$q" ]; then
+      u="$u/info/refs?service=git-upload-pack"
+    else
+      u="$u/info/refs$q&service=git-upload-pack"
+    fi
+
+    # Here we limit the time for the whole operation.
+    #
+    curl -S -s --max-time "$tmo" "$u" >/dev/null
+  fi
+}
diff --git a/build/.gitignore b/build/.gitignore
new file mode 100644
index 0000000..4a730a3
--- /dev/null
+++ b/build/.gitignore
@@ -0,0 +1,3 @@
+config.build
+root/
+bootstrap/
diff --git a/build/bootstrap.build b/build/bootstrap.build
new file mode 100644
index 0000000..3685688
--- /dev/null
+++ b/build/bootstrap.build
@@ -0,0 +1,11 @@
+# file      : build/bootstrap.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+project = bpkg-rep
+
+using version
+using config
+using dist
+using test
+using install
diff --git a/build/export.build b/build/export.build
new file mode 100644
index 0000000..8c676d6
--- /dev/null
+++ b/build/export.build
@@ -0,0 +1,11 @@
+# file      : build/export.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+$out_root/
+{
+  include bpkg-rep/
+}
+
+if ($import.target == exe{bpkg-rep-publish})
+  export $out_root/bpkg-rep/exe{bpkg-rep-publish}
diff --git a/build/root.build b/build/root.build
new file mode 100644
index 0000000..0a2560f
--- /dev/null
+++ b/build/root.build
@@ -0,0 +1,9 @@
+# file      : build/root.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Bash.
+#
+using bash
+
+bpkg-rep/bash{*}: install.subdirs = true
diff --git a/buildfile b/buildfile
new file mode 100644
index 0000000..ad6e61a
--- /dev/null
+++ b/buildfile
@@ -0,0 +1,10 @@
+# file      : buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: {*/ -build/} doc{INSTALL-PUBLISH LICENSE NEWS README} manifest
+
+# Don't install tests or the INSTALL* files.
+#
+tests/:                  install = false
+doc{INSTALL-PUBLISH}@./: install = false
diff --git a/manifest b/manifest
new file mode 100644
index 0000000..6ed76f1
--- /dev/null
+++ b/manifest
@@ -0,0 +1,21 @@
+: 1
+name: bpkg-rep
+version: 0.9.0-a.0.z
+project: build2
+summary: bpkg repository management utilities
+license: MIT
+tags: bpkg, repository, management, utility, publish
+description-file: README
+changes-file: NEWS
+url: https://build2.org
+doc-url: https://build2.org/doc.xhtml
+src-url: https://git.build2.org/cgit/bpkg-rep/tree/
+email: users@build2.org
+build-email: builds@build2.org
+build-exclude: windows*; Requires bash
+build-exclude: macos*; Requires bash >= 4.3
+build-include: *
+requires: bash >= 4.3
+depends: * build2 >= 0.8.0-
+depends: * bpkg >= 0.8.0-
+depends: bpkg [0.9.0-a.0.1 0.9.0-a.1)
diff --git a/repositories.manifest b/repositories.manifest
new file mode 100644
index 0000000..d33e418
--- /dev/null
+++ b/repositories.manifest
@@ -0,0 +1,2 @@
+: 1
+summary: bpkg repository management utilities repository
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..35ec43f
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1,2 @@
+test/
+test-*/
diff --git a/tests/build/.gitignore b/tests/build/.gitignore
new file mode 100644
index 0000000..4a730a3
--- /dev/null
+++ b/tests/build/.gitignore
@@ -0,0 +1,3 @@
+config.build
+root/
+bootstrap/
diff --git a/tests/build/bootstrap.build b/tests/build/bootstrap.build
new file mode 100644
index 0000000..91bc3e9
--- /dev/null
+++ b/tests/build/bootstrap.build
@@ -0,0 +1,9 @@
+# file      : tests/build/bootstrap.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+project = # Unnamed subproject.
+
+using config
+using dist
+using test
diff --git a/tests/build/root.build b/tests/build/root.build
new file mode 100644
index 0000000..1d7601b
--- /dev/null
+++ b/tests/build/root.build
@@ -0,0 +1,7 @@
+# file      : tests/build/root.build
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Setup the utilities that we are testing.
+#
+import publish = bpkg-rep%exe{bpkg-rep-publish}
diff --git a/tests/buildfile b/tests/buildfile
new file mode 100644
index 0000000..36c24d7
--- /dev/null
+++ b/tests/buildfile
@@ -0,0 +1,7 @@
+# file      : tests/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: testscript{*} $publish
+
+testscript{publish}@./: test = $publish
diff --git a/tests/publish.testscript b/tests/publish.testscript
new file mode 100644
index 0000000..95fd6f5
--- /dev/null
+++ b/tests/publish.testscript
@@ -0,0 +1,9 @@
+# file      : tests/publish.testscript
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+: args
+:
+{
+  $* 2>~"/usage: .+/" != 0 : no-args
+}