Staging scripts updates for queue.stage

3 files changed, 134 insertions, 57 deletions

diff --git a/stage b/stage
index bd4cbc0..cf234df 100755
--- a/stage
+++ b/stage
@@ -229,6 +229,9 @@ function sync_dist () # <dist-src> <dist-dst> [<rsync-options>]
 	--prune-empty-dirs $rsync_ops "$@" "$s/" "$d/"
 }
 
+###
+###if false; then
+
 # If we are not preparing packages, publish the distribution now. Otherwise,
 # wait until we have prepared all the packages (which can take a considerable
 # time).
@@ -251,8 +254,6 @@ fi
 # Note: $mode is either empty or -q (which implies -d).
 #
 
-# These are just for testing (though libodb-mysql needs libmysqlclient).
-#
 etc/stage-pkg -c $mode -d zlib packaging/zlib/zlib/libz
 
 etc/stage-pkg -c $mode -d openssl packaging/openssl/openssl/libcrypto
@@ -268,27 +269,11 @@ etc/stage-pkg -c $mode -d cli ../cli/cli/cli
 etc/stage-pkg -c $mode -d cli ../cli/cli/cli-tests
 etc/stage-pkg -c $mode -d cli ../cli/cli/cli-examples
 
-# These are for extra testing before the release (remember to clean up their
-# directories after disabling).
-#
-if false; then
-
-etc/stage-pkg -c $mode -d curl packaging/curl/curl/libcurl
-etc/stage-pkg -c $mode -d curl packaging/curl/curl/curl
-etc/stage-pkg -c $mode -d curl packaging/curl/ca-certificates-curl/libca-certificates-curl
-
-etc/stage-pkg -c $mode -d icu packaging/icu/icu/libicuuc
-etc/stage-pkg -c $mode -d icu packaging/icu/icu/libicui18n
-etc/stage-pkg -c $mode -d icu packaging/icu/icu/libicuio
-
-etc/stage-pkg -c $mode -d xerces-c packaging/xerces-c/xerces-c/libxerces-c
-
-fi
-
 #
 #
 etc/stage-pkg -c $mode -d libapr packaging/libapr/libapr1
 etc/stage-pkg -c $mode -d libapreq packaging/libapreq/libapreq2
+
 etc/stage-pkg -c $mode -d postgresql packaging/postgresql/postgresql/libpq
 
 etc/stage-pkg -c $mode -d sqlite \
@@ -305,10 +290,6 @@ etc/stage-pkg -c $mode -d libstudxml ../studxml/libstudxml
 
 etc/stage-pkg -c $mode -d hello hello/libhello-1.0
 
-etc/stage-pkg -c $mode -d kconfig packaging/kconfig/kconfig/liblkc \
-              packaging/kconfig/kconfig/kconfig-conf \
-	      packaging/kconfig/kconfig/kconfig-qconf
-
 # For testing we also stage other libodb-*.
 #
 etc/stage-pkg -c $mode -d odb \
@@ -343,6 +324,9 @@ etc/stage-pkg -c $mode -g build2 \
 	      libbuild2-kconfig/libbuild2-kconfig \
               libbuild2-kconfig/libbuild2-kconfig-tests
 
+###fi
+###
+
 sync_dist "$dist_src" "$dist_dst" $rsync_del
 
 if [ "$mode" != "-q" ]; then
diff --git a/stage-pkg b/stage-pkg
index 5814d21..fe876c6 100755
--- a/stage-pkg
+++ b/stage-pkg
@@ -10,6 +10,7 @@
 #
 # -g
 #    Distribute and regenerate only without publishing the repository.
+#    In this mode of no packages is specified, then just regenerate.
 #
 # -p
 #    Publish only without distributing and regenerating the repository.
@@ -17,6 +18,13 @@
 # -q
 #    Put packages into the queue instead of staging. Implies -d.
 #
+# -Q
+#    Put packages into staging queue instead of staging.
+#
+# --min-bpkg-version <ver>
+#
+#    Pass --min-bpkg-version to bpkg-rep-create.
+#
 usage="usage: etc/stage-pkg [<options>] <group> <dir>..."
 
 rsync_ops="--progress"
@@ -45,6 +53,7 @@ gen_only=
 pub_only=
 group=
 dirs=()
+rep_create_ops=()
 
 while [ $# -gt 0 ]; do
   case $1 in
@@ -72,6 +81,17 @@ while [ $# -gt 0 ]; do
       dist_only=true
       shift
       ;;
+    -Q)
+      repo_name="QUEUE.STAGE.BUILD2.ORG"
+      repo_dir="staging/queue/1"
+      repo_host1="stage.build2.org:/var/bpkg/queue/1"
+      shift
+      ;;
+    --min-bpkg-version)
+      shift
+      rep_create_ops+=(--min-bpkg-version "$1")
+      shift
+      ;;
     *)
       if [ -z "$group" ]; then
 	group="$1"
@@ -85,45 +105,50 @@ done
 
 if [ -z "$pub_only" ]; then
 
-  if [ -z "$group" -o "${#dirs[@]}" -eq 0 ]; then
-    error "$usage"
-  fi
+  # In the -g mode skip distributing if no packages are specified.
+  #
+  if [ -z "$gen_only" -o -n "$group" ]; then
 
-  mkdir -p /tmp/dist
+    if [ -z "$group" -o "${#dirs[@]}" -eq 0 ]; then
+      error "$usage"
+    fi
 
-  # Dist individual packages into the repository.
-  #
-  function dist() # <group> <dir>
-  {
-    local o="$repo_dir/$1"
-    local b="$(basename $2)"
-    local d="$2-default"
-
-    # If *-default/ exists, use that (old style out of tree configuration).
-    # Otherwise, use the source directory itself (new style forwarded
-    # configuration).
+    mkdir -p /tmp/dist
+
+    # Dist individual packages into the repository.
     #
-    if ! test -d "$d"; then
-      d="$2"
+    function dist() # <group> <dir>
+    {
+      local o="$repo_dir/$1"
+      local b="$(basename $2)"
+      local d="$2-default"
+
+      # If *-default/ exists, use that (old style out of tree configuration).
+      # Otherwise, use the source directory itself (new style forwarded
+      # configuration).
+      #
       if ! test -d "$d"; then
-	error "neither $2-default nor $2 exist"
+	d="$2"
+	if ! test -d "$d"; then
+	  error "neither $2-default nor $2 exist"
+	fi
       fi
-    fi
 
-    mkdir -p "$o"
+      mkdir -p "$o"
 
-    # Clean up old packages.
-    #
-    if [ -n "$clean" ]; then
-      rm -f "$o/$b"-*
-    fi
+      # Clean up old packages.
+      #
+      if [ -n "$clean" ]; then
+	rm -f "$o/$b"-*
+      fi
 
-    b "dist($d/)" config.dist.root=/tmp/dist "config.dist.archives=$owd/$o/tar.gz"
-  }
+      b "dist($d/)" config.dist.root=/tmp/dist "config.dist.archives=$owd/$o/tar.gz"
+    }
 
-  for d in "${dirs[@]}"; do
-    dist "$group" "$d"
-  done
+    for d in "${dirs[@]}"; do
+      dist "$group" "$d"
+    done
+  fi
 
   if [ -n "$dist_only" ]; then
     exit 0
@@ -133,10 +158,10 @@ if [ -z "$pub_only" ]; then
   #
   info "Insert $repo_name signing key and press Enter"
   read
-  etc/rep-update "$repo_dir/" \
-    --openssl-option -engine --openssl-option pkcs11 \
-   --openssl-option -keyform --openssl-option engine \
-   --key "label_SIGN key"
+  etc/rep-update "$repo_dir/" "${rep_create_ops[@]}"  \
+    --openssl-option -engine --openssl-option pkcs11  \
+    --openssl-option -keyform --openssl-option engine \
+    --key "label_SIGN key"
 
   if [ -n "$gen_only" ]; then
     exit 0
diff --git a/stage-queue b/stage-queue
new file mode 100755
index 0000000..5f2e907
--- /dev/null
+++ b/stage-queue
@@ -0,0 +1,68 @@
+#! /usr/bin/env bash
+
+# Stage queued packages (see stage script for details).
+#
+usage="usage: etc/stage-queue"
+
+owd=`pwd`
+trap "{ cd $owd; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+while [ $# -gt 0 ]; do
+  case $1 in
+    *)
+      error "unexpected $1"
+      ;;
+  esac
+done
+
+# Generate package archives.
+#
+
+# These are currently pulled from cppget.org (we cannot have the same
+# versions in multiple places since archive checksums are not stable).
+#
+#etc/stage-pkg -Q -c -d curl packaging/curl/curl/libcurl
+#etc/stage-pkg -Q -c -d curl packaging/curl/curl/curl
+#etc/stage-pkg -Q -c -d curl packaging/curl/ca-certificates-curl/libca-certificates-curl
+
+#etc/stage-pkg -Q -c -d icu packaging/icu/icu/libicuuc
+#etc/stage-pkg -Q -c -d icu packaging/icu/icu/libicui18n
+#etc/stage-pkg -Q -c -d icu packaging/icu/icu/libicuio
+
+etc/stage-pkg -Q -c -d tinycbor packaging/tinycbor/tinycbor/libtinycbor
+
+etc/stage-pkg -Q -c -d xxd packaging/xxd/xxd/xxd
+etc/stage-pkg -Q -c -d byacc packaging/byacc/byacc/byacc
+etc/stage-pkg -Q -c -d reflex packaging/reflex/reflex/reflex
+etc/stage-pkg -Q -c -d openbsd-m4 packaging/openbsd-m4/openbsd-m4/openbsd-m4
+
+etc/stage-pkg -Q -c -d kconfig packaging/kconfig/kconfig/liblkc \
+                               packaging/kconfig/kconfig/kconfig-conf \
+	                       packaging/kconfig/kconfig/kconfig-qconf
+
+etc/stage-pkg -Q -c -d xerces-c packaging/xerces-c/xerces-c/libxerces-c
+
+etc/stage-pkg -Q -c -d libcutl ../cutl/libcutl
+
+etc/stage-pkg -Q -c -d cli ../cli/cli/cli
+etc/stage-pkg -Q -c -d cli ../cli/cli/cli-tests
+etc/stage-pkg -Q -c -d cli ../cli/cli/cli-examples
+
+etc/stage-pkg -Q -c -d libxsd-frontend ../xsd/libxsd-frontend/
+
+etc/stage-pkg -Q -c -d xsd ../xsd/xsd/libxsd
+etc/stage-pkg -Q -c -d xsd ../xsd/xsd/libxsd-tests
+etc/stage-pkg -Q -c -d xsd ../xsd/xsd/xsd
+etc/stage-pkg -Q -c -d xsd ../xsd/xsd/xsd-tests
+etc/stage-pkg -Q -c -d xsd ../xsd/xsd/xsd-examples
+
+etc/stage-pkg -Q -c -d boost packaging/boost/boost/libboost-*/
+
+# Regenerate repository and publish.
+#
+etc/stage-pkg -Q -g --min-bpkg-version 0.13.0
+etc/stage-pkg -Q -p