path: root/stage

#! /usr/bin/env bash

# Stage or queue build2 packages and the toolchain distribution.
#
# -b
#    Packages baseutils/mingw.
# -o
#    Only package baseutils/mingw.
# -s
#    Only prepare the install scripts and regenerate toolchain.sha256.
# -q
#    Put packages and the distribution into the queue instead of staging.
# -p
#    Regenerate and publish distribution. Specifically, move baseutils from
#    queue, regenerate build2-toolchain, install scripts, and the checksum
#    file, and upload everything to public. Also disable the queue toolchain.
#
usage="usage: etc/stage"

rsync_ops="--progress"

owd=`pwd`
trap "{ cd $owd; exit 1; }" ERR
set -o errtrace # Trap in functions.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

# Make sure the build2 tools are runnable.
#
b    --version >/dev/null
bpkg --version >/dev/null
bdep --version >/dev/null

# Make sure the build system if up to date (otherwise update will be triggered
# by libbuild2-* module distribution). Also make sure the package manager is
# up to date since it depend on libbuild2. Update project manager for good
# measure.
#
b build2/build2/ bpkg/bpkg/ bdep/bdep/
b    --version >/dev/null
bpkg --version >/dev/null
bdep --version >/dev/null

v="$(sed -n -re 's/^version: ([^.]+\.[^.]+\.[^-]+(-[ab]\.[^.+]+)?).*$/\1/p' build2-toolchain/manifest)"
vf="$(echo $v | sed -re 's/^([^.]*)\.([^.]*)\.([^-]*).*$/\1.\2.\3/')" # Final.

if [ -z "$v" -o -z "$vf" ]; then
  error "unable to extract version from build2-toolchain/version"
fi

mode=
base=
base_only=
script_only=

dist_src=staging/0
dist_dst=stage.build2.org:/var/www/stage.build2.org/public/0

rsync_del="--delete-after"

while [ $# -gt 0 ]; do
  case $1 in
    -b)
      base=true
      shift
      ;;
    -o)
      base=true
      base_only=true
      shift
      ;;
    -s)
      script_only=true
      shift
      ;;
    -q | -p)
      mode="$1"
      dist_src=cppget.org/queue/0
      dist_dst=build2.org:/var/www/download.build2.org/public/queue

      if [ "$mode" = "-p" ]; then

	queue_dist_src="$dist_src"
	queue_dist_dst="$dist_dst"

	dist_src=cppget.org/public/0
	dist_dst=build2.org:/var/www/download.build2.org/public

	rsync_del= # Do not delete anything existing.
      fi

      shift
      ;;
    *)
      error "unexpected $1"
      ;;
  esac
done

# Note that this structure is seriously hard-wired in the install scripts.
# Specifically, they assume paths in toolchain.sha256 are in the <ver>/<file>
# form.
#
mkdir -p "$dist_src/$v"

# Generate baseutils if requested.
#
if [ "$base" = true ]; then
  cd private/baseutils

  echo "$v" >version

  ./pkdist -n baseutils -d "$vf" -z "/tmp/build2-baseutils-$v-x86_64-windows" \
--sys-p "$vf/list-msys-x86_64" \
--sys-c "$vf/control-msys" \
--sys-s ./pkpost-base-msys \
--min-p "$vf/list-mingw-x86_64" \
--min-c "$vf/control-mingw-x86_64" \
--min-s ./pkpost-base-mingw

  ./pkdist -n mingw -d "mingw-$vf" -x "/tmp/build2-mingw-$v-x86_64-windows" \
--min-p "mingw-$vf/list-mingw-x86_64" \
--min-c "mingw-$vf/control-mingw-x86_64" \
--min-s ./pkpost-mingw-mingw

  cd ../..
fi

# Copy baseutils if requested.
#
if [ "$base" = true -o "$mode" = "-p" ]; then

  if [ "$base" = true ]; then
    d=/tmp
  else
    d="$queue_dist_src/$v"
  fi

  mv "$d/build2-baseutils-$v-x86_64-windows.zip" "$dist_src/$v/"
  mv "$d/build2-baseutils-$v-x86_64-windows.zip.sha256" "$dist_src/$v/"

  mv "$d/build2-mingw-$v-x86_64-windows.tar.xz" "$dist_src/$v/"
  mv "$d/build2-mingw-$v-x86_64-windows.tar.xz.sha256" "$dist_src/$v/"
fi

if [ "$base_only" = true ]; then
  exit 0
fi

# Clean and disable queue toolchain.
#
if [ "$mode" = "-p" ]; then
  rm -rf "$queue_dist_src/$v"
  echo 'disabled' >"$queue_dist_src/toolchain.sha256"
fi

##
##if false; then

mkdir -p /tmp/dist

# Dist build2-toolchain.
#
if [ "$script_only" != true ]; then

  # Cleanup old stuff.
  #
  rm -f "$owd/$dist_src/$v"/build2-toolchain-*

  XZ_OPT=-9 \
  GZIP_OPT=-9 \
  b dist: build2-toolchain-default/ \
    config.dist.root=/tmp/dist \
    "config.dist.archives=$owd/$dist_src/$v/tar.xz $owd/$dist_src/$v/tar.gz" \
    config.dist.checksums=sha256
fi

tv="$(echo "$owd/$dist_src/$v"/build2-toolchain-*.tar.gz | sed -re 's#.*/build2-toolchain-(.+).tar.gz#\1#')"

# Prepare the toolchain.sha256 checksums file.
#
function checksum_line () # <file>
{
  local s
  s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' "$dist_src/$v/$1.sha256")"
  echo "$s *$v/$1" >>"$dist_src/toolchain.sha256"
}

echo "# $tv" >"$dist_src/toolchain.sha256"
checksum_line "build2-toolchain-$tv.tar.xz"              # Must be first.
checksum_line "build2-baseutils-$v-x86_64-windows.zip"
checksum_line "build2-mingw-$v-x86_64-windows.tar.xz"

# Prepare install script/batch files and then append their checksums to
# toolchain.sha256.
#
case "$mode" in
  -q) o=--queue ;;
  -p) o=        ;;
   *) o=--stage ;;
esac

scripts=($(private/install/prepare $o "$dist_src"))

for f in "${scripts[@]}"; do
  info "prepared $f"
  checksum_line "$f"
done

##fi
##

# Sync the distribution.
#
# Note that we do the distribution first so that the toolchain changes are
# visible before package changes. This is relied upon by the Build OS monitor.
#
# We also wse --delay-updates to make things a bit more atomic (we don't want
# to start rebooting before everything is uploaded).
#
function sync_dist () # <dist-src> <dist-dst> [<rsync-options>]
{
  local s="$1"
  local d="$2"

  shift
  shift

  info "Press Enter to start distribution upload to $d"
  read
  rsync -v -rltO -c --chmod=ugo=rwX --delay-updates --copy-unsafe-links \
	--prune-empty-dirs $rsync_ops "$@" "$s/" "$d/"
}

###
###if false; then

# If we are not preparing packages, publish the distribution now. Otherwise,
# wait until we have prepared all the packages (which can take a considerable
# time).
#
if [ "$script_only" -o  "$mode" = "-p" ]; then

  sync_dist "$dist_src" "$dist_dst" $rsync_del

  # Also sync (disable) queue.
  #
  if [ "$mode" = "-p" ]; then
    sync_dist "$queue_dist_src" "$queue_dist_dst" --delete-after
  fi

  exit 0
fi

# Distribute packages, regenerate and sync the repository.
#
# Note: $mode is either empty or -q (which implies -d).

# Extra packages for testing.
#
if false; then
etc/stage-pkg -c $mode -d curl packaging/curl/curl/libcurl
etc/stage-pkg -c $mode -d curl packaging/curl/curl/curl
etc/stage-pkg -c $mode -d curl packaging/curl/ca-certificates-curl/libca-certificates-curl

etc/stage-pkg -c $mode -d icu packaging/icu/icu/libicuuc
etc/stage-pkg -c $mode -d icu packaging/icu/icu/libicui18n
etc/stage-pkg -c $mode -d icu packaging/icu/icu/libicuio
fi

etc/stage-pkg -c $mode -d zlib packaging/zlib/zlib/libz

etc/stage-pkg -c $mode -d openssl packaging/openssl/openssl/libcrypto \
                                  packaging/openssl/openssl/libssl \
                                  packaging/openssl/openssl/openssl

#etc/stage-pkg -c $mode -d mariadb packaging/mariadb/libmariadb
etc/stage-pkg -c $mode -d mysql packaging/mysql/mysql/libmysqlclient

etc/stage-pkg -c $mode -d libcutl ../cutl/libcutl

etc/stage-pkg -c $mode -d cli ../cli/cli/cli
etc/stage-pkg -c $mode -d cli ../cli/cli/cli-tests
etc/stage-pkg -c $mode -d cli ../cli/cli/cli-examples

#
#
etc/stage-pkg -c $mode -d libapr packaging/libapr/libapr1
etc/stage-pkg -c $mode -d libapreq packaging/libapreq/libapreq2

etc/stage-pkg -c $mode -d postgresql \
	      packaging/postgresql/postgresql/libpq \
              packaging/postgresql/postgresql/psql

etc/stage-pkg -c $mode -d sqlite \
	      packaging/sqlite/sqlite/libsqlite3 \
	      packaging/sqlite/sqlite/sqlite3

etc/stage-pkg -c $mode -d libpkg-config libpkg-config/libpkg-config

etc/stage-pkg -c $mode -d cmark-gfm \
	      packaging/cmark-gfm/cmark-gfm/libcmark-gfm \
	      packaging/cmark-gfm/cmark-gfm/libcmark-gfm-extensions

etc/stage-pkg -c $mode -d libstudxml ../studxml/libstudxml

etc/stage-pkg -c $mode -d hello hello/libhello-1.0

# For testing we also stage other libodb-*.
#
# Note: cannot have odb-tests/odb-example since depend on mysql-client (which
# depends on libisocline which doesn't build with -Werror).
#
etc/stage-pkg -c $mode -d odb \
	      ../odb/odb/odb \
	      ../odb/odb/libodb \
	      ../odb/odb/libodb-sqlite \
	      ../odb/odb/libodb-pgsql \
	      ../odb/odb/libodb-mysql \
	      ../odb/odb/libodb-oracle \
	      ../odb/odb/libodb-mssql \
	      ../odb/odb/libodb-boost \
	      ../odb/odb/libodb-qt

#	      ../odb/odb/odb-tests
#	      ../odb/odb/odb-examples


# Regenerate the repository (unless $mode is -q) but do not publish.
#
etc/stage-pkg -c $mode -g build2 \
	      libbutl \
	      libbutl.bash \
	      build2 \
	      libbpkg \
	      bpkg \
	      bpkg-util \
	      bdep \
	      bdep-util \
	      libbbot \
	      brep \
	      bbot \
	      openssl-agent \
              libbuild2-hello/libbuild2-hello \
              libbuild2-hello/libbuild2-hello-tests \
	      libbuild2-kconfig/kconfig-hello \
	      libbuild2-kconfig/libbuild2-kconfig \
              libbuild2-kconfig/libbuild2-kconfig-tests \
	      libbuild2-autoconf/libbuild2-autoconf \
              libbuild2-autoconf/libbuild2-autoconf-tests \
	      libbuild2-qt/libbuild2-qt \
              libbuild2-qt/libbuild2-qt-tests

###fi
###

sync_dist "$dist_src" "$dist_dst" $rsync_del

if [ "$mode" != "-q" ]; then
  etc/stage-pkg -p
fi