aboutsummaryrefslogtreecommitdiff
path: root/etc/private/install/brep-install
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2020-09-03 11:44:20 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2020-09-03 11:47:33 +0200
commitf42eb41a5164780ac8bf5934d0fa6278a6ace6f0 (patch)
tree5dceec3319616297a8100961f83545096716627d /etc/private/install/brep-install
parent35c888632c7cfa8a2c2c3995c79277dd43a179af (diff)
Initial support for private brep instance setup
Diffstat (limited to 'etc/private/install/brep-install')
-rwxr-xr-xetc/private/install/brep-install433
1 files changed, 433 insertions, 0 deletions
diff --git a/etc/private/install/brep-install b/etc/private/install/brep-install
new file mode 100755
index 0000000..0c6c2e6
--- /dev/null
+++ b/etc/private/install/brep-install
@@ -0,0 +1,433 @@
+#! /usr/bin/env bash
+
+# file : etc/private/install/brep-install
+# license : MIT; see accompanying LICENSE file
+
+# Setup HTTP-only brep instance with unsigned package submission support via
+# direct repository publishing (brep/handler/submit/submit-pub).
+#
+# NOTE: this setup should only be used in private/trusted environments.
+#
+# Unless the --setup option is specified, create the 'brep' group and user and
+# re-run itself with the --setup option under this user. In the setup mode
+# install and configure the brep instance, automating the instructions from
+# the INSTALL file, including:
+#
+# - Build the build2 toolchain (installing it to /usr/local/) and brep
+# (installing it to ~brep/install/).
+#
+# - Install PostgreSQL and create brep users/databases.
+#
+# - Installing Apache2 and configure HTTP server with the brep module.
+#
+# Note that the script is written for use on Debian-based distributions so you
+# will need to adjust it to match other distributions or operating systems.
+#
+# Options:
+#
+# --mount
+#
+# Mount the virtio-9p device with id 'state' as the /var/brep directory.
+# This directory is expected to either contain the pkg repository or be
+# empty, in which case an empty repository will be automatically
+# initialized. If this option is unspecified, the directory will be created
+# in the local filesystem.
+#
+# --brep-user
+#
+# User and group ids to use when creating the 'brep' group and user. If
+# unspecified, 63700 is used.
+#
+# --setup
+#
+# Install and configure the brep instance, assuming that the 'brep' user
+# already exists and this script is executed as this user.
+#
+usage="Usage: $0 [<options>]"
+
+# build2 toolchain repository certificate fingerprint. Note: this is a
+# repository the toolchain installation script downloads the build2 packages
+# from.
+#
+toolchain_repo_cert_fp="86:BA:D4:DE:2C:87:1A:EE:38:C7:F1:64:7F:65:77:02:15:79:F3:C4:83:C0:AB:5A:EA:F4:F7:8C:1D:63:30:C6"
+#toolchain_repo_cert_fp="37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65"
+
+# brep package repository URL and certificate fingerprint.
+#
+#brep_repo_url="https://pkg.cppget.org/1/alpha"
+#brep_repo_cert_fp="86:BA:D4:DE:2C:87:1A:EE:38:C7:F1:64:7F:65:77:02:15:79:F3:C4:83:C0:AB:5A:EA:F4:F7:8C:1D:63:30:C6"
+brep_repo_url="https://stage.build2.org/1"
+brep_repo_cert_fp="37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65"
+
+owd=`pwd`
+trap "{ exit 1; }" ERR
+trap "{ cd $owd; }" EXIT
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "error: $*"; exit 1; }
+
+# Trace a command line, quoting empty arguments as well as those that contain
+# spaces.
+#
+function trace () # <cmd> <arg>...
+{
+ local s="+"
+ while [ "$#" -gt 0 ]; do
+ if [ -z "$1" -o -z "${1##* *}" ]; then
+ s="$s \"$1\""
+ else
+ s="$s $1"
+ fi
+
+ shift
+ done
+
+ info "$s"
+}
+
+# Trace and run a command.
+#
+run () # <args>...
+{
+ trace "$@"
+ "$@"
+}
+
+# The chosen fixed id for the 'brep' user. Note: must match the id of the
+# 'brep' user on the host.
+#
+# Note that Linux assigns the [0 99] range for the statically allocated system
+# users and [100 499] -- for dynamic allocations by administrators and post-
+# install scripts. Debian, in turn, assigns the [100 999] range for the
+# dynamically allocated system users and [60000 64999] -- for statically
+# allocated on demand "obscure package users".
+#
+brep_id=63700 # Update the README file on change.
+
+# Parse the command line options and, while at it, compose the options array
+# for potential re-execution as the 'brep' user.
+#
+mount=
+setup=
+ops=()
+
+while [ "$#" -gt 0 ]; do
+ case "$1" in
+ --mount)
+ mount=true
+ ops+=("$1")
+ shift
+ ;;
+ --brep-user)
+ shift
+ brep_id="$1"
+ shift
+ ;;
+ --setup)
+ setup=true
+ shift
+ ;;
+ *)
+ break # The end of options is encountered.
+ ;;
+ esac
+done
+
+if [ "$#" -ne 0 ]; then
+ error "$usage"
+fi
+
+scr_exe="$(realpath "${BASH_SOURCE[0]}")"
+scr_dir="$(dirname "$scr_exe")"
+
+# Unless we are not in the setup mode, non-interactively add the 'brep'
+# user/group and re-execute the script in the setup mode as this user.
+#
+if [ ! "$setup" ]; then
+ run sudo addgroup --gid "$brep_id" brep
+
+ run sudo adduser --uid "$brep_id" --gid "$brep_id" --disabled-password \
+ --gecos "" brep
+
+ run sudo tee -a /etc/sudoers.d/brep >/dev/null <<EOF
+brep ALL=(ALL) NOPASSWD:ALL
+EOF
+
+ run sudo chmod 0440 /etc/sudoers.d/brep
+
+ # Use --session-command rather than --command|-c to make sure that when the
+ # su program receives SIGINT (Ctrl-C) it kills not just its child process
+ # but also all its descendants.
+ #
+ # Note: here we rely on ops to not contain spaces or be empty.
+ #
+ run exec sudo su -l brep --session-command "'$scr_exe' --setup ${ops[@]}"
+fi
+
+# Here we assume that we are executed as brep user.
+#
+run cd "$HOME"
+
+# Mount the brep state directory, if requested. Note that otherwise, the
+# directory will be created later, in the local filesystem by the brep-startup
+# script.
+#
+if [ "$mount" ]; then
+ run sudo mkdir -p /var/brep
+
+ run sudo tee -a /etc/fstab >/dev/null <<EOF
+state /var/brep 9p trans=virtio,version=9p2000.L,posixacl,cache=none,_netdev 0 0
+EOF
+
+ run sudo mount -a
+fi
+
+# Install the prerequisite binary packages.
+#
+run sudo apt-get --yes update
+run sudo apt-get --yes install --no-install-recommends g++
+run sudo apt-get --yes install --no-install-recommends postgresql postgresql-contrib libpq-dev
+run sudo apt-get --yes install --no-install-recommends apache2 libapr1-dev libapreq2-dev apache2-dev
+run sudo apt-get --yes install --no-install-recommends acl rsync
+run sudo apt-get clean
+
+# Install build2 toolchain.
+#
+run mkdir build2-build
+run cd build2-build
+
+# Look for the toolchain installation script in this script directory.
+#
+run cp "$(echo "$scr_dir"/build2-install-*.sh)" .
+run sh ./build2-install-*.sh --no-check --yes --trust "$toolchain_repo_cert_fp"
+#run sh ./build2-install-*.sh --no-check --yes --local
+
+run cd .. # Back to brep home.
+
+# Grant Apache2 read access to the module and configuration.
+#
+run setfacl -m g:www-data:rx "$HOME"
+run setfacl -dm g:www-data:rx "$HOME"
+
+# Install brep.
+#
+run mkdir brep
+run cd brep
+
+run bpkg create \
+ cc \
+ config.cc.coptions="-O3" \
+ config.cc.poptions="-I$(apxs -q includedir)" \
+ config.bin.lib=shared \
+ config.bin.rpath="$HOME/install/lib" \
+ config.install.root="$HOME/install"
+
+run bpkg add "$brep_repo_url"
+run bpkg fetch --trust "$brep_repo_cert_fp"
+run bpkg build --yes brep ?sys:libapr1 ?sys:libapreq2 ?sys:libpq
+run bpkg install brep
+
+run cd .. # Back to brep home.
+
+# Create PostgreSQL user and databases.
+#
+# Note that while we could probably omit the build-related setup, let's keep
+# it to stay close to the instructions in the INSTALL file and to simplify the
+# potential future configuration of the brep instance as a build2 build bot
+# controller.
+#
+run sudo sudo -u postgres psql <<EOF
+CREATE DATABASE brep_package
+TEMPLATE template0
+ENCODING 'UTF8'
+LC_COLLATE 'en_US.UTF8'
+LC_CTYPE 'en_US.UTF8';
+
+CREATE DATABASE brep_build
+TEMPLATE template0
+ENCODING 'UTF8'
+LC_COLLATE 'en_US.UTF8'
+LC_CTYPE 'en_US.UTF8';
+
+CREATE USER brep;
+
+GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO brep;
+
+CREATE USER "www-data" INHERIT IN ROLE brep;
+
+CREATE USER "brep-build" INHERIT IN ROLE brep PASSWORD '-';
+EOF
+
+# Create the "staging" package database for the submit-pub package submission
+# handler.
+#
+run sudo sudo -u postgres psql <<EOF
+CREATE DATABASE brep_submit_package
+TEMPLATE template0
+ENCODING 'UTF8'
+LC_COLLATE 'en_US.UTF8'
+LC_CTYPE 'en_US.UTF8';
+
+GRANT ALL PRIVILEGES ON DATABASE brep_submit_package TO brep;
+EOF
+
+# Make sure the 'brep' and Apache2 user's logins work properly.
+#
+q="SELECT current_database();"
+run psql -d brep_package -c "$q" >/dev/null
+run psql -d brep_build -c "$q" >/dev/null
+run psql -d brep_submit_package -c "$q" >/dev/null
+
+run sudo sudo -u www-data psql -d brep_package -c "$q" >/dev/null
+run sudo sudo -u www-data psql -d brep_build -c "$q" >/dev/null
+
+# Setup the connection between the databases.
+#
+run sudo sudo -u postgres psql -d brep_build <<EOF
+CREATE EXTENSION postgres_fdw;
+
+CREATE SERVER package_server
+FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'brep_package', updatable 'false');
+
+GRANT USAGE ON FOREIGN SERVER package_server to brep;
+
+CREATE USER MAPPING FOR PUBLIC
+SERVER package_server
+OPTIONS (user 'brep-build', password '-');
+EOF
+
+# Allow brep-build user to access the brep_package database.
+#
+f="$(run sudo sudo -u postgres psql -t -A -c "show hba_file;")"
+s="# TYPE DATABASE USER ADDRESS METHOD\nlocal brep_package brep-build md5\n\n"
+
+run sudo sed --in-place=.bak "1s/^/$s/" "$f"
+run sudo systemctl restart postgresql
+
+# Enable creating database tables with columns of the case-insensitive
+# character string type.
+#
+q="CREATE EXTENSION citext;"
+run sudo sudo -u postgres psql -d brep_package <<<"$q"
+run sudo sudo -u postgres psql -d brep_build <<<"$q"
+run sudo sudo -u postgres psql -d brep_submit_package <<<"$q"
+
+# Copy the brep module configuration.
+#
+# Note: must be done before bin/brep-startup execution, which adjusts the
+# configuration.
+#
+run mkdir config
+run cp "$scr_dir/brep-module.conf" config/
+
+# Initialize the brep private instance, in particular creating the database
+# schemas and running the brep loader.
+#
+run mkdir bin/
+run cp "$scr_dir/brep-startup" bin/
+run bin/brep-startup
+
+# Smoke test the database schemas.
+#
+run psql -d brep_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null
+run psql -d brep_build -c 'SELECT package_name FROM build' >/dev/null
+run psql -d brep_build -c 'SELECT DISTINCT name FROM build_package' >/dev/null
+run psql -d brep_submit_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null
+
+# Setup executing the brep-startup script on boot.
+#
+run sudo cp "$scr_dir/brep-startup.service" /etc/systemd/system/
+
+run sudo systemctl start brep-startup.service # Make sure there are no issues.
+run sudo systemctl enable brep-startup.service
+
+# Prepare directories for the package submission service.
+#
+run mkdir submit-data
+run mkdir submit-temp
+run setfacl -m g:www-data:rwx submit-data
+run setfacl -m g:www-data:rwx submit-temp
+
+# Make the Apache2 user owned directories fully accessible by the 'brep' user
+# (which the submit-pub submission handler will run as).
+#
+run setfacl -dm g:brep:rwx submit-data
+run setfacl -dm g:brep:rwx submit-temp
+
+# Add the Apache2 user to sudoers, so the submission handler can re-execute
+# itself as the 'brep' user.
+#
+run sudo tee -a /etc/sudoers.d/www-data >/dev/null <<EOF
+www-data ALL=(ALL) NOPASSWD:ALL
+EOF
+
+run sudo chmod 0440 /etc/sudoers.d/www-data
+
+# Setup the Apache2 module.
+#
+run sudo mkdir -p /var/www/brep/log/
+
+run sudo cp "$scr_dir/brep-apache2.conf" /etc/apache2/sites-available/000-brep.conf
+run sudo cp "$scr_dir/brep-logrotate" /etc/logrotate.d/brep
+
+run sudo a2dissite --purge -q 000-default
+run sudo a2ensite -q 000-brep
+
+run sudo systemctl restart apache2
+run sudo systemctl status apache2 >/dev/null
+
+# Make sure the Apache2 service depends on PostgreSQL and
+# brep-startup.service, so that they are started in proper order.
+#
+run sudo mkdir -p /etc/systemd/system/apache2.service.d/
+run sudo tee /etc/systemd/system/apache2.service.d/postgresql.conf >/dev/null <<EOF
+[Unit]
+Requires=postgresql.service
+After=postgresql.service
+EOF
+
+run sudo tee /etc/systemd/system/apache2.service.d/brep-startup.conf >/dev/null <<EOF
+[Unit]
+Requires=brep-startup.service
+After=brep-startup.service
+EOF
+
+run sudo mkdir -p /etc/systemd/system/postgresql.service.d/
+run sudo tee /etc/systemd/system/postgresql.service.d/apache2.conf >/dev/null <<EOF
+[Unit]
+Wants=apache2.service
+EOF
+
+run sudo systemctl daemon-reload
+
+# Verify that Apache2 is stopped after PostgreSQL is stopped.
+#
+run sudo systemctl stop postgresql
+
+ec="0"
+run sudo systemctl status apache2 >/dev/null || ec="$?"
+
+if [ "$ec" -ne 3 ]; then
+ error "exit code 3 (unit is not active) is expected instead of $ec"
+fi
+
+# Verify that Apache2 is started after PostgreSQL is started.
+#
+run sudo systemctl start postgresql
+
+run sleep 3
+run sudo systemctl status apache2 >/dev/null
+
+# Setup periodic loader execution.
+#
+run sudo cp "$scr_dir/brep-load.service" /etc/systemd/system/
+run sudo cp "$scr_dir/brep-load.timer" /etc/systemd/system/
+
+run sudo systemctl start brep-load.service # Make sure there are no issues.
+
+run sudo systemctl start brep-load.timer
+run sudo systemctl status brep-load.timer >/dev/null
+run sudo systemctl enable brep-load.timer
+run sudo systemctl status brep-load.timer >/dev/null