From af3d4b7038382018d875a8eed01cda78046fc7eb Mon Sep 17 00:00:00 2001
From: Boris Kolpackov <boris@codesynthesis.com>
Date: Tue, 1 Feb 2022 13:31:56 +0200
Subject: Script updates for new single configuration toolchain dev setup

---
 bootstrap | 360 ++++++++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 302 insertions(+), 58 deletions(-)

(limited to 'bootstrap')

diff --git a/bootstrap b/bootstrap
index 7ef2ed4..40e2443 100755
--- a/bootstrap
+++ b/bootstrap
@@ -5,6 +5,12 @@
 #    and only generate PDFs for web publishing? Maybe decide when moving
 #    to ad hoc rules?
 
+# @@ Describe filesystem structure, created configurations.
+
+# --ssh --clean --extra --module rust --cxx g++-7 --cfg gcc7
+# --no-clone --no-symlink --no-libs --no-modules --cxx g++-7 --cfg gcc7-tsan --coptions "-Wall -Wextra -Werror -g -fsanitize=thread"
+# bdep init -d <proj> -A builds/gcc7 @gcc7
+
 # Boostrap the build2 development environment.
 #
 # Note that this script runs git-update-index commands specified in the
@@ -13,8 +19,11 @@
 # This script assumes the following steps have been performed:
 #
 # 1. The latest staged toolchain has been temporarily installed somewhere
-#    other than /usr/local (e.g., /tmp/build2) and is in PATH. Hint: use
-#    --local to speed things up seeing that this is a throw-away installation.
+#    other than /usr/local (e.g., /tmp/build2-install) and is in PATH.
+#    Hint: use --local --no-modules when building the staged toolchain to
+#    speed things up seeing that this is a throw-away installation:
+
+#    sh build2-install-...-stage.sh --local --no-modules /tmp/build2-install
 #
 # 2. The CLI and ODB compilers have been built and are either installed (if
 #    you don't plan to contribute to them) or symlinked in /usr/local/bin/.
@@ -59,7 +68,7 @@
 # mkdir -p ~/work/build2
 # cd ~/work/build2
 # git clone --recursive https://git.build2.org/etc.git
-# PATH="/tmp/build2/bin:$PATH" etc/bootstrap
+# PATH="/tmp/build2-install/bin:$PATH" etc/bootstrap
 #
 # After a successful bootstrap you can remove the staged toolchain.
 #
@@ -83,7 +92,7 @@
 #             b build2/build2/ bpkg/bpkg/ bdep/bdep/
 #
 # If the build system itself is not functional, it can always be rebuilt using
-# the bootstrapped build system (b-boot). For example:
+# the bootstrapped build system (b-boot; built by this script). For example:
 #
 # BDEP_SYNC=0 b-boot build2/build2/
 # BDEP_SYNC=0 b      build2/build2/
@@ -99,8 +108,20 @@
 # --ssh
 #    Use SSH URL for cloning (rw access).
 #
-# --no-symlink
-#    Do not create executable symlinks in /usr/local/bin/.
+# --extra
+#    Initialize additional packages (bbot, brep, etc) that are not part of
+#    the toolchain core (build2, bpkg, and bdep). Note that they are
+#    configured but not updated. Also note that for brep we assume libapr1,
+#    libapreq2, and libpq are available from the system (see brep/INSTALL
+#    for details).
+#
+# --module <name>
+#    Additional (to standard pre-installed) build system module to initialize.
+#    Repeat this option to specify multiple modules. Note: <name> should be
+#    without the libbuild2- prefix.
+#
+# --no-modules
+#    Do not initialize standard pre-installed build system modules.
 #
 # --no-clone
 #    Do not clone the repositories or build the bootstrap build system
@@ -108,11 +129,36 @@
 #    useful for initializing additional configurations (e.g., Clang in
 #    addition to GCC).
 #
+# --clean
+#    Clean (or clone if does not exist) the source directories (remove .bdep/,
+#    forwarded configurations) and the configuration directories. This is
+#    primarily useful to re-initialized an already initialized setup.
+#
+# --no-libs
+#    Do not create an extra configuration for libraries. Having such a
+#    configuration allows you to test changes in libraries without rendering
+#    the rest of the toolchain no longer runnable/functional.
+#
+# --no-symlink
+#    Do not create executable symlinks in /usr/local/bin/.
+#
 # --cxx
 # --cfg
 #    C++ compiler to use and the corresponding build configuration name, for
 #    example, g++-9/gcc9 or clang++-8/clang8; g++/gcc by default.
 #
+# --coptions
+# --loptions
+#    Compiler and linker options to use. By default a debug build with
+#    ASAN/UBSAN and parallel gold linking is configured:
+#
+#    -Wall -Wextra -Werror -g -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer
+#    -fuse-ld=gold -Wl,--threads,--thread-count,4
+#
+#    To configure TSAN, use the following coptions instead:
+#
+#    -Wall -Wextra -Werror -g -fsanitize=thread
+#
 owd="$(pwd)"
 trap "{ cd '$owd'; exit 1; }" ERR
 set -o errtrace # Trap in functions.
@@ -140,15 +186,37 @@ bpkg --version >/dev/null
 bdep --version >/dev/null
 
 url="https://git.build2.org"
+mod_url="https://github.com/build2"
+extra=
+mod=(hello autoconf kconfig)
 sym=true
 clone=true
+clean=
+libs=true
 cxx=g++
 cfg=gcc
+copt="-Wall -Wextra -Werror -g -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer"
+lopt="-fuse-ld=gold -Wl,--threads,--thread-count,4"
+popt=
 
 while [ $# -gt 0 ]; do
   case $1 in
     --ssh)
       url="git.build2.org:/var/scm"
+      mod_url="git@github.com:build2"
+      shift
+      ;;
+    --extra)
+      extra=true
+      shift
+      ;;
+    --module)
+      shift
+      mod+=("$1")
+      shift
+      ;;
+    --no-modules)
+      mod=()
       shift
       ;;
     --no-symlink)
@@ -159,6 +227,14 @@ while [ $# -gt 0 ]; do
       clone=
       shift
       ;;
+    --clean)
+      clean=true
+      shift
+      ;;
+    --no-libs)
+      libs=
+      shift
+      ;;
     --cxx)
       shift
       cxx="$1"
@@ -169,15 +245,70 @@ while [ $# -gt 0 ]; do
       cfg="$1"
       shift
       ;;
+    --coptions)
+      shift
+      copt="$1"
+      shift
+      ;;
+    --loptions)
+      shift
+      lopt="$1"
+      shift
+      ;;
     *)
       error "unexpected $1"
       ;;
   esac
 done
 
-cfg_bs="$cfg-asan-bs"
-cfg_pm="$cfg-asan-pm"
+if [ "$clean" ]; then
+
+  if [ "$sym" ]; then
+    run sudo rm -f /usr/local/bin/b-boot
+    run sudo rm -f /usr/local/bin/b
+    run sudo rm -f /usr/local/bin/bpkg
+    run sudo rm -f /usr/local/bin/bdep
+  fi
+
+  run rm -rf builds/
+
+  for m in "${mod[@]}"; do
+    run rm -rf "libbuild2-$m-build/"
+  done
 
+  run cd build2
+  run make -f bootstrap.gmake CXX="$cxx" CXXFLAGS=-O3 clean
+  run cd -
+
+  def=true
+
+else
+  # Determine if this is the first/default configuration.
+  #
+  if test -d build2/.bdep; then
+    def=
+  else
+    def=true
+  fi
+fi
+
+if [ ! "$def" ]; then
+
+  if [ "$libs" ]; then
+    error "non-default configuration requires --no-libs"
+  fi
+
+  if [ "${#mod[@]}" -ne 0 ]; then
+    error "non-default configuration requires --no-modules"
+  fi
+fi
+
+if [ "$extra" ]; then
+  popt="$popt -I$(apxs -q includedir)"
+fi
+
+# Clone or clean the repository.
+#
 function git_clone () # <url>
 {
   local u d l
@@ -185,7 +316,19 @@ function git_clone () # <url>
   u="$1"
   d="$(basename -s .git "$u")"
 
-  run git clone --recursive "$u"
+  # Clone it if doesn't exist.
+  #
+  if [ "$clean" -a -d "$d" ]; then
+
+    run rm -rf "$d/.bdep/"
+
+    for l in $(find "$d" -name out-root.build); do
+      run rm -f "$l"
+    done
+
+  else
+    run git clone --recursive "$u"
+  fi
 
   # Run git-update-index commands from README-GIT.
   #
@@ -208,14 +351,26 @@ function git_clone () # <url>
   fi
 }
 
-if [ "$clone" ]; then
+if [ "$clone" -o "$clean" ]; then
   git_clone "$url/libbutl.git"
   git_clone "$url/build2.git"
-
   git_clone "$url/libbpkg.git"
   git_clone "$url/bpkg.git"
   git_clone "$url/bdep.git"
 
+  if [ "$extra" ]; then
+    git_clone "$url/libbutl.bash.git"
+    git_clone "$url/bpkg-util.git"
+    git_clone "$url/bdep-util.git"
+    git_clone "$url/libbbot.git"
+    git_clone "$url/bbot.git"
+    git_clone "$url/brep.git"
+  fi
+
+  for m in "${mod[@]}"; do
+    git_clone "$mod_url/libbuild2-$m.git"
+  done
+
   # Build the bootstrap build system (b-boot).
   #
   run cd build2
@@ -225,12 +380,36 @@ if [ "$clone" ]; then
   run cd -
 fi
 
-# @@ TODO: make build system configuration ASAN/TSAN configurable.
+run mkdir -p builds
+
+# The -libs configuration.
+#
+# Note that by doing this first we automatically make sure this will be their
+# default/forwarded configuration.
 #
-# ASAN: -g3 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer
-# TSAN: -g3 -fsanitize=thread
+if [ "$libs" ]; then
 
-run mkdir -p builds
+  run bpkg create -d "builds/$cfg-libs" cc cli \
+    config.cli=cli \
+    config.cxx="$cxx" \
+    config.cc.coptions="$copt" \
+    config.cc.loptions="$lopt" \
+    config.bin.lib=shared \
+    config.install.root=/tmp/install config.dist.root=/tmp/dist
+
+  run bdep init -d libbutl -A "builds/$cfg-libs" "@$cfg-libs"
+  run bdep init -d libbpkg -A "builds/$cfg-libs" "@$cfg-libs"
+
+  if [ "$extra" ]; then
+    run bdep init -d libbutl.bash -A "builds/$cfg-libs" "@$cfg-libs"
+    run bdep init -d libbbot      -A "builds/$cfg-libs" "@$cfg-libs"
+  fi
+fi
+
+# Relative and absolute configuration directories.
+#
+rd="builds/$cfg"
+ad="$owd/$rd"
 
 # Build system configuration.
 #
@@ -240,55 +419,72 @@ run mkdir -p builds
 #
 #   2. Only building shared libraries for speed of development.
 #
-run bpkg create -d "builds/$cfg_bs" cc ?cli \
+run bpkg create -d "$rd" cc ?cli \
+  config.cli=cli \
   config.cxx="$cxx" \
-  config.cc.coptions="-Wall -Wextra -Werror -g3 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer" \
-  config.cc.loptions="-fuse-ld=gold -Wl,--threads,--thread-count,4" \
+  config.cc.poptions="$popt" \
+  config.cc.coptions="$copt" \
+  config.cc.loptions="$lopt" \
   config.bin.lib=shared \
   config.install.root=/tmp/install config.dist.root=/tmp/dist
 
-run bdep init -d libbutl -A "builds/$cfg_bs" "@$cfg_bs" --no-default
-run bdep init -d build2  -A "builds/$cfg_bs" "@$cfg_bs"
+run bdep init -d libbutl -A "$rd" "@$cfg"
+run bdep init -d build2  -A "$rd" "@$cfg"
+run bdep init -d libbpkg -A "$rd" "@$cfg"
+run bdep init -d bpkg    -A "$rd" "@$cfg"
+run bdep init -d bdep    -A "$rd" "@$cfg"
 
-run b build2/build2/
-run build2/build2/b --version
-
-# Package manager (and the rest of the toolchain) configuration.
-#
-run bpkg create -d "builds/$cfg_pm" cc cli \
-  config.cxx="$cxx" \
-  config.cc.coptions="-Wall -Wextra -Werror -g3 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer" \
-  config.cc.loptions="-fuse-ld=gold -Wl,--threads,--thread-count,4" \
-  config.bin.lib=shared \
-  config.install.root=/tmp/install config.dist.root=/tmp/dist
-
-run bdep init -d libbutl -A "builds/$cfg_pm" "@$cfg_pm" --default
-run bdep init -d libbpkg -A "builds/$cfg_pm" "@$cfg_pm"
-run bdep init -d bpkg    -A "builds/$cfg_pm" "@$cfg_pm"
-run bdep init -d bdep    -A "builds/$cfg_pm" "@$cfg_pm"
+if [ "$extra" ]; then
+  run bdep init -d libbutl.bash -A "$rd" "@$cfg"
+  run bdep init -d bpkg-util    -A "$rd" "@$cfg"
+  run bdep init -d bdep-util    -A "$rd" "@$cfg"
+  run bdep init -d libbbot      -A "$rd" "@$cfg"
+  run bdep init -d bbot         -A "$rd" "@$cfg"
+  run bdep init -d brep         -A "$rd" "@$cfg" \
+    ?sys:libapr1 ?sys:libapreq2 ?sys:libpq
+fi
 
 # Generate database support (currently and temporarily handled with a script).
 #
 # Note: this has to be done after bdep-init since we need the libodb headers.
 #       We also have to pre-update version headers.
 #
-run bpkg update -d "builds/$cfg_pm" libodb
-run b "builds/$cfg_pm/libbutl/libbutl/hxx{version}"
-run b "builds/$cfg_pm/libbpkg/libbpkg/hxx{version}"
-run b "builds/$cfg_pm/bpkg/bpkg/hxx{version common-options}"
-run b "builds/$cfg_pm/bdep/bdep/hxx{version common-options project-options}"
+if [ $def ]; then
+  run bpkg update -d "$rd" libodb
+  run b "$rd/libbutl/libbutl/hxx{version}"
+  run b "$rd/libbpkg/libbpkg/hxx{version}"
+  run b "$rd/bpkg/bpkg/hxx{version common-options}"
+  run b "$rd/bdep/bdep/hxx{version common-options project-options}"
+
+  run cd bpkg/bpkg
+  run ./odb.sh
+  run cd -
 
-run cd bpkg/bpkg
-run ./odb.sh
-run cd -
+  run cd bdep/bdep
+  run ./odb.sh
+  run cd -
 
-run cd bdep/bdep
-run ./odb.sh
-run cd -
+  if [ "$extra" ]; then
+    run b "$rd/brep/web/xhtml/hxx{version}"
+    run b "$rd/brep/libbrep/hxx{version}"
 
-run b bpkg/bpkg/ bdep/bdep/
-run bpkg/bpkg/bpkg --version
-run bdep/bdep/bdep --version
+    run cd brep/libbrep
+    run ./odb.sh
+    run cd -
+  fi
+fi
+
+if [ $def ]; then
+  run b build2/build2/ bpkg/bpkg/ bdep/bdep/
+  run build2/build2/b --version
+  run bpkg/bpkg/bpkg --version
+  run bdep/bdep/bdep --version
+else
+  run b "$rd/build2/build2/" "$rd/bpkg/bpkg/" "$rd/bdep/bdep/"
+  run "$rd/build2/build2/b" --version
+  run "$rd/bpkg/bpkg/bpkg" --version
+  run "$rd/bdep/bdep/bdep" --version
+fi
 
 # Add symlinks.
 #
@@ -298,16 +494,64 @@ if [ "$sym" ]; then
   # runnable in the face of compilation errors (which trigger the removal of
   # backlinks).
   #
-  run sudo ln -s "$owd/build2/build2/b-boot"            /usr/local/bin/b-boot
-  run sudo ln -s "$owd/$builds/$cfg_bs/build2/build2/b" /usr/local/bin/b
-  run sudo ln -s "$owd/$builds/$cfg_pm/bpkg/bpkg/bpkg"  /usr/local/bin/bpkg
-  run sudo ln -s "$owd/$builds/$cfg_pm/bdep/bdep/bdep"  /usr/local/bin/bdep
+  run sudo ln -s "$owd/build2/build2/b-boot" /usr/local/bin/b-boot
+  run sudo ln -s "$ad/build2/build2/b"       /usr/local/bin/b
+  run sudo ln -s "$ad/bpkg/bpkg/bpkg"        /usr/local/bin/bpkg
+  run sudo ln -s "$ad/bdep/bdep/bdep"        /usr/local/bin/bdep
 
   run export PATH="/usr/local/bin:$PATH"
-  run which b bpkg bdep
+else
+  run export PATH="$ad/build2/build2:$PATH"
+  run export PATH="$ad/bpkg/bpkg:$PATH"
+  run export PATH="$ad/bdep/bdep:$PATH"
+fi
 
-  # Re-run update using the bootstrapped toolchain (normally should be a
-  # noop).
-  #
+run which b bpkg bdep
+
+# Re-run update using the bootstrapped toolchain (normally should be a
+# noop).
+#
+if [ $def ]; then
   run b build2/build2/ bpkg/bpkg/ bdep/bdep/
+else
+  run b "$rd/build2/build2/" "$rd/bpkg/bpkg/" "$rd/bdep/bdep/"
 fi
+
+# Configure the modules.
+#
+for m in "${mod[@]}"; do
+  d="libbuild2-$m"
+
+  run cd "$d"
+
+  run bdep init --empty
+  run bdep config create @module "../$d-build/module/" --type build2 cc config.config.load=~build2
+  run bdep config create @target "../$d-build/target/" cc config.cxx="$cxx"
+
+  run bdep init @module -d "$d/"
+
+  # See if this module requires bootstrap.
+  #
+  b=$(sed -rn -e 's/^\s*requires\s*:\s*bootstrap\s*$/true/p' "$d/manifest")
+
+  if [ "$b" ]; then
+    info "module $d requires bootstrapping"
+    info "make sure ~/.build/b.option contains the following line and press Enter"
+    info "!config.import.libbuild2_$m=$owd/$d-build/module/"
+    read
+  fi
+
+  # Assume every package other than the module is a test.
+  #
+  # Note: sed regex is from the glue buildfile.
+  #
+  for t in $(sed -rn -e 's/^\s*location\s*:\s*(\S+)\s*$/\1/p' packages.manifest); do
+    if [ "$d/" = "$t" ]; then
+      continue
+    fi
+
+    run bdep init @target -d "$t"
+  done
+
+  run cd -
+done
-- 
cgit v1.1