Implement toolchain fetching and monitoring

2 files changed, 237 insertions, 19 deletions

diff --git a/buildos b/buildos
index 9a0224a..c91c554 100755
--- a/buildos
+++ b/buildos
@@ -9,10 +9,6 @@
 # 4. Monitor for OS and toolchain changes and reboot if detected.
 #
 
-# @@ What do we do in case or error, trap? Send email? Reboot? Probably
-#    reboot bad idea since someone may need to login to examine what's
-#    going on.
-#
 # @@ What will systemd do if we fail? Perhaps configure it to restart
 #    us? Or not since we may hose the logs.
 #
@@ -81,20 +77,192 @@ function restart ()
   sudo systemctl reboot
 }
 
+email "starting buildos monitor" <<EOF
+buildid:       $buildid
+buildid_url:   $buildid_url
+toolchain_url: $toolchain_url
+EOF
+
 if [ -z "$buildid_url" ]; then
   info "no buildos.buildid_url specified, not monitoring for new os builds"
 fi
 
-email "starting buildos monitor" <<EOF
-buildid:     $buildid
-buildid_url: $buildid_url
-EOF
+if [ -z "$toolchain_url" ]; then
+  info "no buildos.toolchain_url specified, not bootstrapping"
+else
+  # The toolchain "sums" file (a list of SHA sums and relative file names, as
+  # produced by shaNNNsum). The first entry should always be build2-toolchain
+  # tar archive itself (which we use to figure out the version). Blank lines
+  # and lines that start with '#' are ignored.
+  #
+  tc_url="$toolchain_url"
+  tc_file="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tc_url")"
+  tc_sum="$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tc_file")"
+  tc_root="/build/tftp"
+  tc_path="$tc_root/$tc_file"
+  tc_ver=
+fi
+
+# Calculate the file checksum using the shaNNNsum utility.
+#
+function tc_checksum () # <file>
+{
+  "${tc_sum}sum" -b "$1" | sed -n -re 's/^([^ ]+) .+$/\1/p'
+}
+
+# Fetch a file from the sums file into $tc_root, verify its checksum, and make
+# a predictable name (without version) symlink.
+#
+function tc_fetch () # <line>
+{
+  local s p f u l
+
+  s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$1")"         # Checksum.
+  p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$1")"    # File path (rel).
+  f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")"      # File name.
+  u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tc_url")/$p" # File URL.
+
+  if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then
+    info "invalid sum line '$1'"
+    return 1
+  fi
+
+  # Extract the version.
+  #
+  if [ -z "$tc_ver" ]; then
+    tc_ver="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")"
+
+    if [ -z "$tc_ver" ]; then
+      info "unable to extract toolchain version from '$f'"
+      return 1
+    fi
+
+    info "toolchain version is $tc_ver"
+    echo "$tc_ver" >"$tc_root/toolchain-version"
+  fi
+
+  # Derive a predictable name link.
+  #
+  l="$(sed -n -re "s/^(.+)-$tc_ver(.*)$/\1\2/p" <<<"$f")"
+
+  if [ -z "$l" ]; then
+    info "unable to derive predicatable name from '$f', '$tc_ver'"
+    return 1
+  fi
+
+  # Fetch the file.
+  #
+  info "fetching $u [$l]"
+
+  if ! curl -f -L -s -S -o "$tc_root/$f" "$u"; then
+    info "unable to fetch $u"
+    return 1
+  fi
+
+  # Verify the checksum.
+  #
+  info "verifying checksum for $f"
+
+  local n
+  n="$(tc_checksum "$tc_root/$f")"
+
+  if [ "$n" != "$s" ]; then
+    info "$tc_sum checksum mismatch for $u"
+    info "  expected:   $s"
+    info "  calculated: $n"
+    return 1
+  fi
 
+  # Make the link.
+  #
+  ln -s "$f" "$tc_root/$l"
+}
+
+# Bootstrap the toolchain.
+#
+function tc_bootstrap ()
+{
+  local l ls=()
+
+  # Fetch files according to the sums file. Skip empty line and those that
+  # start with '#'.
+  #
+  readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tc_path")
+
+  for l in "${ls[@]}"; do
+    if ! tc_fetch "$l"; then
+      return 1 # Diagnostics has already been issued.
+    fi
+  done
+}
+
+# Monitoring loop.
+#
 while true; do
-  info "monitoring..."
 
-  sleep 5
+  # Check for toolchain changes. If this is the first run, bootstrap it.
+  #
+  if [ -n "$tc_url" ]; then
+
+    # Fetch the toolchain sums either to $tc_path if this is the first time
+    # or to $tc_path.new if we are checking for changes.
+    #
+    if [ -e "$tc_path" ]; then
+      f="$tc_path.new"
+    else
+      f="$tc_path"
+    fi
+
+    if curl -f -L -s -S -o "$f" "$tc_url"; then
 
+      # Take care of change detection.
+      #
+      if [ "$f" != "$tc_path" ]; then
+
+	n="$(tc_checksum "$f")"
+
+	if [ "$tc_file_sum" != "$n" ]; then
+	  email "rebooting because of new toolchain" <<EOF
+old_toolchain: $tc_file_sum
+new_toolchain: $n
+EOF
+	  info "new toolchain ($n), rebooting..."
+	  restart
+	fi
+      else
+	# This is the first run, bootstrap the toolchain.
+	#
+	tc_file_sum="$(tc_checksum "$f")"
+
+	# If we fail, we simply wait for a new toolchain to be uploaded (or
+	# some manual intervention).
+	#
+	# Note that because of the pipe tc_bootstrap() will run in subshell
+	# and any variables it sets (like tc_ver) won't be visible to us.
+	#
+	info "bootstrapping toolchain..."
+
+	tc_bootstrap 2>&1 | tee "$tc_root/toolchain.log" 1>&2
+
+	if [ "${PIPESTATUS[0]}" -eq 0 ]; then
+	  tc_ver="$(cat $tc_root/toolchain-version)"
+	  s="bootstrapped toolchain $tc_ver"
+	else
+	  s="failed to bootstrap toolchain, waiting for new version"
+	fi
+
+	email "$s" <<EOF
+toolchain_log: tftp://$hname/toolchain.log
+EOF
+      fi
+    else
+      info "unable to fetch $tc_url, will try again"
+      rm -f "$f"
+    fi
+  fi
+
+  # Check for OS changes.
+  #
   if [ -n "$buildid_url" ]; then
     # Fetch the current id. While normally it will be a TFTP URL, it could also
     # be HTTP(S) so we configure sensible behavior for that.
@@ -112,4 +280,7 @@ EOF
       info "unable to fetch $buildid_url, will try again"
     fi
   fi
+
+  info "monitoring..."
+  sleep 10
 done
diff --git a/doc/manual.cli b/doc/manual.cli
index 04227f9..e61caa2 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -79,11 +79,16 @@ booted locally from the kernel image and initrd directly.
 
 \h#boot-reboot|Reboot|
 
-Build OS can detect when the OS build has been updated and automatically
-reboot the build host. This is achieved by polling the URL specified
-with the \c{buildos.buildid_url} kernel command line parameter. It should
-point to the \c{buildos-buildid} file that comes along the kernel image
-and initrd. See \l{#boot-net Network Boot} for the usage example.
+Build OS can detect when the OS or toolchain have been updated and
+automatically reboot the build host. This is achieved by polling the URLs
+specified with the \c{buildos.buildid_url} and \c{buildos.toolchain_url}
+kernel command line parameters.
+
+The \c{buildos.buildid_url} value should point to the \c{buildos-buildid} file
+that comes along the kernel image and initrd. The \c{buildos.toolchain_url}
+value is the location of the toolchain checksums file as described in
+\l{#config-toolchain Toolchain}. See \l{#boot-net Network Boot} for the usage
+example.
 
 \h#boot-net|Network Boot|
 
@@ -121,13 +126,14 @@ label buildos
   menu label buildos
   kernel /buildos/buildos-image
   initrd /buildos/buildos-initrd
-  append buildos.smtp_relay=example.org buildos.admin_email=admin@example.org buildos.buildid_url=tftp://<tftp-host>/buildos/buildos-buildid
+  append buildos.smtp_relay=example.org buildos.admin_email=admin@example.org buildos.buildid_url=tftp://<os-host>/buildos/buildos-buildid buildos.toolchain_url=https://<toolchain-host>/toolchain.sha256
 EOF
 \
 
-Where \c{<tftp-host>} in \c{buildid_url} is the address of the TFTP server
-(the same address as returned by the DHCP server to PXE clients). Note that
-all the parameters in \c{append} must be specified on a single line.|
+Where \c{<os-host>} is the address of the TFTP server (the same address as
+returned by the DHCP server to PXE clients) and \c{<toolchain-host>} is the
+host that serves the toolchain archives. Note that all the parameters in
+\c{append} must be specified on a single line.|
 
 \li|You can test the setup using QEMU/KVM, for example:
 
@@ -283,4 +289,45 @@ quotes):
 \
 buildos.ssh_key=\"ssh-rsa AAA...OA0DB user@host\"
 \
+
+\h#config-toolchain|Toolchain|
+
+The first step performed by the Build OS monitor is to bootstrap the
+\c{build2} toolchain. The location of the toolchain packages is specified with
+the \c{buildos.toolchain_url} kernel command line parameter. This URL should
+point to the \i{toolchain checksums file}.
+
+Each line in the checksums file is the output of the \c{shaNNNsum(1)} utility,
+that is, the SHANNN sum following by space, an asterisk (\c{*}) which signals
+the binary mode), and the relative file path. Blank lines and lines that start
+with \c{#} are ignored. The extension of the checksums file should be
+\c{.shaNNN} and the first entry should be for the \c{build2-toolchain} \c{tar}
+archive itself (used to derive the toolchain version). For example:
+
+\
+# toolchain.sha256
+ae89[...]87a4 *0.4.0/build2-toolchain-0.4.0.tar.xz
+058d[...]c962 *0.4.0/build2-baseutils-0.4.0-x86_64-windows.zip
+e723[...]c305 *0.4.0/build2-mingw-0.4.0-x86_64-windows.tar.xz
+\
+
+Based on the checksums file the monitor downloads each file into
+\c{/build/tftp/} (the file path is taken as relative to \c{toolchain_url}),
+verifies their checksums, and creates \i{predictable name} symlinks (names
+without the version). Continuing with the above example, the contents of
+\c{/build/tftp/} would be:
+
+\
+build2-toolchain-0.4.0.tar.xz
+build2-baseutils-0.4.0-x86_64-windows.zip
+build2-mingw-0.4.0-x86_64-windows.tar.xz
+
+build2-toolchain-tar.xz             -> build2-toolchain-0.4.0.tar.xz
+build2-baseutils-x86_64-windows.zip -> build2-baseutils-0.4.0-x86_64-windows.zip
+build2-mingw-x86_64-windows.tar.xz  -> build2-mingw-0.4.0-x86_64-windows.tar.xz
+\
+
+While the monitor itself only needs the \c{build2-toolchain} package, build
+machine toolchain bootstrap may require additional packages (which will be
+accessed via TFTP using predictable names).
 "