diff options
| -rwxr-xr-x | buildos | 191 | ||||
| -rw-r--r-- | doc/manual.cli | 65 |
2 files changed, 237 insertions, 19 deletions
@@ -9,10 +9,6 @@ # 4. Monitor for OS and toolchain changes and reboot if detected. # -# @@ What do we do in case or error, trap? Send email? Reboot? Probably -# reboot bad idea since someone may need to login to examine what's -# going on. -# # @@ What will systemd do if we fail? Perhaps configure it to restart # us? Or not since we may hose the logs. # @@ -81,20 +77,192 @@ function restart () sudo systemctl reboot } +email "starting buildos monitor" <<EOF +buildid: $buildid +buildid_url: $buildid_url +toolchain_url: $toolchain_url +EOF + if [ -z "$buildid_url" ]; then info "no buildos.buildid_url specified, not monitoring for new os builds" fi -email "starting buildos monitor" <<EOF -buildid: $buildid -buildid_url: $buildid_url -EOF +if [ -z "$toolchain_url" ]; then + info "no buildos.toolchain_url specified, not bootstrapping" +else + # The toolchain "sums" file (a list of SHA sums and relative file names, as + # produced by shaNNNsum). The first entry should always be build2-toolchain + # tar archive itself (which we use to figure out the version). Blank lines + # and lines that start with '#' are ignored. + # + tc_url="$toolchain_url" + tc_file="$(sed -n -re 's%^.+/([^/]+)$%\1%p' <<<"$tc_url")" + tc_sum="$(sed -n -re 's%^.+\.([^.]+)$%\1%p' <<<"$tc_file")" + tc_root="/build/tftp" + tc_path="$tc_root/$tc_file" + tc_ver= +fi + +# Calculate the file checksum using the shaNNNsum utility. +# +function tc_checksum () # <file> +{ + "${tc_sum}sum" -b "$1" | sed -n -re 's/^([^ ]+) .+$/\1/p' +} + +# Fetch a file from the sums file into $tc_root, verify its checksum, and make +# a predictable name (without version) symlink. +# +function tc_fetch () # <line> +{ + local s p f u l + + s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' <<<"$1")" # Checksum. + p="$(sed -n -re 's/^[^ ]+ \*([^ ]+)$/\1/p' <<<"$1")" # File path (rel). + f="$(sed -n -re 's%^(.+/)?([^/]+)$%\2%p' <<<"$p")" # File name. + u="$(sed -n -re 's%^(.+)/[^/]+$%\1%p' <<<"$tc_url")/$p" # File URL. + + if [ -z "$s" -o -z "$p" -o -z "$f" -o -z "$u" ]; then + info "invalid sum line '$1'" + return 1 + fi + + # Extract the version. + # + if [ -z "$tc_ver" ]; then + tc_ver="$(sed -n -re 's/build2-toolchain-(.+)\.tar.*/\1/p' <<<"$f")" + + if [ -z "$tc_ver" ]; then + info "unable to extract toolchain version from '$f'" + return 1 + fi + + info "toolchain version is $tc_ver" + echo "$tc_ver" >"$tc_root/toolchain-version" + fi + + # Derive a predictable name link. + # + l="$(sed -n -re "s/^(.+)-$tc_ver(.*)$/\1\2/p" <<<"$f")" + + if [ -z "$l" ]; then + info "unable to derive predicatable name from '$f', '$tc_ver'" + return 1 + fi + + # Fetch the file. + # + info "fetching $u [$l]" + + if ! curl -f -L -s -S -o "$tc_root/$f" "$u"; then + info "unable to fetch $u" + return 1 + fi + + # Verify the checksum. + # + info "verifying checksum for $f" + + local n + n="$(tc_checksum "$tc_root/$f")" + + if [ "$n" != "$s" ]; then + info "$tc_sum checksum mismatch for $u" + info " expected: $s" + info " calculated: $n" + return 1 + fi + # Make the link. + # + ln -s "$f" "$tc_root/$l" +} + +# Bootstrap the toolchain. +# +function tc_bootstrap () +{ + local l ls=() + + # Fetch files according to the sums file. Skip empty line and those that + # start with '#'. + # + readarray -t ls < <(sed -e '/^\s*#/d;/^\s*$/d' "$tc_path") + + for l in "${ls[@]}"; do + if ! tc_fetch "$l"; then + return 1 # Diagnostics has already been issued. + fi + done +} + +# Monitoring loop. +# while true; do - info "monitoring..." - sleep 5 + # Check for toolchain changes. If this is the first run, bootstrap it. + # + if [ -n "$tc_url" ]; then + + # Fetch the toolchain sums either to $tc_path if this is the first time + # or to $tc_path.new if we are checking for changes. + # + if [ -e "$tc_path" ]; then + f="$tc_path.new" + else + f="$tc_path" + fi + + if curl -f -L -s -S -o "$f" "$tc_url"; then + # Take care of change detection. + # + if [ "$f" != "$tc_path" ]; then + + n="$(tc_checksum "$f")" + + if [ "$tc_file_sum" != "$n" ]; then + email "rebooting because of new toolchain" <<EOF +old_toolchain: $tc_file_sum +new_toolchain: $n +EOF + info "new toolchain ($n), rebooting..." + restart + fi + else + # This is the first run, bootstrap the toolchain. + # + tc_file_sum="$(tc_checksum "$f")" + + # If we fail, we simply wait for a new toolchain to be uploaded (or + # some manual intervention). + # + # Note that because of the pipe tc_bootstrap() will run in subshell + # and any variables it sets (like tc_ver) won't be visible to us. + # + info "bootstrapping toolchain..." + + tc_bootstrap 2>&1 | tee "$tc_root/toolchain.log" 1>&2 + + if [ "${PIPESTATUS[0]}" -eq 0 ]; then + tc_ver="$(cat $tc_root/toolchain-version)" + s="bootstrapped toolchain $tc_ver" + else + s="failed to bootstrap toolchain, waiting for new version" + fi + + email "$s" <<EOF +toolchain_log: tftp://$hname/toolchain.log +EOF + fi + else + info "unable to fetch $tc_url, will try again" + rm -f "$f" + fi + fi + + # Check for OS changes. + # if [ -n "$buildid_url" ]; then # Fetch the current id. While normally it will be a TFTP URL, it could also # be HTTP(S) so we configure sensible behavior for that. @@ -112,4 +280,7 @@ EOF info "unable to fetch $buildid_url, will try again" fi fi + + info "monitoring..." + sleep 10 done diff --git a/doc/manual.cli b/doc/manual.cli index 04227f9..e61caa2 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -79,11 +79,16 @@ booted locally from the kernel image and initrd directly. \h#boot-reboot|Reboot| -Build OS can detect when the OS build has been updated and automatically -reboot the build host. This is achieved by polling the URL specified -with the \c{buildos.buildid_url} kernel command line parameter. It should -point to the \c{buildos-buildid} file that comes along the kernel image -and initrd. See \l{#boot-net Network Boot} for the usage example. +Build OS can detect when the OS or toolchain have been updated and +automatically reboot the build host. This is achieved by polling the URLs +specified with the \c{buildos.buildid_url} and \c{buildos.toolchain_url} +kernel command line parameters. + +The \c{buildos.buildid_url} value should point to the \c{buildos-buildid} file +that comes along the kernel image and initrd. The \c{buildos.toolchain_url} +value is the location of the toolchain checksums file as described in +\l{#config-toolchain Toolchain}. See \l{#boot-net Network Boot} for the usage +example. \h#boot-net|Network Boot| @@ -121,13 +126,14 @@ label buildos menu label buildos kernel /buildos/buildos-image initrd /buildos/buildos-initrd - append buildos.smtp_relay=example.org buildos.admin_email=admin@example.org buildos.buildid_url=tftp://<tftp-host>/buildos/buildos-buildid + append buildos.smtp_relay=example.org buildos.admin_email=admin@example.org buildos.buildid_url=tftp://<os-host>/buildos/buildos-buildid buildos.toolchain_url=https://<toolchain-host>/toolchain.sha256 EOF \ -Where \c{<tftp-host>} in \c{buildid_url} is the address of the TFTP server -(the same address as returned by the DHCP server to PXE clients). Note that -all the parameters in \c{append} must be specified on a single line.| +Where \c{<os-host>} is the address of the TFTP server (the same address as +returned by the DHCP server to PXE clients) and \c{<toolchain-host>} is the +host that serves the toolchain archives. Note that all the parameters in +\c{append} must be specified on a single line.| \li|You can test the setup using QEMU/KVM, for example: @@ -283,4 +289,45 @@ quotes): \ buildos.ssh_key=\"ssh-rsa AAA...OA0DB user@host\" \ + +\h#config-toolchain|Toolchain| + +The first step performed by the Build OS monitor is to bootstrap the +\c{build2} toolchain. The location of the toolchain packages is specified with +the \c{buildos.toolchain_url} kernel command line parameter. This URL should +point to the \i{toolchain checksums file}. + +Each line in the checksums file is the output of the \c{shaNNNsum(1)} utility, +that is, the SHANNN sum following by space, an asterisk (\c{*}) which signals +the binary mode), and the relative file path. Blank lines and lines that start +with \c{#} are ignored. The extension of the checksums file should be +\c{.shaNNN} and the first entry should be for the \c{build2-toolchain} \c{tar} +archive itself (used to derive the toolchain version). For example: + +\ +# toolchain.sha256 +ae89[...]87a4 *0.4.0/build2-toolchain-0.4.0.tar.xz +058d[...]c962 *0.4.0/build2-baseutils-0.4.0-x86_64-windows.zip +e723[...]c305 *0.4.0/build2-mingw-0.4.0-x86_64-windows.tar.xz +\ + +Based on the checksums file the monitor downloads each file into +\c{/build/tftp/} (the file path is taken as relative to \c{toolchain_url}), +verifies their checksums, and creates \i{predictable name} symlinks (names +without the version). Continuing with the above example, the contents of +\c{/build/tftp/} would be: + +\ +build2-toolchain-0.4.0.tar.xz +build2-baseutils-0.4.0-x86_64-windows.zip +build2-mingw-0.4.0-x86_64-windows.tar.xz + +build2-toolchain-tar.xz -> build2-toolchain-0.4.0.tar.xz +build2-baseutils-x86_64-windows.zip -> build2-baseutils-0.4.0-x86_64-windows.zip +build2-mingw-x86_64-windows.tar.xz -> build2-mingw-0.4.0-x86_64-windows.tar.xz +\ + +While the monitor itself only needs the \c{build2-toolchain} package, build +machine toolchain bootstrap may require additional packages (which will be +accessed via TFTP using predictable names). " |