aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--bbot/agent2
-rw-r--r--bbot/agent.cli7
-rw-r--r--bbot/agent.cxx114
-rw-r--r--bbot/utility2
-rw-r--r--bbot/utility.txx5
-rw-r--r--bbot/worker.cli5
-rw-r--r--bbot/worker.cxx23
-rw-r--r--doc/manual.cli40
-rw-r--r--tests/agent/testscript29
-rw-r--r--tests/integration/buildfile8
-rw-r--r--tests/integration/testscript98
-rw-r--r--tests/integration/tftp-map11
-rw-r--r--tests/worker/build.test12
13 files changed, 261 insertions, 95 deletions
diff --git a/bbot/agent b/bbot/agent
index 14d2995..1271a87 100644
--- a/bbot/agent
+++ b/bbot/agent
@@ -22,8 +22,6 @@ namespace bbot
extern size_t tc_num; // Toolchain number.
extern string tc_id; // Toolchain id.
- extern strings controllers; // Controller URLs.
-
extern string hname; // Our host name.
extern uid_t uid; // Our effective user id.
extern string uname; // Our effective user name.
diff --git a/bbot/agent.cli b/bbot/agent.cli
index c67535e..b71db4e 100644
--- a/bbot/agent.cli
+++ b/bbot/agent.cli
@@ -149,6 +149,13 @@ namespace bbot
"Fake the package building process by creating the aborted build result."
}
+ path --fake-machine
+ {
+ "<file>",
+ "Fake the machine enumeration process by reading the machine header
+ manifest from <file> (or \cb{stdin} if <file> is '\cb{-}')."
+ }
+
path --fake-request
{
"<file>",
diff --git a/bbot/agent.cxx b/bbot/agent.cxx
index 8847e62..60d59e6 100644
--- a/bbot/agent.cxx
+++ b/bbot/agent.cxx
@@ -45,8 +45,6 @@ namespace bbot
size_t tc_num;
string tc_id;
- strings controllers;
-
string hname;
uid_t uid;
string uname;
@@ -257,6 +255,24 @@ try
bootstrapped_machine_manifests rm;
dir_paths rd;
+ if (ops.fake_machine_specified ())
+ {
+ auto mh (
+ parse_manifest<machine_header_manifest> (
+ ops.fake_machine (), "machine header"));
+
+ rm.push_back (
+ bootstrapped_machine_manifest {
+ machine_manifest {mh.id, mh.name, mh.summary, machine_type::kvm},
+ toolchain_manifest {tc_id},
+ bootstrap_manifest {}
+ });
+
+ rd.push_back (dir_path (ops.machines ()) /= mh.name); // For diagnostics.
+
+ return make_pair (move (rm), move (rd));
+ }
+
// The first level are machine volumes.
//
for (const dir_entry& ve: dir_iterator (machines))
@@ -517,6 +533,7 @@ static result_manifest
perform_task (const dir_path& md,
const bootstrapped_machine_manifest& mm,
const task_manifest& tm)
+try
{
tracer trace ("perform_task");
@@ -542,7 +559,36 @@ perform_task (const dir_path& md,
//
// 5. Clean up (force the machine down and delete the snapshot).
//
- try
+
+ // TFTP server mapping (server chroot is --tftp):
+ //
+ // GET requests to .../build/<name>/get/*
+ // PUT requests to .../build/<name>/put/*
+ //
+ auto_rmdir arm ((dir_path (ops.tftp ()) /= "build") /= tc_name);
+
+ dir_path gd (dir_path (arm.path ()) /= "get");
+ dir_path pd (dir_path (arm.path ()) /= "put");
+
+ try_mkdir_p (gd);
+ try_mkdir_p (pd);
+
+ path tf (gd / "manifest"); // Task manifest file.
+ path rf (pd / "manifest"); // Result manifest file.
+
+ serialize_manifest (tm, tf, "task");
+
+ if (ops.fake_machine_specified ())
+ {
+ // Simply wait for the file to appear.
+ //
+ for (size_t i (0); !file_exists (rf); sleep (1))
+ if (i++ % 10 == 0)
+ l3 ([&]{trace << "waiting for result manifest";});
+
+ r = parse_manifest<result_manifest> (rf, "result");
+ }
+ else
{
// <name>-<toolchain>-<xxx>
//
@@ -553,24 +599,8 @@ perform_task (const dir_path& md,
string br ("br1"); // Using private bridge for now.
- // Start the TFTP server (server chroot is --tftp). Map:
- //
- // GET requests to .../build/<name>/get/*
- // PUT requests to .../build/<name>/put/*
+ // Start the TFTP server.
//
- auto_rmdir arm ((dir_path (ops.tftp ()) /= "build") /= tc_name);
-
- dir_path gd (dir_path (arm.path ()) /= "get");
- dir_path pd (dir_path (arm.path ()) /= "put");
-
- try_mkdir_p (gd);
- try_mkdir_p (pd);
-
- path tf (gd / "manifest"); // Task manifest file.
- path rf (pd / "manifest"); // Result manifest file.
-
- serialize_manifest (tm, tf, "task");
-
tftp_server tftpd ("Gr ^/?(.+)$ /build/" + tc_name + "/get/\\1\n" +
"Pr ^/?(.+)$ /build/" + tc_name + "/put/\\1\n");
@@ -640,16 +670,6 @@ perform_task (const dir_path& md,
//
r = parse_manifest<result_manifest> (rf, "result");
- // Update package name/version if the returned value as "unknown".
- //
- if (r.version == bpkg::version ("0"))
- {
- assert (r.status == result_status::abnormal);
-
- r.name = tm.name;
- r.version = tm.version;
- }
-
// If the build terminated abnormally, suspent the machine for
// investigation (note that here we don't wait or return).
//
@@ -664,13 +684,23 @@ perform_task (const dir_path& md,
run_btrfs (trace, "subvolume", "delete", xp);
}
- catch (const system_error& e)
+
+ // Update package name/version if the returned value as "unknown".
+ //
+ if (r.version == bpkg::version ("0"))
{
- fail << "build error: " << e;
+ assert (r.status == result_status::abnormal);
+
+ r.name = tm.name;
+ r.version = tm.version;
}
return r;
}
+catch (const system_error& e)
+{
+ fail << "build error: " << e << endf;
+}
extern "C" void
handle_signal (int sig)
@@ -748,10 +778,15 @@ try
tc_num = ops.toolchain_num ();
tc_id = ops.toolchain_id ();
- if (argc < 2)
+
+ // Controller URLs.
+ //
+ if (argc < 2 && !ops.fake_request_specified ())
fail << "controller url expected" <<
info << "run " << argv[0] << " --help for details";
+ strings controllers;
+
for (int i (1); i != argc; ++i)
controllers.push_back (argv[i]);
@@ -830,18 +865,15 @@ try
if (ops.fake_request_specified ())
{
- const path& f (ops.fake_request ());
- task_manifest t (f.string () != "-"
- ? parse_manifest<task_manifest> (f, "task")
- : parse_manifest<task_manifest> (cin, "stdin", "task"));
-
- url = controllers[0];
+ auto t (parse_manifest<task_manifest> (ops.fake_request (), "task"));
tr = task_response_manifest {
"fake-session", // Dummy session.
string (), // Empty challange.
url, // Empty result URL.
move (t)};
+
+ url = "http://example.org";
}
else
{
@@ -893,9 +925,9 @@ try
// verify it is one of those we sent).
//
size_t i (0);
- for (const bootstrapped_machine_manifest& m: ms)
+ for (const machine_header_manifest& m: tq.machines)
{
- if (m.machine.name == tr.task->machine)
+ if (m.name == tr.task->machine)
break;
++i;
diff --git a/bbot/utility b/bbot/utility
index 59f6e09..f154a9e 100644
--- a/bbot/utility
+++ b/bbot/utility
@@ -141,6 +141,8 @@ namespace bbot
// Manifest parsing and serialization.
//
+ // For parsing, if path is '-', then read from stdin.
+ //
template <typename T>
T
parse_manifest (const path&, const char* what, bool ignore_unknown = true);
diff --git a/bbot/utility.txx b/bbot/utility.txx
index edd674b..4ee4c6f 100644
--- a/bbot/utility.txx
+++ b/bbot/utility.txx
@@ -2,6 +2,8 @@
// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
// license : TBC; see accompanying LICENSE file
+#include <iostream> // cin
+
#include <butl/fdstream>
#include <butl/manifest-parser>
@@ -148,6 +150,9 @@ namespace bbot
try
{
+ if (f.string () == "-")
+ return parse_manifest<T> (std::cin, "stdin", what, ignore_unknown);
+
if (!file_exists (f))
fail << what << " manifest file " << f << " does not exist";
diff --git a/bbot/worker.cli b/bbot/worker.cli
index fd88521..61d3f4c 100644
--- a/bbot/worker.cli
+++ b/bbot/worker.cli
@@ -39,6 +39,11 @@ namespace bbot
{
"\h|OPTIONS|"
+ //
+ // NOTE: when adding new options, consider whether they should be
+ // propagated from startup to build.
+ //
+
bool --help {"Print usage information and exit."}
bool --version {"Print version and exit."}
diff --git a/bbot/worker.cxx b/bbot/worker.cxx
index 0212d26..a2e53b2 100644
--- a/bbot/worker.cxx
+++ b/bbot/worker.cxx
@@ -182,7 +182,9 @@ build (size_t argc, const char* argv[])
for (const variable& v: tm.config)
cfg.push_back (v.unquoted ());
- dir_path dir ("build");
+ // Use target (if present) or machine as configuration directory name.
+ //
+ dir_path dir (tm.target ? tm.target->string () : tm.machine);
r.status |= run_bpkg (trace, r.log,
"create",
@@ -380,10 +382,27 @@ startup ()
// Run it.
//
+ strings os;
+
+ if (ops.systemd_daemon ())
+ os.push_back ("--systemd-daemon");
+
+ if (ops.verbose_specified ())
+ {
+ os.push_back ("--verbose");
+ os.push_back (to_string (ops.verbose ()));
+ }
+
+ if (ops.tftp_host_specified ())
+ {
+ os.push_back ("--tftp-host");
+ os.push_back (ops.tftp_host ());
+ }
+
// Note that we use the effective (absolute) path instead of recall since
// we may have changed the CWD.
//
- run (trace, pp, tg, argv0.effect_string ());
+ run (trace, pp, tg, argv0.effect_string (), os);
}
catch (const failed&)
{
diff --git a/doc/manual.cli b/doc/manual.cli
index 2b2fc21..b96ddd9 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -494,18 +494,19 @@ the environment executable is an error.
Once the environment setup executable is determined, the worker re-executes
itself as that executable passing to it as command line arguments the target
-name (or empty value if not specified) and the path to the \c{bbot} worker to
-be executed once the environment is setup. The environment setup executable is
-executed in the build directory as its current working directory. The build
+name (or empty value if not specified), the path to the \c{bbot} worker to be
+executed once the environment is setup, and any additional options that need
+to be propagated to the re-executed worker. The environment setup executable
+is executed in the build directory as its current working directory. The build
directory contains the build task \c{manifest} file.
The environment setup executable sets up the necessary execution environment
for example by adjusting \c{PATH} or running a suitable \c{vcvars} batch file.
It then re-executes itself as the \c{bbot} worker passing to it as command
-line arguments the list of build system modules (\c{<env-modules>}) and the
-list of configuration variables (\c{<env-config-vars>}). The environment setup
-executable must execute the \c{bbot} worker in the build directory as the
-current working directory.
+line arguments (in addition to worker options) the list of build system
+modules (\c{<env-modules>}) and the list of configuration variables
+(\c{<env-config-vars>}). The environment setup executable must execute the
+\c{bbot} worker in the build directory as the current working directory.
The re-executed \c{bbot} worker then proceeds to test the package from the
repository by executing the following commands (\c{<>}-values are from the
@@ -520,23 +521,30 @@ bpkg -v update <package-name>
bpkg -v test <package-name>
\
-As an example, the following bash script can be used to setup the environment
-for building C and C++ packages with GCC 6 on most Linux distributions.
+As an example, the following POSIX shell script can be used to setup the
+environment for building C and C++ packages with GCC 6 on most Linux
+distributions.
\
-#! /usr/bin/env bash
+#!/bin/sh
-# $1 - target
-# $2 - bbot executable
+# Environment setup script for C/C++ compilation with GCC 6.
+#
+# $1 - target
+# $2 - bbot executable
+# $3+ - bbot options
-trap \"exit 1\" ERR
+set -e # Exit on errors.
-if [ -n \"$1\" ]; then
- echo \"unknown target $1\" 1>&2
+t=\"$1\"
+shift
+
+if test -n \"$t\"; then
+ echo \"unknown target: $t\" 1>&2
exit 1
fi
-exec \"$2\" cc config.c=gcc-6
+exec \"$@\" cc config.c=gcc-6 config.cxx=g++-6
\
\h#arch-controller|Controller Logic|
diff --git a/tests/agent/testscript b/tests/agent/testscript
index e864bf2..6e9d319 100644
--- a/tests/agent/testscript
+++ b/tests/agent/testscript
@@ -137,7 +137,7 @@ rm = $src_base/btrfs-rmdir /build/machines
machine: windows-msvc
EOI
%trace: enumerate_machines:\\.*%*
- error: task from $u for unknown machine windows-msvc
+ error: task from http://example.org for unknown machine windows-msvc
EOE
: result
@@ -159,30 +159,3 @@ rm = $src_base/btrfs-rmdir /build/machines
-$rm
}
-
-#\
-: bootstrap
-:
-{
- test.options += --dump-machines
-
- m = /build/machines/default/linux-gcc
- u = https://example.org/?dummy
-
- +$cp
-
- ln -T -s linux-gcc-1.0 $m/linux-gcc-1
-
- : bootstrap
- :
- $* $u >>EOO 2>>EOE #2>>~"%EOE%d"
- : 1
- id: linux-gcc-1.0
- name: linux-gcc
- summary: Linux with GCC
- EOO
- EOE
-
- #-$rm
-}
-#\
diff --git a/tests/integration/buildfile b/tests/integration/buildfile
new file mode 100644
index 0000000..f054694
--- /dev/null
+++ b/tests/integration/buildfile
@@ -0,0 +1,8 @@
+# file : tests/integration/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : TBC; see accompanying LICENSE file
+
+./: ../../bbot/exe{bbot-agent bbot-worker} test{testscript} file{tftp-map}
+dir{./}: test = ../../bbot/exe{bbot-agent}
+
+include ../../bbot/
diff --git a/tests/integration/testscript b/tests/integration/testscript
new file mode 100644
index 0000000..f43bf02
--- /dev/null
+++ b/tests/integration/testscript
@@ -0,0 +1,98 @@
+# file : tests/integration/testscript
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : TBC; see accompanying LICENSE file
+
+#\
+# Requirement:
+#
+# 1. tftpd-hpa installed (assumed in /usr/sbin/in.tftpd)
+#
+# 2. b, bpkg, curl executables in PATH.
+#
+# TFTP server (tftpd-hpa) setup: from the test out_base, run (sudo is required
+# for --secure/chroot):
+#
+
+sudo /usr/sbin/in.tftpd \
+ --foreground \
+ --address 127.0.0.1:55123 \
+ --user "$(whoami)" \
+ --permissive \
+ --create \
+ --secure \
+ --map-file tftp-map \
+ "$(pwd)"
+
+#\
+
+machine = linux-gcc
+c = gcc
+cxx = g++
+
+# Where we get the task and what we do with the result can be configured
+# independently:
+#
+# - We can poll a controller for a task by specifying its URL or we can
+# read a task manifest from a file (--fake-request).
+#
+# - We can send the result back to the controller or we can dump the result
+# to stdout (--dump-result).
+#
+
+#\
+wait=5s
+controller = https://stage.build2.org/?build-task
+#\
+
+wait=1s
+controller = --fake-request ../task --dump-result
+
+pkg = hello
+ver = 1.0.0
+rep = https://build2.org/pkg/1/stage/stable
+rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94
++cat <<"EOI" >=task
+ : 1
+ name: $pkg
+ version: $ver
+ repository: $rep
+ trust: $rfp
+ machine: $machine
+ EOI
+
+#
+#
+tftp = 127.0.0.1:55123
+
+a = $0
++ sed -e 's/-agent$/-worker/' <"$0" | set w
+
+: agent
+:
+{
+ cat <<"EOI" >=machine-header;
+ : 1
+ id: $machine-1.0
+ name: $machine
+ summary: The $machine fake machine
+ EOI
+ $a --verbose 3 --tftp $~ --fake-machine machine-header $controller \
+ &build/*** >| 2>|
+}
+
+: worker
+:
+{
+ cat <<"EOI" >=default;
+ #!/bin/sh
+
+ t="\$1"
+ shift
+
+ exec "\$@" cc config.c=$c config.cxx=$cxx
+ EOI
+ chmod ugo+x default;
+ sleep $wait;
+ $w --verbose 3 --startup --tftp-host $tftp --environment \
+ &$machine/*** &manifest $~ 2>|
+}
diff --git a/tests/integration/tftp-map b/tests/integration/tftp-map
new file mode 100644
index 0000000..837794d
--- /dev/null
+++ b/tests/integration/tftp-map
@@ -0,0 +1,11 @@
+# file : tests/integration/tftp-map
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : TBC; see accompanying LICENSE file
+
+# Test working directories relative to out_base:
+#
+# agent: test/agent/
+# worker: test/worker/
+#
+Gr ^/?(.+)$ /test/agent/build/default/get/\1
+Pr ^/?(.+)$ /test/agent/build/default/put/\1
diff --git a/tests/worker/build.test b/tests/worker/build.test
index adedccc..3be9150 100644
--- a/tests/worker/build.test
+++ b/tests/worker/build.test
@@ -27,8 +27,8 @@ cat <<"EOI" >=manifest;
target: x86_64-linux-gnu
config: config.cc.coptions=-O3
EOI
-$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
- trace: build: bpkg -v create -d build --wipe config.cc.coptions=-O3 cc
+$* --verbose 3 --tftp-host "$tftp/$@" cc &x86_64-linux-gnu/*** 2>>"EOE";
+ trace: build: bpkg -v create -d x86_64-linux-gnu --wipe config.cc.coptions=-O3 cc
trace: build: bpkg -v add $rep
trace: build: bpkg -v fetch --trust $rfp --trust-no
trace: build: bpkg -v build --configure-only --yes $pkg/$ver
@@ -78,8 +78,8 @@ cat <<"EOI" >=manifest;
machine: linux-gcc
target: x86_64-linux-gnu
EOI
-$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
- trace: build: bpkg -v create -d build --wipe cc
+$* --verbose 3 --tftp-host "$tftp/$@" cc &x86_64-linux-gnu/*** 2>>"EOE";
+ trace: build: bpkg -v create -d x86_64-linux-gnu --wipe cc
trace: build: bpkg -v add $rep
trace: build: bpkg -v fetch --trust $rfp --trust-no
trace: build: bpkg -v build --configure-only --yes bogus/1.2.3
@@ -109,8 +109,8 @@ cat <<"EOI" >=manifest;
target: x86_64-linux-gnu
config: config.cc.loptions=-lbogus
EOI
-$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE";
- trace: build: bpkg -v create -d build --wipe config.cc.loptions=-lbogus cc
+$* --verbose 3 --tftp-host "$tftp/$@" cc &x86_64-linux-gnu/*** 2>>"EOE";
+ trace: build: bpkg -v create -d x86_64-linux-gnu --wipe config.cc.loptions=-lbogus cc
trace: build: bpkg -v add $rep
trace: build: bpkg -v fetch --trust $rfp --trust-no
trace: build: bpkg -v build --configure-only --yes $pkg/$ver
generated by cgit v1.1 at 2024-11-10 00:19:52 +0000