diff options
| -rw-r--r-- | bbot/agent | 5 | ||||
| -rw-r--r-- | bbot/agent.cli | 67 | ||||
| -rw-r--r-- | bbot/agent.cxx | 473 | ||||
| -rw-r--r-- | bbot/bbot-agent@.service | 14 | ||||
| -rw-r--r-- | bbot/bootstrap-manifest | 50 | ||||
| -rw-r--r-- | bbot/bootstrap-manifest.cxx | 155 | ||||
| -rw-r--r-- | bbot/buildfile | 4 | ||||
| -rw-r--r-- | bbot/machine-manifest | 54 | ||||
| -rw-r--r-- | bbot/machine-manifest.cxx | 154 | ||||
| -rw-r--r-- | bbot/machine.cxx | 2 | ||||
| -rw-r--r-- | bbot/tftp.cxx | 2 | ||||
| -rw-r--r-- | bbot/types | 10 | ||||
| -rw-r--r-- | bbot/utility | 53 | ||||
| -rw-r--r-- | bbot/utility.cxx | 2 | ||||
| -rw-r--r-- | bbot/utility.txx | 91 | ||||
| -rw-r--r-- | bbot/worker.cli | 54 | ||||
| -rw-r--r-- | bbot/worker.cxx | 455 | ||||
| -rw-r--r-- | doc/manual.cli | 35 | ||||
| -rw-r--r-- | tests/agent/testscript | 119 | ||||
| -rw-r--r-- | tests/worker/bootstrap.test | 13 | ||||
| -rw-r--r-- | tests/worker/build.test | 133 | ||||
| -rw-r--r-- | tests/worker/buildfile | 26 | ||||
| -rw-r--r-- | tests/worker/startup.test | 101 | ||||
| -rw-r--r-- | unit-tests/bootstrap-manifest/driver.cxx | 1 |
24 files changed, 1706 insertions, 367 deletions
@@ -19,9 +19,12 @@ namespace bbot extern const string bs_prot; // Bootstrap protocol version. extern string tc_name; // Toolchain name. - extern string tc_num; // Toolchain number. + extern size_t tc_num; // Toolchain number. extern string tc_id; // Toolchain id. + extern strings controllers; // Controller URLs. + + extern string hname; // Our host name. extern uid_t uid; // Our effective user id. extern string uname; // Our effective user name. diff --git a/bbot/agent.cli b/bbot/agent.cli index 562860f..3e02807 100644 --- a/bbot/agent.cli +++ b/bbot/agent.cli @@ -11,22 +11,19 @@ include <bbot/common.cli>; namespace bbot { { - "<options> <name> <num> <id> ", + "<options> <url>", " \h|SYNOPSIS| \cb{bbot-agent --help}\n \cb{bbot-agent --version}\n - \c{\b{bbot-agent} [<options>] <name> <num> <id>} + \c{\b{bbot-agent} [<options>] <url>...} \h|DESCRIPTION| \cb{bbot-agent} @@ TODO. - The <name> argument is the toolchain name, <id> \- the toolchain id, - and <num> \- the toolchain number in this deployment. - Note that on termination \cb{bbot-agent} may leave a working machine snapshot behind. It is expected that the caller (normally Build OS monitor) cleans them up before restarting the agent. @@ -45,6 +42,26 @@ namespace bbot "Start as a simple systemd daemon." } + string --toolchain-name = "default" + { + "<str>", + "Toolchain name, \cb{default} by default." + } + + size_t --toolchain-num = 1 + { + "<num>", + "Toolchain number, 1 by default." + } + + string --toolchain-id + { + "<str>", + "Toolchain id. If unspecified or empty, then no re-bootstrapping on + toolchain changes will be performed (which is primarily useful for + testing)." + } + size_t --cpu = 1 { "<num>", @@ -60,8 +77,13 @@ namespace bbot dir_path --machines = "/build/machines/" { "<dir>", - "The location of the build machines with \cb{/build/machines/} being - the default." + "The location of the build machines, \cb{/build/machines/} by default." + } + + dir_path --tftp = "/build/tftp/" + { + "<dir>", + "The location of the TFTP server root, \cb{/build/tftp/} by default." } size_t --bootstrap-timeout = 600 @@ -78,6 +100,13 @@ namespace bbot minutes) by default." } + size_t --request-timeout = 300 + { + "<sec>", + "Maximum number of seconds to wait for controller request completion, + 300 (5 minutes) by default." + } + uint16_t --verbose = 1 { "<level>", @@ -89,15 +118,37 @@ namespace bbot // bool --dump-machines { - "Dump available machines to \c{stdout}, (re)-bootstrapping any if + "Dump the available machines to \cb{stdout}, (re)-bootstrapping any if necessary, and exit." } + bool --dump-task + { + "Dump the received build task to \cb{stdout} and exit." + } + + bool --dump-result + { + "Dump the obtained build result to \cb{stdout} and exit." + } + bool --fake-bootstrap { "Fake the machine bootstrap process by creating the expected bootstrapped machine manifest." } + + bool --fake-build + { + "Fake the package building process by creating the aborted build result." + } + + path --fake-request + { + "<file>", + "Fake the task request process by reading the task manifest from <file> + (or \cb{stdin} if <file> is '\cb{-}')." + } }; " diff --git a/bbot/agent.cxx b/bbot/agent.cxx index 634a94d..eed49d6 100644 --- a/bbot/agent.cxx +++ b/bbot/agent.cxx @@ -28,6 +28,7 @@ #include <bbot/tftp> #include <bbot/machine> +#include <bbot/machine-manifest> #include <bbot/bootstrap-manifest> using namespace std; @@ -41,9 +42,12 @@ namespace bbot const string bs_prot ("1"); string tc_name; - string tc_num; + size_t tc_num; string tc_id; + strings controllers; + + string hname; uid_t uid; string uname; @@ -84,9 +88,9 @@ namespace bbot // template <typename... A> inline void -btrfs (tracer& t, A&&... a) +run_btrfs (tracer& t, A&&... a) { - if (verb >= 3) + if (verb >= 4) run_io (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...); else run_io (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...); @@ -96,7 +100,7 @@ template <typename... A> inline butl::process_exit::code_type btrfs_exit (tracer& t, A&&... a) { - return verb >= 3 + return verb >= 4 ? run_io_exit (t, fdnull (), 2, 2, "btrfs", forward<A> (a)...) : run_io_exit (t, fdnull (), fdnull (), 2, "btrfs", forward<A> (a)...); } @@ -114,7 +118,7 @@ bootstrap_machine (const dir_path& md, bootstrapped_machine_manifest r { mm, - toolchain_manifest {tc_id}, + toolchain_manifest {tc_id.empty () ? "bogus" : tc_id}, bootstrap_manifest { bootstrap_manifest::versions_type { {"bbot", BBOT_VERSION}, @@ -134,12 +138,12 @@ bootstrap_machine (const dir_path& md, { string br ("br1"); // Using private bridge for now. - // Start the TFTP server (server chroot is /build/tftp). Map: + // Start the TFTP server (server chroot is --tftp). Map: // - // GET requests to /build/tftp/toolchain/<name>/* - // PUT requests to /build/tftp/bootstrap/<name>/* + // GET requests to .../toolchain/<name>/* + // PUT requests to .../bootstrap/<name>/* // - auto_rmdir arm (dir_path ("/build/tftp/bootstrap/" + tc_name)); + auto_rmdir arm ((dir_path (ops.tftp ()) /= "bootstrap") /= tc_name); try_mkdir_p (arm.path ()); // Bootstrap result manifest. @@ -150,7 +154,7 @@ bootstrap_machine (const dir_path& md, tftp_server tftpd ("Gr ^/?(.+)$ /toolchain/" + tc_name + "/\\1\n" + "Pr ^/?(.+)$ /bootstrap/" + tc_name + "/\\1\n"); - l2 ([&]{trace << "tftp server on port " << tftpd.port ();}); + l3 ([&]{trace << "tftp server on port " << tftpd.port ();}); // Start the machine. // @@ -205,7 +209,7 @@ bootstrap_machine (const dir_path& md, if (!tftpd.serve ((to = startup_to))) return soft_fail ("bootstrap startup timeout"); - l2 ([&]{trace << "completed startup in " << startup_to - to << "s";}); + l3 ([&]{trace << "completed startup in " << startup_to - to << "s";}); // Next the bootstrap process may download additional toolchain // archives, build things, and then upload the result manifest. So on @@ -217,50 +221,45 @@ bootstrap_machine (const dir_path& md, if (to == 0) return soft_fail ("bootstrap timeout"); - l2 ([&]{trace << "completed bootstrap in " << bootstrap_to - to << "s";}); + l3 ([&]{trace << "completed bootstrap in " << bootstrap_to - to << "s";}); // Shut the machine down cleanly. // if (!m->shutdown ((to = shutdown_to))) return soft_fail ("bootstrap shutdown timeout"); - l2 ([&]{trace << "completed shutdown in " << shutdown_to - to << "s";}); + l3 ([&]{trace << "completed shutdown in " << shutdown_to - to << "s";}); } // Parse the result manifest. // - try - { - r.bootstrap = parse_manifest<bootstrap_manifest> (mf, "bootstrap"); - } - catch (const failed&) - { - error << "invalid bootstrap manifest for machine " << md; - return nullopt; - } + r.bootstrap = parse_manifest<bootstrap_manifest> (mf, "bootstrap"); r.machine.mac = m->mac; // Save the MAC address. } catch (const system_error& e) { - fail << "tftp server error: " << e; + fail << "bootstrap error: " << e; } serialize_manifest (r, md / "manifest", "bootstrapped machine"); return r; } -static machine_header_manifests -enumerate_machines (const dir_path& rd) +// Return available machines and their directories as a parallel array. +// +static pair<bootstrapped_machine_manifests, dir_paths> +enumerate_machines (const dir_path& machines) try { tracer trace ("enumerate_machines"); - machine_header_manifests r; + bootstrapped_machine_manifests rm; + dir_paths rd; // The first level are machine volumes. // - for (const dir_entry& ve: dir_iterator (rd)) + for (const dir_entry& ve: dir_iterator (machines)) { const string vn (ve.path ().string ()); @@ -269,7 +268,7 @@ try if (ve.type () != entry_type::directory || vn[0] == '.') continue; - const dir_path vd (dir_path (rd) /= vn); + const dir_path vd (dir_path (machines) /= vn); // Inside we have machines. // @@ -302,14 +301,14 @@ try // any) and see if we need to re-bootstrap. If so, use the snapshot // as a starting point. Rename to bootstrapped at the end (atomic). // - const dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P> - const dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<too...> + dir_path lp (dir_path (md) /= (mn + '-' + bs_prot)); // -<P> + dir_path tp (dir_path (md) /= (mn + '-' + tc_name)); // -<toolchain> bool te (dir_exists (tp)); auto delete_t = [&tp, &trace] () { - btrfs (trace, "property", "set", "-ts", tp, "ro", "false"); - btrfs (trace, "subvolume", "delete", tp); + run_btrfs (trace, "property", "set", "-ts", tp, "ro", "false"); + run_btrfs (trace, "subvolume", "delete", tp); }; for (size_t retry (0);; ++retry) @@ -355,14 +354,14 @@ try if (te) delete_t (); - l2 ([&]{trace << "skipping " << md << ": no subvolume link";}); + l3 ([&]{trace << "skipping " << md << ": no subvolume link";}); break; } // <name>-<toolchain>-<xxx> // - const dir_path xp (dir_path (md) /= - path::traits::temp_name (mn + '-' + tc_name)); + const dir_path xp ( + dir_path (md) /= path::traits::temp_name (mn + '-' + tc_name)); if (btrfs_exit (trace, "subvolume", "snapshot", sp, xp) != 0) { @@ -422,13 +421,13 @@ try if (bmm->machine.id != mm.id) { - l2 ([&]{trace << "re-bootstrapping " << tp << ": new machine";}); + l3 ([&]{trace << "re-bootstrapping " << tp << ": new machine";}); te = false; } - if (bmm->toolchain.id != tc_id) + if (!tc_id.empty () && bmm->toolchain.id != tc_id) { - l2 ([&]{trace << "re-bootstrapping " << tp << ": new toolchain";}); + l3 ([&]{trace << "re-bootstrapping " << tp << ": new toolchain";}); te = false; } @@ -436,13 +435,13 @@ try { if (i < 0) { - l2 ([&]{trace << "re-bootstrapping " << tp << ": new bbot";}); + l3 ([&]{trace << "re-bootstrapping " << tp << ": new bbot";}); te = false; } else { - l2 ([&]{trace << "ignoring " << tp << ": old bbot";}); - btrfs (trace, "subvolume", "delete", xp); + l3 ([&]{trace << "ignoring " << tp << ": old bbot";}); + run_btrfs (trace, "subvolume", "delete", xp); break; } } @@ -451,7 +450,7 @@ try delete_t (); } else - l2 ([&]{trace << "bootstrapping " << tp;}); + l3 ([&]{trace << "bootstrapping " << tp;}); if (!te) { @@ -463,8 +462,8 @@ try if (!bmm) { - l2 ([&]{trace << "ignoring " << tp << ": failed to bootstrap";}); - btrfs (trace, "subvolume", "delete", xp); + l3 ([&]{trace << "ignoring " << tp << ": failed to bootstrap";}); + run_btrfs (trace, "subvolume", "delete", xp); break; } @@ -477,25 +476,25 @@ try fail << "unable to rename " << xp << " to " << tp; } - // Check the boostrapped bbot version as above and ignore this + l2 ([&]{trace << "bootstrapped " << bmm->machine.name;}); + + // Check the bootstrapped bbot version as above and ignore this // machine if it's newer than us. // if (int i = compare_bbot (bmm->bootstrap)) { assert (i > 0); - l2 ([&]{trace << "ignoring " << tp << ": old bbot";}); + l3 ([&]{trace << "ignoring " << tp << ": old bbot";}); break; } } else - btrfs (trace, "subvolume", "delete", xp); + run_btrfs (trace, "subvolume", "delete", xp); - // Add the machine to the list. + // Add the machine to the lists. // - r.push_back ( - machine_header_manifest (move (mm.id), - move (mm.name), - move (mm.summary))); + rm.push_back (move (*bmm)); + rd.push_back (move (tp)); break; } @@ -507,11 +506,161 @@ try } } - return r; + return make_pair (move (rm), move (rd)); } catch (const system_error& e) { - fail << "unable to iterate over " << rd << ": " << e << endf; + fail << "unable to iterate over " << machines << ": " << e << endf; +} + +static result_manifest +perform_task (const dir_path& md, + const bootstrapped_machine_manifest& mm, + const task_manifest& tm) +{ + tracer trace ("perform_task"); + + result_manifest r { + tm.name, + tm.version, + result_status::abort, + operation_results {}}; + + if (ops.fake_build ()) + return r; + + // The overall plan is as follows: + // + // 1. Snapshot the (bootstrapped) machine. + // + // 2. Save the task manifest to the TFTP directory (to be accessed by the + // worker). + // + // 3. Start the TFTP server and the machine. + // + // 4. Serve TFTP requests while watching out for the result manifest. + // + // 5. Clean up (force the machine down and delete the snapshot). + // + try + { + // <name>-<toolchain>-<xxx> + // + const dir_path xp ( + md.directory () /= path::traits::temp_name (md.leaf ().string ())); + + run_btrfs (trace, "subvolume", "snapshot", md, xp); + + string br ("br1"); // Using private bridge for now. + + // Start the TFTP server (server chroot is --tftp). Map: + // + // GET requests to .../build/<name>/get/* + // PUT requests to .../build/<name>/put/* + // + auto_rmdir arm ((dir_path (ops.tftp ()) /= "build") /= tc_name); + + dir_path gd (dir_path (arm.path ()) /= "get"); + dir_path pd (dir_path (arm.path ()) /= "put"); + + try_mkdir_p (gd); + try_mkdir_p (pd); + + path tf (gd / "manifest"); // Task manifest file. + path rf (pd / "manifest"); // Result manifest file. + + serialize_manifest (tm, tf, "task"); + + tftp_server tftpd ("Gr ^/?(.+)$ /build/" + tc_name + "/get/\\1\n" + + "Pr ^/?(.+)$ /build/" + tc_name + "/put/\\1\n"); + + l3 ([&]{trace << "tftp server on port " << tftpd.port ();}); + + // Start the machine. + // + unique_ptr<machine> m ( + start_machine (xp, + mm.machine, + mm.machine.mac, + br, + tftpd.port ())); + + // Note: the machine handling logic is similar to bootstrap. + // + { + auto mg ( + make_exception_guard ( + [&m, &xp] () + { + info << "trying to force machine " << xp << " down"; + try {m->forcedown ();} catch (const failed&) {} + })); + + auto soft_fail = [&xp, &m, &r] (const char* msg) + { + { + diag_record dr (error); + dr << msg << " for machine " << xp << ", suspending"; + m->print_info (dr); + } + m->suspend (); + m->wait (); + return r; + }; + + // The first request should be the task manifest download. Wait for up + // to 60 seconds for that to arrive. In a sense we use it as an + // indication that the machine has booted and the worker process has + // started. + // + size_t to; + const size_t startup_to (60); + const size_t build_to (ops.build_timeout ()); + + if (!tftpd.serve ((to = startup_to))) + return soft_fail ("build startup timeout"); + + l3 ([&]{trace << "completed startup in " << startup_to - to << "s";}); + + // Next the worker builds things and then uploads the result manifest. + // So on our side we serve TFTP requests while checking for the manifest + // file. + // + for (to = build_to; to != 0 && !file_exists (rf); tftpd.serve (to)) ; + + if (to == 0) + return soft_fail ("build timeout"); + + l3 ([&]{trace << "completed build in " << build_to - to << "s";}); + + // Force the machine down (there is no need wasting time on clean + // shutdown since the next step is to drop the snapshot). + // + m->forcedown (); + } + + run_btrfs (trace, "subvolume", "delete", xp); + + // Parse the result manifest. + // + r = parse_manifest<result_manifest> (rf, "result"); + + // Update package name/version if the returned value as "unknown". + // + if (r.version == bpkg::version ("0")) + { + assert (r.status == result_status::abnormal); + + r.name = tm.name; + r.version = tm.version; + } + } + catch (const system_error& e) + { + fail << "build error: " << e; + } + + return r; } extern "C" void @@ -538,9 +687,11 @@ try verb = ops.verbose (); - uid = getuid (); - uname = getpwuid (uid)->pw_name; - + // Note that unlike other projects, here we distinguish between fail + // (critical error, agent terminates) and error (non-fatal error, agent + // continues to run). This means we should be careful not using fail + // to report normal errors and vice-versa. + // if (ops.systemd_daemon ()) { // Map to systemd severity prefixes (see sd-daemon(3) for details). Note @@ -554,7 +705,7 @@ try info.indent_ = text.indent_ = systemd_indent; - fail.type_ = "<3>"; + fail.type_ = "<2>"; error.type_ = "<3>"; warn.type_ = "<4>"; info.type_ = "<6>"; @@ -568,6 +719,19 @@ try tracer trace ("main"); + uid = getuid (); + uname = getpwuid (uid)->pw_name; + + { + char buf[HOST_NAME_MAX + 1]; + + if (gethostname (buf, sizeof (buf)) == -1) + fail << "unable to obtain hostname: " + << system_error (errno, generic_category ()); // Sanitize. + + hname = buf; + } + // On POSIX ignore SIGPIPE which is signaled to a pipe-writing process if // the pipe reading end is closed. Note that by default this signal // terminates a process. Also note that there is no way to disable this @@ -602,13 +766,16 @@ try return p.wait () ? 0 : 1; } - if (argc != 4) - fail << "toolchain name/id/num excected" << + tc_name = ops.toolchain_name (); + tc_num = ops.toolchain_num (); + tc_id = ops.toolchain_id (); + + if (argc < 2) + fail << "controller url expected" << info << "run " << argv[0] << " --help for details"; - tc_name = argv[1]; - tc_num = argv[2]; - tc_id = argv[3]; + for (int i (1); i != argc; ++i) + controllers.push_back (argv[i]); // Handle SIGHUP and SIGTERM. // @@ -619,26 +786,192 @@ try // The work loop. The steps we go through are: // - // 1. Enumerate the available machines, (re-)bootstrapping any of necessary. + // 1. Enumerate the available machines, (re-)bootstrapping any if necessary. // // 2. Poll controller(s) for build tasks. // - // 3. If no build tasks are available, go to #1 after sleeping a bit. + // 3. If no build tasks are available, go to #1 (after sleeping a bit). // // 4. If a build task is returned, do it, upload the result, and go to #1 - // immediately. + // (immediately). // - for (unsigned int s; (s = 60); sleep (s)) + for (bool sleep (false);; ::sleep (sleep ? 60 : 0), sleep = false) { - machine_header_manifests mms (enumerate_machines (ops.machines ())); + // Enumerate the machines. + // + auto mp (enumerate_machines (ops.machines ())); + bootstrapped_machine_manifests& ms (mp.first); + dir_paths& ds (mp.second); + + // Prepare task request. + // + // @@ TODO: key fingerprint. + // + task_request_manifest tq {hname, "", machine_header_manifests {}}; + + for (const bootstrapped_machine_manifest& m: ms) + tq.machines.emplace_back (m.machine.id, + m.machine.name, + m.machine.summary); if (ops.dump_machines ()) { - for (const machine_header_manifest& mm: mms) - serialize_manifest (mm, cout, "stdout", "machine manifest"); + for (const machine_header_manifest& m: tq.machines) + serialize_manifest (m, cout, "stdout", "machine"); return 0; } + + if (tq.machines.empty ()) + { + warn << "no build machines for toolchain " << tc_name; + sleep = true; + continue; + } + + // Send task requests. + // + // + string url; + task_response_manifest tr; + + if (ops.fake_request_specified ()) + { + const path& f (ops.fake_request ()); + task_manifest t (f.string () != "-" + ? parse_manifest<task_manifest> (f, "task") + : parse_manifest<task_manifest> (cin, "stdin", "task")); + + url = controllers[0]; + + tr = task_response_manifest { + "fake-session", // Dummy session. + string (), // Empty challange. + url, // Empty result URL. + move (t)}; + } + else + { + for (const string& u: controllers) + { + try + { + http_curl c (trace, + path ("-"), + path ("-"), + curl::post, + u, + "--header", "Content-Type: text/manifest", + "--max-time", ops.request_timeout ()); + + serialize_manifest (tq, c.out, u, "task request"); + c.out.close (); + + tr = parse_manifest<task_response_manifest> ( + c.in, u, "task response"); + c.in.close (); + + if (!c.wait ()) + throw_generic_error (EIO); + } + catch (const system_error& e) + { + error << "unable to request task from " << u << ": " << e; + continue; + } + + if (!tr.session.empty ()) // Got a task. + { + url = u; + break; + } + } + } + + if (tr.session.empty ()) // No task from any of the controllers. + { + sleep = true; + continue; + } + + // We have a build task. + // + // First find the index of the machine we were asked to use (and also + // verify it is one of those we sent). + // + size_t i (0); + for (const bootstrapped_machine_manifest& m: ms) + { + if (m.machine.name == tr.task->machine) + break; + + ++i; + } + + if (i == ms.size ()) + { + error << "task from " << url << " for unknown machine " + << tr.task->machine; + + if (ops.dump_task ()) + return 0; + + continue; + } + + const task_manifest& t (*tr.task); + + if (ops.dump_task ()) + { + serialize_manifest (t, cout, "stdout", "task"); + return 0; + } + + const dir_path& d (ds[i]); // The -<toolchain> directory. + const bootstrapped_machine_manifest& m (ms[i]); + + result_manifest r (perform_task (d, m, t)); + + if (ops.dump_result ()) + { + serialize_manifest (r, cout, "stdout", "result"); + return 0; + } + + // Upload the result. + // + // @@ TODO challange + // + result_request_manifest rq {tr.session, "", move (r)}; + { + const string& u (*tr.result_url); + + try + { + http_curl c (trace, + path ("-"), + nullfd, // Not expecting any data in response. + curl::post, + u, + "--header", "Content-Type: text/manifest", + "--max-time", ops.request_timeout ()); + + serialize_manifest (rq, c.out, u, "task request"); + c.out.close (); + + if (!c.wait ()) + throw_generic_error (EIO); + } + catch (const system_error& e) + { + error << "unable to upload result to " << u << ": " << e; + continue; + } + } + + l2 ([&]{trace << "built " << t.name << '/' << t.version << " " + << "on " << t.machine << " " + << "for " << url;}); } } catch (const failed&) diff --git a/bbot/bbot-agent@.service b/bbot/bbot-agent@.service index af760b3..5f160cf 100644 --- a/bbot/bbot-agent@.service +++ b/bbot/bbot-agent@.service @@ -12,9 +12,13 @@ Environment=RAM=1048576 Environment=BOOTSTRAP_TIMEOUT=600 Environment=BUILD_TIMEOUT=1800 +Environment=REQUEST_TIMEOUT=300 -Environment=TOOLCHAIN_ID=123abc +Environment=TOOLCHAIN_NAME=%i Environment=TOOLCHAIN_NUM=1 +Environment=TOOLCHAIN_ID= + +Environment="CONTROLLER_URL=https://example.org/?build-task" ExecStart=/build/bbot/%i/bin/bbot-agent --systemd-daemon \ --verbose ${VERBOSE} \ @@ -22,9 +26,11 @@ ExecStart=/build/bbot/%i/bin/bbot-agent --systemd-daemon \ --ram ${RAM} \ --bootstrap-timeout ${BOOTSTRAP_TIMEOUT} \ --build-timeout ${BUILD_TIMEOUT} \ - %i \ - ${TOOLCHAIN_NUM} \ - ${TOOLCHAIN_ID} + --request-timeout ${REQUEST_TIMEOUT} \ + --toolchain-name ${TOOLCHAIN_NAME} \ + --toolchain-num ${TOOLCHAIN_NUM} \ + --toolchain-id ${TOOLCHAIN_ID} \ + $CONTROLLER_URL User=build Group=build diff --git a/bbot/bootstrap-manifest b/bbot/bootstrap-manifest index 35feaba..94a2a24 100644 --- a/bbot/bootstrap-manifest +++ b/bbot/bootstrap-manifest @@ -12,34 +12,8 @@ #include <bbot/types> #include <bbot/utility> -#include <bbot/machine-manifest> - namespace bbot { - // Toolchain manifest. - // - class toolchain_manifest - { - public: - - // Toolchain id (SHAXXX). - // - string id; - - explicit - toolchain_manifest (string i): id (i) {} - - public: - toolchain_manifest () = default; // VC export. - toolchain_manifest (butl::manifest_parser&, bool ignore_unknown = false); - toolchain_manifest (butl::manifest_parser&, - butl::manifest_name_value start, - bool ignore_unknown = false); - - void - serialize (butl::manifest_serializer&) const; - }; - // Bootstrap result manifest. Uploaded by the worker to the agent's TFTP // server. // @@ -71,30 +45,6 @@ namespace bbot void serialize (butl::manifest_serializer&) const; }; - - // The manifest stored in <name>-<toolchain>/ consists of the machine - // manifest (original), toolchain manifest, and bootstrap manifest. - // - class bootstrapped_machine_manifest - { - public: - machine_manifest machine; - toolchain_manifest toolchain; - bootstrap_manifest bootstrap; - - bootstrapped_machine_manifest (machine_manifest m, - toolchain_manifest t, - bootstrap_manifest b) - : machine (move (m)), toolchain (move (t)), bootstrap (move (b)) {} - - public: - bootstrapped_machine_manifest () = default; // VC export. - bootstrapped_machine_manifest (butl::manifest_parser&, - bool ignore_unknown = false); - - void - serialize (butl::manifest_serializer&) const; - }; } #endif // BBOT_BOOTSTRAP_MANIFEST diff --git a/bbot/bootstrap-manifest.cxx b/bbot/bootstrap-manifest.cxx index d08322a..6a0ff8d 100644 --- a/bbot/bootstrap-manifest.cxx +++ b/bbot/bootstrap-manifest.cxx @@ -17,78 +17,6 @@ namespace bbot using serialization = manifest_serialization; using name_value = manifest_name_value; - // toolchain_manifest - // - toolchain_manifest:: - toolchain_manifest (parser& p, bool iu) - : toolchain_manifest (p, p.next (), iu) - { - // Make sure this is the end. - // - name_value nv (p.next ()); - if (!nv.empty ()) - throw parsing (p.name (), nv.name_line, nv.name_column, - "single toolchain manifest expected"); - } - - toolchain_manifest:: - toolchain_manifest (parser& p, name_value nv, bool iu) - { - auto bad_name = [&p, &nv] (const string& d) - { - throw parsing (p.name (), nv.name_line, nv.name_column, d); - }; - - auto bad_value = [&p, &nv] (const string& d) - { - throw parsing (p.name (), nv.value_line, nv.value_column, d); - }; - - // Make sure this is the start and we support the version. - // - if (!nv.name.empty ()) - bad_name ("start of toolchain manifest expected"); - - if (nv.value != "1") - bad_value ("unsupported format version"); - - // Parse the toolchain manifest. - // - for (nv = p.next (); !nv.empty (); nv = p.next ()) - { - string& n (nv.name); - string& v (nv.value); - - if (n == "id") - { - if (!id.empty ()) - bad_name ("toolchain id redefinition"); - - if (v.empty ()) - bad_value ("empty toolchain id"); - - id = move (v); - } - else if (!iu) - bad_name ("unknown name '" + n + "' in toolchain manifest"); - } - - // Verify all non-optional values were specified. - // - if (id.empty ()) - bad_value ("no toolchain id specified"); - } - - void toolchain_manifest:: - serialize (serializer& s) const - { - // @@ Should we check that all non-optional values are specified? - // - s.next ("", "1"); // Start of manifest. - s.next ("id", id); - s.next ("", ""); // End of manifest. - } - // bootstrap_manifest // bootstrap_manifest:: @@ -181,87 +109,4 @@ namespace bbot s.next ("", ""); // End of manifest. } - - // bootstrapped_machine_manifest - // - bootstrapped_machine_manifest:: - bootstrapped_machine_manifest (parser& p, bool iu) - { - name_value nv (p.next ()); - - auto bad_name = [&p, &nv] (const string& d) - { - throw parsing (p.name (), nv.name_line, nv.name_column, d); - }; - - auto bad_value = [&p, &nv] (const string& d) - { - throw parsing (p.name (), nv.value_line, nv.value_column, d); - }; - - // Make sure this is the start and we support the version. - // - if (!nv.name.empty ()) - bad_name ("start of bootstrapped machine manifest expected"); - - if (nv.value != "1") - bad_value ("unsupported format version"); - - // Parse the bootstrapped machine manifest. Currently there is no values - // expected. - // - for (nv = p.next (); !nv.empty (); nv = p.next ()) - { - if (!iu) - bad_name ("unknown name '" + nv.name + - "' in bootstrapped machine manifest"); - } - - nv = p.next (); - if (nv.empty ()) - bad_value ("machine manifest expected"); - - machine = machine_manifest (p, nv, iu); - - if (!machine.mac) - bad_name ("mac address must be present in machine manifest"); - - nv = p.next (); - if (nv.empty ()) - bad_value ("toolchain manifest expected"); - - toolchain = toolchain_manifest (p, nv, iu); - - nv = p.next (); - if (nv.empty ()) - bad_value ("bootstrap manifest expected"); - - bootstrap = bootstrap_manifest (p, nv, iu); - - // Make sure this is the end. - // - nv = p.next (); - if (!nv.empty ()) - throw parsing (p.name (), nv.name_line, nv.name_column, - "single bootstrapped machine manifest expected"); - } - - void bootstrapped_machine_manifest:: - serialize (serializer& s) const - { - // @@ Should we check that all non-optional values are specified? - // - s.next ("", "1"); // Start of manifest. - s.next ("", ""); // End of manifest. - - if (!machine.mac) - throw serialization (s.name (), - "mac address must be present in machine manifest"); - - machine.serialize (s); - toolchain.serialize (s); - bootstrap.serialize (s); - - s.next ("", ""); // End of stream. - } } diff --git a/bbot/buildfile b/bbot/buildfile index 311775d..467b53e 100644 --- a/bbot/buildfile +++ b/bbot/buildfile @@ -26,7 +26,8 @@ if ($cxx.target.class == "linux") exe{bbot-agent}: \ {hxx cxx}{ agent } {hxx ixx cxx}{ agent-options } \ - {hxx cxx}{ bootstrap-manifest } {hxx ixx cxx}{ common-options } \ + {hxx ixx cxx}{ common-options } \ + {hxx cxx}{ bootstrap-manifest } \ {hxx cxx}{ diagnostics } \ {hxx cxx}{ machine-manifest } \ {hxx cxx}{ machine } \ @@ -43,6 +44,7 @@ if ($cxx.target.class == "linux") exe{bbot-worker}: \ { cxx}{ worker } {hxx ixx cxx}{ worker-options } \ {hxx ixx cxx}{ common-options } \ +{hxx cxx}{ bootstrap-manifest } \ {hxx cxx}{ diagnostics } \ {hxx }{ types } \ {hxx cxx}{ types-parsers } \ diff --git a/bbot/machine-manifest b/bbot/machine-manifest index 77f1c78..3801ae7 100644 --- a/bbot/machine-manifest +++ b/bbot/machine-manifest @@ -9,10 +9,12 @@ #include <butl/manifest-forward> +#include <bbot/manifest> // machine_header + #include <bbot/types> #include <bbot/utility> -#include <bbot/manifest> // machine_header +#include <bbot/bootstrap-manifest> namespace bbot { @@ -53,6 +55,56 @@ namespace bbot void serialize (butl::manifest_serializer&) const; }; + + // Toolchain. + // + class toolchain_manifest + { + public: + + // Toolchain id (SHAXXX). + // + string id; + + explicit + toolchain_manifest (string i): id (i) {} + + public: + toolchain_manifest () = default; // VC export. + toolchain_manifest (butl::manifest_parser&, bool ignore_unknown = false); + toolchain_manifest (butl::manifest_parser&, + butl::manifest_name_value start, + bool ignore_unknown = false); + + void + serialize (butl::manifest_serializer&) const; + }; + + // The manifest stored in <name>-<toolchain>/ consists of the machine + // manifest (original), toolchain manifest, and bootstrap manifest. + // + class bootstrapped_machine_manifest + { + public: + machine_manifest machine; + toolchain_manifest toolchain; + bootstrap_manifest bootstrap; + + bootstrapped_machine_manifest (machine_manifest m, + toolchain_manifest t, + bootstrap_manifest b) + : machine (move (m)), toolchain (move (t)), bootstrap (move (b)) {} + + public: + bootstrapped_machine_manifest () = default; // VC export. + bootstrapped_machine_manifest (butl::manifest_parser&, + bool ignore_unknown = false); + + void + serialize (butl::manifest_serializer&) const; + }; + + using bootstrapped_machine_manifests = vector<bootstrapped_machine_manifest>; } #endif // BBOT_MACHINE_MANIFEST diff --git a/bbot/machine-manifest.cxx b/bbot/machine-manifest.cxx index 8e5100c..5cef054 100644 --- a/bbot/machine-manifest.cxx +++ b/bbot/machine-manifest.cxx @@ -172,4 +172,158 @@ namespace bbot s.next ("", ""); // End of manifest. } + // toolchain_manifest + // + toolchain_manifest:: + toolchain_manifest (parser& p, bool iu) + : toolchain_manifest (p, p.next (), iu) + { + // Make sure this is the end. + // + name_value nv (p.next ()); + if (!nv.empty ()) + throw parsing (p.name (), nv.name_line, nv.name_column, + "single toolchain manifest expected"); + } + + toolchain_manifest:: + toolchain_manifest (parser& p, name_value nv, bool iu) + { + auto bad_name = [&p, &nv] (const string& d) + { + throw parsing (p.name (), nv.name_line, nv.name_column, d); + }; + + auto bad_value = [&p, &nv] (const string& d) + { + throw parsing (p.name (), nv.value_line, nv.value_column, d); + }; + + // Make sure this is the start and we support the version. + // + if (!nv.name.empty ()) + bad_name ("start of toolchain manifest expected"); + + if (nv.value != "1") + bad_value ("unsupported format version"); + + // Parse the toolchain manifest. + // + for (nv = p.next (); !nv.empty (); nv = p.next ()) + { + string& n (nv.name); + string& v (nv.value); + + if (n == "id") + { + if (!id.empty ()) + bad_name ("toolchain id redefinition"); + + if (v.empty ()) + bad_value ("empty toolchain id"); + + id = move (v); + } + else if (!iu) + bad_name ("unknown name '" + n + "' in toolchain manifest"); + } + + // Verify all non-optional values were specified. + // + if (id.empty ()) + bad_value ("no toolchain id specified"); + } + + void toolchain_manifest:: + serialize (serializer& s) const + { + // @@ Should we check that all non-optional values are specified? + // + s.next ("", "1"); // Start of manifest. + s.next ("id", id); + s.next ("", ""); // End of manifest. + } + + // bootstrapped_machine_manifest + // + bootstrapped_machine_manifest:: + bootstrapped_machine_manifest (parser& p, bool iu) + { + name_value nv (p.next ()); + + auto bad_name = [&p, &nv] (const string& d) + { + throw parsing (p.name (), nv.name_line, nv.name_column, d); + }; + + auto bad_value = [&p, &nv] (const string& d) + { + throw parsing (p.name (), nv.value_line, nv.value_column, d); + }; + + // Make sure this is the start and we support the version. + // + if (!nv.name.empty ()) + bad_name ("start of bootstrapped machine manifest expected"); + + if (nv.value != "1") + bad_value ("unsupported format version"); + + // Parse the bootstrapped machine manifest. Currently there is no values + // expected. + // + for (nv = p.next (); !nv.empty (); nv = p.next ()) + { + if (!iu) + bad_name ("unknown name '" + nv.name + + "' in bootstrapped machine manifest"); + } + + nv = p.next (); + if (nv.empty ()) + bad_value ("machine manifest expected"); + + machine = machine_manifest (p, nv, iu); + + if (!machine.mac) + bad_name ("mac address must be present in machine manifest"); + + nv = p.next (); + if (nv.empty ()) + bad_value ("toolchain manifest expected"); + + toolchain = toolchain_manifest (p, nv, iu); + + nv = p.next (); + if (nv.empty ()) + bad_value ("bootstrap manifest expected"); + + bootstrap = bootstrap_manifest (p, nv, iu); + + // Make sure this is the end. + // + nv = p.next (); + if (!nv.empty ()) + throw parsing (p.name (), nv.name_line, nv.name_column, + "single bootstrapped machine manifest expected"); + } + + void bootstrapped_machine_manifest:: + serialize (serializer& s) const + { + // @@ Should we check that all non-optional values are specified? + // + s.next ("", "1"); // Start of manifest. + s.next ("", ""); // End of manifest. + + if (!machine.mac) + throw serialization (s.name (), + "mac address must be present in machine manifest"); + + machine.serialize (s); + toolchain.serialize (s); + bootstrap.serialize (s); + + s.next ("", ""); // End of stream. + } } diff --git a/bbot/machine.cxx b/bbot/machine.cxx index 460e802..0fc38ee 100644 --- a/bbot/machine.cxx +++ b/bbot/machine.cxx @@ -149,7 +149,7 @@ namespace bbot br (br), tap (create_tap (br, port)), port (port), - vnc ("127.0.0.1:" + to_string (5900 + stoul (tc_num))), + vnc ("127.0.0.1:" + to_string (5900 + tc_num)), monitor ("/tmp/" + tc_name + "-monitor") { tracer trace ("kvm_machine"); diff --git a/bbot/tftp.cxx b/bbot/tftp.cxx index 27d58a4..7249214 100644 --- a/bbot/tftp.cxx +++ b/bbot/tftp.cxx @@ -118,7 +118,7 @@ namespace bbot "--map-file", map_.path (), // Path remapping rules. "--user", uname, // Run as our effective user. "--secure", // Chroot to data directory. - "/build/tftp/"); + ops.tftp ()); sec = static_cast<size_t> (timeout.tv_sec); return true; @@ -22,6 +22,8 @@ #include <butl/path> #include <butl/optional> +#include <butl/vector-view> +#include <butl/small-vector> namespace bbot { @@ -66,6 +68,14 @@ namespace bbot using butl::optional; using butl::nullopt; + // <butl/vector-view> + // + using butl::vector_view; + + // <butl/small-vector> + // + using butl::small_vector; + // <butl/path> // using butl::path; diff --git a/bbot/utility b/bbot/utility index 91bce64..59f6e09 100644 --- a/bbot/utility +++ b/bbot/utility @@ -13,7 +13,9 @@ #include <butl/ft/lang> +#include <butl/curl> #include <butl/process> +#include <butl/process-io> #include <butl/utility> // casecmp(), reverse_iterate(), etc #include <butl/fdstream> #include <butl/filesystem> @@ -69,11 +71,11 @@ namespace bbot template <typename I, typename O, typename E, typename P, typename... A> void - run_io (tracer&, I&& in, O&& out, E&& err, P&&, A&&...); + run_io (tracer&, I&& in, O&& out, E&& err, const P&, A&&...); template <typename I, typename O, typename E, typename P, typename... A> process_exit::code_type - run_io_exit (tracer&, I&& in, O&& out, E&& err, P&&, A&&...); + run_io_exit (tracer&, I&& in, O&& out, E&& err, const P&, A&&...); template <typename I, typename O, typename E, typename P, typename... A> process @@ -82,35 +84,61 @@ namespace bbot O&& out, E&& err, const dir_path& cwd, - P&&, + const P&, A&&...); template <typename P> void - run_io_finish (tracer&, process&, P&&); + run_io_finish (tracer&, process&, const P&); template <typename P> process_exit::code_type - run_io_finish_exit (tracer&, process&, P&&); + run_io_finish_exit (tracer&, process&, const P&); template <typename P, typename... A> inline void - run (tracer& t, P&& p, A&&... a) + run (tracer& t, const P& p, A&&... a) { - run_io (t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...); + run_io (t, butl::fdnull (), 2, 2, p, forward<A> (a)...); } template <typename P, typename... A> inline process_exit::code_type - run_exit (tracer& t, P&& p, A&&... a) + run_exit (tracer& t, const P& p, A&&... a) { return run_io_exit ( - t, butl::fdnull (), 2, 2, forward<P> (p), forward<A> (a)...); + t, butl::fdnull (), 2, 2, p, forward<A> (a)...); } void run_trace (tracer&, const char*[], size_t); + // The curl process wrapper (command line tracing, etc). + // + class http_curl: public butl::curl + { + public: + template <typename I, typename O, typename... A> + http_curl (tracer&, + I&& in, + O&& out, + method_type, + const string& url, + A&&... options); + }; + + class tftp_curl: public butl::curl + { + public: + template <typename I, typename O, typename... A> + tftp_curl (tracer&, + I&& in, + O&& out, + method_type, + const string& url, + A&&... options); + }; + // Manifest parsing and serialization. // template <typename T> @@ -118,6 +146,13 @@ namespace bbot parse_manifest (const path&, const char* what, bool ignore_unknown = true); template <typename T> + T + parse_manifest (istream&, + const string& name, + const char* what, + bool ignore_unknown = true); + + template <typename T> void serialize_manifest (const T&, const path&, const char* what); diff --git a/bbot/utility.cxx b/bbot/utility.cxx index 6b7a8ca..83249ad 100644 --- a/bbot/utility.cxx +++ b/bbot/utility.cxx @@ -29,7 +29,7 @@ namespace bbot void run_trace (tracer& t, const char* cmd[], size_t n) { - if (verb >= 2) + if (verb >= 3) { diag_record dr (t); process::print (dr.os, cmd, n); diff --git a/bbot/utility.txx b/bbot/utility.txx index a613b11..edd674b 100644 --- a/bbot/utility.txx +++ b/bbot/utility.txx @@ -11,6 +11,8 @@ namespace bbot { + // run_*() + // template <typename I, typename O, typename E, typename P, typename... A> process run_io_start (tracer& t, @@ -18,7 +20,7 @@ namespace bbot O&& out, E&& err, const dir_path& cwd, - P&& p, + const P& p, A&&... args) { try @@ -40,17 +42,19 @@ namespace bbot template <typename P> process_exit::code_type - run_io_finish_exit (tracer&, process& pr, P&& p) + run_io_finish_exit (tracer&, process& pr, const P& p) { try { pr.wait (); - if (!pr.exit->normal ()) - fail << "process " << p << " terminated abnormally: " - << pr.exit->description (); + const process_exit& e (*pr.exit); - return pr.exit->code (); + if (e.normal ()) + return e.code (); + + fail << "process " << p << " terminated abnormally: " + << e.description () << (e.core () ? " (core dumped)" : "") << endf; } catch (const process_error& e) { @@ -60,7 +64,7 @@ namespace bbot template <typename P> inline void - run_io_finish (tracer& t, process& pr, P&& p) + run_io_finish (tracer& t, process& pr, const P& p) { if (run_io_finish_exit (t, pr, p) != 0) fail << "process " << p << " terminated with non-zero exit code"; @@ -95,6 +99,47 @@ namespace bbot run_io_finish (t, pr, p); } + // curl + // + template <typename I, typename O, typename... A> + http_curl:: + http_curl (tracer& t, + I&& in, + O&& out, + method_type m, + const string& url, + A&&... options) + : butl::curl ([&t] (const char* c[], size_t n) {run_trace (t, c, n);}, + forward<I> (in), + forward<O> (out), + 2, // Diagnostics to stderr. + m, + url, + "-A", BBOT_USER_AGENT, + forward<A> (options)...) + { + } + + template <typename I, typename O, typename... A> + tftp_curl:: + tftp_curl (tracer& t, + I&& in, + O&& out, + method_type m, + const string& url, + A&&... options) + : butl::curl ([&t] (const char* c[], size_t n) {run_trace (t, c, n);}, + forward<I> (in), + forward<O> (out), + 2, // Diagnostics to stderr. + m, + url, + forward<A> (options)...) + { + } + + // *_manifest() + // template <typename T> T parse_manifest (const path& f, const char* what, bool ignore_unknown) @@ -107,23 +152,38 @@ namespace bbot fail << what << " manifest file " << f << " does not exist"; ifdstream ifs (f); - manifest_parser p (ifs, f.string ()); + return parse_manifest<T> (ifs, f.string (), what, ignore_unknown); + } + catch (const system_error& e) // EACCES, etc. + { + fail << "unable to access " << what << " manifest " << f << ": " << e + << endf; + } + } + + template <typename T> + T + parse_manifest (istream& is, + const string& name, + const char* what, + bool ignore_unknown) + { + using namespace butl; + + try + { + manifest_parser p (is, name); return T (p, ignore_unknown); } catch (const manifest_parsing& e) { fail << "invalid " << what << " manifest: " - << f << ':' << e.line << ':' << e.column << ": " << e.description + << name << ':' << e.line << ':' << e.column << ": " << e.description << endf; } catch (const io_error& e) { - fail << "unable to read " << what << " manifest " << f << ": " << e - << endf; - } - catch (const system_error& e) // EACCES, etc. - { - fail << "unable to access " << what << " manifest " << f << ": " << e + fail << "unable to read " << what << " manifest " << name << ": " << e << endf; } } @@ -158,7 +218,6 @@ namespace bbot const string& name, const char* what) { - using namespace std; using namespace butl; try diff --git a/bbot/worker.cli b/bbot/worker.cli index 3691b2d..561d714 100644 --- a/bbot/worker.cli +++ b/bbot/worker.cli @@ -11,18 +11,27 @@ include <bbot/common.cli>; namespace bbot { { - "<options> <file>", + "<options> <module> <conf-var>", " \h|SYNOPSIS| \cb{bbot-worker --help}\n \cb{bbot-worker --version}\n - \c{\b{bbot-worker} [<options>] <toolchain>} + \c{\b{bbot-worker --bootstrap} [<options>]}\n + \c{\b{bbot-worker --startup} [<options>]}\n + \c{\b{bbot-worker} [<options>] <module>... <conf-var>...} \h|DESCRIPTION| \cb{bbot-worker} @@ TODO. + + If the \cb{--bootstrap} mode option is specified, then the worker performs + the initial machine bootstrap and writes the bootstrap result manifest to + \c{stdout}. If the \cb{--startup} mode option is specified, then the + worker performs the environment setup and then re-executes in the build + mode. If neither of the mode options is specified, then the worker + proceeds to performing the build task. " } @@ -32,6 +41,47 @@ namespace bbot bool --help {"Print usage information and exit."} bool --version {"Print version and exit."} + + bool --bootstrap + { + "Perform the inital machine bootstrap insteading of building." + } + + bool --startup + { + "Perform the environment setup and then re-execute for building." + } + + dir_path --build + { + "<dir>", + "The directory to perform the build in. If not specified, then the + current working directory is used." + } + + dir_path --environment + { + "<dir>", + "The directory containing the environment setup executables. If not + specified, then the user's home directory is used." + } + + uint16_t --verbose = 1 + { + "<level>", + "Set the diagnostics verbosity to <level> between 0 and 6 with level 1 + being the default." + } + + // Testing options. + // + string --tftp-host = "196.254.111.222" + { + "<addr>", + "The TFTP host address and, optionally, port to use to download the + build task and to upload the build result. By default the link-local + address 196.254.111.222 with the default TFTP port (69) is used." + } }; " diff --git a/bbot/worker.cxx b/bbot/worker.cxx index e0149fc..fbae8c7 100644 --- a/bbot/worker.cxx +++ b/bbot/worker.cxx @@ -11,12 +11,16 @@ #include <iostream> #include <butl/pager> +#include <butl/filesystem> + +#include <bbot/manifest> #include <bbot/types> #include <bbot/utility> #include <bbot/diagnostics> #include <bbot/worker-options> +#include <bbot/bootstrap-manifest> using namespace std; using namespace butl; @@ -24,7 +28,412 @@ using namespace bbot; namespace bbot { + process_path argv0; worker_options ops; + + dir_path env_dir; + + const size_t tftp_timeout (10); // 10 seconds. +} + +static dir_path +change_wd (const dir_path& d, bool create = false) +try +{ + if (create) + try_mkdir_p (d); + + dir_path r (dir_path::current_directory ()); + + dir_path::current_directory (d); + + return r; +} +catch (const system_error& e) +{ + fail << "unable to change current directory to " << d << ": " << e << endf; +} + +template <typename... A> +static result_status +run_bpkg (tracer& t, string& log, const string& cmd, A&&... a) +{ + try + { + // Trace and log the command line. + // + auto cmdc = [&t, &log] (const char* c[], size_t n) + { + run_trace (t, c, n); + + ostringstream os; + process::print (os, c, n); + log += os.str (); + log += '\n'; + }; + + fdpipe pipe (fdopen_pipe ()); // Text mode seems appropriate. + + process pr ( + process_start (cmdc, + fdnull (), // Never reads from stdout. + 2, // 1>&2 + pipe.out, + dir_path (), + "bpkg", + "-v", + cmd, + forward<A> (a)...)); + + pipe.out.close (); + + // Log the diagnostics. + // + { + ifdstream is (move (pipe.in), fdstream_mode::skip); // Skip on exception. + + for (string l; is.peek () != ifdstream::traits_type::eof (); ) + { + getline (is, l); + log += l; + log += '\n'; + } + } + + if (pr.wait ()) + return result_status::success; + + log += "bpkg " + cmd; + const process_exit& e (*pr.exit); + + if (e.normal ()) + { + log += " exited with code " + to_string (e.code ()) + "\n"; + return result_status::error; + } + else + { + log += " terminated abnormally: " + e.description () + + (e.core () ? " (core dumped)" : "") + "\n"; + return result_status::abnormal; + } + } + catch (const process_error& e) + { + fail << "unable to execute bpkg " << cmd << ": " << e << endf; + } + catch (const io_error& e) + { + fail << "unable to read bpkg " << cmd << " diagnostics: " << e << endf; + } +} + +static void +build (size_t argc, const char* argv[]) +{ + tracer trace ("build"); + + // Our overall plan is as follows: + // + // 1. Parse the task manifest (it should be in CWD). + // + // 2. Run bpkg to create the configuration, add the repository, and + // configure, build, and test the package all while saving the logs in + // the result manifest. + // + // 3. Upload the result manifest. + // + // Note also that we are being "watched" by the startup version of us which + // will upload an appropriate result in case we exit with an error. So here + // for abnormal situations (like a failure to parse the manifest), we just + // fail. + // + task_manifest tm (parse_manifest<task_manifest> (path ("manifest"), "task")); + + result_manifest rm { + tm.name, + tm.version, + result_status::success, + operation_results {} + }; + + auto add_result = [&rm] (string o) -> operation_result& + { + rm.results.push_back ( + operation_result {move (o), result_status::success, ""}); + + return rm.results.back (); + }; + + dir_path owd; + + for (;;) // The "breakout" loop. + { + // Configure. + // + { + operation_result& r (add_result ("configure")); + + // bpkg create <config-vars> <env-module> <env-config-vars> + // + const vector_view<const char*> env (argv + 1, argc - 1); + + vector<string> cfg; + for (const variable& v: tm.config) + cfg.push_back (v.unquoted ()); + + dir_path dir ("build"); + + r.status |= run_bpkg (trace, r.log, + "create", + "-d", dir.string (), + "--wipe", + cfg, + env); + + if (!r.status) + break; + + owd = change_wd (dir); + + // bpkg add <repository-url> + // + r.status |= run_bpkg (trace, r.log, "add", tm.repository.string ()); + + if (!r.status) + break; + + // bpkg fetch + // + string t ("--trust-no"); + + cstrings ts; + for (const string& fp: tm.trust) + { + if (fp == "yes") + t = "--trust-yes"; + else + { + ts.push_back ("--trust"); + ts.push_back (fp.c_str ()); + } + } + + r.status |= run_bpkg (trace, r.log, "fetch", ts, t); + + if (!r.status) + break; + + // bpkg build --configure-only <package-name>/<package-version> + // + r.status |= run_bpkg (trace, r.log, + "build", + "--configure-only", + "--yes", + tm.name + '/' + tm.version.string ()); + + if (!r.status) + break; + + rm.status |= r.status; + } + + // Update. + // + { + operation_result& r (add_result ("update")); + + // bpkg update <package-name> + // + r.status |= run_bpkg (trace, r.log, "update", tm.name); + + if (!r.status) + break; + + rm.status |= r.status; + } + + // Test. + // + { + operation_result& r (add_result ("test")); + + // bpkg test <package-name> + // + r.status |= run_bpkg (trace, r.log, "test", tm.name); + + if (!r.status) + break; + + rm.status |= r.status; + } + + break; + } + + rm.status |= rm.results.back ().status; // Merge last in case of a break. + + if (!owd.empty ()) + change_wd (owd); + + // Upload the result. + // + const string url ("tftp://" + ops.tftp_host () + "/manifest"); + + try + { + tftp_curl c (trace, + path ("-"), + nullfd, + curl::put, + url, + "--max-time", tftp_timeout); + + serialize_manifest (rm, c.out, url, "result"); + c.out.close (); + + if (!c.wait ()) + throw_generic_error (EIO); + } + catch (const system_error& e) + { + fail << "unable to upload result manifest to " << url << ": " << e; + } +} + +static void +startup () +{ + tracer trace ("startup"); + + // Our overall plan is as follows: + // + // 1. Download the task manifest into the build directory (CWD). + // + // 2. Parse it and get the target. + // + // 3. Find the environment setup executable for this target. + // + // 4. Execute the environment setup executable. + // + // 5. If the environment setup executable fails, then upload the (failed) + // result ourselves. + // + const string url ("tftp://" + ops.tftp_host () + "/manifest"); + const path mf ("manifest"); + + // If we fail, try to upload the result manifest (abnormal termination). The + // idea is that the machine gets suspended and we can investigate what's + // going on by logging in and examining the diagnostics (e.g., via + // journalctl, etc). + // + task_manifest tm; + + try + { + // Download the task. + // + try + { + tftp_curl c (trace, + nullfd, + mf, + curl::get, + url, + "--max-time", tftp_timeout); + + if (!c.wait ()) + throw_generic_error (EIO); + } + catch (const system_error& e) + { + fail << "unable to download task manifest from " << url << ": " << e; + } + + // Parse it. + // + tm = parse_manifest<task_manifest> (mf, "task"); + + // Find the environment setup executable. + // + string tg; + process_path pp; + + if (tm.target) + { + tg = tm.target->string (); + + // While the executable path contains a directory (so the PATH search + // does not apply) we still use process::path_search() to automatically + // handle appending platform-specific executable extensions (.exe/.bat, + // etc). + // + pp = process::try_path_search (env_dir / tg, false); + } + + if (pp.empty ()) + pp = process::try_path_search (env_dir / "default", false); + + if (pp.empty ()) + fail << "no environment setup executable in " << env_dir << " " + << "for target '" << tg << "'"; + + // Run it. + // + // Note that we use the effective (absolute) path instead of recall since + // we may have changed the CWD. + // + run (trace, pp, tg, argv0.effect_string ()); + } + catch (const failed&) + { + // If we failed before being able to parse the task manifest, use the + // "unknown" values for the package name and version. + // + result_manifest rm { + tm.name.empty () ? "unknown" : tm.name, + tm.version.empty () ? bpkg::version ("0") : tm.version, + result_status::abnormal, + operation_results {} + }; + + try + { + tftp_curl c (trace, + path ("-"), + nullfd, + curl::put, + url, + "--max-time", tftp_timeout); + + serialize_manifest (rm, c.out, url, "result"); + c.out.close (); + + if (!c.wait ()) + throw_generic_error (EIO); + } + catch (const system_error& e) + { + error << "unable to upload result manifest to " << url << ": " << e; + } + + throw; + } +} + +static void +bootstrap () +{ + bootstrap_manifest bm { + bootstrap_manifest::versions_type { + {"bbot", BBOT_VERSION}, + {"libbbot", LIBBBOT_VERSION}, + {"libbpkg", LIBBPKG_VERSION}, + {"libbutl", LIBBUTL_VERSION} + } + }; + + serialize_manifest (bm, cout, "stdout", "bootstrap"); } int @@ -69,6 +478,8 @@ try cli::argv_scanner scan (argc, argv, true); ops.parse (scan); + verb = ops.verbose (); + // Version. // if (ops.version ()) @@ -94,8 +505,48 @@ try return p.wait () ? 0 : 1; } - warn << "starting up" << - info << "lightly"; + // Figure out our mode. + // + if (ops.bootstrap () && ops.startup ()) + fail << "--bootstrap and --startup are mutually exclusive"; + + enum class mode {boot, start, build} m (mode::build); + + if (ops.bootstrap ()) m = mode::boot; + if (ops.startup ()) m = mode::start; + + // Figure out our path (used for re-exec). + // + argv0 = process::path_search (argv[0], true); + + // Sort out the build directory. + // + if (ops.build_specified ()) + change_wd (ops.build (), true); // Create if does not exist. + + // Sort out the environment directory. + // + try + { + env_dir = ops.environment_specified () + ? ops.environment () + : dir_path::home_directory (); + + if (!dir_exists (env_dir)) + throw_generic_error (ENOENT); + } + catch (const system_error& e) + { + fail << "invalid environment directory: " << e; + } + + switch (m) + { + case mode::boot: bootstrap (); break; + case mode::start: startup (); break; + case mode::build: build (static_cast<size_t> (argc), + const_cast<const char**> (argv)); break; + } } catch (const failed&) { diff --git a/doc/manual.cli b/doc/manual.cli index 5da684a..2b2fc21 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -243,8 +243,9 @@ SYNOPSIS name: <package-name> version: <package-version> -repository: <repository-url> #location: <package-url> +repository: <repository-url> +trust: <repository-fp> machine: <machine-name> target: <target-triplet> @@ -269,6 +270,19 @@ build configuration to use for building the package. The \c{bpkg} repository that contains the package and its dependencies.| +\li|\n\c{trust: <repository-fp>}\n + + The SHA256 repository certificate fingerprint to trust (see the \c{bpkg} + \c{--trust} option for details). This value may be specified multiple times + to establish the authenticity of multiple certificates. If the special + \c{yes} value is specified, then all repositories will be trusted without + authentication (see the \c{bpkg} \c{--trust-yes} option). + + Note that while the controller may return a task with \c{trust} values, + whether they will be used is up to the agent's configuration. For example, + some agents may only trust their internally-specified fingerprints to + prevent the \"man in the middle\" type of attacks.| + \li|\n\c{machine: <machine-name>}\n The name of the build machine to use.| @@ -480,16 +494,18 @@ the environment executable is an error. Once the environment setup executable is determined, the worker re-executes itself as that executable passing to it as command line arguments the target -name (or empty value if not specified), the path to the \c{bbot} worker to be -executed once the environment is setup, and the path to the build task -manifest. +name (or empty value if not specified) and the path to the \c{bbot} worker to +be executed once the environment is setup. The environment setup executable is +executed in the build directory as its current working directory. The build +directory contains the build task \c{manifest} file. The environment setup executable sets up the necessary execution environment for example by adjusting \c{PATH} or running a suitable \c{vcvars} batch file. It then re-executes itself as the \c{bbot} worker passing to it as command -line arguments the path to the build task manifest followed by the list of -build system modules (\c{<env-modules>}) and the list of configuration -variables (\c{<env-config-vars>}). +line arguments the list of build system modules (\c{<env-modules>}) and the +list of configuration variables (\c{<env-config-vars>}). The environment setup +executable must execute the \c{bbot} worker in the build directory as the +current working directory. The re-executed \c{bbot} worker then proceeds to test the package from the repository by executing the following commands (\c{<>}-values are from the @@ -498,7 +514,7 @@ task manifest and environment): \ bpkg -v create <env-module> <config-vars> <env-config-vars> bpkg -v add <repository-url> -bpkg -v fetch +bpkg -v fetch --trust <repository-fp> bpkg -v build --yes --configure-only <package-name>/<package-version> bpkg -v update <package-name> bpkg -v test <package-name> @@ -512,7 +528,6 @@ for building C and C++ packages with GCC 6 on most Linux distributions. # $1 - target # $2 - bbot executable -# $3 - task manifest trap \"exit 1\" ERR @@ -521,7 +536,7 @@ if [ -n \"$1\" ]; then exit 1 fi -exec \"$2\" --build \"$3\" cc config.c=gcc-6 +exec \"$2\" cc config.c=gcc-6 \ \h#arch-controller|Controller Logic| diff --git a/tests/agent/testscript b/tests/agent/testscript index cf92ace..e864bf2 100644 --- a/tests/agent/testscript +++ b/tests/agent/testscript @@ -3,7 +3,7 @@ # license : TBC; see accompanying LICENSE file # The /build/machines directory should be on a btrfs filesystem and have the -# following layout and contents: +# following layout/contents: # # /build/machines/ # └── default/ @@ -17,25 +17,24 @@ # # - The test must be run serially (@@ TODO: serial directive) -test.options = --verbose 2 -test.arguments = stage 1 +test.options = --verbose 3 cp = $src_base/btrfs-cpdir -f /build/machines.orig /build/machines rm = $src_base/btrfs-rmdir /build/machines -#\ : dump-machines : { - m = /build/machines/default/linux-gcc - test.options += --dump-machines --fake-bootstrap + m = /build/machines/default/linux-gcc + u = https://example.org/?dummy + +$cp : no-current-machine-symlink : - $* 123 2>>"EOE" + $* --toolchain-id 123 $u 2>>"EOE" trace: enumerate_machines: skipping $m/: no subvolume link EOE @@ -43,91 +42,140 @@ rm = $src_base/btrfs-rmdir /build/machines : bootstrap : - $* 123 >>EOO 2>>~"%EOE%d" + $* --toolchain-id 123 $u >>EOO 2>>~"%EOE%d" : 1 id: linux-gcc-1.0 name: linux-gcc summary: Linux with GCC EOO - %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-stage-\\.+% - trace: enumerate_machines: bootstrapping $m/linux-gcc-stage/ + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.0 $m/linux-gcc-default-\\.+% + trace: enumerate_machines: bootstrapping $m/linux-gcc-default/ + trace: enumerate_machines: bootstrapped linux-gcc EOE ln -T -f -s linux-gcc-1.1 $m/linux-gcc-1 : re-bootstrap-machine : - $* 123 >>EOO 2>>~"%EOE%d" + $* --toolchain-id 123 $u >>EOO 2>>~"%EOE%d" : 1 id: linux-gcc-1.1 name: linux-gcc summary: Linux with GCC EOO - %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+% - trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new machine - trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false - trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+% + trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new machine + trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false + trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default + trace: enumerate_machines: bootstrapped linux-gcc EOE : re-bootstrap-toolchain : - $* 124 >>EOO 2>>~"%EOE%d" + $* --toolchain-id 124 $u >>EOO 2>>~"%EOE%d" : 1 id: linux-gcc-1.1 name: linux-gcc summary: Linux with GCC EOO - %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+% - trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new toolchain - trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false - trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+% + trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new toolchain + trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false + trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default + trace: enumerate_machines: bootstrapped linux-gcc EOE - sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-stage/manifest + sed -i -e 's/^(bbot-version):.*/\1: 0/' $m/linux-gcc-default/manifest : re-bootstrap-bbot : - $* 124 >>EOO 2>>~"%EOE%d" + $* --toolchain-id 124 $u >>EOO 2>>~"%EOE%d" : 1 id: linux-gcc-1.1 name: linux-gcc summary: Linux with GCC EOO - %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+% - trace: enumerate_machines: re-bootstrapping $m/linux-gcc-stage/: new bbot - trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-stage ro false - trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+% + trace: enumerate_machines: re-bootstrapping $m/linux-gcc-default/: new bbot + trace: enumerate_machines: btrfs property set -ts $m/linux-gcc-default ro false + trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default + trace: enumerate_machines: bootstrapped linux-gcc EOE - sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-stage/manifest + sed -i -e 's/^(bbot-version):.*/\1: 99999900/' $m/linux-gcc-default/manifest : re-bootstrap-bbot-newer : - $* 124 2>>~"%EOE%d" - %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-stage-\\.+% - trace: enumerate_machines: ignoring $m/linux-gcc-stage/: old bbot - %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-stage-\\.+% + $* --toolchain-id 124 $u 2>>~"%EOE%d" + %trace: enumerate_machines: btrfs subvolume snapshot $m/linux-gcc-1.1 $m/linux-gcc-default-\\.+% + trace: enumerate_machines: ignoring $m/linux-gcc-default/: old bbot + %trace: enumerate_machines: btrfs subvolume delete $m/linux-gcc-default-\\.+% EOE -$rm } -#\ - -: bootstrap +: dump-result : { + test.options += --fake-bootstrap --fake-build --fake-request - + m = /build/machines/default/linux-gcc + u = https://example.org/?dummy + + +$cp + +ln -T -s linux-gcc-1.0 $m/linux-gcc-1 + +$* --dump-machines --verbose 1 $u >- # Bootstrap. + + : no-machine + : + $* --dump-task $u <<EOI 2>>~"%EOE%d" + : 1 + name: foo + version: 1.2.3 + repository: https://example.org/1/ + machine: windows-msvc + EOI + %trace: enumerate_machines:\\.*%* + error: task from $u for unknown machine windows-msvc + EOE + : result + : + $* --dump-result $u <<EOI >>EOO 2>>~"%EOE%d" + : 1 + name: foo + version: 1.2.3 + repository: https://example.org/1/ + machine: linux-gcc + EOI + : 1 + name: foo + version: 1.2.3 + status: abort + EOO + %trace: enumerate_machines:\\.*%* + EOE + + -$rm +} + +#\ +: bootstrap +: +{ test.options += --dump-machines + m = /build/machines/default/linux-gcc + u = https://example.org/?dummy + +$cp ln -T -s linux-gcc-1.0 $m/linux-gcc-1 : bootstrap : - $* 123 >>EOO 2>>EOE #2>>~"%EOE%d" + $* $u >>EOO 2>>EOE #2>>~"%EOE%d" : 1 id: linux-gcc-1.0 name: linux-gcc @@ -137,3 +185,4 @@ rm = $src_base/btrfs-rmdir /build/machines #-$rm } +#\ diff --git a/tests/worker/bootstrap.test b/tests/worker/bootstrap.test new file mode 100644 index 0000000..91e4417 --- /dev/null +++ b/tests/worker/bootstrap.test @@ -0,0 +1,13 @@ +# file : tests/worker/bootstrap.test +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : TBC; see accompanying LICENSE file + +test.options = --bootstrap + +$* >>~%EOO% +: 1 +%bbot-version: \d+% +%libbbot-version: \d+% +%libbpkg-version: \d+% +%libbutl-version: \d+% +EOO diff --git a/tests/worker/build.test b/tests/worker/build.test new file mode 100644 index 0000000..adedccc --- /dev/null +++ b/tests/worker/build.test @@ -0,0 +1,133 @@ +# file : tests/worker/build.test +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : TBC; see accompanying LICENSE file + +# Note: requires TFTP server (see buildfile). + +tftp = 127.0.0.1:55123/test + +pkg = hello +ver = 1.0.0 + +#rep = /home/boris/work/build2/hello/repository/1/stable +rep = https://build2.org/pkg/1/stage/stable +#rep = https://build2.org/pkg/1/hello/stable + +rfp = FF:DF:7D:38:67:4E:C3:82:65:7E:EE:1F:D4:80:EC:56:C4:33:5B:65:3F:9B:29:9A:30:56:B9:77:B9:F2:01:94 + +: pass +: +cat <<"EOI" >=manifest; + : 1 + name: $pkg + version: $ver + repository: $rep + trust: $rfp + machine: linux-gcc + target: x86_64-linux-gnu + config: config.cc.coptions=-O3 + EOI +$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE"; + trace: build: bpkg -v create -d build --wipe config.cc.coptions=-O3 cc + trace: build: bpkg -v add $rep + trace: build: bpkg -v fetch --trust $rfp --trust-no + trace: build: bpkg -v build --configure-only --yes $pkg/$ver + trace: build: bpkg -v update $pkg + trace: build: bpkg -v test $pkg + trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest + EOE +cat manifest >>~"%EOO%" + : 1 + name: $pkg + version: $ver + status: success + configure-status: success + update-status: success + test-status: success + configure-log: \\ + %.*%+ + \\ + update-log: \\ + %.*%+ + \\ + test-log: \\ + %.*%+ + \\ + EOO + + +: fail-abnormal +: +cat <<"EOI" >=manifest; + : 1 + foo: bar + EOI +$* --verbose 3 --tftp-host "$tftp/$@" cc 2>>"EOE" != 0 + error: invalid task manifest: manifest:3:1: no task package name specified + EOE + + +: fail-configure +: +cat <<"EOI" >=manifest; + : 1 + name: bogus + version: 1.2.3 + repository: $rep + trust: $rfp + machine: linux-gcc + target: x86_64-linux-gnu + EOI +$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE"; + trace: build: bpkg -v create -d build --wipe cc + trace: build: bpkg -v add $rep + trace: build: bpkg -v fetch --trust $rfp --trust-no + trace: build: bpkg -v build --configure-only --yes bogus/1.2.3 + trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest + EOE +cat manifest >>~"%EOO%" + : 1 + name: bogus + version: 1.2.3 + status: error + configure-status: error + configure-log: \\ + %.*%+ + \\ + EOO + + +: fail-update +: +cat <<"EOI" >=manifest; + : 1 + name: $pkg + version: $ver + repository: $rep + trust: $rfp + machine: linux-gcc + target: x86_64-linux-gnu + config: config.cc.loptions=-lbogus + EOI +$* --verbose 3 --tftp-host "$tftp/$@" cc &build/*** 2>>"EOE"; + trace: build: bpkg -v create -d build --wipe config.cc.loptions=-lbogus cc + trace: build: bpkg -v add $rep + trace: build: bpkg -v fetch --trust $rfp --trust-no + trace: build: bpkg -v build --configure-only --yes $pkg/$ver + trace: build: bpkg -v update $pkg + trace: build: curl -s -S --upload-file - --max-time 10 tftp://$tftp/$@/manifest + EOE +cat manifest >>~"%EOO%" + : 1 + name: $pkg + version: $ver + status: error + configure-status: success + update-status: error + configure-log: \\ + %.*%+ + \\ + update-log: \\ + %.*%+ + \\ + EOO diff --git a/tests/worker/buildfile b/tests/worker/buildfile new file mode 100644 index 0000000..7d0a206 --- /dev/null +++ b/tests/worker/buildfile @@ -0,0 +1,26 @@ +# file : tests/worker/buildfile +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : TBC; see accompanying LICENSE file + +#\ + +Some tests in this directory require a running TFTP server. + +TFTP server (tftp-hpa) setup: from the test out_base, run (sudo is required +for --secure/chroot): + +sudo /usr/sbin/in.tftpd \ + --foreground \ + --address 127.0.0.1:55123 \ + --user "$(whoami)" \ + --permissive \ + --create \ + --secure \ + "$(pwd)" + +#\ + +./: ../../bbot/exe{bbot-worker} test{bootstrap startup build} +dir{./}: test = ../../bbot/exe{bbot-worker} + +include ../../bbot/ diff --git a/tests/worker/startup.test b/tests/worker/startup.test new file mode 100644 index 0000000..d13e6c5 --- /dev/null +++ b/tests/worker/startup.test @@ -0,0 +1,101 @@ +# file : tests/worker/startup.test +# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +# license : TBC; see accompanying LICENSE file + +# Note: requires TFTP server (see buildfile). + +test.options = --startup +tftp = 127.0.0.1:55123/test + +: fail-download +: +$* --tftp-host "$tftp/$@" 2>>"EOE" &manifest != 0; + curl: \(68\) TFTP: File Not Found + error: unable to download task manifest from tftp://$tftp/$@/manifest: input/output error + EOE +diff -u - manifest <<EOO + : 1 + name: unknown + version: 0 + status: abnormal + EOO + +: fail-manifest +: +cat <': 1' >=manifest; +$* --tftp-host "$tftp/$@" 2>>"EOE" != 0; + error: invalid task manifest: manifest:2:1: no task package name specified + EOE +diff -u - manifest <<EOO + : 1 + name: unknown + version: 0 + status: abnormal + EOO + +: fail-env-search +: +cat <<EOI >=manifest; + : 1 + name: libhello + version: 1.2.3 + repository: https://pkg.example.org/1/ + machine: linux-gcc + EOI +$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE" != 0; + error: no environment setup executable in $representation($~) for target '' + EOE +diff -u - manifest <<EOO + : 1 + name: libhello + version: 1.2.3 + status: abnormal + EOO + +: fail-env-run +: +cat <<EOI >=x86_64-linux-gnu; + #!/bin/sh + echo "environment setup failed" + exit 1 + EOI +chmod ugo+x x86_64-linux-gnu; +cat <<EOI >=manifest; + : 1 + name: libhello + version: 1.2.3 + repository: https://pkg.example.org/1/ + machine: linux-gcc + target: x86_64-linux-gnu + EOI +$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE" != 0; + environment setup failed + error: process $~/x86_64-linux-gnu terminated with non-zero exit code + EOE +diff -u - manifest <<EOO + : 1 + name: libhello + version: 1.2.3 + status: abnormal + EOO + +: pass +: +cat <<EOI >=x86_64-linux-gnu; + #!/bin/sh + echo "$1" + echo "$2" + EOI +chmod ugo+x x86_64-linux-gnu; +cat <<EOI >=manifest; + : 1 + name: libhello + version: 1.2.3 + repository: https://pkg.example.org/1/ + machine: linux-gcc + target: x86_64-linux-gnu + EOI +$* --environment $~ --tftp-host "$tftp/$@" 2>>"EOE" + x86_64-linux-gnu + $0 + EOE diff --git a/unit-tests/bootstrap-manifest/driver.cxx b/unit-tests/bootstrap-manifest/driver.cxx index 18bf7b6..46e64bc 100644 --- a/unit-tests/bootstrap-manifest/driver.cxx +++ b/unit-tests/bootstrap-manifest/driver.cxx @@ -12,6 +12,7 @@ #include <bbot/utility> #include <bbot/bootstrap-manifest> +#include <bbot/machine-manifest> using namespace std; using namespace butl; |