diff --git a/.github/actions/truenas-appliance-runner/appliance.sh b/.github/actions/truenas-appliance-runner/appliance.sh index cf51f4c..d00f244 100755 --- a/.github/actions/truenas-appliance-runner/appliance.sh +++ b/.github/actions/truenas-appliance-runner/appliance.sh @@ -12,7 +12,8 @@ # as the baseline; KEY=value on stdout # appliance.sh snapshot save the running appliance, memory and # disks, as -# appliance.sh revert back to , in seconds, no boot +# appliance.sh revert back to , in seconds, no boot, with +# the guest's clock set to now # appliance.sh probe [baseline] change it, revert, check the change went # appliance.sh boot-report where the boot's time went: systemd's # critical chain and slowest units, and @@ -63,6 +64,9 @@ set -euo pipefail DIR="${APPLIANCE_DIR:-${RUNNER_TEMP:-/tmp}/truenas-appliance}" QMP="$DIR/qmp.sock" +# The guest agent's channel: TrueNAS ships qemu-ga, which starts once this +# port (org.qemu.guest_agent.0) appears. revert sets the clock through it. +QGA="$DIR/qga.sock" STATE="$DIR/state.env" hmp() { "${TNCTL[@]}" hmp "$QMP" "$*"; } @@ -195,6 +199,9 @@ start() { -nic "user,model=virtio-net-pci$(hostfwd 80 "$http")$(hostfwd 443 "$https")$(hostfwd 22 "$sshp")" \ "${nics[@]}" \ -device virtio-rng-pci \ + -device virtio-serial-pci,id=vser0 \ + -chardev "socket,id=qga0,path=$QGA,server=on,wait=off" \ + -device "virtserialport,bus=vser0.0,chardev=qga0,name=org.qemu.guest_agent.0" \ -qmp "unix:$QMP,server=on,wait=off" \ -display none -serial "file:$DIR/console.log" \ -daemonize -pidfile "$DIR/qemu.pid" @@ -262,8 +269,10 @@ snapshot() { } # revert: loadvm restores memory and disks as they were, so the API is up -# the moment it returns; its connections from before are gone, and its -# clock is the snapshot's until it catches up. +# the moment it returns; its connections from before are gone. It restores +# the clock as well, minutes behind by now, so the guest agent sets it to +# this machine's: a suite's two-factor codes, tokens and timestamps are +# made on that clock. A clock left behind is a warning, not a failed revert. revert() { local name="${1:?revert }" running || die "no appliance running" @@ -275,6 +284,9 @@ revert() { die "no snapshot $name to revert to; the appliance runs on as it was" fi resume + if ! "${TNCTL[@]}" set-time "$QGA"; then + echo "::warning::truenas-appliance revert: the appliance's clock is the snapshot's, behind this machine's" >&2 + fi "${TNCTL[@]}" wait "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" --timeout 120 >/dev/null awk -v a="$t0" -v b="$(date +%s.%N)" 'BEGIN {printf "%.1f\n", b - a}' } @@ -359,7 +371,7 @@ cpu_seconds() { # over the API, the appliance reverted to , and the group must # be gone; KEY=value on stdout. probe() { - local baseline="${1:-fresh-install}" took left + local baseline="${1:-fresh-install}" took left skew state call() { "${TNCTL[@]}" call "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" "$@"; } call group.create '{"name": "e2e_probe"}' >/dev/null @@ -368,7 +380,17 @@ probe() { took="$(revert "$baseline")" left="$(call group.query '[["name", "=", "e2e_probe"]]' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')" [ "$left" = 0 ] || die "probe: after reverting to $baseline the group is still there" - say "revert to $baseline took ${took}s, and undid the change" + # The clock as middleware reads it ({"$date": ms} on the wire), against this machine's. + # shellcheck disable=SC2016 # $date is the key's name, not a shell variable + skew="$(call system.info | python3 -c ' +import json, sys, time +from datetime import datetime +d = json.load(sys.stdin)["datetime"] +ms = d["$date"] if isinstance(d, dict) else datetime.fromisoformat(d).timestamp() * 1000 +print(f"{ms / 1000 - time.time():.1f}")')" + python3 -c "import sys; sys.exit(abs(float(sys.argv[1])) > 5)" "$skew" \ + || die "probe: after reverting to $baseline the appliance's clock is ${skew}s off" + say "revert to $baseline took ${took}s, undid the change, and left the clock ${skew}s off" echo "APPLIANCE_REVERT_SECONDS=$took" } diff --git a/.github/actions/truenas-appliance-runner/test_tnctl.py b/.github/actions/truenas-appliance-runner/test_tnctl.py index 99c8844..47fd6b8 100644 --- a/.github/actions/truenas-appliance-runner/test_tnctl.py +++ b/.github/actions/truenas-appliance-runner/test_tnctl.py @@ -202,6 +202,104 @@ def serve(): self.assertEqual(e.exception.code, 1) +class FakeAgent: + """qemu-ga on a unix socket: answers guest-sync, guest-get-time and + guest-set-time, with a stale reply already in the channel, as a loadvm + can leave one.""" + + def __init__(self, clock_ns, set_takes=True, hwclock=True): + self.clock_ns = clock_ns + self.set_takes = set_takes + self.hwclock = hwclock + self.seen = [] + self.path = os.path.join(tempfile.mkdtemp(), "qga") + self.srv = socket.socket(socket.AF_UNIX) + self.srv.bind(self.path) + self.srv.listen() + threading.Thread(target=self.serve, daemon=True).start() + + def serve(self): + while True: + try: + conn, _ = self.srv.accept() + except OSError: + return + f = conn.makefile("rwb") + f.write(b'\xff{"return": 7}\n') + f.flush() + for line in f: + msg = json.loads(line) + self.seen.append(msg) + cmd, args = msg["execute"], msg.get("arguments", {}) + if cmd == "guest-sync": + out = {"return": args["id"]} + elif cmd == "guest-get-time": + out = {"return": self.clock_ns} + elif cmd == "guest-set-time": + if self.set_takes: + self.clock_ns = args["time"] + out = {"return": {}} + if not self.hwclock: + # As TrueNAS answers: the system clock is set, then hwclock is missing. + out = {"error": {"class": "GenericError", "desc": ( + "child process has failed to set hardware clock to system time: " + "failed to exec '/sbin/hwclock': No such file or directory\n")}} + else: + out = {"error": {"class": "CommandNotFound", "desc": cmd}} + f.write(json.dumps(out).encode() + b"\n") + f.flush() + conn.close() + + def close(self): + self.srv.close() + + +class TestGuestAgent(unittest.TestCase): + def test_sync_skips_what_was_left_in_the_channel(self): + agent = FakeAgent(clock_ns=123) + self.addCleanup(agent.close) + self.assertEqual(tnctl.qga(agent.path, "guest-get-time"), 123) + self.assertEqual([m["execute"] for m in agent.seen], ["guest-sync", "guest-get-time"]) + with self.assertRaises(tnctl.RPCError): + tnctl.qga(agent.path, "guest-nope") + + def test_set_time_sets_the_guest_clock_to_this_machines(self): + behind = time.time_ns() - 300 * 10**9 + agent = FakeAgent(clock_ns=behind) + self.addCleanup(agent.close) + err = io.StringIO() + with contextlib.redirect_stderr(err): + tnctl.cmd_set_time(args(socket=agent.path)) + self.assertLess(abs(agent.clock_ns - time.time_ns()), tnctl.CLOCK_TOLERANCE_NS) + self.assertIn("it was -300.0s off", err.getvalue()) + + def test_set_time_takes_a_guest_without_hwclock(self): + agent = FakeAgent(clock_ns=time.time_ns() - 300 * 10**9, hwclock=False) + self.addCleanup(agent.close) + err = io.StringIO() + with contextlib.redirect_stderr(err): + tnctl.cmd_set_time(args(socket=agent.path)) + self.assertLess(abs(agent.clock_ns - time.time_ns()), tnctl.CLOCK_TOLERANCE_NS) + self.assertNotIn("hwclock", err.getvalue()) + self.assertIn("it was -300.0s off", err.getvalue()) + + def test_set_time_fails_when_the_clock_does_not_take_it(self): + agent = FakeAgent(clock_ns=time.time_ns() - 300 * 10**9, set_takes=False) + self.addCleanup(agent.close) + with self.assertRaises(SystemExit) as e: + tnctl.cmd_set_time(args(socket=agent.path)) + self.assertIn("still", str(e.exception.code)) + + def test_no_agent_is_a_timeout(self): + path = os.path.join(tempfile.mkdtemp(), "qga") + srv = socket.socket(socket.AF_UNIX) + srv.bind(path) + srv.listen() + self.addCleanup(srv.close) + with self.assertRaises(TimeoutError): + tnctl.qga(path, "guest-get-time", timeout=0.5) + + class TestTailTs(unittest.TestCase): def test_lines_are_timed_and_cleaned(self): d = tempfile.mkdtemp() diff --git a/.github/actions/truenas-appliance-runner/tnctl.py b/.github/actions/truenas-appliance-runner/tnctl.py index 1a9a812..b461442 100755 --- a/.github/actions/truenas-appliance-runner/tnctl.py +++ b/.github/actions/truenas-appliance-runner/tnctl.py @@ -3,7 +3,8 @@ Standard library only, so it runs on any runner with a python3: the installer's and the middleware's JSON-RPC 2.0 over a websocket, and qemu's -QMP over a unix socket. appliance.sh calls it; the image build does too. +QMP and the guest agent over unix sockets. appliance.sh calls it; the image +build does too. tnctl.py install URL [--disk NAME] Drive the installer ISO at ws://.../ws: install to the first disk it @@ -39,6 +40,13 @@ tnctl.py hmp SOCKET "savevm fresh-install" One QMP command, or a human monitor command through QMP. + tnctl.py set-time SOCKET [--timeout S] + Set the guest's clock to this machine's through the guest agent + (qemu-ga, which TrueNAS ships) on SOCKET, and check it took. After a + loadvm the guest's clock is the snapshot's: minutes behind, and two- + factor codes, token expiry and the like are wrong until it catches up. + Logs how far off it was. + URL is ws://host:port; the path (/ws for the installer, /api/current for the middleware) is added here. """ @@ -436,6 +444,82 @@ def send(cmd, args=None): s.close() +def qga(path, command, arguments=None, timeout=10): + """One guest agent command over the virtserialport's socket. + + The agent has no greeting and keeps no session. guest-sync with a fresh + id comes first, and replies until that id are dropped: whatever sits in + the channel (a reply to a client that left, or the buffers a loadvm + restored) is not taken for this command's. A missing or stopped agent + is a timeout. + """ + s = socket.socket(socket.AF_UNIX) + s.settimeout(timeout) + s.connect(path) + f = s.makefile("rwb") + + def send(cmd, args=None): + f.write(json.dumps({"execute": cmd, **({"arguments": args} if args is not None else {})}).encode() + b"\n") + f.flush() + + def read(): + line = f.readline() + if not line: + raise Closed("qga closed") + try: + # guest-sync-delimited puts 0xFF before its reply; a stale one may be here. + return json.loads(line.strip(b"\xff \r\n")) + except ValueError: + return None + + try: + sync = int.from_bytes(os.urandom(4), "big") >> 1 + send("guest-sync", {"id": sync}) + while True: + r = read() + if isinstance(r, dict) and r.get("return") == sync: + break + send(command, arguments) + while True: + r = read() + if not isinstance(r, dict): + continue + if "error" in r: + raise RPCError(command, r["error"]) + return r.get("return") + finally: + # Both: the socket stays open while its file does, and qemu's chardev + # serves one client at a time, so the next call would wait forever. + f.close() + s.close() + + +# How far the guest's clock may be from this machine's once set: the two +# calls' own round trips, with room. +CLOCK_TOLERANCE_NS = 2 * 10**9 + + +def cmd_set_time(a): + was = qga(a.socket, "guest-get-time", timeout=a.timeout) - time.time_ns() + # qemu-ga sets the system clock, then runs hwclock to copy it to the + # hardware clock and fails the call if that does. TrueNAS has no + # /sbin/hwclock, so the call fails with the system clock already set. + # The hardware clock does not matter here (loadvm restores it, and the + # guest reads it only at boot): the system clock decides, below. + refused = None + try: + qga(a.socket, "guest-set-time", {"time": time.time_ns()}, timeout=a.timeout) + except RPCError as e: + refused = e + off = qga(a.socket, "guest-get-time", timeout=a.timeout) - time.time_ns() + if abs(off) > CLOCK_TOLERANCE_NS: + why = f" ({refused})" if refused else "" + raise SystemExit(f"tnctl set-time: the guest's clock is still {off / 1e9:+.1f}s off{why}") + if refused and "hwclock" not in str(refused): + log(f"guest-set-time answered {refused}, but the clock took") + log(f"guest clock set: it was {was / 1e9:+.1f}s off") + + def cmd_qmp(a): r = qmp(a.socket, a.command, json.loads(a.args) if a.args else None) print(json.dumps(r, indent=2)) @@ -474,6 +558,7 @@ def p(name, fn, *args, timeout=600): p("call", cmd_call, "url", "user", "password", "method", timeout=120).add_argument("params", nargs="*") p("qmp", cmd_qmp, "socket", "command").add_argument("args", nargs="?") p("hmp", cmd_hmp, "socket", "line") + p("set-time", cmd_set_time, "socket", timeout=10) a = ap.parse_args() try: a.fn(a) diff --git a/README.md b/README.md index f3e886d..d225467 100644 --- a/README.md +++ b/README.md @@ -575,6 +575,13 @@ Outputs: `host`, `host-http`, `username`, `password` (masked), `ssh`, time went, from systemd and middleware's own log, over ssh (it leaves ssh on with a key and sudo for the admin, so revert afterwards if that matters). +`revert` restores the appliance's memory, and with it the clock it had when +the snapshot was taken. It then sets the clock to the runner's through the +guest agent (TrueNAS ships `qemu-ga`; qemu gives it a channel), so two-factor +codes, tokens and timestamps a suite makes on the runner's clock hold. If +that fails, the revert still succeeds with a warning that the clock is +behind; `probe` fails instead. + **Run the suite on the runner, not in a container.** `truenas-appliance`, qemu and its monitor socket are the runner's, so a spec that reverts the appliance from inside Playwright's image (or any container) cannot reach