From ea525c3177992785c4e7962b1584111357c783a4 Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Tue, 10 Nov 2026 16:42:14 -0300 Subject: [PATCH 1/2] truenas-appliance-runner: set the guest's clock after a revert loadvm restores the appliance's memory, its clock included, so after a revert the appliance runs minutes behind the runner until it catches up, and it has no NTP to catch up with. A suite's two-factor codes, tokens and timestamps, made on the runner's clock, are then refused or wrong (truenas-connect/cui#79's two-factor spec after a retry). qemu now gives the guest agent a channel (virtio-serial, org.qemu.guest_agent.0 on $DIR/qga.sock); TrueNAS ships qemu-ga, which starts once the port appears. tnctl's set-time syncs the channel with guest-sync, sets the clock with guest-set-time, and checks it took. revert runs it after loadvm and warns, without failing, if it cannot; probe checks the clock after its revert and fails if it is over 5 s off. --- .../truenas-appliance-runner/appliance.sh | 32 ++++++-- .../truenas-appliance-runner/test_tnctl.py | 82 +++++++++++++++++++ .../actions/truenas-appliance-runner/tnctl.py | 72 +++++++++++++++- README.md | 7 ++ 4 files changed, 187 insertions(+), 6 deletions(-) diff --git a/.github/actions/truenas-appliance-runner/appliance.sh b/.github/actions/truenas-appliance-runner/appliance.sh index cf51f4c..d00f244 100755 --- a/.github/actions/truenas-appliance-runner/appliance.sh +++ b/.github/actions/truenas-appliance-runner/appliance.sh @@ -12,7 +12,8 @@ # as the baseline; KEY=value on stdout # appliance.sh snapshot save the running appliance, memory and # disks, as -# appliance.sh revert back to , in seconds, no boot +# appliance.sh revert back to , in seconds, no boot, with +# the guest's clock set to now # appliance.sh probe [baseline] change it, revert, check the change went # appliance.sh boot-report where the boot's time went: systemd's # critical chain and slowest units, and @@ -63,6 +64,9 @@ set -euo pipefail DIR="${APPLIANCE_DIR:-${RUNNER_TEMP:-/tmp}/truenas-appliance}" QMP="$DIR/qmp.sock" +# The guest agent's channel: TrueNAS ships qemu-ga, which starts once this +# port (org.qemu.guest_agent.0) appears. revert sets the clock through it. +QGA="$DIR/qga.sock" STATE="$DIR/state.env" hmp() { "${TNCTL[@]}" hmp "$QMP" "$*"; } @@ -195,6 +199,9 @@ start() { -nic "user,model=virtio-net-pci$(hostfwd 80 "$http")$(hostfwd 443 "$https")$(hostfwd 22 "$sshp")" \ "${nics[@]}" \ -device virtio-rng-pci \ + -device virtio-serial-pci,id=vser0 \ + -chardev "socket,id=qga0,path=$QGA,server=on,wait=off" \ + -device "virtserialport,bus=vser0.0,chardev=qga0,name=org.qemu.guest_agent.0" \ -qmp "unix:$QMP,server=on,wait=off" \ -display none -serial "file:$DIR/console.log" \ -daemonize -pidfile "$DIR/qemu.pid" @@ -262,8 +269,10 @@ snapshot() { } # revert: loadvm restores memory and disks as they were, so the API is up -# the moment it returns; its connections from before are gone, and its -# clock is the snapshot's until it catches up. +# the moment it returns; its connections from before are gone. It restores +# the clock as well, minutes behind by now, so the guest agent sets it to +# this machine's: a suite's two-factor codes, tokens and timestamps are +# made on that clock. A clock left behind is a warning, not a failed revert. revert() { local name="${1:?revert }" running || die "no appliance running" @@ -275,6 +284,9 @@ revert() { die "no snapshot $name to revert to; the appliance runs on as it was" fi resume + if ! "${TNCTL[@]}" set-time "$QGA"; then + echo "::warning::truenas-appliance revert: the appliance's clock is the snapshot's, behind this machine's" >&2 + fi "${TNCTL[@]}" wait "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" --timeout 120 >/dev/null awk -v a="$t0" -v b="$(date +%s.%N)" 'BEGIN {printf "%.1f\n", b - a}' } @@ -359,7 +371,7 @@ cpu_seconds() { # over the API, the appliance reverted to , and the group must # be gone; KEY=value on stdout. probe() { - local baseline="${1:-fresh-install}" took left + local baseline="${1:-fresh-install}" took left skew state call() { "${TNCTL[@]}" call "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" "$@"; } call group.create '{"name": "e2e_probe"}' >/dev/null @@ -368,7 +380,17 @@ probe() { took="$(revert "$baseline")" left="$(call group.query '[["name", "=", "e2e_probe"]]' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')" [ "$left" = 0 ] || die "probe: after reverting to $baseline the group is still there" - say "revert to $baseline took ${took}s, and undid the change" + # The clock as middleware reads it ({"$date": ms} on the wire), against this machine's. + # shellcheck disable=SC2016 # $date is the key's name, not a shell variable + skew="$(call system.info | python3 -c ' +import json, sys, time +from datetime import datetime +d = json.load(sys.stdin)["datetime"] +ms = d["$date"] if isinstance(d, dict) else datetime.fromisoformat(d).timestamp() * 1000 +print(f"{ms / 1000 - time.time():.1f}")')" + python3 -c "import sys; sys.exit(abs(float(sys.argv[1])) > 5)" "$skew" \ + || die "probe: after reverting to $baseline the appliance's clock is ${skew}s off" + say "revert to $baseline took ${took}s, undid the change, and left the clock ${skew}s off" echo "APPLIANCE_REVERT_SECONDS=$took" } diff --git a/.github/actions/truenas-appliance-runner/test_tnctl.py b/.github/actions/truenas-appliance-runner/test_tnctl.py index 99c8844..01611f7 100644 --- a/.github/actions/truenas-appliance-runner/test_tnctl.py +++ b/.github/actions/truenas-appliance-runner/test_tnctl.py @@ -202,6 +202,88 @@ def serve(): self.assertEqual(e.exception.code, 1) +class FakeAgent: + """qemu-ga on a unix socket: answers guest-sync, guest-get-time and + guest-set-time, with a stale reply already in the channel, as a loadvm + can leave one.""" + + def __init__(self, clock_ns, set_takes=True): + self.clock_ns = clock_ns + self.set_takes = set_takes + self.seen = [] + self.path = os.path.join(tempfile.mkdtemp(), "qga") + self.srv = socket.socket(socket.AF_UNIX) + self.srv.bind(self.path) + self.srv.listen() + threading.Thread(target=self.serve, daemon=True).start() + + def serve(self): + while True: + try: + conn, _ = self.srv.accept() + except OSError: + return + f = conn.makefile("rwb") + f.write(b'\xff{"return": 7}\n') + f.flush() + for line in f: + msg = json.loads(line) + self.seen.append(msg) + cmd, args = msg["execute"], msg.get("arguments", {}) + if cmd == "guest-sync": + out = {"return": args["id"]} + elif cmd == "guest-get-time": + out = {"return": self.clock_ns} + elif cmd == "guest-set-time": + if self.set_takes: + self.clock_ns = args["time"] + out = {"return": {}} + else: + out = {"error": {"class": "CommandNotFound", "desc": cmd}} + f.write(json.dumps(out).encode() + b"\n") + f.flush() + conn.close() + + def close(self): + self.srv.close() + + +class TestGuestAgent(unittest.TestCase): + def test_sync_skips_what_was_left_in_the_channel(self): + agent = FakeAgent(clock_ns=123) + self.addCleanup(agent.close) + self.assertEqual(tnctl.qga(agent.path, "guest-get-time"), 123) + self.assertEqual([m["execute"] for m in agent.seen], ["guest-sync", "guest-get-time"]) + with self.assertRaises(tnctl.RPCError): + tnctl.qga(agent.path, "guest-nope") + + def test_set_time_sets_the_guest_clock_to_this_machines(self): + behind = time.time_ns() - 300 * 10**9 + agent = FakeAgent(clock_ns=behind) + self.addCleanup(agent.close) + err = io.StringIO() + with contextlib.redirect_stderr(err): + tnctl.cmd_set_time(args(socket=agent.path)) + self.assertLess(abs(agent.clock_ns - time.time_ns()), tnctl.CLOCK_TOLERANCE_NS) + self.assertIn("it was -300.0s off", err.getvalue()) + + def test_set_time_fails_when_the_clock_does_not_take_it(self): + agent = FakeAgent(clock_ns=time.time_ns() - 300 * 10**9, set_takes=False) + self.addCleanup(agent.close) + with self.assertRaises(SystemExit) as e: + tnctl.cmd_set_time(args(socket=agent.path)) + self.assertIn("still", str(e.exception.code)) + + def test_no_agent_is_a_timeout(self): + path = os.path.join(tempfile.mkdtemp(), "qga") + srv = socket.socket(socket.AF_UNIX) + srv.bind(path) + srv.listen() + self.addCleanup(srv.close) + with self.assertRaises(TimeoutError): + tnctl.qga(path, "guest-get-time", timeout=0.5) + + class TestTailTs(unittest.TestCase): def test_lines_are_timed_and_cleaned(self): d = tempfile.mkdtemp() diff --git a/.github/actions/truenas-appliance-runner/tnctl.py b/.github/actions/truenas-appliance-runner/tnctl.py index 1a9a812..f4201c5 100755 --- a/.github/actions/truenas-appliance-runner/tnctl.py +++ b/.github/actions/truenas-appliance-runner/tnctl.py @@ -3,7 +3,8 @@ Standard library only, so it runs on any runner with a python3: the installer's and the middleware's JSON-RPC 2.0 over a websocket, and qemu's -QMP over a unix socket. appliance.sh calls it; the image build does too. +QMP and the guest agent over unix sockets. appliance.sh calls it; the image +build does too. tnctl.py install URL [--disk NAME] Drive the installer ISO at ws://.../ws: install to the first disk it @@ -39,6 +40,13 @@ tnctl.py hmp SOCKET "savevm fresh-install" One QMP command, or a human monitor command through QMP. + tnctl.py set-time SOCKET [--timeout S] + Set the guest's clock to this machine's through the guest agent + (qemu-ga, which TrueNAS ships) on SOCKET, and check it took. After a + loadvm the guest's clock is the snapshot's: minutes behind, and two- + factor codes, token expiry and the like are wrong until it catches up. + Logs how far off it was. + URL is ws://host:port; the path (/ws for the installer, /api/current for the middleware) is added here. """ @@ -436,6 +444,67 @@ def send(cmd, args=None): s.close() +def qga(path, command, arguments=None, timeout=10): + """One guest agent command over the virtserialport's socket. + + The agent has no greeting and keeps no session. guest-sync with a fresh + id comes first, and replies until that id are dropped: whatever sits in + the channel (a reply to a client that left, or the buffers a loadvm + restored) is not taken for this command's. A missing or stopped agent + is a timeout. + """ + s = socket.socket(socket.AF_UNIX) + s.settimeout(timeout) + s.connect(path) + f = s.makefile("rwb") + + def send(cmd, args=None): + f.write(json.dumps({"execute": cmd, **({"arguments": args} if args is not None else {})}).encode() + b"\n") + f.flush() + + def read(): + line = f.readline() + if not line: + raise Closed("qga closed") + try: + # guest-sync-delimited puts 0xFF before its reply; a stale one may be here. + return json.loads(line.strip(b"\xff \r\n")) + except ValueError: + return None + + try: + sync = int.from_bytes(os.urandom(4), "big") >> 1 + send("guest-sync", {"id": sync}) + while True: + r = read() + if isinstance(r, dict) and r.get("return") == sync: + break + send(command, arguments) + while True: + r = read() + if not isinstance(r, dict): + continue + if "error" in r: + raise RPCError(command, r["error"]) + return r.get("return") + finally: + s.close() + + +# How far the guest's clock may be from this machine's once set: the two +# calls' own round trips, with room. +CLOCK_TOLERANCE_NS = 2 * 10**9 + + +def cmd_set_time(a): + was = qga(a.socket, "guest-get-time", timeout=a.timeout) - time.time_ns() + qga(a.socket, "guest-set-time", {"time": time.time_ns()}, timeout=a.timeout) + off = qga(a.socket, "guest-get-time", timeout=a.timeout) - time.time_ns() + if abs(off) > CLOCK_TOLERANCE_NS: + raise SystemExit(f"tnctl set-time: the guest's clock is still {off / 1e9:+.1f}s off") + log(f"guest clock set: it was {was / 1e9:+.1f}s off") + + def cmd_qmp(a): r = qmp(a.socket, a.command, json.loads(a.args) if a.args else None) print(json.dumps(r, indent=2)) @@ -474,6 +543,7 @@ def p(name, fn, *args, timeout=600): p("call", cmd_call, "url", "user", "password", "method", timeout=120).add_argument("params", nargs="*") p("qmp", cmd_qmp, "socket", "command").add_argument("args", nargs="?") p("hmp", cmd_hmp, "socket", "line") + p("set-time", cmd_set_time, "socket", timeout=10) a = ap.parse_args() try: a.fn(a) diff --git a/README.md b/README.md index f3e886d..d225467 100644 --- a/README.md +++ b/README.md @@ -575,6 +575,13 @@ Outputs: `host`, `host-http`, `username`, `password` (masked), `ssh`, time went, from systemd and middleware's own log, over ssh (it leaves ssh on with a key and sudo for the admin, so revert afterwards if that matters). +`revert` restores the appliance's memory, and with it the clock it had when +the snapshot was taken. It then sets the clock to the runner's through the +guest agent (TrueNAS ships `qemu-ga`; qemu gives it a channel), so two-factor +codes, tokens and timestamps a suite makes on the runner's clock hold. If +that fails, the revert still succeeds with a warning that the clock is +behind; `probe` fails instead. + **Run the suite on the runner, not in a container.** `truenas-appliance`, qemu and its monitor socket are the runner's, so a spec that reverts the appliance from inside Playwright's image (or any container) cannot reach From 32e899a0d5830abbc0247b5e0f732847fa9acc1d Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Tue, 10 Nov 2026 16:57:42 -0300 Subject: [PATCH 2/2] truenas-appliance-runner: take a set clock when the guest has no hwclock On a TrueNAS appliance guest-set-time sets the system clock and then fails: qemu-ga runs /sbin/hwclock to copy it to the hardware clock, and TrueNAS does not ship hwclock. revert warned with the clock 0.4 s off (truenas-connect/cui#82). The hardware clock does not matter here, so the clock check after the call decides, and the call's error is reported only when the clock did not take or the error is about something else. qga() also closes its socket's file: socket.close() leaves the connection open while the file lives, an error's traceback kept it alive, and qemu's chardev serves one client at a time, so the next call hung. --- .../truenas-appliance-runner/test_tnctl.py | 18 +++++++++++++++++- .../actions/truenas-appliance-runner/tnctl.py | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.github/actions/truenas-appliance-runner/test_tnctl.py b/.github/actions/truenas-appliance-runner/test_tnctl.py index 01611f7..47fd6b8 100644 --- a/.github/actions/truenas-appliance-runner/test_tnctl.py +++ b/.github/actions/truenas-appliance-runner/test_tnctl.py @@ -207,9 +207,10 @@ class FakeAgent: guest-set-time, with a stale reply already in the channel, as a loadvm can leave one.""" - def __init__(self, clock_ns, set_takes=True): + def __init__(self, clock_ns, set_takes=True, hwclock=True): self.clock_ns = clock_ns self.set_takes = set_takes + self.hwclock = hwclock self.seen = [] self.path = os.path.join(tempfile.mkdtemp(), "qga") self.srv = socket.socket(socket.AF_UNIX) @@ -238,6 +239,11 @@ def serve(self): if self.set_takes: self.clock_ns = args["time"] out = {"return": {}} + if not self.hwclock: + # As TrueNAS answers: the system clock is set, then hwclock is missing. + out = {"error": {"class": "GenericError", "desc": ( + "child process has failed to set hardware clock to system time: " + "failed to exec '/sbin/hwclock': No such file or directory\n")}} else: out = {"error": {"class": "CommandNotFound", "desc": cmd}} f.write(json.dumps(out).encode() + b"\n") @@ -267,6 +273,16 @@ def test_set_time_sets_the_guest_clock_to_this_machines(self): self.assertLess(abs(agent.clock_ns - time.time_ns()), tnctl.CLOCK_TOLERANCE_NS) self.assertIn("it was -300.0s off", err.getvalue()) + def test_set_time_takes_a_guest_without_hwclock(self): + agent = FakeAgent(clock_ns=time.time_ns() - 300 * 10**9, hwclock=False) + self.addCleanup(agent.close) + err = io.StringIO() + with contextlib.redirect_stderr(err): + tnctl.cmd_set_time(args(socket=agent.path)) + self.assertLess(abs(agent.clock_ns - time.time_ns()), tnctl.CLOCK_TOLERANCE_NS) + self.assertNotIn("hwclock", err.getvalue()) + self.assertIn("it was -300.0s off", err.getvalue()) + def test_set_time_fails_when_the_clock_does_not_take_it(self): agent = FakeAgent(clock_ns=time.time_ns() - 300 * 10**9, set_takes=False) self.addCleanup(agent.close) diff --git a/.github/actions/truenas-appliance-runner/tnctl.py b/.github/actions/truenas-appliance-runner/tnctl.py index f4201c5..b461442 100755 --- a/.github/actions/truenas-appliance-runner/tnctl.py +++ b/.github/actions/truenas-appliance-runner/tnctl.py @@ -488,6 +488,9 @@ def read(): raise RPCError(command, r["error"]) return r.get("return") finally: + # Both: the socket stays open while its file does, and qemu's chardev + # serves one client at a time, so the next call would wait forever. + f.close() s.close() @@ -498,10 +501,22 @@ def read(): def cmd_set_time(a): was = qga(a.socket, "guest-get-time", timeout=a.timeout) - time.time_ns() - qga(a.socket, "guest-set-time", {"time": time.time_ns()}, timeout=a.timeout) + # qemu-ga sets the system clock, then runs hwclock to copy it to the + # hardware clock and fails the call if that does. TrueNAS has no + # /sbin/hwclock, so the call fails with the system clock already set. + # The hardware clock does not matter here (loadvm restores it, and the + # guest reads it only at boot): the system clock decides, below. + refused = None + try: + qga(a.socket, "guest-set-time", {"time": time.time_ns()}, timeout=a.timeout) + except RPCError as e: + refused = e off = qga(a.socket, "guest-get-time", timeout=a.timeout) - time.time_ns() if abs(off) > CLOCK_TOLERANCE_NS: - raise SystemExit(f"tnctl set-time: the guest's clock is still {off / 1e9:+.1f}s off") + why = f" ({refused})" if refused else "" + raise SystemExit(f"tnctl set-time: the guest's clock is still {off / 1e9:+.1f}s off{why}") + if refused and "hwclock" not in str(refused): + log(f"guest-set-time answered {refused}, but the clock took") log(f"guest clock set: it was {was / 1e9:+.1f}s off")