Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 27 additions & 5 deletions .github/actions/truenas-appliance-runner/appliance.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
# as the baseline; KEY=value on stdout
# appliance.sh snapshot <name> save the running appliance, memory and
# disks, as <name>
# appliance.sh revert <name> back to <name>, in seconds, no boot
# appliance.sh revert <name> back to <name>, in seconds, no boot, with
# the guest's clock set to now
# appliance.sh probe [baseline] change it, revert, check the change went
# appliance.sh boot-report where the boot's time went: systemd's
# critical chain and slowest units, and
Expand Down Expand Up @@ -63,6 +64,9 @@ set -euo pipefail

DIR="${APPLIANCE_DIR:-${RUNNER_TEMP:-/tmp}/truenas-appliance}"
QMP="$DIR/qmp.sock"
# The guest agent's channel: TrueNAS ships qemu-ga, which starts once this
# port (org.qemu.guest_agent.0) appears. revert sets the clock through it.
QGA="$DIR/qga.sock"
STATE="$DIR/state.env"

hmp() { "${TNCTL[@]}" hmp "$QMP" "$*"; }
Expand Down Expand Up @@ -195,6 +199,9 @@ start() {
-nic "user,model=virtio-net-pci$(hostfwd 80 "$http")$(hostfwd 443 "$https")$(hostfwd 22 "$sshp")" \
"${nics[@]}" \
-device virtio-rng-pci \
-device virtio-serial-pci,id=vser0 \
-chardev "socket,id=qga0,path=$QGA,server=on,wait=off" \
-device "virtserialport,bus=vser0.0,chardev=qga0,name=org.qemu.guest_agent.0" \
-qmp "unix:$QMP,server=on,wait=off" \
-display none -serial "file:$DIR/console.log" \
-daemonize -pidfile "$DIR/qemu.pid"
Expand Down Expand Up @@ -262,8 +269,10 @@ snapshot() {
}

# revert: loadvm restores memory and disks as they were, so the API is up
# the moment it returns; its connections from before are gone, and its
# clock is the snapshot's until it catches up.
# the moment it returns; its connections from before are gone. It restores
# the clock as well, minutes behind by now, so the guest agent sets it to
# this machine's: a suite's two-factor codes, tokens and timestamps are
# made on that clock. A clock left behind is a warning, not a failed revert.
revert() {
local name="${1:?revert <name>}"
running || die "no appliance running"
Expand All @@ -275,6 +284,9 @@ revert() {
die "no snapshot $name to revert to; the appliance runs on as it was"
fi
resume
if ! "${TNCTL[@]}" set-time "$QGA"; then
echo "::warning::truenas-appliance revert: the appliance's clock is the snapshot's, behind this machine's" >&2
fi
"${TNCTL[@]}" wait "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" --timeout 120 >/dev/null
awk -v a="$t0" -v b="$(date +%s.%N)" 'BEGIN {printf "%.1f\n", b - a}'
}
Expand Down Expand Up @@ -359,7 +371,7 @@ cpu_seconds() {
# over the API, the appliance reverted to <baseline>, and the group must
# be gone; KEY=value on stdout.
probe() {
local baseline="${1:-fresh-install}" took left
local baseline="${1:-fresh-install}" took left skew
state
call() { "${TNCTL[@]}" call "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" "$@"; }
call group.create '{"name": "e2e_probe"}' >/dev/null
Expand All @@ -368,7 +380,17 @@ probe() {
took="$(revert "$baseline")"
left="$(call group.query '[["name", "=", "e2e_probe"]]' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')"
[ "$left" = 0 ] || die "probe: after reverting to $baseline the group is still there"
say "revert to $baseline took ${took}s, and undid the change"
# The clock as middleware reads it ({"$date": ms} on the wire), against this machine's.
# shellcheck disable=SC2016 # $date is the key's name, not a shell variable
skew="$(call system.info | python3 -c '
import json, sys, time
from datetime import datetime
d = json.load(sys.stdin)["datetime"]
ms = d["$date"] if isinstance(d, dict) else datetime.fromisoformat(d).timestamp() * 1000
print(f"{ms / 1000 - time.time():.1f}")')"
python3 -c "import sys; sys.exit(abs(float(sys.argv[1])) > 5)" "$skew" \
|| die "probe: after reverting to $baseline the appliance's clock is ${skew}s off"
say "revert to $baseline took ${took}s, undid the change, and left the clock ${skew}s off"
echo "APPLIANCE_REVERT_SECONDS=$took"
}

Expand Down
98 changes: 98 additions & 0 deletions .github/actions/truenas-appliance-runner/test_tnctl.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,104 @@ def serve():
self.assertEqual(e.exception.code, 1)


class FakeAgent:
"""qemu-ga on a unix socket: answers guest-sync, guest-get-time and
guest-set-time, with a stale reply already in the channel, as a loadvm
can leave one."""

def __init__(self, clock_ns, set_takes=True, hwclock=True):
self.clock_ns = clock_ns
self.set_takes = set_takes
self.hwclock = hwclock
self.seen = []
self.path = os.path.join(tempfile.mkdtemp(), "qga")
self.srv = socket.socket(socket.AF_UNIX)
self.srv.bind(self.path)
self.srv.listen()
threading.Thread(target=self.serve, daemon=True).start()

def serve(self):
while True:
try:
conn, _ = self.srv.accept()
except OSError:
return
f = conn.makefile("rwb")
f.write(b'\xff{"return": 7}\n')
f.flush()
for line in f:
msg = json.loads(line)
self.seen.append(msg)
cmd, args = msg["execute"], msg.get("arguments", {})
if cmd == "guest-sync":
out = {"return": args["id"]}
elif cmd == "guest-get-time":
out = {"return": self.clock_ns}
elif cmd == "guest-set-time":
if self.set_takes:
self.clock_ns = args["time"]
out = {"return": {}}
if not self.hwclock:
# As TrueNAS answers: the system clock is set, then hwclock is missing.
out = {"error": {"class": "GenericError", "desc": (
"child process has failed to set hardware clock to system time: "
"failed to exec '/sbin/hwclock': No such file or directory\n")}}
else:
out = {"error": {"class": "CommandNotFound", "desc": cmd}}
f.write(json.dumps(out).encode() + b"\n")
f.flush()
conn.close()

def close(self):
self.srv.close()


class TestGuestAgent(unittest.TestCase):
def test_sync_skips_what_was_left_in_the_channel(self):
agent = FakeAgent(clock_ns=123)
self.addCleanup(agent.close)
self.assertEqual(tnctl.qga(agent.path, "guest-get-time"), 123)
self.assertEqual([m["execute"] for m in agent.seen], ["guest-sync", "guest-get-time"])
with self.assertRaises(tnctl.RPCError):
tnctl.qga(agent.path, "guest-nope")

def test_set_time_sets_the_guest_clock_to_this_machines(self):
behind = time.time_ns() - 300 * 10**9
agent = FakeAgent(clock_ns=behind)
self.addCleanup(agent.close)
err = io.StringIO()
with contextlib.redirect_stderr(err):
tnctl.cmd_set_time(args(socket=agent.path))
self.assertLess(abs(agent.clock_ns - time.time_ns()), tnctl.CLOCK_TOLERANCE_NS)
self.assertIn("it was -300.0s off", err.getvalue())

def test_set_time_takes_a_guest_without_hwclock(self):
agent = FakeAgent(clock_ns=time.time_ns() - 300 * 10**9, hwclock=False)
self.addCleanup(agent.close)
err = io.StringIO()
with contextlib.redirect_stderr(err):
tnctl.cmd_set_time(args(socket=agent.path))
self.assertLess(abs(agent.clock_ns - time.time_ns()), tnctl.CLOCK_TOLERANCE_NS)
self.assertNotIn("hwclock", err.getvalue())
self.assertIn("it was -300.0s off", err.getvalue())

def test_set_time_fails_when_the_clock_does_not_take_it(self):
agent = FakeAgent(clock_ns=time.time_ns() - 300 * 10**9, set_takes=False)
self.addCleanup(agent.close)
with self.assertRaises(SystemExit) as e:
tnctl.cmd_set_time(args(socket=agent.path))
self.assertIn("still", str(e.exception.code))

def test_no_agent_is_a_timeout(self):
path = os.path.join(tempfile.mkdtemp(), "qga")
srv = socket.socket(socket.AF_UNIX)
srv.bind(path)
srv.listen()
self.addCleanup(srv.close)
with self.assertRaises(TimeoutError):
tnctl.qga(path, "guest-get-time", timeout=0.5)


class TestTailTs(unittest.TestCase):
def test_lines_are_timed_and_cleaned(self):
d = tempfile.mkdtemp()
Expand Down
87 changes: 86 additions & 1 deletion .github/actions/truenas-appliance-runner/tnctl.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@

Standard library only, so it runs on any runner with a python3: the
installer's and the middleware's JSON-RPC 2.0 over a websocket, and qemu's
QMP over a unix socket. appliance.sh calls it; the image build does too.
QMP and the guest agent over unix sockets. appliance.sh calls it; the image
build does too.

tnctl.py install URL [--disk NAME]
Drive the installer ISO at ws://.../ws: install to the first disk it
Expand Down Expand Up @@ -39,6 +40,13 @@
tnctl.py hmp SOCKET "savevm fresh-install"
One QMP command, or a human monitor command through QMP.

tnctl.py set-time SOCKET [--timeout S]
Set the guest's clock to this machine's through the guest agent
(qemu-ga, which TrueNAS ships) on SOCKET, and check it took. After a
loadvm the guest's clock is the snapshot's: minutes behind, and two-
factor codes, token expiry and the like are wrong until it catches up.
Logs how far off it was.

URL is ws://host:port; the path (/ws for the installer, /api/current for
the middleware) is added here.
"""
Expand Down Expand Up @@ -436,6 +444,82 @@ def send(cmd, args=None):
s.close()


def qga(path, command, arguments=None, timeout=10):
"""One guest agent command over the virtserialport's socket.

The agent has no greeting and keeps no session. guest-sync with a fresh
id comes first, and replies until that id are dropped: whatever sits in
the channel (a reply to a client that left, or the buffers a loadvm
restored) is not taken for this command's. A missing or stopped agent
is a timeout.
"""
s = socket.socket(socket.AF_UNIX)
s.settimeout(timeout)
s.connect(path)
f = s.makefile("rwb")

def send(cmd, args=None):
f.write(json.dumps({"execute": cmd, **({"arguments": args} if args is not None else {})}).encode() + b"\n")
f.flush()

def read():
line = f.readline()
if not line:
raise Closed("qga closed")
try:
# guest-sync-delimited puts 0xFF before its reply; a stale one may be here.
return json.loads(line.strip(b"\xff \r\n"))
except ValueError:
return None

try:
sync = int.from_bytes(os.urandom(4), "big") >> 1
send("guest-sync", {"id": sync})
while True:
r = read()
if isinstance(r, dict) and r.get("return") == sync:
break
send(command, arguments)
while True:
r = read()
if not isinstance(r, dict):
continue
if "error" in r:
raise RPCError(command, r["error"])
return r.get("return")
finally:
# Both: the socket stays open while its file does, and qemu's chardev
# serves one client at a time, so the next call would wait forever.
f.close()
s.close()


# How far the guest's clock may be from this machine's once set: the two
# calls' own round trips, with room.
CLOCK_TOLERANCE_NS = 2 * 10**9


def cmd_set_time(a):
was = qga(a.socket, "guest-get-time", timeout=a.timeout) - time.time_ns()
# qemu-ga sets the system clock, then runs hwclock to copy it to the
# hardware clock and fails the call if that does. TrueNAS has no
# /sbin/hwclock, so the call fails with the system clock already set.
# The hardware clock does not matter here (loadvm restores it, and the
# guest reads it only at boot): the system clock decides, below.
refused = None
try:
qga(a.socket, "guest-set-time", {"time": time.time_ns()}, timeout=a.timeout)
except RPCError as e:
refused = e
off = qga(a.socket, "guest-get-time", timeout=a.timeout) - time.time_ns()
if abs(off) > CLOCK_TOLERANCE_NS:
why = f" ({refused})" if refused else ""
raise SystemExit(f"tnctl set-time: the guest's clock is still {off / 1e9:+.1f}s off{why}")
if refused and "hwclock" not in str(refused):
log(f"guest-set-time answered {refused}, but the clock took")
log(f"guest clock set: it was {was / 1e9:+.1f}s off")


def cmd_qmp(a):
r = qmp(a.socket, a.command, json.loads(a.args) if a.args else None)
print(json.dumps(r, indent=2))
Expand Down Expand Up @@ -474,6 +558,7 @@ def p(name, fn, *args, timeout=600):
p("call", cmd_call, "url", "user", "password", "method", timeout=120).add_argument("params", nargs="*")
p("qmp", cmd_qmp, "socket", "command").add_argument("args", nargs="?")
p("hmp", cmd_hmp, "socket", "line")
p("set-time", cmd_set_time, "socket", timeout=10)
a = ap.parse_args()
try:
a.fn(a)
Expand Down
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -575,6 +575,13 @@ Outputs: `host`, `host-http`, `username`, `password` (masked), `ssh`,
time went, from systemd and middleware's own log, over ssh (it leaves ssh on
with a key and sudo for the admin, so revert afterwards if that matters).

`revert` restores the appliance's memory, and with it the clock it had when
the snapshot was taken. It then sets the clock to the runner's through the
guest agent (TrueNAS ships `qemu-ga`; qemu gives it a channel), so two-factor
codes, tokens and timestamps a suite makes on the runner's clock hold. If
that fails, the revert still succeeds with a warning that the clock is
behind; `probe` fails instead.

**Run the suite on the runner, not in a container.** `truenas-appliance`,
qemu and its monitor socket are the runner's, so a spec that reverts the
appliance from inside Playwright's image (or any container) cannot reach
Expand Down
Loading