From df4ad384dbde93edd6f4cce2925d10cb5782b84a Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Fri, 13 Nov 2026 22:03:25 -0300 Subject: [PATCH] image: udev's coldplug keeps cpu, so a core attached before systemd comes online The coldplug filtered in 630fa2c replays block, net, virtio, misc, input, rtc, ptp and ttyS0, and not cpu. A core attached while a consumer's init is PID 1 has no udev to see it arrive: what onlines it is that replay reaching 40-spin-hotadd.rules. With v20261113.03 every spin workspace came up with one core whatever it was sized to (spin's kvm run 43527839765). task boot:hotadd attaches a core before systemd starts and reads the guest's online cores after: 0 on v20261113.03's image, 0-1 with this. image/build.sh refuses an image whose filter lost cpu, as it does input. Co-Authored-By: Claude Fable 5.2 --- boot/Taskfile.yml | 9 ++ boot/console_test.go | 9 ++ boot/hotadd_test.go | 86 +++++++++++++++++++ boot/testdata/hotadd-check.sh | 11 +++ boot/testdata/hotadd-init.sh | 8 ++ docs/log.md | 2 + image/build.sh | 4 + .../10-coldplug.conf | 6 +- 8 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 boot/hotadd_test.go create mode 100644 boot/testdata/hotadd-check.sh create mode 100644 boot/testdata/hotadd-init.sh diff --git a/boot/Taskfile.yml b/boot/Taskfile.yml index 7424221..467f42b 100644 --- a/boot/Taskfile.yml +++ b/boot/Taskfile.yml @@ -132,6 +132,15 @@ tasks: cmds: - SPIN_LOCKDOWN_TEST=1 go test ./boot/ -run '^TestTheGuestKernelIsLockedDown$' -count=1 -v -timeout 5m + hotadd: + desc: >- + Check a core attached while an init of the consumer's is PID 1, before systemd and its + udev start, is online once they have, in a disposable guest. Needs a built release; uses + KVM when available, otherwise TCG. + deps: [':tools'] + cmds: + - SPIN_HOTADD_TEST=1 go test ./boot/ -run '^TestACoreAttachedBeforeSystemdStartsComesOnline$' -count=1 -v -timeout 5m + initcalls: desc: >- Where the kernel's own boot goes, initcall by initcall, as a p50 over REPS boots. diff --git a/boot/console_test.go b/boot/console_test.go index e1754dc..7b33036 100644 --- a/boot/console_test.go +++ b/boot/console_test.go @@ -19,6 +19,13 @@ import ( // the serial console, and fails the test unless ok comes first. It returns what the console said. // Under TCG where there is no /dev/kvm: what it checks is behaviour, not time. func checkOnConsole(t *testing.T, rel *machine.Release, root *rawRoot, ok, failed string) string { + t.Helper() + return checkGuest(t, rel, root, ok, failed, func(*machine.Spec) {}, func(string) {}) +} + +// checkGuest is checkOnConsole for a check the host takes part in: shape changes the machine +// before it starts, and said is given the console so far each time the guest has written to it. +func checkGuest(t *testing.T, rel *machine.Release, root *rawRoot, ok, failed string, shape func(*machine.Spec), said func(console string)) string { t.Helper() spec := rel.Spec() spec.BootCPUs = 2 @@ -29,6 +36,7 @@ func checkOnConsole(t *testing.T, rel *machine.Release, root *rawRoot, ok, faile c.Root = "/dev/vda" c.Init = "/sbin/init" spec.Cmdline = c + shape(&spec) if _, err := os.Stat("/dev/kvm"); err != nil { tcg, err := rel.QEMUTCG() if err != nil { @@ -68,6 +76,7 @@ func checkOnConsole(t *testing.T, rel *machine.Release, root *rawRoot, ok, faile if err != nil || strings.Contains(console.String(), failed) { t.Fatalf("guest check failed: %v\n%s\n%s", err, &console, &stderr) } + said(console.String()) } } diff --git a/boot/hotadd_test.go b/boot/hotadd_test.go new file mode 100644 index 0000000..4367b24 --- /dev/null +++ b/boot/hotadd_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +package boot_test + +import ( + "encoding/json" + "maps" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spin-stack/spin-machine/machine" +) + +// A core attached before systemd starts is online once it has: a consumer boots one shape and +// gives a workspace its cores while its own init is PID 1, with no udev there to see them +// arrive, so what onlines them is udev's coldplug replaying their add to 40-spin-hotadd.rules. +// A coldplug filtered to the devices the image uses, and not to cpu, left every such core +// offline: spin's workspaces came up with one core whatever they were sized to (spin's kvm run +// 43527839765, 2026-11-13). +func TestACoreAttachedBeforeSystemdStartsComesOnline(t *testing.T) { + if os.Getenv("SPIN_HOTADD_TEST") != "1" { + t.Skip("set SPIN_HOTADD_TEST=1 to check a core attached before systemd in a built image") + } + out := releaseTree(t) + rel, err := machine.OpenRelease(out) + if err != nil { + t.Fatal(err) + } + base, err := rel.Rootfs() + if err != nil { + t.Fatal(err) + } + root := newRawRoot(t, out, base) + hold, err := os.ReadFile(filepath.Join("testdata", "hotadd-init.sh")) + if err != nil { + t.Fatal(err) + } + root.write("/hotadd-init.sh", string(hold)) + // debugfs writes a file nobody may execute, and the kernel execs its init. + root.edit("sif /hotadd-init.sh mode 0100755") + oneshotAtBoot(t, root, "hotadd-check", "hotadd-check.sh", "") + + // Its own short directory: a socket's path has 108 bytes. + dir, err := os.MkdirTemp("", "q") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + monitor := filepath.Join(dir, "q") + + attached := false + t.Log(checkGuest(t, rel, root, "HOTADD_CHECK_OK", "HOTADD_CHECK_FAILED", + func(s *machine.Spec) { + s.BootCPUs, s.MaxCPUs = 1, 2 + s.Cmdline.Init = "/hotadd-init.sh" + s.Monitors = []machine.Monitor{{Socket: monitor}} + }, + func(console string) { + if attached || !strings.Contains(console, "HOTADD_WAITING") { + return + } + attached = true + q := dial(t, monitor, func() string { return console }) + defer q.close() + var slots []struct { + Type string `json:"type"` + Props map[string]any `json:"props"` + Path string `json:"qom-path"` + } + if err := json.Unmarshal(q.do(t, "query-hotpluggable-cpus", nil), &slots); err != nil { + t.Fatal(err) + } + for _, slot := range slots { + if slot.Path != "" { + continue + } + add := map[string]any{"driver": slot.Type, "id": "core1"} + maps.Copy(add, slot.Props) + q.do(t, "device_add", add) + return + } + t.Fatal("the machine has no free core to attach") + })) +} diff --git a/boot/testdata/hotadd-check.sh b/boot/testdata/hotadd-check.sh new file mode 100644 index 0000000..3e58cc5 --- /dev/null +++ b/boot/testdata/hotadd-check.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 +set -eu +trap 'echo HOTADD_CHECK_FAILED' EXIT + +echo "present: $(cat /sys/devices/system/cpu/present) online: $(cat /sys/devices/system/cpu/online)" +test "$(cat /sys/devices/system/cpu/present)" = "0-1" +test "$(cat /sys/devices/system/cpu/online)" = "0-1" + +trap - EXIT +echo HOTADD_CHECK_OK diff --git a/boot/testdata/hotadd-init.sh b/boot/testdata/hotadd-init.sh new file mode 100644 index 0000000..e1334bf --- /dev/null +++ b/boot/testdata/hotadd-init.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 +# PID 1 until a second core has been attached, then systemd: a consumer's init that is given +# its cores before it hands over, with no udev running to see them arrive. +mount -t sysfs sysfs /sys +echo HOTADD_WAITING > /dev/console +until [ -e /sys/devices/system/cpu/cpu1 ]; do sleep 0.05; done +exec /sbin/init diff --git a/docs/log.md b/docs/log.md index 780a278..f2b11a4 100644 --- a/docs/log.md +++ b/docs/log.md @@ -341,6 +341,8 @@ Measured and not shipped: **11-13 · boot/qemu · branch `first-login-report` (this change).** v20261113.03's report read +11-18% on every row: the report times the serial console's prompt, and the getty drop-in removed in #250 left it Type=idle, waiting for the lane's units. The first login it does not time went 478/500 (v20261113.02, lab 43524334205) -> 381/410 ms (v20261113.03, lab 43524721490), two runs and not interleaved. The report now logs into the release over SSH too (`first_login`), and compare writes it first. The QEMU build asserts static-pie, full RELRO, an NX stack and stack canaries, which come from QEMU's and Alpine's defaults and no flag here. Measured and left out: one sshd for every connection (the vsock socket with Accept=no, an `sshd -D` from the first dispatch), first login 382/393 against 381/410 and a second 40 against 48 ms (lab 43524721490). +**11-13 · image · udev's coldplug keeps cpu (this change).** The coldplug filtered in `630fa2c` left out cpu, and a core attached before systemd starts - spin gives a workspace its cores while its own init is PID 1 - is onlined by that replay reaching `40-spin-hotadd.rules` and by nothing else: with v20261113.03 every workspace came up with one core whatever it was sized to (spin's kvm run 43527839765, four tests "still waiting for the guest to have its floor's two cores"). `task boot:hotadd` attaches a core before systemd and reads `online` after: `0` on v20261113.03's image, `0-1` with cpu in the filter. What cpu costs the first login was not measured; it is one device per core. + --- ## 3. Rework cycles diff --git a/image/build.sh b/image/build.sh index 8552ffa..e5d743e 100755 --- a/image/build.sh +++ b/image/build.sh @@ -398,6 +398,10 @@ stat_in_image /etc/tmpfiles.d/systemd-nologin.conf | grep -q '/dev/null' || { grep -q -- '--subsystem-match=input' || { echo "ERROR: udev's coldplug is not filtered, or lost input (the power button logind shuts down on)" >&2 exit 1; } +"$bin/debugfs" -R "cat /etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf" /work/base.raw 2>/dev/null | + grep -q -- '--subsystem-match=cpu' || { + echo "ERROR: udev's coldplug lost cpu; a core attached before systemd starts stays offline" >&2 + exit 1; } if stat_in_image /etc/systemd/system/systemd-logind.service | grep -q '/dev/null'; then echo "ERROR: systemd-logind.service is masked - it never starts, no login gets a" \ "session, and spin's manager never starts either" >&2 diff --git a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf index 174f001..d9bb125 100644 --- a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf +++ b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf @@ -5,11 +5,13 @@ # vCPUs: filtered, that login came 436 -> 402 ms on the bench fleet (task boot:ssh-first, lab # run 43518915618, 2026-11-13). What it keeps: the disk and the network; virtio; misc (vsock, # fuse, tun) and input (the power button's power-switch tag, which logind's shutdown on -# system_powerdown needs); the clock devices; the serial console. Compared device by device +# system_powerdown needs); the clock devices; the serial console; and cpu, because a core a +# consumer attached while its own init was PID 1 is onlined by this replay reaching +# 40-spin-hotadd.rules and by nothing else (task boot:hotadd). Compared device by device # against the full replay, what differs is the group of /dev/ptmx and /dev/tty (mode 666 # either way), dialout on ttyS1-3 and kmem on /dev/port. A device that arrives later is # udev's as ever: hotplug is a new event, not a replay. [Service] ExecStart= -ExecStart=-udevadm trigger --type=devices --action=add --subsystem-match=block --subsystem-match=net --subsystem-match=virtio --subsystem-match=misc --subsystem-match=input --subsystem-match=rtc --subsystem-match=ptp +ExecStart=-udevadm trigger --type=devices --action=add --subsystem-match=block --subsystem-match=net --subsystem-match=virtio --subsystem-match=misc --subsystem-match=input --subsystem-match=rtc --subsystem-match=ptp --subsystem-match=cpu ExecStart=-udevadm trigger --type=devices --action=add --subsystem-match=tty --sysname-match=ttyS0