diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index e6dba00..4929324 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -135,7 +135,27 @@ install -d "$MNT/var/lib/systemd/linger" && touch "$MNT/var/lib/systemd/linger/s asBuilt.links = map[string]string{ "/etc/systemd/system/sysinit.target.wants/spin-machine-sshd-keygen.service": "/usr/lib/systemd/system/spin-machine-sshd-keygen.service", } - rows := []variant{asBuilt} + // The first login's lane, as the image builds it, put into the published release this row + // boots: the units' drop-ins read from this tree's image, and configure-system.sh's three + // edits done the same way. Measured against the release as it was published, interleaved. + rootfs := filepath.Join("..", "image", "mkosi.extra", "usr", "local", "lib", "spin-base", "rootfs") + for _, d := range []string{"sshd-vsock.socket.d/10-lane.conf", "sshd@.service.d/10-lane.conf", + "systemd-logind.service.d/10-lane.conf", "user-runtime-dir@.service.d/10-lane.conf", + "user@.service.d/10-lane.conf", "systemd-udev-trigger.service.d/10-coldplug.conf"} { + b, err := os.ReadFile(filepath.Join(rootfs, "etc", "systemd", "system", d)) + if err != nil { + t.Fatal(err) + } + asBuilt.files["/etc/systemd/system/"+d] = string(b) + } + asBuilt.setup += `grep -qx 'After=sysinit.target' "$MNT/usr/lib/systemd/system/dbus.socket" +sed -e '/^Wants=sysinit.target$/d' -e '/^After=sysinit.target$/d' "$MNT/usr/lib/systemd/system/dbus.socket" > "$MNT/etc/systemd/system/dbus.socket" +ln -sf /dev/null "$MNT/etc/tmpfiles.d/systemd-nologin.conf" +grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" || + sed -i 's/^\(session.*pam_systemd\.so\)\(.*\)$/\1\2 class=user-early-light/' "$MNT/etc/pam.d/common-session" +grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" +` + rows := []variant{row("as released", ""), asBuilt} type samples struct{ listen, login, accepted, again, prompt []time.Duration } got := map[string]*samples{} diff --git a/docs/log.md b/docs/log.md index 8550125..def577d 100644 --- a/docs/log.md +++ b/docs/log.md @@ -335,6 +335,8 @@ Measured and not shipped: **11-13 · kernel · #255 (this change).** Where the kernel's ~100 ms goes on the bench fleet (boot:initcalls lab 43495583323): initcalls 52 (acpi_init 12, of which `acpi_pci_root_create` - bus 0's enumeration - 11.2 traced on a kernel without forced lockdown, lab 43499868986; inet_init 7.9, which is vmlinux BTF's first parse; the virtio probes ~17), before the first initcall ~15-20 (SPIN_EARLY, lab 43507472288; ~4 of it the probe's own 18 MB log buffer), SMP ~4, the root mount ~6. Kept: patch 0005 back, 0006 (BTF parsed in the background from a core_initcall), `CONFIG_DEFERRED_STRUCT_PAGE_INIT=y`: kernel 96.3 → 87.1 ms (lab 43505126131), first login 466 → 456 as built and 223 → 214 at the floor (lab 43501992969). `driver_async_probe` for rng, balloon and vsock on the command line: 214 → 209 at the floor (lab 43506390106). +**11-13 · image · the first login's lane (this change).** Where systemd's ~250 ms over the floor went, on the bench fleet: ~60 before sshd accepts (sysinit, udev, journald), ~105 in pam_systemd - logind answers a session of class `user` only once the user's manager has started and orders its scope after `systemd-user-sessions` (systemd v259.9 source, `logind-session.c`) - and ~65 in a handshake competing for the CPU. The vsock socket, `sshd@`, dbus, logind, `user-runtime-dir@` and `user@` without default dependencies, no `/run/nologin` at boot (pam_nologin refused the lane's first login: "Access denied for user spin by PAM account configuration"), sessions of class `user-early-light`, and udev's coldplug filtered to block, net, virtio, misc, input, rtc, ptp and ttyS0: first login 478 → 402 ms (lab 43518915618). Measured and left out: priority (`CPUWeight`, `Nice`) on the lane, −2 ms over the filter and a slower second login (63 vs 45 ms). + **11-13 · kernel/qemu · branches `kernel-snapshot`, `qemu-snapshot` (rejected).** QEMU publishes bus 0's configuration space over fw_cfg after the firmware (etc/spin/pci-config) and the kernel answers boot-time configuration reads from it until each dword is written. It works (`PCI: configuration space of 5 devices on bus 0`) and is worth -14 ms against the same kernel without it (lab 43514550646), but only 2-3 ms against 0005, which already removes most repeated reads: 209/217 against 211/222 (lab 43514884598). Not worth a QEMU patch and a 250-line kernel patch. --- @@ -511,6 +513,8 @@ Also the reusable builder, added and reverted on 10-13. | mapped-ram checkpoints (multifd ×4) | Restore login 385/400 vs 386/399 | #246, lab 43483852339 | 11-13 | | sntrup761 (OpenSSH default) | 104 vs 40 ms per login | #245, lab 43479854381 | 11-13 | | Configuration-space snapshot over fw_cfg (QEMU + kernel) | 209 vs 211 with 0005 | lab 43514884598 | 11-13 | +| CPUWeight/IOWeight/Nice on the first login's lane | −2 ms over coldplug filtered; second login 63 vs 45 | lab 43518915618 | 11-13 | +| Login without pam_systemd (diagnostic bound) | 380 vs 436, but no session, no XDG_RUNTIME_DIR | lab 43518044150 | 11-13 | | Lower the vCPU ceiling, drop the balloon or ACPI for boot time | Not tried: hotplug and dynamic memory are an invariant | AGENTS.md | 11-13 | --- diff --git a/image/README.md b/image/README.md index 76e4668..46c3acf 100644 --- a/image/README.md +++ b/image/README.md @@ -62,6 +62,26 @@ gone with it. The drop-in is `/etc/ssh/sshd_config.d/10-spin-kex.conf`, written `configure-system.sh` and asserted by `build.sh`; a drop-in that sets `KexAlgorithms` and is read before it wins, because sshd keeps the first value it reads. +## The first login's lane + +What a first SSH login goes through starts from the boot's first dispatch, ordered only by what +it needs: the vsock socket and the per-connection sshd (after the host keys), dbus, logind, and +spin's runtime directory and manager. None of them waits for `sysinit.target` or `basic.target` +(`etc/systemd/system/*.d/10-lane.conf`). Two settings make that worth it: + +- **No `/run/nologin` at boot.** Until `systemd-user-sessions` removes it late in the boot, + `pam_nologin` refuses every user, and in the lane the first login arrives before that. Shutting + down still writes it. +- **Sessions of class `user-early-light`.** logind holds a session of class `user` until the + user's manager has started and the boot's `systemd-user-sessions` has run. A light, early + session has its scope and `XDG_RUNTIME_DIR` and waits for neither; spin's manager still starts + in the boot, because spin lingers. + +And udev's coldplug replays only the devices something here uses (`10-coldplug.conf`), so the +lane has the CPU. On the bench fleet the first login went 478 -> 402 ms with all of it (`task +boot:ssh-first`, lab run 43518915618, 2026-11-13); `configure-system.sh` and the drop-ins say what +each part did, and `build.sh` asserts each on the finished filesystem. + ## Looking inside ``` diff --git a/image/build.sh b/image/build.sh index d149cce..8552ffa 100755 --- a/image/build.sh +++ b/image/build.sh @@ -370,6 +370,34 @@ stat_in_image /etc/systemd/system/sysinit.target.wants/spin-machine-sshd-keygen. grep -qx 'DefaultDependencies=no' || { echo "ERROR: spin-machine-sshd-keygen.service has default dependencies; it starts after sysinit.target" >&2 exit 1; } +# The first SSH login's lane (configure-system.sh and the units' 10-lane.conf), read back from +# the finished filesystem: each piece alone gains little and leaves the login waiting on the +# next, so a missing one costs the lane without failing anything. +for u in sshd-vsock.socket sshd@.service systemd-logind.service user-runtime-dir@.service user@.service; do + "$bin/debugfs" -R "cat /etc/systemd/system/$u.d/10-lane.conf" /work/base.raw 2>/dev/null | + grep -qx 'DefaultDependencies=no' || { + echo "ERROR: $u is not in the first login's lane (10-lane.conf)" >&2 + exit 1; } +done +dbus=$("$bin/debugfs" -R "cat /etc/systemd/system/dbus.socket" /work/base.raw 2>/dev/null) +printf '%s\n' "$dbus" | grep -q '^ListenStream=/run/dbus/system_bus_socket' || { + echo "ERROR: /etc/systemd/system/dbus.socket is missing or is not the bus's socket" >&2 + exit 1; } +if printf '%s\n' "$dbus" | grep -qx 'After=sysinit.target'; then + echo "ERROR: dbus.socket is ordered after sysinit.target; logind waits for udev and journald" >&2 + exit 1 +fi +stat_in_image /etc/tmpfiles.d/systemd-nologin.conf | grep -q '/dev/null' || { + echo "ERROR: tmpfiles writes /run/nologin at boot; pam_nologin refuses the lane's first login" >&2 + exit 1; } +"$bin/debugfs" -R "cat /etc/pam.d/common-session" /work/base.raw 2>/dev/null | + grep -q '^session.*pam_systemd\.so.*class=user-early-light' || { + echo "ERROR: sessions are not user-early-light; a login waits for the user's manager" >&2 + exit 1; } +"$bin/debugfs" -R "cat /etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf" /work/base.raw 2>/dev/null | + grep -q -- '--subsystem-match=input' || { + echo "ERROR: udev's coldplug is not filtered, or lost input (the power button logind shuts down on)" >&2 + exit 1; } if stat_in_image /etc/systemd/system/systemd-logind.service | grep -q '/dev/null'; then echo "ERROR: systemd-logind.service is masked - it never starts, no login gets a" \ "session, and spin's manager never starts either" >&2 diff --git a/image/mkosi.extra/usr/local/lib/spin-base/configure-system.sh b/image/mkosi.extra/usr/local/lib/spin-base/configure-system.sh index 2a9225a..2af5190 100755 --- a/image/mkosi.extra/usr/local/lib/spin-base/configure-system.sh +++ b/image/mkosi.extra/usr/local/lib/spin-base/configure-system.sh @@ -82,6 +82,34 @@ passwd --lock spin install -d -m 0755 /var/lib/systemd/linger touch /var/lib/systemd/linger/spin +# The rest of the first SSH login's lane (the units' drop-ins say what it is and what it +# measured, in rootfs/etc/systemd/system/*.d/10-lane.conf). +# +# dbus.socket orders itself after sysinit.target in its own [Unit], which a drop-in cannot take +# back, so the lane has a copy without those two lines. +echo "The first login's lane..." +grep -qx 'After=sysinit.target' /usr/lib/systemd/system/dbus.socket +sed -e '/^Wants=sysinit.target$/d' -e '/^After=sysinit.target$/d' \ + /usr/lib/systemd/system/dbus.socket > /etc/systemd/system/dbus.socket +# +# No /run/nologin at boot. systemd-tmpfiles-setup writes it and systemd-user-sessions.service +# removes it late, and until then pam_nologin turns every user away - in the lane, the first +# login's sshd ran first and was refused: "Access denied for user spin by PAM account +# configuration [preauth]". Shutting down still writes it: systemd-user-sessions.service's +# ExecStop does. +ln -sf /dev/null /etc/tmpfiles.d/systemd-nologin.conf +# +# Sessions of class user-early-light. logind answers a session request once the session's +# scope, the runtime directory and, for a class that wants one, the user's service manager are +# started, and orders the scope after systemd-user-sessions.service unless the class is early +# (systemd v259.9, src/login/logind-session.c: session_job_pending, session_start_scope). So a +# login of class user waited for user@1000, a whole systemd --user, and for the end of the +# boot: 472 against 436 ms on the bench fleet (lab run 43518044150). spin's manager still +# starts in the boot - spin lingers - and a light session still has its scope and +# XDG_RUNTIME_DIR; it does not wait for the manager. +sed -i 's/^\(session.*pam_systemd\.so\)\(.*\)$/\1\2 class=user-early-light/' /etc/pam.d/common-session +grep -q '^session.*pam_systemd\.so.*class=user-early-light' /etc/pam.d/common-session + # Sudo without a password, because there is no password. The boundary this machine relies # on is the VM, not the difference between two accounts inside it — but a workload that # expects not to be root by default gets what it expects. diff --git a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/sshd-vsock.socket.d/10-lane.conf b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/sshd-vsock.socket.d/10-lane.conf new file mode 100644 index 0000000..1bd8257 --- /dev/null +++ b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/sshd-vsock.socket.d/10-lane.conf @@ -0,0 +1,13 @@ +# The first SSH login's lane: systemd-ssh-generator's vsock socket. +# +# Without systemd's default dependencies, so this does not wait for sysinit.target (journald, +# udev) or basic.target. Together with the rest of the lane - the vsock socket, the +# per-connection sshd, dbus, logind, the user's runtime directory and manager, all early; +# no /run/nologin at boot and sessions of class user-early-light (configure-system.sh) - the +# first login over vsock came 478 -> 402 ms on the bench fleet (task boot:ssh-first, lab run +# 43518915618, 2026-11-13). Each part alone gained little: the socket early accepted 60 ms +# sooner and the login waited for logind and the user's manager instead. +[Unit] +DefaultDependencies=no +Conflicts=shutdown.target +Before=shutdown.target diff --git a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/sshd@.service.d/10-lane.conf b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/sshd@.service.d/10-lane.conf new file mode 100644 index 0000000..f9e0dc7 --- /dev/null +++ b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/sshd@.service.d/10-lane.conf @@ -0,0 +1,13 @@ +# The first SSH login's lane: the per-connection sshd; it still waits for the host keys (keygen.conf). +# +# Without systemd's default dependencies, so this does not wait for sysinit.target (journald, +# udev) or basic.target. Together with the rest of the lane - the vsock socket, the +# per-connection sshd, dbus, logind, the user's runtime directory and manager, all early; +# no /run/nologin at boot and sessions of class user-early-light (configure-system.sh) - the +# first login over vsock came 478 -> 402 ms on the bench fleet (task boot:ssh-first, lab run +# 43518915618, 2026-11-13). Each part alone gained little: the socket early accepted 60 ms +# sooner and the login waited for logind and the user's manager instead. +[Unit] +DefaultDependencies=no +Conflicts=shutdown.target +Before=shutdown.target diff --git a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-logind.service.d/10-lane.conf b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-logind.service.d/10-lane.conf new file mode 100644 index 0000000..16c40d3 --- /dev/null +++ b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-logind.service.d/10-lane.conf @@ -0,0 +1,13 @@ +# The first SSH login's lane: logind, which pam_systemd asks for the session. +# +# Without systemd's default dependencies, so this does not wait for sysinit.target (journald, +# udev) or basic.target. Together with the rest of the lane - the vsock socket, the +# per-connection sshd, dbus, logind, the user's runtime directory and manager, all early; +# no /run/nologin at boot and sessions of class user-early-light (configure-system.sh) - the +# first login over vsock came 478 -> 402 ms on the bench fleet (task boot:ssh-first, lab run +# 43518915618, 2026-11-13). Each part alone gained little: the socket early accepted 60 ms +# sooner and the login waited for logind and the user's manager instead. +[Unit] +DefaultDependencies=no +Conflicts=shutdown.target +Before=shutdown.target diff --git a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf new file mode 100644 index 0000000..174f001 --- /dev/null +++ b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-udev-trigger.service.d/10-coldplug.conf @@ -0,0 +1,15 @@ +# udev's coldplug for the devices something here uses, not for every device the kernel found. +# +# systemd-udev-trigger replays an add for each of ~170 devices at boot - ACPI's 39, sixteen +# memory blocks, bdi, mem, pci, platform - while the first login's lane runs on the same two +# vCPUs: filtered, that login came 436 -> 402 ms on the bench fleet (task boot:ssh-first, lab +# run 43518915618, 2026-11-13). What it keeps: the disk and the network; virtio; misc (vsock, +# fuse, tun) and input (the power button's power-switch tag, which logind's shutdown on +# system_powerdown needs); the clock devices; the serial console. Compared device by device +# against the full replay, what differs is the group of /dev/ptmx and /dev/tty (mode 666 +# either way), dialout on ttyS1-3 and kmem on /dev/port. A device that arrives later is +# udev's as ever: hotplug is a new event, not a replay. +[Service] +ExecStart= +ExecStart=-udevadm trigger --type=devices --action=add --subsystem-match=block --subsystem-match=net --subsystem-match=virtio --subsystem-match=misc --subsystem-match=input --subsystem-match=rtc --subsystem-match=ptp +ExecStart=-udevadm trigger --type=devices --action=add --subsystem-match=tty --sysname-match=ttyS0 diff --git a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/user-runtime-dir@.service.d/10-lane.conf b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/user-runtime-dir@.service.d/10-lane.conf new file mode 100644 index 0000000..864bd68 --- /dev/null +++ b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/user-runtime-dir@.service.d/10-lane.conf @@ -0,0 +1,13 @@ +# The first SSH login's lane: the user's runtime directory, which a session needs before logind answers. +# +# Without systemd's default dependencies, so this does not wait for sysinit.target (journald, +# udev) or basic.target. Together with the rest of the lane - the vsock socket, the +# per-connection sshd, dbus, logind, the user's runtime directory and manager, all early; +# no /run/nologin at boot and sessions of class user-early-light (configure-system.sh) - the +# first login over vsock came 478 -> 402 ms on the bench fleet (task boot:ssh-first, lab run +# 43518915618, 2026-11-13). Each part alone gained little: the socket early accepted 60 ms +# sooner and the login waited for logind and the user's manager instead. +[Unit] +DefaultDependencies=no +Conflicts=shutdown.target +Before=shutdown.target diff --git a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/user@.service.d/10-lane.conf b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/user@.service.d/10-lane.conf new file mode 100644 index 0000000..13a7d51 --- /dev/null +++ b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/user@.service.d/10-lane.conf @@ -0,0 +1,13 @@ +# The first SSH login's lane: the user's manager, which spin's linger starts in the boot. +# +# Without systemd's default dependencies, so this does not wait for sysinit.target (journald, +# udev) or basic.target. Together with the rest of the lane - the vsock socket, the +# per-connection sshd, dbus, logind, the user's runtime directory and manager, all early; +# no /run/nologin at boot and sessions of class user-early-light (configure-system.sh) - the +# first login over vsock came 478 -> 402 ms on the bench fleet (task boot:ssh-first, lab run +# 43518915618, 2026-11-13). Each part alone gained little: the socket early accepted 60 ms +# sooner and the login waited for logind and the user's manager instead. +[Unit] +DefaultDependencies=no +Conflicts=shutdown.target +Before=shutdown.target