diff --git a/Makefile b/Makefile index 3eb5788e..a3ac99c7 100644 --- a/Makefile +++ b/Makefile @@ -269,6 +269,11 @@ $(BUILD_DIR)/probe-volume-naming: $(BUILD_DIR)/probe-volume-naming.o \ @echo " LD $@" $(Q)$(CC) $(CFLAGS) -o $@ $^ +## Build the disk-write probe test-slab-exit-writes measures with (native macOS) +$(BUILD_DIR)/probe-disk-writes: $(BUILD_DIR)/probe-disk-writes.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + ## Build the filename codec host test (native macOS binary) # casefold.o is a leaf translation unit with no syscall-layer dependencies, so # the test links exactly the code under test and nothing else. diff --git a/docs/internals.md b/docs/internals.md index 3adbc9b4..98e2e656 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -711,8 +711,9 @@ the backing file and sends the snapshot fd over `SCM_RIGHTS`: - Parent stays on `MAP_SHARED` and does NOT remap -- HVF caches the host VA->PA mapping from `hv_vm_map`, and a `MAP_FIXED` remap does not update Stage-2, so a remapping parent would observe stale pages. -- Child maps the snapshot fd `MAP_PRIVATE`, producing an instant CoW - clone with zero data copy. +- Child maps the snapshot fd `MAP_SHARED` and keeps it as its own backing + file, so its writes land in the clone and it can clone that in turn for + its own forks. No data is copied. - The IPC header sets `has_shm = 1` and `num_regions = 0`, skipping memory serialization entirely. - Child calls `guest_init_from_shm()` instead of `guest_init()`, and must @@ -726,10 +727,27 @@ the backing file and sends the snapshot fd over `SCM_RIGHTS`: This path is roughly 50x faster than the legacy IPC copy path on large guest memories. -The CoW path is disabled when hosting Rosetta because HVF caches the host -VA->PA mapping from `hv_vm_map`, and Rosetta's translated code touches -the parent's slab in ways the snapshot model cannot intercept. Rosetta -forks fall back to the legacy IPC copy path. +If `fclonefileat` fails, a native guest sends its live backing fd instead. +The child maps the parent's file `MAP_PRIVATE`, so any page it has not +written still reads the parent's current bytes, and the parent sets +`g->shm_exported`. + +The host diagnostic switch `ELFUSE_DISABLE_FORK_CLONEFILE=1` bypasses the +clone attempt so `test-slab-exit-fork` can exercise the live-fd fallback on +APFS. Other values leave the clone attempt enabled. + +On normal teardown, `guest_destroy` truncates the backing file before its +last close: the file is unlinked, but that close would still write every +dirty page back to disk. It skips the truncate when `shm_exported` is set, because +truncating a file another process maps privately stalls the host while the +kernel exhausts memory. + +If a worker vCPU remains live past the join cap, `guest_destroy` returns +before the truncate and leaves resource cleanup to process exit. + +Rosetta also attempts the APFS clone, but falls back to region copy when +cloning fails or is disabled. The live-fd fallback could expose partially +updated translator state from the parent's slab to the child. macOS rejects `MAP_PRIVATE` on `shm_open` fds (`EINVAL`), so the backing file is created via `mkstemp` + `unlink`. diff --git a/mk/config.mk b/mk/config.mk index d1a97edc..5a23802e 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -63,6 +63,7 @@ NATIVE_TESTS := tests/test-multi-vcpu.c tests/test-rwx.c \ tests/test-casefold-walk-host.c \ tests/test-absock-names-host.c \ tests/probe-volume-naming.c \ + tests/probe-disk-writes.c \ tests/test-dynamic-array-host.c \ tests/test-string-builder-host.c \ tests/test-wakeup-pipe-host.c \ diff --git a/mk/tests.mk b/mk/tests.mk index 069972eb..f3f4c0d6 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -34,6 +34,7 @@ ELFUSE_HOST_NOFILE_MIN ?= $(shell bash "$(CURDIR)/tests/test-config.sh" --host-n test-sysroot-inotify-names test-sysroot-exec-names \ test-sysroot-interp-fallback test-sysroot-interp-cased \ test-sysroot-absock-names test-absock-cleanup test-registry-stale-pid \ + test-slab-exit-writes test-slab-exit-fork \ test-linkat-symlink-fallback test-casefold-host \ test-casefold-walk-host test-absock-names-host \ test-wakeup-pipe-host test-guest-env-host \ @@ -350,6 +351,8 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage check-eintr-contract ch $(call run-lane,test-fuse-alpine,Alpine sysroot FUSE validation) $(call run-lane,test-timeout-disable,timeout=0 validation) $(call run-lane,test-launch-flags,launch flags) + $(call run-lane,test-slab-exit-writes,exit discards dirty guest memory) + $(call run-lane,test-slab-exit-fork,fork children and the exit truncate) $(call run-lane,test-rosetta-cli,rosetta CLI gating) $(call run-lane,test-bench-guardrail,hot-syscall guardrail) $(call run-lane,test-sharun,sharun launcher and probe) @@ -728,6 +731,94 @@ test-absock-cleanup: $(ELFUSE_BIN) $(BUILD_DIR)/test-absock-cleanup fi; \ $(ASSERT_NO_ABSOCK_LEAK) +# Run one lane step with its output in $$tmpdir/NAME.log, failing the lane +# with that log when the step fails. +define SLAB_STEP +step() { name=$$1; shift; "$$@" > "$$tmpdir/$$name.log" 2>&1 || { printf "FAIL: %s\n" "$$name"; cat "$$tmpdir/$$name.log"; exit 1; }; } +endef + +SLAB_GUEST_DEPS := +ifndef GUEST_TEST_BINARIES +SLAB_GUEST_DEPS := $(TEST_DIR)/test-slab-exit +endif + +define SLAB_GUEST +guest="$(TEST_DIR)/test-slab-exit"; \ +if [ -n "$(GUEST_TEST_BINARIES)" ] && [ ! -x "$$guest" ]; then \ + printf "$(YELLOW)SKIP$(RESET) test-slab-exit unavailable in prebuilt bundle\n"; \ + exit 0; \ +fi +endef + +# guest_destroy truncates the slab file so that exit writes none of the +# guest's dirty memory back. The probe records the kernel's block-write count +# from the zombie. A control that writes and fsyncs 32 MiB must register at +# least 24 MiB, or the counter measures nothing here and the lane skips; a run +# that dirties 256 MiB must then stay under 32 MiB. +## exit discards dirty guest memory +test-slab-exit-writes: $(ELFUSE_BIN) $(SLAB_GUEST_DEPS) \ + $(BUILD_DIR)/probe-disk-writes + @$(SYSROOT_SCRATCH); \ + $(SLAB_STEP); \ + $(SLAB_GUEST); \ + probe=$(BUILD_DIR)/probe-disk-writes; \ + step control $$probe "$$tmpdir/n" $(ELFUSE_BIN) "$$guest" \ + write 32 "$$tmpdir/file"; \ + n=$$(cat "$$tmpdir/n"); \ + if [ "$$n" -lt $$((24 << 20)) ]; then \ + printf "$(YELLOW)SKIP$(RESET) disk counter saw %s bytes of a 32 MiB fsync\n" \ + "$$n"; \ + exit 0; \ + fi; \ + printf " %-30s " "256 MiB dirty at exit"; \ + step dirty $$probe "$$tmpdir/n" $(ELFUSE_BIN) "$$guest" dirty 256; \ + n=$$(cat "$$tmpdir/n"); \ + if [ "$$n" -ge $$((32 << 20)) ]; then \ + printf "FAIL: %s bytes written\n" "$$n"; exit 1; \ + fi; \ + printf "OK (%s bytes written)\n" "$$n" + +# The truncate is safe only while no other process maps the slab file. +# orphan has a child read its memory after its parent exited. The parent +# must skip truncation for the live-fd fallback, but not for an APFS clone. +# Force the fallback too, since an APFS clone cannot exercise that guard. +## fork children and the exit truncate +test-slab-exit-fork: $(ELFUSE_BIN) $(SLAB_GUEST_DEPS) + @$(SYSROOT_SCRATCH); \ + $(SLAB_STEP); \ + $(SLAB_GUEST); \ + for mode in normal fallback; do \ + disable_clone=0; \ + if [ "$$mode" = fallback ]; then disable_clone=1; fi; \ + dir="$$tmpdir/$$mode"; \ + mkdir "$$dir"; \ + printf " %-30s " "child outlives parent ($$mode)"; \ + step "$$mode" env ELFUSE_DISABLE_FORK_CLONEFILE=$$disable_clone \ + $(ELFUSE_BIN) --verbose "$$guest" orphan "$$dir"; \ + touch "$$dir/go"; \ + i=0; \ + while [ ! -s "$$dir/result" ] && [ $$i -lt 300 ]; do \ + sleep 0.1; i=$$((i + 1)); \ + done; \ + result=$$(cat "$$dir/result" 2>/dev/null || true); \ + if [ -z "$$result" ]; then \ + printf "FAIL: no result\n"; exit 1; \ + fi; \ + if [ "$$result" != ok ]; then \ + printf "FAIL: %s\n" "$$result"; exit 1; \ + fi; \ + if grep -q 'live shm fd as fallback' "$$tmpdir/$$mode.log"; then \ + if ! grep -q 'maps the slab, not truncated' "$$tmpdir/$$mode.log"; then \ + printf "FAIL: truncate not skipped for live slab\n"; exit 1; \ + fi; \ + elif [ "$$mode" = fallback ]; then \ + printf "FAIL: forced fallback not taken\n"; exit 1; \ + elif grep -q 'maps the slab, not truncated' "$$tmpdir/$$mode.log"; then \ + printf "FAIL: truncate skipped after a clone\n"; exit 1; \ + fi; \ + printf "OK\n"; \ + done + # An exited member's registry record outlives it, and macOS can hand its host # pid to another elfuse process. The recipe plants such a record, host pid of # a live unrelated elfuse run with a start time it does not have, and the diff --git a/src/core/guest.c b/src/core/guest.c index 6d84f0f3..1f310353 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -871,8 +871,19 @@ void guest_destroy(guest_t *g) } g->nregions = 0; g->npreannounced = 0; - /* Close the shm fd if guest memory owns one (parent with shm backing) */ + + /* The backing file is unlinked, yet this last close still writes its dirty + * pages back to disk; truncating first discards them. Not when a fork child + * was handed the file itself: it reads every page it has not written from + * here, and truncating under that private mapping stalls the host while the + * kernel exhausts memory. + */ if (g->shm_fd >= 0) { + if (g->shm_exported) + log_debug( + "guest_destroy: a fork child maps the slab, not truncated"); + else if (ftruncate(g->shm_fd, 0) != 0) + log_debug("guest_destroy: slab truncate: %s", strerror(errno)); close(g->shm_fd); g->shm_fd = -1; } diff --git a/src/core/guest.h b/src/core/guest.h index 9054e5fe..0a467df4 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -451,6 +451,7 @@ typedef struct { typedef struct { void *host_base; /* Host pointer to allocated guest memory */ int shm_fd; /* File fd backing host_base for CoW fork (-1 if MAP_ANON) */ + bool shm_exported; /* A fork child was handed the shm_fd file itself */ uint64_t guest_size; /* Total size (determined by IPA capacity) */ uint64_t ipa_base; /* IPA base for hv_vm_map (GUEST_IPA_BASE) */ diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index c9eb8ec8..0a14620b 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -203,9 +203,10 @@ int fork_child_main(int ipc_fd, guest_t g; if (hdr.has_shm) { - /* CoW fork: receive shm fd via SCM_RIGHTS, then map MAP_PRIVATE. This - * gives the child an instant copy-on-write snapshot of the parent's - * entire guest memory, with no region enumeration or byte copying. + /* CoW fork: receive shm fd via SCM_RIGHTS for guest_init_from_shm to + * map. This gives the child an instant copy-on-write snapshot of the + * parent's entire guest memory, with no region enumeration or byte + * copying. */ int shm_fd = -1, shm_count = 0; if (fork_ipc_recv_fds(ipc_fd, &shm_fd, 1, &shm_count) < 0 || @@ -1769,8 +1770,8 @@ int64_t sys_clone(hv_vcpu_t vcpu, goto fail_snapshot; /* CoW fast path: if shm_fd >= 0, send a snapshot of guest memory to the - * child instead of the per-region copy. The child maps that snapshot - * MAP_PRIVATE; subsequent writes on either side are private. + * child instead of the per-region copy; guest_init_from_shm maps it so the + * child's writes stay out of the parent's memory. * * The parent's own mapping cannot be flipped to MAP_PRIVATE here: hv_vm_map * caches the host VA->PA mapping, and a MAP_FIXED remap invalidates it (the @@ -1833,8 +1834,15 @@ int64_t sys_clone(hv_vcpu_t vcpu, * fork (Redis BGSAVE, checkpointing runtimes). On failure the fallback * differs per design above: Rosetta drops use_shm so the region-copy * path runs; native guests keep use_shm and send the live g->shm_fd. + * Exercise the fallback on APFS hosts. */ - snapshot_shm_fd = fork_snapshot_shm_via_clonefile(g->shm_fd); + const char *disable_clone = getenv("ELFUSE_DISABLE_FORK_CLONEFILE"); + if (disable_clone && strcmp(disable_clone, "1") == 0) { + snapshot_shm_fd = -1; + errno = ENOTSUP; + } else { + snapshot_shm_fd = fork_snapshot_shm_via_clonefile(g->shm_fd); + } if (snapshot_shm_fd < 0) { if (g->is_rosetta) { log_warn( @@ -2010,6 +2018,12 @@ int64_t sys_clone(hv_vcpu_t vcpu, goto fail_snapshot; } + /* Only an admitted child keeps the live slab mapped. The flag stays set + * after it exits, which costs only the write-back. + */ + if (use_shm && snapshot_shm_fd < 0) + g->shm_exported = true; + /* The child's inherited slave copies are live from the moment fork returns, * so they go on the shared count here rather than in the child's own init, * which the parent's close of its copy can beat. @@ -2033,8 +2047,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, close(ipc_sock); /* After CoW fork, parent stays on MAP_SHARED because no remap was done. The - * shm fd is kept open so subsequent forks can also use CoW. The child has - * its own MAP_PRIVATE view of the same file. + * shm fd is kept open so subsequent forks can also use CoW. */ /* CLONE_VFORK suspends the parent until the child exits or execs. The diff --git a/tests/probe-disk-writes.c b/tests/probe-disk-writes.c new file mode 100644 index 00000000..db09efa6 --- /dev/null +++ b/tests/probe-disk-writes.c @@ -0,0 +1,65 @@ +/* + * Record the bytes a command wrote to disk, as the kernel charged them + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Usage: probe-disk-writes OUT CMD [ARG...] + * + * Runs CMD, waits for it to exit without reaping it, and writes the + * ri_diskio_byteswritten count of the zombie to OUT. That count includes the + * write-back a process forces by closing the last descriptor of a file with + * dirty mapped pages, which is what test-slab-exit-writes measures; once + * reaped, the count is gone. OUT keeps the number clear of whatever CMD prints. + * Exits with CMD's status. + */ + +#include +#include +#include +#include +#include +#include + +extern char **environ; + +int main(int argc, char **argv) +{ + if (argc < 3) { + fprintf(stderr, "usage: %s OUT CMD [ARG...]\n", argv[0]); + return 2; + } + + pid_t pid; + int rc = posix_spawnp(&pid, argv[2], NULL, NULL, argv + 2, environ); + if (rc != 0) { + fprintf(stderr, "probe-disk-writes: %s: %s\n", argv[2], strerror(rc)); + return 2; + } + + siginfo_t info; + if (waitid(P_PID, (id_t) pid, &info, WEXITED | WNOWAIT) < 0) { + perror("probe-disk-writes: waitid"); + return 2; + } + struct rusage_info_v4 usage; + if (proc_pid_rusage(pid, RUSAGE_INFO_V4, (rusage_info_t *) &usage) != 0) { + perror("probe-disk-writes: proc_pid_rusage"); + return 2; + } + + int status; + if (waitpid(pid, &status, 0) < 0) { + perror("probe-disk-writes: waitpid"); + return 2; + } + FILE *out = fopen(argv[1], "w"); + if (!out || + fprintf(out, "%llu\n", + (unsigned long long) usage.ri_diskio_byteswritten) < 0 || + fclose(out) != 0) { + perror(argv[1]); + return 2; + } + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); +} diff --git a/tests/test-slab-exit.c b/tests/test-slab-exit.c new file mode 100644 index 00000000..b8c20e4a --- /dev/null +++ b/tests/test-slab-exit.c @@ -0,0 +1,141 @@ +/* + * Guest memory at process exit + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Driven by the test-slab-exit-writes and test-slab-exit-fork lanes in + * mk/tests.mk, which judge the outcome on the host: + * + * dirty MIB write every page of MIB of anonymous memory, then exit + * write MIB FILE write MIB to FILE and fsync it, the lane's control for + * its disk-write counter + * orphan DIR fill memory with a pattern and fork; the parent exits + * at once, the child waits up to a minute for DIR/go + * (created after the parent's process is gone), checks + * the pattern, and writes "ok" or the first bad offset + * to DIR/result. Removing DIR cancels the wait. + */ + +#include +#include +#include +#include +#include +#include + +#include "test-util.h" + +#define MIB (1UL << 20) +#define PATTERN_LEN (8 * MIB) + +static unsigned char *map_anon(size_t len) +{ + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + perror("mmap"); + exit(1); + } + return p; +} + +static int dirty(size_t len) +{ + unsigned char *p = map_anon(len); + for (size_t off = 0; off < len; off += 4096) + p[off] = 1; + return 0; +} + +static int write_file(size_t len, const char *path) +{ + static unsigned char chunk[MIB]; + memset(chunk, 0x5a, sizeof(chunk)); + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd < 0) { + perror(path); + return 1; + } + for (size_t done = 0; done < len; done += sizeof(chunk)) { + if (write_fd_all(fd, chunk, sizeof(chunk)) < 0) { + perror("write"); + return 1; + } + } + if (fsync(fd) != 0) { + perror("fsync"); + return 1; + } + return close(fd); +} + +static unsigned char pattern(size_t off) +{ + return (unsigned char) (off * 131 + 7); +} + +/* Return the first offset whose byte is wrong, or PATTERN_LEN if none is. */ +static size_t check(const unsigned char *p) +{ + size_t off = 0; + while (off < PATTERN_LEN && p[off] == pattern(off)) + off++; + return off; +} + +/* Fill memory with the pattern and fork; returns fork's result in both. */ +static pid_t fill_and_fork(unsigned char **p) +{ + *p = map_anon(PATTERN_LEN); + for (size_t off = 0; off < PATTERN_LEN; off++) + (*p)[off] = pattern(off); + pid_t pid = fork(); + if (pid < 0) { + perror("fork"); + exit(1); + } + return pid; +} + +static int orphan(const char *dir) +{ + int dirfd = open(dir, O_RDONLY | O_DIRECTORY); + if (dirfd < 0) { + perror(dir); + return 1; + } + unsigned char *p; + if (fill_and_fork(&p) > 0) { + close(dirfd); + return 0; + } + + /* Removing the scratch directory cancels the child's wait on lane failure. + */ + int waited = 0; + while (faccessat(dirfd, "go", F_OK, 0) != 0) { + if (access(dir, F_OK) != 0 || ++waited > 600) + _exit(1); + usleep(100000); + } + + char msg[64] = "ok\n"; + size_t bad = check(p); + if (bad != PATTERN_LEN) + snprintf(msg, sizeof(msg), "bad byte at %zu\n", bad); + _exit(file_write_at(dirfd, "result", msg) < 0 ? 1 : 0); +} + +int main(int argc, char **argv) +{ + if (argc == 3 && strcmp(argv[1], "dirty") == 0) + return dirty(strtoul(argv[2], NULL, 10) * MIB); + if (argc == 4 && strcmp(argv[1], "write") == 0) + return write_file(strtoul(argv[2], NULL, 10) * MIB, argv[3]); + if (argc == 3 && strcmp(argv[1], "orphan") == 0) + return orphan(argv[2]); + fprintf(stderr, "usage: %s dirty MIB | write MIB FILE | orphan DIR\n", + argv[0]); + return 2; +}