From 4d66b2307f932f636905a3ce2f71aa5b7c0db45f Mon Sep 17 00:00:00 2001 From: Max042004 Date: Tue, 14 Jul 2026 00:54:35 +0800 Subject: [PATCH 1/2] Optimize anonymous mmap and page-fault paths Make anonymous mappings lazy: sys_mmap records the region while first touch creates page tables, commits host memory, and zeros reused backing. Serve common private anonymous read-write requests from per-vCPU EL1 arenas, with publication rings that let the host reconcile mappings under mmap_lock. Keep shim.S focused on exception entry, register-frame preservation, and HVC dispatch by compiling the EL1 arena consumer as freestanding C. The mmap fast-path allocation policy then has a C home that munmap can share. Materialize untouched guest memory before host access and preserve PROT_NONE reservations. Let partial guest writes materialize lazy destinations, use tracked mremap protections for dirty state, and keep neighboring PTEs intact when mremap grows across block boundaries. Cover lazy reuse, refill, fork, and first-touch behavior. Close #165 --- Makefile | 27 + docs/usage.md | 9 + mk/shim.mk | 24 +- src/core/bootstrap.c | 30 + src/core/guest.c | 539 ++++++++++++++++-- src/core/guest.h | 211 +++++-- src/core/launch.c | 9 + src/core/mmap-fastpath.h | 98 ++++ src/core/shim-globals.c | 121 +++- src/core/shim-globals.h | 7 +- src/core/shim-mmap.c | 221 ++++++++ src/core/shim-mmap.h | 20 + src/core/shim.S | 32 +- src/runtime/fork-state.c | 14 +- src/runtime/fork-state.h | 3 +- src/runtime/forkipc.c | 48 +- src/runtime/futex.c | 15 + src/syscall/exec.c | 42 +- src/syscall/internal.h | 7 + src/syscall/mem.c | 819 +++++++++++++++++++++++++--- src/syscall/proc.c | 33 +- src/syscall/signal.c | 25 + src/syscall/syscall.c | 65 ++- src/syscall/sysvipc.c | 13 +- tests/bench-mmap-lazy.c | 111 ++++ tests/bench-mmap.c | 455 ++++++++++++++++ tests/manifest.txt | 4 + tests/test-fork-ipc-protocol-host.c | 7 +- tests/test-mmap-dirty-stats.sh | 45 ++ tests/test-mmap-fastpath-stats.sh | 113 ++++ tests/test-mmap-fastpath.c | 278 ++++++++++ tests/test-mmap-lazy.c | 771 ++++++++++++++++++++++++++ tests/test-mremap.c | 102 ++++ tests/test-thread-churn.c | 12 +- tests/test-tlbi-encoder-host.c | 53 +- 35 files changed, 4094 insertions(+), 289 deletions(-) create mode 100644 src/core/mmap-fastpath.h create mode 100644 src/core/shim-mmap.c create mode 100644 src/core/shim-mmap.h create mode 100644 tests/bench-mmap-lazy.c create mode 100644 tests/bench-mmap.c create mode 100755 tests/test-mmap-dirty-stats.sh create mode 100755 tests/test-mmap-fastpath-stats.sh create mode 100644 tests/test-mmap-fastpath.c create mode 100644 tests/test-mmap-lazy.c diff --git a/Makefile b/Makefile index 45bedb4f..1bf2aa2c 100644 --- a/Makefile +++ b/Makefile @@ -311,6 +311,33 @@ $(BUILD_DIR)/test-sigsuspend: tests/test-sigsuspend.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -Itests -o $@ $< -lpthread +# bench-mmap has a multi-threaded mmap_lock-contention section; needs -lpthread. +$(BUILD_DIR)/bench-mmap: tests/bench-mmap.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + +# test-mmap-lazy races concurrent first touch from several threads. +$(BUILD_DIR)/test-mmap-lazy: tests/test-mmap-lazy.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + +.PHONY: test-mmap-lazy +test-mmap-lazy: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @sh tests/test-mmap-dirty-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-lazy + +# EL1 consumer-mmap integration/stress test. +$(BUILD_DIR)/test-mmap-fastpath: tests/test-mmap-fastpath.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + +.PHONY: test-mmap-fastpath +test-mmap-fastpath: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @sh tests/test-mmap-fastpath-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-fastpath + # test-thread-churn creates >64 threads to force thread-table slot reuse. $(BUILD_DIR)/test-thread-churn: tests/test-thread-churn.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" diff --git a/docs/usage.md b/docs/usage.md index 2ec20943..5d822f21 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -93,6 +93,15 @@ host `KEY=` imports as `KEY=`. An empty variable name is rejected. Given neither `--env` nor `--clear-env`, the guest inherits the host environment unchanged. `--clear-env` starts from nothing, leaving only what `--env` puts back. +### mmap call fast path + +The aarch64 EL1 consumer fast path is enabled by default for +`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)`. +Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted +arenas, and full consumption rings fall back to the normal host syscall path. +Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host +path so observability and translated-guest behavior are unchanged. + ## Common Launch Patterns Run a statically linked guest binary: diff --git a/mk/shim.mk b/mk/shim.mk index 235c635e..f90a3b19 100644 --- a/mk/shim.mk +++ b/mk/shim.mk @@ -1,11 +1,29 @@ -# EL1 kernel shim assembly pipeline +# EL1 kernel shim pipeline # -# shim.S -> shim.o -> shim.bin -> shim_blob.h (C byte array) +# shim.S + freestanding shim-mmap.c -> shim.o -> shim.bin -> shim_blob.h -$(BUILD_DIR)/shim.o: src/core/shim.S | $(BUILD_DIR) +SHIM_CFLAGS := -O2 -Wall -Wextra -Wpedantic -Wshadow \ + -Wstrict-prototypes -Wmissing-prototypes -Wformat=2 \ + -Wimplicit-fallthrough -Wundef -Wnull-dereference \ + -Wno-unused-parameter -ffreestanding -fno-builtin \ + -fno-stack-protector -fno-unwind-tables \ + -fno-asynchronous-unwind-tables -mno-outline-atomics +SHIM_LD ?= ld + +$(BUILD_DIR)/shim-asm.o: src/core/shim.S | $(BUILD_DIR) @echo " AS $<" $(Q)$(SHIM_AS) $(SHIM_ASFLAGS) -o $@ $< +$(BUILD_DIR)/shim-mmap.o: src/core/shim-mmap.c src/core/shim-mmap.h \ + src/core/mmap-fastpath.h src/core/shim-globals.h | $(BUILD_DIR) + @echo " CC $<" + $(Q)$(CC) $(SHIM_CFLAGS) -MMD -MP -MF $(BUILD_DIR)/shim-mmap.d \ + -Isrc -c -o $@ $< + +$(BUILD_DIR)/shim.o: $(BUILD_DIR)/shim-asm.o $(BUILD_DIR)/shim-mmap.o + @echo " LD $@" + $(Q)$(SHIM_LD) -static -arch arm64 -e _start -o $@ $^ + $(BUILD_DIR)/shim.bin: $(BUILD_DIR)/shim.o @echo " OBJCOPY $@" $(Q)$(OBJCOPY) -O binary $< $@ diff --git a/src/core/bootstrap.c b/src/core/bootstrap.c index 985d1a99..8645a093 100644 --- a/src/core/bootstrap.c +++ b/src/core/bootstrap.c @@ -37,6 +37,7 @@ #include "syscall/signal.h" #include "debug/log.h" +#include "core/mmap-fastpath.h" /* Worst case: 7 fixed regions (shim, shim-data, vDSO, brk, stack, mmap RX, mmap * RW) plus up to ELF_MAX_SEGMENTS for both the executable and the interpreter. @@ -138,6 +139,26 @@ static void register_runtime_regions(guest_t *g, size_t shim_bin_len) guest_invalidate_ptes(g, 0, 0x1000); } +/* ELF/shim/stack bytes are populated directly in the slab before their page + * tables become live. Mark their semantic backing regardless of requested + * permissions: a read-only file segment is still nonzero and must be scrubbed + * if a later MAP_FIXED lazy-anonymous mapping reuses the same slab block. + * Synthetic page-table coverage for unallocated mmap space has no semantic + * region and therefore remains clean. + */ +static void mark_registered_backing_dirty(guest_t *g) +{ + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->end <= r->start) + continue; + uint64_t len = r->end - r->start; + if (r->gpa_base > g->guest_size || len > g->guest_size - r->gpa_base) + continue; + guest_dirty_mark_range(g, r->gpa_base, r->gpa_base + len); + } +} + int guest_bootstrap_probe_elf(const char *elf_path, elf_info_t *info) { memset(info, 0, sizeof(*info)); @@ -563,6 +584,7 @@ int guest_bootstrap_prepare(guest_t *g, } register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); startup_trace_step("register_regions", t0); log_debug("TTBR0=0x%llx, IPA base=0x%llx", (unsigned long long) boot->ttbr0, @@ -756,6 +778,13 @@ int guest_bootstrap_create_vcpu(guest_t *g, */ shim_globals_set_singleton(g); + /* Publish the main vCPU's first arena only after shim_globals_init has + * cleared every recycled control slot. Verbose tracing keeps all shim + * syscall fast paths on HVC so the trace remains complete. + */ + if (!verbose) + mmap_fastpath_prepare_vcpu(g, current_thread); + HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CNTKCTL_EL1, CNTKCTL_EL1_EL0_TIMER_EN)); @@ -864,6 +893,7 @@ int guest_bootstrap_rosetta_post_reset(guest_t *g, g->rosetta_guest_base - g->rosetta_va_base, ROSETTA_PATH); register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); int rosetta_argc = 0; const char **rosetta_argv = NULL; diff --git a/src/core/guest.c b/src/core/guest.c index 1a3ef353..444f583d 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -44,9 +44,10 @@ #include "core/startup-trace.h" #include "debug/log.h" #include "utils.h" -#include "runtime/futex.h" /* futex_interrupt_request */ -#include "runtime/thread.h" /* thread_destroy_all_vcpus */ -#include "syscall/proc.h" /* proc_request_exit_group */ +#include "runtime/futex.h" /* futex_interrupt_request */ +#include "runtime/thread.h" /* thread_destroy_all_vcpus */ +#include "syscall/internal.h" /* mmap_lock (lazy fault-in) */ +#include "syscall/proc.h" /* proc_request_exit_group */ #include "syscall/signal.h" #include "syscall/wakeup-pipe.h" @@ -575,6 +576,7 @@ int guest_init(guest_t *g, uint64_t size, uint32_t ipa_bits) */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + pthread_cond_init(&g->materialize_cond, NULL); return 0; } @@ -687,6 +689,7 @@ int guest_init_from_shm(guest_t *g, */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + pthread_cond_init(&g->materialize_cond, NULL); log_debug( "guest: CoW fork: mapped %llu GiB from shm " @@ -833,6 +836,7 @@ void guest_destroy(guest_t *g) close(g->shm_fd); g->shm_fd = -1; } + pthread_cond_destroy(&g->materialize_cond); } /* Check whether a candidate IPA range [gpa, gpa+size) overlaps the primary @@ -1584,11 +1588,11 @@ static uint64_t gva_contiguous_avail(const guest_t *g, * (MEM_PERM_R/W/X bitmask). The walk continues across adjacent L2/L3 entries * until a mapping, permission, or physical-contiguity break is found. */ -static void *gva_resolve_perm(const guest_t *g, - uint64_t gva, - uint64_t *avail, - int required_perms, - uint64_t avail_limit) +static void *gva_resolve_perm_walk(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit) { /* Always walk page tables to enforce permissions. The guest slab is * identity-mapped (GVA == GPA == offset), but L2 block descriptors carry @@ -1649,14 +1653,114 @@ static void *gva_resolve_perm(const guest_t *g, return NULL; } +/* Host-access fault-in for lazy (deferred-PTE) regions. + * + * A syscall may target guest memory the guest itself has never touched: a + * fresh calloc()-style arena handed straight to read(2), a futex word inside + * an untouched mapping, an iovec into a new heap chunk. The guest-fault path + * (guest_materialize_lazy via the EL1 shim) never runs for those, so the + * page-table walk in gva_resolve_perm_walk fails even though the access is + * legal. Materialize the touched blocks here, then let the caller re-walk. + * + * Locking: takes mmap_lock; callers of the resolve API that already hold + * mmap_lock must use the _nofault variants. Do not call while holding any + * lock that ranks after mmap_lock in the ordering (syscall/internal.h); + * callers that resolve under such locks (futex bucket paths) pre-fault at + * their lock-free entry points so the hook never engages there. + * + * TLBI: guest_materialize_lazy accumulates TLBI requests in the calling + * thread's per-vCPU slot. On a vCPU thread the syscall epilogue emits them. + * On non-vCPU threads the request is lost, which is self-healing: a vCPU + * that still holds a stale negative TLB entry re-faults, and the + * already-valid early-return in guest_materialize_lazy re-issues a page + * TLBI for it without re-zeroing. + * + * Returns 0 if at least one block in [gva, gva+len) is now materialized (or + * already was), -1 if the range intersects no materializable lazy region. + */ +int guest_lazy_faultin_locked(const guest_t *cg, uint64_t gva, uint64_t len) +{ + /* The lazy machinery mutates page tables; the const on the resolve API + * reflects the pure-walk fast path, not this slow path. + */ + guest_t *g = (guest_t *) (uintptr_t) cg; + + if (gva >= g->guest_size) + return -1; /* High-VA / non-identity ranges are never lazy. */ + if (len == 0) + len = 1; + uint64_t end = (len > g->guest_size - gva) ? g->guest_size : gva + len; + + int rc = -1; + for (uint64_t b = gva & ~(uint64_t) (BLOCK_2MIB - 1); b < end; + b += BLOCK_2MIB) { + uint64_t probe = (b > gva) ? b : gva; + if (guest_materialize_lazy(g, probe) == 0) + rc = 0; + } + return rc; +} + +static int gva_lazy_faultin(const guest_t *cg, + uint64_t gva, + uint64_t len, + int required_perms) +{ + (void) required_perms; /* Region prot gates the retry walk, not this. */ + + int rc; + mmap_lock_acquire((guest_t *) (uintptr_t) cg); + rc = guest_lazy_faultin_locked(cg, gva, len); + mmap_lock_release(); + return rc; +} + +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len) +{ + return gva_lazy_faultin(g, gva, len, MEM_PERM_R); +} + +static void *gva_resolve_perm(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit, + bool allow_faultin) +{ + void *ptr = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (!allow_faultin) + return ptr; + + /* Window the fault-in to what the caller actually needs. Length-less + * resolves (guest_ptr / guest_ptr_avail) materialize a single block; + * their callers iterate and re-enter here per chunk. + */ + uint64_t want = (avail_limit == UINT64_MAX) ? 1 : avail_limit; + if (!ptr) { + if (gva_lazy_faultin(g, gva, want, required_perms) < 0) + return NULL; + return gva_resolve_perm_walk(g, gva, avail, required_perms, + avail_limit); + } + if (avail && *avail < want && gva <= UINT64_MAX - *avail && + gva_lazy_faultin(g, gva + *avail, want - *avail, required_perms) == 0) { + void *again = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (again) + ptr = again; + } + return ptr; +} + void *guest_ptr(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX, true); } void *guest_ptr_w(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX, true); } void *guest_ptr_avail(const guest_t *g, @@ -1664,7 +1768,19 @@ void *guest_ptr_avail(const guest_t *g, uint64_t *avail, int required_perms) { - return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX); + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, true); +} + +/* Pure page-table walk without lazy fault-in. For callers that already hold + * mmap_lock (e.g. the stale-TLB re-walk in the EL0 fault handler, which runs + * after guest_materialize_lazy has already been consulted). + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms) +{ + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, false); } void *guest_ptr_bound(const guest_t *g, @@ -1673,7 +1789,7 @@ void *guest_ptr_bound(const guest_t *g, int required_perms, uint64_t len_limit) { - return gva_resolve_perm(g, gva, avail, required_perms, len_limit); + return gva_resolve_perm(g, gva, avail, required_perms, len_limit, true); } static inline int guest_copy(const guest_t *g, @@ -1681,7 +1797,8 @@ static inline int guest_copy(const guest_t *g, void *dst, const void *src, size_t len, - int required_perms) + int required_perms, + bool allow_faultin) { if (len == 0) return 0; @@ -1695,7 +1812,7 @@ static inline int guest_copy(const guest_t *g, while (copied < len) { uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, required_perms, - (uint64_t) (len - copied)); + (uint64_t) (len - copied), allow_faultin); if (!ptr) return -1; size_t chunk = len - copied; @@ -1715,7 +1832,7 @@ static inline int guest_copy(const guest_t *g, int guest_read(const guest_t *g, uint64_t gva, void *dst, size_t len) { - return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R); + return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R, true); } int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) @@ -1731,7 +1848,12 @@ int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) int guest_write(guest_t *g, uint64_t gva, const void *src, size_t len) { - return guest_copy(g, gva, NULL, src, len, MEM_PERM_W); + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, true); +} + +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len) +{ + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, false); } size_t guest_write_partial(guest_t *g, @@ -1743,7 +1865,7 @@ size_t guest_write_partial(guest_t *g, while (done < len) { uint64_t avail; void *dst = gva_resolve_perm(g, gva + done, &avail, MEM_PERM_W, - (uint64_t) (len - done)); + (uint64_t) (len - done), true); if (!dst) return done; @@ -1787,7 +1909,7 @@ int guest_read_str(const guest_t *g, uint64_t gva, char *dst, size_t max) break; uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, MEM_PERM_R, - (uint64_t) (limit - copied)); + (uint64_t) (limit - copied), true); if (!ptr) break; @@ -1877,6 +1999,7 @@ void guest_reset(guest_t *g) if (gpa > g->guest_size || len > g->guest_size - gpa) continue; /* backing lies outside the primary slab */ memset((uint8_t *) g->host_base + gpa, 0, len); + guest_dirty_clear_zeroed_range(g, gpa, gpa + len); } /* Zero page table pool (not tracked in region array) */ @@ -3009,6 +3132,11 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) if (!finalize_block_perms(g, regions, n)) return 0; + for (int r = 0; r < n; r++) { + if (regions[r].perms & MEM_PERM_W) + guest_dirty_mark_range(g, regions[r].gpa_start, regions[r].gpa_end); + } + guest_pt_gen_bump(g); return ttbr0; } @@ -3145,6 +3273,47 @@ int guest_extend_page_tables(guest_t *g, return 0; } +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || !g->ttbr0) + return end; + uint64_t base = g->ipa_base; + uint64_t *l0 = pt_at(g, g->ttbr0 - base); + if (!l0) + return end; + + va &= ~(uint64_t) (BLOCK_2MIB - 1); + while (va < end) { + uint64_t ipa = base + va; + unsigned l0_idx = (unsigned) (ipa / (512ULL * BLOCK_1GIB)); + if (l0_idx >= 512) + return end; + if (!(l0[l0_idx] & PT_VALID)) { + uint64_t next_ipa = (uint64_t) (l0_idx + 1) * 512ULL * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; /* wrap guard */ + va = next_ipa - base; + continue; + } + uint64_t *l1 = pt_at(g, (l0[l0_idx] & 0xFFFFFFFFF000ULL) - base); + if (!l1) + return end; + unsigned l1_idx = + (unsigned) ((ipa % (512ULL * BLOCK_1GIB)) / BLOCK_1GIB); + if (!(l1[l1_idx] & PT_VALID)) { + uint64_t next_ipa = (ipa / BLOCK_1GIB + 1) * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; + va = next_ipa - base; + continue; + } + return va; + } + return end; +} + bool guest_va_block_mapped(const guest_t *g, uint64_t va) { if (!g || !g->ttbr0 || (va & (BLOCK_2MIB - 1))) @@ -3349,8 +3518,13 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) for (uint64_t addr = start; addr < end;) { uint64_t *l2_entry = find_l2_entry(g, addr); if (!l2_entry) { - /* No L2 entry (already unmapped); skip this 2MiB block */ - addr = ALIGN_2MIB_UP(addr + 1); + /* No L2 table (L0/L1 slot absent): nothing to invalidate in this + * block. Skip whole absent 1GiB/512GiB slots at once; a lazy + * multi-GiB mmap invalidates its stale range on every allocation + * and would otherwise pay one four-level walk per 2MiB of empty + * address space. + */ + addr = guest_va_next_present_block(g, ALIGN_2MIB_UP(addr + 1), end); continue; } @@ -3570,6 +3744,8 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) addr = page_end; } + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, start, end); guest_pt_gen_bump(g); return 0; } @@ -3652,15 +3828,131 @@ int guest_install_va_pages(guest_t *g, if (!bcast && changed_hi > changed_lo) tlbi_request_range(changed_lo, changed_hi); + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, gpa, gpa + length); guest_pt_gen_bump(g); return 0; } -/* Lazy page materialization for MAP_NORESERVE. */ +/* Lazy page materialization for deferred-PTE (private anonymous / + * MAP_NORESERVE) regions. + */ -int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start) { - /* Find the noreserve region containing this offset */ + if (!g || block_start >= g->guest_size) + return true; + uint64_t block = block_start / BLOCK_2MIB; + return (g->dirty_blocks[block >> 6] & (1ULL << (block & 63))) != 0; +} + +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = start / BLOCK_2MIB; + uint64_t last = (end - 1) / BLOCK_2MIB; + for (uint64_t block = first; block <= last; block++) + g->dirty_blocks[block >> 6] |= 1ULL << (block & 63); +} + +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = ALIGN_2MIB_UP(start); + uint64_t last = ALIGN_2MIB_DOWN(end); + for (uint64_t addr = first; addr < last; addr += BLOCK_2MIB) { + uint64_t block = addr / BLOCK_2MIB; + g->dirty_blocks[block >> 6] &= ~(1ULL << (block & 63)); + } +} + +static bool materialize_claim_overlaps(const guest_materialize_claim_t *claim, + uint64_t start, + uint64_t end) +{ + return claim->active && start < claim->end && end > claim->start; +} + +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (;;) { + bool overlap = false; + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], start, + end)) { + overlap = true; + break; + } + } + if (!overlap) + return; + mmap_lock_cond_wait(g, &g->materialize_cond); + } +} + +void guest_materialize_wait_all_locked(guest_t *g) +{ + guest_materialize_wait_range_locked(g, 0, UINT64_MAX); +} + +static int materialize_claim_alloc_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + guest_materialize_wait_range_locked(g, start, end); + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + guest_materialize_claim_t *claim = &g->materialize_claims[i]; + if (!claim->active) { + claim->start = start; + claim->end = end; + claim->active = true; + return i; + } + } + return -1; +} + +static void materialize_claim_release_locked(guest_t *g, int slot) +{ + if (slot < 0) + return; + g->materialize_claims[slot].active = false; + pthread_cond_broadcast(&g->materialize_cond); +} + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. Callers + * must hold mmap_lock; used to detect blocks a concurrent fault already + * materialized. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va) +{ + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + return false; + if ((*l2_entry & 3) == 1) + return true; /* 2MiB block descriptor */ + uint64_t *l3 = pt_at(g, (*l2_entry & 0xFFFFFFFFF000ULL) - g->ipa_base); + if (!l3) + return false; + unsigned l3_idx = + (unsigned) (((g->ipa_base + va) % BLOCK_2MIB) / PAGE_SIZE); + return (l3[l3_idx] & PT_VALID) != 0; +} + +static int guest_materialize_lazy_one(guest_t *g, uint64_t fault_offset) +{ +retry:; + /* Find the lazy region containing this offset */ const guest_region_t *region = NULL; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start <= fault_offset && @@ -3671,7 +3963,35 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) } if (!region) - return -1; /* Not a noreserve region */ + return -1; /* Not a lazy region */ + + /* PROT_NONE is a reservation, not a mapping: a fault inside it is a + * genuine SIGSEGV, never a materialization request. Without this check a + * PROT_NONE|MAP_NORESERVE region would be silently granted read + * permission by the perms fallback below. + */ + if (region->prot == LINUX_PROT_NONE) + return -1; + + uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); + uint64_t block_end = block_start + BLOCK_2MIB; + if (block_end > g->guest_size) + block_end = g->guest_size; + + /* Already materialized: another thread (concurrent guest fault or a + * host-side fault-in on a syscall path) completed this block while this + * vCPU was queued on mmap_lock, and the guest may have written real data + * through the new PTEs since. Running the memset below again would wipe + * those writes. The fault that got us here is then either a stale + * negative TLB entry or an in-flight retry. Invalidate the whole + * materialization block so this path follows the same one-RVAE-per-block + * contract as a newly installed block. + */ + if (guest_va_pte_valid(g, fault_offset)) { + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]++; + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + return 0; + } /* Materialize one 2MiB block containing the fault address. This is the * smallest granule that guest_extend_page_tables works with. For the common @@ -3679,12 +3999,17 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) * trade-off: it avoids over-committing the large reservation while keeping * the fault rate manageable. */ - uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); - uint64_t block_end = block_start + BLOCK_2MIB; - - /* Clamp to guest size */ - if (block_end > g->guest_size) - block_end = g->guest_size; + /* A sibling may be zeroing another window in this block without the lock. + * Wait before inspecting regions/PTEs, then restart because a mutator that + * was itself waiting may have changed the region layout first. + */ + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], block_start, + block_end)) { + guest_materialize_wait_range_locked(g, block_start, block_end); + goto retry; + } + } uint64_t materialize_start = (block_start > region->start) ? block_start : region->start; @@ -3705,18 +4030,70 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) if (perms == 0) perms = MEM_PERM_R; /* At minimum readable */ + /* Zero the window BEFORE any PTE becomes valid. The moment a descriptor + * is published, sibling vCPUs with no stale TLB entry can write through + * it without ever faulting; zeroing afterwards (the historical order) + * would wipe such a write. The slab is host memory, so zeroing needs no + * PTEs, and every writer that could touch the window first goes through + * mmap_lock (guest faults and host-side fault-in alike), so nothing can + * write between this memset and the descriptor stores below. Skip pages + * that are already valid: they belong to a previously materialized + * neighbor in the same block and may hold live data. + */ + int claim_slot = -1; + bool dirty = guest_block_may_be_dirty(g, block_start); + if (dirty) { + uint64_t zero_pages[8] = {0}; + bool any_valid = false; + for (uint64_t pg = materialize_start; pg < materialize_end; + pg += PAGE_SIZE) { + unsigned page = (unsigned) ((pg - block_start) / PAGE_SIZE); + if (guest_va_pte_valid(g, pg)) + any_valid = true; + else + zero_pages[page >> 6] |= 1ULL << (page & 63); + } + + claim_slot = materialize_claim_alloc_locked(g, block_start, block_end); + if (claim_slot >= 0) + mmap_lock_release(); + for (unsigned page = 0; page < 512;) { + if (!(zero_pages[page >> 6] & (1ULL << (page & 63)))) { + page++; + continue; + } + unsigned first = page; + do { + page++; + } while (page < 512 && + (zero_pages[page >> 6] & (1ULL << (page & 63)))); + memset((uint8_t *) g->host_base + block_start + + (uint64_t) first * PAGE_SIZE, + 0, (uint64_t) (page - first) * PAGE_SIZE); + } + if (claim_slot >= 0) + mmap_lock_acquire(g); + if (!any_valid && materialize_start == block_start && + materialize_end == block_end) + guest_dirty_clear_zeroed_range(g, block_start, block_end); + } + /* Create page table entries. guest_extend_page_tables creates L2 block * descriptors but skips existing table descriptors (L2->L3 splits). * guest_update_perms handles the L3 case: if guest_invalidate_ptes * previously split the block and invalidated the L3 entries, update_perms * recreates them with correct perms. */ - if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) + if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (partial_block) { - if (guest_split_block(g, block_start) < 0) + if (guest_split_block(g, block_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } /* If this block had no page-table entry before the lazy fault, * guest_extend_page_tables() necessarily created a full 2MiB block. @@ -3726,25 +4103,99 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) */ if (!had_mapping) { if (block_start < materialize_start && - guest_invalidate_ptes(g, block_start, materialize_start) < 0) + guest_invalidate_ptes(g, block_start, materialize_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (materialize_end < block_end && - guest_invalidate_ptes(g, materialize_end, block_end) < 0) + guest_invalidate_ptes(g, materialize_end, block_end) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } } } guest_update_perms(g, materialize_start, materialize_end, perms); - - /* Zero the materialized memory. Only zero within the region boundaries to - * avoid clobbering adjacent data. - */ - if (materialize_end > materialize_start) - memset((uint8_t *) g->host_base + materialize_start, 0, - materialize_end - materialize_start); - - /* The page-table helpers above already requested the matching TLBI; no - * additional flush is needed here. + /* One 2MiB materialization gets one block-sized RVAE1IS. This is cheaper + * than per-page invalidation and covers every negative entry that may have + * been cached for the block while its descriptors were invalid. */ + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + g->materialize_stats[dirty ? GUEST_MATERIALIZE_DIRTY_MEMSET + : GUEST_MATERIALIZE_CLEAN_SKIP]++; + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES] += + materialize_end - materialize_start; + materialize_claim_release_locked(g, claim_slot); return 0; } + +int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +{ + return guest_materialize_lazy_one(g, fault_offset); +} + +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset) +{ + typedef struct { + guest_t *guest; + uint64_t next_block; + unsigned streak; + } fault_around_state_t; + static _Thread_local fault_around_state_t state; + + uint64_t block = ALIGN_2MIB_DOWN(fault_offset); + if (state.guest == g && block == state.next_block) { + if (state.streak < 4) + state.streak++; + } else { + state.guest = g; + state.streak = 0; + } + + const guest_region_t *region = guest_region_find(g, fault_offset); + if (!region || !region->noreserve || region->prot == LINUX_PROT_NONE) + return -1; + + unsigned blocks = 1U << state.streak; + if (blocks > 16) + blocks = 16; + uint64_t region_last = ALIGN_2MIB_UP(region->end); + if (region_last > g->guest_size) + region_last = g->guest_size; + + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + if (guest_block_may_be_dirty(g, ahead) && blocks > 4) { + blocks = 4; + break; + } + } + + int result = -1; + uint64_t last = block; + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + /* The current block must probe the actual FAR. A fast-path mmap can + * extend an already materialized region within the same 2MiB block; + * probing the region/block start would see an older valid page and + * return without installing the newly faulted page. Ahead blocks have + * no FAR, so use their first address inside the region. + */ + uint64_t probe = (i == 0) + ? fault_offset + : (ahead < region->start ? region->start : ahead); + int rc = guest_materialize_lazy_one(g, probe); + if (i == 0) + result = rc; + if (rc < 0) + break; + last = ahead; + } + if (result == 0) + state.next_block = last + BLOCK_2MIB; + return result; +} diff --git a/src/core/guest.h b/src/core/guest.h index 9e85aa59..2d99e9be 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -241,7 +242,11 @@ typedef struct { uint64_t offset; /* File offset (for /proc/self/maps display) */ int backing_fd; /* Duplicated host fd for file-backed mappings, or -1 */ bool shared; /* MAP_SHARED (writes should propagate) */ - bool noreserve; /* MAP_NORESERVE: PTEs deferred until fault */ + bool noreserve; /* Lazy region: PTEs and zeroing deferred until first + * touch. Set for MAP_NORESERVE and for all private + * anonymous mappings (sys_mmap folds the latter into + * the tracked MAP_NORESERVE bit; not guest visible). + */ bool backing_ro; /* MAP_SHARED region whose backing_fd was opened * without write access, so its Linux max_prot is * capped to PROT_READ. sys_mprotect must reject any @@ -278,6 +283,8 @@ typedef struct { * TLBI_BROADCAST -> X8 = 1 (TLBI VMALLE1IS, broadest) * TLBI_RANGE -> X8 = 3, X9 = start VA, X10 = page count * (TLBI VAE1IS loop preserves unrelated TLB entries) + * TLBI_RANGE_LARGE -> X8 = 4, X9 = encoded range operand + * (single TLBI RVAE1IS) * X8 = 2 is reserved for the execve drop-frame marker the shim handles * separately; it is never produced by the accumulator. */ @@ -300,14 +307,12 @@ typedef enum { */ #define TLBI_SELECTIVE_MAX_PAGES 16 -/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. With SCALE=0 the - * RVAE1IS operand encoding covers (NUM+1)*2 pages with NUM in [0..31], so a - * single instruction reaches 64 pages == 256 KiB. Beyond that the host would - * need SCALE=1 (NUM*64 step), which over-invalidates for the typical - * dynamic-linker RELRO / glibc-bring-up storm sizes seen in practice; stay at - * SCALE=0 for now and broadcast above 64 pages. +/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. SCALE=0 covers up to + * 64 pages, SCALE=1 up to 2048 pages, and SCALE=2 much larger ranges. 32768 + * pages (128 MiB) covers the lazy fault-around window with one instruction + * while still fitting tlbi_request_t.pages in uint16_t. */ -#define TLBI_RVAE_MAX_PAGES 64 +#define TLBI_RVAE_MAX_PAGES 32768 /* TLBI RVAE1IS operand bit-field constants. Per ARM ARM DDI 0487J.a D8.7.6 the * operand layout is: @@ -323,28 +328,37 @@ typedef enum { */ #define RVAE_OPERAND_BADDR_MASK ((1ULL << 37) - 1) #define RVAE_OPERAND_NUM_SHIFT 39 +#define RVAE_OPERAND_SCALE_SHIFT 44 #define RVAE_OPERAND_TG_4KB (1ULL << 46) /* Pure encoder: build the TLBI RVAE1IS Xt operand from a 4 KiB-aligned VA and a - * page count in the SCALE=0 range (1..TLBI_RVAE_MAX_PAGES). Lives in the header - * as static inline so tlbi_request_emit_to_vcpu and any future caller - * (host-side unit tests included) compile to the same expression. NUM = - * ceil(pages / 2) - 1 over-invalidates odd page counts by exactly one page, - * which is a perf-only side effect (the extra invalidation evicts a neighbour - * TLB entry that the guest's next access reloads). pages < 2 is clamped to 2 - * because SCALE=0 NUM=0 means 2 pages -- the encoder cannot represent a single - * page through RVAE1IS; single-page callers go through the per-page VAE1IS path - * instead, but the clamp keeps the encoder total in any pathological input. + * page count in the supported SCALE=0..2 range. Lives in the header so the + * emit path and host unit tests use the same expression. Each NUM step covers + * 2^(5*SCALE+1) pages; the normalizer aligns and widens callers to that unit. + * pages < 2 is clamped to 2 because SCALE=0 NUM=0 is the smallest encodable + * range. Single-page callers normally use VAE1IS instead. */ +static inline uint64_t tlbi_rvae_unit_pages(uint64_t pages) +{ + if (pages <= 64) + return 2; /* SCALE=0 */ + if (pages <= 2048) + return 64; /* SCALE=1 */ + return 2048; /* SCALE=2 */ +} + static inline uint64_t tlbi_rvae1is_operand(uint64_t start_va, uint16_t pages) { if (pages < 2) pages = 2; + uint64_t scale = pages <= 64 ? 0 : (pages <= 2048 ? 1 : 2); + uint64_t unit_pages = 1ULL << (5 * scale + 1); uint64_t baddr = (start_va >> 12) & RVAE_OPERAND_BADDR_MASK; - uint64_t num = ((pages + 1) / 2) - 1; + uint64_t num = ((pages + unit_pages - 1) / unit_pages) - 1; if (num > 31) num = 31; - return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | RVAE_OPERAND_TG_4KB; + return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | + (scale << RVAE_OPERAND_SCALE_SHIFT) | RVAE_OPERAND_TG_4KB; } /* Runtime feature flag: TRUE when the host PE implements FEAT_TLBIRANGE @@ -358,10 +372,9 @@ typedef struct { uint8_t icache_flush; /* 1 = the change introduced executable content * visible to EL0, so the shim must IC IALLU * after the TLBI sequence. 0 = data-only - * change, skip the I-cache invalidation. - */ - uint16_t pages; /* Page count when kind == TLBI_RANGE (1..MAX) */ - uint64_t start; /* Page-aligned VA when kind == TLBI_RANGE */ + * change, skip the I-cache invalidation. */ + uint16_t pages; /* Page count for either range kind (1..MAX) */ + uint64_t start; /* Page-aligned VA for either range kind */ } tlbi_request_t; /* Layout contract: 16 bytes (1+1+2+4 padding+8). Documents the padding and pins @@ -414,6 +427,32 @@ typedef struct { uint64_t next; /* Bump offset; (next + BLOCK_2MIB) > size means full */ } guest_overflow_t; +/* One conservative "may contain nonzero bytes" bit per 2 MiB primary-slab + * block. The largest supported slab is 1 TiB, so this costs 64 KiB per guest. + * All access is serialized by mmap_lock. + */ +#define GUEST_DIRTY_BLOCKS_MAX ((1ULL << 40) / BLOCK_2MIB) +#define GUEST_DIRTY_WORDS (GUEST_DIRTY_BLOCKS_MAX / 64) + +enum { + GUEST_MATERIALIZE_CLEAN_SKIP = 0, + GUEST_MATERIALIZE_DIRTY_MEMSET, + GUEST_MATERIALIZE_ALREADY_VALID, + GUEST_MATERIALIZE_WINDOW_BYTES, + GUEST_MATERIALIZE_STATS_N, +}; + +/* Dirty-block zeroing claims. A claim makes its block's invalid PTE window + * stable while the expensive host memset runs without mmap_lock. Waiters use + * one guest-wide condition variable; the fixed table bounds host allocation + * and is ample for the vCPU limit. + */ +#define GUEST_MATERIALIZE_CLAIMS 64 +typedef struct { + uint64_t start, end; + bool active; +} guest_materialize_claim_t; + /* Guest state. */ typedef struct { void *host_base; /* Host pointer to allocated guest memory */ @@ -541,6 +580,11 @@ typedef struct { */ _Atomic uint64_t pt_gen; + uint64_t dirty_blocks[GUEST_DIRTY_WORDS]; + uint64_t materialize_stats[GUEST_MATERIALIZE_STATS_N]; + guest_materialize_claim_t materialize_claims[GUEST_MATERIALIZE_CLAIMS]; + pthread_cond_t materialize_cond; + /* Optional HVC 6 embedder extension hook. * * Native AArch64 guests reach this through HVC 6. When the build enables @@ -662,8 +706,8 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) hv_vcpu_set_reg(vcpu, HV_REG_X11, cpu_tlbi_req.icache_flush ? 1 : 0); break; case TLBI_RANGE_LARGE: { - /* Single-shot TLBI RVAE1IS for ranges in (16..64] pages. The operand - * format and the SCALE=0 / TG=01 / ASID=0 assumptions are documented at + /* Single-shot TLBI RVAE1IS for ranges above the selective cap. The + * SCALE/NUM format and TG=01 assumption are documented at * tlbi_rvae1is_operand above. ASID stays 0 because the shim runs * single-ASID (TCR_EL1.A1=0, TTBR0 ASID=0; rosetta does not allocate a * separate ASID). If a future change introduces non-zero ASIDs, the @@ -685,6 +729,31 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) tlbi_request_clear(); } +/* RVAE1IS requires BaseADDR alignment to its SCALE granule. Widen a requested + * interval to that granule, repeating if widening crosses a SCALE threshold. + * Over-invalidation is architecturally harmless and preserves unrelated TLB + * entries far better than a VMALLE1IS broadcast. + */ +static inline bool tlbi_rvae_normalize(uint64_t *start, uint64_t *end) +{ + for (int pass = 0; pass < 3; pass++) { + uint64_t pages = (*end - *start) >> 12; + if (pages <= TLBI_SELECTIVE_MAX_PAGES) + return true; + uint64_t unit_pages = tlbi_rvae_unit_pages(pages); + uint64_t unit_bytes = unit_pages << 12; + uint64_t s = *start & ~(unit_bytes - 1); + if (*end > UINT64_MAX - (unit_bytes - 1)) + return false; + uint64_t e = (*end + unit_bytes - 1) & ~(unit_bytes - 1); + *start = s; + *end = e; + if (((e - s) >> 12) <= unit_pages * 32) + return ((e - s) >> 12) <= TLBI_RVAE_MAX_PAGES; + } + return false; +} + static inline void tlbi_request_range(uint64_t start, uint64_t end) { if (cpu_tlbi_req.kind == TLBI_BROADCAST) @@ -714,6 +783,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) */ uint64_t large_cap = g_tlbi_range_supported ? TLBI_RVAE_MAX_PAGES : TLBI_SELECTIVE_MAX_PAGES; + if (g_tlbi_range_supported && n > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&s, &e)) { + tlbi_request_broadcast(); + return; + } + n = (e - s) >> 12; + } if (n > large_cap) { tlbi_request_broadcast(); return; @@ -745,6 +821,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) uint64_t us = s < es ? s : es; uint64_t ue = e > ee ? e : ee; uint64_t un = (ue - us) >> 12; + if (g_tlbi_range_supported && un > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&us, &ue)) { + tlbi_request_broadcast(); + return; + } + un = (ue - us) >> 12; + } if (un > large_cap) { tlbi_request_broadcast(); return; @@ -983,8 +1066,8 @@ int guest_install_va_pages(guest_t *g, uint64_t gpa, int perms); -/* Query whether a 2 MiB TTBR0 VA block already has a leaf mapping. - * Returns true only for a present L2 block descriptor. +/* Query whether a 2 MiB TTBR0 VA block already has any L2 entry, either a + * block descriptor or an L3 table descriptor. */ bool guest_va_block_mapped(const guest_t *g, uint64_t va); @@ -1010,6 +1093,38 @@ static inline bool guest_kbuf_user_va_overlap(uint64_t va, uint64_t size) */ void *guest_ptr(const guest_t *g, uint64_t gva); +/* Like guest_ptr_avail but never triggers lazy fault-in. For callers that + * already hold mmap_lock. + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms); + +/* Materialize any lazy (deferred-PTE) blocks intersecting [gva, gva+len) so + * later resolves under locks that rank after mmap_lock (futex buckets) do + * not have to. Takes mmap_lock; call only from lock-free context. Returns 0 + * if anything was (or already is) materialized, -1 otherwise; callers that + * merely pre-fault can ignore the result. + */ +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len); + +/* Same as guest_lazy_faultin for callers that already hold mmap_lock (e.g. + * SC_LOCKED syscall handlers about to guest_read/guest_write a lazy range: + * the resolve-time hook would self-deadlock re-acquiring mmap_lock, so they + * must materialize up front through this variant). + */ +int guest_lazy_faultin_locked(const guest_t *g, uint64_t gva, uint64_t len); + +/* Smallest block-aligned va' in [va, end) whose 1GiB L1 slot is present in + * the page tables, or end if none. Lets range walkers skip absent 1GiB / + * 512GiB slots in O(1) instead of probing every 2MiB block. Locking: callers + * MUST hold mmap_lock. + */ +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end); + /* Get a host pointer for a guest virtual address (write access). * Returns NULL if gva is out of bounds or not writable. */ @@ -1069,6 +1184,12 @@ size_t guest_write_partial(guest_t *g, const void *src, size_t len); +/* Same copy without lazy materialization. Callers that already hold mmap_lock + * can use this after guest_lazy_faultin_locked(); an invalid destination then + * fails instead of recursively trying to acquire mmap_lock. + */ +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len); + /* Optimized host-to-guest copy for small fixed-size outputs. Uses a direct * guest pointer when the full range is contiguous and writable, otherwise falls * back to guest_write() for boundary-crossing safety. @@ -1343,12 +1464,34 @@ bool guest_region_range_has_ro_shared_backing(const guest_t *g, uint64_t start, uint64_t end); -/* Try to materialize a lazy (MAP_NORESERVE) page at the given offset. Called - * from the data/instruction abort handler when the faulting address falls - * within a noreserve region. Creates page table entries for one 2MiB block - * containing the fault address, zeros the memory, and clears the noreserve flag - * for the materialized sub-range. - * Returns 0 on success (caller should TLBI and retry), -1 if the offset is not - * in a noreserve region. +/* Try to materialize a lazy (deferred-PTE: private anonymous or + * MAP_NORESERVE) page at the given offset. Called from the data/instruction + * abort handler when the faulting address falls within a lazy region, and + * from the host-access fault-in path when a syscall targets a lazy range the + * guest has not touched yet. Creates page table entries for one 2MiB block + * containing the fault address. A slab block known to be clean skips zeroing; + * a possibly-dirty block zeros invalid pages before publishing them. A block + * that is already valid returns success without re-zeroing + * (concurrent-fault idempotence). + * Returns 0 on success (caller should TLBI and retry), -1 if the offset is + * not in a lazy region or the region is PROT_NONE. + * Locking: callers MUST hold mmap_lock. */ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset); + +/* Guest-fault variant with per-vCPU sequential fault-around. */ +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset); + +/* Dirty-map and in-flight-claim helpers. Callers hold mmap_lock. */ +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start); +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end); +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end); +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end); +void guest_materialize_wait_all_locked(guest_t *g); + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. + * Locking: callers MUST hold mmap_lock. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va); diff --git a/src/core/launch.c b/src/core/launch.c index bbe068b7..5b8a195b 100644 --- a/src/core/launch.c +++ b/src/core/launch.c @@ -26,6 +26,7 @@ #include "core/bootstrap.h" #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/shim-globals.h" #include "core/sysroot.h" @@ -192,6 +193,14 @@ int elfuse_launch(const launch_args_t *args) 0) goto fail; + /* Tracing/debuggers require every mmap to reach host dispatch so syscall + * observation and region state advance in lockstep. This also prevents a + * trace-gated shim from leaving invisible arena reservations that perturb + * the host slow path's address-hint behavior. + */ + if (args->verbose || args->gdb_port > 0) + mmap_fastpath_disable(&g); + /* GDB setup must happen before the first run so entry-stop and hardware * breakpoints can affect the initial vCPU. */ diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h new file mode 100644 index 00000000..b31172e1 --- /dev/null +++ b/src/core/mmap-fastpath.h @@ -0,0 +1,98 @@ +/* + * Per-vCPU EL1 anonymous-mmap consumer rings. + * + * The host is the sole producer of arenas and the sole consumer of ring + * entries. EL1 only bump-allocates VA and appends descriptions. Control + * blocks live in the EL1-only shim-data mapping and are selected from SP_EL1's + * per-thread stack slot, so no guest-visible register ABI is consumed. + */ + +#pragma once + +#include +#include +#include + +#include "core/guest.h" + +typedef struct thread_entry thread_entry_t; + +#define SHIM_MMAP_CONTROL_BASE 0x20000u +#define SHIM_MMAP_CONTROL_STRIDE 0x800u +#define SHIM_MMAP_RING_SIZE 16u +#define SHIM_MMAP_CTRL_ENABLED 0x1u + +#define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) +#define MMAP_FAST_ARENA_MAX (1ULL * 1024 * 1024 * 1024) +#define MMAP_FAST_HISTORY_MULTIPLIER 16u + +enum { + SHIM_MMAP_COUNTER_SHAPE_MISS = 0, + SHIM_MMAP_COUNTER_CAPACITY_MISS, + SHIM_MMAP_COUNTER_RING_FULL, + SHIM_MMAP_COUNTER_GENERATION_STALE, + SHIM_MMAP_COUNTER_ATTENTION, + SHIM_MMAP_COUNTER_HIT, + SHIM_MMAP_COUNTERS_N, +}; + +typedef struct { + uint64_t addr; + uint64_t len; + uint64_t prot; +} shim_mmap_entry_t; + +typedef struct { + _Atomic uint32_t generation; /* host publish word */ + _Atomic uint32_t consumer_generation; /* EL1 generation ack */ + _Atomic uint32_t flags; /* host-owned enable bits */ + _Atomic uint32_t head; /* host consumer cursor */ + _Atomic uint32_t tail; /* EL1 producer cursor */ + uint32_t _pad0; + _Atomic uint64_t arena_base; + _Atomic uint64_t arena_limit; + _Atomic uint64_t cursor; /* EL1 bump cursor */ + uint64_t next_arena_size; /* most recently selected generation size */ + uint64_t max_len_seen; /* outgoing-generation request history */ + shim_mmap_entry_t ring[SHIM_MMAP_RING_SIZE]; + _Atomic uint64_t counters[SHIM_MMAP_COUNTERS_N]; + uint64_t refill_count; + uint64_t recycle_count; + uint64_t peak_arena_size; +} shim_mmap_control_t; + +/* Provision the main vCPU before guest entry. Worker vCPUs provision lazily on + * their first eligible mmap so short-lived threads do not allocate an unused + * arena. + */ +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t); + +/* Drain every per-vCPU SPSC ring. Caller holds mmap_lock. */ +void mmap_fastpath_drain_locked(guest_t *g); + +/* Refill the current vCPU after an eligible mmap slow-path. request_len is + * page-rounded; requests above MMAP_FAST_ARENA_MAX leave the arena untouched. + * Caller holds mmap_lock. + */ +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len); + +/* Give an explicit slow-path hint precedence over this stopped vCPU's + * unconsumed arena tail. Caller holds mmap_lock. + */ +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length); + +/* Revoke all arenas while sibling vCPUs are quiesced. Caller holds mmap_lock. + */ +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water); + +/* Disable the feature before first guest entry (debugger/observability). */ +void mmap_fastpath_disable(guest_t *g); + +/* Advance *start past an EL1 arena reservation that overlaps length bytes. */ +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr); diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index c9df01b1..0147e2dd 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -18,6 +19,7 @@ #include "hvutil.h" #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/shim-globals.h" #include "core/vdso.h" #include "debug/log.h" @@ -96,6 +98,42 @@ _Static_assert(SHIM_GLOBALS_SIZE <= BLOCK_2MIB, _Static_assert(SHIM_COUNTERS_OFF + SHIM_COUNTERS_N * 8 <= SHIM_IDENTITY_OFF_PGID, "counter array must not overlap the PGID slot"); +_Static_assert(SHIM_MMAP_CONTROL_BASE == 0x20000, + "shim.S mmap fast path hard-codes control base 0x20000"); +_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x800, + "shim.S mmap fast path hard-codes control stride 0x800"); +_Static_assert(SHIM_MMAP_RING_SIZE == 16, + "shim.S mmap fast path hard-codes 16 ring entries"); +_Static_assert(offsetof(shim_mmap_control_t, generation) == 0, + "shim.S mmap generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, consumer_generation) == 4, + "shim.S mmap consumer-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, flags) == 8, + "shim.S mmap flags offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, head) == 12, + "shim.S mmap head offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, tail) == 16, + "shim.S mmap tail offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_base) == 24, + "shim.S mmap arena-base offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_limit) == 32, + "shim.S mmap arena-limit offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, cursor) == 40, + "shim.S mmap cursor offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, next_arena_size) == 48, + "mmap next-arena-size offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, max_len_seen) == 56, + "mmap max-len-seen offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, ring) == 64, + "shim.S mmap ring offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x1C0, + "shim.S mmap counter offset drift"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "per-vCPU mmap control exceeds its shim-data stride"); +_Static_assert(SHIM_MMAP_CONTROL_BASE + + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE <= + BLOCK_2MIB - MAX_THREADS * 4096, + "mmap controls overlap per-vCPU EL1 stacks"); static uint8_t *cache_base(const guest_t *g) { @@ -126,7 +164,13 @@ static void urandom_ring_unlock(uint32_t *lock_p) void shim_globals_init(guest_t *g) { - memset(cache_base(g), 0, SHIM_GLOBALS_SIZE); + /* mmap controls occupy a separate low shim-data range. Init/exec/fork + * child all call this while no sibling can execute, so clearing the whole + * control array also prevents a recycled SP_EL1 slot from inheriting an + * arena published to its previous owner. + */ + memset(cache_base(g), 0, + SHIM_MMAP_CONTROL_BASE + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE); } void shim_globals_publish_pid(guest_t *g, int64_t pid, int64_t ppid) @@ -440,12 +484,10 @@ static const char *const counter_names[SHIM_COUNTERS_N] = { [SHIM_COUNTER_URANDOM_HIT] = "URANDOM_HIT", [SHIM_COUNTER_GETRANDOM_HIT] = "GETRANDOM_HIT", [SHIM_COUNTER_PGSID_HIT] = "PGSID_HIT", - - /* Slots 12..15 (SHIM_COUNTERS_N == 16) are intentionally unnamed; the dump - * prints "(reserved)" so they appear in the output when non-zero, which - * would flag an out-of-band increment. Bind a name here when a future EL1 - * service claims one of these slots. - */ + [SHIM_COUNTER_FAULT_MATERIALIZE] = "FAULT_MATERIALIZE", + [SHIM_COUNTER_FAULT_TLBI_VAE] = "FAULT_TLBI_VAE", + [SHIM_COUNTER_FAULT_TLBI_RVAE] = "FAULT_TLBI_RVAE", + [SHIM_COUNTER_FAULT_TLBI_BCAST] = "FAULT_TLBI_BCAST", }; uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) @@ -458,6 +500,15 @@ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) return __atomic_load_n(slot_p, __ATOMIC_RELAXED); } +void shim_globals_counter_inc(guest_t *g, unsigned slot) +{ + if (!shim_globals_stats_enabled() || slot >= SHIM_COUNTERS_N) + return; + uint8_t *page = (uint8_t *) g->host_base + g->shim_data_base; + uint64_t *slot_p = (uint64_t *) (page + SHIM_COUNTERS_OFF) + slot; + __atomic_fetch_add(slot_p, 1, __ATOMIC_RELAXED); +} + void shim_globals_counters_dump(const guest_t *g) { fprintf(stderr, "shim-stats (pid=%lld)\n", (long long) proc_get_pid()); @@ -469,6 +520,62 @@ void shim_globals_counters_dump(const guest_t *g) fprintf(stderr, " %-20s %llu\n", name ? name : "(reserved)", (unsigned long long) v); } + + static const char *const mmap_counter_names[SHIM_MMAP_COUNTERS_N] = { + [SHIM_MMAP_COUNTER_SHAPE_MISS] = "MMAP_SHAPE_MISS", + [SHIM_MMAP_COUNTER_CAPACITY_MISS] = "MMAP_CAPACITY_MISS", + [SHIM_MMAP_COUNTER_RING_FULL] = "MMAP_RING_FULL", + [SHIM_MMAP_COUNTER_GENERATION_STALE] = "MMAP_GENERATION_STALE", + [SHIM_MMAP_COUNTER_ATTENTION] = "MMAP_ATTENTION", + [SHIM_MMAP_COUNTER_HIT] = "MMAP_HIT", + }; + uint64_t mmap_counters[SHIM_MMAP_COUNTERS_N] = {0}; + uint64_t refill_count = 0, recycle_count = 0; + uint64_t current_max = 0, peak_max = 0; + const uint8_t *shim_data = + (const uint8_t *) g->host_base + g->shim_data_base; + for (int slot = 0; slot < MAX_THREADS; slot++) { + const shim_mmap_control_t *c = + (const shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * + SHIM_MMAP_CONTROL_STRIDE); + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + mmap_counters[i] += + atomic_load_explicit(&c->counters[i], memory_order_relaxed); + refill_count += c->refill_count; + recycle_count += c->recycle_count; + if (c->next_arena_size > current_max) + current_max = c->next_arena_size; + if (c->peak_arena_size > peak_max) + peak_max = c->peak_arena_size; + } + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + fprintf(stderr, " %-20s %llu\n", mmap_counter_names[i], + (unsigned long long) mmap_counters[i]); + fprintf(stderr, " %-20s %llu\n", "MMAP_REFILL", + (unsigned long long) refill_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_RECYCLE", + (unsigned long long) recycle_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_CURRENT", + (unsigned long long) current_max); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_PEAK", + (unsigned long long) peak_max); + uint64_t high_water = + g->mmap_next > MMAP_BASE ? g->mmap_next - MMAP_BASE : 0; + fprintf(stderr, " %-20s %llu\n", "MMAP_HIGH_WATER", + (unsigned long long) high_water); + fprintf(stderr, " %-20s %llu\n", "FAULT_CLEAN_SKIP", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_CLEAN_SKIP]); + fprintf(stderr, " %-20s %llu\n", "FAULT_DIRTY_MEMSET", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_DIRTY_MEMSET]); + fprintf(stderr, " %-20s %llu\n", "FAULT_ALREADY_VALID", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]); + fprintf(stderr, " %-20s %llu\n", "FAULT_WINDOW_BYTES", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES]); } static pthread_once_t stats_once = PTHREAD_ONCE_INIT; diff --git a/src/core/shim-globals.h b/src/core/shim-globals.h index 14cb24f1..049775d4 100644 --- a/src/core/shim-globals.h +++ b/src/core/shim-globals.h @@ -165,7 +165,7 @@ * not in urandom bitmap, len zero, len over inline cap, ring fill below * request, ring wrap, EL0 buffer probe failure). Slots 8..11 record fast-path * hits so bail rates can be computed against a hit denominator. Slots 12..15 - * are reserved. + * attribute lazy-fault materializations and the TLBI wire mode they emit. * * The shim hardcodes the byte offset of each slot; the static_asserts in * shim-globals.c keep the C-side macros and the assembly in sync. @@ -185,6 +185,10 @@ #define SHIM_COUNTER_URANDOM_HIT 9 #define SHIM_COUNTER_GETRANDOM_HIT 10 #define SHIM_COUNTER_PGSID_HIT 11 +#define SHIM_COUNTER_FAULT_MATERIALIZE 12 +#define SHIM_COUNTER_FAULT_TLBI_VAE 13 +#define SHIM_COUNTER_FAULT_TLBI_RVAE 14 +#define SHIM_COUNTER_FAULT_TLBI_BCAST 15 /* Extended identity slots: pgid and sid. * @@ -390,6 +394,7 @@ void shim_globals_refill_urandom_ring(guest_t *g); * when ELFUSE_SHIM_STATS is set. */ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot); +void shim_globals_counter_inc(guest_t *g, unsigned slot); void shim_globals_counters_dump(const guest_t *g); /* ELFUSE_SHIM_STATS env-var gate (idempotent / cached). When enabled the exit diff --git a/src/core/shim-mmap.c b/src/core/shim-mmap.c new file mode 100644 index 00000000..e9c6a2d7 --- /dev/null +++ b/src/core/shim-mmap.c @@ -0,0 +1,221 @@ +/* + * EL1 mmap syscall fast path. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * This file is compiled freestanding and linked into the shim image. It may + * not call host code or a C runtime: the only state it consumes is the saved + * EL0 register frame, TPIDR_EL1's shim-data mapping, and the shared mmap + * control protocol. Keep architecture-only operations in the small helper + * below; the allocator policy remains ordinary C. + */ + +#include "core/shim-mmap.h" + +#include +#include + +#include "core/mmap-fastpath.h" +#include "core/shim-globals.h" + +#define EL1_PAGE_SIZE 0x1000ULL +#define EL1_BLOCK_SIZE 0x200000ULL +#define EL1_SHIM_DATA_SIZE 0x200000ULL + +#define EL1_SYS_MMAP 222ULL + +#define EL1_PROT_READ 1ULL +#define EL1_PROT_WRITE 2ULL +#define EL1_PROT_RW (EL1_PROT_READ | EL1_PROT_WRITE) + +#define EL1_MAP_PRIVATE 0x02ULL +#define EL1_MAP_ANONYMOUS 0x20ULL +#define EL1_MAP_NORESERVE 0x4000ULL + +/* The bump path promises 2 MiB-aligned starts at or above this request size, + * so the host can back them with L2 blocks. */ +#define EL1_ALIGN_THRESHOLD EL1_BLOCK_SIZE + +typedef struct { + uint8_t *shim_data; + shim_mmap_control_t *control; +} el1_mmap_context_t; + +_Static_assert(EL1_SAVED_GPRS * sizeof(uint64_t) == 248, + "saved GPR frame layout changed"); +_Static_assert(sizeof(shim_mmap_entry_t) == 24, + "EL1 mmap publication ABI changed"); + +static inline uint64_t el1_read_tpidr(void) +{ + uint64_t value; + __asm__ volatile("mrs %0, tpidr_el1" : "=r"(value)); + return value; +} + +static inline bool el1_add_overflow(uint64_t left, + uint64_t right, + uint64_t *result) +{ + return __builtin_add_overflow(left, right, result); +} + +static bool el1_page_round(uint64_t length, uint64_t *rounded) +{ + uint64_t value; + if (length == 0 || el1_add_overflow(length, EL1_PAGE_SIZE - 1, &value)) + return false; + *rounded = value & ~(EL1_PAGE_SIZE - 1); + return true; +} + +/* SP_EL1 stack tops are shim_data_end - slot*4KiB and controls are + * shim_data + 0x20000 + slot*2KiB, so the saved frame's page locates the + * control without consuming another system register. */ +static el1_mmap_context_t el1_context(uint64_t *saved_gprs) +{ + uintptr_t shim_data = (uintptr_t) el1_read_tpidr(); + uintptr_t frame = (uintptr_t) saved_gprs; + uintptr_t stack_top = + (frame + EL1_PAGE_SIZE - 1) & ~(uintptr_t) (EL1_PAGE_SIZE - 1); + uintptr_t slot_bytes = shim_data + EL1_SHIM_DATA_SIZE - stack_top; + unsigned slot = (unsigned) (slot_bytes / EL1_PAGE_SIZE); + return (el1_mmap_context_t) { + .shim_data = (uint8_t *) shim_data, + .control = (shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + + (uintptr_t) slot * + SHIM_MMAP_CONTROL_STRIDE), + }; +} + +static inline bool el1_stats_enabled(const el1_mmap_context_t *context) +{ + return context->shim_data[SHIM_GLOBALS_OFF_STATS_EN] != 0; +} + +/* Each vCPU is the sole writer of its own counter slots. */ +static inline void el1_counter_increment(_Atomic uint64_t *counter) +{ + uint64_t value = atomic_load_explicit(counter, memory_order_relaxed); + atomic_store_explicit(counter, value + 1, memory_order_relaxed); +} + +static inline void el1_mmap_counter(const el1_mmap_context_t *context, + unsigned counter) +{ + if (el1_stats_enabled(context)) + el1_counter_increment(&context->control->counters[counter]); +} + +static inline void el1_attention_counter(const el1_mmap_context_t *context) +{ + if (!el1_stats_enabled(context)) + return; + _Atomic uint64_t *counter = + (_Atomic uint64_t *) (context->shim_data + SHIM_COUNTERS_OFF) + + SHIM_COUNTER_ATTN_BAIL; + el1_counter_increment(counter); +} + +static inline bool el1_attention_pending(const el1_mmap_context_t *context) +{ + return atomic_load_explicit((_Atomic uint32_t *) (context->shim_data + + SHIM_GLOBALS_OFF_ATTN), + memory_order_acquire) != 0; +} + +/* Consume a host-prepared, per-vCPU VA arena for the exact anonymous RW shape + * used by allocators: + * + * mmap(NULL, len, PROT_READ|PROT_WRITE, + * MAP_PRIVATE|MAP_ANONYMOUS[|MAP_NORESERVE], fd, off) + * + * Each vCPU is the sole producer of its publication ring and cursor. The host + * acquire-drains all publications whenever it takes mmap_lock. + */ +static bool el1_mmap(el1_mmap_context_t *context, uint64_t *saved_gprs) +{ + shim_mmap_control_t *control = context->control; + + if (el1_attention_pending(context)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_ATTENTION); + el1_attention_counter(context); + return false; + } + + uint64_t prot = saved_gprs[2]; + uint64_t flags = saved_gprs[3]; + uint64_t length; + if (saved_gprs[0] != 0 || prot != EL1_PROT_RW || + (flags & ~EL1_MAP_NORESERVE) != (EL1_MAP_PRIVATE | EL1_MAP_ANONYMOUS) || + !el1_page_round(saved_gprs[1], &length)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_SHAPE_MISS); + return false; + } + + uint32_t generation = + atomic_load_explicit(&control->generation, memory_order_acquire); + if (generation != atomic_load_explicit(&control->consumer_generation, + memory_order_relaxed)) { + /* A revocation deliberately leaves the consumer generation stale. Ack + * only after the acquire load above, then make this syscall take HVC; + * the host will either leave the control disabled or publish a fresh + * arena. + */ + atomic_store_explicit(&control->consumer_generation, generation, + memory_order_relaxed); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_GENERATION_STALE); + return false; + } + + if (!(atomic_load_explicit(&control->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + uint64_t address = + atomic_load_explicit(&control->cursor, memory_order_relaxed); + uint64_t cursor; + if (length >= EL1_ALIGN_THRESHOLD) { + if (el1_add_overflow(address, EL1_BLOCK_SIZE - 1, &address)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + address &= ~(EL1_BLOCK_SIZE - 1); + } + if (el1_add_overflow(address, length, &cursor) || + cursor > + atomic_load_explicit(&control->arena_limit, memory_order_relaxed)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + uint32_t head = atomic_load_explicit(&control->head, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&control->tail, memory_order_relaxed); + if ((uint32_t) (tail - head) >= SHIM_MMAP_RING_SIZE) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_RING_FULL); + return false; + } + + shim_mmap_entry_t *entry = &control->ring[tail & (SHIM_MMAP_RING_SIZE - 1)]; + entry->addr = address; + entry->len = length; + entry->prot = prot; + /* Publish the bump cursor before the entry. */ + atomic_store_explicit(&control->cursor, cursor, memory_order_relaxed); + atomic_store_explicit(&control->tail, tail + 1, memory_order_release); + + saved_gprs[0] = address; + el1_mmap_counter(context, SHIM_MMAP_COUNTER_HIT); + return true; +} + +bool el1_mmap_fastpath(uint64_t saved_gprs[static EL1_SAVED_GPRS]) +{ + if (saved_gprs[8] != EL1_SYS_MMAP) + return false; + el1_mmap_context_t context = el1_context(saved_gprs); + return el1_mmap(&context, saved_gprs); +} diff --git a/src/core/shim-mmap.h b/src/core/shim-mmap.h new file mode 100644 index 00000000..cb8f6aa2 --- /dev/null +++ b/src/core/shim-mmap.h @@ -0,0 +1,20 @@ +/* + * Freestanding EL1 mmap fast path. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The assembly exception shim owns the saved-register frame and calls this + * module only for mmap. A true return means X0 in the saved + * frame contains the completed syscall result; false asks the shim to forward + * the original frame to HVC #5 unchanged. + */ + +#pragma once + +#include +#include + +#define EL1_SAVED_GPRS 31u + +bool el1_mmap_fastpath(uint64_t saved_gprs[static EL1_SAVED_GPRS]); diff --git a/src/core/shim.S b/src/core/shim.S index a59cf972..4994f4a6 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -447,8 +447,22 @@ svc_handler: b.eq getsid_fast cmp x10, #278 /* SYS_getrandom? */ b.eq getrandom_fast + cmp x10, #222 /* SYS_mmap? */ + b.eq mmap_anon_fast b handle_svc_0 +/* The mmap fast path is a freestanding C consumer. The saved frame remains + * authoritative: false leaves it untouched for HVC #5; true places the + * completed result in saved X0. The normal restore tail reloads every other + * guest register, so the C ABI's caller-clobbered set is private to EL1. + */ +mmap_anon_fast: + mov x0, sp + bl _el1_mmap_fastpath + cbz w0, handle_svc_0 + ldr x0, [sp, #0] + b svc_restore_eret + identity_class_fast: mrs x12, tpidr_el1 /* shim-globals base */ ldar w13, [x12] /* attention flag, acquire */ @@ -1210,6 +1224,7 @@ handle_el0_fault: .Lel0_fault_tlbi_full: /* Broadcast TLB + conditional I-cache flush. X11=0 skips IC IALLU. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Lel0_fault_full_no_ic @@ -1230,6 +1245,7 @@ handle_el0_fault: mov x13, x11 ubfx x11, x9, #12, #44 mov x12, x10 + dsb ishst 4: tlbi vae1is, x11 add x11, x11, #1 subs x12, x12, #1 @@ -1244,7 +1260,9 @@ handle_el0_fault: .Lel0_fault_tlbi_rvae: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE). X9 carries the pre-encoded - * operand (baddr | NUM<<39 | TG=01<<46); X11 the I-cache hint. */ + * operand (baddr | NUM<<39 | SCALE<<44 | TG=01<<46); X11 the I-cache + * hint. */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Lel0_fault_rvae_no_ic @@ -1288,6 +1306,7 @@ tlbi_restore_eret: * leak VA bits into the TTL [47:44] or ASID [63:48] operand fields. */ ubfx x0, x0, #12, #44 + dsb ishst tlbi vae1is, x0 dsb ish ic iallu @@ -1363,6 +1382,7 @@ handle_svc_0: * 1 = broadcast TLBI VMALLE1IS * 2 = execve replaced register state (drop frame + flush) * 3 = selective TLBI VAE1IS over X10 pages starting at X9 + * 4 = single-shot TLBI RVAE1IS with encoded operand in X9 * 5. Resume vCPU (execution continues below) */ hvc #5 @@ -1391,6 +1411,7 @@ tlbi_full: * include exec), so the shim must IC IALLU; zero means a data-only * PT change and the I-cache invalidation is skipped. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Ltlbi_full_skip_ic @@ -1426,6 +1447,7 @@ tlbi_selective: * leak VA bits into the TTL [47:44] or ASID [63:48] operand fields. */ ubfx x11, x9, #12, #44 /* x11 = VA[55:12] (current page operand) */ mov x12, x10 /* x12 = remaining page counter */ + dsb ishst 3: tlbi vae1is, x11 add x11, x11, #1 /* next page (operand is in 4 KiB units) */ subs x12, x12, #1 @@ -1441,11 +1463,11 @@ tlbi_selective: tlbi_range_large: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE, ARMv8.4+). The host has * encoded the full operand in X9: baddr (VA >> 12), TTL=0, NUM in bits - * [43:39], SCALE=0, ASID=0. One instruction covers up to 64 pages, - * avoiding the broadcast TLBI VMALLE1IS that the prior selective cap - * forced for 17..64-page ranges. X11 carries the I-cache hint as in - * tlbi_full / tlbi_selective. + * [43:39], SCALE in bits [45:44], ASID=0. One instruction covers up to + * TLBI_RVAE_MAX_PAGES. X11 carries the I-cache hint as in tlbi_full / + * tlbi_selective. */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Ltlbi_rvae_skip_ic diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index add1ca84..0fdc6251 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -202,9 +202,9 @@ int fork_ipc_send_memory_regions(int ipc_sock, const guest_t *g, bool use_shm) #define MAX_USED_REGIONS 16 used_region_t used[MAX_USED_REGIONS]; unsigned int shim_sz = proc_get_shim_size(); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire((guest_t *) (uintptr_t) g); int nregions = guest_get_used_regions(g, shim_sz, used, MAX_USED_REGIONS); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); uint32_t num_regions = (uint32_t) nregions; if (fork_ipc_write_all(ipc_sock, &num_regions, sizeof(num_regions)) < 0) @@ -723,6 +723,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced) { @@ -775,6 +776,9 @@ int fork_ipc_send_process_state(int ipc_sock, fork_ipc_write_all(ipc_sock, regions_snapshot, num_guest_regions * sizeof(guest_region_t)) < 0) return -1; + if (fork_ipc_write_all(ipc_sock, dirty_blocks_snapshot, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) + return -1; if (fork_ipc_write_all(ipc_sock, &num_preannounced, sizeof(num_preannounced)) < 0) @@ -994,6 +998,12 @@ int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig) g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); + if (fork_ipc_read_all(ipc_fd, g->dirty_blocks, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) { + log_error("fork-child: failed to read dirty block bitmap"); + return -1; + } + uint32_t num_preannounced = 0; if (fork_ipc_read_all(ipc_fd, &num_preannounced, sizeof(num_preannounced)) < 0) { diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index c221afe9..e252d16b 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C464FU /* "ELFO" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C4650U /* "ELFP" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ @@ -138,6 +138,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced); int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig); diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index 3a0c4e0d..f91f3514 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -32,6 +32,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "runtime/forkipc.h" #include "runtime/fork-state.h" @@ -517,6 +518,11 @@ int fork_child_main(int ipc_fd, */ shim_globals_rebuild_urandom_bitmap(); + if (!verbose) + mmap_fastpath_prepare_vcpu(&g, current_thread); + else + mmap_fastpath_disable(&g); + /* Now that current_thread is set, apply signal state. This must happen * after thread_register_main() so the per-thread blocked mask and altstack * are properly restored to the thread entry. @@ -588,7 +594,7 @@ typedef struct { vcpu_simd_state_t simd_state; } thread_create_args_t; -static void resolve_clone_stack_range(const guest_t *g, +static void resolve_clone_stack_range(guest_t *g, uint64_t child_stack, uint64_t *start_out, uint64_t *end_out) @@ -604,13 +610,10 @@ static void resolve_clone_stack_range(const guest_t *g, if (sp_off == 0 || sp_off > g->guest_size) return; - /* The region array is mutated under mmap_lock by any concurrent mmap or - * munmap, and clone does not otherwise take it. Reading it unlocked is a - * data race on g->regions and g->nregions, reported by ThreadSanitizer as - * soon as a sibling allocates while another thread clones. Neither caller - * holds a lock here, and mmap_lock is order 1, so taking it is safe. + /* The region array is mutated under mmap_lock. The acquire also drains EL1 + * mmap publications before clone resolves a newly allocated stack. */ - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); const guest_region_t *r = guest_region_find(g, sp_off - 1); if (r) { if (start_out) @@ -618,7 +621,7 @@ static void resolve_clone_stack_range(const guest_t *g, if (end_out) *end_out = r->end; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); } /* Forward declaration: worker entry runs after sys_clone_thread */ @@ -1085,10 +1088,12 @@ static void *thread_create_and_run(void *arg) * how pthread_join works in musl: the joining thread does FUTEX_WAIT on * this address until it becomes 0. * - * Drain any deferred munmap of this thread's stack before waking the - * joiner: the parent may reuse the freed VA as soon as it returns from - * pthread_join, and reuse must not race with the deferred unmap. + * Drain any deferred munmap before publishing clear_child_tid. A joiner + * may observe the zero without ever sleeping in FUTEX_WAIT, then reuse the + * freed VA immediately; ordering only the wake after cleanup leaves a + * window where MAP_FIXED_NOREPLACE still sees the old stack VMA. */ + mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; if (t->clear_child_tid != 0) { uint32_t zero = 0; @@ -1103,7 +1108,6 @@ static void *thread_create_and_run(void *arg) (unsigned long long) t->clear_child_tid); } } - mem_cleanup_deferred_stack_unmaps(g, t); if (wake_ctid) futex_wake_one(g, t->clear_child_tid); @@ -1366,16 +1370,16 @@ static void *vm_clone_thread_run(void *arg) /* Set per-thread TLS pointer and enter worker run loop */ current_thread = t; thread_fork_barrier_check(); - log_debug("vm_clone tid=%lld starting on vCPU", (long long) t->guest_tid); int wait_status = 0; int exit_code = vcpu_run_loop(vcpu, vexit, g, verbose, 0, &wait_status); - /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: drain - * deferred stack munmaps before waking the joiner so the parent does not - * reuse the VA before it is released. + /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: the zero + * itself, not just the futex wake, releases a joiner, so publish it only + * after the deferred stack mapping is gone. */ + mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; if (t->clear_child_tid != 0) { uint32_t zero = 0; @@ -1390,7 +1394,6 @@ static void *vm_clone_thread_run(void *arg) (unsigned long long) t->clear_child_tid); } } - mem_cleanup_deferred_stack_unmaps(g, t); if (wake_ctid) futex_wake_one(g, t->clear_child_tid); @@ -1712,6 +1715,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, mmap_fork_anon_shared_txn_t *anon_shared_txn = NULL; guest_region_t *regions_snapshot = NULL; + uint64_t *dirty_blocks_snapshot = NULL; guest_region_t preannounced_snapshot[GUEST_MAX_PREANNOUNCED]; int snapshot_shm_fd = -1; bool siblings_quiesced = false; @@ -1942,6 +1946,10 @@ int64_t sys_clone(hv_vcpu_t vcpu, } memcpy(regions_snapshot, g->regions, snap_sz); } + dirty_blocks_snapshot = malloc(sizeof(g->dirty_blocks)); + if (!dirty_blocks_snapshot) + goto fail_snapshot; + memcpy(dirty_blocks_snapshot, g->dirty_blocks, sizeof(g->dirty_blocks)); int npreannounced_snapshot = g->npreannounced; if (npreannounced_snapshot > 0) { memcpy(preannounced_snapshot, g->preannounced, @@ -1966,8 +1974,8 @@ int64_t sys_clone(hv_vcpu_t vcpu, uint32_t num_preannounced = (uint32_t) npreannounced_snapshot; if (fork_ipc_send_process_state( ipc_sock, regions_snapshot, num_guest_regions, - regions_tracker_stale_snapshot, preannounced_snapshot, - num_preannounced) < 0) { + regions_tracker_stale_snapshot, dirty_blocks_snapshot, + preannounced_snapshot, num_preannounced) < 0) { log_error("clone: failed to send process state"); goto fail_snapshot; } @@ -2048,6 +2056,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, child_host_pid); free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); return child_guest_pid; @@ -2055,6 +2064,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, fail_snapshot: proc_cancel_child(child_guest_pid); free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); diff --git a/src/runtime/futex.c b/src/runtime/futex.c index 399301df..295376e4 100644 --- a/src/runtime/futex.c +++ b/src/runtime/futex.c @@ -1573,6 +1573,16 @@ int64_t sys_futex(guest_t *g, { int cmd = op & FUTEX_CMD_MASK; + /* Pre-fault lazy mappings before any bucket lock is taken. The word + * resolves below run under per-bucket locks, which rank after mmap_lock; + * materializing there would invert the lock order. A futex word in a + * mapping the guest never touched reads as zero, matching Linux. + */ + guest_lazy_faultin(g, uaddr, sizeof(uint32_t)); + if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE || + cmd == FUTEX_WAKE_OP) + guest_lazy_faultin(g, uaddr2, sizeof(uint32_t)); + switch (cmd) { case FUTEX_WAIT: #if ELFUSE_HAVE_OS_SYNC_WAIT_ON_ADDRESS @@ -1816,6 +1826,11 @@ int64_t sys_futex_waitv(guest_t *g, */ if (!futex_uaddr_is_aligned(elts[i].uaddr)) return -LINUX_EINVAL; + /* Pre-fault lazy mappings: the word resolves below run with every + * bucket lock held, where materializing would invert the lock order + * against mmap_lock. + */ + guest_lazy_faultin(g, elts[i].uaddr, sizeof(uint32_t)); } waitv_shared_t shared; diff --git a/src/syscall/exec.c b/src/syscall/exec.c index 4f92dc3c..ddd54d21 100644 --- a/src/syscall/exec.c +++ b/src/syscall/exec.c @@ -816,14 +816,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, exec_handoff.blocked_mask = current_thread ? current_thread->blocked : 0; pthread_mutex_unlock(&exec_handoff_lock); - /* Release mmap_lock, which this thread's sc_execve wrapper holds, for the - * whole wait: the leader needs it to run the exec, and the teardown that - * reaps this thread needs it free. Re-taken before returning so the - * wrapper's unlock stays balanced, and by then the leader has released it - * around its own teardown. - */ - pthread_mutex_unlock(&mmap_lock); - /* The leader may be parked in a blocking syscall. thread_stop_requested is * true for it while a request is pending, so its wait returns EINTR and its * run loop reaches the service point. @@ -846,7 +838,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, } pthread_mutex_unlock(&exec_handoff_lock); - pthread_mutex_lock(&mmap_lock); return result; } @@ -906,14 +897,8 @@ int64_t exec_run_handoff(hv_vcpu_t vcpu, guest_t *g, bool verbose) saved_mask = signal_save_blocked(); signal_set_blocked(adopt_mask); - /* sys_execve is written to run with mmap_lock held, which on the direct - * path its sc_execve wrapper takes. This path comes from the run loop, so - * take it here instead. - */ - pthread_mutex_lock(&mmap_lock); int64_t rc = sys_execve(vcpu, g, path_gva, argv_gva, envp_gva, verbose, host_path); - pthread_mutex_unlock(&mmap_lock); pthread_mutex_lock(&exec_handoff_lock); if (rc == SYSCALL_EXEC_HAPPENED) { @@ -1401,21 +1386,12 @@ int64_t sys_execve(hv_vcpu_t vcpu, * executing the old image's code, winds down against the memory and fd * table its guest still expects. */ - /* Both callers hold mmap_lock (order 1) across the whole syscall, and the - * teardown must not run under it: a sibling blocked in - * pthread_mutex_lock(&mmap_lock) inside sc_brk, sc_mmap, sc_munmap, - * sc_mprotect, or its own deferred stack unmap is reachable by none of the - * teardown wakes, so it can never reach a stop check and the join below - * would always time out. Measured before this release: four siblings - * looping on mmap/munmap took the fatal path every time. - * - * Dropping it here is safe because nothing in the teardown touches guest - * memory or the region table, and re-acquiring cannot contend: by the time - * it returns 0 no other guest thread is left to hold it. + /* Teardown must run without mmap_lock: a sibling blocked in an mmap-family + * syscall or its deferred stack unmap cannot reach a stop check while the + * lock is held. sys_execve acquires the lock below, immediately before the + * point of no return, after every sibling has stopped. */ - pthread_mutex_unlock(&mmap_lock); int survivors = thread_exec_de_thread(); - pthread_mutex_lock(&mmap_lock); /* The refusal above is a snapshot: a sibling could have created a CLONE_VM * child in the window between it and here. de_thread neither reaps nor @@ -1487,6 +1463,14 @@ int64_t sys_execve(hv_vcpu_t vcpu, return err; } + /* Input copying above may fault in argv/env strings from lazy anonymous + * mappings, so it must run without mmap_lock held. Serialize only after + * all recoverable validation is complete and immediately before replacing + * the guest address space. From this point every failure is fatal and both + * successful return paths release the lock explicitly. + */ + mmap_lock_acquire(g); + /* Point of no return. guest_reset() zeroes all guest memory. The old * process image is gone. All validation that can fail gracefully MUST * happen above this line. Failures below are unrecoverable; elfuse exits @@ -1636,6 +1620,7 @@ int64_t sys_execve(hv_vcpu_t vcpu, unlink(interp.resolved); exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; } @@ -1969,6 +1954,7 @@ int64_t sys_execve(hv_vcpu_t vcpu, exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; too_many_regions: diff --git a/src/syscall/internal.h b/src/syscall/internal.h index de06dd6d..cc5b6f3a 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -46,6 +46,13 @@ typedef int host_fd_t; extern pthread_mutex_t mmap_lock; /* Lock order: 1, mmap/brk + page tables */ extern pthread_mutex_t fd_lock; /* Lock order: 3, FD table */ +/* The only supported mmap_lock entry/exit API. Acquire drains the per-vCPU + * EL1 mmap rings before any caller can inspect semantic region state. + */ +void mmap_lock_acquire(guest_t *g); +void mmap_lock_release(void); +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond); + /* FD table (defined in syscall/fdtable.c). */ extern fd_entry_t fd_table[FD_TABLE_SIZE]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index b79b28c4..98f91ff0 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -21,8 +21,11 @@ #include #include "debug/log.h" +#include "debug/syscall-hist.h" #include "utils.h" +#include "core/mmap-fastpath.h" + #include "proved/align.h" #include "runtime/thread.h" @@ -37,6 +40,395 @@ */ pthread_mutex_t mmap_lock = PTHREAD_MUTEX_INITIALIZER; /* Lock order: 1 */ +static pthread_once_t mmap_fastpath_env_once = PTHREAD_ONCE_INIT; +static bool mmap_fastpath_env_enabled; +static _Atomic bool mmap_fastpath_forced_off; + +static uint64_t find_free_gap_inner(const guest_t *g, + uint64_t length, + uint64_t min_addr, + uint64_t max_addr, + uint64_t align); + +static void mmap_fastpath_read_env(void) +{ + const char *v = getenv("ELFUSE_MMAP_FASTPATH"); + mmap_fastpath_env_enabled = + !v || (strcmp(v, "0") != 0 && strcmp(v, "false") != 0); +} + +static bool mmap_fastpath_available(const guest_t *g) +{ + pthread_once(&mmap_fastpath_env_once, mmap_fastpath_read_env); + return mmap_fastpath_env_enabled && !g->is_rosetta && + !atomic_load_explicit(&mmap_fastpath_forced_off, + memory_order_acquire) && + !syscall_hist_enabled(); +} + +static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) +{ + if (!g || !g->host_base || slot < 0 || slot >= MAX_THREADS) + return NULL; + return (shim_mmap_control_t *) ((uint8_t *) g->host_base + + g->shim_data_base + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); +} + +static void mmap_fastpath_disable_control(shim_mmap_control_t *c) +{ + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->flags, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_base, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, 0, memory_order_relaxed); + atomic_store_explicit(&c->cursor, 0, memory_order_relaxed); + c->next_arena_size = MMAP_FAST_ARENA_MIN; + c->max_len_seen = 0; + atomic_store_explicit(&c->generation, generation, memory_order_release); +} + +void mmap_fastpath_drain_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + uint32_t head = atomic_load_explicit(&c->head, memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&c->tail, memory_order_acquire); + if ((uint32_t) (tail - head) > SHIM_MMAP_RING_SIZE) { + log_fatal( + "mmap fast path: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + } + + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t arena_limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + while (head != tail) { + const shim_mmap_entry_t *e = + &c->ring[head & (SHIM_MMAP_RING_SIZE - 1)]; + uint64_t addr = e->addr; + uint64_t len = e->len; + if ((addr & (GUEST_PAGE_SIZE - 1)) || !len || + (len & (GUEST_PAGE_SIZE - 1)) || addr < arena_base || + addr > arena_limit || len > arena_limit - addr || + e->prot != (LINUX_PROT_READ | LINUX_PROT_WRITE)) { + log_fatal( + "mmap fast path: invalid entry in vCPU slot %d " + "(addr=0x%llx len=0x%llx arena=0x%llx..0x%llx)", + slot, (unsigned long long) addr, (unsigned long long) len, + (unsigned long long) arena_base, + (unsigned long long) arena_limit); + } + if (guest_region_add_ex(g, addr, addr + len, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) { + /* EL1 already returned this address to the guest. Continuing + * without semantic metadata would turn first touch into a false + * SIGSEGV, so fail closed on the violated provisioning + * invariant instead of silently corrupting process state. + */ + log_fatal( + "mmap fast path: region metadata exhausted while " + "draining vCPU slot %d", + slot); + } + if (len > c->max_len_seen) + c->max_len_seen = len; + head++; + } + atomic_store_explicit(&c->head, head, memory_order_release); + } +} + +void mmap_lock_acquire(guest_t *g) +{ + pthread_mutex_lock(&mmap_lock); + mmap_fastpath_drain_locked(g); +} + +void mmap_lock_release(void) +{ + pthread_mutex_unlock(&mmap_lock); +} + +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond) +{ + pthread_cond_wait(cond, &mmap_lock); + /* pthread_cond_wait reacquires mmap_lock directly, so preserve the + * drain-before-region-read invariant of mmap_lock_acquire(). + */ + mmap_fastpath_drain_locked(g); +} + +static bool mmap_fastpath_request_fits(uint64_t cursor, + uint64_t limit, + uint64_t len) +{ + if (!len) + return cursor < limit; + uint64_t start = cursor; + if (len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + return false; + start = ALIGN_UP(start, BLOCK_2MIB); + } + return start <= limit && len <= limit - start; +} + +static uint64_t mmap_fastpath_pow2_clamped(uint64_t value) +{ + if (value <= MMAP_FAST_ARENA_MIN) + return MMAP_FAST_ARENA_MIN; + if (value >= MMAP_FAST_ARENA_MAX) + return MMAP_FAST_ARENA_MAX; + value--; + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + value |= value >> 32; + return value + 1; +} + +static uint64_t mmap_fastpath_arena_size(uint64_t max_len_seen, + uint64_t request_len) +{ + uint64_t adaptive = MMAP_FAST_ARENA_MIN; + if (max_len_seen) { + uint64_t target = + max_len_seen > MMAP_FAST_ARENA_MAX / MMAP_FAST_HISTORY_MULTIPLIER + ? MMAP_FAST_ARENA_MAX + : max_len_seen * MMAP_FAST_HISTORY_MULTIPLIER; + adaptive = mmap_fastpath_pow2_clamped(target); + } + + uint64_t covering = MMAP_FAST_ARENA_MIN; + if (request_len) { + uint64_t target = request_len > MMAP_FAST_ARENA_MAX / 2 + ? MMAP_FAST_ARENA_MAX + : request_len * 2; + covering = mmap_fastpath_pow2_clamped(target); + } + return adaptive > covering ? adaptive : covering; +} + +static void mmap_fastpath_refill_thread_locked(guest_t *g, + thread_entry_t *t, + uint64_t request_len) +{ + if (!t || t->sp_el1_slot < 0) + return; + shim_mmap_control_t *c = mmap_fastpath_control(g, t->sp_el1_slot); + if (!c) + return; + if (!mmap_fastpath_available(g)) { + mmap_fastpath_disable_control(c); + return; + } + + /* Giant requests are deliberately slow-path-only. Do not abandon a useful + * small-request arena or poison its adaptive history. + */ + if (request_len > MMAP_FAST_ARENA_MAX) + return; + + if (request_len > c->max_len_seen) + c->max_len_seen = request_len; + + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint32_t flags = atomic_load_explicit(&c->flags, memory_order_relaxed); + if ((flags & SHIM_MMAP_CTRL_ENABLED) && + mmap_fastpath_request_fits(cursor, limit, request_len)) + return; + + /* The owner is parked in HVC. Make the stranded tail immediately recyclable + * before the gap scan; mappings already served from the prefix were drained + * into regions[] on mmap_lock acquisition. + */ + if (flags & SHIM_MMAP_CTRL_ENABLED) + atomic_store_explicit(&c->cursor, limit, memory_order_relaxed); + + uint64_t arena_size = + mmap_fastpath_arena_size(c->max_len_seen, request_len); + + /* Prefer a real hole below the current high-water mark. Active sibling + * arena tails are excluded by mmap_fastpath_skip_reserved inside the gap + * allocator. Only grow mmap_next when no recyclable hole fits. + */ + uint64_t high = g->mmap_next; + if (high > g->mmap_limit) + high = g->mmap_limit; + uint64_t base = UINT64_MAX; + bool recycled = false; + if (high > MMAP_BASE) { + base = find_free_gap_inner(g, arena_size, MMAP_BASE, high, BLOCK_2MIB); + recycled = base != UINT64_MAX; + } + + if (!recycled) { + if (g->mmap_next > UINT64_MAX - (BLOCK_2MIB - 1)) { + mmap_fastpath_disable_control(c); + return; + } + base = ALIGN_UP(g->mmap_next, BLOCK_2MIB); + if (base > g->mmap_limit || arena_size > g->mmap_limit - base) { + mmap_fastpath_disable_control(c); + return; + } + } + uint64_t new_limit = base + arena_size; + + /* Carve VA only. Clearing stale descriptors once here makes every later + * bump allocation PTE-free without putting page-table work in EL1. Fresh + * bump-tail arenas beyond mmap_end cannot contain stale descriptors. + */ + if (recycled || base < g->mmap_end) { + if (guest_invalidate_ptes(g, base, new_limit) < 0) { + mmap_fastpath_disable_control(c); + return; + } + } + if (!recycled) { + g->mmap_next = new_limit; + if (g->mmap_rw_gap_hint < new_limit) + g->mmap_rw_gap_hint = new_limit; + } + + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->arena_base, base, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, new_limit, memory_order_relaxed); + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + c->next_arena_size = arena_size; + c->max_len_seen = 0; + c->refill_count++; + if (recycled) + c->recycle_count++; + if (arena_size > c->peak_arena_size) + c->peak_arena_size = arena_size; + atomic_store_explicit(&c->flags, SHIM_MMAP_CTRL_ENABLED, + memory_order_relaxed); + /* This vCPU is stopped in HVC (or has never run), so host may acknowledge + * the freshly published descriptor on its behalf. Revocation deliberately + * does not do this, making an in-flight stale generation bail once. + */ + atomic_store_explicit(&c->consumer_generation, generation, + memory_order_relaxed); + atomic_store_explicit(&c->generation, generation, memory_order_release); +} + +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len) +{ + mmap_fastpath_refill_thread_locked(g, current_thread, request_len); +} + +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length) +{ + if (!current_thread || current_thread->sp_el1_slot < 0 || !length || + addr > UINT64_MAX - length) + return; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor >= limit || addr >= limit || addr + length <= cursor) + return; + + /* The owner is stopped in the HVC that reached sys_mmap, so it cannot race + * this descriptor update. Revoke its whole unconsumed tail: the explicit + * hint must remain semantically free, and the post-syscall refill will + * provision a new non-overlapping arena. + */ + mmap_fastpath_disable_control(c); +} + +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t) +{ + mmap_lock_acquire(g); + mmap_fastpath_refill_thread_locked(g, t, 0); + mmap_lock_release(); +} + +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water) +{ + mmap_fastpath_drain_locked(g); + for (int slot = 0; slot < MAX_THREADS; slot++) + mmap_fastpath_disable_control(mmap_fastpath_control(g, slot)); + + if (!shrink_high_water) + return; + uint64_t high = MMAP_BASE; + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= MMAP_BASE && r->start < g->mmap_limit && r->end > high) + high = r->end; + } + g->mmap_next = high; + if (g->mmap_rw_gap_hint > high) + g->mmap_rw_gap_hint = high; +} + +void mmap_fastpath_disable(guest_t *g) +{ + atomic_store_explicit(&mmap_fastpath_forced_off, true, + memory_order_release); + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, true); + mmap_lock_release(); +} + +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr) +{ + if (!g || !start || !length) + return; + bool advanced; + do { + advanced = false; + if (*start > max_addr || length > max_addr - *start) + return; + uint64_t end = *start + length; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_acquire) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t cursor = + atomic_load_explicit(&c->cursor, memory_order_acquire); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor < limit && *start < limit && end > cursor) { + *start = ALIGN_UP(limit, align); + advanced = true; + break; + } + } + } while (advanced); +} + /* Host kernel page size (16 KiB on Apple Silicon, typically 4 KiB on Intel * macOS). MAP_FIXED requires addr/length/offset multiples of this, so an * overlay onto a guest 4 KiB-aligned IPA is only applicable when the IPA @@ -111,6 +503,11 @@ static int64_t sync_shared_aliases_range(guest_t *g, int backing_fd, uint64_t file_start, uint64_t file_end); +static int read_file_range_to_guest(guest_t *g, + uint64_t gpa, + int fd, + uint64_t file_off, + uint64_t len); static int region_count_after_removes(const guest_t *g, const remove_range_t *ranges, @@ -686,6 +1083,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, uint64_t gap_start; if (!align_up_ok(min_addr, align, &gap_start)) return UINT64_MAX; + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); /* Skip the prefix of regions entirely below gap_start in O(log n). After a * successful allocation the gap hint advances near or past the existing @@ -694,6 +1092,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, */ for (int i = guest_region_first_end_above(g, gap_start); i < g->nregions; i++) { + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); /* A region can still slip below gap_start after the align_up_ok advance * below skips past a smaller adjacent region; keep the cheap guard. */ @@ -727,6 +1126,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, } /* Check trailing space after all regions */ + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); if (window_fits(gap_start, length, max_addr)) return gap_start; return UINT64_MAX; /* No suitable gap found */ @@ -1129,6 +1529,7 @@ static int64_t sys_mmap_high_va(guest_t *g, if (!host) goto fail; memset(host, 0, BLOCK_2MIB); + guest_dirty_clear_zeroed_range(g, gpa, gpa + BLOCK_2MIB); /* Detect freshness BEFORE guest_map_va_range so the decision is not * confused by a prior high-VA mmap into the same 2 MiB block. A fresh @@ -1207,28 +1608,11 @@ static int64_t sys_mmap_high_va(guest_t *g, } else if (prot != LINUX_PROT_NONE) { memset(map_host, 0, length); replaced_bytes_dirty = replacing_existing; - uint8_t *dst = map_host; - size_t remaining = length; - off_t file_off = (off_t) offset; - while (remaining > 0) { - ssize_t nr = pread(host_backing_fd, dst, remaining, file_off); - if (nr < 0) { - if (errno == EINTR) - continue; - - /* Real host I/O failure (not EINTR); previously the loop broke - * without setting ret and the syscall returned a "successful" - * partially-zero mapping. - */ - ret = linux_errno(); - goto fail; - } - if (nr == 0) - break; - dst += nr; - remaining -= (size_t) nr; - file_off += nr; - } + uint64_t gpa_for_addr = backing_gpa_start + (addr - va_base); + ret = read_file_range_to_guest(g, gpa_for_addr, host_backing_fd, offset, + length); + if (ret < 0) + goto fail; } /* Install L3 PTEs for the actual mapped range. Fresh blocks were fully @@ -1396,11 +1780,39 @@ static int64_t sys_mmap_high_va(guest_t *g, return ret; } +/* Page-table high-water slot (mmap_rx_end / mmap_end) for a TTBR0 mmap offset, + * or NULL if @off is not in either mmap arena. + */ +static uint64_t *mmap_pt_end_for_off(guest_t *g, uint64_t off) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) + return &g->mmap_rx_end; + if (off >= MMAP_BASE) + return &g->mmap_end; + return NULL; +} + +/* Advance the allocation high-water mark (mmap_rx_next / mmap_next) that fork + * IPC state transfer replays, for whichever arena @off belongs to. + */ +static void mmap_bump_next(guest_t *g, uint64_t off, uint64_t end) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) { + if (end > g->mmap_rx_next) + g->mmap_rx_next = end; + } else if (off >= MMAP_BASE) { + if (end > g->mmap_next) + g->mmap_next = end; + } +} + static int mremap_extend_range(guest_t *g, uint64_t off, uint64_t size, int prot) { + uint64_t *pt_end = mmap_pt_end_for_off(g, off); + if (prot == LINUX_PROT_NONE) { guest_invalidate_ptes(g, off, off + size); return 0; @@ -1411,10 +1823,67 @@ static int mremap_extend_range(guest_t *g, uint64_t ext_end = ALIGN_UP(off + size, BLOCK_2MIB); if (ext_end > g->guest_size) ext_end = g->guest_size; - if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) + size_t nblocks = pt_end ? (size_t) ((ext_end - ext_start) / BLOCK_2MIB) : 0; + bool *block_preexisting = NULL; + if (nblocks) { + block_preexisting = calloc(nblocks, sizeof(*block_preexisting)); + if (!block_preexisting) + return -1; + for (size_t i = 0; i < nblocks; i++) + block_preexisting[i] = + guest_va_block_mapped(g, ext_start + (uint64_t) i * BLOCK_2MIB); + } + if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) { + free(block_preexisting); return -1; - guest_update_perms(g, off, off + size, page_perms); + } + uint64_t saved_pt_end = pt_end ? *pt_end : 0; + if (pt_end && ext_end > *pt_end) + *pt_end = ext_end; + + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + uint64_t keep_start = off > b ? off : b; + uint64_t keep_end = off + size < bend ? off + size : bend; + if (keep_start <= b && keep_end >= bend) + continue; + if (guest_split_block(g, b) < 0) + goto fail; + if (b < keep_start && guest_invalidate_ptes(g, b, keep_start) < 0) + goto fail; + if (keep_end < bend && guest_invalidate_ptes(g, keep_end, bend) < 0) + goto fail; + } + + if (guest_update_perms(g, off, off + size, page_perms) < 0) + goto fail; + free(block_preexisting); return 0; + + /* Roll back: re-invalidate every fresh block whole (idempotent -- the + * forward pass already cleared the non-kept subranges) plus the kept [off, + * off+size) span, returning the range to its pre-extend state. + */ +fail: + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + (void) guest_invalidate_ptes(g, b, bend); + } + (void) guest_invalidate_ptes(g, off, off + size); + if (pt_end) + *pt_end = saved_pt_end; + free(block_preexisting); + return -1; } static int hvf_apply_file_overlay(guest_t *g, @@ -1445,6 +1914,11 @@ static int read_file_range_to_guest(guest_t *g, uint8_t *dst = host_ptr_for_gpa(g, gpa); if (!dst) return -LINUX_EFAULT; + /* A short read, EOF, or later error may still leave nonzero bytes in the + * destination. Mark before the first pread so every exit is conservative. + */ + if (len <= UINT64_MAX - gpa) + guest_dirty_mark_range(g, gpa, gpa + len); size_t remaining = len; while (remaining > 0) { @@ -2011,12 +2485,16 @@ static int rollback_fresh_mmap_allocation(guest_t *g, { if (overlay_installed) hvf_remove_file_overlay(g, overlay_ipa, overlay_len); - if (guest_invalidate_ptes(g, start, start + length) < 0) + uint64_t end = start + length; + uint64_t cur_mmap_end = g->mmap_end; + uint64_t cur_mmap_rx_end = g->mmap_rx_end; + if (guest_invalidate_ptes(g, start, end) < 0) return -LINUX_ENOMEM; g->mmap_next = saved_mmap_next; - g->mmap_end = saved_mmap_end; + g->mmap_end = cur_mmap_end > saved_mmap_end ? cur_mmap_end : saved_mmap_end; g->mmap_rx_next = saved_mmap_rx_next; - g->mmap_rx_end = saved_mmap_rx_end; + g->mmap_rx_end = cur_mmap_rx_end > saved_mmap_rx_end ? cur_mmap_rx_end + : saved_mmap_rx_end; g->mmap_rw_gap_hint = saved_rw_gap_hint; g->mmap_rx_gap_hint = saved_rx_gap_hint; return 0; @@ -2348,6 +2826,8 @@ static int hvf_apply_file_overlay(guest_t *g, return -LINUX_EINTR; /* Being reaped; abandon the overlay */ int err = hvf_apply_file_overlay_quiesced(g, ipa, len, fd, file_off); thread_resume_siblings(); + if (err == 0 && len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); return err; } @@ -2386,6 +2866,12 @@ static int hvf_remove_file_overlay_quiesced(guest_t *g, hvf_remap_segments_best_effort(g, segments, nsegments); return err; } + /* Restoring shm-backed slab pages may reveal an older nonzero snapshot. The + * following munmap/MAP_FIXED path will clear the bit only after it has + * actually zeroed a complete 2 MiB block. + */ + if (len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); for (int i = 0; i < nsegments; i++) { if (hv_vm_map((uint8_t *) g->host_base + segments[i].ipa, @@ -2673,6 +3159,19 @@ int64_t sys_mmap(guest_t *g, bool needs_exec = (prot & LINUX_PROT_EXEC) != 0; bool is_prot_none = (prot == LINUX_PROT_NONE); bool is_noreserve = is_anon && (flags & LINUX_MAP_NORESERVE) != 0; + /* Anonymous mappings defer page-table creation and zeroing to first touch + * (guest fault or host-side access), like MAP_NORESERVE always has. This + * keeps mmap()/munmap() cost independent of length: a multi-GiB reservation + * costs neither an eager PTE walk nor a full-length memset, and + * never-touched blocks consume no page-table pool. PROT_NONE stays a pure + * reservation (faults deliver SIGSEGV, not materialization), and MAP_FIXED + * keeps the eager path because it must atomically replace live mappings. + * Shared anonymous memory stays eager unless the caller opted into + * MAP_NORESERVE (the historical lazy set), since deferred zeroing has never + * been exercised against the fork snapshot paths for it. + */ + bool is_lazy = is_anon && !is_prot_none && + ((flags & LINUX_MAP_SHARED) == 0 || is_noreserve); host_fd_ref_t backing_ref = {.fd = -1, .owned = 0}; int host_backing_fd = -1, track_backing_fd = -1; @@ -2703,13 +3202,24 @@ int64_t sys_mmap(guest_t *g, int replaced_nsnaps = 0; bool replaced_regions_removed = false; int replaced_remove_fd = -1; + /* Linux kernel rejects MAP_FIXED with non-page-aligned address (checked + * below); the flag itself is needed early because it gates the lazy path. + */ + bool is_fixed = + (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); + if (is_fixed) + is_lazy = false; int track_flags = ((flags & LINUX_MAP_SHARED) ? LINUX_MAP_SHARED : LINUX_MAP_PRIVATE); if (is_anon) track_flags |= LINUX_MAP_ANONYMOUS; - /* Preserve MAP_NORESERVE in region metadata before merge checks run. */ - if (is_noreserve) + /* Preserve MAP_NORESERVE in region metadata before merge checks run. The + * same bit doubles as the internal lazy marker: guest_region_add_ex derives + * the region's deferred-PTE flag from it, and it is not guest visible + * (/proc/self/maps prints only prot and shared/private). + */ + if (is_noreserve || is_lazy) track_flags |= LINUX_MAP_NORESERVE; /* The memory syscall layer handles all mmap variants. Aligned file-backed @@ -2744,9 +3254,17 @@ int64_t sys_mmap(guest_t *g, if (length == 0) return -LINUX_ENOMEM; + /* A non-fixed nonzero address is a strong Linux hint. If it lands in the + * current (stopped) vCPU's invisible arena tail, release that tail before + * gap finding so implementation-only VA preparation does not perturb the + * address the application observes. Sibling arenas stay immutable without + * quiesce; their disjoint high-water placement makes self-overlap the + * normal and important case (allocator hinting near its previous result). + */ + if (!is_fixed && addr != 0) + mmap_fastpath_release_current_hint_locked(g, addr, length); + /* Linux kernel rejects MAP_FIXED with non-page-aligned address */ - bool is_fixed = - (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); if (is_fixed && (addr & 4095)) return -LINUX_EINVAL; @@ -2796,6 +3314,7 @@ int64_t sys_mmap(guest_t *g, uint64_t fix_end = off + length; if (guest_range_hits_infra(g, off, fix_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, off, fix_end); result_off = off; @@ -3131,9 +3650,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rx_hwm = result_off + length; - if (rx_hwm > g->mmap_rx_next) - g->mmap_rx_next = rx_hwm; + mmap_bump_next(g, result_off, result_off + length); } else { /* RW (or PROT_NONE, or PROT_READ): allocate from main mmap region. * Honor the address hint if provided and within bounds. Some @@ -3192,9 +3709,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rw_hwm = result_off + length; - if (rw_hwm > g->mmap_next) - g->mmap_next = rw_hwm; + mmap_bump_next(g, result_off, result_off + length); } if (!region_has_capacity_after_removes(g, NULL, 0, 1)) { host_fd_ref_close(&backing_ref); @@ -3219,7 +3734,7 @@ int64_t sys_mmap(guest_t *g, guest_invalidate_ptes(g, result_off, result_off + length); } - if (!is_prot_none && !is_fixed && !is_noreserve) { + if (!is_prot_none && !is_fixed && !is_lazy) { /* Extend page tables for this specific allocation range only. * guest_extend_page_tables skips already-mapped blocks, so calling it * on pre-mapped regions is a no-op. This avoids creating entries for @@ -3281,16 +3796,24 @@ int64_t sys_mmap(guest_t *g, g->mmap_end = ext_end; } - /* Zero the mapped region */ + /* Zero the mapped region. RX mappings cannot be dirtied through their + * published PTEs, so a complete-block zero can make them clean again. + * Other mappings currently use writable stage-1 entries and must stay + * conservatively dirty even if their requested Linux prot is read-only. + */ memset((uint8_t *) g->host_base + result_off, 0, length); + if (needs_exec && !(prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, result_off, result_off + length); } - /* MAP_NORESERVE: invalidate any stale PTEs (like PROT_NONE path) but track - * the region for lazy materialization on first fault. Page table entries - * will be created by guest_materialize_lazy() when the guest first touches - * a page in this range. + /* Lazy (private anonymous, incl. MAP_NORESERVE): invalidate any stale PTEs + * (like the PROT_NONE path) but track the region for lazy materialization + * on first fault. Page table entries will be created by + * guest_materialize_lazy() when the guest first touches a page in this + * range, or by the host-access fault-in path when a syscall targets the + * range before the guest ever touches it. */ - if (is_noreserve && !is_fixed) { + if (is_lazy) { guest_invalidate_ptes(g, result_off, result_off + length); } @@ -3360,6 +3883,7 @@ int64_t sys_mmap(guest_t *g, overlay_ipa = result_off; overlay_len = nf_overlay_len; } else { + guest_dirty_mark_range(g, result_off, result_off + length); uint8_t *dst = (uint8_t *) g->host_base + result_off; size_t remaining = length; off_t file_off = offset; @@ -3538,6 +4062,11 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, old_off, old_off + old_size)) return -LINUX_EINVAL; + if (old_off < g->guest_size) + guest_materialize_wait_range_locked(g, old_off, + old_size > g->guest_size - old_off + ? g->guest_size + : old_off + old_size); /* Verify the whole source range is covered by one logical VMA. A fork-aware * growth can split that VMA at the inherited/private boundary, but no @@ -3588,10 +4117,13 @@ int64_t sys_mremap(guest_t *g, /* Zero the trimmed region on its real backing (high-VA tails live at * gpa_base, not host_base + tail_off). */ - memset(host_ptr_for_gpa(g, src_gpa_base + (tail_off - src_start)), 0, - tail_end - tail_off); + uint64_t tail_gpa = src_gpa_base + (tail_off - src_start); + if (guest_invalidate_ptes(g, tail_off, tail_end) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); + memset(host_ptr_for_gpa(g, tail_gpa), 0, tail_end - tail_off); + guest_dirty_clear_zeroed_range(g, tail_gpa, + tail_gpa + (tail_end - tail_off)); guest_region_remove_reserved(g, tail_off, tail_end, tail_remove_fd); - guest_invalidate_ptes(g, tail_off, tail_end); if (tail_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = tail_off; if (tail_off < g->mmap_rx_gap_hint) @@ -3618,6 +4150,7 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, new_off, new_off + new_size)) return finish_mremap(&source, -LINUX_EINVAL); + guest_materialize_wait_range_locked(g, new_off, new_off + new_size); /* Linux rejects MREMAP_FIXED when old and new ranges overlap */ uint64_t old_end = old_off + old_size, new_end = new_off + new_size; @@ -3759,7 +4292,14 @@ int64_t sys_mremap(guest_t *g, } (void) restore_snapshot_overlays_in_place(g, move.dest_snaps, move.dest_nsnaps); + int pt_err = + restore_snapshot_page_tables(g, new_off, new_off + new_size, + move.dest_snaps, move.dest_nsnaps); + if (pt_err < 0) + restore_err = pt_err; mremap_move_dispose(&move); + if (restore_err < 0) + return finish_mremap(&source, restore_err); return finish_mremap(&source, -LINUX_ENOMEM); } @@ -3816,14 +4356,22 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (move.track.prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping */ if (old_size > 0) { - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove_reserved(g, old_off, old_off + old_size, move.source_remove_fd); move.source_remove_fd = -1; - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -3902,6 +4450,9 @@ int64_t sys_mremap(guest_t *g, } memset((uint8_t *) g->host_base + grow_off, 0, grow_len); + if (!(track.prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, grow_off, + grow_off + grow_len); /* Update region tracking: remove old, add extended */ guest_region_remove_reserved(g, old_off, old_off + old_size, @@ -3914,14 +4465,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, old_off, old_off + new_size); /* Update high-water marks */ - uint64_t hwm = old_off + new_size; - if (old_off >= MMAP_RX_BASE && old_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (old_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, old_off, old_off + new_size); return finish_mremap(&source, (int64_t) old_addr); } @@ -4040,14 +4584,22 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (track.prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping. Any live source overlay was already torn down * before the destination range was touched. */ - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove_reserved(g, old_off, old_off + old_size, source_remove_fd); - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -4060,14 +4612,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, new_off, new_off + new_size); /* Update high-water marks */ - uint64_t hwm = new_off + new_size; - if (new_off >= MMAP_RX_BASE && new_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (new_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, new_off, new_off + new_size); return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } @@ -4157,6 +4702,8 @@ int64_t sys_madvise(guest_t *g, uint64_t addr, uint64_t length, int advice) */ if (!madvise_range_mapped(g, off, length)) return -LINUX_ENOMEM; + if (in_primary) + guest_materialize_wait_range_locked(g, off, off + length); uint64_t end = off + length; for (int i = 0; i < g->nregions; i++) { @@ -4272,11 +4819,38 @@ static int compare_range_pair(const void *a, const void *b) return 0; } +/* Coalesced sub-ranges of a munmap that must be zeroed. Sized so that even a + * pathologically fragmented lazy mapping (alternating materialized and + * untouched blocks) rarely overflows; on overflow the remainder of the region + * overlap is zeroed wholesale, which is always correct (zeroing already-zero + * slab bytes), just slower. + */ +#define MUNMAP_ZERO_RANGES_MAX 128 + +typedef struct { + uint64_t lo, hi; +} zero_range_t; + +static void zero_range_push(zero_range_t *ranges, + int *n, + uint64_t lo, + uint64_t hi) +{ + if (lo >= hi) + return; + if (*n > 0 && ranges[*n - 1].hi == lo) { + ranges[*n - 1].hi = hi; + return; + } + ranges[(*n)++] = (zero_range_t) {lo, hi}; +} + static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) { /* Reject munmap targeting VM infrastructure regions. */ if (guest_range_hits_infra(g, unmap_off, end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, unmap_off, end); /* An interior removal from a file-backed region needs a second owned fd for * the surviving right half. Reserve it before changing overlays, page @@ -4297,17 +4871,19 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) return cleanup_err; } - /* Invalidate PTEs first. This may need to split a 2MiB block which can fail - * if the page table pool is exhausted. Failing before region removal keeps - * metadata consistent. + /* Record which sub-ranges need zeroing BEFORE the PTE invalidation below + * destroys the evidence. Eager regions are zeroed across the whole overlap, + * as before. Lazy (deferred-PTE) regions only need their materialized 2MiB + * blocks zeroed: a block with no L2 mapping was never touched through PTEs, + * host-side fault-in materializes before writing, and the previous unmap of + * that slab range zeroed it -- so its bytes are still zero. This keeps + * munmap cost proportional to memory actually touched instead of to the + * mapping length. */ - if (guest_invalidate_ptes(g, unmap_off, end) < 0) { - if (remove_fd >= 0) - close(remove_fd); - return -LINUX_ENOMEM; - } + zero_range_t zr[MUNMAP_ZERO_RANGES_MAX]; + int nzr = 0; for (int i = 0; i < g->nregions; i++) { - guest_region_t *r = &g->regions[i]; + const guest_region_t *r = &g->regions[i]; if (r->start >= end) break; if (r->end <= unmap_off) @@ -4316,7 +4892,53 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) continue; uint64_t zstart = (r->start > unmap_off) ? r->start : unmap_off; uint64_t zend = (r->end < end) ? r->end : end; - memset((uint8_t *) g->host_base + zstart, 0, zend - zstart); + if (!r->noreserve) { + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: widen the last range instead of dropping any + * span that must be zeroed. Everything between ranges lies + * inside [unmap_off, end) and is being unmapped, so zeroing the + * gap as well is harmless. + */ + zr[nzr - 1].hi = zend; + continue; + } + zero_range_push(zr, &nzr, zstart, zend); + continue; + } + for (uint64_t b = zstart & ~(BLOCK_2MIB - 1); b < zend;) { + if (!guest_va_block_mapped(g, b)) { + /* Skip absent 1GiB/512GiB slots wholesale; a huge untouched + * reservation would otherwise pay one walk per 2MiB. + */ + b = guest_va_next_present_block(g, b + BLOCK_2MIB, zend); + continue; + } + uint64_t lo = (b > zstart) ? b : zstart; + uint64_t hi = (b + BLOCK_2MIB < zend) ? b + BLOCK_2MIB : zend; + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: fold the remainder of this overlap into the + * last range and stop scanning blocks. + */ + zr[nzr - 1].hi = zend; + break; + } + zero_range_push(zr, &nzr, lo, hi); + b += BLOCK_2MIB; + } + } + + /* Invalidate PTEs first. This may need to split a 2MiB block which can fail + * if the page table pool is exhausted. Failing before region removal keeps + * metadata consistent. + */ + if (guest_invalidate_ptes(g, unmap_off, end) < 0) { + if (remove_fd >= 0) + close(remove_fd); + return -LINUX_ENOMEM; + } + for (int i = 0; i < nzr; i++) { + memset((uint8_t *) g->host_base + zr[i].lo, 0, zr[i].hi - zr[i].lo); + guest_dirty_clear_zeroed_range(g, zr[i].lo, zr[i].hi); } guest_region_remove_reserved(g, unmap_off, end, remove_fd); if (unmap_off < g->mmap_rw_gap_hint) @@ -4341,7 +4963,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) if (nranges <= 0) return; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < nranges; i++) { int rc = munmap_guest_range(g, starts[i], ends[i]); if (rc < 0) { @@ -4354,7 +4976,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) } thread_drop_deferred_stack_unmap(t, starts[i], ends[i]); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); } /* sys_munmap. */ @@ -4511,6 +5133,7 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) */ if (guest_range_hits_infra(g, mprot_off, mprot_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, mprot_off, mprot_end); /* Same max_prot check as the high-VA branch above. */ if ((prot & LINUX_PROT_WRITE) && @@ -4528,6 +5151,18 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (prot != LINUX_PROT_NONE) { int page_perms = prot_to_perms(prot); + /* Materialize lazy blocks in the range at their region's + * current prot before the block-granular extend below. The + * extend stamps whole 2MiB blocks with page_perms; on an + * unmaterialized lazy region that would hand every neighbor + * page OUTSIDE [mprot_off, mprot_end) the sub-range's + * permissions (region says RW, PTE says R-only, host-side + * writes EFAULT). guest_materialize_lazy covers block-within- + * region at region prot, so after this the extend is a no-op + * for lazy regions and update_perms below adjusts only the + * requested range. + */ + guest_lazy_faultin_locked(g, mprot_off, mprot_end - mprot_off); if (guest_extend_page_tables(g, mprot_off, mprot_end, page_perms) < 0) return -LINUX_ENOMEM; @@ -4927,7 +5562,13 @@ int mmap_fork_prepare_anon_shared(guest_t *g, if (!txn) return -LINUX_ENOMEM; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); + /* fork callers have quiesced siblings. Drain their last publications, + * revoke every descriptor, and trim never-consumed arena tails before the + * legacy [MMAP_BASE,mmap_next) snapshot range is computed. + */ + mmap_fastpath_revoke_all_locked(g, true); + guest_materialize_wait_all_locked(g); size_t hps = host_page_size_cached(); @@ -5074,7 +5715,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return -LINUX_ENOMEM; } @@ -5087,7 +5728,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return nsnaps; } @@ -5127,7 +5768,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, close(fd); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return 0; } @@ -5146,7 +5787,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, mmap_fork_anon_shared_txn_t *txn = *txn_ptr; int rc = 0; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = txn->noverlays - 1; i >= 0; i--) { const fork_overlay_snapshot_t *ovl = &txn->overlays[i]; @@ -5216,7 +5857,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); mmap_fork_dispose_anon_shared_txn(txn_ptr); return rc; } @@ -5230,7 +5871,7 @@ int mmap_fork_restore_overlays(guest_t *g, const uint64_t *parent_ovl_start, const uint64_t *parent_ovl_end) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int rc = 0; for (int i = 0; i < g->nregions; i++) { @@ -5319,6 +5960,6 @@ int mmap_fork_restore_overlays(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return rc; } diff --git a/src/syscall/proc.c b/src/syscall/proc.c index 281c0227..950d115a 100644 --- a/src/syscall/proc.c +++ b/src/syscall/proc.c @@ -3918,9 +3918,9 @@ int vcpu_run_loop_with_hooks(hv_vcpu_t vcpu, uint32_t fsc_type = (fsc >> 2) & 0xF; if (fsc_type == 0x01) { uint64_t fault_off = far_addr - g->ipa_base; - pthread_mutex_lock(&mmap_lock); - int mat = guest_materialize_lazy(g, fault_off); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + int mat = guest_materialize_lazy_fault(g, fault_off); + mmap_lock_release(); if (mat == 0) { /* Page materialized; the helpers inside * guest_materialize_lazy populated the per-vCPU @@ -3934,6 +3934,25 @@ int vcpu_run_loop_with_hooks(hv_vcpu_t vcpu, * re-fault on the retry, looping until the entry * self-evicts. */ + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_MATERIALIZE); + switch ((tlbi_kind_t) cpu_tlbi_req.kind) { + case TLBI_RANGE: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_VAE); + break; + case TLBI_RANGE_LARGE: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_RVAE); + break; + case TLBI_BROADCAST: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_BCAST); + break; + case TLBI_NONE: + default: + break; + } tlbi_request_emit_to_vcpu(vcpu); break; } @@ -3988,10 +4007,10 @@ int vcpu_run_loop_with_hooks(hv_vcpu_t vcpu, uint64_t live_avail = 0; void *live_pt = NULL; if (stale_plausible) { - pthread_mutex_lock(&mmap_lock); - live_pt = guest_ptr_avail(g, far_addr, &live_avail, - want_perm); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + live_pt = guest_ptr_avail_nofault( + g, far_addr, &live_avail, want_perm); + mmap_lock_release(); } if (live_pt) { /* Bound per vCPU and per (page, faulting PC). A diff --git a/src/syscall/signal.c b/src/syscall/signal.c index 5a21aa8d..1fb47a5f 100644 --- a/src/syscall/signal.c +++ b/src/syscall/signal.c @@ -2260,6 +2260,28 @@ int signal_take_termination_wait_status(void) return status; } +/* Pre-fault the candidate signal-frame windows (current stack and altstack + * top) before sig_lock is taken. The frame write in deliver_signal_locked + * runs under sig_lock; letting it materialize lazy stack pages there would + * acquire mmap_lock in descending lock order. The pre-fault is advisory -- + * the write path still faults in as a backstop -- but it makes the + * under-lock engagement unreachable in practice. Reading the altstack + * fields without sig_lock is benign for the same reason. + */ +static void signal_prefault_frame(hv_vcpu_t vcpu, guest_t *g) +{ + uint64_t need = sizeof(linux_rt_sigframe_t) + 512; + uint64_t sp = 0; + hv_vcpu_get_sys_reg(vcpu, HV_SYS_REG_SP_EL0, &sp); + if (sp > need && sp <= g->guest_size) + guest_lazy_faultin(g, sp - need, need); + thread_entry_t *thr = current_thread; + if (thr && thr->altstack_sp != 0 && + !(thr->altstack_flags & LINUX_SS_DISABLE) && thr->altstack_size > need) + guest_lazy_faultin(g, thr->altstack_sp + thr->altstack_size - need, + need); +} + /* signal_deliver_one() consumed a signal the guest never observes, so the * caller should look at the next one. Distinct from the documented 0/1/-1 * contract of deliver_signal_locked() and never escapes signal_deliver(). @@ -2270,6 +2292,8 @@ static int signal_deliver_one(hv_vcpu_t vcpu, guest_t *g, int *exit_code); int signal_deliver(hv_vcpu_t vcpu, guest_t *g, int *exit_code) { + signal_prefault_frame(vcpu, g); + /* Callers invoke this once per syscall epilogue, so stopping at the first * signal that turns out to be discarded (SIG_IGN, or a SIG_DFL disposition * of ignore/stop/continue) would let a lower-numbered ignored signal mask a @@ -2366,6 +2390,7 @@ int signal_deliver_fault(hv_vcpu_t vcpu, guest_t *g, int signum, int *exit_code) * threads faulting on the same signal collapse into one bit so one fault is * lost. Deliver directly here, never touching sig_state.pending. */ + signal_prefault_frame(vcpu, g); pthread_mutex_lock(&sig_lock); /* Linux force_sig_info_to_task(): a forced synchronous fault cannot be diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index a0e86d7e..e51ab24a 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -71,6 +71,7 @@ #include "syscall/time.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "debug/syscall-hist.h" @@ -180,9 +181,9 @@ typedef int64_t (*syscall_handler_t)(guest_t *g, { \ (void) g; (void) x0; (void) x1; (void) x2; \ (void) x3; (void) x4; (void) x5; (void) verbose; \ - pthread_mutex_lock(&mmap_lock); \ + mmap_lock_acquire(g); \ int64_t r = (body); \ - pthread_mutex_unlock(&mmap_lock); \ + mmap_lock_release(); \ return r; \ } @@ -499,16 +500,16 @@ static void sc_sync_regions_inline(guest_t *g) * position) cannot make us skip an entry permanently. */ for (int i = 0;; i++) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); if (i >= g->nregions) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); break; } const guest_region_t *r = &g->regions[i]; int duped = -1; if (r->shared && r->backing_fd >= 0) duped = dup(r->backing_fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); if (duped < 0) continue; (void) fsync(duped); @@ -539,7 +540,7 @@ static int64_t sc_sync_impl(guest_t *g) } pthread_mutex_unlock(&fd_lock); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < g->nregions && n < (int) cap; i++) { const guest_region_t *r = &g->regions[i]; if (!r->shared || r->backing_fd < 0) @@ -549,7 +550,7 @@ static int64_t sc_sync_impl(guest_t *g) continue; hosts[n++] = duped; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); /* fsync each dup outside both locks so a slow disk does not stall * concurrent FD or memory operations on other threads. @@ -735,7 +736,7 @@ static int64_t sc_mincore(guest_t *g, * never early-returns on a hole. */ uint8_t chunk[512]; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int ri = guest_region_first_end_above(g, addr); for (uint64_t done = 0; done < npages;) { uint64_t batch = npages - done; @@ -750,13 +751,19 @@ static int64_t sc_mincore(guest_t *g, if (!mapped) has_hole = true; } - if (guest_write(g, vec + done, chunk, batch) < 0) { - pthread_mutex_unlock(&mmap_lock); + /* sc_mincore holds mmap_lock while regions[] is swept. Materialize a + * valid lazy output block through the locked entry point, then use a + * no-fault copy so an invalid vec returns EFAULT instead of trying to + * acquire mmap_lock recursively. + */ + (void) guest_lazy_faultin_locked(g, vec + done, batch); + if (guest_write_nofault(g, vec + done, chunk, batch) < 0) { + mmap_lock_release(); return -LINUX_EFAULT; } done += batch; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return has_hole ? -LINUX_ENOMEM : 0; } @@ -973,6 +980,19 @@ static int64_t sc_set_tid_address(guest_t *g, return proc_get_pid(); } +static uint64_t mmap_fastpath_eligible_length(uint64_t addr, + uint64_t length, + uint64_t prot, + uint64_t flags) +{ + if (addr != 0 || prot != (LINUX_PROT_READ | LINUX_PROT_WRITE) || + (flags & ~(uint64_t) LINUX_MAP_NORESERVE) != + (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS) || + length == 0 || length > UINT64_MAX - (GUEST_PAGE_SIZE - 1)) + return 0; + return (length + GUEST_PAGE_SIZE - 1) & ~(GUEST_PAGE_SIZE - 1); +} + static int64_t sc_mmap(guest_t *g, uint64_t x0, uint64_t x1, @@ -982,9 +1002,12 @@ static int64_t sc_mmap(guest_t *g, uint64_t x5, bool verbose) { - pthread_mutex_lock(&mmap_lock); + uint64_t refill_len = mmap_fastpath_eligible_length(x0, x1, x2, x3); + mmap_lock_acquire(g); int64_t r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); - pthread_mutex_unlock(&mmap_lock); + if (r >= 0 && refill_len) + mmap_fastpath_refill_current_locked(g, refill_len); + mmap_lock_release(); log_debug(" mmap(0x%llx, 0x%llx) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) (uint64_t) r); @@ -1001,9 +1024,9 @@ static int64_t sc_mremap(guest_t *g, bool verbose) { (void) x5; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int64_t r = sys_mremap(g, x0, x1, x2, (int) x3, x4); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); log_debug(" mremap(0x%llx, 0x%llx, 0x%llx, 0x%x) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) x2, (int) x3, @@ -2178,10 +2201,7 @@ static int64_t sc_execve(guest_t *g, (void) x3; (void) x4; (void) x5; - pthread_mutex_lock(&mmap_lock); - int64_t r = sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); - pthread_mutex_unlock(&mmap_lock); - return r; + return sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); } static int64_t sc_execveat(guest_t *g, @@ -2197,8 +2217,9 @@ static int64_t sc_execveat(guest_t *g, hv_vcpu_t vcpu = current_thread->vcpu; int dirfd = (int) x0, flags = (int) x4; - /* Resolve the target path before taking mmap_lock (path resolution may call - * fd_to_host / openat which do not need mmap_lock). + /* Resolve the target path before entering the exec transaction. Path + * resolution may call fd_to_host / openat and does not need mmap_lock; + * sys_execve takes it at the point of no return. */ uint64_t path_gva = x1; char resolved[LINUX_PATH_MAX]; @@ -2259,7 +2280,6 @@ static int64_t sc_execveat(guest_t *g, need_resolve = true; } - pthread_mutex_lock(&mmap_lock); int64_t r; if (need_resolve) { /* Use the host-resolved path directly so execveat does not copy a host @@ -2269,7 +2289,6 @@ static int64_t sc_execveat(guest_t *g, } else { r = sys_execve(vcpu, g, path_gva, x2, x3, verbose, NULL); } - pthread_mutex_unlock(&mmap_lock); return r; } diff --git a/src/syscall/sysvipc.c b/src/syscall/sysvipc.c index 503b2510..096f61fc 100644 --- a/src/syscall/sysvipc.c +++ b/src/syscall/sysvipc.c @@ -255,7 +255,12 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) return gva; /* propagate mmap error */ } - /* Copy host shm content into guest memory */ + /* Copy host shm content into guest memory. sys_shmat runs under + * mmap_lock (SC_LOCKED), so the resolve-time lazy fault-in inside + * guest_write would self-deadlock on it; materialize the fresh anonymous + * mapping through the locked variant first. + */ + guest_lazy_faultin_locked(g, (uint64_t) gva, seg_size); if (guest_write(g, (uint64_t) gva, host_addr, seg_size) < 0) { shmdt(host_addr); return -LINUX_EFAULT; @@ -314,7 +319,11 @@ int64_t sys_shmdt(guest_t *g, uint64_t shmaddr_gva) /* Write back guest modifications to host shm (unless read-only) */ if (!entry.rdonly) { - /* Read guest memory back to host shm buffer */ + /* Read guest memory back to host shm buffer. Same SC_LOCKED + * self-deadlock hazard as the shmat copy-in: pages the guest never + * touched may still be unmaterialized. + */ + guest_lazy_faultin_locked(g, entry.guest_gva, entry.size); guest_read(g, entry.guest_gva, entry.host_addr, entry.size); } diff --git a/tests/bench-mmap-lazy.c b/tests/bench-mmap-lazy.c new file mode 100644 index 00000000..377aae70 --- /dev/null +++ b/tests/bench-mmap-lazy.c @@ -0,0 +1,111 @@ +/* + * Guest microbenchmark for anonymous private mmap latency vs size. + * + * Measures mmap(), first-touch, and munmap() latency for MAP_PRIVATE | + * MAP_ANONYMOUS mappings from 4 KiB to 32 GiB. A lazy (deferred page-table) + * implementation should show size-independent mmap/munmap cost; an eager + * implementation scales linearly with length and exhausts resources on the + * multi-GiB sizes. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static uint64_t now_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void bench_size(uint64_t size, int iters) +{ + uint64_t t_map = 0, t_touch = 0, t_unmap = 0; + int ok = 0; + + for (int i = 0; i < iters; i++) { + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = now_ns(); + if (p == MAP_FAILED) { + printf("%10llu KiB: mmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + /* First touch: one write at the start and one mid-mapping. */ + volatile char *c = p; + c[0] = 1; + c[size / 2] = 1; + uint64_t t2 = now_ns(); + int rc = munmap(p, size); + uint64_t t3 = now_ns(); + if (rc != 0) { + printf("%10llu KiB: munmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + t_map += t1 - t0; + t_touch += t2 - t1; + t_unmap += t3 - t2; + ok++; + } + printf( + "%10llu KiB: mmap %10llu ns touch2 %10llu ns munmap %10llu ns " + "(%d iters)\n", + (unsigned long long) (size >> 10), + (unsigned long long) (t_map / (uint64_t) ok), + (unsigned long long) (t_touch / (uint64_t) ok), + (unsigned long long) (t_unmap / (uint64_t) ok), ok); +} + +int main(int argc, char **argv) +{ + static const struct { + uint64_t size; + int iters; + } cases[] = { + {4ull << 10, 200}, {64ull << 10, 200}, {2ull << 20, 100}, + {64ull << 20, 20}, {512ull << 20, 10}, {2ull << 30, 5}, + {8ull << 30, 3}, {16ull << 30, 3}, {32ull << 30, 3}, + {64ull << 30, 1}, {128ull << 30, 1}, {256ull << 30, 1}, + }; + /* Optional argv[1]: cap size in GiB (eager implementations commit host + * memory for every byte mapped; the full matrix would thrash small hosts). + */ + uint64_t cap = ~0ull; + if (argc > 1) + cap = (uint64_t) atoll(argv[1]) << 30; + + setvbuf(stdout, NULL, _IONBF, 0); + printf("anon private mmap latency vs size\n"); + for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) + if (cases[i].size <= cap) + bench_size(cases[i].size, cases[i].iters); + + /* Full-touch throughput sanity: 64 MiB written end to end. */ + uint64_t size = 64ull << 20; + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf("full-touch mmap failed: %s\n", strerror(errno)); + return 1; + } + memset(p, 0xa5, size); + uint64_t t1 = now_ns(); + munmap(p, size); + printf("mmap+memset 64MiB: %llu ns (%.2f GiB/s)\n", + (unsigned long long) (t1 - t0), + (double) size / 1.073741824 / (double) (t1 - t0)); + return 0; +} diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c new file mode 100644 index 00000000..c57e9d03 --- /dev/null +++ b/tests/bench-mmap.c @@ -0,0 +1,455 @@ +/* + * Comprehensive anonymous-mmap microbenchmark for elfuse. + * + * Measures the guest-visible cost of the mmap subsystem in isolation: + * allocation, teardown, first-touch faults, permission splitting, and remap. It + * is self-contained -- no external harness -- and is meant to be run under + * elfuse (./build/elfuse ./build/bench-mmap) but also runs on any aarch64-linux + * host for a ground-truth comparison. + * + * Timing: reads CNTVCT_EL0 directly at EL0 (enabled by CNTKCTL_EL1.EL0VCTEN in + * bootstrap.c), so a measurement costs an isb + mrs, not a clock_gettime SVC. + * On Apple Silicon CNTFRQ is ~24 MHz (~41.7 ns/tick); amortizing over an + * adaptive batch drives the effective resolution well below one tick. This is + * the key fairness property: clock_gettime on a static guest falls through to + * the ~2 us SVC path and swamps any sub-us operation. + * + * Every case takes one untimed warmup pass (to pay the one-time arena carve and + * page-table extension) and reports the median and min over ITERS runs. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +/* CNTVCT timing */ + +static double ns_per_tick; + +static inline uint64_t rd(void) +{ + uint64_t v; + __asm__ volatile("isb\n\tmrs %0, cntvct_el0" : "=r"(v)); + return v; +} + +static void clock_init(void) +{ + uint64_t f; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(f)); + if (f == 0) + f = 24000000; /* defensive: assume 24 MHz if RES0 */ + ns_per_tick = 1e9 / (double) f; +} + +static double ns(uint64_t ticks) +{ + return (double) ticks * ns_per_tick; +} + +static int cmp_d(const void *a, const void *b) +{ + double x = *(const double *) a, y = *(const double *) b; + return (x > y) - (x < y); +} + +static double median(double *v, int n) +{ + qsort(v, n, sizeof(*v), cmp_d); + return (n & 1) ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); +} + +#define ITERS 15 +#define MAXB 64 +#define KIB (1ULL << 10) +#define MIB (1ULL << 20) +#define GIB (1ULL << 30) + +/* Batch size: amortize the coarse counter over B ops while bounding the live + * address footprint of one timed batch to ~256 MiB. + */ +static int batch_for(uint64_t size) +{ + uint64_t b = (256 * MIB) / size; + if (b < 1) + b = 1; + if (b > MAXB) + b = MAXB; + return (int) b; +} + +static const char *human(uint64_t s, char *buf) +{ + if (s >= GIB) + sprintf(buf, "%llu GiB", (unsigned long long) (s / GIB)); + else if (s >= MIB) + sprintf(buf, "%llu MiB", (unsigned long long) (s / MIB)); + else + sprintf(buf, "%llu KiB", (unsigned long long) (s / KIB)); + return buf; +} + +/* A. mmap + munmap latency vs size (steady state) NULL-hint allocate then free, + * batched. Iterations after the first reuse freed address space, so this is the + * realistic repeated-allocation number a workload sees, not the one-shot fresh + * case (that is section B). + */ +static void bench_size_sweep(void) +{ + static const uint64_t sizes[] = { + 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, + 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, + }; + printf( + "== A. mmap / munmap latency vs size (steady state, NULL hint) ==\n"); + printf("%-10s %6s %12s %12s\n", "size", "batch", "mmap ns", "munmap ns"); + void *ptr[MAXB]; + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + int b = batch_for(size); + double mm[ITERS], um[ITERS]; + int ok = 1; + for (int it = -1; it < ITERS && ok; it++) { + uint64_t t0 = rd(); + for (int i = 0; i < b; i++) + ptr[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + for (int i = 0; i < b; i++) + if (ptr[i] == MAP_FAILED) + ok = 0; + if (!ok) + break; + uint64_t t2 = rd(); + for (int i = 0; i < b; i++) + munmap(ptr[i], size); + uint64_t t3 = rd(); + if (it >= 0) { + mm[it] = ns(t1 - t0) / b; + um[it] = ns(t3 - t2) / b; + } + } + char hb[16]; + if (!ok) { + printf("%-10s %6d %12s %12s\n", human(size, hb), b, "FAILED", "-"); + continue; + } + printf("%-10s %6d %12.1f %12.1f\n", human(size, hb), b, + median(mm, ITERS), median(um, ITERS)); + } + printf("\n"); +} + +/* B. fresh bump-tail mmap (isolates the lazy_fresh_range path) Allocate a + * bounded sequential run WITHOUT freeing, so every mapping lands at or above + * the arena high-water -- exactly the case lazy_fresh_range skips the stale-PTE + * scan for. The run is kept small enough (<= 1000 regions, well under + * GUEST_MAX_REGIONS, footprint <= 2 GiB) that region bookkeeping and page-table + * extension do not dominate, and every result is failure-checked. Run this + * binary against an opt-off build to read the skip's contribution as the + * difference on this identical code path -- a MAP_FIXED "recycled" compare + * would instead measure the region-snapshot replacement path, not the skip. + */ +static void bench_fresh(void) +{ + static const uint64_t sizes[] = {4 * KIB, 64 * KIB, MIB, + 2 * MIB, 8 * MIB, 128 * MIB, + GIB, 8 * GIB, 32 * GIB}; + printf( + "== B. fresh bump-tail mmap, per-mmap ns (lazy_fresh_range path) ==\n"); + printf("%-10s %8s %14s\n", "size", "count", "fresh mmap ns"); + void *run[1000]; + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + int n = (int) (2 * GIB / size); + if (n < 1) + n = 1; + if (n > 1000) + n = 1000; + /* warmup one fresh mapping so the arena high-water is already primed */ + void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (w != MAP_FAILED) + munmap(w, size); + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + char hb[16]; + if (failed) { + printf("%-10s %8d %14s (%d failed)\n", human(size, hb), n, + "PARTIAL", failed); + continue; + } + printf("%-10s %8d %14.1f\n", human(size, hb), n, ns(t1 - t0) / n); + } + printf("\n"); +} + +/* C. first-touch page-fault cost Touch one byte per page at a 16 KiB stride so + * no two touches share a macOS host page; every touch is a genuine fault (HVC + * #11 -> host fault handler -> page-table install + zero). Reports per-fault + * ns. + */ +static void bench_fault(void) +{ + const uint64_t stride = 16 * KIB; + const int pages = 512; + uint64_t size = stride * (uint64_t) (pages + 1); + printf("== C. first-touch fault cost (16 KiB stride, %d pages) ==\n", + pages); + double per[ITERS]; + for (int it = -1; it < ITERS; it++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED: %s\n\n", strerror(errno)); + return; + } + uint64_t t0 = rd(); + for (int i = 0; i < pages; i++) + p[(uint64_t) i * stride] = 1; + uint64_t t1 = rd(); + munmap((void *) p, size); + if (it >= 0) + per[it] = ns(t1 - t0) / pages; + } + printf(" per-fault: median %.1f ns min %.1f ns\n\n", median(per, ITERS), + per[0]); +} + +/* D. mprotect split cost Flip the middle 4 KiB of a 2 MiB RW block to + * PROT_READ, forcing guest_split_block to convert the L2 block into 512 L3 + * pages. Restore between iterations so each run does a fresh split. + */ +static void bench_mprotect_split(void) +{ + printf( + "== D. mprotect split (2 MiB block -> L3, protect middle 4 KiB) ==\n"); + double sp[ITERS]; + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint8_t *mid = p + MIB; + uint64_t t0 = rd(); + int rc = mprotect(mid, 4 * KIB, PROT_READ); + uint64_t t1 = rd(); + munmap(p, 2 * MIB); + if (rc != 0) { + printf(" mprotect FAILED: %s\n\n", strerror(errno)); + return; + } + if (it >= 0) + sp[it] = ns(t1 - t0); + } + printf(" split: median %.1f ns min %.1f ns\n\n", median(sp, ITERS), + sp[0]); +} + +/* E. mremap grow: in-place vs forced move */ +static void bench_mremap(void) +{ + printf("== E. mremap grow 4 KiB -> 8 KiB ==\n"); + double inp[ITERS], mov[ITERS]; + + /* In-place: no blocker, the following page is free. */ + for (int it = -1; it < ITERS; it++) { + void *p = mmap(NULL, 4 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + munmap(p, 4 * KIB); + printf(" mremap in-place FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (it >= 0) + inp[it] = ns(t1 - t0); + } + + /* Forced move: a PROT_READ blocker sits immediately after, so the grow must + * relocate. + */ + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 8 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + /* free the tail page and pin it read-only so in-place growth is blocked + * but the head is still a 4 KiB mapping. + */ + munmap(p + 4 * KIB, 4 * KIB); + void *blk = mmap(p + 4 * KIB, 4 * KIB, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + printf(" mremap move FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (blk != MAP_FAILED) + munmap(blk, 4 * KIB); + if (it >= 0) + mov[it] = ns(t1 - t0); + } + printf(" in-place: median %.1f ns min %.1f ns\n", median(inp, ITERS), + inp[0]); + printf(" move: median %.1f ns min %.1f ns\n\n", median(mov, ITERS), + mov[0]); +} + +/* F. multi-threaded fresh mmap under mmap_lock Several threads hammer fresh + * bump-tail mmaps concurrently. mmap serializes on mmap_lock, so this exposes + * both lock contention and any per-mmap TLBI shootdown cost -- the one place a + * "skip the invalidate on fresh ranges" optimization could pay off that a + * single-threaded run cannot see. Threads do not free during the timed run + * (every mapping stays fresh); total live regions are capped under + * GUEST_MAX_REGIONS. Compare against an opt-off build to read the skip's + * multi-threaded contribution. + */ +typedef struct { + uint64_t size; + int n; + void **buf; + double per_op_ns; + int failed; +} mt_arg_t; + +static pthread_barrier_t mt_barrier; + +/* CNTVCT_EL0 reads a constant on worker vCPUs (EL0VCTEN is set for the main + * vCPU only), so the MT worker brackets its whole loop with clock_gettime and + * amortizes the one SVC pair over n mmaps. + */ +static uint64_t mono_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void *mt_worker(void *p) +{ + mt_arg_t *a = p; + pthread_barrier_wait(&mt_barrier); + uint64_t t0 = mono_ns(); + for (int i = 0; i < a->n; i++) + a->buf[i] = mmap(NULL, a->size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = mono_ns(); + a->per_op_ns = (double) (t1 - t0) / a->n; + for (int i = 0; i < a->n; i++) { + if (a->buf[i] == MAP_FAILED) + a->failed++; + else + munmap(a->buf[i], a->size); + } + return NULL; +} + +#define MT_MAX_THREADS 4 +#define MT_REGION_CAP 3000 /* keep T*n well under GUEST_MAX_REGIONS (4096) */ + +static void bench_mt(void) +{ + static const uint64_t sizes[] = {4 * KIB, 2 * MIB}; + static const int threads[] = {2, 4}; + printf( + "== F. multi-threaded fresh mmap, per-op ns (mmap_lock contention) " + "==\n"); + printf("%-10s %8s %12s %12s\n", "size", "threads", "mean ns", "max ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + for (unsigned t = 0; t < sizeof(threads) / sizeof(threads[0]); t++) { + int T = threads[t]; + int n = MT_REGION_CAP / T; + if (n < 1) + n = 1; + mt_arg_t arg[MT_MAX_THREADS]; + pthread_t th[MT_MAX_THREADS]; + int ok = 1; + for (int i = 0; i < T; i++) { + arg[i].size = size; + arg[i].n = n; + arg[i].per_op_ns = 0; + arg[i].failed = 0; + arg[i].buf = calloc(n, sizeof(void *)); + if (!arg[i].buf) + ok = 0; + } + /* prime the arena high-water so the timed run is genuinely fresh */ + void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (w != MAP_FAILED) + munmap(w, size); + pthread_barrier_init(&mt_barrier, NULL, (unsigned) T); + for (int i = 0; i < T && ok; i++) + if (pthread_create(&th[i], NULL, mt_worker, &arg[i]) != 0) + ok = 0; + double sum = 0, mx = 0; + int failed = 0; + for (int i = 0; i < T; i++) { + pthread_join(th[i], NULL); + sum += arg[i].per_op_ns; + if (arg[i].per_op_ns > mx) + mx = arg[i].per_op_ns; + failed += arg[i].failed; + free(arg[i].buf); + } + pthread_barrier_destroy(&mt_barrier); + char hb[16]; + if (!ok || failed) + printf("%-10s %8d %12s\n", human(size, hb), T, "FAILED"); + else + printf("%-10s %8d %12.1f %12.1f\n", human(size, hb), T, sum / T, + mx); + } + } + printf("\n"); +} + +int main(void) +{ + clock_init(); + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_size_sweep(); + bench_fresh(); + bench_fault(); + bench_mprotect_split(); + bench_mremap(); + bench_mt(); + return 0; +} diff --git a/tests/manifest.txt b/tests/manifest.txt index a75089fb..8df38768 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -113,6 +113,10 @@ test-guard-page test-mmap-hint test-mmap-sigbus-efault +[section] Lazy anonymous mmap tests +test-mmap-lazy +test-mmap-fastpath + [section] mremap tests test-mremap test-mremap-infra diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 0f420098..010e8e91 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -20,9 +20,10 @@ #define PREVIOUS_ELFL_MAGIC 0x454C464CU #define PREVIOUS_ELFM_MAGIC 0x454C464DU #define PREVIOUS_ELFN_MAGIC 0x454C464EU +#define PREVIOUS_ELFO_MAGIC 0x454C464FU -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464FU, - "fork IPC protocol magic must remain ELFO until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C4650U, + "fork IPC protocol magic must remain ELFP until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); @@ -34,6 +35,8 @@ _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFM_MAGIC, "start_stack header field requires rejecting ELFM peers"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFN_MAGIC, "region fork metadata requires rejecting ELFN peers"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFO_MAGIC, + "dirty-bitmap wire must reject old ELFO children/parents"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-mmap-dirty-stats.sh b/tests/test-mmap-dirty-stats.sh new file mode 100755 index 00000000..9d8e2715 --- /dev/null +++ b/tests/test-mmap-dirty-stats.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Counter-backed dirty-map materialization integration checks. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-lazy} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-dirty-map.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" \ + > "$TMPDIR_CASE/out" 2> "$TMPDIR_CASE/err" + +counter() +{ + key=$1 + awk -v key="$key" \ + '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/err" +} + +require_ge() +{ + key=$1 + floor=$2 + value=$(counter "$key") || { + printf 'missing dirty-map counter %s\n' "$key" >&2 + return 1 + } + if [ "$value" -lt "$floor" ]; then + printf '%s=%s, expected >= %s\n' "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_ge FAULT_CLEAN_SKIP 1 +require_ge FAULT_DIRTY_MEMSET 1 +require_ge FAULT_ALREADY_VALID 1 +require_ge FAULT_WINDOW_BYTES 2097152 + +printf ' clean-block zero skip OK\n' +printf ' dirty-block selective memset OK\n' +printf ' already-valid early return OK\n' +printf ' materialized-window bytes OK\n' +printf 'test-mmap-dirty-stats: PASS\n' diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh new file mode 100755 index 00000000..fa92a8a7 --- /dev/null +++ b/tests/test-mmap-fastpath-stats.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# Counter-backed refill, adaptive sizing, giant-request guard, and VA recycle +# integration checks for the EL1 anonymous-mmap consumer fast path. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-fastpath} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-mmap-stats.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +run_case() +{ + case_name=$1 + out="$TMPDIR_CASE/$case_name.out" + err="$TMPDIR_CASE/$case_name.err" + ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" "--stats-$case_name" \ + > "$out" 2> "$err" +} + +counter() +{ + case_name=$1 + key=$2 + value=$(awk -v key="$key" '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/$case_name.err") || { + printf 'missing counter %s in case %s\n' "$key" "$case_name" >&2 + return 1 + } + printf '%s\n' "$value" +} + +require_ge() +{ + case_name=$1 + key=$2 + floor=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -lt "$floor" ]; then + printf '%s: %s=%s, expected >= %s\n' \ + "$case_name" "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_eq() +{ + case_name=$1 + key=$2 + expected=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -ne "$expected" ]; then + printf '%s: %s=%s, expected %s\n' \ + "$case_name" "$key" "$value" "$expected" >&2 + return 1 + fi +} + +require_le() +{ + case_name=$1 + key=$2 + ceiling=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -gt "$ceiling" ]; then + printf '%s: %s=%s, expected <= %s\n' \ + "$case_name" "$key" "$value" "$ceiling" >&2 + return 1 + fi +} + +run_case np2-10m +require_ge np2-10m MMAP_HIT 80 +require_ge np2-10m MMAP_CAPACITY_MISS 1 +require_ge np2-10m MMAP_RING_FULL 1 +printf ' sustained 10 MiB stream OK\n' + +run_case np2-48m +require_ge np2-48m MMAP_HIT 40 +require_ge np2-48m MMAP_CAPACITY_MISS 1 +printf ' sustained 48 MiB stream OK\n' + +run_case np2-100m +require_ge np2-100m MMAP_HIT 24 +require_ge np2-100m MMAP_CAPACITY_MISS 1 +printf ' sustained 100 MiB stream OK\n' + +run_case escalation +require_ge escalation MMAP_HIT 45 +require_eq escalation MMAP_ARENA_CURRENT 1073741824 +printf ' 10 MiB -> 512 MiB escalation OK\n' + +run_case giant-guard +require_ge giant-guard MMAP_HIT 34 +require_le giant-guard MMAP_ARENA_PEAK 536870912 +printf ' >1 GiB giant request guard OK\n' + +run_case adaptive-small +require_eq adaptive-small MMAP_ARENA_CURRENT 67108864 +require_eq adaptive-small MMAP_ARENA_PEAK 67108864 +printf ' small-stream arena floor OK\n' + +run_case adaptive-decay +require_eq adaptive-decay MMAP_ARENA_CURRENT 67108864 +require_eq adaptive-decay MMAP_ARENA_PEAK 1073741824 +printf ' one-generation arena decay OK\n' + +run_case recycle +require_ge recycle MMAP_RECYCLE 1 +require_le recycle MMAP_HIGH_WATER 201326592 +printf ' arena VA recycling OK\n' + +printf 'test-mmap-fastpath-stats: PASS\n' diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c new file mode 100644 index 00000000..4a9aa21e --- /dev/null +++ b/tests/test-mmap-fastpath.c @@ -0,0 +1,278 @@ +/* + * EL1 consumer-mmap fast-path integration tests. + * + * Run through the dedicated make target without an ELFUSE_MMAP_FASTPATH + * override so the default-enabled configuration is exercised. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +static sigjmp_buf segv_jmp; + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static int maps_extent_for(uintptr_t needle, + uintptr_t *lo_out, + uintptr_t *hi_out) +{ + int fd = open("/proc/self/maps", O_RDONLY); + if (fd < 0) + return -1; + char buf[16384]; + ssize_t n = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (n <= 0) + return -1; + buf[n] = '\0'; + + char *line = buf; + while (*line) { + unsigned long long lo, hi; + if (sscanf(line, "%llx-%llx", &lo, &hi) == 2 && needle >= lo && + needle < hi) { + *lo_out = (uintptr_t) lo; + *hi_out = (uintptr_t) hi; + return 0; + } + char *nl = strchr(line, '\n'); + if (!nl) + break; + line = nl + 1; + } + return -1; +} + +static void test_fidelity(void) +{ + TEST("unconsumed arena is absent and faults"); + struct sigaction sa = {.sa_handler = segv_handler}; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, NULL) != 0) { + FAIL("sigaction"); + return; + } + + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[0] = 0x5a; /* drains the publication through the fault-side lock */ + + volatile uint8_t *unconsumed = p + 4096; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *unconsumed; + FAIL("wild read into unconsumed arena did not SIGSEGV"); + munmap(p, 4096); + return; + } + + uintptr_t lo = 0, hi = 0; + if (maps_extent_for((uintptr_t) p, &lo, &hi) < 0 || lo != (uintptr_t) p || + hi != (uintptr_t) p + 4096) { + FAIL("/proc/self/maps exposed more than the consumed page"); + munmap(p, 4096); + return; + } + munmap(p, 4096); + PASS(); +} + +static void test_exhaustion_fallback(void) +{ + TEST("arena exhaustion falls back to host mmap"); + const size_t len = 80ULL << 20; /* larger than the first 64MiB arena */ + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("80MiB mmap"); + return; + } + if (p[0] != 0 || p[len - 1] != 0) { + FAIL("fallback mapping was not zero-filled"); + munmap(p, len); + return; + } + p[0] = 1; + p[len - 1] = 2; + if (munmap(p, len) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +typedef struct { + int iterations; + _Atomic int *failed; +} storm_arg_t; + +static void *storm_worker(void *opaque) +{ + storm_arg_t *arg = opaque; + for (int i = 0; i < arg->iterations; i++) { + size_t len = (size_t) ((i & 7) + 1) * 4096; + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + p[0] = (uint8_t) i; + p[len - 1] = (uint8_t) (i ^ 0x5a); + if (munmap(p, len) != 0) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + } + return NULL; +} + +static void test_mt_storm_and_fork_exec(void) +{ + TEST("multi-vCPU mmap storm with fork+exec revocation"); + enum { NTHREADS = 8 }; + pthread_t threads[NTHREADS]; + _Atomic int failed = 0; + storm_arg_t arg = {.iterations = 400, .failed = &failed}; + + int made = 0; + for (; made < NTHREADS; made++) { + if (pthread_create(&threads[made], NULL, storm_worker, &arg) != 0) { + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + break; + } + } + + pid_t pid = fork(); + if (pid == 0) { + char *const argv[] = {(char *) "/proc/self/exe", NULL}; + char *const envp[] = {(char *) "ELFUSE_FASTPATH_EXEC_CHILD=1", NULL}; + execve(argv[0], argv, envp); + _exit(111); + } + if (pid < 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + + for (int i = 0; i < made; i++) + pthread_join(threads[i], NULL); + + if (pid > 0) { + int status = 0; + if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + } + + /* The parent's arenas were revoked for the fork snapshot. This pair makes + * the first call take the generation fallback and verifies service resumes. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + else { + p[0] = 7; + munmap(p, 4096); + } + + if (__atomic_load_n(&failed, __ATOMIC_RELAXED)) { + FAIL("storm/fork/exec worker failure"); + return; + } + PASS(); +} + +static int stats_stream(size_t len, int iterations, bool release_each) +{ + for (int i = 0; i < iterations; i++) { + void *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (release_each && munmap(p, len) != 0) + return 1; + } + return 0; +} + +static int run_stats_case(const char *name) +{ + if (strcmp(name, "np2-10m") == 0) + return stats_stream(10ULL << 20, 96, false); + if (strcmp(name, "np2-48m") == 0) + return stats_stream(48ULL << 20, 48, false); + if (strcmp(name, "np2-100m") == 0) + return stats_stream(100ULL << 20, 30, false); + if (strcmp(name, "escalation") == 0) { + if (stats_stream(10ULL << 20, 48, false) != 0) + return 1; + return stats_stream(512ULL << 20, 3, false); + } + if (strcmp(name, "giant-guard") == 0) { + if (stats_stream(10ULL << 20, 32, false) != 0) + return 1; + for (int i = 0; i < 6; i++) { + if (stats_stream(2ULL << 30, 1, false) != 0 || + stats_stream(10ULL << 20, 1, false) != 0) + return 1; + } + return 0; + } + if (strcmp(name, "adaptive-small") == 0) + return stats_stream(64ULL << 10, 1100, false); + if (strcmp(name, "adaptive-decay") == 0) { + if (stats_stream(64ULL << 10, 1100, false) != 0 || + stats_stream(500ULL << 20, 1, false) != 0) + return 1; + /* Ring-full fallbacks do not consume the arena cursor, so exceed the + * nominal 16384 pages enough to force a true 1GiB capacity rollover. + */ + return stats_stream(64ULL << 10, 18000, false); + } + if (strcmp(name, "recycle") == 0) + return stats_stream(64ULL << 10, 6000, true); + return 2; +} + +int main(int argc, char **argv) +{ + if (argc == 2 && strncmp(argv[1], "--stats-", 8) == 0) + return run_stats_case(argv[1] + 8); + + if (getenv("ELFUSE_FASTPATH_EXEC_CHILD")) { + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + p[0] = 0xa5; + return p[0] == 0xa5 ? 0 : 1; + } + + test_fidelity(); + test_exhaustion_fallback(); + test_mt_storm_and_fork_exec(); + + printf("\ntest-mmap-fastpath: %d passed, %d failed - %s\n", passes, fails, + fails ? "FAIL" : "PASS"); + return fails ? 1 : 0; +} diff --git a/tests/test-mmap-lazy.c b/tests/test-mmap-lazy.c new file mode 100644 index 00000000..8541b819 --- /dev/null +++ b/tests/test-mmap-lazy.c @@ -0,0 +1,771 @@ +/* + * Lazy anonymous mmap regression tests + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Private anonymous mappings defer page-table creation and zeroing to first + * touch. These tests pin down the guest-visible contract of that laziness: + * huge reservations succeed and read as zeros, address reuse never leaks + * stale bytes, host-side syscall access (read/write/futex) works on memory + * the guest never touched, PROT_NONE stays a faulting reservation, data + * survives PROT_NONE round trips and fork, and concurrent first touch from + * multiple threads never loses a write to the deferred zeroing. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#define BLOCK_2MIB (2ULL << 20) + +#ifndef FUTEX_WAIT +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#endif + +/* Largest plain anonymous RW mapping the kernel grants. On elfuse the lazy + * path must take this well past physical memory; on real Linux the result + * depends on the overcommit heuristic, so the tests only require >= 1 GiB + * and probe downward. + */ +static void *map_largest(size_t *out_size) +{ + static const size_t sizes[] = { + 64ULL << 30, + 16ULL << 30, + 4ULL << 30, + 1ULL << 30, + }; + for (unsigned i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + void *p = mmap(NULL, sizes[i], PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p != MAP_FAILED) { + *out_size = sizes[i]; + return p; + } + } + return NULL; +} + +static void test_huge_sparse(void) +{ + TEST("huge mmap + sparse touch"); + size_t size = 0; + volatile uint8_t *p = map_largest(&size); + if (!p || size < (1ULL << 30)) { + FAIL("no >=1GiB anonymous mapping granted"); + return; + } + /* Sparse probes: start, one per size/8 stride, last page. All must read + * zero and accept writes. + */ + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != 0) { + FAIL("fresh mapping reads nonzero"); + munmap((void *) p, size); + return; + } + p[off] = (uint8_t) (i + 1); + } + if (p[size - 1] != 0) { + FAIL("last page reads nonzero"); + munmap((void *) p, size); + return; + } + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != (uint8_t) (i + 1)) { + FAIL("sparse write lost"); + munmap((void *) p, size); + return; + } + } + if (munmap((void *) p, size) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +static void test_zero_reuse(void) +{ + TEST("address reuse reads zero"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap 1"); + return; + } + memset(p, 0xa5, size); + munmap(p, size); + uint8_t *q = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap 2"); + return; + } + /* The allocator typically reuses the freed range; either way no byte may + * be nonzero. Check one page per 2MiB block plus both ends. + */ + for (size_t off = 0; off < size; off += 4096) { + if (q[off] != 0) { + FAIL("stale data after reuse"); + munmap(q, size); + return; + } + } + munmap(q, size); + PASS(); +} + +static void test_hinted_tail_zero(void) +{ + TEST("hinted tail mmap reads zero"); + size_t size = 2ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap base"); + return; + } + p[0] = 0x3c; + p[size - 1] = 0xc3; + + uint8_t *hint = p + size; + uint8_t *q = mmap(hint, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap hint"); + munmap(p, size); + return; + } + if (q[0] != 0 || q[size - 1] != 0 || p[0] != 0x3c || p[size - 1] != 0xc3) { + FAIL("hinted tail leaked stale bytes or clobbered neighbor"); + munmap(q, size); + munmap(p, size); + return; + } + munmap(q, size); + munmap(p, size); + PASS(); +} + +static void test_partial_block_reuse(void) +{ + TEST("partial-block reuse preserves neighbor"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[17] = 0xa5; + p[half + 17] = 0x5a; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED || q != p) { + FAIL("freed half was not reused at hint"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + if (q[17] != 0 || q[half - 1] != 0 || p[half + 17] != 0x5a) { + FAIL("partial zero clobbered neighbor or leaked stale data"); + munmap(q, half); + munmap(p + half, half); + return; + } + munmap(q, half); + munmap(p + half, half); + PASS(); +} + +static void test_fork_clean_reuse(void) +{ + TEST("fork child sees zero on clean-block reuse"); + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap 1"); + return; + } + memset(p, 0xcc, BLOCK_2MIB); + if (munmap(p, BLOCK_2MIB) != 0) { + FAIL("munmap"); + return; + } + uint8_t *q = mmap(p, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap 2"); + return; + } + pid_t pid = fork(); + if (pid == 0) { + if (q[0] != 0 || q[BLOCK_2MIB - 1] != 0) + _exit(1); + q[123] = 0x77; + _exit(q[124] == 0 ? 0 : 2); + } + int st = 0; + if (pid < 0 || waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || + WEXITSTATUS(st) != 0 || q[123] != 0) { + FAIL("fork clean-block state"); + munmap(q, BLOCK_2MIB); + return; + } + munmap(q, BLOCK_2MIB); + PASS(); +} + +static void test_file_overlay_reuse(void) +{ + TEST("file overlay teardown then lazy reuse"); + char path[] = "/tmp/elfuse-dirty-map.XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(path); + if (ftruncate(fd, BLOCK_2MIB) != 0) { + FAIL("ftruncate"); + close(fd); + return; + } + uint8_t first = 0xa7, last = 0x5c; + if (pwrite(fd, &first, 1, 17) != 1 || + pwrite(fd, &last, 1, BLOCK_2MIB - 1) != 1) { + FAIL("pwrite"); + close(fd); + return; + } + + uint8_t *reserve = mmap(NULL, 2 * BLOCK_2MIB, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserve == MAP_FAILED) { + FAIL("reserve"); + close(fd); + return; + } + uintptr_t aligned = + ((uintptr_t) reserve + BLOCK_2MIB - 1) & ~(BLOCK_2MIB - 1); + munmap(reserve, 2 * BLOCK_2MIB); + uint8_t *target = (uint8_t *) aligned; + + uint8_t *file = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, 0); + if (file != target || file[17] != first || file[BLOCK_2MIB - 1] != last) { + FAIL("file mmap"); + if (file != MAP_FAILED) + munmap(file, BLOCK_2MIB); + close(fd); + return; + } + file[BLOCK_2MIB / 2] = 0xe1; + if (munmap(file, BLOCK_2MIB) != 0) { + FAIL("file munmap"); + close(fd); + return; + } + close(fd); + + uint8_t *anon = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anon != target || anon[17] != 0 || anon[BLOCK_2MIB / 2] != 0 || + anon[BLOCK_2MIB - 1] != 0) { + FAIL("stale file bytes after lazy reuse"); + if (anon != MAP_FAILED) + munmap(anon, BLOCK_2MIB); + return; + } + munmap(anon, BLOCK_2MIB); + PASS(); +} + +static void test_read_into_lazy(void) +{ + TEST("read() into untouched mapping"); + size_t size = 6ULL << 20; + uint8_t *buf = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (buf == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + static const char msg[] = "lazy-host-access-payload"; + /* Unaligned target crossing into the mapping's third 2MiB block. */ + size_t off = (4ULL << 20) + 123; + if (write(fds[1], msg, sizeof(msg)) != (ssize_t) sizeof(msg) || + read(fds[0], buf + off, sizeof(msg)) != (ssize_t) sizeof(msg)) { + FAIL("pipe copy through untouched buffer"); + goto out; + } + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload corrupted"); + goto out; + } + /* A guest touch elsewhere in the same 2MiB block must not re-zero the + * host-written payload (deferred-zeroing idempotence). + */ + buf[(4ULL << 20) + 64 * 1024] = 7; + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload clobbered by later fault in same block"); + goto out; + } + /* Untouched parts of the mapping still read zero. */ + for (size_t i = 0; i < 4096; i++) { + if (buf[i] != 0) { + FAIL("nonzero byte in untouched block"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(buf, size); +} + +static void test_write_from_lazy(void) +{ + TEST("write() from untouched mapping"); + size_t size = 2ULL << 20; + uint8_t *src = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (src == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + uint8_t back[512]; + memset(back, 0xff, sizeof(back)); + if (write(fds[1], src + 4096, sizeof(back)) != (ssize_t) sizeof(back) || + read(fds[0], back, sizeof(back)) != (ssize_t) sizeof(back)) { + FAIL("pipe copy from untouched buffer"); + goto out; + } + for (size_t i = 0; i < sizeof(back); i++) { + if (back[i] != 0) { + FAIL("untouched buffer sent nonzero bytes"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(src, size); +} + +static void test_prot_none_roundtrip(void) +{ + TEST("mprotect NONE round trip keeps data"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + memset(p, 0x5c, 8192); + p[size - 1] = 0x77; + if (mprotect(p, size, PROT_NONE) != 0 || + mprotect(p, size, PROT_READ | PROT_WRITE) != 0) { + FAIL("mprotect"); + munmap(p, size); + return; + } + if (p[0] != 0x5c || p[8191] != 0x5c || p[size - 1] != 0x77 || + p[16384] != 0) { + FAIL("data lost or stale bytes after round trip"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static void test_reserve_commit(void) +{ + TEST("PROT_NONE reserve + mprotect commit"); + size_t size = 1ULL << 30; + uint8_t *p = mmap(NULL, size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (p == MAP_FAILED) { + FAIL("reserve"); + return; + } + uint8_t *slab = p + (512ULL << 20); + if (mprotect(slab, 8ULL << 20, PROT_READ | PROT_WRITE) != 0) { + FAIL("commit"); + munmap(p, size); + return; + } + for (size_t off = 0; off < (8ULL << 20); off += 4096) { + if (slab[off] != 0) { + FAIL("committed slab reads nonzero"); + munmap(p, size); + return; + } + } + slab[0] = 1; + slab[(8ULL << 20) - 1] = 2; + if (slab[0] != 1 || slab[(8ULL << 20) - 1] != 2) { + FAIL("committed slab write lost"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static sigjmp_buf segv_jmp; + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static void test_prot_none_faults(void) +{ + TEST("PROT_NONE|NORESERVE still faults"); + size_t size = 16ULL << 20; + volatile uint8_t *p = + mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + struct sigaction sa = {0}, old_sa; + sa.sa_handler = segv_handler; + sigaction(SIGSEGV, &sa, &old_sa); + int faulted = 0; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) p[BLOCK_2MIB + 5]; + } else { + faulted = 1; + } + sigaction(SIGSEGV, &old_sa, NULL); + munmap((void *) p, size); + /* A lazy materializer that ignores prot would silently hand the guest a + * readable zero page here instead of SIGSEGV. + */ + EXPECT_TRUE(faulted, "read from PROT_NONE reservation did not fault"); +} + +static void test_fork_lazy(void) +{ + TEST("fork with partially touched mapping"); + size_t size = 8ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + memset(p, 0x42, 4096); /* touch only block 0 */ + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork"); + munmap(p, size); + return; + } + if (pid == 0) { + /* Child: inherited data intact, untouched block reads zero and is + * privately writable. + */ + if (p[0] != 0x42 || p[4095] != 0x42) + _exit(1); + if (p[4ULL << 20] != 0) + _exit(2); + p[4ULL << 20] = 0x99; + if (p[(4ULL << 20) + 1] != 0) + _exit(3); + _exit(0); + } + int st = 0; + if (waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || WEXITSTATUS(st) != 0) { + FAIL("child saw wrong memory"); + munmap(p, size); + return; + } + /* Parent: child's private write must not leak back. */ + if (p[4ULL << 20] != 0) { + FAIL("child write leaked into parent"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static void test_futex_untouched(void) +{ + TEST("futex on untouched mapping"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + uint32_t *word = (uint32_t *) (p + (2ULL << 20) + 256); + /* WAKE on never-touched memory: no waiters, must not fault. */ + long r = syscall(SYS_futex, word, FUTEX_WAKE, 1, NULL, NULL, 0); + if (r != 0) { + FAIL("FUTEX_WAKE on untouched word"); + munmap(p, size); + return; + } + /* WAIT with expected=1: the word reads as zero, so EAGAIN. */ + r = syscall(SYS_futex, word, FUTEX_WAIT, 1, NULL, NULL, 0); + if (!(r == -1 && errno == EAGAIN)) { + FAIL("FUTEX_WAIT did not read zero from untouched word"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +/* Concurrent first touch: every thread writes its own slot in the same fresh + * 2MiB block, racing the deferred zeroing. A materializer that re-zeros an + * already-populated block loses some slots. + */ +#define MT_THREADS 4 +#define MT_ITERS 64 + +typedef struct { + uint8_t *base; + int idx; + pthread_barrier_t *barrier; +} mt_arg_t; + +static void *mt_touch(void *argp) +{ + mt_arg_t *a = argp; + pthread_barrier_wait(a->barrier); + a->base[a->idx * 64] = (uint8_t) (a->idx + 1); + /* Also touch a private block so several materializations race. */ + a->base[BLOCK_2MIB * (unsigned) (a->idx + 1) + 17] = + (uint8_t) (0x10 + a->idx); + return NULL; +} + +static void test_mt_first_touch(void) +{ + TEST("concurrent first touch"); + for (int iter = 0; iter < MT_ITERS; iter++) { + size_t size = BLOCK_2MIB * (MT_THREADS + 2); + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + pthread_barrier_t barrier; + pthread_barrier_init(&barrier, NULL, MT_THREADS); + pthread_t th[MT_THREADS]; + mt_arg_t args[MT_THREADS]; + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (mt_arg_t) {p, i, &barrier}; + if (pthread_create(&th[i], NULL, mt_touch, &args[i]) != 0) { + FAIL("pthread_create"); + return; + } + } + for (int i = 0; i < MT_THREADS; i++) + pthread_join(th[i], NULL); + pthread_barrier_destroy(&barrier); + for (int i = 0; i < MT_THREADS; i++) { + if (p[i * 64] != (uint8_t) (i + 1) || + p[BLOCK_2MIB * (unsigned) (i + 1) + 17] != + (uint8_t) (0x10 + i)) { + FAIL("write lost to concurrent materialization"); + munmap(p, size); + return; + } + } + munmap(p, size); + } + PASS(); +} + +typedef struct { + uint8_t *base; + size_t half; + int idx; + pthread_barrier_t *barrier; + int *error; +} claim_race_arg_t; + +static void *claim_race_touch(void *argp) +{ + claim_race_arg_t *a = argp; + pthread_barrier_wait(a->barrier); + a->base[64 * (unsigned) a->idx] = (uint8_t) (a->idx + 1); + return NULL; +} + +static void *claim_race_mutate(void *argp) +{ + claim_race_arg_t *a = argp; + uint8_t *neighbor = a->base + a->half; + uint8_t *hole = neighbor + 4096; + pthread_barrier_wait(a->barrier); + for (int i = 0; i < 16; i++) { + if (mprotect(neighbor, a->half, PROT_NONE) != 0 || + mprotect(neighbor, a->half, PROT_READ) != 0 || + munmap(hole, 4096) != 0) { + *a->error = 1; + return NULL; + } + void *r = mmap(hole, 4096, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (r != hole) { + *a->error = 1; + return NULL; + } + } + return NULL; +} + +static void test_claim_mutation_race(void) +{ + TEST("first-touch claim vs adjacent mutations"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[half + 17] = 0x6d; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q != p) { + FAIL("reuse"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + + pthread_barrier_t barrier; + pthread_barrier_init(&barrier, NULL, MT_THREADS + 1); + pthread_t workers[MT_THREADS], mutator; + claim_race_arg_t args[MT_THREADS + 1]; + int error = 0; + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (claim_race_arg_t) {q, half, i, &barrier, &error}; + pthread_create(&workers[i], NULL, claim_race_touch, &args[i]); + } + args[MT_THREADS] = (claim_race_arg_t) {q, half, 0, &barrier, &error}; + pthread_create(&mutator, NULL, claim_race_mutate, &args[MT_THREADS]); + for (int i = 0; i < MT_THREADS; i++) + pthread_join(workers[i], NULL); + pthread_join(mutator, NULL); + pthread_barrier_destroy(&barrier); + + for (int i = 0; i < MT_THREADS; i++) { + if (q[64 * (unsigned) i] != (uint8_t) (i + 1)) + error = 1; + } + if (p[half + 17] != 0x6d) + error = 1; + munmap(q, half); + munmap(p + half, half); + if (error) { + FAIL("claim/mutation race corrupted data"); + return; + } + PASS(); +} + +static void test_adjacent_region_extension(void) +{ + TEST("adjacent fast-mmap region extension"); + enum { N_PAGES = 64 }; + uint8_t *pages[N_PAGES]; + int allocated = 0; + bool ok = true; + + for (int i = 0; i < N_PAGES; i++) { + pages[i] = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (pages[i] == MAP_FAILED) { + ok = false; + break; + } + allocated++; + pages[i][0] = (uint8_t) (i + 1); + } + for (int i = 0; ok && i < allocated; i++) { + if (pages[i][0] != (uint8_t) (i + 1)) + ok = false; + } + for (int i = 0; i < allocated; i++) + munmap(pages[i], 4096); + + if (!ok) { + FAIL("adjacent lazy mappings did not materialize independently"); + return; + } + PASS(); +} + +int main(void) +{ + test_huge_sparse(); + test_zero_reuse(); + test_hinted_tail_zero(); + test_partial_block_reuse(); + test_fork_clean_reuse(); + test_file_overlay_reuse(); + test_read_into_lazy(); + test_write_from_lazy(); + test_prot_none_roundtrip(); + test_reserve_commit(); + test_prot_none_faults(); + test_fork_lazy(); + test_futex_untouched(); + test_mt_first_touch(); + test_claim_mutation_race(); + test_adjacent_region_extension(); + + SUMMARY("test-mmap-lazy"); + return fails ? 1 : 0; +} diff --git a/tests/test-mremap.c b/tests/test-mremap.c index dc97321b..bf8abe33 100644 --- a/tests/test-mremap.c +++ b/tests/test-mremap.c @@ -120,6 +120,106 @@ static void test_grow_maymove(void) munmap(q, 4096 * 4); } +static void test_grow_move_adjacent_fault(void) +{ + TEST("mremap fixed move keeps adjacent page unmapped"); + void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *dest = + mmap(NULL, 4096 * 3, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (dest == MAP_FAILED) { + FAIL("mmap dest"); + munmap(p, 4096); + return; + } + if (munmap(dest + 8192, 4096) < 0) { + FAIL("munmap guard"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + ((char *) p)[0] = 0x5a; + + void *q = mremap(p, 4096, 8192, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (q == MAP_FAILED) { + FAIL("mremap move"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + if (q != dest || ((char *) q)[0] != 0x5a || ((char *) q)[4096] != 0) { + FAIL("mremap data"); + munmap(q, 8192); + return; + } + + pid_t pid = fork(); + if (pid == 0) { + volatile unsigned char value = *((volatile unsigned char *) q + 8192); + (void) value; + _exit(1); + } + int status = 0; + if (pid < 0 || waitpid(pid, &status, 0) != pid) { + FAIL("fork/wait"); + munmap(q, 8192); + return; + } + if ((WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV) || + (WIFEXITED(status) && WEXITSTATUS(status) == 139)) + PASS(); + else + FAIL("adjacent page became readable"); + munmap(q, 8192); +} + +static void test_fixed_preserves_neighbor_l3(void) +{ + TEST("mremap fixed preserves neighboring lazy PTEs"); + char *source = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (source == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *anchor = mmap(NULL, 8192, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anchor == MAP_FAILED) { + FAIL("mmap anchor"); + munmap(source, 4096); + return; + } + char *dest = anchor + 4096; + if (munmap(dest, 4096) < 0) { + FAIL("munmap dest"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + source[0] = 0x11; + anchor[0] = 0x7e; + + void *moved = + mremap(source, 4096, 4096, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (moved == MAP_FAILED) { + FAIL("mremap fixed"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + if (moved != dest || dest[0] != 0x11 || anchor[0] != 0x7e) + FAIL("neighboring lazy page changed"); + else + PASS(); + + munmap(dest, 4096); + munmap(anchor, 4096); +} + /* Test 3: grow without MAYMOVE fails if blocked */ static void test_grow_no_maymove(void) @@ -403,6 +503,8 @@ int main(void) test_shrink(); test_grow_maymove(); + test_grow_move_adjacent_fault(); + test_fixed_preserves_neighbor_l3(); test_grow_no_maymove(); test_fixed(); test_same_size(); diff --git a/tests/test-thread-churn.c b/tests/test-thread-churn.c index b6a92243..9360ada6 100644 --- a/tests/test-thread-churn.c +++ b/tests/test-thread-churn.c @@ -24,9 +24,13 @@ int passes = 0, fails = 0; -#define SEQUENTIAL_ROUNDS 150 +/* Keep this a slot-reuse regression rather than an HVF vCPU lifecycle soak. + * Eighty sequential workers exceed the 64-slot table by more than 16, and two + * 32-worker batches repeat reuse while unrelated workers remain live. + */ +#define SEQUENTIAL_ROUNDS 80 #define BATCH_SIZE 32 -#define BATCH_ROUNDS 6 +#define BATCH_ROUNDS 2 static void *churn_fn(void *arg) { @@ -40,7 +44,7 @@ static void *churn_fn(void *arg) */ static void test_sequential_churn(void) { - TEST("sequential churn (150 threads)"); + TEST("sequential churn (80 threads)"); for (int i = 0; i < SEQUENTIAL_ROUNDS; i++) { int ran = 0; @@ -67,7 +71,7 @@ static void test_sequential_churn(void) */ static void test_batch_churn(void) { - TEST("batch churn (6x32 threads)"); + TEST("batch churn (2x32 threads)"); for (int round = 0; round < BATCH_ROUNDS; round++) { pthread_t threads[BATCH_SIZE]; diff --git a/tests/test-tlbi-encoder-host.c b/tests/test-tlbi-encoder-host.c index 4757a2f0..accc26b7 100644 --- a/tests/test-tlbi-encoder-host.c +++ b/tests/test-tlbi-encoder-host.c @@ -47,12 +47,12 @@ static void check_field(const char *label, uint64_t got, uint64_t expect) /* Decompose the operand per ARM ARM D8.7.6 and compare each field against the * expected value. baseADDR is VA>>12 masked to 37 bits; TG must be 01 (4 KiB); - * SCALE must be 0; TTL must be 0; ASID must be 0. NUM derives from the page - * count via the ceil(pages/2) - 1 SCALE=0 encoding. + * TTL and ASID must be 0. NUM derives from the selected SCALE unit. */ static void verify_operand(uint64_t start_va, uint16_t pages, - uint64_t expect_num) + uint64_t expect_num, + uint64_t expect_scale) { uint64_t op = tlbi_rvae1is_operand(start_va, pages); @@ -76,7 +76,7 @@ static void verify_operand(uint64_t start_va, check_field(label, num, expect_num); snprintf(label, sizeof(label), "SCALE (pages=%u)", (unsigned) pages); - check_field(label, scale, 0); + check_field(label, scale, expect_scale); snprintf(label, sizeof(label), "TG (start=0x%llx)", (unsigned long long) start_va); @@ -100,30 +100,47 @@ int main(void) * pages 63 -> NUM 31 (covers 64) * pages 64 -> NUM 31 (covers 64) */ - verify_operand(0x10000000ULL, 2, 0); - verify_operand(0x10000000ULL, 3, 1); - verify_operand(0x10000000ULL, 16, 7); - verify_operand(0x10000000ULL, 17, 8); - verify_operand(0x10000000ULL, 32, 15); - verify_operand(0x10000000ULL, 63, 31); - verify_operand(0x10000000ULL, 64, 31); + verify_operand(0x10000000ULL, 2, 0, 0); + verify_operand(0x10000000ULL, 3, 1, 0); + verify_operand(0x10000000ULL, 16, 7, 0); + verify_operand(0x10000000ULL, 17, 8, 0); + verify_operand(0x10000000ULL, 32, 15, 0); + verify_operand(0x10000000ULL, 63, 31, 0); + verify_operand(0x10000000ULL, 64, 31, 0); + + /* SCALE=1 covers 64 pages per NUM step; SCALE=2 covers 2048. A lazy + * 2 MiB block is 512 pages and must therefore encode as SCALE=1, NUM=7. + */ + verify_operand(0x10000000ULL, 512, 7, 1); + verify_operand(0x10000000ULL, 2048, 31, 1); + verify_operand(0x10000000ULL, 8192, 3, 2); /* Boundary VAs. 4 KiB-aligned, low-VA, MMAP_BASE (8 GiB), high-VA just * below the 48-bit BaseADDR truncation point. */ - verify_operand(0x00000000ULL, 32, 15); /* zero base */ - verify_operand(0x200000000ULL, 32, 15); /* MMAP_BASE */ - verify_operand(0x800000000000ULL, 32, 15); /* Rosetta image */ - verify_operand(0x0000FFFFF0000000ULL, 32, 15); /* KBUF_USER_VA */ + verify_operand(0x00000000ULL, 32, 15, 0); /* zero base */ + verify_operand(0x200000000ULL, 32, 15, 0); /* MMAP_BASE */ + verify_operand(0x800000000000ULL, 32, 15, 0); /* Rosetta image */ + verify_operand(0x0000FFFFF0000000ULL, 32, 15, 0); /* KBUF_USER_VA */ /* Pathological inputs the clamp must catch: * pages = 0 -> clamped to 2 -> NUM 0 * pages = 1 -> clamped to 2 -> NUM 0 (callers never reach here) * pages = UINT16_MAX -> NUM clamped to 31 (saturating) */ - verify_operand(0x10000000ULL, 0, 0); - verify_operand(0x10000000ULL, 1, 0); - verify_operand(0x10000000ULL, UINT16_MAX, 31); + verify_operand(0x10000000ULL, 0, 0, 0); + verify_operand(0x10000000ULL, 1, 0, 0); + verify_operand(0x10000000ULL, UINT16_MAX, 31, 2); + + /* The accumulator widens a 2 MiB request to the SCALE=1 granule and keeps + * it on the single-shot RVAE path instead of degrading to broadcast. + */ + g_tlbi_range_supported = true; + tlbi_request_clear(); + tlbi_request_range(0x200000000ULL, 0x200200000ULL); + check_field("2MiB accumulator kind", cpu_tlbi_req.kind, TLBI_RANGE_LARGE); + check_field("2MiB accumulator pages", cpu_tlbi_req.pages, 512); + check_field("2MiB accumulator start", cpu_tlbi_req.start, 0x200000000ULL); /* TG bit is the architectural lynchpin -- if the encoder ever drops it the * integration tests on Apple Silicon would still pass. Pin a direct bit-46 From 1c0c386663b17fd25fa7d14aee7c6b83b125f598 Mon Sep 17 00:00:00 2001 From: Max042004 Date: Fri, 17 Jul 2026 14:18:33 +0800 Subject: [PATCH 2/2] Implement the deferred munmap fast path Extend the freestanding C EL1 fast path to retire compatible anonymous mappings, invalidate their translations before return, and defer host metadata cleanup until mmap_lock is next acquired. Return drained arena generations to per-vCPU allocators and refill arenas from recent registration history so mmap and munmap remain effective under reuse and mixed mapping sizes. Preserve the producer window around vCPU kicks and fork, and expose counters for both munmap fallback reasons. Keep dirty backing lazy on every anonymous munmap path; measurements show that eager zeroing and HVF backing replacement cost more than zeroing on reuse. Prove the guest-influenced arena sizing arithmetic, isolate benchmark samples by process, and cover retirement, reuse, refill, fallback, and cross-vCPU publication behavior. --- docs/internals.md | 8 + docs/usage.md | 3 +- mk/shim.mk | 6 +- mk/verify.mk | 30 ++ scripts/check-mutants.py | 35 ++ src/core/bootstrap.c | 9 +- src/core/guest.c | 251 +++++++++- src/core/guest.h | 33 +- src/core/mmap-fastpath.h | 117 ++++- src/core/shim-globals.c | 82 ++- src/core/shim-mmap.c | 652 +++++++++++++++++++++++- src/core/shim-mmap.h | 4 +- src/core/shim.S | 10 +- src/proved/align.h | 2 +- src/proved/mmap-fastpath.h | 236 +++++++++ src/runtime/forkipc.c | 12 + src/syscall/internal.h | 5 + src/syscall/mem.c | 797 +++++++++++++++++++++++++++--- src/syscall/proc.c | 55 +++ src/syscall/signal.c | 12 +- src/syscall/syscall.c | 18 +- tests/bench-mmap-fresh | 80 +++ tests/bench-mmap-isolated | 158 ++++++ tests/bench-mmap.c | 542 ++++++++++++++++---- tests/test-mmap-fastpath-stats.sh | 45 +- tests/test-mmap-fastpath.c | 364 +++++++++++++- tests/test-mmap-lazy.c | 76 +++ 27 files changed, 3380 insertions(+), 262 deletions(-) create mode 100644 src/proved/mmap-fastpath.h create mode 100755 tests/bench-mmap-fresh create mode 100755 tests/bench-mmap-isolated diff --git a/docs/internals.md b/docs/internals.md index 2061f72a..26176cc7 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -457,6 +457,14 @@ goes above the structured area, never below. Post-push masking ### `mmap` Notes +Private anonymous mappings are lazy at 2 MiB materialization granularity. A +host-side hierarchical bitmap records which low-VA 2 MiB blocks contain any +valid TTBR0 PTE, independently of the dirty-block bitmap. `munmap` and recycled +fast-path arenas use this index to visit only materialized blocks, so untouched +multi-GiB reservations have length-independent teardown. When every mapping in +a per-vCPU arena has been released and the index confirms that no PTE remains, +the arena cursor rewinds in place instead of taking a refill HVC. + Aligned file-backed `MAP_SHARED` (fixed or non-fixed) installs a real host `mmap(MAP_FIXED|MAP_SHARED, fd)` overlay onto the guest slab so the kernel page cache keeps the mapping coherent with the file (and diff --git a/docs/usage.md b/docs/usage.md index 5d822f21..b6f75830 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -96,7 +96,8 @@ host `KEY=` imports as `KEY=`. An empty variable name is rejected. Given neither ### mmap call fast path The aarch64 EL1 consumer fast path is enabled by default for -`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)`. +`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)` up to +32 GiB per request. Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted arenas, and full consumption rings fall back to the normal host syscall path. Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host diff --git a/mk/shim.mk b/mk/shim.mk index f90a3b19..7e167170 100644 --- a/mk/shim.mk +++ b/mk/shim.mk @@ -2,12 +2,16 @@ # # shim.S + freestanding shim-mmap.c -> shim.o -> shim.bin -> shim_blob.h +# Disable RCpc codegen so acquire loads remain cumulative LDARs. The retire +# snapshot rule carries cross-vCPU causality through different atomic words; +# LDAPR is intentionally too weak for that protocol. SHIM_CFLAGS := -O2 -Wall -Wextra -Wpedantic -Wshadow \ -Wstrict-prototypes -Wmissing-prototypes -Wformat=2 \ -Wimplicit-fallthrough -Wundef -Wnull-dereference \ -Wno-unused-parameter -ffreestanding -fno-builtin \ -fno-stack-protector -fno-unwind-tables \ - -fno-asynchronous-unwind-tables -mno-outline-atomics + -fno-asynchronous-unwind-tables -mno-outline-atomics \ + -Xclang -target-feature -Xclang -rcpc SHIM_LD ?= ld $(BUILD_DIR)/shim-asm.o: src/core/shim.S | $(BUILD_DIR) diff --git a/mk/verify.mk b/mk/verify.mk index be780133..5470af85 100644 --- a/mk/verify.mk +++ b/mk/verify.mk @@ -311,6 +311,36 @@ VERIFY_ALIGN_SCAN := src/proved/align.h VERIFY_ALIGN_CLAIM := for ANY address, alignment, and search window VERIFY_ALIGN_UNPROVED := the region-array walk around them stays test-covered +# Includes align.h: request_fits calls align_up_ok and window_fits, so this +# proof must discharge their contracts too, not merely assume them, the same +# reason VERIFY_ELF appends VERIFY_UTILS_FCTS. +# +# MIN_GOALS is the complete count from a real run. Keep the floor at that count +# so a future edit that quietly drops a contract or a runtime-error obligation +# cannot turn a smaller proof into a pass. +# +# mmap_fastpath_pow2_clamped was originally the classic bit-smear +# round-up-to-power-of-two; that form's bound and power-of-two properties +# were confirmed unreachable by these provers (a single OR step already +# times out), the same wall align_up_ok's own history describes one level +# down. Rewritten to a doubling loop with an axiomatized power-of-two ghost +# invariant -- same inputs, same outputs, linear arithmetic instead of +# bitwise -- and it discharges completely; see the comment above it. +VERIFY_MMAPFASTPATH_SRC := src/proved/mmap-fastpath.h +VERIFY_MMAPFASTPATH_FCTS := mmap_fastpath_request_fits mmap_fastpath_pow2_clamped \ + mmap_fastpath_window_max mmap_fastpath_arena_size \ + align_up_ok window_fits +VERIFY_MMAPFASTPATH_MIN_GOALS ?= 93 +VERIFY_MMAPFASTPATH_MODEL := typed +VERIFY_MMAPFASTPATH_SCAN := src/proved/mmap-fastpath.h src/proved/align.h +VERIFY_MMAPFASTPATH_CLAIM := for ANY cursor/limit/len and ANY registration history +VERIFY_MMAPFASTPATH_UNPROVED := mmap_fastpath_window_max reporting an actual array member \ + rather than just an upper bound (the loop-invariant \ + preservation step for that claim times out even with a \ + ghost witness index, confirmed unreachable, see the comment \ + above it); the atomic control-block bookkeeping around all \ + four stays test-covered + VERIFY_PATHDEPTH_SRC := src/proved/pathdepth.h VERIFY_PATHDEPTH_FCTS := path_depth_push path_depth_pop VERIFY_PATHDEPTH_MIN_GOALS ?= 24 diff --git a/scripts/check-mutants.py b/scripts/check-mutants.py index 00d7bfd5..8c8b32e0 100755 --- a/scripts/check-mutants.py +++ b/scripts/check-mutants.py @@ -131,6 +131,41 @@ def _load(stem, name): " return start <= limit && length <= limit - start;", " return start <= limit && length <= limit - start + 1;", ), + # ---- verify-mmapfastpath ---------------------------------------------- + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_request_fits", + "accept every non-empty request (an allocation can pass the limit)", + " return window_fits(start, len, limit);", + " return true;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_pow2_clamped", + "clamp an oversized arena request to the lower bound", + " if (value >= MMAP_FAST_ARENA_MAX)\n" + " return MMAP_FAST_ARENA_MAX;", + " if (value >= MMAP_FAST_ARENA_MAX)\n" + " return MMAP_FAST_ARENA_MIN;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_window_max", + "discard a new maximum (the arena can be undersized)", + " if (window[i] > max)\n max = window[i];", + " if (window[i] > max)\n max = 0;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_arena_size", + "return an arena size above the configured maximum", + " return adaptive > covering ? adaptive : covering;", + " return UINT64_MAX;", + ), # ---- verify-cmsg ------------------------------------------------------- ( "cmsg", diff --git a/src/core/bootstrap.c b/src/core/bootstrap.c index 8645a093..58f6d813 100644 --- a/src/core/bootstrap.c +++ b/src/core/bootstrap.c @@ -323,7 +323,14 @@ static bool build_boot_regions(mem_region_t *regions, * to the vDSO page when splitting the block; otherwise vdso_build cannot * write into it through guest_ptr. */ - if (!append_boot_region(regions, nregions, g->shim_base, + /* EL1 fast munmap walks and atomically clears the live TTBR0 tree. Give + * the page-table pool an identity VA visible only to EL1; EL0 remains + * unable to inspect or corrupt descriptors, and every guest syscall still + * rejects the encompassing infrastructure range. + */ + if (!append_boot_region(regions, nregions, g->pt_pool_base, g->pt_pool_end, + MEM_PERM_RW_EL1_ONLY) || + !append_boot_region(regions, nregions, g->shim_base, g->shim_base + shim_bin_len, MEM_PERM_RX) || /* shim_data is EL1-only: the guest must not directly read or write the diff --git a/src/core/guest.c b/src/core/guest.c index 444f583d..f37b4512 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -40,6 +40,7 @@ #include #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "proved/gva.h" #include "core/startup-trace.h" #include "debug/log.h" @@ -160,6 +161,7 @@ static const void *guest_host_memchr(const void *src, /* Forward declaration (defined in the page table section below) */ static int desc_to_perms(uint64_t desc); +static uint64_t *find_l2_entry(guest_t *g, uint64_t va); /* Page table pool allocator. */ @@ -353,6 +355,48 @@ static inline void pte_store_release(uint64_t *entry, uint64_t desc) __atomic_store_n(entry, desc, __ATOMIC_RELEASE); } +/* Low-VA TTBR0 occupancy index. All mutators run under mmap_lock (or during + * single-threaded bootstrap), so plain bitmap operations are sufficient. The + * page-table descriptors themselves remain release-published for lock-free + * guest walkers; this host-only index is never consulted by a vCPU. + */ +static inline bool guest_pte_present_index(uint64_t va, uint64_t *block_out) +{ + if (va >= GUEST_PTE_PRESENT_LIMIT) + return false; + *block_out = va / BLOCK_2MIB; + return true; +} + +static inline void guest_pte_present_set(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] |= 1ULL << (block & 63); + g->pte_present_summary[word >> 6] |= 1ULL << (word & 63); +} + +static inline void guest_pte_present_clear(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] &= ~(1ULL << (block & 63)); + if (g->pte_present_blocks[word] == 0) + g->pte_present_summary[word >> 6] &= ~(1ULL << (word & 63)); +} + +static inline bool guest_pte_present_test(const guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return false; + return (g->pte_present_blocks[block >> 6] & (1ULL << (block & 63))) != 0; +} + /* Public API */ /* FEAT_TLBIRANGE probe -- runs exactly once via pthread_once. ARMv8.4 @@ -1266,9 +1310,11 @@ int guest_map_va_range(guest_t *g, * guest_split_block instead. Skip silently to mirror upstream's * sys_mmap_high_va "reuse existing GPA" behavior. */ + guest_pte_present_set(g, va); continue; } pte_store_release(&l2[l2_idx], make_block_desc(cur_gpa, perms)); + guest_pte_present_set(g, va); if (!bcast) { if (va < changed_lo) changed_lo = va; @@ -2036,6 +2082,8 @@ void guest_reset(guest_t *g) g->mmap_rw_gap_hint = 0; g->mmap_rx_gap_hint = 0; g->ttbr0 = 0; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); tlbi_request_clear(); g->elf_load_min = ELF_DEFAULT_BASE; @@ -3023,6 +3071,9 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) { uint64_t base = g->ipa_base; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + /* Allocate L0 table */ uint64_t l0_gpa = pt_alloc_page(g); if (!l0_gpa) @@ -3119,6 +3170,8 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) * to the primary-buffer GPA where the bytes actually are. */ l2[l2_idx] = make_block_desc(output_ipa, block_perms); + if (lookup_addr >= base) + guest_pte_present_set(g, lookup_addr - base); } } @@ -3255,9 +3308,12 @@ int guest_extend_page_tables(guest_t *g, * an explicit PT_VALID test so the intent survives a future * descriptor-bit renumbering. */ - if (l2[l2_idx] & PT_VALID) + if (l2[l2_idx] & PT_VALID) { + guest_pte_present_set(g, addr); continue; + } pte_store_release(&l2[l2_idx], make_block_desc(ipa, perms)); + guest_pte_present_set(g, addr); if (!bcast) { if (addr < changed_lo) changed_lo = addr; @@ -3273,9 +3329,9 @@ int guest_extend_page_tables(guest_t *g, return 0; } -uint64_t guest_va_next_present_block(const guest_t *g, - uint64_t va, - uint64_t end) +static uint64_t guest_va_next_page_table_block(const guest_t *g, + uint64_t va, + uint64_t end) { if (!g || !g->ttbr0) return end; @@ -3314,10 +3370,81 @@ uint64_t guest_va_next_present_block(const guest_t *g, return end; } +static uint64_t guest_va_next_indexed_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (va >= end || va >= GUEST_PTE_PRESENT_LIMIT) + return end; + if (end > GUEST_PTE_PRESENT_LIMIT) + end = GUEST_PTE_PRESENT_LIMIT; + + uint64_t first_block = va / BLOCK_2MIB; + uint64_t last_block = (end - 1) / BLOCK_2MIB; + uint64_t first_word = first_block >> 6; + uint64_t last_word = last_block >> 6; + uint64_t bits = + g->pte_present_blocks[first_word] & (~0ULL << (first_block & 63)); + if (first_word == last_word && (last_block & 63) != 63) + bits &= (1ULL << ((last_block & 63) + 1)) - 1; + if (bits) + return (first_word * 64 + (uint64_t) __builtin_ctzll(bits)) * + BLOCK_2MIB; + + uint64_t word = first_word + 1; + while (word <= last_word) { + uint64_t summary_word = word >> 6; + uint64_t summary_last = last_word >> 6; + uint64_t summary = + g->pte_present_summary[summary_word] & (~0ULL << (word & 63)); + if (summary_word == summary_last && (last_word & 63) != 63) + summary &= (1ULL << ((last_word & 63) + 1)) - 1; + if (summary) { + uint64_t present_word = + summary_word * 64 + (uint64_t) __builtin_ctzll(summary); + uint64_t present = g->pte_present_blocks[present_word]; + if (present_word == last_word && (last_block & 63) != 63) + present &= (1ULL << ((last_block & 63) + 1)) - 1; + if (present) + return (present_word * 64 + + (uint64_t) __builtin_ctzll(present)) * + BLOCK_2MIB; + } + if (summary_word == summary_last) + break; + word = (summary_word + 1) * 64; + } + return end; +} + +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || va >= end) + return end; + + if (va < GUEST_PTE_PRESENT_LIMIT) { + uint64_t low_end = + end < GUEST_PTE_PRESENT_LIMIT ? end : GUEST_PTE_PRESENT_LIMIT; + uint64_t next = guest_va_next_indexed_block(g, va, low_end); + if (next < low_end || end <= GUEST_PTE_PRESENT_LIMIT) + return next; + va = GUEST_PTE_PRESENT_LIMIT; + } + + /* Non-identity/high-VA mappings live outside the compact low-VA index. + * Preserve the page-table walker for those uncommon ranges. + */ + return guest_va_next_page_table_block(g, va, end); +} + bool guest_va_block_mapped(const guest_t *g, uint64_t va) { if (!g || !g->ttbr0 || (va & (BLOCK_2MIB - 1))) return false; + if (va < GUEST_PTE_PRESENT_LIMIT) + return guest_pte_present_test(g, va); uint64_t base = g->ipa_base; uint64_t *l0 = pt_at(g, g->ttbr0 - base); @@ -3346,6 +3473,71 @@ bool guest_va_block_mapped(const guest_t *g, uint64_t va) return (l2[l2_idx] & PT_VALID) != 0; } +void guest_rebuild_pte_present(guest_t *g) +{ + if (!g) + return; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + if (!g->ttbr0) + return; + + uint64_t limit = g->guest_size; + if (limit > GUEST_PTE_PRESENT_LIMIT) + limit = GUEST_PTE_PRESENT_LIMIT; + for (uint64_t va = 0; va < limit; va += BLOCK_2MIB) { + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + continue; + if ((*l2_entry & 3) == 1) { + guest_pte_present_set(g, va); + continue; + } + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (!l3) + continue; + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + guest_pte_present_set(g, va); + break; + } + } + } +} + +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start) + return; + uint64_t block = ALIGN_2MIB_DOWN(start); + while (block < end) { + bool present = false; + uint64_t *l2_entry = find_l2_entry(g, block); + if (l2_entry && (*l2_entry & PT_VALID)) { + if ((*l2_entry & 3) == 1) { + present = true; + } else { + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (l3) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (pte_load_acquire(&l3[i]) & PT_VALID) { + present = true; + break; + } + } + } + } + } + if (present) + guest_pte_present_set(g, block); + else + guest_pte_present_clear(g, block); + block += BLOCK_2MIB; + } +} + /* L3 page table splitting. */ /* L3 page descriptor: bits[1:0]=11 = valid page at level 3. This is distinct @@ -3502,6 +3694,7 @@ int guest_split_block(guest_t *g, uint64_t block_gpa) int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) { uint64_t base = g->ipa_base; + bool any_changed = false; /* Page-align the range. The ALIGN_UP step on end could wrap to 0 for inputs * within PAGE_SIZE-1 of UINT64_MAX, silently turning the invalidation into @@ -3516,14 +3709,20 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) return 0; for (uint64_t addr = start; addr < end;) { + uint64_t indexed_block = ALIGN_2MIB_DOWN(addr); + if (indexed_block < GUEST_PTE_PRESENT_LIMIT && + !guest_pte_present_test(g, indexed_block)) { + addr = + guest_va_next_present_block(g, indexed_block + BLOCK_2MIB, end); + continue; + } uint64_t *l2_entry = find_l2_entry(g, addr); if (!l2_entry) { - /* No L2 table (L0/L1 slot absent): nothing to invalidate in this - * block. Skip whole absent 1GiB/512GiB slots at once; a lazy - * multi-GiB mmap invalidates its stale range on every allocation - * and would otherwise pay one four-level walk per 2MiB of empty - * address space. + /* No L2 table: nothing to invalidate in this block. The low-VA + * occupancy index jumps directly to the next block containing a + * valid PTE; high VA retains the page-table hierarchy fallback. */ + guest_pte_present_clear(g, indexed_block); addr = guest_va_next_present_block(g, ALIGN_2MIB_UP(addr + 1), end); continue; } @@ -3533,6 +3732,7 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) /* Not mapped at all: skip */ if (!(*l2_entry & 1)) { + guest_pte_present_clear(g, block_start); addr = block_end; continue; } @@ -3546,6 +3746,8 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) * broadcast. */ pte_store_release(l2_entry, 0); + guest_pte_present_clear(g, block_start); + any_changed = true; tlbi_request_range(base + block_start, base + block_end); addr = block_end; continue; @@ -3571,12 +3773,14 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) uint64_t page_end = (end < block_end) ? end : block_end; uint64_t changed_lo = UINT64_MAX, changed_hi = 0; bool bcast = tlbi_request_is_broadcast(); + bool block_changed = false; for (uint64_t pa = page_start; pa < page_end; pa += PAGE_SIZE) { unsigned l3_idx = (unsigned) (((base + pa) % BLOCK_2MIB) / PAGE_SIZE); if (l3[l3_idx] != 0) { pte_store_release(&l3[l3_idx], 0); /* Invalid descriptor */ + block_changed = true; if (!bcast) { if (pa < changed_lo) changed_lo = pa; @@ -3588,10 +3792,27 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + if (block_changed) + any_changed = true; + + bool block_present = false; + if (page_start > block_start || page_end < block_end) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + block_present = true; + break; + } + } + } + if (block_present) + guest_pte_present_set(g, block_start); + else + guest_pte_present_clear(g, block_start); addr = page_end; } - guest_pt_gen_bump(g); + if (any_changed) + guest_pt_gen_bump(g); return 0; } @@ -3667,6 +3888,7 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) pte_store_release(l2_entry, make_block_desc(ipa, perms)); tlbi_request_range(base + block_start, base + block_end); } + guest_pte_present_set(g, block_start); addr = block_end; continue; } @@ -3741,6 +3963,7 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + guest_pte_present_set(g, block_start); addr = page_end; } @@ -3824,6 +4047,7 @@ int guest_install_va_pages(guest_t *g, changed_hi = v + PAGE_SIZE; } } + guest_pte_present_set(g, v); } if (!bcast && changed_hi > changed_lo) @@ -3988,6 +4212,7 @@ retry:; * contract as a newly installed block. */ if (guest_va_pte_valid(g, fault_offset)) { + mmap_fastpath_note_materialized_locked(g, block_start, block_end); g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]++; tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); return 0; @@ -4056,7 +4281,7 @@ retry:; claim_slot = materialize_claim_alloc_locked(g, block_start, block_end); if (claim_slot >= 0) - mmap_lock_release(); + mmap_lock_drop_keep_gate(); for (unsigned page = 0; page < 512;) { if (!(zero_pages[page >> 6] & (1ULL << (page & 63)))) { page++; @@ -4072,7 +4297,7 @@ retry:; 0, (uint64_t) (page - first) * PAGE_SIZE); } if (claim_slot >= 0) - mmap_lock_acquire(g); + mmap_lock_reacquire_with_gate(g); if (!any_valid && materialize_start == block_start && materialize_end == block_end) guest_dirty_clear_zeroed_range(g, block_start, block_end); @@ -4125,6 +4350,8 @@ retry:; : GUEST_MATERIALIZE_CLEAN_SKIP]++; g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES] += materialize_end - materialize_start; + mmap_fastpath_note_materialized_locked(g, materialize_start, + materialize_end); materialize_claim_release_locked(g, claim_slot); return 0; } diff --git a/src/core/guest.h b/src/core/guest.h index 2d99e9be..351c8af4 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -434,6 +434,18 @@ typedef struct { #define GUEST_DIRTY_BLOCKS_MAX ((1ULL << 40) / BLOCK_2MIB) #define GUEST_DIRTY_WORDS (GUEST_DIRTY_BLOCKS_MAX / 64) +/* Host-side occupancy index for low-VA TTBR0 mappings. One bit per 2 MiB + * block records whether that block contains at least one valid L2/L3 PTE; a + * second-level bitmap records which occupancy words are non-zero. This lets + * huge lazy mmap/munmap ranges skip untouched address space without walking + * one page-table slot per GiB. The index is separate from dirty_blocks: a + * read-only mapping can have valid PTEs without dirty backing bytes. + */ +#define GUEST_PTE_PRESENT_BLOCKS_MAX GUEST_DIRTY_BLOCKS_MAX +#define GUEST_PTE_PRESENT_WORDS (GUEST_PTE_PRESENT_BLOCKS_MAX / 64) +#define GUEST_PTE_PRESENT_SUMMARY_WORDS (GUEST_PTE_PRESENT_WORDS / 64) +#define GUEST_PTE_PRESENT_LIMIT (GUEST_PTE_PRESENT_BLOCKS_MAX * BLOCK_2MIB) + enum { GUEST_MATERIALIZE_CLEAN_SKIP = 0, GUEST_MATERIALIZE_DIRTY_MEMSET, @@ -580,6 +592,8 @@ typedef struct { */ _Atomic uint64_t pt_gen; + uint64_t pte_present_blocks[GUEST_PTE_PRESENT_WORDS]; + uint64_t pte_present_summary[GUEST_PTE_PRESENT_SUMMARY_WORDS]; uint64_t dirty_blocks[GUEST_DIRTY_WORDS]; uint64_t materialize_stats[GUEST_MATERIALIZE_STATS_N]; guest_materialize_claim_t materialize_claims[GUEST_MATERIALIZE_CLAIMS]; @@ -1071,6 +1085,17 @@ int guest_install_va_pages(guest_t *g, */ bool guest_va_block_mapped(const guest_t *g, uint64_t va); +/* Rebuild the low-VA PTE occupancy index from TTBR0. Used after fork restores + * page-table pages into a freshly initialized guest_t. + */ +void guest_rebuild_pte_present(guest_t *g); + +/* Reconcile the host-only 2 MiB occupancy index after EL1 has invalidated + * descriptors in [start,end). This observes PTEs only; it never writes a + * descriptor or requests another TLBI. Caller holds mmap_lock. + */ +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end); + /* Returns true when the VA range [va, va+size) overlaps the user-VA kbuf alias * window [KBUF_USER_VA, KBUF_USER_VA+KBUF_SIZE). Callers that install TTBR0 * mappings (the future rosetta_finalize, sys_mmap MAP_FIXED touching this @@ -1116,10 +1141,10 @@ int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len); */ int guest_lazy_faultin_locked(const guest_t *g, uint64_t gva, uint64_t len); -/* Smallest block-aligned va' in [va, end) whose 1GiB L1 slot is present in - * the page tables, or end if none. Lets range walkers skip absent 1GiB / - * 512GiB slots in O(1) instead of probing every 2MiB block. Locking: callers - * MUST hold mmap_lock. +/* Smallest 2MiB-block-aligned va' in [va, end) containing at least one valid + * TTBR0 PTE, or end if none. Low VA uses the host-side hierarchical occupancy + * bitmap; non-identity/high VA falls back to the page-table hierarchy. Locking: + * callers MUST hold mmap_lock. */ uint64_t guest_va_next_present_block(const guest_t *g, uint64_t va, diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h index b31172e1..0e787ad1 100644 --- a/src/core/mmap-fastpath.h +++ b/src/core/mmap-fastpath.h @@ -1,30 +1,54 @@ /* * Per-vCPU EL1 anonymous-mmap consumer rings. * - * The host is the sole producer of arenas and the sole consumer of ring - * entries. EL1 only bump-allocates VA and appends descriptions. Control - * blocks live in the EL1-only shim-data mapping and are selected from SP_EL1's - * per-thread stack slot, so no guest-visible register ABI is consumed. + * The host produces arenas, consumes mmap/munmap publications, and returns + * metadata-committed holes through a reverse SPSC ring. EL1 first-fit consumes + * those private extents before bump-allocating fresh VA. Control blocks live + * in the EL1-only shim-data mapping and are selected from SP_EL1's per-thread + * stack slot, so no guest-visible register ABI is consumed. */ #pragma once #include +#include #include #include #include "core/guest.h" +#include "proved/mmap-fastpath.h" typedef struct thread_entry thread_entry_t; #define SHIM_MMAP_CONTROL_BASE 0x20000u -#define SHIM_MMAP_CONTROL_STRIDE 0x800u -#define SHIM_MMAP_RING_SIZE 16u +#define SHIM_MMAP_CONTROL_STRIDE 0x1000u +#define SHIM_MMAP_RING_SIZE 32u #define SHIM_MMAP_CTRL_ENABLED 0x1u +#define SHIM_MMAP_CTRL_TLBIRANGE 0x2u -#define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) -#define MMAP_FAST_ARENA_MAX (1ULL * 1024 * 1024 * 1024) -#define MMAP_FAST_HISTORY_MULTIPLIER 16u +/* Host page-table writers set this gate before changing an arena descriptor or + * a stage-1 PTE. Each EL1 producer announces itself in its private control + * before rechecking the gate. This is a writer-vs-per-vCPU-reader handshake, + * not an allocator lock: fast munmap producers never write a shared cache + * line or wait for one another. + */ +#define SHIM_MMAP_PT_GATE_OFF 0x1160u + +#define SHIM_MUNMAP_RETIRE_RING_SIZE 32u +#define SHIM_MUNMAP_RETIRE_OFF 0x400u +#define SHIM_MUNMAP_RETIRE_BYTES_SOFT (256ULL * 1024 * 1024) +#define SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK 0x3fu +#define SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT 6u +#define SHIM_MUNMAP_RETIRE_F_CHARGE_MASK 0xffffffc0u + +/* MMAP_FAST_ARENA_MIN/MAX/TARGET_ENTRIES and MMAP_FAST_PUBLICATION_WINDOW are + * defined in proved/mmap-fastpath.h, included above: that header's sizing + * arithmetic is proved against those exact values, so this is the one + * definition rather than a copy a proof cannot see drift from. + * + * MMAP_FAST_PUBLICATION_WINDOW must stay a power of two: the sequence counter + * indexes the window by masking. + */ enum { SHIM_MMAP_COUNTER_SHAPE_MISS = 0, @@ -42,6 +66,25 @@ typedef struct { uint64_t prot; } shim_mmap_entry_t; +typedef struct munmap_retire_entry { + uint64_t addr; + uint64_t length; + uint32_t arena_generation; + uint32_t flags; +} munmap_retire_entry_t; + +typedef struct munmap_retire_ring { + _Atomic uint32_t head; /* host consumer */ + _Atomic uint32_t tail; /* EL1 producer */ + _Atomic uint64_t produced_bytes; /* EL1 producer, monotonic */ + _Atomic uint64_t consumed_bytes; /* host consumer, monotonic */ + _Atomic uint32_t producer_active; + /* Advisory; never forces the producer to exit. */ + _Atomic uint32_t cleanup_requested; + munmap_retire_entry_t entries[SHIM_MUNMAP_RETIRE_RING_SIZE]; +} munmap_retire_ring_t; + + typedef struct { _Atomic uint32_t generation; /* host publish word */ _Atomic uint32_t consumer_generation; /* EL1 generation ack */ @@ -53,14 +96,35 @@ typedef struct { _Atomic uint64_t arena_limit; _Atomic uint64_t cursor; /* EL1 bump cursor */ uint64_t next_arena_size; /* most recently selected generation size */ - uint64_t max_len_seen; /* outgoing-generation request history */ + uint64_t publication_seq; /* host-only: registrations, rotates the window */ shim_mmap_entry_t ring[SHIM_MMAP_RING_SIZE]; _Atomic uint64_t counters[SHIM_MMAP_COUNTERS_N]; uint64_t refill_count; uint64_t recycle_count; uint64_t peak_arena_size; + _Atomic uint32_t materialized_generation; + uint32_t _pad1; + _Atomic uint64_t materialized_start; + _Atomic uint64_t materialized_end; + uint8_t _pad2[SHIM_MUNMAP_RETIRE_OFF - 0x3a0]; + munmap_retire_ring_t retire; + /* Host-only refill policy history; see MMAP_FAST_PUBLICATION_WINDOW. Kept + * past the rings so adding it moves no offset the EL1 side depends on. + */ + uint64_t publication_window[MMAP_FAST_PUBLICATION_WINDOW]; + /* EL1-incremented, host-read: how many munmap() calls found the + * retirement ring near full (see el1_munmap's near-full check) and fell + * back to a synchronous host trap instead of publishing into the ring. + * Placed past every EL1-addressed offset, like publication_window above. + */ + _Atomic uint64_t munmap_retire_near_full; } shim_mmap_control_t; +_Static_assert(offsetof(shim_mmap_control_t, retire) == SHIM_MUNMAP_RETIRE_OFF, + "EL1 retire-ring offset ABI"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "mmap control exceeds per-vCPU stride"); + /* Provision the main vCPU before guest entry. Worker vCPUs provision lazily on * their first eligible mmap so short-lived threads do not allocate an unused * arena. @@ -70,12 +134,45 @@ void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t); /* Drain every per-vCPU SPSC ring. Caller holds mmap_lock. */ void mmap_fastpath_drain_locked(guest_t *g); +/* Opportunistically drain publications and retirements at a natural VM exit. + * Safe before every exit handler; it acquires mmap_lock internally. When a + * fork-family syscall is pending, skip the arena top-up that the syscall will + * immediately revoke. + */ +void mmap_fastpath_drain_vmexit(guest_t *g, bool fork_family_pending); + +/* True when the stopped current vCPU was interrupted in the middle of its EL1 + * producer critical section. A cancellation exit must resume it before host + * drain can close the PT gate. + */ +bool mmap_fastpath_current_producer_active(const guest_t *g); + +/* Mark every arena intersecting a successfully materialized lazy range. Caller + * holds mmap_lock. EL1 may skip its PTE walk only while this marker differs + * from the arena's current generation. + */ +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end); + /* Refill the current vCPU after an eligible mmap slow-path. request_len is * page-rounded; requests above MMAP_FAST_ARENA_MAX leave the arena untouched. * Caller holds mmap_lock. */ void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len); +/* Fulfil an eligible mmap that reached HVC (capacity/ring/generation fallback) + * directly from the current vCPU's refilled arena. Metadata is committed by + * the host immediately, so no mmap publication entry is needed. Caller holds + * mmap_lock. + */ +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); + /* Give an explicit slow-path hint precedence over this stopped vCPU's * unconsumed arena tail. Caller holds mmap_lock. */ diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index 0147e2dd..c38d2349 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -99,35 +99,76 @@ _Static_assert(SHIM_COUNTERS_OFF + SHIM_COUNTERS_N * 8 <= SHIM_IDENTITY_OFF_PGID, "counter array must not overlap the PGID slot"); _Static_assert(SHIM_MMAP_CONTROL_BASE == 0x20000, - "shim.S mmap fast path hard-codes control base 0x20000"); -_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x800, - "shim.S mmap fast path hard-codes control stride 0x800"); -_Static_assert(SHIM_MMAP_RING_SIZE == 16, - "shim.S mmap fast path hard-codes 16 ring entries"); + "EL1 mmap fast path hard-codes control base 0x20000"); +_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x1000, + "EL1 mmap fast path hard-codes control stride 0x1000"); +_Static_assert(SHIM_MMAP_RING_SIZE == 32, + "EL1 mmap fast path hard-codes 32 ring entries"); _Static_assert(offsetof(shim_mmap_control_t, generation) == 0, - "shim.S mmap generation offset drift"); + "EL1 mmap generation offset drift"); _Static_assert(offsetof(shim_mmap_control_t, consumer_generation) == 4, - "shim.S mmap consumer-generation offset drift"); + "EL1 mmap consumer-generation offset drift"); _Static_assert(offsetof(shim_mmap_control_t, flags) == 8, - "shim.S mmap flags offset drift"); + "EL1 mmap flags offset drift"); _Static_assert(offsetof(shim_mmap_control_t, head) == 12, - "shim.S mmap head offset drift"); + "EL1 mmap head offset drift"); _Static_assert(offsetof(shim_mmap_control_t, tail) == 16, - "shim.S mmap tail offset drift"); + "EL1 mmap tail offset drift"); _Static_assert(offsetof(shim_mmap_control_t, arena_base) == 24, - "shim.S mmap arena-base offset drift"); + "EL1 mmap arena-base offset drift"); _Static_assert(offsetof(shim_mmap_control_t, arena_limit) == 32, - "shim.S mmap arena-limit offset drift"); + "EL1 mmap arena-limit offset drift"); _Static_assert(offsetof(shim_mmap_control_t, cursor) == 40, - "shim.S mmap cursor offset drift"); + "EL1 mmap cursor offset drift"); _Static_assert(offsetof(shim_mmap_control_t, next_arena_size) == 48, "mmap next-arena-size offset drift"); -_Static_assert(offsetof(shim_mmap_control_t, max_len_seen) == 56, - "mmap max-len-seen offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, publication_seq) == 56, + "mmap publication-seq offset drift"); _Static_assert(offsetof(shim_mmap_control_t, ring) == 64, - "shim.S mmap ring offset drift"); -_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x1C0, - "shim.S mmap counter offset drift"); + "EL1 mmap ring offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x340, + "EL1 mmap counter offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_generation) == 0x388, + "EL1 mmap materialized-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_start) == 0x390 && + offsetof(shim_mmap_control_t, materialized_end) == 0x398, + "EL1 mmap materialized bounds offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, retire) == 0x400, + "EL1 munmap retire offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, produced_bytes) == 8, + "EL1 munmap produced-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, consumed_bytes) == 16, + "EL1 munmap consumed-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, producer_active) == 24, + "EL1 munmap active offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, cleanup_requested) == 28, + "EL1 munmap cleanup-request offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, entries) == 32, + "EL1 munmap entries offset drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_RING_SIZE == 32 && MAX_THREADS == 64, + "EL1 munmap ring/arena scan constants drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_BYTES_SOFT == 0x10000000ULL, + "EL1 munmap byte advisory threshold drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK == 0x3f && + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT == 6, + "EL1 munmap retire flag encoding drift"); +/* Host-only, so no EL1 offset depends on it; the assert pins it past every + * ring instead, which is what keeps adding it from moving the shared layout. + */ +_Static_assert(offsetof(shim_mmap_control_t, publication_window) > + offsetof(shim_mmap_control_t, retire), + "publication window must not precede the EL1-visible rings"); +_Static_assert((MMAP_FAST_PUBLICATION_WINDOW & + (MMAP_FAST_PUBLICATION_WINDOW - 1)) == 0, + "publication window size must be a power of two"); +_Static_assert((MMAP_FAST_ARENA_MAX >> 12) <= + (SHIM_MUNMAP_RETIRE_F_CHARGE_MASK >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT), + "munmap retire flags cannot encode maximum arena charge"); +_Static_assert(SHIM_MMAP_PT_GATE_OFF >= SHIM_GLOBALS_SIZE && + SHIM_MMAP_PT_GATE_OFF + sizeof(uint32_t) <= + SHIM_MMAP_CONTROL_BASE, + "host PT gate overlaps shim globals or mmap controls"); _Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, "per-vCPU mmap control exceeds its shim-data stride"); _Static_assert(SHIM_MMAP_CONTROL_BASE + @@ -532,6 +573,7 @@ void shim_globals_counters_dump(const guest_t *g) uint64_t mmap_counters[SHIM_MMAP_COUNTERS_N] = {0}; uint64_t refill_count = 0, recycle_count = 0; uint64_t current_max = 0, peak_max = 0; + uint64_t munmap_retire_near_full = 0; const uint8_t *shim_data = (const uint8_t *) g->host_base + g->shim_data_base; for (int slot = 0; slot < MAX_THREADS; slot++) { @@ -548,6 +590,8 @@ void shim_globals_counters_dump(const guest_t *g) current_max = c->next_arena_size; if (c->peak_arena_size > peak_max) peak_max = c->peak_arena_size; + munmap_retire_near_full += atomic_load_explicit( + &c->munmap_retire_near_full, memory_order_relaxed); } for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) fprintf(stderr, " %-20s %llu\n", mmap_counter_names[i], @@ -556,6 +600,8 @@ void shim_globals_counters_dump(const guest_t *g) (unsigned long long) refill_count); fprintf(stderr, " %-20s %llu\n", "MMAP_RECYCLE", (unsigned long long) recycle_count); + fprintf(stderr, " %-20s %llu\n", "MUNMAP_RETIRE_NEAR_FULL", + (unsigned long long) munmap_retire_near_full); fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_CURRENT", (unsigned long long) current_max); fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_PEAK", diff --git a/src/core/shim-mmap.c b/src/core/shim-mmap.c index e9c6a2d7..824e4c67 100644 --- a/src/core/shim-mmap.c +++ b/src/core/shim-mmap.c @@ -1,18 +1,19 @@ /* - * EL1 mmap syscall fast path. + * EL1 mmap-family syscall fast paths. * * Copyright 2026 elfuse contributors * SPDX-License-Identifier: Apache-2.0 * * This file is compiled freestanding and linked into the shim image. It may * not call host code or a C runtime: the only state it consumes is the saved - * EL0 register frame, TPIDR_EL1's shim-data mapping, and the shared mmap - * control protocol. Keep architecture-only operations in the small helper - * below; the allocator policy remains ordinary C. + * EL0 register frame, TPIDR_EL1's shim-data mapping, TTBR0_EL1, and the shared + * mmap control protocol. Keep architecture-only operations in the small + * helpers below; the synchronization and allocator policy remain ordinary C. */ #include "core/shim-mmap.h" +#include #include #include @@ -21,8 +22,10 @@ #define EL1_PAGE_SIZE 0x1000ULL #define EL1_BLOCK_SIZE 0x200000ULL +#define EL1_WINDOW_1G 0x40000000ULL #define EL1_SHIM_DATA_SIZE 0x200000ULL +#define EL1_SYS_MUNMAP 215ULL #define EL1_SYS_MMAP 222ULL #define EL1_PROT_READ 1ULL @@ -33,19 +36,39 @@ #define EL1_MAP_ANONYMOUS 0x20ULL #define EL1_MAP_NORESERVE 0x4000ULL +#define EL1_DESC_ADDR_MASK 0x0000fffffffff000ULL +#define EL1_DESC_TYPE_MASK 0x3ULL +#define EL1_DESC_BLOCK 0x1ULL +#define EL1_DESC_TABLE 0x3ULL +#define EL1_PTE_VALID 0x1ULL + /* The bump path promises 2 MiB-aligned starts at or above this request size, * so the host can back them with L2 blocks. */ #define EL1_ALIGN_THRESHOLD EL1_BLOCK_SIZE +/* Arena descriptors are one per vCPU slot. shim-globals.c pins this against + * MAX_THREADS; the host thread header is not includable freestanding. */ +#define EL1_ARENA_SLOTS 64u + typedef struct { uint8_t *shim_data; shim_mmap_control_t *control; + _Atomic uint32_t *gate; + unsigned slot; } el1_mmap_context_t; +typedef enum { + EL1_GATE_ENTERED, + EL1_GATE_CLOSED_BEFORE_ANNOUNCE, + EL1_GATE_CLOSED_AFTER_ANNOUNCE, +} el1_gate_result_t; + _Static_assert(EL1_SAVED_GPRS * sizeof(uint64_t) == 248, "saved GPR frame layout changed"); _Static_assert(sizeof(shim_mmap_entry_t) == 24, "EL1 mmap publication ABI changed"); +_Static_assert(sizeof(munmap_retire_entry_t) == 24, + "EL1 retire publication ABI changed"); static inline uint64_t el1_read_tpidr(void) { @@ -54,6 +77,75 @@ static inline uint64_t el1_read_tpidr(void) return value; } +static inline uint64_t el1_read_ttbr0(void) +{ + uint64_t value; + __asm__ volatile("mrs %0, ttbr0_el1" : "=r"(value)); + return value; +} + +static inline void el1_dsb_ishst(void) +{ + __asm__ volatile("dsb ishst" ::: "memory"); +} + +static inline void el1_dsb_ish(void) +{ + __asm__ volatile("dsb ish" ::: "memory"); +} + +static inline void el1_isb(void) +{ + __asm__ volatile("isb" ::: "memory"); +} + +static inline void el1_tlbi_vae1is(uint64_t operand) +{ + __asm__ volatile("tlbi vae1is, %0" : : "r"(operand) : "memory"); +} + +static inline void el1_tlbi_rvae1is(uint64_t operand) +{ + __asm__ volatile("tlbi rvae1is, %0" : : "r"(operand) : "memory"); +} + +static inline void el1_tlbi_all(void) +{ + __asm__ volatile("tlbi vmalle1is" ::: "memory"); +} + +static inline uint64_t el1_desc_address(uint64_t descriptor) +{ + return descriptor & EL1_DESC_ADDR_MASK; +} + +static inline uint64_t el1_desc_type(uint64_t descriptor) +{ + return descriptor & EL1_DESC_TYPE_MASK; +} + +/* Guest VA is identity-mapped over the arena, so a descriptor's output address + * is also the VA of the next table. */ +static inline _Atomic uint64_t *el1_table(uint64_t descriptor) +{ + return (_Atomic uint64_t *) (uintptr_t) el1_desc_address(descriptor); +} + +static inline uint64_t el1_block_start(uint64_t address) +{ + return address & ~(EL1_BLOCK_SIZE - 1); +} + +static inline uint64_t el1_next_block(uint64_t address) +{ + return el1_block_start(address) + EL1_BLOCK_SIZE; +} + +static inline uint64_t el1_min(uint64_t left, uint64_t right) +{ + return left < right ? left : right; +} + static inline bool el1_add_overflow(uint64_t left, uint64_t right, uint64_t *result) @@ -71,7 +163,7 @@ static bool el1_page_round(uint64_t length, uint64_t *rounded) } /* SP_EL1 stack tops are shim_data_end - slot*4KiB and controls are - * shim_data + 0x20000 + slot*2KiB, so the saved frame's page locates the + * shim_data + 0x20000 + slot*4KiB, so the saved frame's page locates the * control without consuming another system register. */ static el1_mmap_context_t el1_context(uint64_t *saved_gprs) { @@ -86,6 +178,8 @@ static el1_mmap_context_t el1_context(uint64_t *saved_gprs) .control = (shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + (uintptr_t) slot * SHIM_MMAP_CONTROL_STRIDE), + .gate = (_Atomic uint32_t *) (shim_data + SHIM_MMAP_PT_GATE_OFF), + .slot = slot, }; } @@ -125,15 +219,43 @@ static inline bool el1_attention_pending(const el1_mmap_context_t *context) memory_order_acquire) != 0; } -/* Consume a host-prepared, per-vCPU VA arena for the exact anonymous RW shape - * used by allocators: +/* Join the host-writer handshake. Arena revoke and refill both run with the + * gate closed; announcing this per-vCPU producer prevents the + * host from invalidating a descriptor between the generation check and the + * publication. The second gate read closes the check-vs-announce race: if the + * host closed it in between, withdraw without touching a PTE. */ +static el1_gate_result_t el1_gate_enter(el1_mmap_context_t *context) +{ + if (atomic_load_explicit(context->gate, memory_order_acquire) != 0) + return EL1_GATE_CLOSED_BEFORE_ANNOUNCE; + + atomic_store_explicit(&context->control->retire.producer_active, 1, + memory_order_release); + if (atomic_load_explicit(context->gate, memory_order_acquire) != 0) { + atomic_store_explicit(&context->control->retire.producer_active, 0, + memory_order_release); + return EL1_GATE_CLOSED_AFTER_ANNOUNCE; + } + return EL1_GATE_ENTERED; +} + +static inline void el1_gate_leave(el1_mmap_context_t *context) +{ + atomic_store_explicit(&context->control->retire.producer_active, 0, + memory_order_release); +} + +/* + * mmap: consume a host-prepared, per-vCPU VA arena for the exact anonymous RW + * shape used by allocators: * * mmap(NULL, len, PROT_READ|PROT_WRITE, * MAP_PRIVATE|MAP_ANONYMOUS[|MAP_NORESERVE], fd, off) * - * Each vCPU is the sole producer of its publication ring and cursor. The host - * acquire-drains all publications whenever it takes mmap_lock. + * Each vCPU is the sole producer of its mmap publication ring and cursor. The + * host acquire-drains all publications whenever it takes mmap_lock. */ + static bool el1_mmap(el1_mmap_context_t *context, uint64_t *saved_gprs) { shim_mmap_control_t *control = context->control; @@ -154,6 +276,15 @@ static bool el1_mmap(el1_mmap_context_t *context, uint64_t *saved_gprs) return false; } + switch (el1_gate_enter(context)) { + case EL1_GATE_ENTERED: + break; + case EL1_GATE_CLOSED_BEFORE_ANNOUNCE: + case EL1_GATE_CLOSED_AFTER_ANNOUNCE: + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + uint32_t generation = atomic_load_explicit(&control->generation, memory_order_acquire); if (generation != atomic_load_explicit(&control->consumer_generation, @@ -165,21 +296,33 @@ static bool el1_mmap(el1_mmap_context_t *context, uint64_t *saved_gprs) */ atomic_store_explicit(&control->consumer_generation, generation, memory_order_relaxed); + el1_gate_leave(context); el1_mmap_counter(context, SHIM_MMAP_COUNTER_GENERATION_STALE); return false; } if (!(atomic_load_explicit(&control->flags, memory_order_relaxed) & SHIM_MMAP_CTRL_ENABLED)) { + el1_gate_leave(context); el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); return false; } + /* Reserve publication capacity before mutating the bump cursor. */ + uint32_t head = atomic_load_explicit(&control->head, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&control->tail, memory_order_relaxed); + if ((uint32_t) (tail - head) >= SHIM_MMAP_RING_SIZE) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_RING_FULL); + return false; + } + uint64_t address = atomic_load_explicit(&control->cursor, memory_order_relaxed); uint64_t cursor; if (length >= EL1_ALIGN_THRESHOLD) { if (el1_add_overflow(address, EL1_BLOCK_SIZE - 1, &address)) { + el1_gate_leave(context); el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); return false; } @@ -188,34 +331,503 @@ static bool el1_mmap(el1_mmap_context_t *context, uint64_t *saved_gprs) if (el1_add_overflow(address, length, &cursor) || cursor > atomic_load_explicit(&control->arena_limit, memory_order_relaxed)) { + el1_gate_leave(context); el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); return false; } - uint32_t head = atomic_load_explicit(&control->head, memory_order_acquire); - uint32_t tail = atomic_load_explicit(&control->tail, memory_order_relaxed); - if ((uint32_t) (tail - head) >= SHIM_MMAP_RING_SIZE) { - el1_mmap_counter(context, SHIM_MMAP_COUNTER_RING_FULL); - return false; - } - shim_mmap_entry_t *entry = &control->ring[tail & (SHIM_MMAP_RING_SIZE - 1)]; entry->addr = address; entry->len = length; entry->prot = prot; - /* Publish the bump cursor before the entry. */ + /* publish the bump cursor before the entry */ atomic_store_explicit(&control->cursor, cursor, memory_order_relaxed); atomic_store_explicit(&control->tail, tail + 1, memory_order_release); + el1_gate_leave(context); saved_gprs[0] = address; el1_mmap_counter(context, SHIM_MMAP_COUNTER_HIT); return true; } -bool el1_mmap_fastpath(uint64_t saved_gprs[static EL1_SAVED_GPRS]) +/* + * munmap: invalidate an anonymous arena range synchronously at EL1, then + * release-publish a metadata retirement to this vCPU's SPSC ring. The host + * consumes it at a later natural VM exit. No shared allocator cursor is + * modified here. + * + * The two-pass page-table walk is deliberate. Pass one proves every partial + * 2 MiB block already has an L3 table, so pass two cannot discover a need to + * split after it has cleared an earlier descriptor. Whole L2 blocks and L3 + * leaves are the only descriptors this path writes. + */ + +/* Locate the one arena generation containing the whole range. Scanning 64 + * cache-cold descriptors is only the cross-vCPU case; the owner's control + * normally matches early. generation is re-read after bounds so a descriptor + * publication can never be observed torn. */ +static shim_mmap_control_t *el1_munmap_find_arena( + const el1_mmap_context_t *context, + uint64_t start, + uint64_t end, + uint32_t *generation_out, + uint32_t *flags_out, + unsigned *slot_out) +{ + for (unsigned slot = 0; slot < EL1_ARENA_SLOTS; slot++) { + shim_mmap_control_t *candidate = + (shim_mmap_control_t *) (context->shim_data + + SHIM_MMAP_CONTROL_BASE + + (uintptr_t) slot * + SHIM_MMAP_CONTROL_STRIDE); + uint32_t flags = + atomic_load_explicit(&candidate->flags, memory_order_acquire); + if (!(flags & SHIM_MMAP_CTRL_ENABLED)) + continue; + uint32_t generation = + atomic_load_explicit(&candidate->generation, memory_order_acquire); + if (start < + atomic_load_explicit(&candidate->arena_base, memory_order_relaxed)) + continue; + if (end > + atomic_load_explicit(&candidate->cursor, memory_order_acquire)) + continue; + if (generation != + atomic_load_explicit(&candidate->generation, memory_order_acquire)) + continue; + *generation_out = generation; + *flags_out = flags; + *slot_out = slot; + return candidate; + } + return NULL; +} + +static inline uint64_t el1_rvae_operand(uint64_t address, + uint64_t units, + unsigned scale) +{ + uint64_t base = (address >> 12) & ((1ULL << 37) - 1); + return base | ((units - 1) << 39) | ((uint64_t) scale << 44) | (1ULL << 46); +} + +/* Encode SCALE=0..3 RVAE1IS batches directly from the changed envelope. Each + * instruction covers up to 32 scale units: 64 pages at SCALE=0, 2048 at + * SCALE=1, 65536 at SCALE=2, and 2097152 (8 GiB) at SCALE=3. Widening to the + * scale-unit boundary only invalidates neighboring TLB entries; it cannot + * change mappings. Ranges larger than one instruction are emitted as adjacent + * batches while retaining a single completion DSB. */ +static void el1_tlbi_pages(uint64_t start, uint64_t end, bool range_supported) +{ + uint64_t pages = (end - start) / EL1_PAGE_SIZE; + if (pages <= 8) { + uint64_t operand = (start >> 12) & ((1ULL << 44) - 1); + for (uint64_t page = 0; page < pages; page++) + el1_tlbi_vae1is(operand + page); + return; + } + + if (!range_supported) { + el1_tlbi_all(); + return; + } + + unsigned scale; + unsigned unit_shift; + if (pages <= 64) { + scale = 0; + unit_shift = 13; /* 2 pages = 8 KiB */ + } else if (pages <= 0x800) { + scale = 1; + unit_shift = 18; /* 64 pages = 256 KiB */ + } else if (pages <= 0x10000) { + scale = 2; + unit_shift = 23; /* 2048 pages = 8 MiB */ + } else { + scale = 3; + unit_shift = 28; /* 65536 pages = 256 MiB */ + } + + uint64_t unit = 1ULL << unit_shift; + uint64_t mask = unit - 1; + uint64_t first = start & ~mask; + uint64_t last; + if (el1_add_overflow(end, mask, &last)) { + el1_tlbi_all(); + return; + } + last &= ~mask; + while (first < last) { + uint64_t units = (last - first) >> unit_shift; + if (units > 32) + units = 32; + el1_tlbi_rvae1is(el1_rvae_operand(first, units, scale)); + first += units << unit_shift; + } +} + +typedef enum { + EL1_VALIDATE_GENERIC, /* walk per block, mixed L2/L3 shapes */ + EL1_VALIDATE_L2_BLOCKS, /* proven L2-block-only under one L0 slot */ + EL1_VALIDATE_BAD, /* corrupt or unsupported descriptor shape */ +} el1_validate_t; + +/* Common large-anonymous case: the materialized envelope is 2 MiB aligned and + * consists only of L2 block descriptors (or holes retired by a sibling). Walk + * L0 once, then L1 once per 1 GiB window and scan the L2 entries linearly. The + * generic path remains the fallback for any L3 table or unusual upper shape. */ +static el1_validate_t el1_validate_l2_blocks(uint64_t start, + uint64_t end, + uint64_t *pages) +{ + if (((start | end) & (EL1_BLOCK_SIZE - 1)) != 0) + return EL1_VALIDATE_GENERIC; + if ((((end - 1) ^ start) >> 39) != 0) /* must stay in one L0 slot */ + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = + atomic_load_explicit(&l0[(start >> 39) & 0x1ff], memory_order_acquire); + uint64_t type = el1_desc_type(l0_desc); + if (type == 0) { + *pages = 0; + return EL1_VALIDATE_L2_BLOCKS; + } + if (type != EL1_DESC_TABLE) + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t counted = 0; + uint64_t address = start; + while (address < end) { + uint64_t window_end = + el1_min(((address >> 30) + 1) << 30, end); /* next 1 GiB */ + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + uint64_t l1_type = el1_desc_type(l1_desc); + if (l1_type == 0) { + address = window_end; + continue; + } + if (l1_type != EL1_DESC_TABLE) + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l2 = el1_table(l1_desc); + unsigned index = (unsigned) ((address >> 21) & 0x1ff); + while (address < window_end) { + uint64_t l2_desc = + atomic_load_explicit(&l2[index], memory_order_relaxed); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type != 0) { + if (l2_type != EL1_DESC_BLOCK) /* L2 block, never L3 table */ + return EL1_VALIDATE_GENERIC; + counted += EL1_BLOCK_SIZE / EL1_PAGE_SIZE; + } + index++; + address += EL1_BLOCK_SIZE; + } + } + *pages = counted; + return EL1_VALIDATE_L2_BLOCKS; +} + +/* Validation pass. Pending bytes are charged by materialized backing rather + * than virtual length, so *pages counts only descriptors that actually map. */ +static el1_validate_t el1_validate_generic(uint64_t request_start, + uint64_t request_end, + uint64_t start, + uint64_t end, + uint64_t *pages) +{ + uint64_t counted = 0; + uint64_t address = start; + while (address < end) { + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = atomic_load_explicit(&l0[(address >> 39) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l0_desc) == 0) { + address = el1_next_block(address); + continue; + } + if (el1_desc_type(l0_desc) != EL1_DESC_TABLE) + return EL1_VALIDATE_BAD; + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l1_desc) == 0) { + address = el1_next_block(address); + continue; + } + if (el1_desc_type(l1_desc) != EL1_DESC_TABLE) + return EL1_VALIDATE_BAD; + + _Atomic uint64_t *l2 = el1_table(l1_desc); + uint64_t l2_desc = atomic_load_explicit(&l2[(address >> 21) & 0x1ff], + memory_order_acquire); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type == 0) { + address = el1_next_block(address); + continue; + } + if (l2_type == EL1_DESC_TABLE) { + _Atomic uint64_t *l3 = el1_table(l2_desc); + uint64_t block_end = el1_min(el1_next_block(address), end); + while (address < block_end) { + if (atomic_load_explicit(&l3[(address >> 12) & 0x1ff], + memory_order_acquire) & + EL1_PTE_VALID) + counted++; + address += EL1_PAGE_SIZE; + } + continue; + } + if (l2_type != EL1_DESC_BLOCK) + return EL1_VALIDATE_BAD; + + /* An L2 block can only be cleared as a whole. */ + uint64_t block_start = el1_block_start(address); + if (request_start > block_start || + request_end < block_start + EL1_BLOCK_SIZE) + return EL1_VALIDATE_BAD; + counted += EL1_BLOCK_SIZE / EL1_PAGE_SIZE; + address = block_start + EL1_BLOCK_SIZE; + } + *pages = counted; + return EL1_VALIDATE_GENERIC; +} + +/* Mutation pass. Host writers are gated; sibling fast munmaps can only change + * the same descriptors to zero, so repeated invalidation is safe. */ +static bool el1_clear_generic(uint64_t start, uint64_t end) { - if (saved_gprs[8] != EL1_SYS_MMAP) + uint64_t address = start; + while (address < end) { + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = atomic_load_explicit(&l0[(address >> 39) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l0_desc) != EL1_DESC_TABLE) { + address = el1_next_block(address); + continue; + } + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l1_desc) != EL1_DESC_TABLE) { + address = el1_next_block(address); + continue; + } + + _Atomic uint64_t *l2 = el1_table(l1_desc); + _Atomic uint64_t *l2_entry = &l2[(address >> 21) & 0x1ff]; + uint64_t l2_desc = atomic_load_explicit(l2_entry, memory_order_acquire); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type == 0) { + address = el1_next_block(address); + continue; + } + if (l2_type == EL1_DESC_BLOCK) { + atomic_store_explicit(l2_entry, 0, memory_order_release); + address = el1_next_block(address); + continue; + } + if (l2_type != EL1_DESC_TABLE) + return false; /* validation invariant */ + + _Atomic uint64_t *l3 = el1_table(l2_desc); + uint64_t block_end = el1_min(el1_next_block(address), end); + while (address < block_end) { + atomic_store_explicit(&l3[(address >> 12) & 0x1ff], 0, + memory_order_release); + address += EL1_PAGE_SIZE; + } + } + return true; +} + +/* Validation proved one L0 table and L2-block-only leaves. Re-walk L1 once per + * 1 GiB window, then use ordinary aligned 64-bit stores. Those stores are + * atomic; the DSB ISHST below supplies the required publication order, so a + * release store on every descriptor is unnecessary. */ +static void el1_clear_l2_blocks(uint64_t start, uint64_t end) +{ + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = + atomic_load_explicit(&l0[(start >> 39) & 0x1ff], memory_order_relaxed); + _Atomic uint64_t *l1 = el1_table(l0_desc); + + uint64_t address = start; + while (address < end) { + uint64_t window_end = el1_min(((address >> 30) + 1) << 30, end); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_relaxed); + if (el1_desc_type(l1_desc) == 0) { + address = window_end; + continue; + } + + _Atomic uint64_t *l2 = el1_table(l1_desc); + unsigned index = (unsigned) ((address >> 21) & 0x1ff); + while (address < window_end) { + /* Validation plus the closed host gate proves this slot is a block + * or already zero. A sibling producer can only move it toward + * zero, so an unconditional aligned store is safe and avoids a + * second load/branch. + */ + atomic_store_explicit(&l2[index], 0, memory_order_relaxed); + index++; + address += EL1_BLOCK_SIZE; + } + } +} + +static bool el1_munmap(el1_mmap_context_t *context, uint64_t *saved_gprs) +{ + shim_mmap_control_t *control = context->control; + munmap_retire_ring_t *retire = &control->retire; + + if (el1_attention_pending(context)) { + el1_attention_counter(context); + return false; + } + + uint64_t start = saved_gprs[0]; + uint64_t length = saved_gprs[1]; + uint64_t end; + if (length == 0 || (start & (EL1_PAGE_SIZE - 1)) || + (length & (EL1_PAGE_SIZE - 1)) || el1_add_overflow(start, length, &end)) return false; + + uint32_t head = atomic_load_explicit(&retire->head, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&retire->tail, memory_order_relaxed); + if ((uint32_t) (tail - head) >= SHIM_MUNMAP_RETIRE_RING_SIZE - 1) { + atomic_fetch_add_explicit(&control->munmap_retire_near_full, 1, + memory_order_relaxed); + return false; /* near-full => batched HVC drain */ + } + uint64_t produced = + atomic_load_explicit(&retire->produced_bytes, memory_order_relaxed); + + switch (el1_gate_enter(context)) { + case EL1_GATE_ENTERED: + break; + case EL1_GATE_CLOSED_BEFORE_ANNOUNCE: + case EL1_GATE_CLOSED_AFTER_ANNOUNCE: + return false; + } + + uint32_t generation; + uint32_t arena_flags; + unsigned arena_slot; + shim_mmap_control_t *owner = el1_munmap_find_arena( + context, start, end, &generation, &arena_flags, &arena_slot); + if (!owner) { + el1_gate_leave(context); + return false; + } + + /* Reserve and fill, but do not advance tail until PTE+TLBI completion. */ + munmap_retire_entry_t *entry = + &retire->entries[tail & (SHIM_MUNMAP_RETIRE_RING_SIZE - 1)]; + entry->addr = start; + entry->length = length; + entry->arena_generation = generation; + entry->flags = arena_slot; + + /* Refill invalidates every stale descriptor before publishing a new + * generation. Until the host records a lazy materialization in that + * generation, the entire arena is known PTE-empty and no walk/TLBI is + * needed. The host PT gate makes the generation marker stable here. + */ + uint64_t pages = 0; + uint64_t walk_start = 0; + uint64_t walk_end = 0; + bool walked = false; + if (atomic_load_explicit(&owner->materialized_generation, + memory_order_acquire) == generation) { + uint64_t materialized_start = atomic_load_explicit( + &owner->materialized_start, memory_order_relaxed); + uint64_t materialized_end = atomic_load_explicit( + &owner->materialized_end, memory_order_relaxed); + walk_start = start > materialized_start ? start : materialized_start; + walk_end = el1_min(end, materialized_end); + walked = walk_start < walk_end; + } + + bool l2_blocks = false; + if (walked) { + el1_validate_t verdict = + el1_validate_l2_blocks(walk_start, walk_end, &pages); + if (verdict == EL1_VALIDATE_L2_BLOCKS) { + l2_blocks = true; + } else { + verdict = + el1_validate_generic(start, end, walk_start, walk_end, &pages); + if (verdict == EL1_VALIDATE_BAD) { + el1_gate_leave(context); + return false; + } + } + + /* Encode the charged 4 KiB page count in the retire flags so the host's + * single-writer consumed sequence advances by exactly the same amount. + */ + entry->flags = (uint32_t) (pages << SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT) | + (uint32_t) arena_slot; + } + + if (walked && pages != 0) { + uint64_t consumed = + atomic_load_explicit(&retire->consumed_bytes, memory_order_acquire); + uint64_t charge = pages << 12; + uint64_t pending; + if (el1_add_overflow(produced - consumed, charge, &pending) || + el1_add_overflow(produced, charge, &produced)) { + el1_gate_leave(context); + return false; + } + /* Crossing the soft threshold records cleanup_requested but never + * forces this producer to HVC; another natural exit consumes the ring. + */ + if (pending > SHIM_MUNMAP_RETIRE_BYTES_SOFT) + atomic_store_explicit(&retire->cleanup_requested, 1, + memory_order_release); + + if (l2_blocks) { + el1_clear_l2_blocks(walk_start, walk_end); + } else if (!el1_clear_generic(walk_start, walk_end)) { + el1_gate_leave(context); + return false; + } + + /* Descriptor stores -> broadcast invalidation -> completed visibility, + * before the retire tail release makes metadata cleanup eligible. + */ + el1_dsb_ishst(); + el1_tlbi_pages(walk_start, walk_end, + (arena_flags & SHIM_MMAP_CTRL_TLBIRANGE) != 0); + el1_dsb_ish(); + el1_isb(); + } + + atomic_store_explicit(&retire->produced_bytes, produced, + memory_order_relaxed); + atomic_store_explicit(&retire->tail, tail + 1, memory_order_release); + el1_gate_leave(context); + saved_gprs[0] = 0; + return true; +} + +bool el1_mmap_fastpath(uint64_t saved_gprs[static EL1_SAVED_GPRS]) +{ el1_mmap_context_t context = el1_context(saved_gprs); - return el1_mmap(&context, saved_gprs); + switch (saved_gprs[8]) { + case EL1_SYS_MMAP: + return el1_mmap(&context, saved_gprs); + case EL1_SYS_MUNMAP: + return el1_munmap(&context, saved_gprs); + default: + return false; + } } diff --git a/src/core/shim-mmap.h b/src/core/shim-mmap.h index cb8f6aa2..ebe27a2d 100644 --- a/src/core/shim-mmap.h +++ b/src/core/shim-mmap.h @@ -1,11 +1,11 @@ /* - * Freestanding EL1 mmap fast path. + * Freestanding EL1 mmap-family fast paths. * * Copyright 2026 elfuse contributors * SPDX-License-Identifier: Apache-2.0 * * The assembly exception shim owns the saved-register frame and calls this - * module only for mmap. A true return means X0 in the saved + * module only for mmap-family syscalls. A true return means X0 in the saved * frame contains the completed syscall result; false asks the shim to forward * the original frame to HVC #5 unchanged. */ diff --git a/src/core/shim.S b/src/core/shim.S index 4994f4a6..f1868154 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -26,7 +26,7 @@ * 4 = single-shot TLBI RVAE1IS (FEAT_TLBIRANGE); * X9 carries the pre-encoded RVAE1IS operand * (baddr | NUM<<39 | SCALE<<44 | TTL<<37 | - * ASID<<48; SCALE=0, TTL=0, ASID=0 today) + * ASID<<48; TTL=0 and ASID=0) * X11 carries the I-cache hint for X8 in {1, 3, 4}: * 1 = IC IALLU after the TLBI sequence (new * executable content visible to EL0), 0 = skip the @@ -447,16 +447,18 @@ svc_handler: b.eq getsid_fast cmp x10, #278 /* SYS_getrandom? */ b.eq getrandom_fast + cmp x10, #215 /* SYS_munmap? */ + b.eq mmap_family_fast cmp x10, #222 /* SYS_mmap? */ - b.eq mmap_anon_fast + b.eq mmap_family_fast b handle_svc_0 -/* The mmap fast path is a freestanding C consumer. The saved frame remains +/* mmap and munmap share a freestanding C consumer. The saved frame remains * authoritative: false leaves it untouched for HVC #5; true places the * completed result in saved X0. The normal restore tail reloads every other * guest register, so the C ABI's caller-clobbered set is private to EL1. */ -mmap_anon_fast: +mmap_family_fast: mov x0, sp bl _el1_mmap_fastpath cbz w0, handle_svc_0 diff --git a/src/proved/align.h b/src/proved/align.h index 42608aea..b1eda0d6 100644 --- a/src/proved/align.h +++ b/src/proved/align.h @@ -60,7 +60,7 @@ ensures binary: \result == 0 || \result == 1; ensures rejects_only_on_wrap: \result != 0 <==> (x % align == 0 - || (x / align + 1) * align <= UINT64_MAX); + || x / align < UINT64_MAX / align); ensures aligned: \result != 0 ==> (\exists integer k; *out == k * align); ensures never_below: \result != 0 ==> *out >= x; diff --git a/src/proved/mmap-fastpath.h b/src/proved/mmap-fastpath.h new file mode 100644 index 00000000..f25361cc --- /dev/null +++ b/src/proved/mmap-fastpath.h @@ -0,0 +1,236 @@ +/* + * EL1 mmap fast-path arena sizing and capacity arithmetic: the parts a proof + * can reach + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Each per-vCPU arena is a bump allocator: EL1 hands out [cursor, cursor+len) + * and advances cursor until a request no longer fits, at which point the host + * refills or grows the arena from the guest's own mmap history. Every value + * here is guest-influenced (request_len is the guest's mmap length, window_max + * is derived from a history of guest-chosen lengths), so a slip either wedges + * the allocator (undersizes an arena forever) or, in + * mmap_fastpath_request_fits, accepts a request that runs past arena_limit. + * + * Split out of mem.c because mem.c cannot be given to Frama-C: it includes + * sys/mman.h and the HVF headers, which the analyzer's libc does not model. + * These functions need nothing but stdint.h plus proved/align.h, so + * make verify-mmapfastpath proves this header directly. + * + * MMAP_FAST_ARENA_MIN/MAX/TARGET_ENTRIES and MMAP_FAST_PUBLICATION_WINDOW live + * here rather than in core/mmap-fastpath.h so the sizing policy has one + * definition; core/mmap-fastpath.h includes this header for them rather than + * repeating the numbers where a proof cannot see whether they still match. + */ + +#pragma once + +#include +#include + +#include "proved/align.h" + +#define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) +#define MMAP_FAST_ARENA_MAX (32ULL * 1024 * 1024 * 1024) +#define MMAP_FAST_ARENA_TARGET_ENTRIES 32u + +/* Registrations the arena sizer looks back over. Sixteen is long enough to + * outlive a burst of one allocation size (so a phase change does not resize + * the arena on its first mapping) and short enough that an outlier washes out + * within a few dozen mappings. + */ +#define MMAP_FAST_PUBLICATION_WINDOW 16u + +/* Whether a request of len bytes still fits the bump cursor before limit. + * + * Requests at least block_align wide are served block-aligned, matching the + * fast path's block-granular VA carve (guest_invalidate_ptes clears a whole + * block, so a sub-block bump allocation inside a block already carrying a + * live mapping would straddle stale and fresh PTEs). Requests below that + * threshold draw straight from cursor. + * + * len == 0 is its own case rather than falling out of the general fits test: + * mmap_fastpath_topup_locked calls this with len == 0 to ask "is there any + * room left", and an arena with cursor == limit has none, so that case needs + * cursor < limit rather than window_fits's cursor <= limit (which would call + * a completely full arena still fitting a zero-length request). + * + * Only the len >= block_align case is stated one-directionally + * (\result ==> room existed at the unaligned cursor, not the converse): its + * true value comes from align_up_ok, which bounds the aligned start + * (never_below, rounds_up_once) rather than pinning it to a closed-form + * expression restated here. That is enough to rule out the dangerous + * direction -- this function reporting a fit that ends up past limit -- and + * chasing the converse would mean re-deriving align_up_ok's own arithmetic in + * this contract instead of resting on its proof. + */ +/*@ + requires block_align > 0; + assigns \nothing; + ensures zero_len: + len == 0 ==> (\result <==> cursor < limit); + ensures small_request: + (len != 0 && len < block_align) ==> + (\result <==> (cursor <= limit && len <= limit - cursor)); + ensures large_request_sound: + (len != 0 && len >= block_align && \result) ==> + (cursor <= limit && len <= limit - cursor); + */ +static inline bool mmap_fastpath_request_fits(uint64_t cursor, + uint64_t limit, + uint64_t len, + uint64_t block_align) +{ + if (!len) + return cursor < limit; + uint64_t start = cursor; + if (len >= block_align && !align_up_ok(cursor, block_align, &start)) + return false; + return window_fits(start, len, limit); +} + +/* Ratio the doubling loop below relies on: MMAP_FAST_ARENA_MAX is exactly + * MMAP_FAST_ARENA_MIN doubled 9 times. Guards the loop's trip-count bound + * against either constant changing without updating the other. + */ +_Static_assert( + MMAP_FAST_ARENA_MAX == MMAP_FAST_ARENA_MIN * 512, + "mmap_fastpath_pow2_clamped's loop bound assumes MAX == MIN << 9"); + +/*@ + axiomatic ArenaPow2 { + logic integer arena_pow2(integer k); + axiom arena_pow2_zero: arena_pow2(0) == 1; + axiom arena_pow2_succ: + \forall integer k; k >= 0 ==> arena_pow2(k + 1) == 2 * arena_pow2(k); + } +*/ + +/* Smallest power of two in [MMAP_FAST_ARENA_MIN, MMAP_FAST_ARENA_MAX] that is + * at least value, or the nearer bound when value falls outside that range. + * + * This used to be the classic bit-smear round-up-to-power-of-two (value--; + * value |= value >> 1; ...; return value + 1;). Every claim reaching through + * that form -- including a bound as simple as \result >= MMAP_FAST_ARENA_MIN + * -- turned out to be a bitvector-shaped goal (OR only sets bits, so + * value | value >> k >= value for any k, but that is a property of the bit + * pattern, not of linear arithmetic), and neither alt-ergo nor z3 discharges + * even a single OR step of it here; proved/align.h's own history is the same + * shape one level down. Same semantics, different algorithm: this doubles + * from MMAP_FAST_ARENA_MIN instead, which is linear arithmetic all the way + * through and discharges completely, including the power-of-two and + * covering properties the smear form could not get past its two boundary + * branches. It costs at most 9 iterations (see the _Static_assert above) + * where the smear was 6 fixed shifts; both run once per arena refill, not + * per mmap, so the difference does not reach the hot path this feeds. + */ +/*@ + assigns \nothing; + ensures bounded_low: \result >= MMAP_FAST_ARENA_MIN; + ensures bounded_high: \result <= MMAP_FAST_ARENA_MAX; + ensures covers: + value <= MMAP_FAST_ARENA_MAX ==> \result >= value; + ensures clamp_low: + value <= MMAP_FAST_ARENA_MIN ==> \result == MMAP_FAST_ARENA_MIN; + ensures clamp_high: + value >= MMAP_FAST_ARENA_MAX ==> \result == MMAP_FAST_ARENA_MAX; + */ +static inline uint64_t mmap_fastpath_pow2_clamped(uint64_t value) +{ + if (value <= MMAP_FAST_ARENA_MIN) + return MMAP_FAST_ARENA_MIN; + if (value >= MMAP_FAST_ARENA_MAX) + return MMAP_FAST_ARENA_MAX; + uint64_t p = MMAP_FAST_ARENA_MIN; + /*@ ghost int k = 0; */ + /*@ + loop invariant p_eq: p == MMAP_FAST_ARENA_MIN * arena_pow2(k); + loop invariant k_range: 0 <= k <= 9; + loop assigns p, k; + loop variant 9 - k; + */ + while (p < value) { + p += p; + /*@ ghost k = k + 1; */ + } + return p; +} + +/* Largest registration still inside a fixed-size history window. window must + * point at MMAP_FAST_PUBLICATION_WINDOW live entries; the caller passes + * shim_mmap_control_t's publication_window array, which this header does not + * name so it never has to see that struct's _Atomic fields. + * + * Only the upper bound is stated, not that \result is itself one of the + * entries: a ghost witness index tracking argmax alongside max proves its own + * base case and the two ensures it would support (both go through by taking + * the invariant as given), but the loop-invariant preservation step itself + * times out under both alt-ergo and z3 -- at 90s, ten times FRAMAC_TIMEOUT, + * not just 30s -- while the plain upper-bound invariant below proves in + * milliseconds. The upper bound is also the only property arena_size's + * sizing math actually needs. + */ +/*@ + requires \valid_read(window + (0 .. MMAP_FAST_PUBLICATION_WINDOW - 1)); + assigns \nothing; + ensures upper_bound: + \forall integer i; 0 <= i < MMAP_FAST_PUBLICATION_WINDOW ==> + window[i] <= \result; + */ +static inline uint64_t mmap_fastpath_window_max( + const uint64_t window[MMAP_FAST_PUBLICATION_WINDOW]) +{ + uint64_t max = 0; + /*@ + loop invariant bound: 0 <= i <= MMAP_FAST_PUBLICATION_WINDOW; + loop invariant prefix_le_max: + \forall integer j; 0 <= j < i ==> window[j] <= max; + loop assigns i, max; + loop variant MMAP_FAST_PUBLICATION_WINDOW - i; + */ + for (unsigned i = 0; i < MMAP_FAST_PUBLICATION_WINDOW; i++) + if (window[i] > max) + max = window[i]; + return max; +} + +/* Target arena size covering both the recent registration history + * (window_max) and the request that is about to be served (request_len), + * clamped to [MMAP_FAST_ARENA_MIN, MMAP_FAST_ARENA_MAX]. + * + * The bound ensures below rest on mmap_fastpath_pow2_clamped's own + * bounded_low/bounded_high: adaptive and covering are each either + * MMAP_FAST_ARENA_MIN (the guard's false branch) or a pow2_clamped result, + * both now fully covered by that function's contract. -wp-rte separately + * proves this function's own arithmetic sound regardless: the two + * multiplication guards (window_max > MAX / target_entries, + * request_len > MAX / 2) mean neither window_max * target_entries nor + * request_len * 2 can overflow, and there is no division-by-zero. + */ +/*@ + assigns \nothing; + ensures result_ge_min: \result >= MMAP_FAST_ARENA_MIN; + ensures result_le_max: \result <= MMAP_FAST_ARENA_MAX; + */ +static inline uint64_t mmap_fastpath_arena_size(uint64_t window_max, + uint64_t request_len) +{ + uint64_t adaptive = MMAP_FAST_ARENA_MIN; + if (window_max) { + const uint64_t target_entries = MMAP_FAST_ARENA_TARGET_ENTRIES; + uint64_t target = window_max > MMAP_FAST_ARENA_MAX / target_entries + ? MMAP_FAST_ARENA_MAX + : window_max * target_entries; + adaptive = mmap_fastpath_pow2_clamped(target); + } + + uint64_t covering = MMAP_FAST_ARENA_MIN; + if (request_len) { + uint64_t target = request_len > MMAP_FAST_ARENA_MAX / 2 + ? MMAP_FAST_ARENA_MAX + : request_len * 2; + covering = mmap_fastpath_pow2_clamped(target); + } + return adaptive > covering ? adaptive : covering; +} diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index f91f3514..12c65fc9 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -271,6 +271,7 @@ int fork_child_main(int ipc_fd, guest_destroy(&g); return 1; } + guest_rebuild_pte_present(&g); if (fork_ipc_recv_fd_table(ipc_fd, &g) < 0) { log_error("fork-child: failed to receive fd table"); @@ -1509,6 +1510,17 @@ int64_t sys_clone(hv_vcpu_t vcpu, if ((flags & ~(uint64_t) 0xff) & LINUX_CLONE3_NS_FLAGS) return -LINUX_EINVAL; + /* Once an anonymous arena allocation becomes a live thread stack, munmap + * must pass through thread_collect_and_defer_stack_ranges(). Revoke arena + * generations before publishing the stack to the thread table so no EL1 + * fast munmap can bypass that lifetime rule. + */ + if (child_stack != 0) { + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); + } + /* CLONE_THREAD: create a new thread in the same VM (not a new process) */ if (flags & LINUX_CLONE_THREAD) { return sys_clone_thread(vcpu, g, flags, child_stack, stack_map_start, diff --git a/src/syscall/internal.h b/src/syscall/internal.h index cc5b6f3a..df980e39 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -52,6 +52,11 @@ extern pthread_mutex_t fd_lock; /* Lock order: 3, FD table */ void mmap_lock_acquire(guest_t *g); void mmap_lock_release(void); void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond); +/* Temporarily drop mmap_lock while retaining the host PT-gate reference, then + * reacquire without taking a second reference. Used only by lazy zeroing. + */ +void mmap_lock_drop_keep_gate(void); +void mmap_lock_reacquire_with_gate(guest_t *g); /* FD table (defined in syscall/fdtable.c). */ extern fd_entry_t fd_table[FD_TABLE_SIZE]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 98f91ff0..87a462ef 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "debug/log.h" #include "debug/syscall-hist.h" @@ -27,6 +28,7 @@ #include "core/mmap-fastpath.h" #include "proved/align.h" +#include "proved/mmap-fastpath.h" #include "runtime/thread.h" #include "syscall/linux-wire.h" @@ -49,6 +51,13 @@ static uint64_t find_free_gap_inner(const guest_t *g, uint64_t min_addr, uint64_t max_addr, uint64_t align); +static bool mmap_fastpath_rewind_control_if_clean_locked( + guest_t *g, + shim_mmap_control_t *c); +static void mmap_fastpath_refill_thread_locked(guest_t *g, + thread_entry_t *t, + uint64_t request_len, + bool speculative); static void mmap_fastpath_read_env(void) { @@ -75,6 +84,76 @@ static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); } + +static _Atomic uint32_t *mmap_fastpath_pt_gate(const guest_t *g) +{ + if (!g || !g->host_base) + return NULL; + return (_Atomic uint32_t *) ((uint8_t *) g->host_base + g->shim_data_base + + SHIM_MMAP_PT_GATE_OFF); +} + +/* mmap_lock serializes host writers. The gate extends that exclusion to EL1 + * fast munmap without making the per-vCPU producers contend with each other: + * after publishing gate=closed, wait for each producer's private active word. + */ +static void mmap_fastpath_host_gate_close(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate) + return; + uint32_t previous = + atomic_fetch_add_explicit(gate, 1, memory_order_acq_rel); + if (previous != 0) + return; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + while (atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0) + sched_yield(); + } +} + +static void mmap_fastpath_host_gate_open(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (gate) { + uint32_t count = atomic_load_explicit(gate, memory_order_relaxed); + while (count != 0 && !atomic_compare_exchange_weak_explicit( + gate, &count, count - 1, memory_order_release, + memory_order_relaxed)) { + } + /* exec resets the entire shim-data page while holding mmap_lock, + * including this implementation-only counter. Seeing zero here is + * therefore an already-open gate, not an underflow. + */ + } +} + +static _Thread_local guest_t *mmap_lock_guest; + +/* Record a length the guest actually obtained from this arena. Only real + * registrations enter the window: a request that missed and was served by the + * generic path never became arena traffic, and sizing the next arena for it + * would grow arenas for a workload that does not use them. + */ +static void mmap_fastpath_note_registration(shim_mmap_control_t *c, + uint64_t len) +{ + if (!len) + return; + c->publication_window[c->publication_seq & + (MMAP_FAST_PUBLICATION_WINDOW - 1)] = len; + c->publication_seq++; +} + +static void mmap_fastpath_window_reset(shim_mmap_control_t *c) +{ + for (unsigned i = 0; i < MMAP_FAST_PUBLICATION_WINDOW; i++) + c->publication_window[i] = 0; + c->publication_seq = 0; +} + static void mmap_fastpath_disable_control(shim_mmap_control_t *c) { uint32_t generation = @@ -85,12 +164,18 @@ static void mmap_fastpath_disable_control(shim_mmap_control_t *c) atomic_store_explicit(&c->arena_base, 0, memory_order_relaxed); atomic_store_explicit(&c->arena_limit, 0, memory_order_relaxed); atomic_store_explicit(&c->cursor, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); c->next_arena_size = MMAP_FAST_ARENA_MIN; - c->max_len_seen = 0; + /* Teardown (exec, fast-path disable) ends the workload the history + * described, so the next one starts from the minimum arena. + */ + mmap_fastpath_window_reset(c); atomic_store_explicit(&c->generation, generation, memory_order_release); } -void mmap_fastpath_drain_locked(guest_t *g) +static void mmap_fastpath_drain_publications_locked(guest_t *g) { if (!g || !g->host_base) return; @@ -141,90 +226,416 @@ void mmap_fastpath_drain_locked(guest_t *g) "draining vCPU slot %d", slot); } - if (len > c->max_len_seen) - c->max_len_seen = len; + mmap_fastpath_note_registration(c, len); head++; } atomic_store_explicit(&c->head, head, memory_order_release); } } -void mmap_lock_acquire(guest_t *g) +static void munmap_retire_commit_locked(guest_t *g, + const munmap_retire_entry_t *e, + uint64_t backing_start, + uint64_t backing_end) +{ + uint64_t start = e->addr; + uint64_t end = start + e->length; + + /* Publication drain ran first, so every mapping causally preceding this + * retirement is now represented in regions[]. A non-anonymous overlay in + * an arena indicates a missing revocation and must fail closed: EL1 has + * already invalidated the PTEs, so silently retaining such metadata would + * permit a later fault path to recreate them. + */ + for (int i = guest_region_first_end_above(g, start); i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->end <= start) + continue; + if (!(r->flags & LINUX_MAP_ANONYMOUS) || + (r->flags & LINUX_MAP_SHARED) || r->backing_fd >= 0 || + r->overlay_active) { + log_fatal( + "munmap retire: non-fast mapping in arena " + "[0x%llx-0x%llx)", + (unsigned long long) start, (unsigned long long) end); + } + } + + guest_materialize_wait_range_locked(g, start, end); + + /* The PTE occupancy evidence was consumed by EL1, so use the conservative + * dirty bitmap to avoid touching huge never-materialized reservations. + * Retain the dirty bits and let a future lazy materialization zero only + * the backing that is actually reused. In particular, do not charge an + * unrelated VM exit (often the next mapping's first fault) with an eager + * memset of the retired range. guest_materialize_lazy_one() zeros dirty + * backing before publishing any new descriptor, so a future reader can + * never observe stale bytes. + * + * An earlier version of this function eagerly replaced the backing of + * large dirty runs (unmap + F_PUNCHHOLE + fresh mmap + remap) once the + * run's page-accurate materialized byte count crossed a size threshold, + * on the theory that handing the zero-fill to the host's demand-zero path + * beats a future software memset. Round-trip measurement (munmap + * immediately followed by a full re-touch of the same range, not just the + * munmap call in isolation) showed the opposite at every size tried from + * 16 MiB to 512 MiB: the replace path's own unmap/remap cost, paid + * synchronously while holding mmap_lock, exceeded the deferred memset it + * was meant to avoid, and grew faster than linearly with size. Retaining + * dirty bits unconditionally is cheaper in every case measured, so this + * function no longer special-cases large runs. + */ + + guest_region_remove(g, start, end); + if (backing_end > backing_start) + guest_retire_ptes_committed(g, backing_start, backing_end); + if (start < g->mmap_rw_gap_hint) + g->mmap_rw_gap_hint = start; + if (start < g->mmap_rx_gap_hint) + g->mmap_rx_gap_hint = start; +} + +void mmap_fastpath_drain_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + /* Acquire-snapshot every retirement tail before consuming any mmap + * publication. This is the cross-vCPU causal ordering required for + * "A mmap; publish pointer; B munmap": the acquire observes B's retire, + * then publication drain establishes A's semantic region before removal. + */ + uint32_t retire_tails[MAX_THREADS]; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + retire_tails[slot] = + atomic_load_explicit(&c->retire.tail, memory_order_acquire); + } + + mmap_fastpath_drain_publications_locked(g); + + bool retired_any = false; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = + atomic_load_explicit(&producer->retire.head, memory_order_relaxed); + uint32_t tail = retire_tails[slot]; + if ((uint32_t) (tail - head) > SHIM_MUNMAP_RETIRE_RING_SIZE) { + log_fatal( + "munmap retire: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + } + + while (head != tail) { + const munmap_retire_entry_t *e = + &producer->retire + .entries[head & (SHIM_MUNMAP_RETIRE_RING_SIZE - 1)]; + uint32_t arena_slot = + e->flags & SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK; + uint64_t charged_pages = + (e->flags & SHIM_MUNMAP_RETIRE_F_CHARGE_MASK) >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT; + uint64_t charged_bytes = charged_pages * GUEST_PAGE_SIZE; + if (arena_slot >= MAX_THREADS || !e->length || + (e->addr & (GUEST_PAGE_SIZE - 1)) || + (e->length & (GUEST_PAGE_SIZE - 1)) || + e->addr > UINT64_MAX - e->length || charged_bytes > e->length) { + log_fatal("munmap retire: invalid entry in vCPU slot %d", slot); + } + + shim_mmap_control_t *arena = + mmap_fastpath_control(g, (int) arena_slot); + uint32_t generation = + atomic_load_explicit(&arena->generation, memory_order_acquire); + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + uint64_t end = e->addr + e->length; + if (generation != e->arena_generation || e->addr < base || + end > cursor) { + log_fatal( + "munmap retire: stale arena generation/range " + "(producer=%d arena=%u gen=%u/%u)", + slot, arena_slot, e->arena_generation, generation); + } + + uint64_t backing_start = 0, backing_end = 0; + if (charged_bytes != 0) { + backing_start = atomic_load_explicit(&arena->materialized_start, + memory_order_relaxed); + backing_end = atomic_load_explicit(&arena->materialized_end, + memory_order_relaxed); + if (backing_start < e->addr) + backing_start = e->addr; + if (backing_end > end) + backing_end = end; + if (backing_end <= backing_start) + log_fatal( + "munmap retire: charged entry has no materialized " + "bounds (producer=%d arena=%u range=0x%llx..0x%llx " + "marker=0x%llx..0x%llx charged=0x%llx)", + slot, arena_slot, (unsigned long long) e->addr, + (unsigned long long) end, + (unsigned long long) atomic_load_explicit( + &arena->materialized_start, memory_order_relaxed), + (unsigned long long) atomic_load_explicit( + &arena->materialized_end, memory_order_relaxed), + (unsigned long long) charged_bytes); + } + + munmap_retire_commit_locked(g, e, backing_start, backing_end); + uint64_t consumed = atomic_load_explicit( + &producer->retire.consumed_bytes, memory_order_relaxed); + atomic_store_explicit(&producer->retire.consumed_bytes, + consumed + charged_bytes, + memory_order_release); + head++; + retired_any = true; + } + atomic_store_explicit(&producer->retire.head, head, + memory_order_release); + /* The PT gate is closed while draining, so no producer can race this + * acknowledgement. Ring fullness remains the only hard per-vCPU + * backpressure; byte pressure is deliberately advisory. + */ + atomic_store_explicit(&producer->retire.cleanup_requested, 0, + memory_order_release); + } + + /* A stopped owner whose whole arena retired can collapse all published + * sub-extents back into its bump cursor. Like envelope reset, this must + * wait for the complete snapshot: overlapping retire records from sibling + * producers may otherwise observe a prematurely reset arena. */ + if (current_thread && current_thread->sp_el1_slot >= 0) + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); + + /* EL1 may publish several charged retirements before this drain. Their + * PTEs are all already invalid, so clearing an arena's materialized + * envelope after the first commit would make later entries in the same + * snapshot lose their backing bounds. Restore the PTE-empty proof only + * after every snapshotted retirement has consumed the old envelope. */ + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *arena = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&arena->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + if (base < cursor && + guest_va_next_present_block(g, base, cursor) >= cursor) { + atomic_store_explicit(&arena->materialized_start, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_end, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_generation, 0, + memory_order_release); + } + } + + /* EL1 PTE stores cannot update guest_t's host-only cache generation. One + * bump per batch invalidates every host GVA translation cache after all + * retirement entries have committed. + */ + if (retired_any) + guest_pt_gen_bump(g); +} + +/* Top up the calling thread's arena before it runs dry. + * + * Refills otherwise happen only after the EL1 side has already missed: the + * fast path exhausts its arena, bails to HVC, and the host refills on the way + * through the slow path -- so the mapping that discovers the exhaustion always + * pays for it. Any host path that takes mmap_lock is already positioned to + * refill for free, having paid for the lock and the gate close. + * + * The water mark is the largest recent registration rather than a fixed + * fraction: what makes an arena useless is being unable to hold the mappings + * this workload actually makes, so an arena is spent exactly when its tail no + * longer covers one of them. That also bounds what the refill abandons -- + * relocating strands the unused tail until a later gap scan recovers it, and + * this way the strand is one typical mapping rather than a slice of an arena + * whose size nothing ties to the workload. A control with no history yet + * never triggers, which keeps startup from refilling on its first lock. + * + * Cheap enough for the lock path: an unavailable fast path or a thread with no + * slot costs one predictable branch, and a healthy arena costs three relaxed + * loads plus the window scan. Only the caller's own slot is considered; + * scanning all of them here would put a MAX_THREADS loop on every acquisition. + */ +static void mmap_fastpath_topup_locked(guest_t *g) +{ + if (!mmap_fastpath_available(g) || !current_thread || + current_thread->sp_el1_slot < 0) + return; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor > limit) + return; + + uint64_t low_water = mmap_fastpath_window_max(c->publication_window); + if (!low_water || limit - cursor > low_water) + return; + + mmap_fastpath_refill_thread_locked(g, current_thread, 0, true); +} + +static void mmap_lock_acquire_common(guest_t *g) { pthread_mutex_lock(&mmap_lock); + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; mmap_fastpath_drain_locked(g); } +void mmap_lock_acquire(guest_t *g) +{ + mmap_lock_acquire_common(g); + mmap_fastpath_topup_locked(g); +} + +static void mmap_lock_acquire_for_fork(guest_t *g) +{ + /* Fork is about to revoke every arena. Keep the drain-before-region-read + * invariant without allocating a replacement arena that cannot survive + * this critical section. + */ + mmap_lock_acquire_common(g); +} + void mmap_lock_release(void) { + mmap_fastpath_host_gate_open(mmap_lock_guest); + mmap_lock_guest = NULL; pthread_mutex_unlock(&mmap_lock); } void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond) { + mmap_fastpath_host_gate_open(g); + mmap_lock_guest = NULL; pthread_cond_wait(cond, &mmap_lock); /* pthread_cond_wait reacquires mmap_lock directly, so preserve the * drain-before-region-read invariant of mmap_lock_acquire(). */ + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; mmap_fastpath_drain_locked(g); } -static bool mmap_fastpath_request_fits(uint64_t cursor, - uint64_t limit, - uint64_t len) +void mmap_lock_drop_keep_gate(void) { - if (!len) - return cursor < limit; - uint64_t start = cursor; - if (len >= BLOCK_2MIB) { - if (start > UINT64_MAX - (BLOCK_2MIB - 1)) - return false; - start = ALIGN_UP(start, BLOCK_2MIB); - } - return start <= limit && len <= limit - start; + /* Dirty lazy-materialization drops mmap_lock around a potentially large + * memset. Retain this thread's gate reference so EL1 cannot retire the + * invalid PTE window and let the materializer recreate it afterwards. + * Another host thread may temporarily acquire mmap_lock; the refcounted + * gate remains closed until this owner finishes the materialization. + */ + mmap_lock_guest = NULL; + pthread_mutex_unlock(&mmap_lock); } -static uint64_t mmap_fastpath_pow2_clamped(uint64_t value) +void mmap_lock_reacquire_with_gate(guest_t *g) { - if (value <= MMAP_FAST_ARENA_MIN) - return MMAP_FAST_ARENA_MIN; - if (value >= MMAP_FAST_ARENA_MAX) - return MMAP_FAST_ARENA_MAX; - value--; - value |= value >> 1; - value |= value >> 2; - value |= value >> 4; - value |= value >> 8; - value |= value >> 16; - value |= value >> 32; - return value + 1; + pthread_mutex_lock(&mmap_lock); + mmap_lock_guest = g; + /* EL1 mmap publication does not need the PT gate and may have progressed + * during the memset, so refresh semantic metadata before resuming. + */ + mmap_fastpath_drain_locked(g); } -static uint64_t mmap_fastpath_arena_size(uint64_t max_len_seen, - uint64_t request_len) +void mmap_fastpath_drain_vmexit(guest_t *g, bool fork_family_pending) { - uint64_t adaptive = MMAP_FAST_ARENA_MIN; - if (max_len_seen) { - uint64_t target = - max_len_seen > MMAP_FAST_ARENA_MAX / MMAP_FAST_HISTORY_MULTIPLIER - ? MMAP_FAST_ARENA_MAX - : max_len_seen * MMAP_FAST_HISTORY_MULTIPLIER; - adaptive = mmap_fastpath_pow2_clamped(target); - } - - uint64_t covering = MMAP_FAST_ARENA_MIN; - if (request_len) { - uint64_t target = request_len > MMAP_FAST_ARENA_MAX / 2 - ? MMAP_FAST_ARENA_MAX - : request_len * 2; - covering = mmap_fastpath_pow2_clamped(target); - } - return adaptive > covering ? adaptive : covering; + if (fork_family_pending) + mmap_lock_acquire_for_fork(g); + else + mmap_lock_acquire(g); + mmap_lock_release(); } +bool mmap_fastpath_current_producer_active(const guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return false; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + return c && atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0; +} + +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start >= limit || end <= base) + continue; + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed); + uint64_t lo = start > base ? start : base; + uint64_t hi = end < limit ? end : limit; + uint32_t materialized = atomic_load_explicit( + &c->materialized_generation, memory_order_relaxed); + if (materialized == generation) { + uint64_t old_lo = atomic_load_explicit(&c->materialized_start, + memory_order_relaxed); + uint64_t old_hi = atomic_load_explicit(&c->materialized_end, + memory_order_relaxed); + if (old_lo < lo) + lo = old_lo; + if (old_hi > hi) + hi = old_hi; + } + atomic_store_explicit(&c->materialized_start, lo, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, hi, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, generation, + memory_order_release); + /* Fast-path arenas are allocated from disjoint VA ranges. Once this + * block has updated its owner, no later vCPU control can overlap it; + * avoid another 63 control-page probes on the single-vCPU hot path. */ + break; + } +} + +/* mmap_fastpath_request_fits, mmap_fastpath_pow2_clamped, + * mmap_fastpath_window_max, and mmap_fastpath_arena_size live in + * proved/mmap-fastpath.h (included above): they are pure arithmetic over + * scalars, unlike everything else in this file, so make verify-mmapfastpath + * proves them directly instead of leaving them reviewed-by-eye. + */ + +/* speculative: refill because the arena is nearly spent, not because a request + * failed to fit. The fits check below would otherwise abandon such a call + * immediately -- a zero-length request fits any arena with a byte to spare. + */ static void mmap_fastpath_refill_thread_locked(guest_t *g, thread_entry_t *t, - uint64_t request_len) + uint64_t request_len, + bool speculative) { if (!t || t->sp_el1_slot < 0) return; @@ -242,16 +653,29 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, if (request_len > MMAP_FAST_ARENA_MAX) return; - if (request_len > c->max_len_seen) - c->max_len_seen = request_len; - uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); uint64_t limit = atomic_load_explicit(&c->arena_limit, memory_order_relaxed); uint32_t flags = atomic_load_explicit(&c->flags, memory_order_relaxed); - if ((flags & SHIM_MMAP_CTRL_ENABLED) && - mmap_fastpath_request_fits(cursor, limit, request_len)) - return; + uint64_t arena_size = mmap_fastpath_arena_size( + mmap_fastpath_window_max(c->publication_window), request_len); + if (!speculative && (flags & SHIM_MMAP_CTRL_ENABLED) && + mmap_fastpath_request_fits(cursor, limit, request_len, BLOCK_2MIB)) { + /* A capacity miss enters HVC, whose drain can rewind a completely + * retired arena before this refill check. Retaining that undersized + * arena merely because one more request fits makes the same miss recur + * every few operations and turns mmap latency into a periodic sawtooth. + * Grow an empty arena to the adaptive target; never relocate one that + * still contains allocations served by the current generation. + */ + bool empty = cursor == arena_base; + bool target_sized = + arena_base <= limit && limit - arena_base >= arena_size; + if (!empty || target_sized) + return; + } /* The owner is parked in HVC. Make the stranded tail immediately recyclable * before the gap scan; mappings already served from the prefix were drained @@ -260,9 +684,6 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, if (flags & SHIM_MMAP_CTRL_ENABLED) atomic_store_explicit(&c->cursor, limit, memory_order_relaxed); - uint64_t arena_size = - mmap_fastpath_arena_size(c->max_len_seen, request_len); - /* Prefer a real hole below the current high-water mark. Active sibling * arena tails are excluded by mmap_fastpath_skip_reserved inside the gap * allocator. Only grow mmap_next when no recyclable hole fits. @@ -313,15 +734,23 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, atomic_store_explicit(&c->arena_base, base, memory_order_relaxed); atomic_store_explicit(&c->arena_limit, new_limit, memory_order_relaxed); atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); c->next_arena_size = arena_size; - c->max_len_seen = 0; + /* The window deliberately survives the generation change: sizing the next + * arena from the traffic that filled the previous one is the whole point, + * and clearing it here would restart the history at every refill. + */ c->refill_count++; if (recycled) c->recycle_count++; if (arena_size > c->peak_arena_size) c->peak_arena_size = arena_size; - atomic_store_explicit(&c->flags, SHIM_MMAP_CTRL_ENABLED, - memory_order_relaxed); + uint32_t control_flags = SHIM_MMAP_CTRL_ENABLED; + if (g_tlbi_range_supported) + control_flags |= SHIM_MMAP_CTRL_TLBIRANGE; + atomic_store_explicit(&c->flags, control_flags, memory_order_relaxed); /* This vCPU is stopped in HVC (or has never run), so host may acknowledge * the freshly published descriptor on its behalf. Revocation deliberately * does not do this, making an in-flight stale generation bail once. @@ -333,7 +762,132 @@ static void mmap_fastpath_refill_thread_locked(guest_t *g, void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len) { - mmap_fastpath_refill_thread_locked(g, current_thread, request_len); + mmap_fastpath_refill_thread_locked(g, current_thread, request_len, false); +} + +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + mmap_fastpath_refill_thread_locked(g, current_thread, request_len, false); + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + return false; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + return false; + uint64_t end = start + request_len; + + if (guest_region_add_ex( + g, start, end, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL, -1) < 0) + return false; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + mmap_fastpath_note_registration(c, request_len); + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + return true; +} + +/* Service mmap publication-ring backpressure without turning it into a + * global PT-gate rendezvous. The calling vCPU is stopped in HVC, so the host + * may advance only that vCPU's existing bump cursor after draining its prior + * publications. Sibling EL1 allocators own disjoint arenas and may continue + * publishing concurrently. + * + * This path deliberately refuses every operation that could require a host + * writer transaction: a pending retirement, a closed gate, arena refill or + * generation change all fall back to mmap_lock_acquire(). The acquire + * snapshots of every retire tail preserve the usual drain-before-metadata + * ordering for causally prior munmaps. */ +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!g || !addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + pthread_mutex_lock(&mmap_lock); + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate || atomic_load_explicit(gate, memory_order_acquire) != 0) + goto miss; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = + atomic_load_explicit(&producer->retire.head, memory_order_relaxed); + uint32_t tail = + atomic_load_explicit(&producer->retire.tail, memory_order_acquire); + if (head != tail) + goto miss; + } + + mmap_fastpath_drain_publications_locked(g); + + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_acquire); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED) || + atomic_load_explicit(&c->consumer_generation, memory_order_relaxed) != + generation) + goto miss; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start == base) { + uint64_t target = mmap_fastpath_arena_size( + mmap_fastpath_window_max(c->publication_window), request_len); + if (base > limit || limit - base < target) + goto miss; + } + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + goto miss; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + goto miss; + uint64_t end = start + request_len; + + if (guest_region_add_ex( + g, start, end, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL, -1) < 0) + goto miss; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + mmap_fastpath_note_registration(c, request_len); + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + pthread_mutex_unlock(&mmap_lock); + return true; + +miss: + pthread_mutex_unlock(&mmap_lock); + return false; } void mmap_fastpath_release_current_hint_locked(guest_t *g, @@ -348,24 +902,77 @@ void mmap_fastpath_release_current_hint_locked(guest_t *g, if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & SHIM_MMAP_CTRL_ENABLED)) return; - uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); uint64_t limit = atomic_load_explicit(&c->arena_limit, memory_order_relaxed); - if (cursor >= limit || addr >= limit || addr + length <= cursor) + if (base >= limit || addr >= limit || addr + length <= base) return; /* The owner is stopped in the HVC that reached sys_mmap, so it cannot race - * this descriptor update. Revoke its whole unconsumed tail: the explicit - * hint must remain semantically free, and the post-syscall refill will - * provision a new non-overlapping arena. + * this descriptor update. Revoke its bump tail and committed free extents: + * the explicit hint must take precedence over every EL1-reserved hole, and + * the post-syscall refill will provision a new non-overlapping arena. */ mmap_fastpath_disable_control(c); } +/* Reuse a fully released arena in place. mmap_lock acquisition has drained + * this vCPU's publication ring, and sys_munmap has removed the last semantic + * region before calling here. The PTE occupancy index is the final guard: an + * arena is rewound only when no live metadata and no valid descriptor remain. + * The owner is stopped in HVC, so resetting its private bump cursor cannot + * race EL1. Keeping base/limit/generation unchanged avoids a host refill on + * the next mmap -- especially important when one 32 GiB request consumes the + * entire maximum-sized arena. + */ +static bool mmap_fastpath_rewind_control_if_clean_locked(guest_t *g, + shim_mmap_control_t *c) +{ + if (!g || !c) + return false; + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + if (base >= limit || cursor <= base) + return false; + + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= limit) + break; + if (r->end > base) + return false; + } + if (guest_va_next_present_block(g, base, limit) < limit) + return false; + + /* The stopped owner can safely discard pending sub-extents because the + * whole arena is becoming one bump-allocatable extent again. */ + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + c->recycle_count++; + return true; +} + +static void mmap_fastpath_rewind_current_if_clean_locked(guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return; + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); +} + void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t) { mmap_lock_acquire(g); - mmap_fastpath_refill_thread_locked(g, t, 0); + mmap_fastpath_refill_thread_locked(g, t, 0, false); mmap_lock_release(); } @@ -3273,6 +3880,14 @@ int64_t sys_mmap(guest_t *g, */ bool is_noreplace = (flags & LINUX_MAP_FIXED_NOREPLACE) != 0; + /* A fixed mapping may replace an address previously handed out by an EL1 + * arena with a file/shared/stack-like mapping. Revoke all descriptors + * before making that semantic transition so a later fast munmap cannot + * classify it from the stale arena bounds. + */ + if (is_fixed) + mmap_fastpath_revoke_all_locked(g, false); + uint64_t result_off; /* Result as offset (0-based) */ if (is_fixed) { /* Addresses above TASK_SIZE (bit 63 set or beyond user VA range) are @@ -4872,13 +5487,25 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) } /* Record which sub-ranges need zeroing BEFORE the PTE invalidation below - * destroys the evidence. Eager regions are zeroed across the whole overlap, - * as before. Lazy (deferred-PTE) regions only need their materialized 2MiB - * blocks zeroed: a block with no L2 mapping was never touched through PTEs, - * host-side fault-in materializes before writing, and the previous unmap of - * that slab range zeroed it -- so its bytes are still zero. This keeps - * munmap cost proportional to memory actually touched instead of to the - * mapping length. + * destroys the evidence. A pure anonymous, private, non-overlaid region + * needs no eager zeroing at all: its dirty bitmap already reflects every + * write that ever touched it -- the same guarantee + * munmap_retire_commit_locked() relies on to skip zeroing entirely on the + * EL1 fast munmap path, gated by the identical + * anonymous/private/no-backing-fd/no-overlay check below -- and + * hvf_remove_file_overlay_quiesced() explicitly marks a restored + * overlay's backing dirty via guest_dirty_mark_range() before this + * function ever sees it. guest_materialize_lazy_one() zeros dirty + * backing before publishing any new descriptor, so deferring here costs + * nothing at reuse time either; it only stops paying up front to zero + * bytes a future mapping may never touch. + * + * Anything else -- an actual file-backed region, or one whose overlay + * cleanup above could not tear down -- keeps the eager policy below: + * eager regions are zeroed across the whole overlap, and lazy + * (deferred-PTE, MAP_NORESERVE) regions only need their materialized + * 2MiB blocks zeroed. This keeps that fallback's cost proportional to + * memory actually touched instead of to the mapping length. */ zero_range_t zr[MUNMAP_ZERO_RANGES_MAX]; int nzr = 0; @@ -4890,6 +5517,10 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) continue; if (r->prot == LINUX_PROT_NONE) continue; + if ((r->flags & LINUX_MAP_ANONYMOUS) && + !(r->flags & LINUX_MAP_SHARED) && r->backing_fd < 0 && + !r->overlay_active) + continue; uint64_t zstart = (r->start > unmap_off) ? r->start : unmap_off; uint64_t zend = (r->end < end) ? r->end : end; if (!r->noreserve) { @@ -4907,8 +5538,9 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) } for (uint64_t b = zstart & ~(BLOCK_2MIB - 1); b < zend;) { if (!guest_va_block_mapped(g, b)) { - /* Skip absent 1GiB/512GiB slots wholesale; a huge untouched - * reservation would otherwise pay one walk per 2MiB. + /* Jump through the PTE occupancy index to the next materialized + * block. A huge untouched reservation therefore does no work + * proportional to its virtual length. */ b = guest_va_next_present_block(g, b + BLOCK_2MIB, zend); continue; @@ -5051,6 +5683,7 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) thread_finish_deferred_stack_ranges(txns, nranges); } } + mmap_fastpath_rewind_current_if_clean_locked(g); return 0; } @@ -5078,6 +5711,12 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (addr > UINT64_MAX - length) return -LINUX_EINVAL; + /* Permission and VMA-shape changes are slow-path boundaries. Retire any + * already-published unmaps, then invalidate arena generations before the + * metadata/PTE edit so EL1 cannot act on the old anonymous classification. + */ + mmap_fastpath_revoke_all_locked(g, false); + if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { uint64_t mprot_end = addr + length; @@ -5562,7 +6201,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, if (!txn) return -LINUX_ENOMEM; - mmap_lock_acquire(g); + mmap_lock_acquire_for_fork(g); /* fork callers have quiesced siblings. Drain their last publications, * revoke every descriptor, and trim never-consumed arena tails before the * legacy [MMAP_BASE,mmap_next) snapshot range is computed. diff --git a/src/syscall/proc.c b/src/syscall/proc.c index 950d115a..9e38378b 100644 --- a/src/syscall/proc.c +++ b/src/syscall/proc.c @@ -37,6 +37,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/futex.h" @@ -80,6 +81,23 @@ static _Atomic bool rosetta_enabled = true; */ static _Atomic bool rosetta_active = false; +static bool vcpu_exit_is_fork_family_syscall(hv_vcpu_t vcpu, + const hv_vcpu_exit_t *vexit) +{ + if (vexit->reason != HV_EXIT_REASON_EXCEPTION) + return false; + + uint64_t syndrome = vexit->exception.syndrome; + uint32_t ec = (uint32_t) ((syndrome >> 26) & 0x3f); + uint16_t imm = (uint16_t) (syndrome & 0xffff); + if (ec != 0x16 || imm != 5) + return false; + + uint64_t nr = 0; + hv_vcpu_get_reg(vcpu, HV_REG_X8, &nr); + return nr == SYS_clone || nr == SYS_clone3; +} + /* Process table for tracking direct and adopted fork children. Start small so * lifecycle tests exercise growth deterministically; expand under pid_lock as * the fork family grows. No pointer into this array survives unlocking. @@ -3670,6 +3688,43 @@ int vcpu_run_loop_with_hooks(hv_vcpu_t vcpu, drain_external_guest_signal(); + /* A kick (hv_vcpus_exit, reason CANCELED) carries no guest-side work + * of its own, so it can land anywhere -- including inside the EL1 fast + * path's producer window, with retire.producer_active published for + * this vCPU's own slot. Only the guest clears that word, and every + * host path that takes mmap_lock waits for it in + * mmap_fastpath_host_gate_close(); signal delivery is one, faulting + * the signal frame in through guest_lazy_faultin(). Waiting there + * would block this thread on a vCPU that cannot run until this very + * thread re-enters it: a self-deadlock, since the kicked vCPU is the + * one being kicked out. Resume instead. The window is a bounded, + * non-blocking instruction sequence, so it retires at once, and the + * kick's intent survives: whatever it wanted attention for is still + * pending at the exit this lands on, and the preemption timer forces + * one even if the guest never traps again. + */ + while (vexit->reason == HV_EXIT_REASON_CANCELED && + mmap_fastpath_current_producer_active(g)) { + HV_CHECK_CTX(hv_vcpu_run(vcpu), vcpu, g); + drain_external_guest_signal(); + } + + /* Every return from HVF is a natural retirement point. Drain before + * dispatching syscalls, page faults, MAP_FIXED, fork/exec, signals, or + * exit so no host path can consult pre-munmap region metadata and + * rematerialize an EL1-invalidated page. The helper also drains mmap + * publications before the acquire-snapshotted retire entries. + */ + bool munmap_producer_active = mmap_fastpath_current_producer_active(g); + if (!munmap_producer_active) + mmap_fastpath_drain_vmexit( + g, vcpu_exit_is_fork_family_syscall(vcpu, vexit)); + else if (vexit->reason == HV_EXIT_REASON_EXCEPTION) + log_error( + "%s: exception exit interrupted an active EL1 munmap " + "producer", + prefix); + /* Main: disarm timeout */ if (is_main) alarm(0); diff --git a/src/syscall/signal.c b/src/syscall/signal.c index 1fb47a5f..0d52445d 100644 --- a/src/syscall/signal.c +++ b/src/syscall/signal.c @@ -35,12 +35,14 @@ #include "hvutil.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/thread.h" #include "syscall/linux-wire.h" -#include "syscall/fd.h" /* signalfd_notify */ +#include "syscall/fd.h" /* signalfd_notify */ +#include "syscall/internal.h" #include "syscall/proc.h" /* proc_get_pid, proc_get_uid, SYSCALL_EXEC_HAPPENED */ #include "proved/sigframe.h" #include "syscall/signal.h" @@ -1861,6 +1863,14 @@ int64_t signal_sigaltstack(guest_t *g, uint64_t ss_gva, uint64_t old_ss_gva) */ if (ss.ss_sp > UINT64_MAX - ss.ss_size) return -LINUX_EINVAL; + + /* Alternate stacks have the same lifetime sensitivity as clone + * stacks: once registered, their unmap must take the host path + * instead of being classified only as an anonymous arena range. + */ + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); t->altstack_sp = ss.ss_sp; t->altstack_flags = 0; t->altstack_size = ss.ss_size; diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index e51ab24a..f7fdc060 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -1003,10 +1003,21 @@ static int64_t sc_mmap(guest_t *g, bool verbose) { uint64_t refill_len = mmap_fastpath_eligible_length(x0, x1, x2, x3); + uint64_t arena_addr = 0; + if (refill_len && mmap_fastpath_allocate_current_publication_only( + g, refill_len, &arena_addr)) + return (int64_t) arena_addr; + mmap_lock_acquire(g); - int64_t r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); - if (r >= 0 && refill_len) - mmap_fastpath_refill_current_locked(g, refill_len); + int64_t r; + if (refill_len && + mmap_fastpath_allocate_current_locked(g, refill_len, &arena_addr)) { + r = (int64_t) arena_addr; + } else { + r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); + if (r >= 0 && refill_len) + mmap_fastpath_refill_current_locked(g, refill_len); + } mmap_lock_release(); log_debug(" mmap(0x%llx, 0x%llx) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, @@ -1025,6 +1036,7 @@ static int64_t sc_mremap(guest_t *g, { (void) x5; mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); int64_t r = sys_mremap(g, x0, x1, x2, (int) x3, x4); mmap_lock_release(); log_debug(" mremap(0x%llx, 0x%llx, 0x%llx, 0x%x) \xe2\x86\x92 0x%llx", diff --git a/tests/bench-mmap-fresh b/tests/bench-mmap-fresh new file mode 100755 index 00000000..633baa4f --- /dev/null +++ b/tests/bench-mmap-fresh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Accumulate auxiliary fresh bump-tail mmap timing across independent elfuse +# processes. +# One process cannot retain enough large fresh mappings to overcome CNTVCT's +# 41.7-ns tick without exhausting guest VA. Each invocation below gets a new +# guest, while its in-guest timer excludes process startup from the measurement. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" +# 1024 ticks = about 42.7 us on Apple Silicon. This bounds the aggregate +# counter quantization to roughly 0.1%, while keeping the largest cases timely. +TARGET_TICKS="${BENCH_MMAP_FRESH_TARGET_TICKS:-1024}" +MAX_RUNS="${BENCH_MMAP_FRESH_MAX_RUNS:-8192}" + +# size in bytes : mappings per fresh guest run +CASES=( + "4096:1000" + "65536:1000" + "1048576:1000" + "2097152:1000" + "8388608:256" + "134217728:16" + "1073741824:8" + "8589934592:4" + "34359738368:2" +) + +# Optional positional cases use the same size:count spelling as CASES, e.g. +# tests/bench-mmap-fresh 1073741824:8 8589934592:4 +if [ "$#" -gt 0 ]; then + CASES=("$@") +fi + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi + +printf 'Auxiliary fresh bump-tail mmap: target %s aggregate CNTVCT ticks\n' \ + "$TARGET_TICKS" +printf '%-10s %8s %6s %14s\n' size count runs 'fresh mmap ns' + +for case in "${CASES[@]}"; do + size="${case%%:*}" + count="${case##*:}" + total_ticks=0 + total_ops=0 + runs=0 + ns_per_tick= + + while awk -v ticks="$total_ticks" -v target="$TARGET_TICKS" \ + 'BEGIN { exit !(ticks < target) }'; do + if [ "$runs" -ge "$MAX_RUNS" ]; then + echo "fresh benchmark exceeded ${MAX_RUNS} runs for ${size}" >&2 + exit 1 + fi + if ! output="$("$ELFUSE" "$BENCH" fresh "$size" "$count" 2>&1)"; then + echo "fresh guest run failed for size=${size}, count=${count}:" >&2 + echo "$output" >&2 + exit 1 + fi + raw_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ticks=/) {sub(/^ticks=/, "", $i); print $i}}' <<< "$output")" + read_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^read_ticks=/) {sub(/^read_ticks=/, "", $i); print $i}}' <<< "$output")" + ns_per_tick="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ns_per_tick=/) {sub(/^ns_per_tick=/, "", $i); print $i}}' <<< "$output")" + if [ -z "$raw_ticks" ] || [ -z "$read_ticks" ] || [ -z "$ns_per_tick" ]; then + echo "unexpected benchmark output: $output" >&2 + exit 1 + fi + total_ticks="$(awk -v total="$total_ticks" -v raw="$raw_ticks" -v read="$read_ticks" 'BEGIN { print total + raw - read }')" + total_ops=$((total_ops + count)) + runs=$((runs + 1)) + done + + ns_per_op="$(awk -v ticks="$total_ticks" -v ops="$total_ops" -v ns="$ns_per_tick" 'BEGIN { printf "%.2f", ticks * ns / ops }')" + human="$(awk -v n="$size" 'BEGIN { if (n >= 1073741824) printf "%g GiB", n / 1073741824; else if (n >= 1048576) printf "%g MiB", n / 1048576; else printf "%g KiB", n / 1024 }')" + printf '%-10s %8s %6s %14s\n' "$human" "$count" "$runs" "$ns_per_op" +done diff --git a/tests/bench-mmap-isolated b/tests/bench-mmap-isolated new file mode 100755 index 00000000..429cdeee --- /dev/null +++ b/tests/bench-mmap-isolated @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# mmap/munmap performance tests isolated across elfuse processes. +# +# Section A launches a brand new elfuse process for every timed mmap/munmap +# pair. Sections B and C launch one process per size, so a later size +# cannot inherit an escalated arena, fragmented guest VA, or adaptive-sizing +# history from an earlier one. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" +# One timed call is comparable to CNTVCT's 41.7-ns period. Accumulate corrected +# ticks across fresh processes before converting them to a per-call result. +ONESHOT_TARGET_TICKS="${BENCH_MMAP_ONESHOT_TARGET_TICKS:-256}" +ONESHOT_MAX_RUNS="${BENCH_MMAP_ONESHOT_MAX_RUNS:-4096}" + +# Sections A and B use the same size list. +SIZES=( + 4096 16384 65536 262144 1048576 2097152 8388608 + 67108864 268435456 1073741824 4294967296 17179869184 34359738368 +) + +# size:ops pairs matching section C's cases[] table (dirtying every 4 KiB +# page bounds total work, so larger sizes use fewer operations). +DIRTY_CASES=( + "4096:2048" "16384:2048" "65536:2048" "262144:1024" "1048576:512" + "2097152:256" "8388608:128" "67108864:32" "268435456:24" "1073741824:8" +) + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi + +human() +{ + awk -v n="$1" 'BEGIN { + if (n >= 1073741824) printf "%g GiB", n / 1073741824 + else if (n >= 1048576) printf "%g MiB", n / 1048576 + else printf "%g KiB", n / 1024 + }' +} + +# field +# Prints the value of the first "=" token in . +field() +{ + awk -v key="$2" '{ + for (i = 1; i <= NF; i++) { + if (index($i, key "=") == 1) { + sub("^" key "=", "", $i) + print $i + found = 1 + } + } + if (!found) + exit 1 + }' <<< "$1" +} + +run_a() +{ + printf 'A. one timed mmap / munmap fast-path pair per fresh elfuse process\n' + printf ' aggregate target: %s corrected CNTVCT ticks per call type\n' \ + "$ONESHOT_TARGET_TICKS" + printf '%-10s %8s %12s %12s\n' size guests 'mmap ns' 'munmap ns' + for size in "${SIZES[@]}"; do + mmap_ticks=0 + munmap_ticks=0 + runs=0 + ns_per_tick= + failed=0 + while awk -v mmap_ticks="$mmap_ticks" -v munmap_ticks="$munmap_ticks" \ + -v target="$ONESHOT_TARGET_TICKS" \ + 'BEGIN { exit ! (mmap_ticks < target || munmap_ticks < target) }'; do + if [ "$runs" -ge "$ONESHOT_MAX_RUNS" ]; then + printf 'one-shot benchmark exceeded %s runs for %s\n' \ + "$ONESHOT_MAX_RUNS" "$size" >&2 + return 1 + fi + if ! out="$("$ELFUSE" "$BENCH" a-one "$size" 2>&1)"; then + failed=1 + break + fi + raw_mmap_ticks="$(field "$out" mmap_ticks)" + raw_munmap_ticks="$(field "$out" munmap_ticks)" + read_ticks="$(field "$out" read_ticks)" + ns_per_tick="$(field "$out" ns_per_tick)" + mmap_ticks="$(awk -v total="$mmap_ticks" -v raw="$raw_mmap_ticks" \ + -v read="$read_ticks" 'BEGIN { print total + raw - read }')" + munmap_ticks="$(awk -v total="$munmap_ticks" \ + -v raw="$raw_munmap_ticks" -v read="$read_ticks" \ + 'BEGIN { print total + raw - read }')" + runs=$((runs + 1)) + done + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %12s %12s\n' "$(human "$size")" - FAILED - + continue + fi + mmap_ns="$(awk -v ticks="$mmap_ticks" -v runs="$runs" \ + -v ns="$ns_per_tick" 'BEGIN { print ticks * ns / runs }')" + munmap_ns="$(awk -v ticks="$munmap_ticks" -v runs="$runs" \ + -v ns="$ns_per_tick" 'BEGIN { print ticks * ns / runs }')" + printf '%-10s %8s %12.1f %12.1f\n' "$(human "$size")" "$runs" \ + "$mmap_ns" "$munmap_ns" + done + printf '\n' +} + +run_b() +{ + printf 'B. munmap after materializing one 4 KiB page (isolated)\n' + printf '%-10s %8s %12s\n' size ops 'munmap ns' + for size in "${SIZES[@]}"; do + if ! out="$("$ELFUSE" "$BENCH" b-one "$size" 2>&1)"; then + printf '%-10s %8s %12s\n' "$(human "$size")" - FAILED + continue + fi + ops="$(field "$out" ops)" + munmap_ns="$(field "$out" munmap_ns)" + printf '%-10s %8s %12.1f\n' "$(human "$size")" "$ops" "$munmap_ns" + done + printf '\n' +} + +run_c() +{ + printf 'C. munmap after dirtying every 4 KiB page (isolated)\n' + printf '%-10s %8s %12s %12s %12s\n' size ops 'p50 ns' 'p95 ns' 'max ns' + for case in "${DIRTY_CASES[@]}"; do + size="${case%%:*}" + ops="${case##*:}" + if ! out="$("$ELFUSE" "$BENCH" c-one "$size" "$ops" 2>&1)"; then + printf '%-10s %8s %12s %12s %12s\n' "$(human "$size")" "$ops" \ + FAILED - - + continue + fi + p50_ns="$(field "$out" p50_ns)" + p95_ns="$(field "$out" p95_ns)" + max_ns="$(field "$out" max_ns)" + printf '%-10s %8s %12.1f %12.1f %12.1f\n' "$(human "$size")" "$ops" \ + "$p50_ns" "$p95_ns" "$max_ns" + done + printf '\n' +} + +case "${1:-all}" in + a) run_a ;; + b) run_b ;; + c) run_c ;; + all) run_a; run_b; run_c ;; + *) + echo "usage: $0 [a|b|c|all]" >&2 + exit 2 + ;; +esac diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c index c57e9d03..2412d56e 100644 --- a/tests/bench-mmap.c +++ b/tests/bench-mmap.c @@ -14,8 +14,11 @@ * the key fairness property: clock_gettime on a static guest falls through to * the ~2 us SVC path and swamps any sub-us operation. * - * Every case takes one untimed warmup pass (to pay the one-time arena carve and - * page-table extension) and reports the median and min over ITERS runs. + * Every in-process case takes one untimed warmup pass (to pay the one-time + * arena carve and page-table extension). Most sections report aggregate + * samples; section C retains every operation so its normal latency and long + * tail remain visible. Section A is driven host-side so every timed mmap and + * munmap pair gets a new elfuse process. * * Copyright 2026 elfuse contributors * SPDX-License-Identifier: Apache-2.0 @@ -25,6 +28,7 @@ #define _GNU_SOURCE #endif #include +#include #include #include #include @@ -70,23 +74,41 @@ static double median(double *v, int n) return (n & 1) ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); } +/* R-7/sample quantile, matching the common (n - 1) * p interpolation. The + * input must already be sorted. */ +static double sorted_quantile(const double *v, unsigned n, double p) +{ + double pos = (double) (n - 1) * p; + unsigned lo = (unsigned) pos; + unsigned hi = lo + (lo + 1 < n); + return v[lo] + (v[hi] - v[lo]) * (pos - lo); +} + #define ITERS 15 -#define MAXB 64 +#define MIN_TIMED_TICKS 4096 #define KIB (1ULL << 10) #define MIB (1ULL << 20) #define GIB (1ULL << 30) +#define DIRTY_VMEXIT_STRIDE (8 * MIB) -/* Batch size: amortize the coarse counter over B ops while bounding the live - * address footprint of one timed batch to ~256 MiB. +/* Calibrate the rd()/rd() interval around a timed operation. Its median cost + * is subtracted from aggregate samples so the counter-read overhead is not + * attributed to mmap or munmap. */ -static int batch_for(uint64_t size) +static double rd_pair_ticks(void) { - uint64_t b = (256 * MIB) / size; - if (b < 1) - b = 1; - if (b > MAXB) - b = MAXB; - return (int) b; + enum { CAL_SAMPLES = 15, CAL_OPS = 4096 }; + double samples[CAL_SAMPLES]; + for (int sample = 0; sample < CAL_SAMPLES; sample++) { + uint64_t total = 0; + for (int op = 0; op < CAL_OPS; op++) { + uint64_t t0 = rd(); + uint64_t t1 = rd(); + total += t1 - t0; + } + samples[sample] = (double) total / CAL_OPS; + } + return median(samples, CAL_SAMPLES); } static const char *human(uint64_t s, char *buf) @@ -100,65 +122,55 @@ static const char *human(uint64_t s, char *buf) return buf; } -/* A. mmap + munmap latency vs size (steady state) NULL-hint allocate then free, - * batched. Iterations after the first reuse freed address space, so this is the - * realistic repeated-allocation number a workload sees, not the one-shot fresh - * case (that is section B). +/* A. One mmap and one munmap fast-path sample in a fresh guest. The host-side + * driver starts a new elfuse process for every invocation of this function. + * Prime the requested arena size outside the timed interval, then force a host + * drain so the arena is empty and its cursor is rewound. The two measured + * calls can then take the EL1 paths even when size exceeds the initial 64 MiB + * arena. Raw ticks are returned because one fast call is comparable to the + * counter period; the driver aggregates independent one-call samples before + * converting them to nanoseconds. */ -static void bench_size_sweep(void) +static int bench_fastpath_once(uint64_t size) { - static const uint64_t sizes[] = { - 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, - 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, - }; + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED || munmap(warmup, size) != 0) + return 1; + + /* Drain the warmup retirement and rewind the now-empty arena. */ + (void) fcntl(-1, F_GETFD); + + uint64_t mmap_start = rd(); + void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t mmap_end = rd(); + if (ptr == MAP_FAILED) + return 1; + + uint64_t munmap_start = rd(); + int rc = munmap(ptr, size); + uint64_t munmap_end = rd(); + if (rc != 0) + return 1; + printf( - "== A. mmap / munmap latency vs size (steady state, NULL hint) ==\n"); - printf("%-10s %6s %12s %12s\n", "size", "batch", "mmap ns", "munmap ns"); - void *ptr[MAXB]; - for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { - uint64_t size = sizes[s]; - int b = batch_for(size); - double mm[ITERS], um[ITERS]; - int ok = 1; - for (int it = -1; it < ITERS && ok; it++) { - uint64_t t0 = rd(); - for (int i = 0; i < b; i++) - ptr[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - uint64_t t1 = rd(); - for (int i = 0; i < b; i++) - if (ptr[i] == MAP_FAILED) - ok = 0; - if (!ok) - break; - uint64_t t2 = rd(); - for (int i = 0; i < b; i++) - munmap(ptr[i], size); - uint64_t t3 = rd(); - if (it >= 0) { - mm[it] = ns(t1 - t0) / b; - um[it] = ns(t3 - t2) / b; - } - } - char hb[16]; - if (!ok) { - printf("%-10s %6d %12s %12s\n", human(size, hb), b, "FAILED", "-"); - continue; - } - printf("%-10s %6d %12.1f %12.1f\n", human(size, hb), b, - median(mm, ITERS), median(um, ITERS)); - } - printf("\n"); + "fast-once size=%llu mmap_ticks=%llu munmap_ticks=%llu " + "read_ticks=%.6f ns_per_tick=%.12f\n", + (unsigned long long) size, (unsigned long long) (mmap_end - mmap_start), + (unsigned long long) (munmap_end - munmap_start), rd_pair_ticks(), + ns_per_tick); + return 0; } -/* B. fresh bump-tail mmap (isolates the lazy_fresh_range path) Allocate a - * bounded sequential run WITHOUT freeing, so every mapping lands at or above - * the arena high-water -- exactly the case lazy_fresh_range skips the stale-PTE - * scan for. The run is kept small enough (<= 1000 regions, well under - * GUEST_MAX_REGIONS, footprint <= 2 GiB) that region bookkeeping and page-table - * extension do not dominate, and every result is failure-checked. Run this - * binary against an opt-off build to read the skip's contribution as the - * difference on this identical code path -- a MAP_FIXED "recycled" compare +/* Auxiliary: fresh bump-tail mmap isolates the lazy_fresh_range path. Allocate + * sequential run WITHOUT freeing, so every mapping lands at or above the arena + * high-water -- exactly the case lazy_fresh_range skips the stale-PTE scan for. + * Small mappings use the original 2-GiB footprint cap; large mappings use a + * minimum count chosen to retain multiple samples without exceeding 64 GiB of + * live fresh VA. + * Run this binary against an opt-off build to read the skip's contribution as + * the difference on this identical code path -- a MAP_FIXED "recycled" compare * would instead measure the region-snapshot replacement path, not the skip. */ static void bench_fresh(void) @@ -167,7 +179,8 @@ static void bench_fresh(void) 2 * MIB, 8 * MIB, 128 * MIB, GIB, 8 * GIB, 32 * GIB}; printf( - "== B. fresh bump-tail mmap, per-mmap ns (lazy_fresh_range path) ==\n"); + "== Auxiliary: fresh bump-tail mmap, per-mmap ns " + "(lazy_fresh_range path) ==\n"); printf("%-10s %8s %14s\n", "size", "count", "fresh mmap ns"); void *run[1000]; for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { @@ -177,6 +190,12 @@ static void bench_fresh(void) n = 1; if (n > 1000) n = 1000; + if (size == GIB) + n = 8; + else if (size == 8 * GIB) + n = 4; + else if (size == 32 * GIB) + n = 2; /* warmup one fresh mapping so the arena high-water is already primed */ void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -205,19 +224,61 @@ static void bench_fresh(void) printf("\n"); } -/* C. first-touch page-fault cost Touch one byte per page at a 16 KiB stride so - * no two touches share a macOS host page; every touch is a genuine fault (HVC - * #11 -> host fault handler -> page-table install + zero). Reports per-fault - * ns. +/* One fresh bump-tail run for the host-side driver. A new elfuse process is + * used for each invocation, so the driver can accumulate many counter ticks + * without exhausting one guest's VA space. */ +static int bench_fresh_one(uint64_t size, int n) +{ + void *run[1000]; + if (n < 1 || n > (int) (sizeof(run) / sizeof(run[0]))) + return 2; + + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED) + return 1; + munmap(warmup, size); + + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + if (failed) + return 1; + + printf( + "fresh size=%llu count=%d ticks=%llu read_ticks=%.6f " + "ns_per_tick=%.12f\n", + (unsigned long long) size, n, (unsigned long long) (t1 - t0), + rd_pair_ticks(), ns_per_tick); + return 0; +} + +/* Auxiliary: first-touch cost. Touch one byte per macOS 16 KiB host page. Each + * demands a distinct physical backing page, but it is not necessarily a + * distinct HVC: elfuse installs Stage-1 descriptors in 2 MiB windows and the + * fault-around policy may install several windows per exit. Report both the + * whole sweep and its per-host-page amortization; calling the latter a + * "per-fault" cost would substantially overcount guest translation faults. */ -static void bench_fault(void) +static void bench_fault(int pages, int drain_between) { const uint64_t stride = 16 * KIB; - const int pages = 512; uint64_t size = stride * (uint64_t) (pages + 1); - printf("== C. first-touch fault cost (16 KiB stride, %d pages) ==\n", - pages); - double per[ITERS]; + printf( + "== Auxiliary: first-touch fault cost (16 KiB stride, %d pages%s) " + "==\n", + pages, drain_between ? ", forced retire drain" : ""); + double sweep[ITERS]; for (int it = -1; it < ITERS; it++) { volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -230,21 +291,31 @@ static void bench_fault(void) p[(uint64_t) i * stride] = 1; uint64_t t1 = rd(); munmap((void *) p, size); + if (drain_between) + (void) fcntl(-1, F_GETFD); if (it >= 0) - per[it] = ns(t1 - t0) / pages; + sweep[it] = ns(t1 - t0); } - printf(" per-fault: median %.1f ns min %.1f ns\n\n", median(per, ITERS), - per[0]); + qsort(sweep, ITERS, sizeof(*sweep), cmp_d); + printf(" sweep: p50 %.1f us p95 %.1f us max %.1f us\n", + sorted_quantile(sweep, ITERS, 0.50) / 1000.0, + sorted_quantile(sweep, ITERS, 0.95) / 1000.0, + sweep[ITERS - 1] / 1000.0); + printf(" amortized/touch: p50 %.1f ns p95 %.1f ns max %.1f ns\n\n", + sorted_quantile(sweep, ITERS, 0.50) / pages, + sorted_quantile(sweep, ITERS, 0.95) / pages, + sweep[ITERS - 1] / pages); } -/* D. mprotect split cost Flip the middle 4 KiB of a 2 MiB RW block to +/* Auxiliary: mprotect split cost. Flip the middle 4 KiB of a 2 MiB RW block to * PROT_READ, forcing guest_split_block to convert the L2 block into 512 L3 * pages. Restore between iterations so each run does a fresh split. */ static void bench_mprotect_split(void) { printf( - "== D. mprotect split (2 MiB block -> L3, protect middle 4 KiB) ==\n"); + "== Auxiliary: mprotect split " + "(2 MiB block -> L3, protect middle 4 KiB) ==\n"); double sp[ITERS]; for (int it = -1; it < ITERS; it++) { uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, @@ -269,10 +340,10 @@ static void bench_mprotect_split(void) sp[0]); } -/* E. mremap grow: in-place vs forced move */ +/* Auxiliary: mremap grow, in-place vs forced move. */ static void bench_mremap(void) { - printf("== E. mremap grow 4 KiB -> 8 KiB ==\n"); + printf("== Auxiliary: mremap grow 4 KiB -> 8 KiB ==\n"); double inp[ITERS], mov[ITERS]; /* In-place: no blocker, the following page is free. */ @@ -331,7 +402,7 @@ static void bench_mremap(void) mov[0]); } -/* F. multi-threaded fresh mmap under mmap_lock Several threads hammer fresh +/* Auxiliary: multi-threaded fresh mmap under mmap_lock. Several threads hammer * bump-tail mmaps concurrently. mmap serializes on mmap_lock, so this exposes * both lock contention and any per-mmap TLBI shootdown cost -- the one place a * "skip the invalidate on fresh ranges" optimization could pay off that a @@ -388,8 +459,8 @@ static void bench_mt(void) static const uint64_t sizes[] = {4 * KIB, 2 * MIB}; static const int threads[] = {2, 4}; printf( - "== F. multi-threaded fresh mmap, per-op ns (mmap_lock contention) " - "==\n"); + "== Auxiliary: multi-threaded fresh mmap, per-op ns " + "(mmap_lock contention) ==\n"); printf("%-10s %8s %12s %12s\n", "size", "threads", "mean ns", "max ns"); for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { uint64_t size = sizes[s]; @@ -441,13 +512,306 @@ static void bench_mt(void) printf("\n"); } -int main(void) +/* B. Teardown after materialization. The store is deliberately outside the + * timed interval: it takes the lazy first-touch fault and installs the first + * page, then the counter brackets only munmap(). Each row has exactly one + * materialized 4-KiB page; touching every page would instead benchmark + * faulting and zeroing gigabytes of memory. + * + * bench_munmap_materialized_measure() holds the timing loop for exactly one + * size, shared by the in-process sweep below and the "b-one" isolated-process + * driver mode (see tests/bench-mmap-isolated). + */ +static int bench_munmap_materialized_measure(uint64_t size, + double timer_ticks, + unsigned *reported_ops, + double *munmap_ns) +{ + double unmap_ns[ITERS]; + int ok = 1; + for (int it = -1; it < ITERS && ok; it++) { + uint64_t unmap_ticks = 0; + unsigned ops = 0; + do { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + p[0] = 1; /* materialize before starting the timed interval */ + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + unmap_ticks += t1 - t0; + if (rc != 0) { + ok = 0; + break; + } + ops++; + } while (unmap_ticks < MIN_TIMED_TICKS); + + if (it >= 0 && ok) { + unmap_ns[it] = + ((double) unmap_ticks / ops - timer_ticks) * ns_per_tick; + *reported_ops = ops; + } + } + if (!ok) + return 0; + *munmap_ns = median(unmap_ns, ITERS); + return 1; +} + +static void bench_munmap_materialized(void) +{ + double timer_ticks = rd_pair_ticks(); + static const uint64_t sizes[] = { + 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, + 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, + }; + + printf("== B. munmap after materializing one 4 KiB page ==\n"); + printf("%-10s %8s %12s\n", "size", "ops", "munmap ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + unsigned reported_ops = 0; + double munmap_ns = 0; + char hb[16]; + if (!bench_munmap_materialized_measure(size, timer_ticks, &reported_ops, + &munmap_ns)) + printf("%-10s %8s %12s\n", human(size, hb), "-", "FAILED"); + else + printf("%-10s %8u %12.1f\n", human(size, hb), reported_ops, + munmap_ns); + } + printf("\n"); +} + +/* C. Teardown after every page was dirtied. Page stores happen before the + * timed interval, so this reports only munmap's handling of the materialized, + * dirty mapping. Each size has a fixed operation count and every munmap is + * retained separately. In particular, one slow first operation cannot end an + * adaptive batch and become the whole sample. Counts decrease with size to + * bound total dirtying work. The 1-GiB case uses eight operations, and the + * untimed store loop takes periodic VM exits so every vCPU-run interval stays + * below elfuse's watchdog; interpolated p95 remains distinct from max. + * + * bench_munmap_dirty_measure() holds the timing loop for exactly one size, + * shared by the in-process sweep below and the "c-one" isolated-process + * driver mode (see tests/bench-mmap-isolated). + */ +static int bench_munmap_dirty_measure(uint64_t size, + unsigned ops, + double timer_ticks, + double *p50_ns, + double *p95_ns, + double *max_ns) +{ + double *unmap_ns = malloc((size_t) ops * sizeof(*unmap_ns)); + int ok = 1; + if (!unmap_ns) + ok = 0; + + /* One full untimed warmup pays setup without consuming an observation. */ + for (int64_t op = -1; op < (int64_t) ops && ok; op++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + for (uint64_t off = 0; off < size; off += 4 * KIB) { + p[off] = (uint8_t) (off >> 12); + /* Large already-backed runs can execute in EL0 long enough + * for the benchmark's 10-second vCPU watchdog to fire under + * macOS memory pressure. This deliberately failing fcntl is + * a guaranteed, side-effect-free HVC outside the timed region. + * It also drains the preceding retirement in bounded chunks. + */ + if (off != 0 && (off & (DIRTY_VMEXIT_STRIDE - 1)) == 0) + (void) fcntl(-1, F_GETFD); + } + + /* Establish an identical clean host-retirement boundary before + * every measurement. The interval below then contains EL1's + * descriptor/TLBI path, never the previous operation's cleanup. + */ + (void) fcntl(-1, F_GETFD); + + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + if (rc != 0) { + ok = 0; + break; + } + if (op >= 0) { + double ticks = (double) (t1 - t0) - timer_ticks; + if (ticks < 0.0) + ticks = 0.0; + unmap_ns[op] = ticks * ns_per_tick; + } + } + + if (!ok) { + free(unmap_ns); + return 0; + } + qsort(unmap_ns, ops, sizeof(*unmap_ns), cmp_d); + *p50_ns = sorted_quantile(unmap_ns, ops, 0.50); + *p95_ns = sorted_quantile(unmap_ns, ops, 0.95); + *max_ns = unmap_ns[ops - 1]; + free(unmap_ns); + return 1; +} + +static void bench_munmap_dirty(void) +{ + static const struct { + uint64_t size; + unsigned ops; + } cases[] = { + {4 * KIB, 2048}, {16 * KIB, 2048}, {64 * KIB, 2048}, {256 * KIB, 1024}, + {MIB, 512}, {2 * MIB, 256}, {8 * MIB, 128}, {64 * MIB, 32}, + {256 * MIB, 24}, {GIB, 8}, + }; + double timer_ticks = rd_pair_ticks(); + + printf("== C. munmap after dirtying every 4 KiB page ==\n"); + printf("%-10s %8s %12s %12s %12s\n", "size", "ops", "p50 ns", "p95 ns", + "max ns"); + for (unsigned s = 0; s < sizeof(cases) / sizeof(cases[0]); s++) { + uint64_t size = cases[s].size; + unsigned ops = cases[s].ops; + double p50_ns = 0, p95_ns = 0, max_ns = 0; + char hb[16]; + if (!bench_munmap_dirty_measure(size, ops, timer_ticks, &p50_ns, + &p95_ns, &max_ns)) + printf("%-10s %8u %12s %12s %12s\n", human(size, hb), ops, "FAILED", + "-", "-"); + else + printf("%-10s %8u %12.1f %12.1f %12.1f\n", human(size, hb), ops, + p50_ns, p95_ns, max_ns); + } + printf("\n"); +} + +int main(int argc, char **argv) { clock_init(); + if (argc == 2 && strcmp(argv[1], "b") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_munmap_materialized(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "c") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_munmap_dirty(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "mt") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_mt(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "fault") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault(512, 0); + return 0; + } + if (argc == 3 && strcmp(argv[1], "fault") == 0) { + char *end = NULL; + errno = 0; + long pages = strtol(argv[2], &end, 0); + if (errno || !end || *end || pages < 1 || pages > 1048576) + return 2; + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault((int) pages, 0); + return 0; + } + if (argc == 2 && strcmp(argv[1], "fault-drain") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault(512, 1); + return 0; + } + if (argc == 4 && strcmp(argv[1], "fresh") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long count = strtol(argv[3], &end, 0); + if (errno || !end || *end || count < 1 || count > 1000) + return 2; + return bench_fresh_one(size, (int) count); + } + /* One section-A sample. The host-side driver starts a new elfuse process + * for every invocation and aggregates the raw one-call timings. */ + if (argc == 3 && strcmp(argv[1], "a-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + return bench_fastpath_once(size); + } + /* One section-B data point for the host-side driver. */ + if (argc == 3 && strcmp(argv[1], "b-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + double timer_ticks = rd_pair_ticks(); + unsigned reported_ops = 0; + double munmap_ns = 0; + if (!bench_munmap_materialized_measure(size, timer_ticks, &reported_ops, + &munmap_ns)) { + printf("munmap-materialized size=%llu FAILED\n", + (unsigned long long) size); + return 1; + } + printf("munmap-materialized size=%llu ops=%u munmap_ns=%.6f\n", + (unsigned long long) size, reported_ops, munmap_ns); + return 0; + } + /* One section-C data point for the host-side driver. The op count is + * explicit because C's cases[] table pairs it to size, and the driver must + * pass the same value the table would give. */ + if (argc == 4 && strcmp(argv[1], "c-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long ops_arg = strtol(argv[3], &end, 0); + if (errno || !end || *end || ops_arg < 1 || ops_arg > 1000000) + return 2; + unsigned ops = (unsigned) ops_arg; + double timer_ticks = rd_pair_ticks(); + double p50_ns = 0, p95_ns = 0, max_ns = 0; + if (!bench_munmap_dirty_measure(size, ops, timer_ticks, &p50_ns, + &p95_ns, &max_ns)) { + printf("munmap-dirty size=%llu FAILED\n", + (unsigned long long) size); + return 1; + } + printf( + "munmap-dirty size=%llu ops=%u p50_ns=%.6f p95_ns=%.6f " + "max_ns=%.6f\n", + (unsigned long long) size, ops, p50_ns, p95_ns, max_ns); + return 0; + } + if (argc != 1) + return 2; printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); - bench_size_sweep(); + bench_munmap_materialized(); + bench_munmap_dirty(); bench_fresh(); - bench_fault(); + bench_fault(512, 0); bench_mprotect_split(); bench_mremap(); bench_mt(); diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh index fa92a8a7..3068cc95 100755 --- a/tests/test-mmap-fastpath-stats.sh +++ b/tests/test-mmap-fastpath-stats.sh @@ -69,10 +69,14 @@ require_le() fi } +run_case ring-full +require_ge ring-full MMAP_HIT 32 +require_ge ring-full MMAP_RING_FULL 1 +printf ' 32-entry ring fallback OK\n' + run_case np2-10m require_ge np2-10m MMAP_HIT 80 require_ge np2-10m MMAP_CAPACITY_MISS 1 -require_ge np2-10m MMAP_RING_FULL 1 printf ' sustained 10 MiB stream OK\n' run_case np2-48m @@ -87,27 +91,52 @@ printf ' sustained 100 MiB stream OK\n' run_case escalation require_ge escalation MMAP_HIT 45 -require_eq escalation MMAP_ARENA_CURRENT 1073741824 +require_eq escalation MMAP_ARENA_CURRENT 17179869184 printf ' 10 MiB -> 512 MiB escalation OK\n' run_case giant-guard require_ge giant-guard MMAP_HIT 34 -require_le giant-guard MMAP_ARENA_PEAK 536870912 -printf ' >1 GiB giant request guard OK\n' +require_eq giant-guard MMAP_ARENA_PEAK 34359738368 +printf ' 2 GiB request uses fast path OK\n' run_case adaptive-small require_eq adaptive-small MMAP_ARENA_CURRENT 67108864 require_eq adaptive-small MMAP_ARENA_PEAK 67108864 printf ' small-stream arena floor OK\n' -run_case adaptive-decay -require_eq adaptive-decay MMAP_ARENA_CURRENT 67108864 -require_eq adaptive-decay MMAP_ARENA_PEAK 1073741824 -printf ' one-generation arena decay OK\n' +run_case adaptive-retention +require_eq adaptive-retention MMAP_ARENA_CURRENT 17179869184 +require_eq adaptive-retention MMAP_ARENA_PEAK 17179869184 +printf ' large arena retained OK\n' + +run_case adaptive-rewind-growth +require_ge adaptive-rewind-growth MMAP_CAPACITY_MISS 1 +require_eq adaptive-rewind-growth MMAP_ARENA_CURRENT 268435456 +printf ' rewound arena grows to target OK\n' run_case recycle require_ge recycle MMAP_RECYCLE 1 require_le recycle MMAP_HIGH_WATER 201326592 printf ' arena VA recycling OK\n' +# Mixed-size churn now recycles VA through the arena rewind and the host gap +# allocator alone. MMAP_RECYCLE is the load-bearing assertion: it is nonzero +# only when a refill actually reclaimed a hole below the high-water mark, so it +# fails the moment VA recovery stops and the allocator only walks forward. The +# hit count keeps the case honest -- the other two would also pass if the fast +# path stopped being taken at all -- and the bound catches growth that recovery +# is too slow to contain. +run_case mixed-churn +require_ge mixed-churn MMAP_HIT 100 +require_ge mixed-churn MMAP_RECYCLE 1 +require_le mixed-churn MMAP_HIGH_WATER 402653184 +printf ' mixed-size churn recycles VA OK\n' + +# Fork drains and revokes every arena. A near-empty current arena must not be +# topped up between those operations; only the initial vCPU arena is counted. +run_case fork-no-topup +require_eq fork-no-topup MMAP_HIT 1 +require_eq fork-no-topup MMAP_REFILL 1 +printf ' fork skips arena top-up OK\n' + printf 'test-mmap-fastpath-stats: PASS\n' diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c index 4a9aa21e..9b8af480 100644 --- a/tests/test-mmap-fastpath.c +++ b/tests/test-mmap-fastpath.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,8 @@ int passes = 0, fails = 0; static sigjmp_buf segv_jmp; +static inline void spin_hint(void); + static void segv_handler(int sig) { (void) sig; @@ -64,8 +67,9 @@ static void test_fidelity(void) { TEST("unconsumed arena is absent and faults"); struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) != 0) { + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { FAIL("sigaction"); return; } @@ -74,6 +78,7 @@ static void test_fidelity(void) MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); return; } p[0] = 0x5a; /* drains the publication through the fault-side lock */ @@ -83,6 +88,7 @@ static void test_fidelity(void) (void) *unconsumed; FAIL("wild read into unconsumed arena did not SIGSEGV"); munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); return; } @@ -91,9 +97,26 @@ static void test_fidelity(void) hi != (uintptr_t) p + 4096) { FAIL("/proc/self/maps exposed more than the consumed page"); munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + if (munmap(p, 4096) != 0) { + FAIL("munmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + /* No syscall may intervene between munmap and this load: EL1 must have + * invalidated the Stage-1 descriptor and completed broadcast TLBI before + * returning, even though host region cleanup is still deferred. + */ + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *(volatile uint8_t *) p; + FAIL("access immediately after munmap did not SIGSEGV"); + sigaction(SIGSEGV, &old_sa, NULL); return; } - munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); PASS(); } @@ -121,6 +144,268 @@ static void test_exhaustion_fallback(void) PASS(); } +typedef struct { + volatile uint8_t *p; + _Atomic int ready; + _Atomic int go; + _Atomic int result; +} large_tlbi_arg_t; + +static void *large_tlbi_worker(void *opaque) +{ + large_tlbi_arg_t *arg = opaque; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) arg->p[0]; /* seed a translation on this sibling vCPU */ + atomic_store_explicit(&arg->ready, 1, memory_order_release); + while (!atomic_load_explicit(&arg->go, memory_order_acquire)) + spin_hint(); + (void) arg->p[0]; + atomic_store_explicit(&arg->result, -1, memory_order_release); + } else { + atomic_store_explicit(&arg->result, 1, memory_order_release); + } + return NULL; +} + +static void test_large_l2_range_tlbi(void) +{ + TEST("SCALE=3 RVAE1IS invalidates sibling L2 translation"); + const size_t len = 320ULL << 20; /* exceeds SCALE=2's 256MiB maximum */ + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + /* One touch per 2MiB materializes L2 block descriptors without making the + * test resident at every guest page. The resulting TLBI envelope requires + * SCALE=3 when FEAT_TLBIRANGE is enabled. + */ + for (size_t off = 0; off < len; off += 2ULL << 20) + p[off] = (uint8_t) (off >> 21); + + large_tlbi_arg_t arg = {.p = p}; + pthread_t worker; + if (pthread_create(&worker, NULL, large_tlbi_worker, &arg) != 0) { + FAIL("pthread_create"); + munmap((void *) p, len); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + if (munmap((void *) p, len) != 0) { + FAIL("munmap"); + atomic_store_explicit(&arg.go, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + /* No syscall may intervene here: the sibling must observe EL1's broadcast + * invalidation before any host drain gets a chance to remove metadata. + */ + atomic_store_explicit(&arg.go, 1, memory_order_release); + int result; + while (!(result = atomic_load_explicit(&arg.result, memory_order_acquire))) + spin_hint(); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + if (result < 0) { + FAIL("stale sibling translation survived large-range TLBI"); + return; + } + PASS(); +} + +typedef struct { + _Atomic uintptr_t ptr; + _Atomic int ready; + _Atomic int done; + _Atomic int release; +} handoff_arg_t; + +static inline void spin_hint(void) +{ + __asm__ volatile("yield" ::: "memory"); +} + +static void *handoff_worker(void *opaque) +{ + handoff_arg_t *arg = opaque; + uint8_t *warm = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warm == MAP_FAILED) { + atomic_store_explicit(&arg->done, -1, memory_order_release); + atomic_store_explicit(&arg->ready, 1, memory_order_release); + return NULL; + } + warm[0] = 1; + atomic_store_explicit(&arg->ready, 1, memory_order_release); + uintptr_t ptr; + while (!(ptr = atomic_load_explicit(&arg->ptr, memory_order_acquire))) + spin_hint(); + int rc = munmap((void *) ptr, 4096); + atomic_store_explicit(&arg->done, rc == 0 ? 1 : -1, memory_order_release); + while (!atomic_load_explicit(&arg->release, memory_order_acquire)) + spin_hint(); + munmap(warm, 4096); + return NULL; +} + +static void test_cross_vcpu_handoff(void) +{ + TEST("cross-vCPU mmap publication then munmap retirement"); + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + handoff_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, handoff_worker, &arg) != 0) { + FAIL("pthread_create"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + /* Keep the worker alive after munmap so its next thread-exit syscall + * cannot drain either ring. The fault below is the first natural VM exit + * after A's mmap publication and B's retirement. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + atomic_store_explicit(&arg.ptr, (uintptr_t) p, memory_order_release); + int done; + while (!(done = atomic_load_explicit(&arg.done, memory_order_acquire))) + spin_hint(); + + int faulted = 0; + if (done > 0 && sigsetjmp(segv_jmp, 1) == 0) + (void) *(volatile uint8_t *) p; + else if (done > 0) + faulted = 1; + + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + if (done < 0) { + FAIL("worker munmap"); + return; + } + if (!faulted) { + FAIL("retired cross-vCPU mapping remained accessible"); + return; + } + PASS(); +} + +static void test_mixed_size_recycled_va_reads_zero(void) +{ + TEST("recycled mixed-size VA reads zero"); + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + + for (int i = 0; i < 128; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + if (p[0] != 0 || p[len / 2] != 0 || p[len - 1] != 0) { + munmap((void *) p, len); + FAIL("recycled VA exposed stale bytes"); + return; + } + p[0] = (uint8_t) (i + 1); + p[len / 2] = (uint8_t) (i ^ 0x5a); + p[len - 1] = (uint8_t) (i ^ 0xa5); + if (munmap((void *) p, len) != 0) { + FAIL("munmap"); + return; + } + } + PASS(); +} + +static void test_repeated_munmap_does_not_alias(void) +{ + TEST("repeated munmap does not alias two live mappings"); + const size_t len = 64ULL << 10; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("initial mmap"); + return; + } + p[0] = 1; + if (munmap((void *) p, len) != 0 || munmap((void *) p, len) != 0) { + FAIL("repeated munmap"); + return; + } + + /* This fault drains both retire records. Only the first still has region + * coverage, so only it may return VA to the allocator; the second must not + * hand the same extent out a second time. */ + volatile uint8_t *bridge = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bridge == MAP_FAILED) { + FAIL("bridge mmap"); + return; + } + bridge[0] = 2; + + volatile uint8_t *a = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + volatile uint8_t *b = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (a == MAP_FAILED || b == MAP_FAILED || a == b) { + if (a != MAP_FAILED) + munmap((void *) a, len); + if (b != MAP_FAILED && b != a) + munmap((void *) b, len); + munmap((void *) bridge, len); + FAIL("duplicate extent allocation"); + return; + } + a[0] = 0x31; + b[0] = 0x42; + if (a[0] != 0x31 || b[0] != 0x42) { + FAIL("distinct mappings aliased"); + return; + } + munmap((void *) a, len); + munmap((void *) b, len); + munmap((void *) bridge, len); + PASS(); +} + typedef struct { int iterations; _Atomic int *failed; @@ -215,8 +500,57 @@ static int stats_stream(size_t len, int iterations, bool release_each) return 0; } +static int stats_mixed_churn(void) +{ + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + for (int i = 0; i < 160; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (p[0] != 0 || p[len - 1] != 0) + return 1; + p[0] = 1; + p[len - 1] = 2; + if (munmap((void *) p, len) != 0) + return 1; + } + return 0; +} + +static int stats_fork_no_topup(void) +{ + /* Leave exactly half of the initial 64 MiB arena free. The 32 MiB + * registration becomes the low-water mark, so the ordinary mmap lock + * acquire would speculatively refill immediately before fork revokes all + * arenas. + */ + void *p = mmap(NULL, 32ULL << 20, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + + pid_t pid = fork(); + if (pid == 0) + _exit(0); + if (pid < 0) + return 1; + + int status = 0; + if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) + return 1; + return 0; +} + static int run_stats_case(const char *name) { + if (strcmp(name, "ring-full") == 0) + return stats_stream(64ULL << 10, 40, false); if (strcmp(name, "np2-10m") == 0) return stats_stream(10ULL << 20, 96, false); if (strcmp(name, "np2-48m") == 0) @@ -240,17 +574,27 @@ static int run_stats_case(const char *name) } if (strcmp(name, "adaptive-small") == 0) return stats_stream(64ULL << 10, 1100, false); - if (strcmp(name, "adaptive-decay") == 0) { - if (stats_stream(64ULL << 10, 1100, false) != 0 || - stats_stream(500ULL << 20, 1, false) != 0) + if (strcmp(name, "adaptive-retention") == 0) { + if (stats_stream(64ULL << 10, 1100, false) != 0) return 1; - /* Ring-full fallbacks do not consume the arena cursor, so exceed the - * nominal 16384 pages enough to force a true 1GiB capacity rollover. + /* The first request selects a 16GiB arena, the next 32 consume it, and + * the last forces a capacity rollover that must retain the target. */ - return stats_stream(64ULL << 10, 18000, false); + return stats_stream(500ULL << 20, 34, false); } + if (strcmp(name, "adaptive-rewind-growth") == 0) + /* The first 64MiB arena holds eight 8MiB mappings. The ninth mmap + * takes the capacity fallback after the matching munmaps let host + * drain rewind the arena; refill must grow it to the 32-entry target + * instead of retaining an arena that will miss every eight calls. + */ + return stats_stream(8ULL << 20, 41, true); if (strcmp(name, "recycle") == 0) return stats_stream(64ULL << 10, 6000, true); + if (strcmp(name, "mixed-churn") == 0) + return stats_mixed_churn(); + if (strcmp(name, "fork-no-topup") == 0) + return stats_fork_no_topup(); return 2; } @@ -270,6 +614,10 @@ int main(int argc, char **argv) test_fidelity(); test_exhaustion_fallback(); + test_large_l2_range_tlbi(); + test_cross_vcpu_handoff(); + test_mixed_size_recycled_va_reads_zero(); + test_repeated_munmap_does_not_alias(); test_mt_storm_and_fork_exec(); printf("\ntest-mmap-fastpath: %d passed, %d failed - %s\n", passes, fails, diff --git a/tests/test-mmap-lazy.c b/tests/test-mmap-lazy.c index 8541b819..1e53c5b2 100644 --- a/tests/test-mmap-lazy.c +++ b/tests/test-mmap-lazy.c @@ -747,6 +747,81 @@ static void test_adjacent_region_extension(void) PASS(); } +static void test_large_retire_reuse(void) +{ + TEST("large multi-block retire preserves neighbors and fork zeroes"); + const size_t body_len = 96ULL << 20; + const size_t total_len = body_len + (6ULL << 20); + uint8_t *p = mmap(NULL, total_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + + uintptr_t aligned = + ((uintptr_t) p + BLOCK_2MIB * 2 - 1) & ~(uintptr_t) (BLOCK_2MIB - 1); + uint8_t *body = (uint8_t *) aligned; + size_t left_len = (size_t) (body - p); + size_t right_len = total_len - left_len - body_len; + if (left_len < BLOCK_2MIB || right_len < BLOCK_2MIB) { + FAIL("guard alignment"); + munmap(p, total_len); + return; + } + + memset(p, 0xa5, total_len); + if (munmap(body, body_len) != 0) { + FAIL("retire body"); + munmap(p, total_len); + return; + } + + /* MAP_FIXED is a metadata-reading slow path, so it must first drain the + * EL1 retirement. The 96 MiB dirty body spans many 2 MiB blocks, so this + * exercises retire-then-reuse at multi-block scale rather than the + * single-block case the smaller tests above already cover. + */ + uint8_t *q = mmap(body, body_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (q != body) { + FAIL("fixed reuse"); + munmap(p, left_len); + munmap(body + body_len, right_len); + return; + } + + bool ok = p[0] == 0xa5 && body[-1] == 0xa5 && body[body_len] == 0xa5 && + p[total_len - 1] == 0xa5; + for (size_t off = 0; ok && off < body_len; off += 4096) + ok = q[off] == 0; + + pid_t pid = -1; + int st = 0; + if (ok) + pid = fork(); + if (pid == 0) { + for (size_t off = 0; off < body_len; off += BLOCK_2MIB) { + if (q[off] != 0) + _exit(1); + } + q[123] = 0x77; + _exit(q[124] == 0 ? 0 : 2); + } + if (pid < 0 || waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || + WEXITSTATUS(st) != 0 || q[123] != 0) + ok = false; + + munmap(p, left_len); + munmap(q, body_len); + munmap(body + body_len, right_len); + if (!ok) { + FAIL("reuse leaked data, clobbered a neighbor, or broke fork"); + return; + } + PASS(); +} + int main(void) { test_huge_sparse(); @@ -765,6 +840,7 @@ int main(void) test_mt_first_touch(); test_claim_mutation_race(); test_adjacent_region_extension(); + test_large_retire_reuse(); SUMMARY("test-mmap-lazy"); return fails ? 1 : 0;