From 571441cd05a902765f794faeecf1060669334f9d Mon Sep 17 00:00:00 2001 From: YuutaTsubasa <81959102+YuutaTsubasa@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:28:10 +0800 Subject: [PATCH 1/3] Label checkpoints hide their permit call; register helpers use the generated base The generated code checkpoints at every label (172,524 of them, 17% loop heads). The permit call on the checkpoint's rare path is an ordinary call as far as the compiler knows, and the context is passed to every callee, so every guest register in a host register had to be stored before each label and reloaded after it; a loop without calls kept its counter and pointers in memory, each iteration a store and a reload. On AArch64 Linux and Android (SFR_HIDDEN_CHECKPOINT, CMake option, default ON) the call is now made from a non-volatile inline asm statement that names no memory, through a trampoline; the count the permit sets comes back in x0 and is written back where the compiler sees it, and the permit's exceptions are caught on the far side and rethrown. The checkpoint also tells the compiler the thread state lies apart from the context and guest memory (the generated sfr::guest_checkpoint(ctx) now passes the function's base through a macro), so such a loop keeps its count in a register too. The GPR save/restore hooks (6% of the AYN Thor main thread) found guest memory's base through active_memory: three dependent loads before the first register could move, their hottest instruction. They now use the base the generated code passes them, which is the same pointer. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 7 ++++ src/diagnostic_hooks.h | 69 +++++++++++++++++++++++++++++++++- src/diagnostic_main.cpp | 43 +++++++++++++++++++++ src/guest_register_hooks.cpp | 72 ++++++++++++++++++------------------ src/guest_registers.h | 21 ++++++++--- 5 files changed, 170 insertions(+), 42 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 298ca3ba..473eae1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,13 @@ option(SFR_DIRECT_MEMORY "Generated code accesses guest memory directly" ON) if(SFR_DIRECT_MEMORY) add_compile_definitions(SFR_DIRECT_MEMORY=1) endif() +# The generated code's label checkpoints call the permit from inline assembly +# where it is written, AArch64 Linux and Android (src/diagnostic_hooks.h, +# docs/performance.md). OFF builds the plain call there too, for comparison. +option(SFR_HIDDEN_CHECKPOINT "Label checkpoints hide their permit call from the compiler (AArch64 Linux/Android)" ON) +if(NOT SFR_HIDDEN_CHECKPOINT) + add_compile_definitions(SFR_HIDDEN_CHECKPOINT=0) +endif() if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/XenonRecomp/CMakeLists.txt") message(FATAL_ERROR "Run scripts/bootstrap.py to obtain the pinned XenonRecomp tools.") endif() diff --git a/src/diagnostic_hooks.h b/src/diagnostic_hooks.h index 84bb1fe7..57528ac9 100644 --- a/src/diagnostic_hooks.h +++ b/src/diagnostic_hooks.h @@ -155,8 +155,70 @@ struct HookScope { // The checkpoint at every loop label of the generated code, with the state // its context holds: through TLS, each label not dominated by another paid // the resolver again. +// +// SFR_HIDDEN_CHECKPOINT (docs/performance.md, 2026-10-20 generated code): the +// permit call on the rare path is made from inline assembly, so the compiler +// sees no call at a label. A call there, however rarely taken, may read and +// write the whole context as far as the compiler knows (the context is passed +// to every callee), so every guest register had to be stored before each +// label and loaded again after it, and a loop without calls kept its +// registers in memory: each iteration's counter, pointers and checkpoint +// count went through a store and a reload. The permit reads and writes no +// context (it hands the core over, notices cancellation and sets the count +// again) and the generated code keeps nothing else across a label that the +// permit changes. Its exceptions are caught on the far side and thrown again +// here. Only where the assembly is written: AArch64 Linux and Android, which +// keep no red zone below the stack pointer for the call to overwrite. +#if !defined(SFR_HIDDEN_CHECKPOINT) +#if defined(__aarch64__) && defined(__linux__) +#define SFR_HIDDEN_CHECKPOINT 1 +#else +#define SFR_HIDDEN_CHECKPOINT 0 +#endif +#endif +#if SFR_HIDDEN_CHECKPOINT +[[noreturn]] void rethrow_checkpoint_failure(); +#endif template -inline void guest_checkpoint(Context& ctx) { +inline void guest_checkpoint_at(Context& ctx, [[maybe_unused]] const uint8_t* base) { +#if SFR_HIDDEN_CHECKPOINT + if constexpr (requires { ctx.reserved.u64; }) { + // Bound by the enter_function every generated function starts with. + auto* entry = reinterpret_cast(ctx.reserved.u64); + // Thread-local state, never inside a context or guest memory: a loop + // without calls then keeps its count in a register too. (Stated + // before the test: inside it, the compiler stopped using it.) + __builtin_assume_separate_storage(entry, &ctx); + __builtin_assume_separate_storage(entry, base); + if (entry) [[likely]] { + if (entry->checkpoint_countdown) [[likely]] { + --entry->checkpoint_countdown; + return; + } + } + // sfr_checkpoint_trampoline (diagnostic_main.cpp) takes the state in + // x0 (zero for a context not bound yet, whose count is the thread's) + // and returns in x0 the count, with bit 63 set if the permit threw. + // The statement is not volatile and names no memory: a volatile one, + // or a memory operand, counts as reading and writing all memory, the + // context included. The count the permit set is therefore written + // back here, where the compiler sees it, and the permit is called + // without reading the count from memory (a loop may hold it in a + // register). The clobbers are every register a call may change but + // x18, the platform register, which a callee leaves alone. + register uint64_t result asm("x0") = reinterpret_cast(entry); + asm("bl sfr_checkpoint_trampoline" + : "+r"(result) + : + : "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", + "x15", "x16", "x17", "x30", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", + "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc"); + if (result >> 63) [[unlikely]] rethrow_checkpoint_failure(); + if (entry) entry->checkpoint_countdown = uint32_t(result); + return; + } +#endif GuestEntryState& entry = entry_thread_state(ctx).entry; if (entry.checkpoint_countdown) [[likely]] { --entry.checkpoint_countdown; @@ -164,6 +226,11 @@ inline void guest_checkpoint(Context& ctx) { } guest_checkpoint_permit(); } +// The generated code's sfr::guest_checkpoint(ctx) also hands over the +// function's own base parameter; sfr::guest_checkpoint() stays the plain one. +#define SFR_GUEST_CHECKPOINT_FORM(first, form, ...) form +#define guest_checkpoint(...) \ + SFR_GUEST_CHECKPOINT_FORM(__VA_ARGS__ __VA_OPT__(,) guest_checkpoint_at(__VA_ARGS__, base), guest_checkpoint()) // Every guest function entry; inline, as it runs millions of times a second. // A template so that the body sees the generated PPCContext, which is // defined after this header. diff --git a/src/diagnostic_main.cpp b/src/diagnostic_main.cpp index cd3a39b3..cfd7e4f9 100644 --- a/src/diagnostic_main.cpp +++ b/src/diagnostic_main.cpp @@ -814,6 +814,49 @@ void guest_checkpoint_permit() { // (an attached thread holds both for a short import or hook). if (core_permit) core_permit->checkpoint(handoff && execution_permit->detached()); } +#if SFR_HIDDEN_CHECKPOINT +// The label checkpoints' permit call (diagnostic_hooks.h): the trampoline +// below is reached by a bl from inline assembly and keeps the frame chain; +// the exception, if any, waits here until the checkpoint throws it again. +static thread_local std::exception_ptr checkpoint_failure; +extern "C" __attribute__((visibility("hidden"), used)) uint64_t sfr_checkpoint_permit_caught(uint64_t bound) noexcept { + try { + GuestEntryState& entry = guest_thread_state.entry; + // A context not bound yet counts down the thread's state here, as + // guest_checkpoint() does; a bound one came because its count ran out. + if (!bound && entry.checkpoint_countdown) --entry.checkpoint_countdown; + else guest_checkpoint_permit(); + return entry.checkpoint_countdown; + } catch (...) { + checkpoint_failure = std::current_exception(); + return uint64_t(1) << 63; + } +} +void rethrow_checkpoint_failure() { + std::exception_ptr failure = std::move(checkpoint_failure); + checkpoint_failure = nullptr; + std::rethrow_exception(failure); +} +__asm__( + ".text\n" + ".p2align 4\n" + ".globl sfr_checkpoint_trampoline\n" + ".hidden sfr_checkpoint_trampoline\n" + ".type sfr_checkpoint_trampoline, %function\n" + "sfr_checkpoint_trampoline:\n" + ".cfi_startproc\n" + "stp x29, x30, [sp, #-16]!\n" + ".cfi_def_cfa_offset 16\n" + ".cfi_offset x29, -16\n" + ".cfi_offset x30, -8\n" + "mov x29, sp\n" + "bl sfr_checkpoint_permit_caught\n" + "ldp x29, x30, [sp], #16\n" + ".cfi_def_cfa_offset 0\n" + "ret\n" + ".cfi_endproc\n" + ".size sfr_checkpoint_trampoline, . - sfr_checkpoint_trampoline\n"); +#endif [[noreturn]] void unsupported_function(PPCContext&, const char* name, uint32_t address, const char* reason) { guest_checkpoint(); diff --git a/src/guest_register_hooks.cpp b/src/guest_register_hooks.cpp index 58e912a0..e7136259 100644 --- a/src/guest_register_hooks.cpp +++ b/src/guest_register_hooks.cpp @@ -8,251 +8,251 @@ PPC_FUNC_IMPL(__imp____savegprlr_14); SFR_CONCURRENT_HOOK(__savegprlr_14) { sfr::enter_function(ctx, "__savegprlr_14", 0x82A56030); - if (!sfr::try_save_gprs<14>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<14>(*sfr::active_memory, ctx, base)) __imp____savegprlr_14(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_14); SFR_CONCURRENT_HOOK(__restgprlr_14) { sfr::enter_function(ctx, "__restgprlr_14", 0x82A56080); - if (!sfr::try_restore_gprs<14>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<14>(*sfr::active_memory, ctx, base)) __imp____restgprlr_14(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_15); SFR_CONCURRENT_HOOK(__savegprlr_15) { sfr::enter_function(ctx, "__savegprlr_15", 0x82A56034); - if (!sfr::try_save_gprs<15>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<15>(*sfr::active_memory, ctx, base)) __imp____savegprlr_15(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_15); SFR_CONCURRENT_HOOK(__restgprlr_15) { sfr::enter_function(ctx, "__restgprlr_15", 0x82A56084); - if (!sfr::try_restore_gprs<15>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<15>(*sfr::active_memory, ctx, base)) __imp____restgprlr_15(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_16); SFR_CONCURRENT_HOOK(__savegprlr_16) { sfr::enter_function(ctx, "__savegprlr_16", 0x82A56038); - if (!sfr::try_save_gprs<16>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<16>(*sfr::active_memory, ctx, base)) __imp____savegprlr_16(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_16); SFR_CONCURRENT_HOOK(__restgprlr_16) { sfr::enter_function(ctx, "__restgprlr_16", 0x82A56088); - if (!sfr::try_restore_gprs<16>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<16>(*sfr::active_memory, ctx, base)) __imp____restgprlr_16(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_17); SFR_CONCURRENT_HOOK(__savegprlr_17) { sfr::enter_function(ctx, "__savegprlr_17", 0x82A5603C); - if (!sfr::try_save_gprs<17>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<17>(*sfr::active_memory, ctx, base)) __imp____savegprlr_17(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_17); SFR_CONCURRENT_HOOK(__restgprlr_17) { sfr::enter_function(ctx, "__restgprlr_17", 0x82A5608C); - if (!sfr::try_restore_gprs<17>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<17>(*sfr::active_memory, ctx, base)) __imp____restgprlr_17(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_18); SFR_CONCURRENT_HOOK(__savegprlr_18) { sfr::enter_function(ctx, "__savegprlr_18", 0x82A56040); - if (!sfr::try_save_gprs<18>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<18>(*sfr::active_memory, ctx, base)) __imp____savegprlr_18(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_18); SFR_CONCURRENT_HOOK(__restgprlr_18) { sfr::enter_function(ctx, "__restgprlr_18", 0x82A56090); - if (!sfr::try_restore_gprs<18>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<18>(*sfr::active_memory, ctx, base)) __imp____restgprlr_18(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_19); SFR_CONCURRENT_HOOK(__savegprlr_19) { sfr::enter_function(ctx, "__savegprlr_19", 0x82A56044); - if (!sfr::try_save_gprs<19>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<19>(*sfr::active_memory, ctx, base)) __imp____savegprlr_19(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_19); SFR_CONCURRENT_HOOK(__restgprlr_19) { sfr::enter_function(ctx, "__restgprlr_19", 0x82A56094); - if (!sfr::try_restore_gprs<19>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<19>(*sfr::active_memory, ctx, base)) __imp____restgprlr_19(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_20); SFR_CONCURRENT_HOOK(__savegprlr_20) { sfr::enter_function(ctx, "__savegprlr_20", 0x82A56048); - if (!sfr::try_save_gprs<20>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<20>(*sfr::active_memory, ctx, base)) __imp____savegprlr_20(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_20); SFR_CONCURRENT_HOOK(__restgprlr_20) { sfr::enter_function(ctx, "__restgprlr_20", 0x82A56098); - if (!sfr::try_restore_gprs<20>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<20>(*sfr::active_memory, ctx, base)) __imp____restgprlr_20(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_21); SFR_CONCURRENT_HOOK(__savegprlr_21) { sfr::enter_function(ctx, "__savegprlr_21", 0x82A5604C); - if (!sfr::try_save_gprs<21>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<21>(*sfr::active_memory, ctx, base)) __imp____savegprlr_21(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_21); SFR_CONCURRENT_HOOK(__restgprlr_21) { sfr::enter_function(ctx, "__restgprlr_21", 0x82A5609C); - if (!sfr::try_restore_gprs<21>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<21>(*sfr::active_memory, ctx, base)) __imp____restgprlr_21(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_22); SFR_CONCURRENT_HOOK(__savegprlr_22) { sfr::enter_function(ctx, "__savegprlr_22", 0x82A56050); - if (!sfr::try_save_gprs<22>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<22>(*sfr::active_memory, ctx, base)) __imp____savegprlr_22(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_22); SFR_CONCURRENT_HOOK(__restgprlr_22) { sfr::enter_function(ctx, "__restgprlr_22", 0x82A560A0); - if (!sfr::try_restore_gprs<22>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<22>(*sfr::active_memory, ctx, base)) __imp____restgprlr_22(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_23); SFR_CONCURRENT_HOOK(__savegprlr_23) { sfr::enter_function(ctx, "__savegprlr_23", 0x82A56054); - if (!sfr::try_save_gprs<23>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<23>(*sfr::active_memory, ctx, base)) __imp____savegprlr_23(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_23); SFR_CONCURRENT_HOOK(__restgprlr_23) { sfr::enter_function(ctx, "__restgprlr_23", 0x82A560A4); - if (!sfr::try_restore_gprs<23>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<23>(*sfr::active_memory, ctx, base)) __imp____restgprlr_23(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_24); SFR_CONCURRENT_HOOK(__savegprlr_24) { sfr::enter_function(ctx, "__savegprlr_24", 0x82A56058); - if (!sfr::try_save_gprs<24>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<24>(*sfr::active_memory, ctx, base)) __imp____savegprlr_24(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_24); SFR_CONCURRENT_HOOK(__restgprlr_24) { sfr::enter_function(ctx, "__restgprlr_24", 0x82A560A8); - if (!sfr::try_restore_gprs<24>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<24>(*sfr::active_memory, ctx, base)) __imp____restgprlr_24(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_25); SFR_CONCURRENT_HOOK(__savegprlr_25) { sfr::enter_function(ctx, "__savegprlr_25", 0x82A5605C); - if (!sfr::try_save_gprs<25>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<25>(*sfr::active_memory, ctx, base)) __imp____savegprlr_25(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_25); SFR_CONCURRENT_HOOK(__restgprlr_25) { sfr::enter_function(ctx, "__restgprlr_25", 0x82A560AC); - if (!sfr::try_restore_gprs<25>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<25>(*sfr::active_memory, ctx, base)) __imp____restgprlr_25(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_26); SFR_CONCURRENT_HOOK(__savegprlr_26) { sfr::enter_function(ctx, "__savegprlr_26", 0x82A56060); - if (!sfr::try_save_gprs<26>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<26>(*sfr::active_memory, ctx, base)) __imp____savegprlr_26(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_26); SFR_CONCURRENT_HOOK(__restgprlr_26) { sfr::enter_function(ctx, "__restgprlr_26", 0x82A560B0); - if (!sfr::try_restore_gprs<26>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<26>(*sfr::active_memory, ctx, base)) __imp____restgprlr_26(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_27); SFR_CONCURRENT_HOOK(__savegprlr_27) { sfr::enter_function(ctx, "__savegprlr_27", 0x82A56064); - if (!sfr::try_save_gprs<27>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<27>(*sfr::active_memory, ctx, base)) __imp____savegprlr_27(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_27); SFR_CONCURRENT_HOOK(__restgprlr_27) { sfr::enter_function(ctx, "__restgprlr_27", 0x82A560B4); - if (!sfr::try_restore_gprs<27>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<27>(*sfr::active_memory, ctx, base)) __imp____restgprlr_27(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_28); SFR_CONCURRENT_HOOK(__savegprlr_28) { sfr::enter_function(ctx, "__savegprlr_28", 0x82A56068); - if (!sfr::try_save_gprs<28>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<28>(*sfr::active_memory, ctx, base)) __imp____savegprlr_28(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_28); SFR_CONCURRENT_HOOK(__restgprlr_28) { sfr::enter_function(ctx, "__restgprlr_28", 0x82A560B8); - if (!sfr::try_restore_gprs<28>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<28>(*sfr::active_memory, ctx, base)) __imp____restgprlr_28(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_29); SFR_CONCURRENT_HOOK(__savegprlr_29) { sfr::enter_function(ctx, "__savegprlr_29", 0x82A5606C); - if (!sfr::try_save_gprs<29>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<29>(*sfr::active_memory, ctx, base)) __imp____savegprlr_29(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_29); SFR_CONCURRENT_HOOK(__restgprlr_29) { sfr::enter_function(ctx, "__restgprlr_29", 0x82A560BC); - if (!sfr::try_restore_gprs<29>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<29>(*sfr::active_memory, ctx, base)) __imp____restgprlr_29(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_30); SFR_CONCURRENT_HOOK(__savegprlr_30) { sfr::enter_function(ctx, "__savegprlr_30", 0x82A56070); - if (!sfr::try_save_gprs<30>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<30>(*sfr::active_memory, ctx, base)) __imp____savegprlr_30(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_30); SFR_CONCURRENT_HOOK(__restgprlr_30) { sfr::enter_function(ctx, "__restgprlr_30", 0x82A560C0); - if (!sfr::try_restore_gprs<30>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<30>(*sfr::active_memory, ctx, base)) __imp____restgprlr_30(ctx, base); } PPC_FUNC_IMPL(__imp____savegprlr_31); SFR_CONCURRENT_HOOK(__savegprlr_31) { sfr::enter_function(ctx, "__savegprlr_31", 0x82A56074); - if (!sfr::try_save_gprs<31>(*sfr::active_memory, ctx)) + if (!sfr::try_save_gprs<31>(*sfr::active_memory, ctx, base)) __imp____savegprlr_31(ctx, base); } PPC_FUNC_IMPL(__imp____restgprlr_31); SFR_CONCURRENT_HOOK(__restgprlr_31) { sfr::enter_function(ctx, "__restgprlr_31", 0x82A560C4); - if (!sfr::try_restore_gprs<31>(*sfr::active_memory, ctx)) + if (!sfr::try_restore_gprs<31>(*sfr::active_memory, ctx, base)) __imp____restgprlr_31(ctx, base); } diff --git a/src/guest_registers.h b/src/guest_registers.h index 32dfd933..67e766f6 100644 --- a/src/guest_registers.h +++ b/src/guest_registers.h @@ -36,15 +36,18 @@ template bool eligible_stack(uint32_t sp) { // address without any check (SFR_DIRECT_MEMORY), and so does the original // body these replace: the save and restore then do the same, without asking // the checked path about a stack page every call (2-4% of a race's main -// thread on Windows and the AYN Thor). +// thread on Windows and the AYN Thor). The hooks pass the base the generated +// code was given (memory.base()): reading it from the GuestMemory through +// active_memory was three dependent loads before the first register could +// move, the hottest instruction of these helpers on the Thor. template -bool try_save_gprs(GuestMemory& memory, const Context& ctx) { +bool try_save_gprs(GuestMemory& memory, const Context& ctx, uint8_t* base) { const uint32_t sp = ctx.r1.u32; constexpr uint32_t distance = 8 * (33 - First); #if SFR_DIRECT_MEMORY if (GuestMemory::direct_guest_access) [[likely]] { if (sp & 7 || sp < distance) return false; - detail::save_gprs(memory.base() + (sp - distance), ctx, std::make_index_sequence<32 - First>{}); + detail::save_gprs(base + (sp - distance), ctx, std::make_index_sequence<32 - First>{}); return true; } #endif @@ -55,13 +58,13 @@ bool try_save_gprs(GuestMemory& memory, const Context& ctx) { return true; } template -bool try_restore_gprs(GuestMemory& memory, Context& ctx) { +bool try_restore_gprs(GuestMemory& memory, Context& ctx, const uint8_t* base) { const uint32_t sp = ctx.r1.u32; constexpr uint32_t distance = 8 * (33 - First); #if SFR_DIRECT_MEMORY if (GuestMemory::direct_guest_access) [[likely]] { if (sp & 7 || sp < distance) return false; - detail::restore_gprs(memory.base() + (sp - distance), ctx, std::make_index_sequence<32 - First>{}); + detail::restore_gprs(base + (sp - distance), ctx, std::make_index_sequence<32 - First>{}); return true; } #endif @@ -71,4 +74,12 @@ bool try_restore_gprs(GuestMemory& memory, Context& ctx) { detail::restore_gprs(bytes, ctx, std::make_index_sequence<32 - First>{}); return true; } +template +bool try_save_gprs(GuestMemory& memory, const Context& ctx) { + return try_save_gprs(memory, ctx, memory.base()); +} +template +bool try_restore_gprs(GuestMemory& memory, Context& ctx) { + return try_restore_gprs(memory, ctx, memory.base()); +} } From d3f76ccba0e6245e50102f927f3bc0a55f404ee7 Mon Sep 17 00:00:00 2001 From: YuutaTsubasa <81959102+YuutaTsubasa@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:45:07 +0800 Subject: [PATCH 2/3] Checkpoint only loop heads among the generated labels Every label of the generated code called sfr::guest_checkpoint(ctx), but only 29,100 of the 172,524 are loop heads (a label a goto at or after it returns to, switch cases included). A forward-only label is passed at most once between two function entries, which checkpoint themselves, so its checkpoint only counted down; two thirds of the checkpoint time on the AYN Thor's main thread was at such labels, and each one is a load, a decrement and a store on one count that every checkpoint of the thread chains through. The generator now emits the checkpoint only at loop heads. scripts/loop_head_checkpoints.py does the same to an existing output directory (the shipped generated code is not regenerated) and --check verifies the copy is the input less those lines. Guest-visible behaviour is unchanged; the permit's count now spans more guest work. Co-Authored-By: Claude Opus 5.5 --- scripts/generate_diagnostic.py | 23 ++++-- scripts/loop_head_checkpoints.py | 107 ++++++++++++++++++++++++++++ tests/test_diagnostic_generation.py | 22 +++--- tests/test_loop_head_checkpoints.py | 77 ++++++++++++++++++++ 4 files changed, 214 insertions(+), 15 deletions(-) create mode 100644 scripts/loop_head_checkpoints.py create mode 100644 tests/test_loop_head_checkpoints.py diff --git a/scripts/generate_diagnostic.py b/scripts/generate_diagnostic.py index 7c035601..c9c53415 100644 --- a/scripts/generate_diagnostic.py +++ b/scripts/generate_diagnostic.py @@ -1558,6 +1558,23 @@ def inspect_body(body, address, symbols, events, jump_tables=None): return end, reasons, details +def checkpoint_loop_heads(body): + """sfr::guest_checkpoint(ctx) after each label of one function that a goto at + or after it returns to (a loop head, a switch case included). A label only + reached forwards is passed at most once between two entries, which + checkpoint themselves (docs/performance.md, 2026-10-20: generated code; + scripts/loop_head_checkpoints.py does the same to existing output).""" + seen, heads = set(), set() + for line in body.split('\n'): + label = re.match(r'(loc_[0-9A-Fa-f]+):\r?$', line) + if label: + seen.add(label[1]) + heads.update(target for target in re.findall(r'goto (loc_[0-9A-Fa-f]+);', line) if target in seen) + return re.sub(r'^(loc_[0-9A-Fa-f]+):(\r?\n)', + lambda match: match[0] + ('\tsfr::guest_checkpoint(ctx);' + match[2] if match[1] in heads else ''), + body, flags=re.MULTILINE) + + def generate(input_dir, log_path, output_dir, jump_table_path=None): input_dir, log_path, output_dir = map(lambda p: Path(p).resolve(), (input_dir, log_path, output_dir)) @@ -1596,7 +1613,7 @@ def generate(input_dir, log_path, output_dir, jump_table_path=None): 'limitations': ['Requires checked scalar memory and indirect-call hooks.', 'mftb requires the runtime monotonic time-base hook; raw host RDTSC is not guest time.', 'Reservation hooks require exclusive guest execution with no intervening ordinary stores or imports.', - 'Every retained branch label calls the runtime execution safe point, including arithmetic-only loops.', + 'Every function entry and loop head (a label a later goto returns to) calls the runtime execution safe point, including arithmetic-only loops.', 'Full vector memory hooks require 16-byte checked access and reversed register bytes.', 'Vector word stores use checked four-byte access and the same reversed register representation.', 'Remaining instruction translations are not proven correct.', @@ -1688,9 +1705,7 @@ def generate(input_dir, log_path, output_dir, jump_table_path=None): rewritten_body = rewrite_cache_zero(rewritten_body, addresses[name])[0] rewritten_body = rewrite_native_resource_coherency(rewritten_body, addresses[name], body) rewritten_body = rewrite_country_ctr(rewritten_body, addresses[name], body) - rewritten_body = re.sub(r'^(loc_[0-9A-Fa-f]+:)(\r?\n)', - lambda match: match[1] + match[2] + '\tsfr::guest_checkpoint(ctx);' + match[2], - rewritten_body, flags=re.MULTILINE) + rewritten_body = checkpoint_loop_heads(rewritten_body) chunks.append((rewritten_body + '}').replace( 'PPC_FUNC_PROLOGUE();', 'PPC_FUNC_PROLOGUE();\n\tSFR_FAST_PATH();\n\tsfr::enter_function(ctx, ' + json.dumps(name) + f', 0x{addresses[name]:08X});', 1)) diff --git a/scripts/loop_head_checkpoints.py b/scripts/loop_head_checkpoints.py new file mode 100644 index 00000000..faf835c2 --- /dev/null +++ b/scripts/loop_head_checkpoints.py @@ -0,0 +1,107 @@ +"""Copy generated diagnostic code keeping label checkpoints only at loop heads. + +The generated code calls sfr::guest_checkpoint(ctx) after every label (172,524 +in the shipped output). A label that no goto at or after it in the same +function jumps to is reached at most once between two function entries, so its +checkpoint only counts down towards the next permit call; the function's entry +(sfr::enter_function) and its loop heads still checkpoint, so every path that +repeats still passes one. Guest-visible behaviour is unchanged: a checkpoint +reads and writes no guest state. What changes is how much guest work the +countdown spans (docs/performance.md, 2026-10-20: generated code). + +This works on an existing output directory, without regenerating it (the +shipped generated code must not be regenerated with a newer generator). + +usage: python scripts/loop_head_checkpoints.py + python scripts/loop_head_checkpoints.py --check +The output holds every file of the input; only the ppc_recomp.*.cpp files lose +checkpoint lines, and --check verifies that the output is exactly the input +less those lines. +""" +import re +import shutil +import sys +from pathlib import Path + +CHECKPOINT = '\tsfr::guest_checkpoint(ctx);' +LABEL = re.compile(r'(loc_[0-9A-Fa-f]+):\r?$') +GOTO = re.compile(r'goto (loc_[0-9A-Fa-f]+);') +GENERATED = re.compile(r'ppc_recomp\.\d+\.cpp') + + +def forward_checkpoints(lines): + """Indices of checkpoint lines that follow a label no goto returns to.""" + drop = [] + starts = [i for i, line in enumerate(lines) if line.startswith('PPC_FUNC_IMPL(')] + [len(lines)] + for begin, end in zip(starts, starts[1:]): + labels, loops = {}, set() + for i in range(begin, end): + line = lines[i] + match = LABEL.match(line) + if match: + labels[match[1]] = i + for target in GOTO.findall(line): + if target in labels: # defined at or above this goto: it can come back + loops.add(target) + for name, i in labels.items(): + if name not in loops and i + 1 < end and lines[i + 1].rstrip('\r') == CHECKPOINT: + drop.append(i + 1) + return drop + + +def transform(text): + lines = text.split('\n') + drop = set(forward_checkpoints(lines)) + kept = sum(1 for line in lines if line.rstrip('\r') == CHECKPOINT) - len(drop) + return '\n'.join(line for i, line in enumerate(lines) if i not in drop), len(drop), kept + + +def check(source, output): + """The output is the source less checkpoint lines, in order; nothing else.""" + a, b = source.split('\n'), output.split('\n') + j = 0 + for line in a: + if j < len(b) and b[j] == line: + j += 1 + elif line.rstrip('\r') != CHECKPOINT: + return False + return j == len(b) + + +def main(argv): + verify = argv[:1] == ['--check'] + if verify: + argv = argv[1:] + if len(argv) != 2: + sys.exit(__doc__) + source, output = Path(argv[0]), Path(argv[1]) + if source.resolve() == output.resolve(): + sys.exit('the output must be another directory') + output.mkdir(parents=True, exist_ok=True) + removed = kept = 0 + for path in sorted(source.iterdir()): + if not path.is_file(): + continue + target = output / path.name + if not GENERATED.fullmatch(path.name): + if verify: + if target.read_bytes() != path.read_bytes(): + sys.exit(f'{path.name} differs') + else: + shutil.copyfile(path, target) + continue + text = path.read_bytes().decode('utf-8') + result, dropped, left = transform(text) + removed += dropped + kept += left + if verify: + written = target.read_bytes().decode('utf-8') + if written != result or not check(text, written): + sys.exit(f'{path.name} is not the input less forward checkpoints') + else: + target.write_bytes(result.encode('utf-8')) + print(f'label checkpoints kept {kept} (loop heads), removed {removed}') + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tests/test_diagnostic_generation.py b/tests/test_diagnostic_generation.py index 4d9cd1b5..2cb4425f 100644 --- a/tests/test_diagnostic_generation.py +++ b/tests/test_diagnostic_generation.py @@ -144,7 +144,7 @@ def test_format_ctr_restores_local_targets_preserving_calls_table_and_lhzu(self) self.assertEqual([(int(a, 16), int(b, 16)) for a, b in cases], [(t, t) for t in targets]) for target in targets: self.assertEqual(result.count(f'loc_{target:X}:'), 1) - self.assertIn(f'loc_{target:X}:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn(f'loc_{target:X}:\n\tsfr::guest_checkpoint(ctx);', result) self.assertIn('default: throw sfr::RuntimeStop("format-ctr-target", ctx.ctr.u32, ' '"unrecognized local format branch target");', result) self.assertEqual([m[1].rstrip('\r') for m in diagnostic.INSTRUCTION.finditer(result)], @@ -259,8 +259,8 @@ def test_country_ctr_restores_38_exact_local_targets_and_checkpoints(self): [(target, target) for target in targets]) for target in targets: self.assertEqual(result.count(f'loc_{target:X}:'), 1) - self.assertIn(f'loc_{target:X}:\n\tsfr::guest_checkpoint(ctx);', result) - self.assertIn(f'loc_{targets[-1] + 4:X}:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn(f'loc_{target:X}:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn(f'loc_{targets[-1] + 4:X}:\n\tsfr::guest_checkpoint(ctx);', result) self.assertIn('default: throw sfr::RuntimeStop("country-ctr-target", ctx.ctr.u32, ' '"unrecognized local country branch target");', result) self.assertNotIn('PPC_CALL_INDIRECT_FUNC(ctx.ctr.u32);', result) @@ -356,7 +356,7 @@ def test_partial_vector_loads_rewrite_four_variants_preserving_ea_and_checkpoint for hook, reg, ea in [('left', 63, 'ctx.r3.u32'), ('right', 127, 'ctx.r31.u32 + ctx.r0.u32'), ('left', 31, 'ctx.r31.u32'), ('right', 0, 'ctx.r3.u32 + ctx.r3.u32')]: self.assertIn(f'temp.u32 = {ea};\n\tsfr::load_vector_{hook}(temp.u32, ctx.v{reg}.u8);', result) - self.assertIn('loc_1010:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn('loc_1010:\n\tsfr::guest_checkpoint(ctx);', result) self.assertEqual(len(list(diagnostic.INSTRUCTION.finditer(result))), 5) self.assertNotIn('simde_mm_', result) for path, original in originals.items(): @@ -527,7 +527,7 @@ def test_native_coherency_emission_preserves_comments_cpu_rewrites_and_end_check self.assertNotIn('ctx.r3.u64 = 1;', result) self.assertNotIn('loc_1008:', result) self.assertNotIn('goto loc_100C;', result) - self.assertIn('loc_100C:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn('loc_100C:\n\tsfr::guest_checkpoint(ctx);', result) self.assertIn('\treturn;', result) self.assertEqual([m[1].rstrip('\r') for m in diagnostic.INSTRUCTION.finditer(result)], [m[1].rstrip('\r') for m in diagnostic.INSTRUCTION.finditer(raw)]) @@ -591,7 +591,7 @@ def test_addme_logged_empty_blocks_preserve_sources_aliases_and_labels(self): self.assertEqual(report['counts']['retained_functions'], 1) for d, a in registers: self.assertIn(f'sfr::addme(ctx.r{d}.u64, ctx.r{a}.u64, ctx.xer.ca);', result) - self.assertIn('ctx.xer.ca);\nloc_1004:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn('ctx.xer.ca);\nloc_1004:\n\tsfr::guest_checkpoint(ctx);', result) self.assertEqual(source.read_bytes(), original) def test_addme_rejects_variants_nonempty_blocks_and_conflicting_logs(self): @@ -639,7 +639,7 @@ def test_addc_logged_empty_blocks_preserve_sources_aliases_and_labels(self): self.assertEqual(report['counts']['retained_functions'], 1) for d, a, b in registers: self.assertIn(f'sfr::addc(ctx.r{d}.u64, ctx.r{a}.u64, ctx.r{b}.u64, ctx.xer.ca);', result) - self.assertIn('ctx.xer.ca);\nloc_1004:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn('ctx.xer.ca);\nloc_1004:\n\tsfr::guest_checkpoint(ctx);', result) self.assertEqual(source.read_bytes(), original) def test_addc_rejects_variants_nonempty_blocks_and_conflicting_logs(self): @@ -688,7 +688,7 @@ def test_dcbz_exact_memory_blocks_preserve_operands_labels_and_input(self): for ra, rb in pairs: ea = ('' if ra in ('0', 'r0') else f'ctx.{ra}.u32 + ') + f'ctx.r{rb}.u32' self.assertIn(f'sfr::zero_cache_block(uint32_t({ea}));', result) - self.assertIn('loc_1010:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn('loc_1010:\n\tsfr::guest_checkpoint(ctx);', result) self.assertNotIn('memset(base', result) self.assertEqual(source.read_bytes(), original) @@ -732,7 +732,7 @@ def test_bdzf_exact_logged_blocks_preserve_all_condition_bits_and_checkpoints(se self.assertEqual(report['counts']['retained_bdzf'], 32) for cr, bit in bits: self.assertIn(f'if (sfr::branch_counter_zero_false(ctx.ctr.u64, ctx.cr{cr}.{bit})) goto loc_1080;', result) - self.assertIn('loc_1080:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn('loc_1080:\n\tsfr::guest_checkpoint(ctx);', result) self.assertEqual(source.read_bytes(), original) self.assertEqual(report['log']['unrecognized_events'], 32) @@ -782,7 +782,7 @@ def test_arithmetic_loop_labels_include_execution_safe_points(self): result = (self.output / 'ppc_recomp.0.cpp').read_text() self.assertEqual(report['counts']['retained_functions'], 1) self.assertIn('loc_1000:\n\tsfr::guest_checkpoint(ctx);', result) - self.assertIn('loc_1004:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn('loc_1004:\n\tsfr::guest_checkpoint(ctx);', result) self.assertIn('ctx.r3.s64 = ctx.r3.s64 + 1;', result) self.assertIn('goto loc_1000;', result) @@ -1276,7 +1276,7 @@ def test_eqv_real_pc_resolves_duplicate_logs_once_and_preserves_source(self): result, report = self.run_generation() self.assertEqual(result.count(expected), 1) self.assertIn('\t__nop();', result) - self.assertIn('loc_8222B538:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertNotIn('loc_8222B538:\n\tsfr::guest_checkpoint(ctx);', result) self.assertEqual(source.read_bytes(), before) self.assertEqual(report['counts']['retained_eqv'], 1) self.assertEqual(report['counts']['retained_functions'], 1) diff --git a/tests/test_loop_head_checkpoints.py b/tests/test_loop_head_checkpoints.py new file mode 100644 index 00000000..9f1658db --- /dev/null +++ b/tests/test_loop_head_checkpoints.py @@ -0,0 +1,77 @@ +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +import loop_head_checkpoints as loops + +SOURCE = '''#include "diagnostic_hooks.h" + +PPC_FUNC_IMPL(__imp__sub_1000) { +\tPPC_FUNC_PROLOGUE(); +\tsfr::enter_function(ctx, "sub_1000", 0x1000); +loc_1004: +\tsfr::guest_checkpoint(ctx); +\tif (ctx.cr6.eq) goto loc_1010; +loc_1008: +\tsfr::guest_checkpoint(ctx); +\t--ctx.ctr.u64; +\tif (ctx.ctr.u32 != 0) goto loc_1008; +loc_1010: +\tsfr::guest_checkpoint(ctx); +\tswitch (ctx.r3.u32) { +\tcase 0: goto loc_1004; +\tcase 1: goto loc_1018; +\t} +loc_1018: +\tsfr::guest_checkpoint(ctx); +\treturn; +} + +PPC_FUNC_IMPL(__imp__sub_1020) { +\tPPC_FUNC_PROLOGUE(); +\tsfr::enter_function(ctx, "sub_1020", 0x1020); +\tgoto loc_1008; +loc_1008: +\tsfr::guest_checkpoint(ctx); +\treturn; +} +''' + + +class LoopHeadCheckpointTest(unittest.TestCase): + def test_keeps_only_labels_a_later_goto_returns_to(self): + result, removed, kept = loops.transform(SOURCE) + # sub_1000: loc_1004 (switch case 0 below it) and loc_1008 (bdnz) are loop heads; + # loc_1010 and loc_1018 are only reached forwards. + self.assertIn('loc_1004:\n\tsfr::guest_checkpoint(ctx);', result) + self.assertIn('loc_1008:\n\tsfr::guest_checkpoint(ctx);\n\t--ctx.ctr.u64;', result) + self.assertIn('loc_1010:\n\tswitch', result) + self.assertIn('loc_1018:\n\treturn;', result) + # a label of another function with the same name is judged in its own function + self.assertIn('\tgoto loc_1008;\nloc_1008:\n\treturn;', result) + self.assertEqual((removed, kept), (3, 2)) + self.assertTrue(loops.check(SOURCE, result)) + + def test_check_rejects_any_other_change(self): + result, _, _ = loops.transform(SOURCE) + self.assertFalse(loops.check(SOURCE, result.replace('ctx.ctr.u32 != 0', 'ctx.ctr.u32 == 0'))) + self.assertFalse(loops.check(SOURCE, result.replace('\treturn;\n}', '}', 1))) + + def test_directory_copy_and_check(self): + with tempfile.TemporaryDirectory() as work: + source, output = Path(work) / 'in', Path(work) / 'out' + source.mkdir() + (source / 'ppc_recomp.0.cpp').write_text(SOURCE, newline='') + (source / 'ppc_context.h').write_text('// header\n', newline='') + loops.main([str(source), str(output)]) + loops.main(['--check', str(source), str(output)]) + self.assertEqual((output / 'ppc_context.h').read_text(), '// header\n') + (output / 'ppc_recomp.0.cpp').write_text(SOURCE, newline='') + with self.assertRaises(SystemExit): + loops.main(['--check', str(source), str(output)]) + + +if __name__ == '__main__': + unittest.main() From af27e1a3ff8a75a7df68a429a8058addac442378 Mon Sep 17 00:00:00 2001 From: YuutaTsubasa <81959102+YuutaTsubasa@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:04:51 +0800 Subject: [PATCH 3/3] docs: generated code on the Thor main thread, checkpoint A/B Profile of the generated code (statement and pattern breakdown), what was changed and measured (hidden label checkpoints, the helpers' base, loop-head-only checkpoints), how to ship the loop-head part with the current generated code, and the remaining proposals ranked. Co-Authored-By: Claude Opus 5.5 --- docs/checkpoint-interval.md | 5 +- docs/performance.md | 175 ++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 1 deletion(-) diff --git a/docs/checkpoint-interval.md b/docs/checkpoint-interval.md index 4d5380c5..2c7050c9 100644 --- a/docs/checkpoint-interval.md +++ b/docs/checkpoint-interval.md @@ -1,7 +1,10 @@ # Guest checkpoint interval Generated guest code visits a checkpoint at function entries and loop labels. -Calling the execution permit less often reduces host CPU overhead. Every +Output of the current generator (or of `scripts/loop_head_checkpoints.py`) +checkpoints only labels a loop returns to, so the same interval spans more +guest work than with checkpoints at every label (performance.md, 2026-10-20: +generated code). Calling the execution permit less often reduces host CPU overhead. Every platform defaults to 256 entries between permit calls: Windows since Issue #33 (knuckleslee's checkpoint-interval contribution), Android, Linux and macOS since the AYN Thor measurement below (they kept 32 until then). diff --git a/docs/performance.md b/docs/performance.md index 37691f16..b3e97d38 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -2329,3 +2329,178 @@ full timing did. Measured but left alone: the graphics lock per hook (an uncontended mutex pair, ~2%), the constants swap (8 KB a draw), the pacing spin on Android (a 2 ms margin, as sleeps overshoot by ~0.8 ms there). + +## 2026-10-20: Generated code on the main thread + +After the runtime work above, generated guest code is the largest single part of +a race frame's main thread on the AYN Thor. This pass looked at what that code +compiles to and what is cheaper without regenerating it (releases build the +shipped generated code, which a newer generator would not reproduce). + +**Profile.** simpleperf on the Thor (release 0.10.13 `libmain.so` repackaged as +profileable, Dolphin Resort moving race `CLAUDE_INPUT=kicks`, frames ~12,000 to +14,000, 30 s at 2 kHz, frame metrics off), main thread 50,164 samples (84% +busy): + +| Part of the main thread | Share | +| --- | --- | +| Generated guest code (`__imp__sub_*`) | 32.4% | +| GPR/FPR save-restore helpers (`__savegprlr_*`, `__restgprlr_*`, `try_*_gprs`) | 6.2% | +| Hand-written hooks (`sub_824F56E8` gather 4.8%) | 5.7% | +| Entry/permit/indirect (`enter_function_observed` 1.7%, `call_indirect` 1.1%) | 3.6% | +| Other runtime (graphics HLE, memcpy, libc, kernel, driver) | the rest | + +The generated code is flat: 810 functions were sampled; the ten hottest hold +10% of the thread, the hundred hottest 25%. Nothing is worth hand-optimising +one function at a time; what matters is how all of them are compiled. + +Each sample's address was mapped to the generated statement through the debug +line tables (the outermost inlined frame in a `ppc_recomp.*.cpp`), and the +statement to the PowerPC instruction in the comment above it: + +| Statement | Share of generated code | +| --- | --- | +| guest loads (`PPC_LOAD_*`) | 19.7% | +| integer moves and arithmetic (mostly `mr` and `addi` reloading the context after a call) | 18.4% | +| CR compares (`ctx.crN.compare`) | 15.5% | +| label checkpoints (`sfr::guest_checkpoint(ctx)`) | 14.2% | +| `sfr::enter_function` | 10.1% | +| guest stores | 7.1% | +| function prologue/epilogue | 5.3% | +| `ctx.lr` writes, branches, vector, scalar FP, calls | 1-2% each | +| FPSCR flush-mode switches (`msr fpcr`) | 0.8% | +| XER, indirect calls | 0.5%, 0.3% | + +Samples land on the instruction after a stall, so the compares mostly carry +the latency of the guest load that feeds them: 30% of these samples follow a +load from guest memory, 17% a reload from the context or the thread state. +Statically there are 30,726 functions, 172,524 labels with a checkpoint (of +which 29,100, 17%, are loop heads), and 35,403 backward branches, half of them +with a call inside the loop; 35% of the generated-code samples lie inside a +loop. + +**What the patterns cost, and what was done.** + +- *Label checkpoints* (14% of generated code, plus what they do to the code + around them). The permit call on the checkpoint's rare path is an ordinary + call as far as the compiler knows, and the context is passed to every + callee, so every guest register held in a host register had to be stored + before each label and loaded again after it, and a loop without calls kept + its counter and pointers in memory: each iteration a store and a reload, + plus the count's own load-decrement-store. All the checkpoints of a thread + also form one chain through that count. An upper bound (label checkpoints + removed altogether, not shippable) is in the table below. Two changes: + `SFR_HIDDEN_CHECKPOINT` (CMake option, on by default; AArch64 Linux and + Android only) makes the rare call from a non-volatile `asm` statement that + names no memory, through `sfr_checkpoint_trampoline`; the count the permit + sets comes back in x0 and is written where the compiler sees it, and the + permit's exceptions are caught on the far side and rethrown. With + `__builtin_assume_separate_storage` on the thread state against the context + and the guest base (the generated `sfr::guest_checkpoint(ctx)` now passes the + function's `base` through a macro), a loop without calls keeps its guest + registers and its count in host registers. Second, only loop heads keep a + checkpoint (`scripts/loop_head_checkpoints.py` for existing output, and the + generator for new output): a forward-only label is passed at most once + between two function entries, which checkpoint themselves, and two thirds of + the checkpoint samples were at such labels. Guest-visible behaviour is the + same; the permit's count of 256 now spans more guest work. +- *GPR save/restore helpers* (6.2%). Their hottest instruction waited for + `active_memory->base()`: a GOT load, the pointer, then the field, before the + first register could move. They now use the `base` the generated code passes + them (the same pointer). +- *Byte swaps.* `rev` is one cycle on the Cortex-X3 and the compiler already + folds a load-swap-swap-store copy into a plain copy (seen in the copy loops); + nothing to gain. +- *Volatile guest accesses* must stay (guest-concurrency rules: threads run in + parallel); on AArch64 they are plain loads and stores, and they did not stop + the compiler from keeping context fields in registers once the checkpoint + call was hidden. +- *CR/XER bookkeeping.* A compare stores four CR bytes and reads XER.SO only + where the CR field must be in memory: before a call, a return, or (until + now) a label. Hiding the checkpoint call removes the label case; the rest + needs the generator's `cr_as_local`, i.e. regeneration. +- *FPSCR.* 29,633 flush-mode switches in the source, 0.8% of the generated + time (0.26% of the thread). `-ffp-model=strict` stays: it is what keeps the + compiler from moving floating-point work across the `msr fpcr` statements. +- *Indirect calls.* `call_indirect` is 1.1% of the thread: a call, an 18 MB + table indexed by address/4 (cache and TLB misses) and an audit flag. An + inline lookup with a smaller table, or per-site caches, could save perhaps + 0.1 ms; not done. +- *Vector code* is 1.4% of generated code; SIMDe's NEON paths are fine. +- *Context spills around calls* are inherent to the context-in-memory model; + XenonRecomp's `non_volatile_as_local`, `cr_as_local` and the like (Unleashed + Recompiled uses them) would remove most of them, but need regeneration. + +**Compiler and linker flags.** + +- `-Wl,-Bsymbolic` (data as well as functions): lld does not turn the + non-adjacent `adrp`/`ldr` GOT pairs into direct accesses, so no instruction + changes; not pursued. `-fvisibility=hidden` would make globals direct + (`adrp`+`add`) across the runtime, but needs `SDL_main` and the JNI entry + points exported explicitly; untested. +- `-ffp-model=strict` stays (see FPSCR). +- A hot-first symbol order (`--symbol-ordering-file` from the profile) relinks + in a second and moves the 2,000 sampled functions to the start of the 50 MB + text; built but not measured (proposal 5 below). +- ThinLTO could inline the save/restore helpers (6%) into their callers but + means a link over 30,000 functions; PGO for the generated code would bring + layout and branch weights but needs a profile collected on the device. Both + are larger projects. + +The hidden checkpoint makes the text 5% larger (49.6 to 52.3 MB: values live +in callee-saved registers, and the rare path spills); with loop-head-only +checkpoints it is 43.6 MB. + +**Thor A/B** (same harness as above: Dolphin Resort, `CLAUDE_INPUT=kicks`, +frame metrics off, HOST_THREADS over frames 13,000-21,000, 67 windows of 120 +frames; every build is this branch's source on the shipped generated code, +versionCode 61, runs interleaved with other agents' runs). `cpu_ms` counts +the pacing spin too, so it flattens near the budget; the forced-skip runs +(`SFR_TEST_RACE_DRAW_MS=10`: drawn frames take 10 ms longer, so 55-60% of +frames go undrawn) give `undrawn_ms` over ~4,700 frames each, the frame's work +without drawing: + +| Build | drawn_ms | undrawn_ms | cpu_ms | frames skipped of 8040 | +| --- | --- | --- | --- | --- | +| baseline (0.10.13 source) | 16.13 / 15.97 | 9.78 / 9.86 | 14.47 / 14.33 | 320 / 211 | +| hidden checkpoint + helpers' base | 15.75 / 16.03 | 8.90 / 9.40 | 13.93 / 14.18 | 80 / 160 | +| + loop-head checkpoints only | 15.96 | 8.31 | 13.81 | 39 | +| no label checkpoints at all (bound, not shippable) | 16.01 | 7.28 | 13.52 | 12 | + +| Build, `SFR_TEST_RACE_DRAW_MS=10` | undrawn_ms | cpu_ms | frames skipped | +| --- | --- | --- | --- | +| baseline | 9.53 / 8.99 | 11.38 / 10.88 | 4993 / 4728 | +| hidden checkpoint + helpers' base | 9.00 | 11.13 | 4876 | +| + loop-head checkpoints only | 8.41 / 8.60 | 10.64 / 10.86 | 4628 / 4703 | + +So the two together take about 0.75 ms (8%) off an undrawn frame's work and +0.6 ms of main-thread CPU off a drawn one, and the skipped frames of a normal +race fall from 200-300 to ~40. Races ran at speed 0.999 and drew correctly +(screenshots at presents 14,400 and 19,200). On Windows (clang-cl, where only +the loop-head part and the helpers' base apply) the Vulkan race with the +validation layer loaded reported nothing, and the D3D12 race ran to its +present limit, on both the shipped and the loop-head-only code; unit tests +pass on Windows (165) and Linux (158), Python 356 (11 skipped for want of the +game files). + +**Shipping the loop-head part** needs the release's generated directory +copied through `python scripts/loop_head_checkpoints.py ` +(`--check` verifies the copy) and the build pointed at the copy with +`-DSFR_DIAGNOSTIC_DIR`. The generator now emits only loop-head checkpoints +itself, so new output needs nothing. + +**Proposals not done, by expected gain and risk:** + +1. Regenerate with XenonRecomp's local-register options (`non_volatile_as_local`, + `cr_as_local`, `ctr_as_local`, `xer_as_local`): removes most context + traffic around calls and CR stores (the "integer/other" and compare shares + above). Largest gain, but a regeneration that has to be validated against + the shipped code and the runtime's hooks, which read the context. +2. ThinLTO (or inline wrappers) for the GPR save/restore helpers, 6% of the + thread: each costs a call, `enter_function` and the save itself. +3. `call_indirect`: an inline lookup or per-site caches instead of the 18 MB + table (~1% of the thread). +4. `-fvisibility=hidden` for the game library (globals by `adrp`+`add` + instead of through the GOT), after exporting `SDL_main` and the JNI entries. +5. A hot-first symbol order or PGO layout for the 50 MB text (untested here: + the Thor time went to the checkpoint A/B).