From 271529aa1047aeb900b3fd8b29035e2aced9f6dd Mon Sep 17 00:00:00 2001 From: quake Date: Sat, 5 Sep 2026 14:30:59 +0900 Subject: [PATCH] bench(core): add memory receipts --- docs/bench/core-memory-2026-09-05.json | 28 + docs/bench/core-memory-2026-09-05.md | 39 ++ docs/bench/core-memory-ppsspp-2026-09-05.json | 46 ++ docs/bench/core-memory-ppsspp-2026-09-05.md | 77 +++ engine/core/examples/membench.rs | 562 ++++++++++++++++++ hosts/psp/src/main.rs | 73 ++- hosts/psp/src/switch.rs | 4 + .../references/metrics.md | 3 + tests/core-memory-bench.test.ts | 138 +++++ tests/psp-bench-parser.test.ts | 39 ++ tests/psp-bench.test.ts | 62 ++ tools/bench-ppsspp-parser.ts | 58 ++ tools/bench-ppsspp.ts | 52 +- 13 files changed, 1137 insertions(+), 44 deletions(-) create mode 100644 docs/bench/core-memory-2026-09-05.json create mode 100644 docs/bench/core-memory-2026-09-05.md create mode 100644 docs/bench/core-memory-ppsspp-2026-09-05.json create mode 100644 docs/bench/core-memory-ppsspp-2026-09-05.md create mode 100644 engine/core/examples/membench.rs create mode 100644 tests/core-memory-bench.test.ts create mode 100644 tests/psp-bench-parser.test.ts create mode 100644 tests/psp-bench.test.ts create mode 100644 tools/bench-ppsspp-parser.ts diff --git a/docs/bench/core-memory-2026-09-05.json b/docs/bench/core-memory-2026-09-05.json new file mode 100644 index 000000000..2cb729de6 --- /dev/null +++ b/docs/bench/core-memory-2026-09-05.json @@ -0,0 +1,28 @@ +{ + "command": "cargo run --manifest-path engine/core/Cargo.toml --example membench --quiet", + "toolchain": { + "rustc": "rustc 1.97.1 (8bab26f4f 2026-07-14)", + "cargo": "cargo 1.97.1 (c980f4866 2026-06-30)" + }, + "git_revision": "b7dd46f", + "origin_main_revision": "2d20ddad228db52fa29ad7d7d06f44e8672164af", + "benchmark_implementation_revision": "0821ddf", + "workload": { + "shape": "99 nodes; 24 steady ticks; 8 rounds of 4-node subtree churn; 16 text ticks; 12 burst ticks", + "text_mode": "atlas", + "texture_mode": "atlas" + }, + "receipt": { + "peak_requested_bytes": "29591", + "final_requested_bytes": "10793", + "allocation_count": "11682", + "total_allocated_bytes": "6225569", + "avg_layout_us": "1210", + "max_layout_us": "5163", + "nodes": "99", + "structural_relayouts": "16", + "text_mode": "atlas", + "texture_mode": "atlas", + "drawlist_checksum": "cc6a0b00efdba151" + } +} diff --git a/docs/bench/core-memory-2026-09-05.md b/docs/bench/core-memory-2026-09-05.md new file mode 100644 index 000000000..36a027b19 --- /dev/null +++ b/docs/bench/core-memory-2026-09-05.md @@ -0,0 +1,39 @@ +# Core Memory Benchmark Baseline + +The benchmark harness was based on the latest `origin/main` revision +`2d20ddad228db52fa29ad7d7d06f44e8672164af`; the first receipt was measured at +harness commit `b7dd46f`. The benchmark implementation was introduced +separately at `0821ddf`. `git_revision` identifies the revision where the +receipt was measured, while `origin_main_revision` identifies the source +revision used as the harness base. This receipt was not measured from +`origin/main` itself. + +The byte values below are **allocator-independent requested-byte metrics** from +the counting allocator. They are not process RSS, committed pages, or PSP arena +high-water measurements. Timing values are host measurements and are excluded +from deterministic receipt comparisons. + +## Reproduction + +```text +cargo run --manifest-path engine/core/Cargo.toml --example membench --quiet +``` + +Toolchain: `rustc 1.97.1 (8bab26f4f 2026-07-14)`, `cargo 1.97.1 (c980f4866 2026-06-30)`. + +Workload shape: 99 nodes; 24 steady ticks; 8 rounds of 4-node subtree churn; +16 text ticks; 12 burst ticks. Text and texture modes are both `atlas`. + +## Receipt + +| Field | Value | Meaning | +| --- | ---: | --- | +| `peak_requested_bytes` | 29591 | Peak requested bytes | +| `final_requested_bytes` | 10793 | Final live requested bytes | +| `allocation_count` | 11682 | Count of measured allocation events | +| `total_allocated_bytes` | 6225569 | Total requested bytes across measured allocation events | +| `avg_layout_us` | 1210 | Host timing, excluded from canonical comparison | +| `max_layout_us` | 5163 | Host timing, excluded from canonical comparison | +| `nodes` | 99 | Workload node count | +| `structural_relayouts` | 16 | Structural relayout count | +| `drawlist_checksum` | `cc6a0b00efdba151` | Deterministic drawlist checksum | diff --git a/docs/bench/core-memory-ppsspp-2026-09-05.json b/docs/bench/core-memory-ppsspp-2026-09-05.json new file mode 100644 index 000000000..71b8f7080 --- /dev/null +++ b/docs/bench/core-memory-ppsspp-2026-09-05.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "PASS", + "selected_app": "stats", + "workload_role": "representative workload for the shared workload profile; not a byte-identical event sequence", + "framework": "solid", + "samples": 3, + "report_path": "dist/bench/ppsspp-bench-2026-09-05T05-07-18-714Z.json", + "checksum": "c88e7bcedc5d42a5", + "commands": { + "baseline": { + "command": "PSP_SDK=\"$PSP_SDK\" BENCH_PPSSPP_TIMEOUT=60 bun tools/bench-ppsspp.ts --apps=stats --samples=3", + "status": "PASS", + "exit_status": 0 + }, + "memory_scan": { + "command": "PSP_SDK=\"$PSP_SDK\" BENCH_PPSSPP_TIMEOUT=60 bun tools/bench-ppsspp.ts --apps=stats --samples=3 --memory-scan", + "status": "PASS", + "exit_status": 0 + } + }, + "execution_context": { + "checkout": "bench/core-memory-receipt", + "git_revision_used_for_both_attempts": "3e0b772", + "later_metadata_commits_benchmarked": false + }, + "revisions": { + "git_revision": "3e0b772", + "ppsspp_revision": "f929a74" + }, + "metrics": { + "arena_bump_bytes": 2654944, + "arena_tail_free_bytes": 15232288, + "avg_tick_us": 516, + "avg_work_us": 4686, + "max_work_us": 62816, + "uncapped_arena_bump_bytes": 2654944, + "min_pass_arena_bytes": 2883584, + "safe_arena_bytes": 3670016 + }, + "field_definitions": { + "report_path": "Expected generated JSON report path under dist/bench/ (ppsspp-bench-.json); null means no report was generated.", + "checksum": "Checksum of the representative workload drawlist used by the run; null means no benchmark ran. Compare with the Rust profile and report differences explicitly." + }, + "caveat": "PPSSPP measurements are emulator evidence, not real PSP hardware proof. Arena high-water is a practical allocator capacity requirement, not a precise live-object heap profile. PSP checksum c88e7bcedc5d42a5 differs from the Rust baseline checksum cc6a0b00efdba151 because the PSP journey is not byte-identical to the synthetic Rust workload." +} diff --git a/docs/bench/core-memory-ppsspp-2026-09-05.md b/docs/bench/core-memory-ppsspp-2026-09-05.md new file mode 100644 index 000000000..b1b056ac1 --- /dev/null +++ b/docs/bench/core-memory-ppsspp-2026-09-05.md @@ -0,0 +1,77 @@ +# Core Memory PSP Receipt + +Status: **PASS** + +Receipt schema version: **1** + +Both benchmark commands reached the `stats` PSP app under PPSSPPHeadless. + +The PSP target is the **`stats` representative workload**, using the existing +fixed input script and PSP JSONL schema. It corresponds to phases in the Rust +synthetic workload profile; the two journeys are not byte-identical event +sequences. No second demo or Taffy A/B run was added. + +## Commands + +Both commands ran on revision `3e0b772`. The PPSSPP source revision was +`f929a74`. + +```bash +git status --short --branch +``` + +Baseline: + +```bash +PSP_SDK="$PSP_SDK" BENCH_PPSSPP_TIMEOUT=60 bun tools/bench-ppsspp.ts --apps=stats --samples=3 +``` + +Status: **PASS**; exit status: `0`. + +Memory scan: + +```bash +PSP_SDK="$PSP_SDK" BENCH_PPSSPP_TIMEOUT=60 bun tools/bench-ppsspp.ts --apps=stats --samples=3 --memory-scan +``` + +Status: **PASS**; exit status: `0`. + +## Revisions + +| item | value | +|---|---| +| git revision | `3e0b772` | +| PPSSPP revision | `f929a74` | +| selected app | `stats` | +| framework | `solid` | +| samples | `3` | +| report path | `dist/bench/ppsspp-bench-2026-09-05T05-07-18-714Z.json` | +| checksum | `c88e7bcedc5d42a5` | + +`report_path` points to the authoritative generated JSON report under +`dist/bench/`. `checksum` is the representative workload drawlist checksum. +The PSP checksum `c88e7bcedc5d42a5` differs from the Rust baseline checksum +`cc6a0b00efdba151`; the workloads are not byte-identical. + +## Metrics + +| metric | value | +|---|---:| +| arena bump | `2,654,944` bytes (`2.53 MiB`) | +| arena tail free | `15,232,288` bytes | +| avg tick | `516us` | +| avg work | `4,686us` | +| max work | `62,816us` | +| uncapped arena bump | `2,654,944` bytes | +| minimum passing arena | `2,883,584` bytes (`2.75 MiB`) | +| safe arena | `3,670,016` bytes (`3.50 MiB`) | + +## Environment + +The fixed PSP SDK was loaded from `/Users/quake/.cache/pocket-stack/psp/env.sh`. +`PPSSPPHeadless` was built at `/Users/quake/ppsspp-src/build/PPSSPPHeadless`. + +**PPSSPP results, when available, are emulator evidence rather than real PSP +hardware proof.** `arena_bump_bytes` is a practical allocator capacity +requirement and includes size-class fragmentation; it is not a precise +live-object heap profile. diff --git a/engine/core/examples/membench.rs b/engine/core/examples/membench.rs new file mode 100644 index 000000000..d7945c030 --- /dev/null +++ b/engine/core/examples/membench.rs @@ -0,0 +1,562 @@ +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::UnsafeCell; +use std::ops::{Deref, DerefMut}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Instant; + +use pocketjs_core::{spec, Ui}; + +struct CountingAlloc; + +static COUNTING: AtomicBool = AtomicBool::new(false); +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +static TOTAL: AtomicUsize = AtomicUsize::new(0); +static COUNT: AtomicUsize = AtomicUsize::new(0); + +const LEDGER_CAPACITY: usize = 4096; + +#[derive(Clone, Copy)] +struct LedgerEntry { + ptr: usize, + size: usize, + measured: bool, +} + +struct AllocationLedger { + entries: [LedgerEntry; LEDGER_CAPACITY], +} + +impl AllocationLedger { + const EMPTY: LedgerEntry = LedgerEntry { + ptr: 0, + size: 0, + measured: false, + }; + + const fn new() -> Self { + Self { + entries: [Self::EMPTY; LEDGER_CAPACITY], + } + } + + fn alloc(&mut self, ptr: usize, size: usize, measured: bool) { + let slot = self + .entries + .iter_mut() + .find(|entry| entry.ptr == 0) + .expect("allocation ledger capacity exceeded"); + *slot = LedgerEntry { + ptr, + size, + measured, + }; + } + + fn realloc( + &mut self, + old_ptr: usize, + new_ptr: usize, + new_size: usize, + ) -> Option<(usize, bool)> { + let entry = self.entries.iter_mut().find(|entry| entry.ptr == old_ptr); + let Some(entry) = entry else { + self.alloc(new_ptr, new_size, false); + return None; + }; + let old = (entry.size, entry.measured); + entry.ptr = new_ptr; + entry.size = new_size; + Some(old) + } + + fn dealloc(&mut self, ptr: usize) -> Option<(usize, bool)> { + let entry = self.entries.iter_mut().find(|entry| entry.ptr == ptr)?; + let result = (entry.size, entry.measured); + *entry = Self::EMPTY; + Some(result) + } + + fn begin_measurement(&mut self) { + for entry in &mut self.entries { + if entry.ptr != 0 { + entry.measured = false; + } + } + } +} + +struct LedgerLock { + locked: AtomicBool, + ledger: UnsafeCell, +} + +unsafe impl Sync for LedgerLock {} + +struct LedgerGuard<'a> { + lock: &'a LedgerLock, +} + +impl LedgerLock { + const fn new() -> Self { + Self { + locked: AtomicBool::new(false), + ledger: UnsafeCell::new(AllocationLedger::new()), + } + } + + fn lock(&self) -> LedgerGuard<'_> { + while self + .locked + .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + std::hint::spin_loop(); + } + LedgerGuard { lock: self } + } +} + +impl Deref for LedgerGuard<'_> { + type Target = AllocationLedger; + + fn deref(&self) -> &Self::Target { + // The guard owns the lock for the lifetime of this reference. + unsafe { &*self.lock.ledger.get() } + } +} + +impl DerefMut for LedgerGuard<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + // The guard owns the lock, so no other mutable reference exists. + unsafe { &mut *self.lock.ledger.get() } + } +} + +impl Drop for LedgerGuard<'_> { + fn drop(&mut self) { + self.lock.locked.store(false, Ordering::Release); + } +} + +static ALLOCATION_LEDGER: LedgerLock = LedgerLock::new(); + +#[global_allocator] +static ALLOCATOR: CountingAlloc = CountingAlloc; + +impl CountingAlloc { + fn record_alloc(ptr: *mut u8, size: usize) { + let measured = COUNTING.load(Ordering::Relaxed); + ALLOCATION_LEDGER.lock().alloc(ptr as usize, size, measured); + if !measured { + return; + } + LIVE.fetch_add(size, Ordering::Relaxed); + TOTAL.fetch_add(size, Ordering::Relaxed); + COUNT.fetch_add(1, Ordering::Relaxed); + Self::record_peak(); + } + + fn record_peak() { + let live = LIVE.load(Ordering::Relaxed); + let mut peak = PEAK.load(Ordering::Relaxed); + while live > peak { + match PEAK.compare_exchange_weak(peak, live, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => break, + Err(current) => peak = current, + } + } + } + + fn record_dealloc(ptr: *mut u8) { + if let Some((size, measured)) = ALLOCATION_LEDGER.lock().dealloc(ptr as usize) { + if measured { + LIVE.fetch_sub(size, Ordering::Relaxed); + } + } + } +} + +unsafe impl GlobalAlloc for CountingAlloc { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = System.alloc(layout); + if !ptr.is_null() { + Self::record_alloc(ptr, layout.size()); + } + ptr + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let ptr = System.alloc_zeroed(layout); + if !ptr.is_null() { + Self::record_alloc(ptr, layout.size()); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + System.dealloc(ptr, layout); + Self::record_dealloc(ptr); + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new_ptr = System.realloc(ptr, layout, new_size); + if !new_ptr.is_null() { + if let Some((old_size, measured)) = + ALLOCATION_LEDGER + .lock() + .realloc(ptr as usize, new_ptr as usize, new_size) + { + if measured { + LIVE.fetch_sub(old_size, Ordering::Relaxed); + LIVE.fetch_add(new_size, Ordering::Relaxed); + TOTAL.fetch_add(new_size, Ordering::Relaxed); + COUNT.fetch_add(1, Ordering::Relaxed); + Self::record_peak(); + } + } + } + new_ptr + } +} + +fn begin_measurement() { + ALLOCATION_LEDGER.lock().begin_measurement(); + LIVE.store(0, Ordering::Relaxed); + PEAK.store(0, Ordering::Relaxed); + TOTAL.store(0, Ordering::Relaxed); + COUNT.store(0, Ordering::Relaxed); + COUNTING.store(true, Ordering::Relaxed); +} + +fn end_measurement() { + COUNTING.store(false, Ordering::Relaxed); +} + +fn push_u16(out: &mut Vec, value: u16) { + out.extend_from_slice(&value.to_le_bytes()); +} + +fn push_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_le_bytes()); +} + +fn style_blob() -> Vec { + // Three records: view, text, and image. The records use only fixed props, + // so parsing them cannot introduce workload-dependent style allocations. + let mut out = Vec::with_capacity(12 + 3 * 32); + push_u32(&mut out, spec::style_table::MAGIC); + push_u16(&mut out, spec::style_table::VERSION); + push_u16(&mut out, 3); + push_u16(&mut out, 0); + push_u16(&mut out, 0); + + let records = [ + vec![ + (spec::prop::BG_COLOR, 0xff202830), + (spec::prop::PADDING_T, 4.0f32.to_bits()), + (spec::prop::PADDING_B, 4.0f32.to_bits()), + ], + vec![ + (spec::prop::TEXT_COLOR, 0xffffffff), + (spec::prop::HEIGHT, 18.0f32.to_bits()), + ], + vec![(spec::prop::BG_COLOR, 0xff405060)], + ]; + for props in records { + out.push(spec::style_table::VARIANT_BASE); + out.push(props.len() as u8); + for (prop, value) in props { + out.push(prop); + out.push(0); + push_u32(&mut out, value); + } + } + out +} + +fn texture_blob() -> Vec { + let mut atlas = Vec::with_capacity(16 * 16 * 4); + for i in 0..(16 * 16) { + atlas.extend_from_slice(&[ + (i & 0xff) as u8, + ((i * 3) & 0xff) as u8, + ((i * 7) & 0xff) as u8, + 0xff, + ]); + } + atlas +} + +fn font_atlas_blob() -> Vec { + const FIRST_GLYPH: u32 = 32; + const GLYPH_COUNT: u16 = 95; + const CELL_W: u8 = 8; + const CELL_H: u8 = 16; + const BYTES_PER_GLYPH: usize = CELL_W as usize * CELL_H as usize; + + let mut atlas = Vec::with_capacity( + spec::font_atlas::HEADER_SIZE + + GLYPH_COUNT as usize * spec::font_atlas::CMAP_ENTRY_SIZE + + GLYPH_COUNT as usize * BYTES_PER_GLYPH, + ); + push_u32(&mut atlas, spec::font_atlas::MAGIC); + push_u16(&mut atlas, spec::font_atlas::VERSION); + push_u16(&mut atlas, GLYPH_COUNT); + atlas.extend_from_slice(&[CELL_W, CELL_H, 12, 18, 0, 0, 1, 0]); + for gid in 0..GLYPH_COUNT { + push_u32(&mut atlas, FIRST_GLYPH + u32::from(gid)); + push_u16(&mut atlas, gid); + atlas.extend_from_slice(&[CELL_W, 0]); + } + for gid in 0..GLYPH_COUNT { + for y in 0..CELL_H as usize { + for x in 0..CELL_W as usize { + // Keep the space transparent while giving every printable + // glyph deterministic coverage to exercise atlas sampling. + let covered = gid != 0 + && (1..=6).contains(&x) + && (2..=13).contains(&y) + && (x + y + gid as usize) % 3 != 0; + atlas.push(if covered { 255 } else { 0 }); + } + } + } + atlas +} + +fn timing_capacity() -> usize { + 24 + 8 + 16 + 12 +} + +fn structural_tick(tick: usize) -> bool { + tick % 4 == 0 +} + +fn structural_probe(ui: &mut Ui, parent: i32, height: f64) { + let node = ui.create_node(spec::NodeType::View as u8); + ui.set_style(node, 0); + ui.set_prop(node, spec::prop::HEIGHT, height); + ui.insert_before(parent, node, 0); + ui.destroy_node(node); +} + +fn hash_draw(words: &[u32], checksum: &mut u64) { + for word in words { + *checksum ^= u64::from(*word); + *checksum = checksum.wrapping_mul(0x100000001b3); + } +} + +fn draw_and_hash(ui: &mut Ui, checksum: &mut u64) { + let words = &ui.draw().words; + assert!( + words.iter().any(|&word| word == spec::draw_op::GLYPH_RUN), + "benchmark draw must emit atlas-backed glyphs" + ); + hash_draw(words, checksum); +} + +fn tick_and_measure(ui: &mut Ui, checksum: &mut u64, total_us: &mut u128, max_us: &mut u128) { + let started = Instant::now(); + ui.tick(); + let elapsed = started.elapsed().as_micros(); + *total_us += elapsed; + *max_us = (*max_us).max(elapsed); + draw_and_hash(ui, checksum); +} + +fn main() { + const STEADY_TICKS: usize = 24; + const CHURN_ROUNDS: usize = 8; + const CHURN_SIZE: usize = 4; + const TEXT_TICKS: usize = 16; + const BURST_TICKS: usize = 12; + + // Fixture bytes, source strings, handles, churn ids, and timing state are + // prepared before measurement. Setup allocations are intentionally outside + // the workload receipt. + let styles = style_blob(); + let atlas = texture_blob(); + let font_atlas = font_atlas_blob(); + let source_strings = [ + String::from("PocketJS memory benchmark"), + String::from("deterministic retained core workload"), + ]; + let mut handles = Vec::with_capacity(1 + 1 + 32 + CHURN_SIZE); + let mut churn_ids = Vec::with_capacity(CHURN_SIZE); + let mut text_ids = Vec::with_capacity(32); + let mut timings = Vec::with_capacity(timing_capacity()); + timings.clear(); + + let mut ui = Ui::new(); + ui.set_viewport(spec::SCREEN_W as f32, spec::SCREEN_H as f32); + assert!(ui.load_styles(&styles)); + assert!(ui.load_font_atlas(&font_atlas)); + let font = ui.font_atlas(0).unwrap(); + let (workload_gid, _) = font + .lookup('P' as u32) + .expect("workload glyph must be mapped"); + assert!( + font.glyph_rows(workload_gid) + .iter() + .any(|&coverage| coverage != 0), + "benchmark font atlas must contain non-zero coverage for P" + ); + handles.push(ui.upload_texture(&atlas, 16, 16, spec::psm::PSM_8888)); + let texture = handles[0]; + + begin_measurement(); + let mut checksum = 0xcbf29ce484222325; + let mut total_us = 0u128; + let mut max_us = 0u128; + + // Phase 1: initial tree and resources. + let panel = ui.create_node(spec::NodeType::View as u8); + ui.set_style(panel, 0); + ui.insert_before(spec::ROOT_ID, panel, 0); + let title = ui.create_node(spec::NodeType::Text as u8); + ui.set_style(title, 1); + ui.set_text(title, &source_strings[0]); + ui.insert_before(panel, title, 0); + for i in 0..32 { + let row = ui.create_node(spec::NodeType::View as u8); + ui.set_style(row, 0); + ui.set_prop(row, spec::prop::HEIGHT, 24.0 + (i % 3) as f64); + ui.insert_before(panel, row, 0); + let text = ui.create_node(spec::NodeType::Text as u8); + ui.set_style(text, 1); + ui.set_text(text, &source_strings[1]); + ui.insert_before(row, text, 0); + text_ids.push(text); + let image = ui.create_node(spec::NodeType::Image as u8); + ui.set_style(image, 2); + ui.set_prop(image, spec::prop::WIDTH, 16.0); + ui.set_prop(image, spec::prop::HEIGHT, 16.0); + ui.set_image(image, texture); + ui.insert_before(row, image, 0); + } + draw_and_hash(&mut ui, &mut checksum); + + // Phase 2: steady style-only ticks. + for i in 0..STEADY_TICKS { + ui.set_prop(panel, spec::prop::OPACITY, 0.85 + (i % 4) as f64 * 0.03); + tick_and_measure(&mut ui, &mut checksum, &mut total_us, &mut max_us); + timings.push(i); + } + + // Phase 3: fixed-size subtree creation and destruction. + let mut structural_relayouts = 1u64; + for round in 0..CHURN_ROUNDS { + churn_ids.clear(); + for i in 0..CHURN_SIZE { + let node = ui.create_node(spec::NodeType::View as u8); + ui.set_style(node, 0); + ui.set_prop(node, spec::prop::HEIGHT, (20 + i + round) as f64); + ui.insert_before(panel, node, 0); + churn_ids.push(node); + } + for node in churn_ids.iter().copied() { + ui.destroy_node(node); + } + tick_and_measure(&mut ui, &mut checksum, &mut total_us, &mut max_us); + structural_relayouts += 1; + timings.push(STEADY_TICKS + round); + } + + // Phase 4: text changes with a structural relayout at a fixed interval. + for i in 0..TEXT_TICKS { + let text = text_ids[i % text_ids.len()]; + let value = if i % 4 == 0 { + "structural text update" + } else { + "steady text update" + }; + ui.set_text(text, value); + if structural_tick(i) { + structural_probe(&mut ui, panel, 22.0 + i as f64); + structural_relayouts += 1; + } + tick_and_measure(&mut ui, &mut checksum, &mut total_us, &mut max_us); + timings.push(STEADY_TICKS + CHURN_ROUNDS + i); + } + + // Phase 5: one fixed burst over the peak path. + for i in 0..BURST_TICKS { + let text = text_ids[(i * 7) % text_ids.len()]; + ui.set_text(text, if i & 1 == 0 { "burst A" } else { "burst B" }); + ui.set_prop(panel, spec::prop::GAP, (i % 5) as f64); + if structural_tick(i) { + structural_probe(&mut ui, panel, 30.0 + i as f64); + structural_relayouts += 1; + } + tick_and_measure(&mut ui, &mut checksum, &mut total_us, &mut max_us); + timings.push(STEADY_TICKS + CHURN_ROUNDS + TEXT_TICKS + i); + } + + let nodes = 1 + 1 + 1 + 32 * 3; + end_measurement(); + // Ui::tick includes animation bookkeeping, so this is a tick/layout proxy. + let avg_layout_us = total_us / timings.len() as u128; + println!("peak_requested_bytes={}", PEAK.load(Ordering::Relaxed)); + println!("final_requested_bytes={}", LIVE.load(Ordering::Relaxed)); + println!("allocation_count={}", COUNT.load(Ordering::Relaxed)); + println!("total_allocated_bytes={}", TOTAL.load(Ordering::Relaxed)); + println!("avg_layout_us={avg_layout_us}"); + println!("max_layout_us={max_us}"); + println!("nodes={nodes}"); + println!("structural_relayouts={structural_relayouts}"); + println!("text_mode=atlas"); + println!("texture_mode=atlas"); + println!("drawlist_checksum={checksum:016x}"); + assert_eq!( + checksum, 0xcc6a0b00efdba151, + "deterministic benchmark drawlist changed" + ); +} + +#[cfg(test)] +mod tests { + use super::{font_atlas_blob, structural_tick, timing_capacity, AllocationLedger}; + use pocketjs_core::Ui; + + #[test] + fn reserves_all_measurement_entries() { + assert_eq!(timing_capacity(), 60); + } + + #[test] + fn structural_schedule_is_fixed_and_coalesced() { + assert_eq!((0..16).filter(|&tick| structural_tick(tick)).count(), 4); + assert_eq!((0..12).filter(|&tick| structural_tick(tick)).count(), 3); + } + + #[test] + fn fixture_atlas_loads_and_has_workload_glyphs() { + let mut ui = Ui::new(); + assert!(ui.load_font_atlas(&font_atlas_blob())); + let font = ui.font_atlas(0).unwrap(); + let (workload_gid, _) = font.lookup('P' as u32).unwrap(); + assert!(font + .glyph_rows(workload_gid) + .iter() + .any(|&coverage| coverage != 0)); + assert_eq!(ui.measure_text("P", 0), 8.0); + } + + #[test] + fn ledger_preserves_pre_measurement_realloc_state() { + let mut ledger = AllocationLedger::new(); + ledger.alloc(0x1000, 16, false); + ledger.begin_measurement(); + + let event = ledger.realloc(0x1000, 0x2000, 32); + assert_eq!(event, Some((16, false))); + assert_eq!(ledger.dealloc(0x2000), Some((32, false))); + + ledger.alloc(0x3000, 8, true); + let event = ledger.realloc(0x3000, 0x4000, 12); + assert_eq!(event, Some((8, true))); + assert_eq!(ledger.dealloc(0x4000), Some((12, true))); + } +} diff --git a/hosts/psp/src/main.rs b/hosts/psp/src/main.rs index c3181e313..c161e2fa6 100644 --- a/hosts/psp/src/main.rs +++ b/hosts/psp/src/main.rs @@ -32,6 +32,10 @@ use pocketjs_psp::{arena, audio_mod, dbg, ffi, ge, host, pak, svc, switch, veil, psp::module!("pocketjs", 1, 1); const CORE_TICKS_PER_SECOND: u32 = 60; +#[cfg(feature = "bench")] +const DRAWLIST_CHECKSUM_OFFSET: u64 = 0xcbf29ce484222325; +#[cfg(feature = "bench")] +const DRAWLIST_CHECKSUM_PRIME: u64 = 0x100000001b3; // The full launcher originally consumed about 42 ms of CPU work on real PSP // hardware. Batching cuts that to about 12 ms, but the texture-heavy GE pass // still completes across the third vblank. A multi-app package therefore @@ -42,8 +46,6 @@ const MULTI_APP_SIM_HZ: u32 = 20; // App bundles live in switch::APPS (build.rs generates the table; entry 0 is // the app POCKETJS_APP selected, multi-app builds append the registry). Each // entry's js is NUL-terminated there for JS_Eval (input[len] == '\0'). -#[cfg(feature = "bench")] -static POCKETJS_APP_NAME: &str = env!("POCKETJS_APP"); static POCKETJS_TRACE: &str = env!("POCKETJS_TRACE"); // Arena bump high-water at the last host-forced collection (frame loop's @@ -161,6 +163,7 @@ struct BenchState { previous_frame_start_us: u64, frame_interval_sum_us: u64, max_frame_interval_us: u64, + drawlist_checksum: u64, } #[cfg(feature = "bench")] @@ -184,12 +187,15 @@ impl BenchState { previous_frame_start_us: 0, frame_interval_sum_us: 0, max_frame_interval_us: 0, + drawlist_checksum: DRAWLIST_CHECKSUM_OFFSET, } } } #[cfg(feature = "bench")] static mut BENCH: BenchState = BenchState::new(); +#[cfg(feature = "bench")] +static mut BENCH_HAS_GUEST: bool = false; #[cfg(feature = "bench")] #[inline] @@ -235,6 +241,12 @@ unsafe fn bench_reset_file() { #[cfg(feature = "bench")] unsafe fn bench_init() { bench_reset_file(); + bench_start_guest(); + BENCH_HAS_GUEST = false; +} + +#[cfg(feature = "bench")] +unsafe fn bench_start_guest() { BENCH = BenchState::new(); BENCH.run_start_us = bench_now_us(); } @@ -264,6 +276,12 @@ unsafe fn bench_window() -> (u32, u32) { } } +#[cfg(feature = "bench")] +#[inline] +fn bench_in_window(frame_count: u32, start: u32, n: u32) -> bool { + frame_count >= start && frame_count - start < n +} + #[cfg(feature = "bench")] #[allow(clippy::too_many_arguments)] unsafe fn bench_record_frame( @@ -277,7 +295,7 @@ unsafe fn bench_record_frame( present_us: u64, ) { let (start, n) = bench_window(); - if frame_count < start || frame_count >= start + n { + if !bench_in_window(frame_count, start, n) { return; } let js_us = after_js.saturating_sub(t0); @@ -306,10 +324,24 @@ unsafe fn bench_record_frame( } } +#[cfg(feature = "bench")] +unsafe fn bench_record_drawlist(frame_count: u32, words: &[u32]) { + let (start, n) = bench_window(); + if !bench_in_window(frame_count, start, n) { + return; + } + for word in words { + BENCH.drawlist_checksum ^= u64::from(*word); + BENCH.drawlist_checksum = BENCH + .drawlist_checksum + .wrapping_mul(DRAWLIST_CHECKSUM_PRIME); + } +} + #[cfg(feature = "bench")] unsafe fn bench_record_gpu(frame_count: u32, gpu_us: u64) { let (start, n) = bench_window(); - if frame_count < start || frame_count >= start + n { + if !bench_in_window(frame_count, start, n) { return; } BENCH.gpu_sum_us = BENCH.gpu_sum_us.saturating_add(gpu_us); @@ -326,7 +358,11 @@ unsafe fn bench_record_frame0_complete() { #[cfg(feature = "bench")] unsafe fn bench_maybe_flush(frame_count: u32) { let (start, n) = bench_window(); - if n == 0 || frame_count != start + n - 1 || BENCH.frames == 0 { + if n == 0 + || !bench_in_window(frame_count, start, n) + || frame_count - start != n - 1 + || BENCH.frames == 0 + { return; } let frames = BENCH.frames as u64; @@ -335,8 +371,8 @@ unsafe fn bench_maybe_flush(frame_count: u32) { let stack_free_bytes = sys::sceKernelGetThreadStackFreeSize(sys::SceUid(sys::sceKernelGetThreadId())).max(0); let line = alloc::format!( - "{{\"app\":\"{}\",\"sim_hz\":{},\"frames\":{},\"window_start\":{},\"window_n\":{},\"eval_us\":{},\"boot_to_eval_begin_us\":{},\"boot_to_frame0_us\":{},\"avg_frame_interval_us\":{},\"max_frame_interval_us\":{},\"avg_js_us\":{},\"avg_jobs_us\":{},\"avg_tick_us\":{},\"avg_draw_us\":{},\"avg_render_us\":{},\"avg_work_us\":{},\"max_work_us\":{},\"avg_gpu_us\":{},\"max_gpu_us\":{},\"stack_free_bytes\":{},\"bundle_bytes\":{},\"pak_bytes\":{},\"arena_capacity_bytes\":{},\"arena_bump_bytes\":{},\"arena_tail_free_bytes\":{},\"arena_init_free_bytes\":{},\"arena_configured_bytes\":{}}}\n", - POCKETJS_APP_NAME, + "{{\"app\":\"{}\",\"sim_hz\":{},\"frames\":{},\"window_start\":{},\"window_n\":{},\"eval_us\":{},\"boot_to_eval_begin_us\":{},\"boot_to_frame0_us\":{},\"avg_frame_interval_us\":{},\"max_frame_interval_us\":{},\"avg_js_us\":{},\"avg_jobs_us\":{},\"avg_tick_us\":{},\"avg_draw_us\":{},\"avg_render_us\":{},\"avg_work_us\":{},\"max_work_us\":{},\"avg_gpu_us\":{},\"max_gpu_us\":{},\"stack_free_bytes\":{},\"bundle_bytes\":{},\"pak_bytes\":{},\"arena_capacity_bytes\":{},\"arena_bump_bytes\":{},\"arena_tail_free_bytes\":{},\"arena_init_free_bytes\":{},\"arena_configured_bytes\":{},\"drawlist_checksum\":\"{:016x}\"}}\n", + switch::current_output(), if switch::multi() { MULTI_APP_SIM_HZ } else { CORE_TICKS_PER_SECOND }, BENCH.frames, start, @@ -356,13 +392,18 @@ unsafe fn bench_maybe_flush(frame_count: u32) { BENCH.gpu_sum_us / frames, BENCH.max_gpu_us, stack_free_bytes, - switch::guest_bytes(0).map(|g| g.js.len().saturating_sub(1)).unwrap_or(0), - switch::guest_bytes(0).map(|g| g.pak.len()).unwrap_or(0), + switch::guest_bytes(switch::current()) + .map(|g| g.js.len().saturating_sub(1)) + .unwrap_or(0), + switch::guest_bytes(switch::current()) + .map(|g| g.pak.len()) + .unwrap_or(0), arena_stats.capacity_bytes, arena_stats.bump_bytes, arena_stats.tail_free_bytes, arena_stats.init_free_bytes, arena_stats.configured_bytes, + BENCH.drawlist_checksum, ); bench_write(line.as_bytes()); } @@ -479,6 +520,12 @@ unsafe fn run_guest( last_present_vcount: &mut u32, ) -> usize { switch::set_current(app_index); + #[cfg(feature = "bench")] + if BENCH_HAS_GUEST { + bench_start_guest(); + } else { + BENCH_HAS_GUEST = true; + } // Package mode extracts js/pak zero-copy from the entry's embedded // `.pocket`; single-app mode reads the classic inline embed. A package // that fails to parse routes through the broken-guest rule. @@ -818,6 +865,8 @@ unsafe fn run_guest( bench_after_render, bench_after_present.saturating_sub(bench_before_sync), ); + #[cfg(feature = "bench")] + bench_record_drawlist(guest_frame, core::slice::from_raw_parts(words_ptr, words_len)); if guest_frame == 0 { trace("frame 0: rendered"); } @@ -948,13 +997,13 @@ unsafe fn cap_dump_frame(frame_count: u32) { // Defaults: skip boot transients + 150 ms mount transitions; 32 frames. let cap_start = capture_env_u32(POCKETJS_CAP_START, 16); let cap_n = capture_env_u32(POCKETJS_CAP_N, 32); - if frame_count < cap_start || frame_count >= cap_start + cap_n { + if frame_count < cap_start || frame_count - cap_start >= cap_n { return; } let idx = frame_count - cap_start; #[cfg(feature = "bench")] if POCKETJS_BENCH_DUMP_FRAMES != "1" { - if idx + 1 == cap_n { + if idx == cap_n - 1 { sys::sceKernelExitGame(); } return; @@ -994,7 +1043,7 @@ unsafe fn cap_dump_frame(frame_count: u32) { sys::sceIoWrite(fd, addr as *const c_void, 512 * 272 * 4); sys::sceIoClose(fd); } - if idx + 1 == cap_n { + if idx == cap_n - 1 { sys::sceKernelExitGame(); } } diff --git a/hosts/psp/src/switch.rs b/hosts/psp/src/switch.rs index 2bc03da5e..dc957212d 100644 --- a/hosts/psp/src/switch.rs +++ b/hosts/psp/src/switch.rs @@ -83,6 +83,10 @@ pub unsafe fn set_current(index: usize) { CURRENT = index; } +pub unsafe fn current_output() -> &'static str { + APPS[CURRENT].output +} + pub unsafe fn resume() -> Option { usize::try_from(RESUME).ok() } diff --git a/skills/pocketjs-psp-benchmark/references/metrics.md b/skills/pocketjs-psp-benchmark/references/metrics.md index 63a7e7adc..77423ac3e 100644 --- a/skills/pocketjs-psp-benchmark/references/metrics.md +++ b/skills/pocketjs-psp-benchmark/references/metrics.md @@ -25,6 +25,7 @@ Read this reference when explaining `tools/bench-ppsspp.ts` output. - `arena_tail_free_bytes`: unused tail capacity in the arena at report time. It does not include blocks held on free lists. - `arena_init_free_bytes`: PSP user partition max-free value observed when the arena initialized. - `arena_configured_bytes`: requested `POCKETJS_ARENA_BYTES`; `0` means the production default of max free memory minus margin. +- `drawlist_checksum`: deterministic FNV-1a-style checksum over the `u32` draw-list words returned by `ui.draw()` across the measured window. It uses seed `0xcbf29ce484222325` and prime `0x100000001b3`; use it to report rendered-workload differences, not to claim identical input event sequences. ## Memory Scan Fields @@ -37,3 +38,5 @@ Read this reference when explaining `tools/bench-ppsspp.ts` output. ## Caveats PPSSPP measurements are deterministic and useful for regression tracking, but they are not real PSP hardware proof. The arena high-water is a practical capacity requirement for this allocator, not a precise live-object heap profile, because freed blocks remain reserved in size-class free lists. + +The core Rust harness is a deterministic synthetic workload profile. PSP uses the existing `stats` app as a representative workload with corresponding phases. The two journeys are not byte-identical event sequences; reports must state any drawlist checksum difference explicitly. diff --git a/tests/core-memory-bench.test.ts b/tests/core-memory-bench.test.ts new file mode 100644 index 000000000..85f269319 --- /dev/null +++ b/tests/core-memory-bench.test.ts @@ -0,0 +1,138 @@ +import { $ } from "bun"; +import { expect, test } from "bun:test"; +import { fileURLToPath } from "node:url"; + +const ROOT = fileURLToPath(new URL("..", import.meta.url)); +const COMMAND = [ + "cargo", + "run", + "--manifest-path", + "engine/core/Cargo.toml", + "--example", + "membench", + "--quiet", +]; + +const REQUIRED_FIELDS = [ + "peak_requested_bytes", + "final_requested_bytes", + "allocation_count", + "total_allocated_bytes", + "avg_layout_us", + "max_layout_us", + "nodes", + "structural_relayouts", + "text_mode", + "texture_mode", + "drawlist_checksum", +] as const; + +const TIMING_FIELDS = new Set(["avg_layout_us", "max_layout_us"]); +type Receipt = Record<(typeof REQUIRED_FIELDS)[number], string>; + +function parseReceipt(output: string): Receipt { + const lines = output.endsWith("\n") + ? output.slice(0, -1).split("\n") + : output.split("\n"); + expect(lines).toHaveLength(REQUIRED_FIELDS.length); + + const seen = new Set(); + const entries = lines.map((line) => { + const separator = line.indexOf("="); + expect(separator).toBeGreaterThan(0); + expect(line.indexOf("=", separator + 1)).toBe(-1); + + const field = line.slice(0, separator); + const value = line.slice(separator + 1); + expect((REQUIRED_FIELDS as readonly string[]).includes(field)).toBe(true); + expect(seen.has(field)).toBe(false); + expect(value).not.toBe(""); + seen.add(field); + return [field, value]; + }); + + expect(seen.size).toBe(REQUIRED_FIELDS.length); + const receipt = Object.fromEntries(entries) as Partial; + + for (const field of REQUIRED_FIELDS) { + expect(receipt[field]).toBeDefined(); + expect(receipt[field]).not.toBe(""); + } + + for (const field of REQUIRED_FIELDS) { + if (TIMING_FIELDS.has(field)) continue; + if (field === "text_mode" || field === "texture_mode") { + expect(receipt[field]).toBe("atlas"); + } else if (field === "drawlist_checksum") { + expect(receipt[field]).toMatch(/^[0-9a-f]{16}$/); + } else { + expect(receipt[field]).toMatch(/^\d+$/); + const value = Number(receipt[field]); + expect(Number.isFinite(value)).toBe(true); + expect(Number.isInteger(value)).toBe(true); + expect(value).toBeGreaterThanOrEqual(0); + } + } + + for (const field of TIMING_FIELDS) { + expect(receipt[field]).toMatch(/^\d+$/); + const value = Number(receipt[field]); + expect(Number.isFinite(value)).toBe(true); + expect(Number.isInteger(value)).toBe(true); + expect(value).toBeGreaterThanOrEqual(0); + } + + return receipt as Receipt; +} + +const VALID_RECEIPT = REQUIRED_FIELDS.map((field) => { + if (field === "text_mode" || field === "texture_mode") return `${field}=atlas`; + if (field === "drawlist_checksum") return `${field}=0000000000000000`; + return `${field}=0`; +}).join("\n"); + +test("receipt parser rejects unknown and duplicate fields", () => { + expect(() => parseReceipt(`${VALID_RECEIPT}\nunknown=0`)).toThrow(); + expect(() => parseReceipt(`${VALID_RECEIPT}\npeak_requested_bytes=0`)).toThrow(); +}); + +test("receipt parser rejects non-finite and non-integer numbers", () => { + expect(() => + parseReceipt( + VALID_RECEIPT.replace( + "peak_requested_bytes=0", + `peak_requested_bytes=${"9".repeat(400)}`, + ), + ), + ).toThrow(); + expect(() => + parseReceipt(VALID_RECEIPT.replace("peak_requested_bytes=0", "peak_requested_bytes=1.5")), + ).toThrow(); +}); + +function canonicalReceipt(receipt: Receipt): Partial { + return Object.fromEntries( + REQUIRED_FIELDS.filter((field) => !TIMING_FIELDS.has(field)).map((field) => [ + field, + receipt[field], + ]), + ); +} + +test( + "core memory receipt is complete, stable, and matches its baseline", + { timeout: 30_000 }, + async () => { + const run = () => + $`${COMMAND[0]} ${COMMAND.slice(1)}`.cwd(ROOT).quiet().text(); + const first = parseReceipt(await run()); + const second = parseReceipt(await run()); + + expect(canonicalReceipt(first)).toEqual(canonicalReceipt(second)); + + const baseline = (await Bun.file( + new URL("../docs/bench/core-memory-2026-09-05.json", import.meta.url), + ).json()) as { receipt: Receipt }; + expect(canonicalReceipt(first)).toEqual(canonicalReceipt(baseline.receipt)); + }, +); diff --git a/tests/psp-bench-parser.test.ts b/tests/psp-bench-parser.test.ts new file mode 100644 index 000000000..59e0fb874 --- /dev/null +++ b/tests/psp-bench-parser.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from "bun:test"; +import { parseBenchOutput } from "../tools/bench-ppsspp-parser.ts"; + +function row(app: string, checksum?: string, bundle = 10, pak = 20): string { + return JSON.stringify({ + app, + frames: 80, + window_n: 80, + bundle_bytes: bundle, + pak_bytes: pak, + ...(checksum === undefined ? {} : { drawlist_checksum: checksum }), + }); +} + +test("selects the requested app from multi-app PSP output", () => { + const parsed = parseBenchOutput( + `${row("hero", "1111111111111111", 11, 22)}\n${row("cards", undefined, 33, 44)}`, + "cards", + 1, + ); + + expect(parsed.app).toBe("cards"); + expect(parsed.bundle_bytes).toBe(33); + expect(parsed.pak_bytes).toBe(44); + expect(parsed.drawlist_checksum).toBeUndefined(); +}); + +test("keeps one-line legacy PSP output compatible", () => { + const parsed = parseBenchOutput(row("legacy-app"), "stats", 1); + + expect(parsed.app).toBe("legacy-app"); + expect(parsed.drawlist_checksum).toBeUndefined(); +}); + +test("reports a missing requested app in multi-app output", () => { + expect(() => parseBenchOutput(`${row("hero")}\n${row("cards")}`, "stats", 1)).toThrow( + "stats sample 1: requested app not found in PSP bench output (hero, cards)", + ); +}); diff --git a/tests/psp-bench.test.ts b/tests/psp-bench.test.ts new file mode 100644 index 000000000..a37abd591 --- /dev/null +++ b/tests/psp-bench.test.ts @@ -0,0 +1,62 @@ +import { $ } from "bun"; +import { expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const NUMERIC_FIELDS = [ + "eval_us", + "boot_to_frame0_us", + "avg_frame_interval_us", + "max_frame_interval_us", + "avg_js_us", + "avg_jobs_us", + "avg_tick_us", + "avg_draw_us", + "avg_render_us", + "avg_work_us", + "max_work_us", + "stack_free_bytes", + "bundle_bytes", + "pak_bytes", + "arena_capacity_bytes", + "arena_bump_bytes", + "arena_tail_free_bytes", + "arena_init_free_bytes", + "arena_configured_bytes", +] as const; + +test("PPSSPP bench report preserves drawlist checksums", async () => { + const dir = await mkdtemp(join(tmpdir(), "pocketjs-psp-bench-")); + const rawPath = join(dir, "input.jsonl"); + const outDir = join(dir, "out"); + const row: Record = { + kind: "sample", + app: "stats", + framework: "solid", + sample: 1, + sim_hz: 60, + frames: 100, + window_start: 28, + window_n: 100, + drawlist_checksum: "0123456789abcdef", + host_wall_ms: 1, + arena_limit_bytes: 0, + }; + for (const field of NUMERIC_FIELDS) row[field] = 1; + + try { + await writeFile(rawPath, `${JSON.stringify(row)}\n`); + await $`bun tools/bench-ppsspp.ts --apps=stats --from-raw=${rawPath} --out-dir=${outDir}`.quiet(); + const reportPath = (await Array.fromAsync(new Bun.Glob("*.json").scan({ cwd: outDir })))[0]; + const report = JSON.parse(await readFile(join(outDir, reportPath), "utf8")); + expect(report.drawlist_checksums.stats.solid).toEqual(["0123456789abcdef"]); + + const markdownPath = reportPath.replace(/\.json$/, ".md"); + expect(await readFile(join(outDir, markdownPath), "utf8")).toContain( + "Drawlist checksums: 0123456789abcdef", + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/tools/bench-ppsspp-parser.ts b/tools/bench-ppsspp-parser.ts new file mode 100644 index 000000000..944d5d592 --- /dev/null +++ b/tools/bench-ppsspp-parser.ts @@ -0,0 +1,58 @@ +export interface BenchLine { + app: string; + sim_hz: number; + frames: number; + window_start: number; + window_n: number; + eval_us: number; + boot_to_eval_begin_us: number; + boot_to_frame0_us: number; + avg_frame_interval_us: number; + max_frame_interval_us: number; + avg_js_us: number; + avg_jobs_us: number; + avg_tick_us: number; + avg_draw_us: number; + avg_render_us: number; + avg_work_us: number; + max_work_us: number; + stack_free_bytes: number; + bundle_bytes: number; + pak_bytes: number; + arena_capacity_bytes: number; + arena_bump_bytes: number; + arena_tail_free_bytes: number; + arena_init_free_bytes: number; + arena_configured_bytes: number; + drawlist_checksum?: string; +} + +function isRequestedApp(row: BenchLine, app: string): boolean { + const base = row.app.split(".")[0]; + return base === `${app}-main` || base === app; +} + +export function parseBenchOutput(output: string, requestedApp: string, sample: number): BenchLine { + const lines = output.trim().split("\n").filter(Boolean); + let rows: BenchLine[]; + try { + rows = lines.map((line) => JSON.parse(line) as BenchLine); + } catch (error) { + throw new Error(`${requestedApp} sample ${sample}: invalid PSP bench JSON: ${String(error)}`); + } + + if (rows.length === 0) { + throw new Error(`${requestedApp} sample ${sample}: PSP bench output was empty`); + } + // Older PSP hosts emit one row and did not identify the requested app reliably. + const parsed = rows.length === 1 ? rows[0] : rows.find((row) => isRequestedApp(row, requestedApp)); + if (!parsed) { + throw new Error( + `${requestedApp} sample ${sample}: requested app not found in PSP bench output (${rows.map((row) => row.app).join(", ")})`, + ); + } + if (parsed.drawlist_checksum !== undefined && !/^[0-9a-f]{16}$/.test(parsed.drawlist_checksum)) { + throw new Error(`${requestedApp} sample ${sample}: invalid drawlist checksum`); + } + return parsed; +} diff --git a/tools/bench-ppsspp.ts b/tools/bench-ppsspp.ts index b2e320f69..a54fd14a0 100644 --- a/tools/bench-ppsspp.ts +++ b/tools/bench-ppsspp.ts @@ -13,6 +13,7 @@ import { $ } from "bun"; import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { parseFramework, type PocketFramework } from "../framework/compiler/jsx-plugin.ts"; +import { parseBenchOutput, type BenchLine } from "./bench-ppsspp-parser.ts"; interface Spec { app: string; @@ -21,34 +22,6 @@ interface Spec { capN: number; } -interface BenchLine { - app: string; - sim_hz: number; - frames: number; - window_start: number; - window_n: number; - eval_us: number; - boot_to_eval_begin_us: number; - boot_to_frame0_us: number; - avg_frame_interval_us: number; - max_frame_interval_us: number; - avg_js_us: number; - avg_jobs_us: number; - avg_tick_us: number; - avg_draw_us: number; - avg_render_us: number; - avg_work_us: number; - max_work_us: number; - stack_free_bytes: number; - bundle_bytes: number; - pak_bytes: number; - arena_capacity_bytes: number; - arena_bump_bytes: number; - arena_tail_free_bytes: number; - arena_init_free_bytes: number; - arena_configured_bytes: number; -} - interface Sample extends BenchLine { framework: PocketFramework; sample: number; @@ -303,6 +276,7 @@ const report = { ]), ), comparison: frameworks.length > 1 ? buildComparison(samplesOut, selectedSpecs) : undefined, + drawlist_checksums: summarizeChecksums(samplesOut, selectedSpecs), memory_scan: memoryScanReport, }; writeFileSync(summaryPath, JSON.stringify(report, null, 2)); @@ -360,10 +334,7 @@ async function runBenchSample( throw new Error(`${spec.app} sample ${sample}: ${benchFile} missing`); } const lines = readFileSync(benchFile, "utf8").trim().split("\n").filter(Boolean); - if (lines.length !== 1) { - throw new Error(`${spec.app} sample ${sample}: expected 1 bench line, got ${lines.length}`); - } - const parsed = JSON.parse(lines[0]) as BenchLine; + const parsed = parseBenchOutput(lines.join("\n"), spec.app, sample); if (parsed.frames !== spec.capN || parsed.window_n !== spec.capN) { throw new Error(`${spec.app} sample ${sample}: bench window mismatch (${parsed.frames}/${parsed.window_n}, expected ${spec.capN})`); } @@ -559,6 +530,20 @@ function summarizeApp( >; } +function summarizeChecksums(rows: Sample[], specs: Spec[]) { + return Object.fromEntries( + specs.map((spec) => [ + spec.app, + Object.fromEntries( + frameworks.map((fw) => [ + fw, + [...new Set(rows.filter((r) => isAppRow(r, spec.app, fw)).map((r) => r.drawlist_checksum).filter(Boolean))], + ]), + ), + ]), + ); +} + // --------------------------------------------------------------------------- // Cross-framework comparison (PR #6 methodology): per metric, the geometric // mean over apps of ratio(framework mean / baseline mean), baseline = the @@ -642,6 +627,7 @@ function renderMarkdown(report: { frame_budget_us: number; apps: Record>>>; comparison?: ReturnType; + drawlist_checksums: Record>; memory_scan?: ReturnType; }) { const lines = [ @@ -693,6 +679,8 @@ function renderMarkdown(report: { ); } lines.push(""); + const checksums = report.drawlist_checksums[app]?.[fw] ?? []; + lines.push(`Drawlist checksums: ${checksums.length ? checksums.join(", ") : "unavailable"}`, ""); } }