perf: measure warmed collection rebind VMs

Author: fffonion

examples/collection_rebind_bench.rs

@@ -2,7 +2,7 @@ use std::env;
 use std::hint::black_box;
 use std::time::{Duration, Instant};
 
-use vm::{JitConfig, Program, Value, Vm, VmStatus, compile_source};
+use vm::{JitConfig, JitTraceTerminal, Program, Value, Vm, VmStatus, compile_source};
 
 const DEFAULT_WIDTH: usize = 256;
 const DEFAULT_ITERATIONS: usize = 50_000;
@@ -38,12 +38,25 @@ impl ExecMode {
     }
 }
 
+struct Measurement {
+    samples: Vec<Duration>,
+    native_traces: usize,
+    call_boundary_traces: usize,
+    loop_back_traces: usize,
+    native_execs: u64,
+    trace_exits: u64,
+    native_loop_backs: u64,
+    helper_fallbacks: u64,
+    generic_builtin_calls: u64,
+}
+
 #[derive(Clone, Copy, Debug)]
 struct Config {
     width: usize,
     iterations: usize,
     samples: usize,
     jit: bool,
+    jit_only: bool,
 }
 
 impl Config {
@@ -53,6 +66,7 @@ impl Config {
             iterations: DEFAULT_ITERATIONS,
             samples: DEFAULT_SAMPLES,
             jit: false,
+            jit_only: false,
         };
         let mut args = env::args().skip(1);
         while let Some(arg) = args.next() {
@@ -61,6 +75,10 @@ impl Config {
                 "--iterations" => config.iterations = parse_usize("--iterations", args.next())?,
                 "--samples" => config.samples = parse_usize("--samples", args.next())?,
                 "--jit" => config.jit = true,
+                "--jit-only" => {
+                    config.jit = true;
+                    config.jit_only = true;
+                }
                 other => return Err(format!("unknown argument '{other}'")),
             }
         }
@@ -87,7 +105,9 @@ fn main() {
 
 fn run() -> Result<(), String> {
     let config = Config::parse()?;
-    let modes: &[ExecMode] = if config.jit {
+    let modes: &[ExecMode] = if config.jit_only {
+        &[ExecMode::Jit]
+    } else if config.jit {
         &[ExecMode::Interpreter, ExecMode::Jit]
     } else {
         &[ExecMode::Interpreter]
@@ -96,23 +116,32 @@ fn run() -> Result<(), String> {
     for workload in [Workload::Array, Workload::Map] {
         let program = compile_workload(workload, config.width, config.iterations)?;
         for mode in modes {
-            let samples = measure(&program, workload, *mode, config)?;
-            let mut sorted = samples.clone();
+            let measurement = measure(&program, workload, *mode, config)?;
+            let mut sorted = measurement.samples.clone();
             sorted.sort_unstable();
             let median = sorted[sorted.len() / 2];
-            let sample_ns = samples
+            let sample_ns = measurement
+                .samples
                 .iter()
                 .map(Duration::as_nanos)
                 .map(|value| value.to_string())
                 .collect::<Vec<_>>()
                 .join(",");
             println!(
-                "workload={} mode={} width={} iterations={} median_ns={} samples_ns={}",
+                "workload={} mode={} width={} iterations={} warmup_runs=1 reused_vm=true median_ns={} native_traces={} call_boundary_traces={} loop_back_traces={} native_execs={} trace_exits={} native_loop_backs={} helper_fallbacks={} generic_builtin_calls={} samples_ns={}",
                 workload.label(),
                 mode.label(),
                 config.width,
                 config.iterations,
                 median.as_nanos(),
+                measurement.native_traces,
+                measurement.call_boundary_traces,
+                measurement.loop_back_traces,
+                measurement.native_execs,
+                measurement.trace_exits,
+                measurement.native_loop_backs,
+                measurement.helper_fallbacks,
+                measurement.generic_builtin_calls,
                 sample_ns
             );
         }
@@ -157,34 +186,78 @@ fn measure(
     workload: Workload,
     mode: ExecMode,
     config: Config,
-) -> Result<Vec<Duration>, String> {
-    run_once(program, workload, mode, config)?;
+) -> Result<Measurement, String> {
+    let mut vm = configured_vm(program, mode);
+    let warmup = vm
+        .run()
+        .map_err(|err| format!("{} {} warmup failed: {err}", workload.label(), mode.label()))?;
+    verify_result(&vm, warmup, config)?;
+    if matches!(mode, ExecMode::Jit) && vm.jit_native_exec_count() == 0 {
+        return Err(format!(
+            "{} warmup did not execute native traces:\n{}",
+            workload.label(),
+            vm.dump_jit_info()
+        ));
+    }
+
+    let snapshot_before = vm.jit_snapshot();
+    let metrics_before = snapshot_before.metrics;
+    let native_traces = snapshot_before.traces.len();
+    let call_boundary_traces = snapshot_before
+        .traces
+        .iter()
+        .filter(|trace| trace.has_call)
+        .count();
+    let loop_back_traces = snapshot_before
+        .traces
+        .iter()
+        .filter(|trace| trace.terminal == JitTraceTerminal::LoopBack)
+        .count();
     let mut samples = Vec::with_capacity(config.samples);
+    let mut generic_builtin_calls = 0u64;
     for _ in 0..config.samples {
-        let mut vm = configured_vm(program, mode);
+        vm.reset_for_reuse();
         let started = Instant::now();
         let status = vm
             .run()
             .map_err(|err| format!("{} {} run failed: {err}", workload.label(), mode.label()))?;
         let elapsed = started.elapsed();
         verify_result(&vm, status, config)?;
+        generic_builtin_calls = generic_builtin_calls
+            .saturating_add(vm.interpreter_metrics_snapshot().generic_builtin_call_count);
         black_box(vm.stack());
         samples.push(elapsed);
     }
-    Ok(samples)
-}
-
-fn run_once(
-    program: &Program,
-    workload: Workload,
-    mode: ExecMode,
-    config: Config,
-) -> Result<(), String> {
-    let mut vm = configured_vm(program, mode);
-    let status = vm
-        .run()
-        .map_err(|err| format!("{} {} warmup failed: {err}", workload.label(), mode.label()))?;
-    verify_result(&vm, status, config)
+    let snapshot_after = vm.jit_snapshot();
+    if snapshot_after.traces.len() != native_traces {
+        return Err(format!(
+            "{} {} compiled additional traces during measured runs: warmup={} measured={}",
+            workload.label(),
+            mode.label(),
+            native_traces,
+            snapshot_after.traces.len()
+        ));
+    }
+    let metrics_after = snapshot_after.metrics;
+    Ok(Measurement {
+        samples,
+        native_traces,
+        call_boundary_traces,
+        loop_back_traces,
+        native_execs: metrics_after
+            .native_trace_exec_count
+            .saturating_sub(metrics_before.native_trace_exec_count),
+        trace_exits: metrics_after
+            .trace_exit_count
+            .saturating_sub(metrics_before.trace_exit_count),
+        native_loop_backs: metrics_after
+            .native_loop_back_count
+            .saturating_sub(metrics_before.native_loop_back_count),
+        helper_fallbacks: metrics_after
+            .helper_fallback_count
+            .saturating_sub(metrics_before.helper_fallback_count),
+        generic_builtin_calls,
+    })
 }
 
 fn configured_vm(program: &Program, mode: ExecMode) -> Vm {