fix(gpu): review follow-ups on the round-4 residency PR

Makefile

@@ -590,7 +590,7 @@ test-cuda-integration:
 test-cuda-fallback:
 	$(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features test-cuda-faults \
 	    --test cuda_fallback_tests -- --ignored --nocapture --test-threads=1
-	cargo test -p lambda-vm-prover --release --features lambda-vm-prover/cuda \
+	$(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features lambda-vm-prover/cuda \
 	    --test gpu_force_downgrade -- --ignored --nocapture --test-threads=1
 
 # The prover/stark/crypto/ecsm test suite with the GPU (cuda) path enabled (requires NVIDIA

crypto/math-cuda/src/barycentric.rs

@@ -586,3 +586,28 @@ pub fn gather_rows_ext3_on_device(
     stream.synchronize()?;
     Ok(host)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::bary_num_chunks;
+
+    /// Pins which of the three terms binds, per regime. Pure arithmetic — the
+    /// kernels' parity across chunk counts is covered by
+    /// `tests/barycentric_multi.rs`, which allocates a GPU.
+    #[test]
+    fn bary_num_chunks_branches() {
+        // Rows-bound: the domain is too short to split further, whatever the
+        // grid wants. 2^14/8192 = 2, under the occupancy term's 2048/100 = 20.
+        assert_eq!(bary_num_chunks(100, 1 << 14), 2);
+        // Occupancy-bound: the columns alone nearly fill the grid, so the
+        // domain is split less than its length would allow. 2048/256 = 8,
+        // under the rows term's 2^17/8192 = 16.
+        assert_eq!(bary_num_chunks(256, 1 << 17), 8);
+        // Cap-bound: at production shapes both terms clear 64 (512 and 128).
+        assert_eq!(bary_num_chunks(4, 1 << 20), 64);
+        // Degenerate inputs still yield a launchable grid (>= 1 chunk).
+        assert_eq!(bary_num_chunks(0, 0), 1);
+        assert_eq!(bary_num_chunks(usize::MAX, 1 << 20), 1);
+        assert_eq!(bary_num_chunks(1, 0), 1);
+    }
+}

crypto/math-cuda/tests/barycentric_multi.rs

@@ -140,14 +140,17 @@ fn run_ext3(log_trace: u32, blowup: usize, num_cols: usize, k_points: usize, see
 #[test]
 fn bary_base_multi_matches_single_point() {
     // Covers: k=1 degenerate, the production k=2, the kernel cap k=8, a
-    // single-chunk tiny n, and a column count that forces the chunk heuristic
-    // to its occupancy branch.
+    // single-chunk tiny n, a multi-chunk mid case, and the 64-chunk cap —
+    // the most chunks any shape can ask for, so parity is pinned at both
+    // ends of the chunk range. (`bary_num_chunks`'s own branch selection is
+    // covered by its unit tests; only the kernels are exercised here.)
     for (log_t, blowup, cols, k) in [
         (4u32, 2usize, 3usize, 1usize),
         (8, 4, 10, 2),
         (12, 2, 5, 3),
         (14, 2, 100, 2),
         (10, 2, 4, 8),
+        (20, 2, 4, 2),
     ] {
         run_base(log_t, blowup, cols, k, 3000 + log_t as u64 + k as u64);
     }
@@ -161,6 +164,7 @@ fn bary_ext3_multi_matches_single_point() {
         (10, 2, 3, 3),
         (14, 2, 40, 2),
         (10, 2, 4, 8),
+        (19, 2, 2, 2),
     ] {
         run_ext3(log_t, blowup, cols, k, 4000 + log_t as u64 + k as u64);
     }

crypto/stark/src/gpu_lde.rs

@@ -46,6 +46,12 @@ use crate::trace::LDETraceTable;
 /// measured sweep optimum on ethrex continuations (2^14 beats 2^15..2^19 and
 /// also beats "everything on GPU", where sub-2^14 tables lose to launch
 /// overhead). Override via env var for tuning.
+///
+/// The same value gates the whole dispatch layer, not just the commit: R2
+/// decompose, the R3 inv-denoms/barycentric contexts, R4 DEEP and the FRI
+/// fold all admit on it, so moving it moves every one of those floors
+/// together. The device-only envelope is the one gate that does NOT ride on
+/// it — see [`DEFAULT_DEVICE_ONLY_MIN_LDE`].
 const DEFAULT_GPU_LDE_THRESHOLD: usize = 1 << 14;
 
 fn gpu_lde_threshold() -> usize {
@@ -66,6 +72,15 @@ fn gpu_lde_threshold() -> usize {
 /// eligibility (the LOCKSTEP note below). Keep device-only to the large-table
 /// envelope where those paths are exercised; mid tables keep a host copy so a
 /// dispatch decline degrades to CPU instead of aborting.
+///
+/// That degradation covers the sites that READ the LDE — they all gate on
+/// `host_trace_empty()` and take their host arm. It does NOT cover the R4
+/// Merkle-proof gather: the host tree is root-only for every GPU-committed
+/// table (the tree stays resident from [`DEFAULT_GPU_LDE_THRESHOLD`] upward,
+/// whatever `retain_host_lde` says), so a declined `gather_proofs_dev` has
+/// nothing to fall back to and aborts regardless of the host LDE. Lowering
+/// the commit threshold therefore widens that one abort site even though it
+/// leaves this envelope alone.
 const DEFAULT_DEVICE_ONLY_MIN_LDE: usize = 1 << 19;
 
 fn gpu_device_only_threshold() -> usize {
@@ -2633,8 +2648,9 @@ where
 /// returning one [`Proof`] per position in the same order. Byte-identical to
 /// the host `MerkleTree::get_proof_by_pos` (guarded by the `merkle_gather`
 /// parity test), so R4 query openings can source proofs from the resident
-/// device tree instead of the host tree. Returns `None` on any cudarc error
-/// (the caller then falls back to the host tree).
+/// device tree instead of the host tree. Returns `None` on any cudarc error —
+/// which every caller treats as a hard abort, NOT a fallback: a resident tree
+/// leaves the host tree root-only, so there is no host path to walk.
 pub(crate) fn gather_proofs_dev(
     tree: &math_cuda::lde::GpuMerkleTree,
     positions: &[usize],
@@ -3162,7 +3178,8 @@ mod split_tree_tests {
     /// isolates the tree layout/hashing under test.
     #[test]
     fn split_trees_match_cpu_subset_commits() {
-        // Above the dispatch threshold (2^19 LDE) so the GPU path must engage.
+        // This shape's LDE is 2^19, well above the dispatch threshold, so the
+        // GPU path must engage.
         let n: usize = 1 << 18;
         let blowup: usize = 2;
         let m: usize = 5;

crypto/stark/src/prover.rs

@@ -2687,9 +2687,11 @@ pub trait IsStarkProver<
                 !lde_trace.host_trace_empty(),
                 "R4 {what} opening fell back to the host tree, but it is device-only (empty)"
             );
-            // A root-only host tree means the nodes are device-resident: the
-            // host walk would emit an empty path for position 0 instead of
-            // failing, so a broken proofs↔tree pairing must abort here.
+            // A root-only host tree means the nodes are device-resident, so a
+            // broken proofs↔tree pairing must abort here. `get_proof_by_pos`
+            // already refuses a root-only tree, but the panic it produces
+            // downstream reads "FRI query index in bounds" — this names the
+            // real cause instead.
             assert!(
                 !tree.is_root_only(),
                 "R4 {what} opening fell back to a root-only host tree (nodes device-resident)"

crypto/stark/src/trace.rs

@@ -54,8 +54,10 @@ where
 }
 
 /// Device-resident row-major main trace, pre-uploaded ahead of the prove.
-/// Excluded from logical trace equality and opaque in `Debug`, matching
-/// [`ResidentMainTrace`].
+/// Opaque in `Debug` like [`ResidentMainTrace`], and fully excluded from
+/// logical trace equality: this is a cache of data the host trace still owns,
+/// so two traces that differ only here are equal. (`ResidentMainTrace` still
+/// compares its row count, because it can be the sole owner of the data.)
 #[cfg(feature = "cuda")]
 #[derive(Clone)]
 pub(crate) struct PreUploadedMainTrace {

scripts/profiling/README.md

@@ -104,6 +104,7 @@ the phase that enqueued them even if they execute later.
 | `capture_env.sh` | env JSON to stdout — attach to anything you measure by hand |
 | `phase_table.py [--util u.csv]… tl.json…` | aggregate timelines; `--instances LABEL` adds per-instance tables for deeper repeated spans, `--min-pct X` hides noise rows |
 | `nsys_phase_busy.py report.sqlite [--top N]` | the GPU busy report from `nsys export --type sqlite` |
+| `h2d_histo.py report.sqlite` | H2D/D2H bytes grouped by (phase, innermost NVTX range, transfer size) — names the dominant uploaders inside a phase. Prints the top 20 per direction |
 | `nvml_sampler.py -o out.csv [-i 0.1]` | standalone 10 Hz GPU util sampler (epoch-ns timestamps, aligns with span `start_ns`) |
 | `timeline_to_perfetto.py tl.json > trace.json` | span tree for ui.perfetto.dev |
 
@@ -122,6 +123,15 @@ Useful prover knobs for A/B experiments (pre-existing, see plan §11):
 `LAMBDA_VM_GPU_BARY_THRESHOLD`, `LAMBDA_VM_VRAM_BUDGET_MB`,
 `TABLE_PARALLELISM`.
 
+Residency and diagnostic knobs:
+
+| var | effect |
+|---|---|
+| `LAMBDA_VM_GPU_DEVICE_ONLY_THRESHOLD=<lde_size>` | minimum LDE size for the device-only envelope (default 2^19), independent of `LAMBDA_VM_GPU_LDE_THRESHOLD`. Raise it to shed device-only tables without giving up GPU commits — a finer instrument than `LAMBDA_VM_DISABLE_DEVICE_ONLY=1` |
+| `LAMBDA_VM_TRACE_PREUPLOAD_MB=<mb>` | budget for pre-uploading the epoch's biggest main traces from the builder thread, so R1 commits D2D-copy instead of paying their H2D. Default 0 (off); capped at a quarter of the device VRAM budget. Wall-neutral on a 5090 and it competes with the prove peak on small cards, so it is for PCIe-bound setups |
+| `LAMBDA_VM_GPU_FORCE_DOWNGRADE=1` | test hook: decline the device R2 path unconditionally, so every device-only table exercises the host recovery. Used by the `gpu_force_downgrade` test |
+| `LAMBDA_VM_GPU_XCHECK=1` | after each table, re-run the verifier's composition consistency check in-process; on a mismatch, recompute each device stage on host, report which one diverged, and abort. For localizing silent device-side corruption |
+
 ## Continuations: per-epoch data for parallelization
 
 `prove_continuation` is instrumented independently of the monolithic path

scripts/profiling/h2d_histo.py

@@ -33,7 +33,7 @@ def main():
     api = load_api_calls(con, tset)
     chain_at = build_range_lookup(nvtx)
 
-    def chain_for(corr, start):
+    def chain_for(corr):
         if corr in api:
             api_start, tid = api[corr]
             c = chain_at(tid, api_start)
@@ -55,7 +55,7 @@ def innermost(chain):
     for start, end, kind, nbytes, corr in memcpys:
         if kind not in ("h2d", "d2h"):
             continue
-        chain = chain_for(corr, start)
+        chain = chain_for(corr)
         key = (kind, coarse_of(chain), innermost(chain), nbytes)
         h = hist[key]
         h[0] += 1