From 7702140784d5c71023d4eebd2f743ce1b5c855b9 Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Fri, 12 Jun 2026 01:33:47 -0400 Subject: [PATCH 01/12] =?UTF-8?q?nanowm=5Frollout=5Fspeedup:=20core=20?= =?UTF-8?q?=E2=80=94=20scoring/settings/runner=20(apply=20patch=20->=20CSG?= =?UTF-8?q?O=20rollout=20->=20speedup=20+=20LPIPS=20guardrail)=20+=20bf16?= =?UTF-8?q?=20reference.patch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../nanowm_rollout_speedup/reference.patch | 13 ++ .../speedup_eval/runner.py | 117 ++++++++++++++++++ .../speedup_eval/scoring.py | 68 ++++++++++ .../speedup_eval/settings.py | 56 +++++++++ 4 files changed, 254 insertions(+) create mode 100644 2.0/problems/nanowm_rollout_speedup/reference.patch create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/scoring.py create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py diff --git a/2.0/problems/nanowm_rollout_speedup/reference.patch b/2.0/problems/nanowm_rollout_speedup/reference.patch new file mode 100644 index 00000000..e1f6a410 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/reference.patch @@ -0,0 +1,13 @@ +diff --git a/src/diffusion/gaussian_diffusion.py b/src/diffusion/gaussian_diffusion.py +index f0c799a..3726b95 100644 +--- a/src/diffusion/gaussian_diffusion.py ++++ b/src/diffusion/gaussian_diffusion.py +@@ -838,7 +838,7 @@ class GaussianDiffusion: + # Backup for context restoration + img_prev = img.clone() + +- with th.no_grad(): ++ with th.no_grad(), th.autocast("cuda", dtype=th.bfloat16): # [speedup-ref] bf16 sampling + out = self.dfot_ddim_sample( + model, + img, diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py new file mode 100644 index 00000000..9ef83a78 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py @@ -0,0 +1,117 @@ +"""Apply a sampler patch to a clean nano-world-model checkout, run the FIXED +CSGO rollout, and measure (generation wall-clock, LPIPS-vs-GT). + +The judge fixes the rollout invocation (rollout_length / history / num_steps / +scheduling); the agent's patch may only change the *sampling internals* +(`src/diffusion/**`, `src/sample/sampling_utils.py`) to make that invocation +faster at iso-quality. Speedup = baseline_seconds / patched_seconds; quality = +LPIPS vs ground truth, guardrailed against the unpatched baseline. + +Runnable standalone for local/Modal validation: + python -m speedup_eval.runner --repo --patch \ + --out metrics.json [--steps 50 --clips 4] +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +from . import settings as S + + +def apply_patch(clean_repo: Path, patch_path: Path | None) -> Path: + work = Path(tempfile.mkdtemp(prefix="nwm_speedup_")) + repo = work / "repo" + # symlink the heavy/immutable parts, copy only src/ so patches are cheap + repo.mkdir(parents=True) + for child in clean_repo.iterdir(): + if child.name == "src": + shutil.copytree(child, repo / "src") + else: + (repo / child.name).symlink_to(child) + if patch_path is not None and str(patch_path) != "-" and Path(patch_path).stat().st_size > 0: + r = subprocess.run(["git", "apply", "--unsafe-paths", "--directory", str(repo), + str(patch_path)], capture_output=True, text=True) + if r.returncode != 0: + # fall back to patch(1) (git apply needs a git root / clean context) + r = subprocess.run(["patch", "-p1", "-d", str(repo), "-i", str(patch_path)], + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError(f"patch failed: {r.stderr[:2000]}") + return repo + + +def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, steps: int, + device: str = "cuda") -> float: + """Run rollout.py; return generation wall-clock seconds (rollout-region timed + inside; falls back to total wall).""" + save.mkdir(parents=True, exist_ok=True) + env = dict(os.environ) + env["CSGO_DATA_DIR"] = str(S.CSGO_DATA) + env["NWM_TIME_FILE"] = str(save / "gen_seconds.txt") + cmd = [sys.executable, "-u", "src/sample/rollout.py", "--config", str(config), + "--ckpt", str(ckpt), "--save_path", str(save), "--num_samples", str(clips), + "--batch_size", str(S.BATCH_SIZE), "--rollout_length", str(S.ROLLOUT_LENGTH), + "--history_length", str(S.HISTORY_LENGTH), "--scheduling_mode", S.SCHEDULING_MODE, + "--num_sampling_steps", str(steps), "--history_stabilization_level", str(S.HISTORY_STAB), + "--fps", "8"] + t0 = time.time() + r = subprocess.run(cmd, cwd=str(repo), env=env, capture_output=True, text=True) + wall = time.time() - t0 + if r.returncode != 0: + raise RuntimeError(f"rollout failed (exit {r.returncode}): {r.stderr[-3000:]}") + tf = save / "gen_seconds.txt" + if tf.exists(): + try: + return float(tf.read_text().strip()) + except Exception: + pass + return wall + + +def lpips_vs_gt(repo: Path, save: Path) -> float: + out_csv = save / "metrics.csv" + r = subprocess.run([sys.executable, "src/sample/evaluate_metrics.py", "--video_dir", str(save), + "--history_length", str(S.HISTORY_LENGTH), "--output_csv", str(out_csv)], + cwd=str(repo), env={**os.environ, "CSGO_DATA_DIR": str(S.CSGO_DATA)}, + capture_output=True, text=True) + if not out_csv.exists(): + raise RuntimeError(f"metrics failed: {r.stderr[-2000:]}") + import pandas as pd + return float(pd.read_csv(out_csv)["lpips"].mean()) + + +def evaluate(clean_repo: Path, patch_path: Path | None, config: Path, ckpt: Path, + clips: int, steps: int, device: str = "cuda") -> dict: + repo = apply_patch(clean_repo, patch_path) + try: + save = repo / "_rollout_out" + secs = run_rollout(repo, config, ckpt, save, clips, steps, device) + lp = lpips_vs_gt(repo, save) + return {"gen_seconds": secs, "lpips": lp, "clips": clips, "steps": steps} + finally: + shutil.rmtree(repo.parent, ignore_errors=True) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--repo", required=True) + ap.add_argument("--patch", default="-") + ap.add_argument("--config", required=True) + ap.add_argument("--ckpt", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--clips", type=int, default=S.QUICK_CLIPS) + ap.add_argument("--steps", type=int, default=S.NUM_SAMPLING_STEPS) + ap.add_argument("--device", default="cuda") + a = ap.parse_args() + m = evaluate(Path(a.repo), None if a.patch == "-" else Path(a.patch), Path(a.config), + Path(a.ckpt), a.clips, a.steps, a.device) + Path(a.out).write_text(json.dumps(m, indent=2)) + print(json.dumps(m, indent=2)) diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/scoring.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/scoring.py new file mode 100644 index 00000000..1074d65f --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/scoring.py @@ -0,0 +1,68 @@ +"""Scoring math for nanowm_rollout_speedup (shared by the judge evaluator and +the agent-facing public test; keep the two in sync). + +Mirrors the Frontier-CS 2.0 latency-optimization convention (cf. +vllm_llm_serving_optimization): geometric-mean per-clip speedup over a fixed +baseline, mapped through 100*log2, gated by an accuracy guardrail. Here +"accuracy" is rollout fidelity to ground truth (lower LPIPS = better), so the +guardrail penalizes patched LPIPS rising above the baseline's by more than a +tolerance. +""" +from __future__ import annotations + +import math + + +def geometric_mean(values: list[float]) -> float: + if not values: + return 0.0 + return math.exp(sum(math.log(max(v, 1e-9)) for v in values) / len(values)) + + +def paired_speedups(baseline_s: dict[str, float], patched_s: dict[str, float]) -> list[float]: + """Per-clip baseline_seconds / patched_seconds (floored at 0.01).""" + out: list[float] = [] + for cid, p in patched_s.items(): + b = baseline_s.get(cid) + if b is None or p <= 0 or b <= 0: + continue + out.append(max(b / p, 0.01)) + return out + + +def score_from_speedup(speedup: float) -> float: + if speedup <= 0: + return 0.0 + return max(0.0, min(100.0, 100.0 * math.log(speedup, 2))) + + +def quality_multiplier(baseline_lpips: float, patched_lpips: float, tolerance: float) -> float: + """1.0 while patched LPIPS stays within `tolerance` relative of baseline; + decays inverse-proportionally beyond that. (Lower LPIPS = better, so the + penalty is on RISES above baseline.)""" + base = max(baseline_lpips, 1e-9) + rel_rise = max(0.0, (patched_lpips - baseline_lpips) / base) + if rel_rise <= tolerance: + return 1.0 + return max(0.0, min(1.0, tolerance / rel_rise)) + + +def provisional_score( + baseline_seconds: dict[str, float], + patched_seconds: dict[str, float], + baseline_lpips: float, + patched_lpips: float, + tolerance: float, +) -> dict[str, float]: + speedups = paired_speedups(baseline_seconds, patched_seconds) + gm = geometric_mean(speedups) if speedups else 0.0 + latency_score = score_from_speedup(gm) + qmult = quality_multiplier(baseline_lpips, patched_lpips, tolerance) + return { + "geomean_speedup": gm, + "latency_score": latency_score, + "quality_multiplier": qmult, + "score": max(0.0, min(100.0, latency_score * qmult)), + "score_unbounded": max(0.0, 100.0 * math.log(max(gm, 1e-9), 2)) * qmult, + "clips_scored": float(len(speedups)), + } diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py new file mode 100644 index 00000000..79942cfc --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py @@ -0,0 +1,56 @@ +"""Task settings for nanowm_rollout_speedup, overridable via config.yaml +`evaluation` block (read by the judge) or env vars (local runs).""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +TASK_CONFIG_PATH = Path(os.environ.get("FRONTIER_NWM_TASK_CONFIG", "/judge/task_config.json")) + + +def _eval_cfg() -> dict: + try: + return (json.loads(TASK_CONFIG_PATH.read_text()).get("evaluation", {}) or {}) + except Exception: + return {} + + +_CFG = _eval_cfg() + + +def _get(name: str, default): + if name in os.environ: + v = os.environ[name] + try: + return type(default)(v) + except Exception: + return v + return _CFG.get(name.lower().replace("frontier_nwm_", ""), default) + + +# Domain / model +MODEL = _get("FRONTIER_NWM_MODEL", "nanowm_l2_csgo") +DATASET = _get("FRONTIER_NWM_DATASET", "game/csgo") +# Rollout protocol (the FIXED invocation; the agent's patch changes the sampler +# internals, not these). +ROLLOUT_LENGTH = int(_get("FRONTIER_NWM_ROLLOUT_LENGTH", 50)) +HISTORY_LENGTH = int(_get("FRONTIER_NWM_HISTORY_LENGTH", 4)) +NUM_SAMPLING_STEPS = int(_get("FRONTIER_NWM_NUM_STEPS", 50)) # nominal reference budget +SCHEDULING_MODE = _get("FRONTIER_NWM_SCHEDULING", "sequential") +HISTORY_STAB = float(_get("FRONTIER_NWM_HISTORY_STAB", 0.02)) +# Eval set sizes +QUICK_CLIPS = int(_get("FRONTIER_NWM_QUICK_CLIPS", 4)) +FINAL_CLIPS = int(_get("FRONTIER_NWM_FINAL_CLIPS", 16)) +BATCH_SIZE = int(_get("FRONTIER_NWM_BATCH_SIZE", 4)) +# Quality guardrail: patched rollout LPIPS may rise at most this (relative) +# above the baseline (unpatched seq@50) LPIPS before the score is penalized. +QUALITY_TOLERANCE = float(_get("FRONTIER_NWM_QUALITY_TOLERANCE", 0.03)) +# Image-asset layout (baked into the images; overridable locally). +REPO = Path(_get("FRONTIER_NWM_REPO", "/opt/nanowm/nano-world-model")) +CKPT = Path(_get("FRONTIER_NWM_CKPT", "/opt/nanowm/ckpts/nanowm-l2-csgo/model_state_dict.pt")) +CSGO_DATA = Path(_get("FRONTIER_NWM_CSGO_DATA", "/opt/nanowm/data/csgo")) +VAL_FILES = Path(_get("FRONTIER_NWM_VAL_FILES", "/opt/nanowm/data/csgo_subset/val_files.txt")) +VAL_STARTS = Path(_get("FRONTIER_NWM_VAL_STARTS", "/opt/nanowm/data/csgo_subset/val_starts.npy")) +BASELINE_CACHE = Path(_get("FRONTIER_NWM_BASELINE_CACHE", "/opt/nanowm/baseline/baseline_metrics.json")) +SMOKE = os.environ.get("FRONTIER_NWM_SMOKE", "0") == "1" From 5fafb26e316fd5f49014fd3ea0d1a44e672b225c Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Fri, 12 Jun 2026 01:41:13 -0400 Subject: [PATCH 02/12] =?UTF-8?q?nanowm=5Frollout=5Fspeedup:=20full=20#145?= =?UTF-8?q?-style=20scaffold=20=E2=80=94=20config,=20readme,=20DESIGN,=20e?= =?UTF-8?q?valuator=20(patch=20policy=20validated:=20accepts=20ref,=20reje?= =?UTF-8?q?cts=20metric-edit+env-leak),=20orchestrate+modal=5Fapp=20(Modal?= =?UTF-8?q?=20GPU,=20judge=20CPU),=20harbor/app,=20docker=20agent+judge,?= =?UTF-8?q?=20infra=20patch,=20evaluate.sh.=20Patch-policy=20+=20smoke=20v?= =?UTF-8?q?alidated=20on=20CPU;=20GPU=20runner=20validating=20on=20H100?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 2.0/problems/nanowm_rollout_speedup/DESIGN.md | 83 +++++++ .../nanowm_rollout_speedup/config.yaml | 60 ++++++ .../docker/agent/Dockerfile | 32 +++ .../docker/build_images.sh | 32 +++ .../docker/judge/Dockerfile | 35 +++ .../nanowm_rollout_speedup/evaluate.sh | 14 ++ .../nanowm_rollout_speedup/evaluator.py | 204 ++++++++++++++++++ .../harbor/app/README.md | 19 ++ .../harbor/app/make_submission.sh | 7 + .../harbor/app/public_test.py | 28 +++ .../harbor/app/public_test.sh | 4 + .../harbor/app/solution.patch | 0 .../0001-rollout-chunked-decode.patch | 23 ++ 2.0/problems/nanowm_rollout_speedup/readme | 73 +++++++ .../nanowm_rollout_speedup/reference.py | 20 ++ .../speedup_eval/__init__.py | 1 + .../speedup_eval/modal_app.py | 66 ++++++ .../speedup_eval/orchestrate.py | 91 ++++++++ 18 files changed, 792 insertions(+) create mode 100644 2.0/problems/nanowm_rollout_speedup/DESIGN.md create mode 100644 2.0/problems/nanowm_rollout_speedup/config.yaml create mode 100644 2.0/problems/nanowm_rollout_speedup/docker/agent/Dockerfile create mode 100755 2.0/problems/nanowm_rollout_speedup/docker/build_images.sh create mode 100644 2.0/problems/nanowm_rollout_speedup/docker/judge/Dockerfile create mode 100755 2.0/problems/nanowm_rollout_speedup/evaluate.sh create mode 100644 2.0/problems/nanowm_rollout_speedup/evaluator.py create mode 100644 2.0/problems/nanowm_rollout_speedup/harbor/app/README.md create mode 100755 2.0/problems/nanowm_rollout_speedup/harbor/app/make_submission.sh create mode 100644 2.0/problems/nanowm_rollout_speedup/harbor/app/public_test.py create mode 100755 2.0/problems/nanowm_rollout_speedup/harbor/app/public_test.sh create mode 100644 2.0/problems/nanowm_rollout_speedup/harbor/app/solution.patch create mode 100644 2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch create mode 100644 2.0/problems/nanowm_rollout_speedup/readme create mode 100644 2.0/problems/nanowm_rollout_speedup/reference.py create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/__init__.py create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/modal_app.py create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py diff --git a/2.0/problems/nanowm_rollout_speedup/DESIGN.md b/2.0/problems/nanowm_rollout_speedup/DESIGN.md new file mode 100644 index 00000000..8c998253 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/DESIGN.md @@ -0,0 +1,83 @@ +# Design — nanowm_rollout_speedup + +## 1. Task + +Optimize the inference **latency** of a frozen video world model's +autoregressive long-rollout, at iso-quality. The agent submits a Python-only +patch to the diffusion **sampling** layer of Nano World Models (arXiv:2605.23993); +the judge runs a fixed NanoWM-L/2 CSGO 50-frame rollout, scores wall-clock +speedup over the unpatched baseline, gated by an LPIPS-vs-GT quality guardrail. + +Shape and scoring mirror `vllm_llm_serving_optimization` (#145): patch a source +tree, build/run on a Modal GPU from a CPU judge, score `100·log2(speedup)` with +an accuracy guardrail. **No change to the general 2.0 adapter/template** — GPU is +per-problem via Modal. + +## 2. Why this is a real task (calibration) + +Measured on Della H100, NanoWM-L/2 CSGO, 50-frame rollout, 12 held-out episodes, +LPIPS vs ground truth: + +| DDIM steps | speedup | LPIPS-vs-GT | Δ vs seq@50 | +|---|---|---|---| +| 50 (baseline) | 1× | 0.517 | — | +| 20 | 2.5× | 0.543 | +5% | +| 10 | 5× | 0.543 | +5% | +| 5 | 10× | 0.579 | +12% | +| 2 | 25× | 0.676 | +31% | + +CSGO has a genuine steps↔quality frontier (paper Fig. 6): naive step-cutting +degrades quality fast (seq@5 = +12%, seq@2 = +31%). With a 3% guardrail, even +seq@20 (+5%) fails, so a winning patch must reproduce ~50-step quality with less +compute via real fast-sampling techniques. (Contrast: on the visually-trivial +PushT domain, seq@2 reproduces seq@250 within the stochastic noise floor — no +frontier; CSGO was chosen precisely because the frontier is real.) + +## 3. Patch policy (validated before running) + +Python-only (`.py`/`.pyi`), ≤256 KB, no file deletion, safe paths. + +- **Allowed:** `src/diffusion/**.py`, `src/sample/sampling_utils.py` (the sampling + layer: scheduling matrices, the DDIM/diffusion-forcing loop, solvers, caches). +- **Denied:** model (`src/models/**`), VAE (`src/latent_codecs/**`), metric + (`src/sample/evaluate_metrics.py`), harness (`src/sample/rollout.py`), data + (`src/wm_datasets/**`), training/eval/utils, native/build/deps. +- **Rejected added-line tokens:** `FRONTIER_*/JUDGE_/HARBOR_/MODAL_/HF_TOKEN`, + metric/GT/timing identifiers, `os.environ`, `subprocess`, `socket`, + `time.sleep`, `while True`, hard-coded judge paths — i.e. no benchmark + detection, env-var leakage, output hard-coding, or timing short-circuits. + +The rollout invocation (length/context/nominal-steps/scheduling) is fixed by the +judge; the agent changes only sampler internals (cf. #145 fixing the serving +config and patching the scheduler). + +## 4. GPU on Modal (judge stays CPU) + +`speedup_eval/modal_app.py` runs one rollout in a Modal GPU function from baked +assets (NanoWM checkout + L/2 CSGO ckpt + held-out CSGO episode subset); +`orchestrate.run_pair` computes the cached vanilla baseline once and the patched +run per submission. A `local` backend (`orchestrate._run_local`) runs the same +`speedup_eval.runner` on a directly-visible GPU — this is the path validated on +Della H100. End-to-end Modal execution awaits maintainer Modal credentials. + +## 5. Scoring + +`speedup_eval/scoring.py` (shared with the public test): geomean rollout speedup +→ `clip(100·log2, 0, 100) · quality_multiplier`, where the multiplier is 1.0 +within `quality_tolerance` LPIPS rise and decays inverse-proportionally beyond. +`score_unbounded` keeps rewarding speedup past 2×. + +## 6. Reference solution + +`reference.patch`: a one-line bf16-autocast wrap of the sampling loop in +`gaussian_diffusion.dfot_sample_loop` — a quality-preserving speedup that beats +the fp32 baseline (CI requires reference > baseline). The intended frontier +(DPM-Solver++, caching, distillation) is left to the agent. + +## 7. Open items for maintainers + +- Modal end-to-end run + a deployed `modal_app` app name / GPU type confirmation. +- Bake-asset provenance for the judge image (ckpt + held-out CSGO subset + cached + baseline metrics); the held-out episode ids are the only hidden component. +- CI smoke (`FRONTIER_NWM_SMOKE=1`, CPU) validates the patch policy + empty + reference; confirm this matches the repo's CI expectation for GPU/Modal tasks. diff --git a/2.0/problems/nanowm_rollout_speedup/config.yaml b/2.0/problems/nanowm_rollout_speedup/config.yaml new file mode 100644 index 00000000..939b420d --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/config.yaml @@ -0,0 +1,60 @@ +tag: ai +runtime: + language: patch + timeout_seconds: 21600 + environment: >- + Python-only patch against a clean NanoWM checkout (Nano World Models, + arXiv:2605.23993); Modal GPU runs the NanoWM-L/2 CSGO 50-frame long-rollout; + speedup-vs-baseline judge with an LPIPS rollout-quality guardrail + apt_packages: + - bash + - ca-certificates + - curl + - git + - python3 + - python3-pip + judge_apt_packages: + - bash + - ca-certificates + - curl + - git + - python3 + - python3-pip + judge_pip_packages: + - modal + docker: + # Experimental local images; build with docker/build_images.sh before a local + # Harbor trial. Both bake a clean NanoWM checkout + the L/2 CSGO ckpt; the + # judge image additionally vendors the held-out CSGO episode subset, the + # LPIPS scorer, and the cached vanilla baseline metrics. + image: frontiercs/nanowm-rollout-speedup-agent:experimental-v0 + judge_image: frontiercs/nanowm-rollout-speedup-judge:experimental-v0 +environment: + cpus: 8 + memory_mb: 32768 + storage_mb: 32768 + build_timeout_seconds: 5400 +evaluation: + # GPU served on Modal (one per environment); judge container is CPU-only. + model: nanowm_l2_csgo + dataset: game/csgo + gpu: L40S + # FIXED rollout invocation (the agent's patch changes sampler internals, not these). + rollout_length: 50 + history_length: 4 + num_steps: 50 # nominal reference DDIM budget + scheduling: sequential + history_stab: 0.02 + # Quality guardrail: patched rollout LPIPS-vs-GT may rise at most this + # (relative) above the unpatched seq@50 baseline before the score is penalized. + # Calibrated: seq@20 is already +5% over seq@50, so a 3% tolerance forces real + # fast-sampling work (DPM-Solver++, caching, distillation), not naive step cuts. + quality_tolerance: 0.03 + quick_clips: 4 # iterative (agent-role) public feedback + final_clips: 16 # final (verifier-role) evaluation + batch_size: 4 + baseline_cache_path: /opt/nanowm/baseline/baseline_metrics.json +submission: + kind: file + path: /app/solution.patch + max_queue_size: 2 diff --git a/2.0/problems/nanowm_rollout_speedup/docker/agent/Dockerfile b/2.0/problems/nanowm_rollout_speedup/docker/agent/Dockerfile new file mode 100644 index 00000000..c4de13c8 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/docker/agent/Dockerfile @@ -0,0 +1,32 @@ +# Agent workspace image for nanowm_rollout_speedup (CPU-only client). +# The agent edits the NanoWM checkout under /app/nano-world-model and submits a +# patch; GPU evaluation happens on Modal (judge side). Build via ../build_images.sh. +FROM python:3.11-slim + +ENV DEBIAN_FRONTEND=noninteractive PIP_NO_CACHE_DIR=1 +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates bash ripgrep patch && rm -rf /var/lib/apt/lists/* + +# Pre-install Claude Code + Codex CLI (parity with other 2.0 agent images). +RUN curl -fsSL https://claude.ai/install.sh | bash -s -- && \ + echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc +ENV NVM_DIR="/root/.nvm" +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash && \ + . "$NVM_DIR/nvm.sh" && nvm install 22 && nvm alias default 22 && \ + npm install -g @openai/codex@latest && ln -sf "$(which node)" /usr/local/bin/node + +WORKDIR /app +# Clean NanoWM checkout the agent patches (pinned commit; native build not needed). +ARG NANOWM_COMMIT=main +RUN git clone https://github.com/simchowitzlabpublic/nano-world-model /app/nano-world-model && \ + cd /app/nano-world-model && git checkout "$NANOWM_COMMIT" +# Task-local infra patches the agent builds on (decode-OOM + get_seq_length); +# applied so the harness rollout works, outside the agent's editable scope. +COPY task_ctx/infra_patches/ /tmp/infra_patches/ +RUN cd /app/nano-world-model && for p in /tmp/infra_patches/*.patch; do \ + [ -e "$p" ] && git apply "$p" || true; done && \ + git add -A && git -c user.email=t@e -c user.name=t commit -q -m base || true + +COPY task_ctx/harbor_app/ /app/ +COPY task_ctx/task_pkg/ /app/task/ +RUN chmod +x /app/*.sh 2>/dev/null || true diff --git a/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh b/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh new file mode 100755 index 00000000..298b4c98 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build the nanowm_rollout_speedup agent + judge images. +# bash 2.0/problems/nanowm_rollout_speedup/docker/build_images.sh [tag] +# Requires the hidden assets staged under $NWM_ASSETS (default below): +# ckpts/nanowm-l2-csgo/model_state_dict.pt +# data/csgo/1-200/*.hdf5 (held-out CSGO episode subset) +# data/csgo_subset/{val_files.txt,val_starts.npy} +# baseline/baseline_metrics.json (optional; computed on first trial otherwise) +set -euo pipefail +TAG="${1:-experimental-v0}" +NANOWM_COMMIT="${NANOWM_COMMIT:-main}" +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROB=$(cd "$SCRIPT_DIR/.." && pwd) +NWM_ASSETS="${NWM_ASSETS:-/scratch/gpfs/KARTHIKN/wc9403/project/fcs-2/data}" + +CTX=$(mktemp -d); trap 'rm -rf "$CTX"' EXIT +mkdir -p "$CTX/task_ctx" +cp -r "$PROB/speedup_eval" "$CTX/task_ctx/task_pkg" +cp "$PROB/evaluator.py" "$CTX/task_ctx/evaluator.py" +cp -r "$PROB/harbor/app" "$CTX/task_ctx/harbor_app" +cp -r "$PROB/infra_patches" "$CTX/task_ctx/infra_patches" 2>/dev/null || mkdir -p "$CTX/task_ctx/infra_patches" +# judge-only hidden assets +mkdir -p "$CTX/task_ctx/assets" +cp -r "$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k" "$CTX/task_ctx/assets/ckpts/nanowm-l2-csgo" 2>/dev/null || true +cp -r "$NWM_ASSETS/csgo" "$CTX/task_ctx/assets/data/csgo" 2>/dev/null || true +cp -r "$NWM_ASSETS/csgo_subset" "$CTX/task_ctx/assets/data/csgo_subset" 2>/dev/null || true + +docker build --target "" --build-arg NANOWM_COMMIT="$NANOWM_COMMIT" \ + -t "frontiercs/nanowm-rollout-speedup-agent:$TAG" -f "$SCRIPT_DIR/agent/Dockerfile" "$CTX" +docker build --build-arg NANOWM_COMMIT="$NANOWM_COMMIT" \ + -t "frontiercs/nanowm-rollout-speedup-judge:$TAG" -f "$SCRIPT_DIR/judge/Dockerfile" "$CTX" +echo "Built frontiercs/nanowm-rollout-speedup-{agent,judge}:$TAG" diff --git a/2.0/problems/nanowm_rollout_speedup/docker/judge/Dockerfile b/2.0/problems/nanowm_rollout_speedup/docker/judge/Dockerfile new file mode 100644 index 00000000..f5846dd0 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/docker/judge/Dockerfile @@ -0,0 +1,35 @@ +# Judge image for nanowm_rollout_speedup (CPU-only; drives a Modal GPU). +# Bakes the task package + clean NanoWM checkout + L/2 CSGO ckpt + held-out CSGO +# episode subset + cached vanilla baseline; the GPU rollout runs on Modal. +FROM python:3.11-slim + +ENV DEBIAN_FRONTEND=noninteractive PIP_NO_CACHE_DIR=1 FRONTIER_NWM_PKG=/opt/nanowm/task +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates bash patch && rm -rf /var/lib/apt/lists/* +RUN pip install modal omegaconf hydra-core + +WORKDIR /opt/nanowm +ARG NANOWM_COMMIT=main +RUN git clone https://github.com/simchowitzlabpublic/nano-world-model /opt/nanowm/nano-world-model && \ + cd /opt/nanowm/nano-world-model && git checkout "$NANOWM_COMMIT" +COPY task_ctx/infra_patches/ /tmp/infra_patches/ +RUN cd /opt/nanowm/nano-world-model && for p in /tmp/infra_patches/*.patch; do \ + [ -e "$p" ] && git apply "$p" || true; done + +# Hidden assets baked by build_images.sh into the build context: +# task_ctx/assets/ckpts/nanowm-l2-csgo/model_state_dict.pt +# task_ctx/assets/csgo/1-200/*.hdf5 (held-out episode subset) +# task_ctx/assets/csgo_subset/{val_files.txt,val_starts.npy} +# task_ctx/assets/baseline/baseline_metrics.json (optional precomputed) +COPY task_ctx/assets/ /opt/nanowm/ +COPY task_ctx/task_pkg/ /opt/nanowm/task/ +COPY task_ctx/evaluator.py /opt/nanowm/task/evaluator.py + +# Modal builds the GPU image from this same checkout/assets at deploy time. +ENV FRONTIER_NWM_REPO=/opt/nanowm/nano-world-model \ + FRONTIER_NWM_CKPT=/opt/nanowm/ckpts/nanowm-l2-csgo/model_state_dict.pt \ + FRONTIER_NWM_CSGO_DATA=/opt/nanowm/data/csgo \ + FRONTIER_NWM_VAL_FILES=/opt/nanowm/data/csgo_subset/val_files.txt \ + FRONTIER_NWM_VAL_STARTS=/opt/nanowm/data/csgo_subset/val_starts.npy \ + FRONTIER_NWM_BASELINE_CACHE=/opt/nanowm/baseline/baseline_metrics.json \ + NWM_BAKE_DIR=/opt/nanowm diff --git a/2.0/problems/nanowm_rollout_speedup/evaluate.sh b/2.0/problems/nanowm_rollout_speedup/evaluate.sh new file mode 100755 index 00000000..9a87262b --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/evaluate.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Local CLI evaluation for nanowm_rollout_speedup. +# Full runs need a GPU (local backend) or Modal credentials, plus the baked +# NanoWM assets (FRONTIER_NWM_* env, see speedup_eval/settings.py). Without a GPU +# this falls back to the smoke path (patch-policy validation only), which is what +# repository CI exercises. +set -euo pipefail +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +export FRONTIER_NWM_PKG="${FRONTIER_NWM_PKG:-$SCRIPT_DIR}" +if ! python3 -c 'import torch; assert torch.cuda.is_available()' >/dev/null 2>&1; then + export FRONTIER_NWM_SMOKE="${FRONTIER_NWM_SMOKE:-1}" +fi +SOLUTION="${1:-$SCRIPT_DIR/reference.patch}" +exec python3 "$SCRIPT_DIR/evaluator.py" "$SOLUTION" diff --git a/2.0/problems/nanowm_rollout_speedup/evaluator.py b/2.0/problems/nanowm_rollout_speedup/evaluator.py new file mode 100644 index 00000000..1cff1129 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/evaluator.py @@ -0,0 +1,204 @@ +"""nanowm_rollout_speedup evaluator (Frontier-CS 2.0 contract). + +The agent submits a Python-only patch against a clean NanoWM checkout that +speeds up the diffusion SAMPLING for a fixed CSGO long-rollout invocation +(NanoWM-L/2, 50 frames, 4 context, nominal 50 DDIM steps). The judge applies the +patch, runs the rollout on a Modal GPU, and scores wall-clock speedup over the +unpatched baseline, gated by a rollout-quality (LPIPS-vs-GT) guardrail. + +This file holds the self-contained static patch policy + scoring. Heavy +orchestration (Modal sandbox, baseline caching) lives in `speedup_eval/`. When +the GPU/Modal stack is unconfigured (local CI), the evaluator validates the +patch policy and returns a smoke score so the empty reference patch passes. +""" +from __future__ import annotations + +import hashlib +import os +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_PKG = os.environ.get("FRONTIER_NWM_PKG", str(Path(__file__).resolve().parent)) +if _PKG not in sys.path: + sys.path.insert(0, _PKG) + +from speedup_eval import scoring, settings as S # noqa: E402 + +MAX_PATCH_BYTES = 256_000 + +# --- Patch policy ------------------------------------------------------------ +# The agent may only change the diffusion SAMPLING layer (Python source). The +# model architecture, VAE, datasets, metric, and harness are frozen. +ALLOWED = ( + "src/diffusion/*.py", + "src/diffusion/**/*.py", + "src/sample/sampling_utils.py", +) +DENIED = ( + "src/models/**", "src/latent_codecs/**", # frozen model + VAE + "src/sample/evaluate_metrics.py", "src/sample/plot_metrics.py", # the metric + "src/sample/rollout.py", # the judge harness + "src/wm_datasets/**", # data loading / no bench detection + "src/experiments/**", "src/main.py", "src/utils/**", + "**/*.so", "**/*.pyx", "**/*.c", "**/*.cpp", "**/*.cu", # no native + "**/setup.py", "**/pyproject.toml", "**/requirements*.txt", +) +# Forbidden substrings in ADDED lines (no benchmark detection / env leakage / +# timing short-circuits / hardcoded ground truth). +FORBIDDEN_TOKENS = ( + "FRONTIER_", "JUDGE_", "HARBOR_", "MODAL_", "HF_TOKEN", "NWM_TIME", + "csgo_validation_start_indices", "val_starts", "val_files", "gen_seconds", + "evaluate_metrics", "lpips", "ground_truth", "gt_frames", + "os.environ", "subprocess", "socket", "/judge", "/opt/nanowm/baseline", + "time.sleep", "while True", +) + + +@dataclass +class PatchFile: + old_path: str = "" + new_path: str = "" + added_lines: list[str] = field(default_factory=list) + + @property + def path(self) -> str: + return self.new_path if self.new_path != "/dev/null" else self.old_path + + +def _match(path: str, patterns: tuple[str, ...]) -> bool: + from fnmatch import fnmatch + return any(fnmatch(path, pat) for pat in patterns) + + +def _parse_patch(text: str) -> list[PatchFile]: + files: list[PatchFile] = [] + cur: PatchFile | None = None + for line in text.splitlines(): + if line.startswith("diff --git"): + if cur: + files.append(cur) + cur = PatchFile() + elif line.startswith("--- "): + if cur is None: + cur = PatchFile() + cur.old_path = line[4:].strip().removeprefix("a/") + elif line.startswith("+++ "): + if cur is None: + cur = PatchFile() + cur.new_path = line[4:].strip().removeprefix("b/") + elif line.startswith("+") and not line.startswith("+++"): + if cur is not None: + cur.added_lines.append(line[1:]) + if cur: + files.append(cur) + return files + + +def _safe(path: str) -> bool: + p = Path(path) + return not p.is_absolute() and ".." not in p.parts + + +def validate_patch(patch_path: Path) -> tuple[bool, str, dict[str, Any]]: + if not patch_path.exists(): + return False, "solution patch does not exist", {"valid_patch": 0} + size = patch_path.stat().st_size + if size > MAX_PATCH_BYTES: + return False, f"patch too large ({size} > {MAX_PATCH_BYTES})", {"valid_patch": 0} + text = patch_path.read_text(encoding="utf-8", errors="replace") + metrics: dict[str, Any] = { + "patch_bytes": size, + "patch_sha256": hashlib.sha256(text.encode("utf-8", "replace")).hexdigest(), + } + files = _parse_patch(text) + metrics["files_changed"] = [f.path for f in files] + if not files and size > 0: + return False, "could not parse any file hunks from patch", {**metrics, "valid_patch": 0} + for pf in files: + for cand in {pf.old_path, pf.new_path, pf.path}: + if cand and cand != "/dev/null" and not _safe(cand): + return False, f"unsafe patch path: {cand}", {**metrics, "valid_patch": 0} + if pf.new_path == "/dev/null": + return False, f"deleting source files is out of scope: {pf.old_path}", {**metrics, "valid_patch": 0} + path = pf.path + if _match(path, DENIED) or not path.endswith((".py", ".pyi")): + return False, f"path not allowed (frozen/native/harness): {path}", {**metrics, "valid_patch": 0} + if not _match(path, ALLOWED): + return False, f"path outside the allowlisted sampling layer: {path}", {**metrics, "valid_patch": 0} + blob = "\n".join(pf.added_lines) + for tok in FORBIDDEN_TOKENS: + if tok in blob: + return False, f"forbidden token in added lines ({tok}) in {path}", {**metrics, "valid_patch": 0} + metrics["valid_patch"] = 1 + return True, "patch accepted by static policy", metrics + + +# --- Evaluate ---------------------------------------------------------------- +def _resolve_patch(solution_path: str) -> Path: + p = Path(solution_path) + if p.is_dir(): + cand = p / "solution.patch" + if cand.exists(): + return cand + hits = sorted(p.rglob("*.patch")) + if hits: + return hits[0] + return cand # nonexistent -> validate_patch reports + return p + + +def prepare() -> dict: + return {"task": "nanowm_rollout_speedup", "smoke": S.SMOKE, + "quality_tolerance": S.QUALITY_TOLERANCE, "nominal_steps": S.NUM_SAMPLING_STEPS} + + +def evaluate(solution_path: str): + role = "final" if os.environ.get("FRONTIER_SUBMISSION_ROLE") == "final" else "agent" + patch_path = _resolve_patch(solution_path) + ok, msg, pmetrics = validate_patch(patch_path) + if not ok: + return 0.0, 0.0, f"patch rejected: {msg}", pmetrics + + # Local CI / no-GPU smoke: validate policy only, pass the (empty) reference. + if S.SMOKE or not S.CKPT.exists(): + return 1.0, 1.0, "smoke: patch policy valid (GPU/Modal unconfigured)", {**pmetrics, "smoke": 1} + + # Heavy path: run baseline (cached) + patched rollout on the GPU sandbox. + from speedup_eval import orchestrate # Modal/GPU orchestration + clips = S.FINAL_CLIPS if role == "final" else S.QUICK_CLIPS + try: + baseline, patched = orchestrate.run_pair(patch_path, clips=clips, role=role) + except Exception as exc: # concise public error only + return 0.0, 0.0, f"evaluation failed: {type(exc).__name__}", {**pmetrics, "role": role} + + res = scoring.provisional_score( + {"all": baseline["gen_seconds"]}, {"all": patched["gen_seconds"]}, + baseline["lpips"], patched["lpips"], tolerance=S.QUALITY_TOLERANCE, + ) + metrics = { + **pmetrics, "role": role, "clips": clips, + "baseline_seconds": round(baseline["gen_seconds"], 2), + "patched_seconds": round(patched["gen_seconds"], 2), + "baseline_lpips": round(baseline["lpips"], 4), + "patched_lpips": round(patched["lpips"], 4), + "geomean_speedup": round(res["geomean_speedup"], 3), + "quality_multiplier": round(res["quality_multiplier"], 3), + "score_formula": "100*log2(speedup)*quality_mult; quality_mult<1 if LPIPS rises >tol over baseline", + } + msg = (f"speedup {res['geomean_speedup']:.2f}x " + f"(lpips {patched['lpips']:.3f} vs baseline {baseline['lpips']:.3f}, " + f"tol {S.QUALITY_TOLERANCE:.0%}); score {res['score']:.1f}") + return res["score"], res["score_unbounded"], msg, metrics + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("usage: evaluator.py ") + sys.exit(2) + import json + s, u, m, mt = evaluate(sys.argv[1]) + print(json.dumps({"score": s, "score_unbounded": u, "message": m, "metrics": mt}, indent=2)) + print(s) diff --git a/2.0/problems/nanowm_rollout_speedup/harbor/app/README.md b/2.0/problems/nanowm_rollout_speedup/harbor/app/README.md new file mode 100644 index 00000000..77e72eac --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/harbor/app/README.md @@ -0,0 +1,19 @@ +# nanowm_rollout_speedup — agent workspace + +Patch the diffusion **sampling** code of the NanoWM checkout at +`/app/nano-world-model` to speed up the fixed CSGO 50-frame rollout at +iso-quality, then submit the unified diff as `/app/solution.patch`. + +Workflow: +1. Edit `nano-world-model/src/diffusion/**` or `src/sample/sampling_utils.py`. +2. `bash /app/public_test.sh` — static patch-policy check on your diff. +3. `bash /app/make_submission.sh` — writes `/app/solution.patch` from your edits. +4. `bash /app/submit.sh` — enqueue for the black-box judge (quick set). + +Rules (see the task statement): Python-only; allowed = `src/diffusion/**.py`, +`src/sample/sampling_utils.py`; the model/VAE/metric/harness/data are frozen and +denied; no env-var/benchmark/timing tricks. The judge fixes the rollout call — +you change the sampler internals. + +Scored on wall-clock speedup vs the unpatched baseline, gated by an LPIPS-vs-GT +quality guardrail (≤3% rise). See `../readme` and `../DESIGN.md`. diff --git a/2.0/problems/nanowm_rollout_speedup/harbor/app/make_submission.sh b/2.0/problems/nanowm_rollout_speedup/harbor/app/make_submission.sh new file mode 100755 index 00000000..becc45f6 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/harbor/app/make_submission.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Write /app/solution.patch from the agent's edits to /app/nano-world-model. +set -euo pipefail +cd /app/nano-world-model +git add -A 2>/dev/null || true +git diff --staged > /app/solution.patch 2>/dev/null || git diff > /app/solution.patch +echo "wrote /app/solution.patch ($(wc -l < /app/solution.patch) lines)" diff --git a/2.0/problems/nanowm_rollout_speedup/harbor/app/public_test.py b/2.0/problems/nanowm_rollout_speedup/harbor/app/public_test.py new file mode 100644 index 00000000..2dd23e3d --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/harbor/app/public_test.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Agent-facing public test: static patch-policy check (no GPU). + +Mirrors the judge's `validate_patch` so the agent can confirm a submission will +be accepted before enqueuing. Does NOT run the rollout or score speedup — that +happens on the judge's GPU. +""" +import sys +from pathlib import Path + +# locate the shared policy: prefer the baked task package, else the local copy. +for cand in ("/judge", "/opt/nanowm/task", str(Path(__file__).resolve().parents[2])): + if (Path(cand) / "evaluator.py").exists(): + sys.path.insert(0, cand) + break + +try: + from evaluator import validate_patch +except Exception: + print("NOTE: judge evaluator not available in this workspace; cannot run the " + "exact policy check here. Submit and read the judge feedback.") + sys.exit(0) + +patch = Path(sys.argv[1] if len(sys.argv) > 1 else "/app/solution.patch") +ok, msg, metrics = validate_patch(patch) +print(f"valid_patch={int(ok)} files={metrics.get('files_changed')}") +print(msg) +sys.exit(0 if ok else 1) diff --git a/2.0/problems/nanowm_rollout_speedup/harbor/app/public_test.sh b/2.0/problems/nanowm_rollout_speedup/harbor/app/public_test.sh new file mode 100755 index 00000000..a39757d5 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/harbor/app/public_test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +python3 "$SCRIPT_DIR/public_test.py" "${1:-/app/solution.patch}" diff --git a/2.0/problems/nanowm_rollout_speedup/harbor/app/solution.patch b/2.0/problems/nanowm_rollout_speedup/harbor/app/solution.patch new file mode 100644 index 00000000..e69de29b diff --git a/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch new file mode 100644 index 00000000..411fd2dd --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch @@ -0,0 +1,23 @@ +diff --git a/src/sample/rollout.py b/src/sample/rollout.py +index b83f0ee..3b62b99 100644 +--- a/src/sample/rollout.py ++++ b/src/sample/rollout.py +@@ -225,8 +225,16 @@ def main(args): + generated_latents = torch.cat([generated_latents, new_latent], dim=1) + + print(f"Decoding and saving batch results...") +- gen_frames_batch = decode_latents(vae, generated_latents, vae_precision=vae_precision) +- gt_frames_batch = decode_latents(vae, gt_latents, vae_precision=vae_precision) ++ # Decode episode-by-episode: flattening B*T frames through the VAE in ++ # one call OOMs for long rollouts (e.g. 16 eps x 19 frames @ 256^2). ++ gen_frames_batch = torch.cat([ ++ decode_latents(vae, generated_latents[i:i+1], vae_precision=vae_precision) ++ for i in range(generated_latents.shape[0]) ++ ]) ++ gt_frames_batch = torch.cat([ ++ decode_latents(vae, gt_latents[i:i+1], vae_precision=vae_precision) ++ for i in range(gt_latents.shape[0]) ++ ]) + + for i in range(current_batch_size): + sample_id = start_idx + i diff --git a/2.0/problems/nanowm_rollout_speedup/readme b/2.0/problems/nanowm_rollout_speedup/readme new file mode 100644 index 00000000..d77f614f --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/readme @@ -0,0 +1,73 @@ +# NanoWM Rollout Speedup — fast diffusion sampling for a frozen video world model + +## Problem + +You are given a clean checkout of **Nano World Models** (arXiv:2605.23993) and +its frozen **NanoWM-L/2 CSGO** checkpoint — a diffusion-forcing video world model. +The judge runs a **fixed** autoregressive long-rollout: from 4 context frames, +generate **50 future frames** of held-out CSGO gameplay, sequential scheduling, +nominal **50 DDIM steps**. + +Your job: **make that rollout faster** by submitting a **Python-only patch** to +the diffusion **sampling** code, **without degrading rollout quality**. Score is +wall-clock speedup over the unpatched baseline, gated by a quality guardrail. + +This is a real fast-sampling problem: the paper's Fig. 6 shows DDIM step count +genuinely trades off against rollout quality on CSGO (unlike saturated toy +domains). Naively cutting steps degrades quality and fails the guardrail; to win +you must reproduce ~50-step quality with less compute — DPM-Solver++ / higher-order +or exponential integrators, KV/feature caching across denoising steps and frames, +mixed precision, `torch.compile`, fused attention, redundancy elimination, etc. + +## What you submit + +A unified-diff patch at **`/app/solution.patch`** against the checkout in +`/app/nano-world-model`. **Python source only**, and only within the diffusion +sampling layer: + +**Allowed:** `src/diffusion/**.py`, `src/sample/sampling_utils.py` +**Denied:** the model architecture (`src/models/**`), VAE (`src/latent_codecs/**`), +the metric (`src/sample/evaluate_metrics.py`), the rollout harness +(`src/sample/rollout.py`), data loading (`src/wm_datasets/**`), training/eval +harness, and any native/build/dependency files. New `.py` files inside the +allowed areas are fine. Patches are validated **before** running. + +The rollout invocation (length, context, nominal step count, scheduling) is +**fixed by the judge** — you change the sampler internals, not the call. Patches +that read judge/Modal/HF env vars, hard-code episode ids or ground truth, +short-circuit/sleep, or special-case the benchmark are rejected. + +## Evaluation & scoring + +- The judge applies your patch to a clean checkout and runs the fixed CSGO + rollout on hidden held-out episodes on a **GPU (served via Modal)**. Iterative + (`bash /app/submit.sh`) uses a small quick set; the final verifier uses a + larger disjoint set. +- **Quality guardrail:** rollout **LPIPS vs ground truth** must not rise more + than `quality_tolerance` (default **3%**) above the unpatched seq@50 baseline. + (Calibration: seq@20 is already +5% over seq@50, so naive step-cutting fails + this — real fast-sampling is required.) +- **Score:** + +``` +geomean_speedup = baseline_seconds / patched_seconds (rollout generation) +score = clip(100 * log2(geomean_speedup), 0, 100) * quality_multiplier +``` + + `quality_multiplier` is 1.0 within tolerance and decays inverse-proportionally + beyond it. `score_unbounded` keeps rewarding speedup past 2× (the bounded score + caps at 100). A patch that degrades quality past tolerance is penalized toward + 0; one that crashes, exceeds limits, or violates the patch policy scores 0. + +## Resource budget + +CPU agent + judge containers (8 CPU / 32 GB); one Modal GPU per evaluation. +Evaluation timeout 21600 s. Submission queue depth 2. + +## Getting started + +`/app/nano-world-model` is the checkout you patch. `bash /app/public_test.sh` +runs a tiny local policy check on your `solution.patch`. See `AGENT.md` and +`harbor/app/README.md` for the submission workflow, and the paper / `docs/` for +the sampling code you'll be optimizing (`src/diffusion/df_sample.py`, +`gaussian_diffusion.py`). diff --git a/2.0/problems/nanowm_rollout_speedup/reference.py b/2.0/problems/nanowm_rollout_speedup/reference.py new file mode 100644 index 00000000..64e937fe --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/reference.py @@ -0,0 +1,20 @@ +"""CI/local reference entry for nanowm_rollout_speedup. + +The real reference solution is `reference.patch` (a bf16-autocast sampling +speedup). The Frontier-CS 2.0 CLI's default `evaluator.py ` path +expects a Python file; this shim points the evaluator at reference.patch so the +standard `frontier eval` validation works. In smoke/no-GPU mode the evaluator +validates the patch policy and returns a passing score. +""" +import sys +from pathlib import Path + +REFERENCE_PATCH = str(Path(__file__).resolve().parent / "reference.patch") + +if __name__ == "__main__": + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import json + from evaluator import evaluate + s, u, m, mt = evaluate(REFERENCE_PATCH) + print(json.dumps({"score": s, "score_unbounded": u, "message": m, "metrics": mt}, indent=2)) + print(s) diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/__init__.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/__init__.py new file mode 100644 index 00000000..9c5b1828 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/__init__.py @@ -0,0 +1 @@ +"""nanowm_rollout_speedup evaluation package (shared by judge + public test).""" diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/modal_app.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/modal_app.py new file mode 100644 index 00000000..b36b5515 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/modal_app.py @@ -0,0 +1,66 @@ +"""Modal app: run one NanoWM CSGO rollout (patched or vanilla) on one GPU. + +Mirrors the GPU-on-Modal pattern of vllm_llm_serving_optimization: the judge +container stays CPU-only and calls into a Modal GPU function. The NanoWM +checkout, L/2 CSGO checkpoint, and held-out CSGO episode subset are baked into +the Modal image at build time; the agent's patch text travels in per call. + +Parametrized via env vars (set by the judge): + NWM_MODAL_GPU Modal GPU string (default "L40S") + NWM_MODAL_APP Modal app name + NWM_HF_SECRET Modal Secret name (unused once weights are baked) + +Deploy: modal deploy speedup_eval/modal_app.py +Call: run_rollout_remote(patch_text, clips, steps) -> metrics dict + +NOTE: validated structurally against the #145 modal pattern; end-to-end Modal +execution is pending maintainer Modal credentials (Della has no Modal access). +The LOCAL backend (orchestrate._run_local) is the path validated on H100. +""" +from __future__ import annotations + +import os + +GPU = os.environ.get("NWM_MODAL_GPU", "L40S") +APP_NAME = os.environ.get("NWM_MODAL_APP", "nanowm-rollout-speedup") +REMOTE_ROOT = "/opt/nanowm" + +try: + import modal + + image = ( + modal.Image.from_registry("nvidia/cuda:12.1.0-devel-ubuntu22.04", add_python="3.11") + .pip_install( + "torch==2.4.1", "torchvision==0.19.1", "numpy<2", "scipy==1.15.3", + "lpips", "diffusers[torch]==0.24.0", "omegaconf", "hydra-core", + "decord", "imageio", "imageio-ffmpeg", "opencv-python-headless", + "scikit-image", "pandas", "einops", "timm", "pytorch-lightning==2.4.0", + "huggingface_hub==0.25.2", "transformers==4.46.3", + ) + # bake the clean checkout + ckpt + CSGO subset + task package + .add_local_dir(os.environ.get("NWM_BAKE_DIR", "/opt/nanowm"), REMOTE_ROOT, copy=True) + ) + app = modal.App(APP_NAME) + + @app.function(gpu=GPU, image=image, timeout=3600) + def _rollout(patch_text: str, clips: int, steps: int) -> dict: + import sys, tempfile + from pathlib import Path + sys.path.insert(0, f"{REMOTE_ROOT}/task") + from speedup_eval import runner, orchestrate # noqa + patch = None + if patch_text.strip(): + p = Path(tempfile.mkstemp(suffix=".patch")[1]) + p.write_text(patch_text) + patch = p + cfg = orchestrate._compose_config() + from speedup_eval import settings as S + return runner.evaluate(S.REPO, patch, cfg, S.CKPT, clips=clips, steps=steps, device="cuda") + + def run_rollout_remote(patch_text: str, clips: int, steps: int) -> dict: + with app.run(): + return _rollout.remote(patch_text, clips, steps) + +except ImportError: # modal not installed (e.g. local validation) + def run_rollout_remote(patch_text: str, clips: int, steps: int) -> dict: # type: ignore + raise RuntimeError("modal not available; use the LOCAL backend (FRONTIER_NWM_BACKEND=local)") diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py new file mode 100644 index 00000000..b02709f0 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py @@ -0,0 +1,91 @@ +"""GPU orchestration for nanowm_rollout_speedup. + +`run_pair(patch_path, clips, role)` returns (baseline_metrics, patched_metrics), +each {"gen_seconds", "lpips", "clips", "steps"}. + +Two backends, selected automatically: + - MODAL (FRONTIER_NWM_BACKEND=modal or a Modal token is present): runs the + rollout in a Modal GPU sandbox (the maintainer-facing path, mirroring + vllm_llm_serving_optimization; judge container stays CPU-only). + - LOCAL (default when a CUDA device is visible): runs the runner directly on + the local GPU. Used for Della validation and any GPU-equipped judge. + +The unpatched baseline is computed once and cached (baseline_cache_path); the +patched run always re-applies the agent patch to a clean checkout. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +from . import settings as S + + +def _compose_config() -> Path: + """Write the fixed CSGO rollout config (subset val list) the runner consumes.""" + out = Path(os.environ.get("FRONTIER_NWM_WORKDIR", "/tmp/nwm_speedup")) / "csgo_config.yaml" + out.parent.mkdir(parents=True, exist_ok=True) + if out.exists(): + return out + from hydra import compose, initialize_config_dir + from omegaconf import OmegaConf + with initialize_config_dir(config_dir=str(S.REPO / "src/configs"), version_base=None): + cfg = compose(config_name="config", overrides=[ + f"model={S.MODEL}", f"dataset={S.DATASET}", "experiment=evaluate_only", + "wandb.enabled=false", + f"dataset.loader.val_file_list={S.VAL_FILES}", + f"dataset.loader.train_file_list={S.VAL_FILES}", + f"dataset.loader.val_start_indices={S.VAL_STARTS}", + ]) + out.write_text(OmegaConf.to_yaml(cfg, resolve=False)) + return out + + +def _run_local(patch_path: Path | None, clips: int) -> dict: + from . import runner + return runner.evaluate(S.REPO, patch_path, _compose_config(), S.CKPT, + clips=clips, steps=S.NUM_SAMPLING_STEPS, device="cuda") + + +def _run_modal(patch_path: Path | None, clips: int) -> dict: + """Run one rollout inside a Modal GPU sandbox. Mirrors #145's modal_app: + the patch text + config travel into the sandbox, which runs speedup_eval.runner + on the baked NanoWM checkout and returns the metrics dict.""" + from .modal_app import run_rollout_remote # deploys/looks up the Modal app + patch_text = "" if patch_path is None else Path(patch_path).read_text(errors="replace") + return run_rollout_remote(patch_text=patch_text, clips=clips, steps=S.NUM_SAMPLING_STEPS) + + +def _backend(): + b = os.environ.get("FRONTIER_NWM_BACKEND", "").lower() + if b in ("modal", "local"): + return b + if os.environ.get("MODAL_TOKEN_ID") or os.environ.get("MODAL_TOKEN_SECRET"): + return "modal" + return "local" + + +def _baseline(clips: int, run) -> dict: + cache = S.BASELINE_CACHE + if cache.exists(): + try: + data = json.loads(cache.read_text()) + if data.get("clips", 0) >= clips: + return data + except Exception: + pass + m = run(None, clips) + try: + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(json.dumps(m)) + except OSError: + pass + return m + + +def run_pair(patch_path: Path, clips: int, role: str = "agent") -> tuple[dict, dict]: + run = _run_modal if _backend() == "modal" else _run_local + baseline = _baseline(clips, run) + patched = run(Path(patch_path), clips) + return baseline, patched From 02a8d6b4d12a64e6c1bc5e9832cdf25979516c6a Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Fri, 12 Jun 2026 01:41:39 -0400 Subject: [PATCH 03/12] nanowm_rollout_speedup: 2.0/README entry --- 2.0/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/2.0/README.md b/2.0/README.md index b414435d..14651169 100644 --- a/2.0/README.md +++ b/2.0/README.md @@ -81,3 +81,14 @@ single ICCAD2015 design, `superblue1`. Its problem ID is `bboplace_direct_iccad2015`. It follows the same JSON interface and single design evaluation flow as `bboplace_direct_ispd2005`, with the ICCAD2015 baseline for `superblue1`. + +## NanoWM Rollout Speedup + +This systems problem asks agents to speed up diffusion sampling for a frozen +video world model. Its problem ID is `nanowm_rollout_speedup`. Agents submit a +Python-only patch to the sampling layer of Nano World Models (arXiv:2605.23993); +the judge runs a fixed NanoWM-L/2 CSGO 50-frame long-rollout on a Modal GPU and +scores wall-clock speedup over the unpatched baseline, gated by an LPIPS rollout- +quality guardrail (so naive step-cutting fails — real fast-sampling is required). +Mirrors `vllm_llm_serving_optimization`: patch + latency + accuracy guardrail + +Modal GPU, CPU judge. From 55940fce8e55353253b0fbcaa8ba9164fd2ab660 Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Fri, 12 Jun 2026 01:44:19 -0400 Subject: [PATCH 04/12] =?UTF-8?q?nanowm=5Frollout=5Fspeedup:=20record=20va?= =?UTF-8?q?lidated=20reference=20result=20(1.17x=20bf16,=20score=2022.4,?= =?UTF-8?q?=20qmult=201.0)=20=E2=80=94=20end-to-end=20H100=20local=20backe?= =?UTF-8?q?nd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 2.0/problems/nanowm_rollout_speedup/DESIGN.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/2.0/problems/nanowm_rollout_speedup/DESIGN.md b/2.0/problems/nanowm_rollout_speedup/DESIGN.md index 8c998253..35c4e647 100644 --- a/2.0/problems/nanowm_rollout_speedup/DESIGN.md +++ b/2.0/problems/nanowm_rollout_speedup/DESIGN.md @@ -74,6 +74,14 @@ within `quality_tolerance` LPIPS rise and decays inverse-proportionally beyond. the fp32 baseline (CI requires reference > baseline). The intended frontier (DPM-Solver++, caching, distillation) is left to the agent. +**Validated end-to-end on Della H100 (local backend, 4 CSGO clips, 2026-06-12):** +baseline 299.9 s / LPIPS 0.548 → bf16-patched 256.7 s / LPIPS 0.521 ⇒ **1.17× +speedup, quality_multiplier 1.0, score 22.4** (reference > baseline ✓). Patch +policy validated (accepts reference; rejects metric edits + env-var leakage); +smoke path returns 1.0 with the empty reference on CPU. The frontier (seq@2 = ++31% LPIPS) leaves wide headroom above the 1.17× reference for real fast-sampling +patches. **Pending:** end-to-end Modal execution (maintainer credentials). + ## 7. Open items for maintainers - Modal end-to-end run + a deployed `modal_app` app name / GPU type confirmation. From 5065f7cb2b14fb3250474dcf9b85c8ac0e72df97 Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Sat, 13 Jun 2026 01:37:34 -0400 Subject: [PATCH 05/12] =?UTF-8?q?nanowm=5Frollout=5Fstability:=20drift=20t?= =?UTF-8?q?ask=20(dual=20of=20speedup)=20=E2=80=94=20minimize=2080-frame?= =?UTF-8?q?=20tail-drift=20at=20iso-wall-clock.=20Reference=20(stab=3D0.20?= =?UTF-8?q?)=20reliably=20beats=20baseline=20(t~2.5/22=20clips).=20Reuses?= =?UTF-8?q?=20framework;=20patch=20policy=20+=20smoke=20validated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 2.0/README.md | 11 + .../nanowm_rollout_stability/DESIGN.md | 52 +++++ .../nanowm_rollout_stability/config.yaml | 45 ++++ .../docker/agent/Dockerfile | 32 +++ .../docker/build_images.sh | 32 +++ .../docker/judge/Dockerfile | 35 +++ .../nanowm_rollout_stability/evaluate.sh | 14 ++ .../nanowm_rollout_stability/evaluator.py | 206 ++++++++++++++++++ .../harbor/app/README.md | 19 ++ .../harbor/app/make_submission.sh | 7 + .../harbor/app/public_test.py | 28 +++ .../harbor/app/public_test.sh | 4 + .../harbor/app/solution.patch | 0 .../0001-rollout-chunked-decode.patch | 26 +++ 2.0/problems/nanowm_rollout_stability/readme | 60 +++++ .../nanowm_rollout_stability/reference.patch | 12 + .../nanowm_rollout_stability/reference.py | 20 ++ .../stability_eval/__init__.py | 1 + .../stability_eval/modal_app.py | 66 ++++++ .../stability_eval/orchestrate.py | 91 ++++++++ .../stability_eval/runner.py | 124 +++++++++++ .../stability_eval/scoring.py | 39 ++++ .../stability_eval/settings.py | 61 ++++++ 23 files changed, 985 insertions(+) create mode 100644 2.0/problems/nanowm_rollout_stability/DESIGN.md create mode 100644 2.0/problems/nanowm_rollout_stability/config.yaml create mode 100644 2.0/problems/nanowm_rollout_stability/docker/agent/Dockerfile create mode 100755 2.0/problems/nanowm_rollout_stability/docker/build_images.sh create mode 100644 2.0/problems/nanowm_rollout_stability/docker/judge/Dockerfile create mode 100755 2.0/problems/nanowm_rollout_stability/evaluate.sh create mode 100644 2.0/problems/nanowm_rollout_stability/evaluator.py create mode 100644 2.0/problems/nanowm_rollout_stability/harbor/app/README.md create mode 100755 2.0/problems/nanowm_rollout_stability/harbor/app/make_submission.sh create mode 100644 2.0/problems/nanowm_rollout_stability/harbor/app/public_test.py create mode 100755 2.0/problems/nanowm_rollout_stability/harbor/app/public_test.sh create mode 100644 2.0/problems/nanowm_rollout_stability/harbor/app/solution.patch create mode 100644 2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-chunked-decode.patch create mode 100644 2.0/problems/nanowm_rollout_stability/readme create mode 100644 2.0/problems/nanowm_rollout_stability/reference.patch create mode 100644 2.0/problems/nanowm_rollout_stability/reference.py create mode 100644 2.0/problems/nanowm_rollout_stability/stability_eval/__init__.py create mode 100644 2.0/problems/nanowm_rollout_stability/stability_eval/modal_app.py create mode 100644 2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py create mode 100644 2.0/problems/nanowm_rollout_stability/stability_eval/runner.py create mode 100644 2.0/problems/nanowm_rollout_stability/stability_eval/scoring.py create mode 100644 2.0/problems/nanowm_rollout_stability/stability_eval/settings.py diff --git a/2.0/README.md b/2.0/README.md index 14651169..5f88657d 100644 --- a/2.0/README.md +++ b/2.0/README.md @@ -92,3 +92,14 @@ scores wall-clock speedup over the unpatched baseline, gated by an LPIPS rollout quality guardrail (so naive step-cutting fails — real fast-sampling is required). Mirrors `vllm_llm_serving_optimization`: patch + latency + accuracy guardrail + Modal GPU, CPU judge. + +## NanoWM Rollout Stability + +The dual of `nanowm_rollout_speedup`: minimize long-horizon **drift** at fixed +compute. Its problem ID is `nanowm_rollout_stability`. Agents submit a Python-only +patch to the NanoWM sampling layer; the judge runs a fixed 80-frame NanoWM-L/2 +CSGO rollout (50 steps) on a Modal GPU and scores the relative reduction in +tail-frame (≥60) LPIPS-vs-GT over the unpatched baseline, gated by a wall-clock +guardrail (so drift can't be bought with more compute). A history-stabilization +reference reliably beats baseline (validated t≈2.5/22 clips); beating it +substantially is the open challenge. diff --git a/2.0/problems/nanowm_rollout_stability/DESIGN.md b/2.0/problems/nanowm_rollout_stability/DESIGN.md new file mode 100644 index 00000000..3e285c3a --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/DESIGN.md @@ -0,0 +1,52 @@ +# Design — nanowm_rollout_stability + +## 1. Task (the dual of nanowm_rollout_speedup) +- **speedup**: minimize wall-clock at iso-quality. +- **stability**: minimize long-horizon **drift** (tail-frame LPIPS-vs-GT) at + iso-**wall-clock**. + +Agent submits a Python-only patch to the NanoWM diffusion sampling layer; judge +runs a fixed 80-frame NanoWM-L/2 CSGO rollout (50 steps) on a Modal GPU and +scores relative tail-drift (frames ≥ 60) reduction vs the unpatched baseline, +gated by a wall-clock guardrail. Same #145-style shape (patch + metric + +guardrail + Modal GPU; general adapter/template untouched). + +## 2. Why it's a valid HARD task (calibration) +Long CSGO rollouts saturate to a ~0.65-LPIPS "plausible-but-wrong" tail; at fixed +compute the drift is largely model-intrinsic, so reducing it is genuinely hard. +But it is NOT impossible: a paired test (NanoWM-L/2, 80-frame, tail f≥60) shows +the rollout PROCEDURE moves drift reliably above noise — + +| setting (fixed steps=50) | tail-drift | +|---|---| +| stab=0.02 (baseline) | 0.652 | +| stab=0.20 (reference) | **0.629** | + +reduction 0.023 ± 0.009 SE, **73% per-clip win, t≈2.5 over 22 clips**. So a +reliably-better-than-baseline reference exists (task is well-posed/solvable); +substantially beating it requires real drift-reduction work (the open challenge). +The earlier 6-clip read showed it "within noise" — small effects need enough +clips, hence final_clips=24. + +## 3. Fixed-compute constraint (why it's distinct from speedup) +Adding denoising steps reduces drift (paper Fig 6) but costs compute — that's the +speedup axis. A **wall-clock guardrail** (patched gen time ≤ baseline × 1.10, +else the score multiplier decays) forces the agent to reduce drift *at iso- +compute*: stabilization, scheduling-matrix design, drift-aware caching that frees +time for re-grounding, periodic context re-anchoring, error correction, solvers. + +## 4. Patch policy / GPU / scoring +Identical to nanowm_rollout_speedup: Python-only allowlist (`src/diffusion/**`, +`src/sample/sampling_utils.py`), deny model/VAE/metric/harness/data; Modal GPU +from a CPU judge (`stability_eval/{runner,orchestrate,modal_app}.py`); smoke +score 1.0 when GPU/Modal unconfigured (local CI). Score = +`clip(100·(base_tail−patched_tail)/base_tail,0,100)·wallclock_mult`. + +## 5. Reference +`reference.patch`: one-line history-stabilization bump (stab→0.20) in +`df_sample.dfot_sample` — the calibrated reliable drift reducer (§2). + +## 6. Open items for maintainers +Modal end-to-end run; bake-asset provenance (ckpt + held-out CSGO subset + +cached baseline); confirm final_clips gives stable scoring on the judge's hidden +set (the small effect needs ~20+ clips). diff --git a/2.0/problems/nanowm_rollout_stability/config.yaml b/2.0/problems/nanowm_rollout_stability/config.yaml new file mode 100644 index 00000000..de9728d0 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/config.yaml @@ -0,0 +1,45 @@ +tag: ai +runtime: + language: patch + timeout_seconds: 21600 + environment: >- + Python-only patch against a clean NanoWM checkout (Nano World Models, + arXiv:2605.23993); Modal GPU runs the NanoWM-L/2 CSGO 80-frame long-rollout; + minimize long-horizon drift (tail-frame LPIPS) at iso-wall-clock + apt_packages: [bash, ca-certificates, curl, git, python3, python3-pip] + judge_apt_packages: [bash, ca-certificates, curl, git, python3, python3-pip] + judge_pip_packages: [modal] + docker: + image: frontiercs/nanowm-rollout-stability-agent:experimental-v0 + judge_image: frontiercs/nanowm-rollout-stability-judge:experimental-v0 +environment: + cpus: 8 + memory_mb: 32768 + storage_mb: 32768 + build_timeout_seconds: 5400 +evaluation: + model: nanowm_l2_csgo + dataset: game/csgo + gpu: L40S + # LONG rollout so error accumulates into a drifted tail; FIXED steps + a + # wall-clock budget => the agent improves the rollout PROCEDURE at iso-compute + # (stabilization / scheduling / drift-aware caching), not by adding steps. + rollout_length: 80 + history_length: 4 + num_steps: 50 # fixed compute budget + scheduling: sequential + history_stab: 0.02 # baseline default (repo long_rollout setting) + drift_tail_start: 60 # drift = mean LPIPS over frames >= this + # Wall-clock guardrail: patched gen time may rise at most this over baseline, + # else drift is being bought with compute (the speedup task's axis). + wallclock_tolerance: 0.10 + # Drift reductions are small; enough clips to resolve above per-clip noise + # (validated: stab=0.20 reference beats baseline at 22 clips, t~2.5, 73% win). + quick_clips: 8 + final_clips: 24 + batch_size: 2 + baseline_cache_path: /opt/nanowm/baseline/stability_baseline.json +submission: + kind: file + path: /app/solution.patch + max_queue_size: 2 diff --git a/2.0/problems/nanowm_rollout_stability/docker/agent/Dockerfile b/2.0/problems/nanowm_rollout_stability/docker/agent/Dockerfile new file mode 100644 index 00000000..c4de13c8 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/docker/agent/Dockerfile @@ -0,0 +1,32 @@ +# Agent workspace image for nanowm_rollout_speedup (CPU-only client). +# The agent edits the NanoWM checkout under /app/nano-world-model and submits a +# patch; GPU evaluation happens on Modal (judge side). Build via ../build_images.sh. +FROM python:3.11-slim + +ENV DEBIAN_FRONTEND=noninteractive PIP_NO_CACHE_DIR=1 +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates bash ripgrep patch && rm -rf /var/lib/apt/lists/* + +# Pre-install Claude Code + Codex CLI (parity with other 2.0 agent images). +RUN curl -fsSL https://claude.ai/install.sh | bash -s -- && \ + echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc +ENV NVM_DIR="/root/.nvm" +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash && \ + . "$NVM_DIR/nvm.sh" && nvm install 22 && nvm alias default 22 && \ + npm install -g @openai/codex@latest && ln -sf "$(which node)" /usr/local/bin/node + +WORKDIR /app +# Clean NanoWM checkout the agent patches (pinned commit; native build not needed). +ARG NANOWM_COMMIT=main +RUN git clone https://github.com/simchowitzlabpublic/nano-world-model /app/nano-world-model && \ + cd /app/nano-world-model && git checkout "$NANOWM_COMMIT" +# Task-local infra patches the agent builds on (decode-OOM + get_seq_length); +# applied so the harness rollout works, outside the agent's editable scope. +COPY task_ctx/infra_patches/ /tmp/infra_patches/ +RUN cd /app/nano-world-model && for p in /tmp/infra_patches/*.patch; do \ + [ -e "$p" ] && git apply "$p" || true; done && \ + git add -A && git -c user.email=t@e -c user.name=t commit -q -m base || true + +COPY task_ctx/harbor_app/ /app/ +COPY task_ctx/task_pkg/ /app/task/ +RUN chmod +x /app/*.sh 2>/dev/null || true diff --git a/2.0/problems/nanowm_rollout_stability/docker/build_images.sh b/2.0/problems/nanowm_rollout_stability/docker/build_images.sh new file mode 100755 index 00000000..ec0df373 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/docker/build_images.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build the nanowm_rollout_speedup agent + judge images. +# bash 2.0/problems/nanowm_rollout_speedup/docker/build_images.sh [tag] +# Requires the hidden assets staged under $NWM_ASSETS (default below): +# ckpts/nanowm-l2-csgo/model_state_dict.pt +# data/csgo/1-200/*.hdf5 (held-out CSGO episode subset) +# data/csgo_subset/{val_files.txt,val_starts.npy} +# baseline/baseline_metrics.json (optional; computed on first trial otherwise) +set -euo pipefail +TAG="${1:-experimental-v0}" +NANOWM_COMMIT="${NANOWM_COMMIT:-main}" +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROB=$(cd "$SCRIPT_DIR/.." && pwd) +NWM_ASSETS="${NWM_ASSETS:-/scratch/gpfs/KARTHIKN/wc9403/project/fcs-2/data}" + +CTX=$(mktemp -d); trap 'rm -rf "$CTX"' EXIT +mkdir -p "$CTX/task_ctx" +cp -r "$PROB/stability_eval" "$CTX/task_ctx/task_pkg" +cp "$PROB/evaluator.py" "$CTX/task_ctx/evaluator.py" +cp -r "$PROB/harbor/app" "$CTX/task_ctx/harbor_app" +cp -r "$PROB/infra_patches" "$CTX/task_ctx/infra_patches" 2>/dev/null || mkdir -p "$CTX/task_ctx/infra_patches" +# judge-only hidden assets +mkdir -p "$CTX/task_ctx/assets" +cp -r "$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k" "$CTX/task_ctx/assets/ckpts/nanowm-l2-csgo" 2>/dev/null || true +cp -r "$NWM_ASSETS/csgo" "$CTX/task_ctx/assets/data/csgo" 2>/dev/null || true +cp -r "$NWM_ASSETS/csgo_subset" "$CTX/task_ctx/assets/data/csgo_subset" 2>/dev/null || true + +docker build --target "" --build-arg NANOWM_COMMIT="$NANOWM_COMMIT" \ + -t "frontiercs/nanowm-rollout-stability-agent:$TAG" -f "$SCRIPT_DIR/agent/Dockerfile" "$CTX" +docker build --build-arg NANOWM_COMMIT="$NANOWM_COMMIT" \ + -t "frontiercs/nanowm-rollout-stability-judge:$TAG" -f "$SCRIPT_DIR/judge/Dockerfile" "$CTX" +echo "Built frontiercs/nanowm-rollout-stability-{agent,judge}:$TAG" diff --git a/2.0/problems/nanowm_rollout_stability/docker/judge/Dockerfile b/2.0/problems/nanowm_rollout_stability/docker/judge/Dockerfile new file mode 100644 index 00000000..f5846dd0 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/docker/judge/Dockerfile @@ -0,0 +1,35 @@ +# Judge image for nanowm_rollout_speedup (CPU-only; drives a Modal GPU). +# Bakes the task package + clean NanoWM checkout + L/2 CSGO ckpt + held-out CSGO +# episode subset + cached vanilla baseline; the GPU rollout runs on Modal. +FROM python:3.11-slim + +ENV DEBIAN_FRONTEND=noninteractive PIP_NO_CACHE_DIR=1 FRONTIER_NWM_PKG=/opt/nanowm/task +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates bash patch && rm -rf /var/lib/apt/lists/* +RUN pip install modal omegaconf hydra-core + +WORKDIR /opt/nanowm +ARG NANOWM_COMMIT=main +RUN git clone https://github.com/simchowitzlabpublic/nano-world-model /opt/nanowm/nano-world-model && \ + cd /opt/nanowm/nano-world-model && git checkout "$NANOWM_COMMIT" +COPY task_ctx/infra_patches/ /tmp/infra_patches/ +RUN cd /opt/nanowm/nano-world-model && for p in /tmp/infra_patches/*.patch; do \ + [ -e "$p" ] && git apply "$p" || true; done + +# Hidden assets baked by build_images.sh into the build context: +# task_ctx/assets/ckpts/nanowm-l2-csgo/model_state_dict.pt +# task_ctx/assets/csgo/1-200/*.hdf5 (held-out episode subset) +# task_ctx/assets/csgo_subset/{val_files.txt,val_starts.npy} +# task_ctx/assets/baseline/baseline_metrics.json (optional precomputed) +COPY task_ctx/assets/ /opt/nanowm/ +COPY task_ctx/task_pkg/ /opt/nanowm/task/ +COPY task_ctx/evaluator.py /opt/nanowm/task/evaluator.py + +# Modal builds the GPU image from this same checkout/assets at deploy time. +ENV FRONTIER_NWM_REPO=/opt/nanowm/nano-world-model \ + FRONTIER_NWM_CKPT=/opt/nanowm/ckpts/nanowm-l2-csgo/model_state_dict.pt \ + FRONTIER_NWM_CSGO_DATA=/opt/nanowm/data/csgo \ + FRONTIER_NWM_VAL_FILES=/opt/nanowm/data/csgo_subset/val_files.txt \ + FRONTIER_NWM_VAL_STARTS=/opt/nanowm/data/csgo_subset/val_starts.npy \ + FRONTIER_NWM_BASELINE_CACHE=/opt/nanowm/baseline/baseline_metrics.json \ + NWM_BAKE_DIR=/opt/nanowm diff --git a/2.0/problems/nanowm_rollout_stability/evaluate.sh b/2.0/problems/nanowm_rollout_stability/evaluate.sh new file mode 100755 index 00000000..11198e68 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/evaluate.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Local CLI evaluation for nanowm_rollout_speedup. +# Full runs need a GPU (local backend) or Modal credentials, plus the baked +# NanoWM assets (FRONTIER_NWM_* env, see stability_eval/settings.py). Without a GPU +# this falls back to the smoke path (patch-policy validation only), which is what +# repository CI exercises. +set -euo pipefail +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +export FRONTIER_NWM_PKG="${FRONTIER_NWM_PKG:-$SCRIPT_DIR}" +if ! python3 -c 'import torch; assert torch.cuda.is_available()' >/dev/null 2>&1; then + export FRONTIER_NWM_SMOKE="${FRONTIER_NWM_SMOKE:-1}" +fi +SOLUTION="${1:-$SCRIPT_DIR/reference.patch}" +exec python3 "$SCRIPT_DIR/evaluator.py" "$SOLUTION" diff --git a/2.0/problems/nanowm_rollout_stability/evaluator.py b/2.0/problems/nanowm_rollout_stability/evaluator.py new file mode 100644 index 00000000..b21a5171 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/evaluator.py @@ -0,0 +1,206 @@ +"""nanowm_rollout_speedup evaluator (Frontier-CS 2.0 contract). + +The agent submits a Python-only patch against a clean NanoWM checkout that +speeds up the diffusion SAMPLING for a fixed CSGO long-rollout invocation +(NanoWM-L/2, 50 frames, 4 context, nominal 50 DDIM steps). The judge applies the +patch, runs the rollout on a Modal GPU, and scores wall-clock speedup over the +unpatched baseline, gated by a rollout-quality (LPIPS-vs-GT) guardrail. + +This file holds the self-contained static patch policy + scoring. Heavy +orchestration (Modal sandbox, baseline caching) lives in `stability_eval/`. When +the GPU/Modal stack is unconfigured (local CI), the evaluator validates the +patch policy and returns a smoke score so the empty reference patch passes. +""" +from __future__ import annotations + +import hashlib +import os +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_PKG = os.environ.get("FRONTIER_NWM_PKG", str(Path(__file__).resolve().parent)) +if _PKG not in sys.path: + sys.path.insert(0, _PKG) + +from stability_eval import scoring, settings as S # noqa: E402 + +MAX_PATCH_BYTES = 256_000 + +# --- Patch policy ------------------------------------------------------------ +# The agent may only change the diffusion SAMPLING layer (Python source). The +# model architecture, VAE, datasets, metric, and harness are frozen. +ALLOWED = ( + "src/diffusion/*.py", + "src/diffusion/**/*.py", + "src/sample/sampling_utils.py", +) +DENIED = ( + "src/models/**", "src/latent_codecs/**", # frozen model + VAE + "src/sample/evaluate_metrics.py", "src/sample/plot_metrics.py", # the metric + "src/sample/rollout.py", # the judge harness + "src/wm_datasets/**", # data loading / no bench detection + "src/experiments/**", "src/main.py", "src/utils/**", + "**/*.so", "**/*.pyx", "**/*.c", "**/*.cpp", "**/*.cu", # no native + "**/setup.py", "**/pyproject.toml", "**/requirements*.txt", +) +# Forbidden substrings in ADDED lines (no benchmark detection / env leakage / +# timing short-circuits / hardcoded ground truth). +FORBIDDEN_TOKENS = ( + "FRONTIER_", "JUDGE_", "HARBOR_", "MODAL_", "HF_TOKEN", "NWM_TIME", + "csgo_validation_start_indices", "val_starts", "val_files", "gen_seconds", + "evaluate_metrics", "lpips", "ground_truth", "gt_frames", + "os.environ", "subprocess", "socket", "/judge", "/opt/nanowm/baseline", + "time.sleep", "while True", +) + + +@dataclass +class PatchFile: + old_path: str = "" + new_path: str = "" + added_lines: list[str] = field(default_factory=list) + + @property + def path(self) -> str: + return self.new_path if self.new_path != "/dev/null" else self.old_path + + +def _match(path: str, patterns: tuple[str, ...]) -> bool: + from fnmatch import fnmatch + return any(fnmatch(path, pat) for pat in patterns) + + +def _parse_patch(text: str) -> list[PatchFile]: + files: list[PatchFile] = [] + cur: PatchFile | None = None + for line in text.splitlines(): + if line.startswith("diff --git"): + if cur: + files.append(cur) + cur = PatchFile() + elif line.startswith("--- "): + if cur is None: + cur = PatchFile() + cur.old_path = line[4:].strip().removeprefix("a/") + elif line.startswith("+++ "): + if cur is None: + cur = PatchFile() + cur.new_path = line[4:].strip().removeprefix("b/") + elif line.startswith("+") and not line.startswith("+++"): + if cur is not None: + cur.added_lines.append(line[1:]) + if cur: + files.append(cur) + return files + + +def _safe(path: str) -> bool: + p = Path(path) + return not p.is_absolute() and ".." not in p.parts + + +def validate_patch(patch_path: Path) -> tuple[bool, str, dict[str, Any]]: + if not patch_path.exists(): + return False, "solution patch does not exist", {"valid_patch": 0} + size = patch_path.stat().st_size + if size > MAX_PATCH_BYTES: + return False, f"patch too large ({size} > {MAX_PATCH_BYTES})", {"valid_patch": 0} + text = patch_path.read_text(encoding="utf-8", errors="replace") + metrics: dict[str, Any] = { + "patch_bytes": size, + "patch_sha256": hashlib.sha256(text.encode("utf-8", "replace")).hexdigest(), + } + files = _parse_patch(text) + metrics["files_changed"] = [f.path for f in files] + if not files and size > 0: + return False, "could not parse any file hunks from patch", {**metrics, "valid_patch": 0} + for pf in files: + for cand in {pf.old_path, pf.new_path, pf.path}: + if cand and cand != "/dev/null" and not _safe(cand): + return False, f"unsafe patch path: {cand}", {**metrics, "valid_patch": 0} + if pf.new_path == "/dev/null": + return False, f"deleting source files is out of scope: {pf.old_path}", {**metrics, "valid_patch": 0} + path = pf.path + if _match(path, DENIED) or not path.endswith((".py", ".pyi")): + return False, f"path not allowed (frozen/native/harness): {path}", {**metrics, "valid_patch": 0} + if not _match(path, ALLOWED): + return False, f"path outside the allowlisted sampling layer: {path}", {**metrics, "valid_patch": 0} + blob = "\n".join(pf.added_lines) + for tok in FORBIDDEN_TOKENS: + if tok in blob: + return False, f"forbidden token in added lines ({tok}) in {path}", {**metrics, "valid_patch": 0} + metrics["valid_patch"] = 1 + return True, "patch accepted by static policy", metrics + + +# --- Evaluate ---------------------------------------------------------------- +def _resolve_patch(solution_path: str) -> Path: + p = Path(solution_path) + if p.is_dir(): + cand = p / "solution.patch" + if cand.exists(): + return cand + hits = sorted(p.rglob("*.patch")) + if hits: + return hits[0] + return cand # nonexistent -> validate_patch reports + return p + + +def prepare() -> dict: + return {"task": "nanowm_rollout_stability", "smoke": S.SMOKE, + "wallclock_tolerance": S.WALLCLOCK_TOLERANCE, "rollout_length": S.ROLLOUT_LENGTH, + "drift_tail_start": S.DRIFT_TAIL_START, "fixed_steps": S.NUM_SAMPLING_STEPS} + + +def evaluate(solution_path: str): + role = "final" if os.environ.get("FRONTIER_SUBMISSION_ROLE") == "final" else "agent" + patch_path = _resolve_patch(solution_path) + ok, msg, pmetrics = validate_patch(patch_path) + if not ok: + return 0.0, 0.0, f"patch rejected: {msg}", pmetrics + + # Local CI / no-GPU smoke: validate policy only, pass the (empty) reference. + if S.SMOKE or not S.CKPT.exists(): + return 1.0, 1.0, "smoke: patch policy valid (GPU/Modal unconfigured)", {**pmetrics, "smoke": 1} + + # Heavy path: run baseline (cached) + patched rollout on the GPU sandbox. + from stability_eval import orchestrate # Modal/GPU orchestration + clips = S.FINAL_CLIPS if role == "final" else S.QUICK_CLIPS + try: + baseline, patched = orchestrate.run_pair(patch_path, clips=clips, role=role) + except Exception as exc: # concise public error only + return 0.0, 0.0, f"evaluation failed: {type(exc).__name__}", {**pmetrics, "role": role} + + res = scoring.provisional_score( + baseline["tail_lpips"], patched["tail_lpips"], + baseline["gen_seconds"], patched["gen_seconds"], tolerance=S.WALLCLOCK_TOLERANCE, + ) + metrics = { + **pmetrics, "role": role, "clips": clips, + "baseline_tail_lpips": round(baseline["tail_lpips"], 4), + "patched_tail_lpips": round(patched["tail_lpips"], 4), + "rel_reduction_pct": round(res["rel_reduction_pct"], 2), + "baseline_seconds": round(baseline["gen_seconds"], 2), + "patched_seconds": round(patched["gen_seconds"], 2), + "wallclock_multiplier": round(res["wallclock_multiplier"], 3), + "score_formula": ("100*(baseline_tail-patched_tail)/baseline_tail * wallclock_mult; " + "wallclock_mult<1 if gen time rises >tol over baseline (no buying drift with compute)"), + } + msg = (f"tail-drift {patched['tail_lpips']:.3f} vs baseline {baseline['tail_lpips']:.3f} " + f"({res['rel_reduction_pct']:+.1f}%, {patched['gen_seconds']:.0f}s/budget {baseline['gen_seconds']:.0f}s); " + f"score {res['score']:.1f}") + return res["score"], res["score_unbounded"], msg, metrics + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("usage: evaluator.py ") + sys.exit(2) + import json + s, u, m, mt = evaluate(sys.argv[1]) + print(json.dumps({"score": s, "score_unbounded": u, "message": m, "metrics": mt}, indent=2)) + print(s) diff --git a/2.0/problems/nanowm_rollout_stability/harbor/app/README.md b/2.0/problems/nanowm_rollout_stability/harbor/app/README.md new file mode 100644 index 00000000..d8bab948 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/harbor/app/README.md @@ -0,0 +1,19 @@ +# nanowm_rollout_speedup — agent workspace + +Patch the diffusion **sampling** code of the NanoWM checkout at +`/app/nano-world-model` to reduce drift in the fixed CSGO 50-frame rollout at +iso-quality, then submit the unified diff as `/app/solution.patch`. + +Workflow: +1. Edit `nano-world-model/src/diffusion/**` or `src/sample/sampling_utils.py`. +2. `bash /app/public_test.sh` — static patch-policy check on your diff. +3. `bash /app/make_submission.sh` — writes `/app/solution.patch` from your edits. +4. `bash /app/submit.sh` — enqueue for the black-box judge (quick set). + +Rules (see the task statement): Python-only; allowed = `src/diffusion/**.py`, +`src/sample/sampling_utils.py`; the model/VAE/metric/harness/data are frozen and +denied; no env-var/benchmark/timing tricks. The judge fixes the rollout call — +you change the sampler internals. + +Scored on wall-clock speedup vs the unpatched baseline, gated by an LPIPS-vs-GT +quality guardrail (≤3% rise). See `../readme` and `../DESIGN.md`. diff --git a/2.0/problems/nanowm_rollout_stability/harbor/app/make_submission.sh b/2.0/problems/nanowm_rollout_stability/harbor/app/make_submission.sh new file mode 100755 index 00000000..becc45f6 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/harbor/app/make_submission.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Write /app/solution.patch from the agent's edits to /app/nano-world-model. +set -euo pipefail +cd /app/nano-world-model +git add -A 2>/dev/null || true +git diff --staged > /app/solution.patch 2>/dev/null || git diff > /app/solution.patch +echo "wrote /app/solution.patch ($(wc -l < /app/solution.patch) lines)" diff --git a/2.0/problems/nanowm_rollout_stability/harbor/app/public_test.py b/2.0/problems/nanowm_rollout_stability/harbor/app/public_test.py new file mode 100644 index 00000000..2dd23e3d --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/harbor/app/public_test.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Agent-facing public test: static patch-policy check (no GPU). + +Mirrors the judge's `validate_patch` so the agent can confirm a submission will +be accepted before enqueuing. Does NOT run the rollout or score speedup — that +happens on the judge's GPU. +""" +import sys +from pathlib import Path + +# locate the shared policy: prefer the baked task package, else the local copy. +for cand in ("/judge", "/opt/nanowm/task", str(Path(__file__).resolve().parents[2])): + if (Path(cand) / "evaluator.py").exists(): + sys.path.insert(0, cand) + break + +try: + from evaluator import validate_patch +except Exception: + print("NOTE: judge evaluator not available in this workspace; cannot run the " + "exact policy check here. Submit and read the judge feedback.") + sys.exit(0) + +patch = Path(sys.argv[1] if len(sys.argv) > 1 else "/app/solution.patch") +ok, msg, metrics = validate_patch(patch) +print(f"valid_patch={int(ok)} files={metrics.get('files_changed')}") +print(msg) +sys.exit(0 if ok else 1) diff --git a/2.0/problems/nanowm_rollout_stability/harbor/app/public_test.sh b/2.0/problems/nanowm_rollout_stability/harbor/app/public_test.sh new file mode 100755 index 00000000..a39757d5 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/harbor/app/public_test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +python3 "$SCRIPT_DIR/public_test.py" "${1:-/app/solution.patch}" diff --git a/2.0/problems/nanowm_rollout_stability/harbor/app/solution.patch b/2.0/problems/nanowm_rollout_stability/harbor/app/solution.patch new file mode 100644 index 00000000..e69de29b diff --git a/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-chunked-decode.patch b/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-chunked-decode.patch new file mode 100644 index 00000000..6d941a56 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-chunked-decode.patch @@ -0,0 +1,26 @@ +diff --git a/src/sample/rollout.py b/src/sample/rollout.py +index b83f0ee..dc8b03c 100644 +--- a/src/sample/rollout.py ++++ b/src/sample/rollout.py +@@ -225,8 +225,19 @@ def main(args): + generated_latents = torch.cat([generated_latents, new_latent], dim=1) + + print(f"Decoding and saving batch results...") +- gen_frames_batch = decode_latents(vae, generated_latents, vae_precision=vae_precision) +- gt_frames_batch = decode_latents(vae, gt_latents, vae_precision=vae_precision) ++ # Decode in (episode, frame-chunk) tiles: flattening B*T frames through ++ # the VAE in one call OOMs for long rollouts (a single 150-frame decode ++ # allocates ~19 GiB). Chunk both episode and frame dims. ++ _DEC_FCHUNK = 24 ++ def _decode_chunked(lat): ++ outs = [] ++ for i in range(lat.shape[0]): ++ fr = [decode_latents(vae, lat[i:i+1, j:j+_DEC_FCHUNK], vae_precision=vae_precision) ++ for j in range(0, lat.shape[1], _DEC_FCHUNK)] ++ outs.append(torch.cat(fr, dim=1)) ++ return torch.cat(outs) ++ gen_frames_batch = _decode_chunked(generated_latents) ++ gt_frames_batch = _decode_chunked(gt_latents) + + for i in range(current_batch_size): + sample_id = start_idx + i diff --git a/2.0/problems/nanowm_rollout_stability/readme b/2.0/problems/nanowm_rollout_stability/readme new file mode 100644 index 00000000..5b16a64b --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/readme @@ -0,0 +1,60 @@ +# NanoWM Rollout Stability — minimize long-horizon drift at fixed compute + +## Problem + +You are given a clean checkout of Nano World Models (arXiv:2605.23993) and its +frozen NanoWM-L/2 CSGO checkpoint. The judge runs a **fixed 80-frame** +autoregressive rollout (4 context frames, sequential, **50 DDIM steps**). Long +autoregressive rollouts accumulate perceptual error — by the tail of the rollout +the prediction has drifted into a "plausible but wrong" state (paper Finding #5). + +Your job: **minimize that drift** — the mean LPIPS-vs-ground-truth over the +**tail frames (index ≥ 60)** — by submitting a **Python-only patch** to the +diffusion **sampling** code, **without using more compute** (a wall-clock +budget = the unpatched baseline's generation time is enforced). + +This is a hard, open problem: simply adding denoising steps reduces drift but is +disallowed (it costs compute — that's the *speedup* task). At fixed compute you +must use the budget *smarter*: history stabilization, scheduling-matrix design, +drift-aware KV/feature caching that frees time for re-grounding, periodic +context re-anchoring, error-feedback correction, better solvers, etc. + +## What you submit + +A unified-diff patch at `/app/solution.patch` against `/app/nano-world-model`. +**Python source only**, within the diffusion sampling layer: +**Allowed:** `src/diffusion/**.py`, `src/sample/sampling_utils.py`. +**Denied:** model (`src/models/**`), VAE, the metric, the rollout harness +(`src/sample/rollout.py`), data loading, training/eval harness, native/build +files. No env-var/benchmark/timing tricks. Validated before running. + +## Evaluation & scoring + +- Judge applies your patch, runs the fixed 80-frame CSGO rollout on hidden + episodes (Modal GPU), measures **tail-drift** (mean LPIPS-vs-GT, frames ≥ 60) + and **generation wall-clock**. Quick set for iterative `submit.sh`; a larger + disjoint set for the final verifier (enough clips to resolve small drift + reductions above per-clip noise). +- **Score:** + +``` +score = clip(100 * (baseline_tail_drift - patched_tail_drift) / baseline_tail_drift, 0, 100) + * wallclock_multiplier +``` + + `wallclock_multiplier` is 1.0 while patched generation time stays within 10% + of the baseline, and decays beyond (so you cannot buy drift reduction with + more compute). A patch that does not reduce drift, exceeds the wall-clock + budget, crashes, or violates the patch policy scores 0. + +## Reference & difficulty + +`reference.patch` raises history stabilization (a one-line sampling change) — it +reliably reduces tail-drift a few percent over the baseline at iso-compute +(validated: 73% per-clip win, t≈2.5 over 22 clips), proving the task is solvable. +Substantially beating it is the open challenge. + +## Resource budget + +CPU agent + judge; one Modal GPU per evaluation. Evaluation timeout 21600 s. +See `AGENT.md`, `harbor/app/README.md`, and `DESIGN.md`. diff --git a/2.0/problems/nanowm_rollout_stability/reference.patch b/2.0/problems/nanowm_rollout_stability/reference.patch new file mode 100644 index 00000000..df50c606 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/reference.patch @@ -0,0 +1,12 @@ +diff --git a/src/diffusion/df_sample.py b/src/diffusion/df_sample.py +index 43a298a..9f23e70 100644 +--- a/src/diffusion/df_sample.py ++++ b/src/diffusion/df_sample.py +@@ -287,6 +287,7 @@ def dfot_sample( + truncate_at = clean_rows[0].item() + 1 + scheduling_matrix = scheduling_matrix[:truncate_at] + ++ history_stabilization_level = 0.20 # [stability-ref] stronger history stabilization reduces long-horizon drift + if history_stabilization_level > 0.0 and n_context_frames > 0 and context is not None: + assert 0.0 < history_stabilization_level < 1.0 + t_stab = int(round(history_stabilization_level * (diffusion.num_timesteps - 1))) diff --git a/2.0/problems/nanowm_rollout_stability/reference.py b/2.0/problems/nanowm_rollout_stability/reference.py new file mode 100644 index 00000000..64e937fe --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/reference.py @@ -0,0 +1,20 @@ +"""CI/local reference entry for nanowm_rollout_speedup. + +The real reference solution is `reference.patch` (a bf16-autocast sampling +speedup). The Frontier-CS 2.0 CLI's default `evaluator.py ` path +expects a Python file; this shim points the evaluator at reference.patch so the +standard `frontier eval` validation works. In smoke/no-GPU mode the evaluator +validates the patch policy and returns a passing score. +""" +import sys +from pathlib import Path + +REFERENCE_PATCH = str(Path(__file__).resolve().parent / "reference.patch") + +if __name__ == "__main__": + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import json + from evaluator import evaluate + s, u, m, mt = evaluate(REFERENCE_PATCH) + print(json.dumps({"score": s, "score_unbounded": u, "message": m, "metrics": mt}, indent=2)) + print(s) diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/__init__.py b/2.0/problems/nanowm_rollout_stability/stability_eval/__init__.py new file mode 100644 index 00000000..9c5b1828 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/__init__.py @@ -0,0 +1 @@ +"""nanowm_rollout_speedup evaluation package (shared by judge + public test).""" diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/modal_app.py b/2.0/problems/nanowm_rollout_stability/stability_eval/modal_app.py new file mode 100644 index 00000000..2482352e --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/modal_app.py @@ -0,0 +1,66 @@ +"""Modal app: run one NanoWM CSGO rollout (patched or vanilla) on one GPU. + +Mirrors the GPU-on-Modal pattern of vllm_llm_serving_optimization: the judge +container stays CPU-only and calls into a Modal GPU function. The NanoWM +checkout, L/2 CSGO checkpoint, and held-out CSGO episode subset are baked into +the Modal image at build time; the agent's patch text travels in per call. + +Parametrized via env vars (set by the judge): + NWM_MODAL_GPU Modal GPU string (default "L40S") + NWM_MODAL_APP Modal app name + NWM_HF_SECRET Modal Secret name (unused once weights are baked) + +Deploy: modal deploy stability_eval/modal_app.py +Call: run_rollout_remote(patch_text, clips, steps) -> metrics dict + +NOTE: validated structurally against the #145 modal pattern; end-to-end Modal +execution is pending maintainer Modal credentials (Della has no Modal access). +The LOCAL backend (orchestrate._run_local) is the path validated on H100. +""" +from __future__ import annotations + +import os + +GPU = os.environ.get("NWM_MODAL_GPU", "L40S") +APP_NAME = os.environ.get("NWM_MODAL_APP", "nanowm-rollout-speedup") +REMOTE_ROOT = "/opt/nanowm" + +try: + import modal + + image = ( + modal.Image.from_registry("nvidia/cuda:12.1.0-devel-ubuntu22.04", add_python="3.11") + .pip_install( + "torch==2.4.1", "torchvision==0.19.1", "numpy<2", "scipy==1.15.3", + "lpips", "diffusers[torch]==0.24.0", "omegaconf", "hydra-core", + "decord", "imageio", "imageio-ffmpeg", "opencv-python-headless", + "scikit-image", "pandas", "einops", "timm", "pytorch-lightning==2.4.0", + "huggingface_hub==0.25.2", "transformers==4.46.3", + ) + # bake the clean checkout + ckpt + CSGO subset + task package + .add_local_dir(os.environ.get("NWM_BAKE_DIR", "/opt/nanowm"), REMOTE_ROOT, copy=True) + ) + app = modal.App(APP_NAME) + + @app.function(gpu=GPU, image=image, timeout=3600) + def _rollout(patch_text: str, clips: int, steps: int) -> dict: + import sys, tempfile + from pathlib import Path + sys.path.insert(0, f"{REMOTE_ROOT}/task") + from stability_eval import runner, orchestrate # noqa + patch = None + if patch_text.strip(): + p = Path(tempfile.mkstemp(suffix=".patch")[1]) + p.write_text(patch_text) + patch = p + cfg = orchestrate._compose_config() + from stability_eval import settings as S + return runner.evaluate(S.REPO, patch, cfg, S.CKPT, clips=clips, steps=steps, device="cuda") + + def run_rollout_remote(patch_text: str, clips: int, steps: int) -> dict: + with app.run(): + return _rollout.remote(patch_text, clips, steps) + +except ImportError: # modal not installed (e.g. local validation) + def run_rollout_remote(patch_text: str, clips: int, steps: int) -> dict: # type: ignore + raise RuntimeError("modal not available; use the LOCAL backend (FRONTIER_NWM_BACKEND=local)") diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py b/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py new file mode 100644 index 00000000..f23dcabd --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py @@ -0,0 +1,91 @@ +"""GPU orchestration for nanowm_rollout_speedup. + +`run_pair(patch_path, clips, role)` returns (baseline_metrics, patched_metrics), +each {"gen_seconds", "lpips", "clips", "steps"}. + +Two backends, selected automatically: + - MODAL (FRONTIER_NWM_BACKEND=modal or a Modal token is present): runs the + rollout in a Modal GPU sandbox (the maintainer-facing path, mirroring + vllm_llm_serving_optimization; judge container stays CPU-only). + - LOCAL (default when a CUDA device is visible): runs the runner directly on + the local GPU. Used for Della validation and any GPU-equipped judge. + +The unpatched baseline is computed once and cached (baseline_cache_path); the +patched run always re-applies the agent patch to a clean checkout. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +from . import settings as S + + +def _compose_config() -> Path: + """Write the fixed CSGO rollout config (subset val list) the runner consumes.""" + out = Path(os.environ.get("FRONTIER_NWM_WORKDIR", "/tmp/nwm_speedup")) / "csgo_config.yaml" + out.parent.mkdir(parents=True, exist_ok=True) + if out.exists(): + return out + from hydra import compose, initialize_config_dir + from omegaconf import OmegaConf + with initialize_config_dir(config_dir=str(S.REPO / "src/configs"), version_base=None): + cfg = compose(config_name="config", overrides=[ + f"model={S.MODEL}", f"dataset={S.DATASET}", "experiment=evaluate_only", + "wandb.enabled=false", + f"dataset.loader.val_file_list={S.VAL_FILES}", + f"dataset.loader.train_file_list={S.VAL_FILES}", + f"dataset.loader.val_start_indices={S.VAL_STARTS}", + ]) + out.write_text(OmegaConf.to_yaml(cfg, resolve=False)) + return out + + +def _run_local(patch_path: Path | None, clips: int) -> dict: + from . import runner + return runner.evaluate(S.REPO, patch_path, _compose_config(), S.CKPT, + clips=clips, steps=S.NUM_SAMPLING_STEPS, device="cuda") + + +def _run_modal(patch_path: Path | None, clips: int) -> dict: + """Run one rollout inside a Modal GPU sandbox. Mirrors #145's modal_app: + the patch text + config travel into the sandbox, which runs stability_eval.runner + on the baked NanoWM checkout and returns the metrics dict.""" + from .modal_app import run_rollout_remote # deploys/looks up the Modal app + patch_text = "" if patch_path is None else Path(patch_path).read_text(errors="replace") + return run_rollout_remote(patch_text=patch_text, clips=clips, steps=S.NUM_SAMPLING_STEPS) + + +def _backend(): + b = os.environ.get("FRONTIER_NWM_BACKEND", "").lower() + if b in ("modal", "local"): + return b + if os.environ.get("MODAL_TOKEN_ID") or os.environ.get("MODAL_TOKEN_SECRET"): + return "modal" + return "local" + + +def _baseline(clips: int, run) -> dict: + cache = S.BASELINE_CACHE + if cache.exists(): + try: + data = json.loads(cache.read_text()) + if data.get("clips", 0) >= clips: + return data + except Exception: + pass + m = run(None, clips) + try: + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(json.dumps(m)) + except OSError: + pass + return m + + +def run_pair(patch_path: Path, clips: int, role: str = "agent") -> tuple[dict, dict]: + run = _run_modal if _backend() == "modal" else _run_local + baseline = _baseline(clips, run) + patched = run(Path(patch_path), clips) + return baseline, patched diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py b/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py new file mode 100644 index 00000000..8108e4ce --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py @@ -0,0 +1,124 @@ +"""Apply a sampler patch to a clean nano-world-model checkout, run the FIXED +CSGO rollout, and measure (generation wall-clock, LPIPS-vs-GT). + +The judge fixes the rollout invocation (rollout_length / history / num_steps / +scheduling); the agent's patch may only change the *sampling internals* +(`src/diffusion/**`, `src/sample/sampling_utils.py`) to make that invocation +reduce drift at iso-wall-clock. Speedup = baseline_seconds / patched_seconds; quality = +LPIPS vs ground truth, guardrailed against the unpatched baseline. + +Runnable standalone for local/Modal validation: + python -m stability_eval.runner --repo --patch \ + --out metrics.json [--steps 50 --clips 4] +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +from . import settings as S + + +def apply_patch(clean_repo: Path, patch_path: Path | None) -> Path: + work = Path(tempfile.mkdtemp(prefix="nwm_speedup_")) + repo = work / "repo" + # symlink the heavy/immutable parts, copy only src/ so patches are cheap + repo.mkdir(parents=True) + for child in clean_repo.iterdir(): + if child.name == "src": + shutil.copytree(child, repo / "src") + else: + (repo / child.name).symlink_to(child) + if patch_path is not None and str(patch_path) != "-" and Path(patch_path).stat().st_size > 0: + r = subprocess.run(["git", "apply", "--unsafe-paths", "--directory", str(repo), + str(patch_path)], capture_output=True, text=True) + if r.returncode != 0: + # fall back to patch(1) (git apply needs a git root / clean context) + r = subprocess.run(["patch", "-p1", "-d", str(repo), "-i", str(patch_path)], + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError(f"patch failed: {r.stderr[:2000]}") + return repo + + +def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, steps: int, + device: str = "cuda") -> float: + """Run rollout.py; return generation wall-clock seconds (rollout-region timed + inside; falls back to total wall).""" + save.mkdir(parents=True, exist_ok=True) + env = dict(os.environ) + env["CSGO_DATA_DIR"] = str(S.CSGO_DATA) + env["NWM_TIME_FILE"] = str(save / "gen_seconds.txt") + cmd = [sys.executable, "-u", "src/sample/rollout.py", "--config", str(config), + "--ckpt", str(ckpt), "--save_path", str(save), "--num_samples", str(clips), + "--batch_size", str(S.BATCH_SIZE), "--rollout_length", str(S.ROLLOUT_LENGTH), + "--history_length", str(S.HISTORY_LENGTH), "--scheduling_mode", S.SCHEDULING_MODE, + "--num_sampling_steps", str(steps), "--history_stabilization_level", str(S.HISTORY_STAB), + "--fps", "8"] + t0 = time.time() + r = subprocess.run(cmd, cwd=str(repo), env=env, capture_output=True, text=True) + wall = time.time() - t0 + if r.returncode != 0: + raise RuntimeError(f"rollout failed (exit {r.returncode}): {r.stderr[-3000:]}") + tf = save / "gen_seconds.txt" + if tf.exists(): + try: + return float(tf.read_text().strip()) + except Exception: + pass + return wall + + +def lpips_vs_gt(repo: Path, save: Path) -> tuple[float, float]: + """Returns (mean_lpips, tail_lpips) where tail = frames >= S.DRIFT_TAIL_START + (the drifted region of a long rollout).""" + out_csv = save / "metrics.csv" + r = subprocess.run([sys.executable, "src/sample/evaluate_metrics.py", "--video_dir", str(save), + "--history_length", str(S.HISTORY_LENGTH), "--output_csv", str(out_csv)], + cwd=str(repo), env={**os.environ, "CSGO_DATA_DIR": str(S.CSGO_DATA)}, + capture_output=True, text=True) + if not out_csv.exists(): + raise RuntimeError(f"metrics failed: {r.stderr[-2000:]}") + import pandas as pd + df = pd.read_csv(out_csv) + mean = float(df["lpips"].mean()) + tail_start = getattr(S, "DRIFT_TAIL_START", 60) + g = df.groupby("frame_idx")["lpips"].mean() + tail = float(g[g.index >= tail_start].mean()) if (g.index >= tail_start).any() else mean + return mean, tail + + +def evaluate(clean_repo: Path, patch_path: Path | None, config: Path, ckpt: Path, + clips: int, steps: int, device: str = "cuda") -> dict: + repo = apply_patch(clean_repo, patch_path) + try: + save = repo / "_rollout_out" + secs = run_rollout(repo, config, ckpt, save, clips, steps, device) + lp, tail = lpips_vs_gt(repo, save) + return {"gen_seconds": secs, "lpips": lp, "tail_lpips": tail, "clips": clips, "steps": steps} + finally: + shutil.rmtree(repo.parent, ignore_errors=True) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--repo", required=True) + ap.add_argument("--patch", default="-") + ap.add_argument("--config", required=True) + ap.add_argument("--ckpt", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--clips", type=int, default=S.QUICK_CLIPS) + ap.add_argument("--steps", type=int, default=S.NUM_SAMPLING_STEPS) + ap.add_argument("--device", default="cuda") + a = ap.parse_args() + m = evaluate(Path(a.repo), None if a.patch == "-" else Path(a.patch), Path(a.config), + Path(a.ckpt), a.clips, a.steps, a.device) + Path(a.out).write_text(json.dumps(m, indent=2)) + print(json.dumps(m, indent=2)) diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/scoring.py b/2.0/problems/nanowm_rollout_stability/stability_eval/scoring.py new file mode 100644 index 00000000..6b8e5b22 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/scoring.py @@ -0,0 +1,39 @@ +"""Scoring for nanowm_rollout_stability — the DUAL of the speedup task. + +Speedup: minimize wall-clock at iso-quality. Stability: minimize long-horizon +DRIFT (tail-frame LPIPS-vs-GT) at iso-WALL-CLOCK. Score = relative tail-drift +reduction over the unpatched baseline, gated by a wall-clock guardrail (so drift +can't simply be bought with more compute -- that's the speedup axis). +""" +from __future__ import annotations + + +def drift_reduction_score(baseline_tail: float, patched_tail: float) -> float: + """Relative tail-drift reduction, in [0, 100]. 0 if patched is no better.""" + if baseline_tail <= 0: + return 0.0 + rel = (baseline_tail - patched_tail) / baseline_tail + return max(0.0, min(100.0, 100.0 * rel)) + + +def wallclock_multiplier(baseline_seconds: float, patched_seconds: float, tolerance: float) -> float: + """1.0 while patched wall-clock stays within `tolerance` of baseline; decays + inverse-proportionally beyond (can't buy drift reduction with more compute).""" + base = max(baseline_seconds, 1e-9) + rel_over = max(0.0, (patched_seconds - baseline_seconds) / base) + if rel_over <= tolerance: + return 1.0 + return max(0.0, min(1.0, tolerance / rel_over)) + + +def provisional_score(baseline_tail, patched_tail, baseline_seconds, patched_seconds, tolerance): + drift_score = drift_reduction_score(baseline_tail, patched_tail) + wmult = wallclock_multiplier(baseline_seconds, patched_seconds, tolerance) + return { + "tail_drift_reduction": (baseline_tail - patched_tail), + "rel_reduction_pct": (100.0 * (baseline_tail - patched_tail) / max(baseline_tail, 1e-9)), + "drift_score": drift_score, + "wallclock_multiplier": wmult, + "score": max(0.0, min(100.0, drift_score * wmult)), + "score_unbounded": max(0.0, drift_score * wmult), + } diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py b/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py new file mode 100644 index 00000000..8c1948c8 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py @@ -0,0 +1,61 @@ +"""Task settings for nanowm_rollout_speedup, overridable via config.yaml +`evaluation` block (read by the judge) or env vars (local runs).""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +TASK_CONFIG_PATH = Path(os.environ.get("FRONTIER_NWM_TASK_CONFIG", "/judge/task_config.json")) + + +def _eval_cfg() -> dict: + try: + return (json.loads(TASK_CONFIG_PATH.read_text()).get("evaluation", {}) or {}) + except Exception: + return {} + + +_CFG = _eval_cfg() + + +def _get(name: str, default): + if name in os.environ: + v = os.environ[name] + try: + return type(default)(v) + except Exception: + return v + return _CFG.get(name.lower().replace("frontier_nwm_", ""), default) + + +# Domain / model +MODEL = _get("FRONTIER_NWM_MODEL", "nanowm_l2_csgo") +DATASET = _get("FRONTIER_NWM_DATASET", "game/csgo") +# Rollout protocol. LONG rollout (80 frames) so error accumulates into a +# drifted tail; FIXED steps + wall-clock budget (the agent improves the rollout +# PROCEDURE at iso-compute, not by adding steps). +ROLLOUT_LENGTH = int(_get("FRONTIER_NWM_ROLLOUT_LENGTH", 80)) +HISTORY_LENGTH = int(_get("FRONTIER_NWM_HISTORY_LENGTH", 4)) +NUM_SAMPLING_STEPS = int(_get("FRONTIER_NWM_NUM_STEPS", 50)) # fixed compute budget +SCHEDULING_MODE = _get("FRONTIER_NWM_SCHEDULING", "sequential") +HISTORY_STAB = float(_get("FRONTIER_NWM_HISTORY_STAB", 0.02)) # baseline default +# Drift metric: mean LPIPS over the drifted tail (frames >= this index). +DRIFT_TAIL_START = int(_get("FRONTIER_NWM_DRIFT_TAIL_START", 60)) +# Eval set sizes — drift reductions are small, so enough clips to resolve them +# above per-clip noise (validated: 22 clips gives t~2.5 for the reference). +QUICK_CLIPS = int(_get("FRONTIER_NWM_QUICK_CLIPS", 8)) +FINAL_CLIPS = int(_get("FRONTIER_NWM_FINAL_CLIPS", 24)) +BATCH_SIZE = int(_get("FRONTIER_NWM_BATCH_SIZE", 2)) +# Wall-clock guardrail: the patched rollout must not exceed the baseline's +# generation wall-clock by more than this (relative) — else drift is being +# bought with more compute (the speedup task's axis), so penalize. +WALLCLOCK_TOLERANCE = float(_get("FRONTIER_NWM_WALLCLOCK_TOLERANCE", 0.10)) +# Image-asset layout (baked into the images; overridable locally). +REPO = Path(_get("FRONTIER_NWM_REPO", "/opt/nanowm/nano-world-model")) +CKPT = Path(_get("FRONTIER_NWM_CKPT", "/opt/nanowm/ckpts/nanowm-l2-csgo/model_state_dict.pt")) +CSGO_DATA = Path(_get("FRONTIER_NWM_CSGO_DATA", "/opt/nanowm/data/csgo")) +VAL_FILES = Path(_get("FRONTIER_NWM_VAL_FILES", "/opt/nanowm/data/csgo_subset/val_files.txt")) +VAL_STARTS = Path(_get("FRONTIER_NWM_VAL_STARTS", "/opt/nanowm/data/csgo_subset/val_starts.npy")) +BASELINE_CACHE = Path(_get("FRONTIER_NWM_BASELINE_CACHE", "/opt/nanowm/baseline/baseline_metrics.json")) +SMOKE = os.environ.get("FRONTIER_NWM_SMOKE", "0") == "1" From e5e71a7005bf79f24ac5910c2a5f684c4f41ed28 Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Sat, 13 Jun 2026 03:59:00 -0400 Subject: [PATCH 06/12] nanowm_rollout_stability: validated end-to-end (stab=0.20 ref 5.5% tail-drift reduction @iso-wall-clock, score 5.54 > baseline) --- 2.0/problems/nanowm_rollout_stability/DESIGN.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/2.0/problems/nanowm_rollout_stability/DESIGN.md b/2.0/problems/nanowm_rollout_stability/DESIGN.md index 3e285c3a..fca7b546 100644 --- a/2.0/problems/nanowm_rollout_stability/DESIGN.md +++ b/2.0/problems/nanowm_rollout_stability/DESIGN.md @@ -45,6 +45,9 @@ score 1.0 when GPU/Modal unconfigured (local CI). Score = ## 5. Reference `reference.patch`: one-line history-stabilization bump (stab→0.20) in `df_sample.dfot_sample` — the calibrated reliable drift reducer (§2). +**Validated end-to-end on H100 (16 clips, local backend):** baseline tail-drift +0.658 (1928 s) → reference 0.622 (1925 s, iso-wall-clock) ⇒ **5.5% reduction, +wallclock_mult 1.0, score 5.54 > baseline** ✓. Patch policy + smoke pass. ## 6. Open items for maintainers Modal end-to-end run; bake-asset provenance (ckpt + held-out CSGO subset + From e0ec4144cbf6381d7cd7356a31dedaa4fe40804e Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Sat, 13 Jun 2026 04:01:45 -0400 Subject: [PATCH 07/12] nanowm_rollout_speedup: infra patch = frame-chunked decode (24) for long rollouts --- .../0001-rollout-chunked-decode.patch | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch index 411fd2dd..6d941a56 100644 --- a/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch +++ b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch @@ -1,23 +1,26 @@ diff --git a/src/sample/rollout.py b/src/sample/rollout.py -index b83f0ee..3b62b99 100644 +index b83f0ee..dc8b03c 100644 --- a/src/sample/rollout.py +++ b/src/sample/rollout.py -@@ -225,8 +225,16 @@ def main(args): +@@ -225,8 +225,19 @@ def main(args): generated_latents = torch.cat([generated_latents, new_latent], dim=1) print(f"Decoding and saving batch results...") - gen_frames_batch = decode_latents(vae, generated_latents, vae_precision=vae_precision) - gt_frames_batch = decode_latents(vae, gt_latents, vae_precision=vae_precision) -+ # Decode episode-by-episode: flattening B*T frames through the VAE in -+ # one call OOMs for long rollouts (e.g. 16 eps x 19 frames @ 256^2). -+ gen_frames_batch = torch.cat([ -+ decode_latents(vae, generated_latents[i:i+1], vae_precision=vae_precision) -+ for i in range(generated_latents.shape[0]) -+ ]) -+ gt_frames_batch = torch.cat([ -+ decode_latents(vae, gt_latents[i:i+1], vae_precision=vae_precision) -+ for i in range(gt_latents.shape[0]) -+ ]) ++ # Decode in (episode, frame-chunk) tiles: flattening B*T frames through ++ # the VAE in one call OOMs for long rollouts (a single 150-frame decode ++ # allocates ~19 GiB). Chunk both episode and frame dims. ++ _DEC_FCHUNK = 24 ++ def _decode_chunked(lat): ++ outs = [] ++ for i in range(lat.shape[0]): ++ fr = [decode_latents(vae, lat[i:i+1, j:j+_DEC_FCHUNK], vae_precision=vae_precision) ++ for j in range(0, lat.shape[1], _DEC_FCHUNK)] ++ outs.append(torch.cat(fr, dim=1)) ++ return torch.cat(outs) ++ gen_frames_batch = _decode_chunked(generated_latents) ++ gt_frames_batch = _decode_chunked(gt_latents) for i in range(current_batch_size): sample_id = start_idx + i From f10a319c463a9f296738a03fac6621d4f07cdfe4 Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Sat, 13 Jun 2026 11:39:48 -0400 Subject: [PATCH 08/12] fix(2.0/nanowm): use language: python so validate-benchmark20 resolves The CI validator (scripts/validate_problems.py -> get_language_config) only supports {python, cpp, rust}; `language: patch` raised `ValueError: Unsupported language: patch` and crashed the whole validate step with a traceback before any per-problem logic ran. Mirror vllm_llm_serving_optimization (#145), the canonical Modal-GPU 2.0 task: declare `language: python`, keep the real solution in reference.patch, and make reference.py a docstring-only placeholder. Also align the tag to `systems` (matches #145 and the sibling systems-optimization tasks duckdb_e2e_query_optimization / vector_db_ann). Submission contract is unchanged: agents still submit /app/solution.patch; the static patch policy + Modal-GPU scoring in {speedup,stability}_eval are untouched. The judge stays CPU-only (no runtime.docker.gpu), exactly like #145. Verified locally: language resolves to python, reference.py is found, both evaluators return 1.0 on the no-GPU smoke path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../nanowm_rollout_speedup/config.yaml | 8 ++++-- .../nanowm_rollout_speedup/reference.py | 26 ++++++------------- .../nanowm_rollout_stability/config.yaml | 8 ++++-- .../nanowm_rollout_stability/reference.py | 26 ++++++------------- 4 files changed, 28 insertions(+), 40 deletions(-) diff --git a/2.0/problems/nanowm_rollout_speedup/config.yaml b/2.0/problems/nanowm_rollout_speedup/config.yaml index 939b420d..80f330f4 100644 --- a/2.0/problems/nanowm_rollout_speedup/config.yaml +++ b/2.0/problems/nanowm_rollout_speedup/config.yaml @@ -1,6 +1,10 @@ -tag: ai +tag: systems runtime: - language: patch + # Submission is a Python-only source patch (the real reference is + # reference.patch). `language: python` keeps the file extension/CLI conventions + # standard (mirrors vllm_llm_serving_optimization, #145); there is no separate + # "patch" language in the framework. + language: python timeout_seconds: 21600 environment: >- Python-only patch against a clean NanoWM checkout (Nano World Models, diff --git a/2.0/problems/nanowm_rollout_speedup/reference.py b/2.0/problems/nanowm_rollout_speedup/reference.py index 64e937fe..42836a38 100644 --- a/2.0/problems/nanowm_rollout_speedup/reference.py +++ b/2.0/problems/nanowm_rollout_speedup/reference.py @@ -1,20 +1,10 @@ -"""CI/local reference entry for nanowm_rollout_speedup. +"""Reference placeholder for the nanowm_rollout_speedup task. -The real reference solution is `reference.patch` (a bf16-autocast sampling -speedup). The Frontier-CS 2.0 CLI's default `evaluator.py ` path -expects a Python file; this shim points the evaluator at reference.patch so the -standard `frontier eval` validation works. In smoke/no-GPU mode the evaluator -validates the patch policy and returns a passing score. +The Harbor task submits /app/solution.patch (a Python-only patch against a clean +Nano World Models checkout, arXiv:2605.23993). This Python file exists only so the +Frontier-CS 2.0 task layout stays conventional (cf. vllm_llm_serving_optimization, +#145); the actual reference solution lives in reference.patch — a one-line +bf16-autocast wrap of the diffusion sampling loop that yields a quality-preserving +speedup over the fp32 baseline. See DESIGN.md for the calibration and the end-to-end +Della H100 validation. """ -import sys -from pathlib import Path - -REFERENCE_PATCH = str(Path(__file__).resolve().parent / "reference.patch") - -if __name__ == "__main__": - sys.path.insert(0, str(Path(__file__).resolve().parent)) - import json - from evaluator import evaluate - s, u, m, mt = evaluate(REFERENCE_PATCH) - print(json.dumps({"score": s, "score_unbounded": u, "message": m, "metrics": mt}, indent=2)) - print(s) diff --git a/2.0/problems/nanowm_rollout_stability/config.yaml b/2.0/problems/nanowm_rollout_stability/config.yaml index de9728d0..85927251 100644 --- a/2.0/problems/nanowm_rollout_stability/config.yaml +++ b/2.0/problems/nanowm_rollout_stability/config.yaml @@ -1,6 +1,10 @@ -tag: ai +tag: systems runtime: - language: patch + # Submission is a Python-only source patch (the real reference is + # reference.patch). `language: python` keeps the file extension/CLI conventions + # standard (mirrors vllm_llm_serving_optimization, #145); there is no separate + # "patch" language in the framework. + language: python timeout_seconds: 21600 environment: >- Python-only patch against a clean NanoWM checkout (Nano World Models, diff --git a/2.0/problems/nanowm_rollout_stability/reference.py b/2.0/problems/nanowm_rollout_stability/reference.py index 64e937fe..e8ef56cb 100644 --- a/2.0/problems/nanowm_rollout_stability/reference.py +++ b/2.0/problems/nanowm_rollout_stability/reference.py @@ -1,20 +1,10 @@ -"""CI/local reference entry for nanowm_rollout_speedup. +"""Reference placeholder for the nanowm_rollout_stability task. -The real reference solution is `reference.patch` (a bf16-autocast sampling -speedup). The Frontier-CS 2.0 CLI's default `evaluator.py ` path -expects a Python file; this shim points the evaluator at reference.patch so the -standard `frontier eval` validation works. In smoke/no-GPU mode the evaluator -validates the patch policy and returns a passing score. +The Harbor task submits /app/solution.patch (a Python-only patch against a clean +Nano World Models checkout, arXiv:2605.23993). This Python file exists only so the +Frontier-CS 2.0 task layout stays conventional (cf. vllm_llm_serving_optimization, +#145); the actual reference solution lives in reference.patch — it raises the +diffusion-forcing history-stabilization level to reduce long-horizon tail drift at +iso-wall-clock. See DESIGN.md for the calibration and the end-to-end Della H100 +validation. """ -import sys -from pathlib import Path - -REFERENCE_PATCH = str(Path(__file__).resolve().parent / "reference.patch") - -if __name__ == "__main__": - sys.path.insert(0, str(Path(__file__).resolve().parent)) - import json - from evaluator import evaluate - s, u, m, mt = evaluate(REFERENCE_PATCH) - print(json.dumps({"score": s, "score_unbounded": u, "message": m, "metrics": mt}, indent=2)) - print(s) From be45521cda5aea0b97e53db9aa4732fb9018adcc Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Sun, 14 Jun 2026 02:22:38 -0400 Subject: [PATCH 09/12] fix(2.0/nanowm): deterministic seeding + region timer (audit P1/P3/P4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial audit found the rollout metric was unseeded and the baseline cached-not-paired: ~2-3% run-to-run noise (≈ the effect sizes) let a no-op patch score nonzero (gameable), and the bf16 reference showed an impossible -4.97% LPIPS "improvement" that was pure noise. Fix (judge infrastructure, outside the agent's editable sampling scope; the shared 2.0 adapter/template is untouched): - Deterministic clip-keyed seeding in the rollout harness so the baseline (unpatched) and patched arms draw identical per-clip initial noise (common random numbers); batch boundaries are clip-aligned so QUICK is a noise-identical prefix of FINAL. Regenerated into infra_patches/0001-rollout-judge-infra.patch (was chunked-decode only). - Real per-region sampling timer (writes NWM_TIME_FILE) so the speedup metric isolates the patchable region (was dead code -> full-process wall-clock). - *_eval: settings add SEED; runner passes --seed; orchestrate baseline cache keyed by (== clips, seed) so a cached baseline is a valid CRN partner. Re-validated on Della H100 (3 seeds): a no-op patch now scores 0.15 / 0.000 (ungameable), residual wall noise 0.24%, bf16 LPIPS delta +1.7% (expected), stability drift reduction 6.8% +/- 1.2% pooled paired t=5.15 p<1e-4. Canonical numbers reconciled across DESIGN/PR_SUMMARY (speedup 1.17x/22.3, stability 6.8%/6.8); LPIPS tables labeled. Standing audit conclusions C1-C6 unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- 2.0/problems/nanowm_rollout_speedup/DESIGN.md | 22 ++-- .../0001-rollout-chunked-decode.patch | 26 ----- .../0001-rollout-judge-infra.patch | 110 ++++++++++++++++++ .../speedup_eval/orchestrate.py | 5 +- .../speedup_eval/runner.py | 2 +- .../speedup_eval/settings.py | 5 + .../nanowm_rollout_stability/DESIGN.md | 29 +++-- .../0001-rollout-chunked-decode.patch | 26 ----- .../0001-rollout-judge-infra.patch | 110 ++++++++++++++++++ .../stability_eval/orchestrate.py | 5 +- .../stability_eval/runner.py | 2 +- .../stability_eval/settings.py | 5 + 12 files changed, 271 insertions(+), 76 deletions(-) delete mode 100644 2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch create mode 100644 2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-judge-infra.patch delete mode 100644 2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-chunked-decode.patch create mode 100644 2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-judge-infra.patch diff --git a/2.0/problems/nanowm_rollout_speedup/DESIGN.md b/2.0/problems/nanowm_rollout_speedup/DESIGN.md index 35c4e647..7d539ff9 100644 --- a/2.0/problems/nanowm_rollout_speedup/DESIGN.md +++ b/2.0/problems/nanowm_rollout_speedup/DESIGN.md @@ -16,7 +16,8 @@ per-problem via Modal. ## 2. Why this is a real task (calibration) Measured on Della H100, NanoWM-L/2 CSGO, 50-frame rollout, 12 held-out episodes, -LPIPS vs ground truth: +LPIPS vs ground truth (means over **all** frames incl. the 4 context frames — the +same convention the judge scores on; generated-only means are ~0.045 higher): | DDIM steps | speedup | LPIPS-vs-GT | Δ vs seq@50 | |---|---|---|---| @@ -74,13 +75,18 @@ within `quality_tolerance` LPIPS rise and decays inverse-proportionally beyond. the fp32 baseline (CI requires reference > baseline). The intended frontier (DPM-Solver++, caching, distillation) is left to the agent. -**Validated end-to-end on Della H100 (local backend, 4 CSGO clips, 2026-06-12):** -baseline 299.9 s / LPIPS 0.548 → bf16-patched 256.7 s / LPIPS 0.521 ⇒ **1.17× -speedup, quality_multiplier 1.0, score 22.4** (reference > baseline ✓). Patch -policy validated (accepts reference; rejects metric edits + env-var leakage); -smoke path returns 1.0 with the empty reference on CPU. The frontier (seq@2 = -+31% LPIPS) leaves wide headroom above the 1.17× reference for real fast-sampling -patches. **Pending:** end-to-end Modal execution (maintainer credentials). +**Validated end-to-end on Della H100 (local backend, 16 CSGO clips, seed 42, +sampling-region timed):** baseline 1102.1 s / LPIPS 0.523 → bf16-patched 944.0 s / +LPIPS 0.532 ⇒ **1.17× speedup, LPIPS +1.7% (within the 3% guardrail), +quality_multiplier 1.0, score 22.3** (reference > baseline ✓). The judge seeds the +rollout deterministically per clip (common random numbers), so the baseline and +patched arms share initial noise: a **no-op patch scores 0.15 (≈0, ungameable)** +and residual wall-clock noise is **0.24%** (the region timer excludes model/VAE/ +dataset load and the VAE decode the patch cannot touch). Patch policy validated +(accepts reference; rejects metric edits + env-var leakage); smoke path returns +1.0 with the empty reference on CPU. The frontier (seq@2 = +31% LPIPS) leaves wide +headroom above the 1.17× reference for real fast-sampling patches. **Pending:** +end-to-end Modal execution (maintainer credentials). ## 7. Open items for maintainers diff --git a/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch deleted file mode 100644 index 6d941a56..00000000 --- a/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-chunked-decode.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/src/sample/rollout.py b/src/sample/rollout.py -index b83f0ee..dc8b03c 100644 ---- a/src/sample/rollout.py -+++ b/src/sample/rollout.py -@@ -225,8 +225,19 @@ def main(args): - generated_latents = torch.cat([generated_latents, new_latent], dim=1) - - print(f"Decoding and saving batch results...") -- gen_frames_batch = decode_latents(vae, generated_latents, vae_precision=vae_precision) -- gt_frames_batch = decode_latents(vae, gt_latents, vae_precision=vae_precision) -+ # Decode in (episode, frame-chunk) tiles: flattening B*T frames through -+ # the VAE in one call OOMs for long rollouts (a single 150-frame decode -+ # allocates ~19 GiB). Chunk both episode and frame dims. -+ _DEC_FCHUNK = 24 -+ def _decode_chunked(lat): -+ outs = [] -+ for i in range(lat.shape[0]): -+ fr = [decode_latents(vae, lat[i:i+1, j:j+_DEC_FCHUNK], vae_precision=vae_precision) -+ for j in range(0, lat.shape[1], _DEC_FCHUNK)] -+ outs.append(torch.cat(fr, dim=1)) -+ return torch.cat(outs) -+ gen_frames_batch = _decode_chunked(generated_latents) -+ gt_frames_batch = _decode_chunked(gt_latents) - - for i in range(current_batch_size): - sample_id = start_idx + i diff --git a/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-judge-infra.patch b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-judge-infra.patch new file mode 100644 index 00000000..4c2b40b7 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-judge-infra.patch @@ -0,0 +1,110 @@ +diff --git a/src/sample/rollout.py b/src/sample/rollout.py +index b83f0ee..d4bfd6a 100644 +--- a/src/sample/rollout.py ++++ b/src/sample/rollout.py +@@ -9,6 +9,7 @@ Uses the sliced dataset for evaluation consistency. + """ + import os + import sys ++import time + sys.path.append(os.path.split(sys.path[0])[0]) + + import torch +@@ -131,9 +132,20 @@ def main(args): + print(f" History length: {history_length}") + print(f" Rollout length: {rollout_length}") + ++ gen_seconds = 0.0 # accumulated sampling-region wall-clock (excludes load/decode) + for start_idx in range(0, num_samples, batch_size): + end_idx = min(start_idx + batch_size, num_samples) + current_batch_size = end_idx - start_idx ++ ++ # Deterministic, clip-keyed RNG (judge infra, outside the agent's editable ++ # sampling scope): seed by the global clip index so the rollout is ++ # reproducible and the baseline (unpatched) and patched arms draw the SAME ++ # initial noise per clip (common random numbers). Batch boundaries are ++ # clip-aligned, so a QUICK K-clip eval is a noise-identical prefix of FINAL. ++ _seed = int(getattr(args, "seed", 42)) + start_idx ++ torch.manual_seed(_seed) ++ if device == "cuda": ++ torch.cuda.manual_seed_all(_seed) + + print(f"Processing batch {start_idx//batch_size + 1} (slices {start_idx} to {end_idx-1})...") + +@@ -181,7 +193,12 @@ def main(args): + gt_latents = encode_frames(vae, gt_visual, device, vae_precision=vae_precision) + generated_latents = gt_latents[:, :history_length] + +- # Now every sample in the batch has exactly rollout_length frames ++ # Now every sample in the batch has exactly rollout_length frames. ++ # Time ONLY the diffusion sampling region — the part the agent's patch can ++ # affect; model/VAE/dataset load and the VAE decode are excluded. ++ if device == "cuda": ++ torch.cuda.synchronize() ++ _samp_t0 = time.time() + for t in tqdm(range(history_length, rollout_length), desc=f"Rollout progress"): + context_latents = generated_latents[:, -history_length:] + +@@ -224,9 +241,25 @@ def main(args): + new_latent = pred_latents[:, history_length:history_length+1] + generated_latents = torch.cat([generated_latents, new_latent], dim=1) + ++ if device == "cuda": ++ torch.cuda.synchronize() ++ gen_seconds += time.time() - _samp_t0 ++ + print(f"Decoding and saving batch results...") +- gen_frames_batch = decode_latents(vae, generated_latents, vae_precision=vae_precision) +- gt_frames_batch = decode_latents(vae, gt_latents, vae_precision=vae_precision) ++ # Decode in (episode, frame-chunk) tiles + free generation memory first: ++ # for 320x512 CSGO frames the VAE decoder activations are huge, and the ++ # long rollout leaves little headroom, so a single big decode OOMs. ++ torch.cuda.empty_cache() ++ _DEC_FCHUNK = 6 ++ def _decode_chunked(lat): ++ outs = [] ++ for i in range(lat.shape[0]): ++ fr = [decode_latents(vae, lat[i:i+1, j:j+_DEC_FCHUNK], vae_precision=vae_precision) ++ for j in range(0, lat.shape[1], _DEC_FCHUNK)] ++ outs.append(torch.cat(fr, dim=1)) ++ return torch.cat(outs) ++ gen_frames_batch = _decode_chunked(generated_latents) ++ gt_frames_batch = _decode_chunked(gt_latents) + + for i in range(current_batch_size): + sample_id = start_idx + i +@@ -234,6 +267,15 @@ def main(args): + save_video(gt_frames_batch[i], str(save_dir / f"sample_{sample_id:04d}_gt.mp4"), fps=args.fps) + save_comparison_video(gt_frames_batch[i], gen_frames_batch[i], str(save_dir / f"sample_{sample_id:04d}_compare.mp4"), fps=args.fps) + ++ # Expose the sampling-region wall-clock to the judge so the speedup metric ++ # isolates the patchable region (falls back to total wall if file unset). ++ _tf = os.environ.get("NWM_TIME_FILE") ++ if _tf: ++ try: ++ Path(_tf).write_text(f"{gen_seconds}\n") ++ except Exception: ++ pass ++ + print(f"Done! Processed {num_samples} samples.") + + if __name__ == "__main__": +@@ -259,6 +301,9 @@ if __name__ == "__main__": + help="DFoT stabilization noise level in [0, 1) for context frames. " + "0.0 disables; 0.02 is the DFoT default recommended starting point.") + parser.add_argument("--use_fp16", action="store_true", help="Use fp16") ++ parser.add_argument("--seed", type=int, default=42, ++ help="Deterministic RNG seed (judge infra). Per-clip seed = seed + clip_index " ++ "so the baseline and patched arms share initial noise (common random numbers).") + parser.add_argument("--vae_model_path", type=str, default=None, + help="Override the VAE path from the training config (e.g. local dir for offline nodes).") + +@@ -282,7 +327,7 @@ if __name__ == "__main__": + + # Top-level / non-hierarchical CLI args + for key in ('ckpt', 'save_path', 'num_samples', 'batch_size', 'rollout_length', +- 'history_length', 'fps', 'eta', 'use_fp16', 'vae_model_path'): ++ 'history_length', 'fps', 'eta', 'use_fp16', 'vae_model_path', 'seed'): + val = cli_args.get(key) + if val is not None: + cli_overrides[key] = val diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py index b02709f0..5ba87dbe 100644 --- a/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py @@ -71,11 +71,14 @@ def _baseline(clips: int, run) -> dict: if cache.exists(): try: data = json.loads(cache.read_text()) - if data.get("clips", 0) >= clips: + # Deterministic seeding makes the cached baseline a valid common-random + # -numbers partner ONLY for the same clip set (== not >=) and same seed. + if data.get("clips") == clips and data.get("seed") == S.SEED: return data except Exception: pass m = run(None, clips) + m["seed"] = S.SEED try: cache.parent.mkdir(parents=True, exist_ok=True) cache.write_text(json.dumps(m)) diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py index 9ef83a78..8525b2ea 100644 --- a/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py @@ -61,7 +61,7 @@ def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, st "--batch_size", str(S.BATCH_SIZE), "--rollout_length", str(S.ROLLOUT_LENGTH), "--history_length", str(S.HISTORY_LENGTH), "--scheduling_mode", S.SCHEDULING_MODE, "--num_sampling_steps", str(steps), "--history_stabilization_level", str(S.HISTORY_STAB), - "--fps", "8"] + "--fps", "8", "--seed", str(S.SEED)] t0 = time.time() r = subprocess.run(cmd, cwd=str(repo), env=env, capture_output=True, text=True) wall = time.time() - t0 diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py index 79942cfc..0218506d 100644 --- a/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py @@ -43,6 +43,11 @@ def _get(name: str, default): QUICK_CLIPS = int(_get("FRONTIER_NWM_QUICK_CLIPS", 4)) FINAL_CLIPS = int(_get("FRONTIER_NWM_FINAL_CLIPS", 16)) BATCH_SIZE = int(_get("FRONTIER_NWM_BATCH_SIZE", 4)) +# Deterministic RNG seed (judge infra): per-clip seed = SEED + clip_index, so the +# baseline (unpatched) and patched arms draw identical initial noise per clip +# (common random numbers). Makes a no-op patch score exactly 0 and the cached +# baseline a valid CRN partner. Vary only for multi-seed robustness runs. +SEED = int(_get("FRONTIER_NWM_SEED", 42)) # Quality guardrail: patched rollout LPIPS may rise at most this (relative) # above the baseline (unpatched seq@50) LPIPS before the score is penalized. QUALITY_TOLERANCE = float(_get("FRONTIER_NWM_QUALITY_TOLERANCE", 0.03)) diff --git a/2.0/problems/nanowm_rollout_stability/DESIGN.md b/2.0/problems/nanowm_rollout_stability/DESIGN.md index fca7b546..debd0e48 100644 --- a/2.0/problems/nanowm_rollout_stability/DESIGN.md +++ b/2.0/problems/nanowm_rollout_stability/DESIGN.md @@ -17,16 +17,20 @@ compute the drift is largely model-intrinsic, so reducing it is genuinely hard. But it is NOT impossible: a paired test (NanoWM-L/2, 80-frame, tail f≥60) shows the rollout PROCEDURE moves drift reliably above noise — -| setting (fixed steps=50) | tail-drift | +| setting (fixed steps=50) | tail-drift (mean over 3 seeds) | |---|---| -| stab=0.02 (baseline) | 0.652 | -| stab=0.20 (reference) | **0.629** | - -reduction 0.023 ± 0.009 SE, **73% per-clip win, t≈2.5 over 22 clips**. So a -reliably-better-than-baseline reference exists (task is well-posed/solvable); -substantially beating it requires real drift-reduction work (the open challenge). -The earlier 6-clip read showed it "within noise" — small effects need enough -clips, hence final_clips=24. +| stab=0.02 (baseline) | 0.655 | +| stab=0.20 (reference) | **0.610** | + +With deterministic common-random-numbers pairing (the judge seeds each clip, so +baseline and patched share initial noise), the reference reduces tail-drift by +**6.8% ± 1.2% across 3 seeds** (per-seed 8.3 / 6.2 / 6.1%, 74% per-clip win); +pooled over 66 paired clip-runs the effect is **paired t=5.15, p<1e-4, Wilcoxon +p<1e-4**. So a clearly-better-than-baseline reference exists (task well-posed/ +solvable); substantially beating it requires real drift-reduction work (the open +challenge). CRN pairing was essential: an unpaired/unseeded read diluted the +effect into noise (the original marginal single-seed t≈2.4), so the judge now +seeds deterministically and a no-op patch scores exactly 0. ## 3. Fixed-compute constraint (why it's distinct from speedup) Adding denoising steps reduces drift (paper Fig 6) but costs compute — that's the @@ -45,9 +49,10 @@ score 1.0 when GPU/Modal unconfigured (local CI). Score = ## 5. Reference `reference.patch`: one-line history-stabilization bump (stab→0.20) in `df_sample.dfot_sample` — the calibrated reliable drift reducer (§2). -**Validated end-to-end on H100 (16 clips, local backend):** baseline tail-drift -0.658 (1928 s) → reference 0.622 (1925 s, iso-wall-clock) ⇒ **5.5% reduction, -wallclock_mult 1.0, score 5.54 > baseline** ✓. Patch policy + smoke pass. +**Validated end-to-end on H100 (22 clips × 3 seeds, local backend, CRN-paired):** +tail-drift 0.655 → 0.610 at iso-wall-clock (gen-time Δ ≤0.2%, wallclock_mult 1.0) +⇒ **6.8% ± 1.2% reduction, score 6.8 > baseline** ✓ (pooled paired t=5.15, +p<1e-4; a no-op patch scores 0.000). Patch policy + smoke pass. ## 6. Open items for maintainers Modal end-to-end run; bake-asset provenance (ckpt + held-out CSGO subset + diff --git a/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-chunked-decode.patch b/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-chunked-decode.patch deleted file mode 100644 index 6d941a56..00000000 --- a/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-chunked-decode.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/src/sample/rollout.py b/src/sample/rollout.py -index b83f0ee..dc8b03c 100644 ---- a/src/sample/rollout.py -+++ b/src/sample/rollout.py -@@ -225,8 +225,19 @@ def main(args): - generated_latents = torch.cat([generated_latents, new_latent], dim=1) - - print(f"Decoding and saving batch results...") -- gen_frames_batch = decode_latents(vae, generated_latents, vae_precision=vae_precision) -- gt_frames_batch = decode_latents(vae, gt_latents, vae_precision=vae_precision) -+ # Decode in (episode, frame-chunk) tiles: flattening B*T frames through -+ # the VAE in one call OOMs for long rollouts (a single 150-frame decode -+ # allocates ~19 GiB). Chunk both episode and frame dims. -+ _DEC_FCHUNK = 24 -+ def _decode_chunked(lat): -+ outs = [] -+ for i in range(lat.shape[0]): -+ fr = [decode_latents(vae, lat[i:i+1, j:j+_DEC_FCHUNK], vae_precision=vae_precision) -+ for j in range(0, lat.shape[1], _DEC_FCHUNK)] -+ outs.append(torch.cat(fr, dim=1)) -+ return torch.cat(outs) -+ gen_frames_batch = _decode_chunked(generated_latents) -+ gt_frames_batch = _decode_chunked(gt_latents) - - for i in range(current_batch_size): - sample_id = start_idx + i diff --git a/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-judge-infra.patch b/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-judge-infra.patch new file mode 100644 index 00000000..4c2b40b7 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-judge-infra.patch @@ -0,0 +1,110 @@ +diff --git a/src/sample/rollout.py b/src/sample/rollout.py +index b83f0ee..d4bfd6a 100644 +--- a/src/sample/rollout.py ++++ b/src/sample/rollout.py +@@ -9,6 +9,7 @@ Uses the sliced dataset for evaluation consistency. + """ + import os + import sys ++import time + sys.path.append(os.path.split(sys.path[0])[0]) + + import torch +@@ -131,9 +132,20 @@ def main(args): + print(f" History length: {history_length}") + print(f" Rollout length: {rollout_length}") + ++ gen_seconds = 0.0 # accumulated sampling-region wall-clock (excludes load/decode) + for start_idx in range(0, num_samples, batch_size): + end_idx = min(start_idx + batch_size, num_samples) + current_batch_size = end_idx - start_idx ++ ++ # Deterministic, clip-keyed RNG (judge infra, outside the agent's editable ++ # sampling scope): seed by the global clip index so the rollout is ++ # reproducible and the baseline (unpatched) and patched arms draw the SAME ++ # initial noise per clip (common random numbers). Batch boundaries are ++ # clip-aligned, so a QUICK K-clip eval is a noise-identical prefix of FINAL. ++ _seed = int(getattr(args, "seed", 42)) + start_idx ++ torch.manual_seed(_seed) ++ if device == "cuda": ++ torch.cuda.manual_seed_all(_seed) + + print(f"Processing batch {start_idx//batch_size + 1} (slices {start_idx} to {end_idx-1})...") + +@@ -181,7 +193,12 @@ def main(args): + gt_latents = encode_frames(vae, gt_visual, device, vae_precision=vae_precision) + generated_latents = gt_latents[:, :history_length] + +- # Now every sample in the batch has exactly rollout_length frames ++ # Now every sample in the batch has exactly rollout_length frames. ++ # Time ONLY the diffusion sampling region — the part the agent's patch can ++ # affect; model/VAE/dataset load and the VAE decode are excluded. ++ if device == "cuda": ++ torch.cuda.synchronize() ++ _samp_t0 = time.time() + for t in tqdm(range(history_length, rollout_length), desc=f"Rollout progress"): + context_latents = generated_latents[:, -history_length:] + +@@ -224,9 +241,25 @@ def main(args): + new_latent = pred_latents[:, history_length:history_length+1] + generated_latents = torch.cat([generated_latents, new_latent], dim=1) + ++ if device == "cuda": ++ torch.cuda.synchronize() ++ gen_seconds += time.time() - _samp_t0 ++ + print(f"Decoding and saving batch results...") +- gen_frames_batch = decode_latents(vae, generated_latents, vae_precision=vae_precision) +- gt_frames_batch = decode_latents(vae, gt_latents, vae_precision=vae_precision) ++ # Decode in (episode, frame-chunk) tiles + free generation memory first: ++ # for 320x512 CSGO frames the VAE decoder activations are huge, and the ++ # long rollout leaves little headroom, so a single big decode OOMs. ++ torch.cuda.empty_cache() ++ _DEC_FCHUNK = 6 ++ def _decode_chunked(lat): ++ outs = [] ++ for i in range(lat.shape[0]): ++ fr = [decode_latents(vae, lat[i:i+1, j:j+_DEC_FCHUNK], vae_precision=vae_precision) ++ for j in range(0, lat.shape[1], _DEC_FCHUNK)] ++ outs.append(torch.cat(fr, dim=1)) ++ return torch.cat(outs) ++ gen_frames_batch = _decode_chunked(generated_latents) ++ gt_frames_batch = _decode_chunked(gt_latents) + + for i in range(current_batch_size): + sample_id = start_idx + i +@@ -234,6 +267,15 @@ def main(args): + save_video(gt_frames_batch[i], str(save_dir / f"sample_{sample_id:04d}_gt.mp4"), fps=args.fps) + save_comparison_video(gt_frames_batch[i], gen_frames_batch[i], str(save_dir / f"sample_{sample_id:04d}_compare.mp4"), fps=args.fps) + ++ # Expose the sampling-region wall-clock to the judge so the speedup metric ++ # isolates the patchable region (falls back to total wall if file unset). ++ _tf = os.environ.get("NWM_TIME_FILE") ++ if _tf: ++ try: ++ Path(_tf).write_text(f"{gen_seconds}\n") ++ except Exception: ++ pass ++ + print(f"Done! Processed {num_samples} samples.") + + if __name__ == "__main__": +@@ -259,6 +301,9 @@ if __name__ == "__main__": + help="DFoT stabilization noise level in [0, 1) for context frames. " + "0.0 disables; 0.02 is the DFoT default recommended starting point.") + parser.add_argument("--use_fp16", action="store_true", help="Use fp16") ++ parser.add_argument("--seed", type=int, default=42, ++ help="Deterministic RNG seed (judge infra). Per-clip seed = seed + clip_index " ++ "so the baseline and patched arms share initial noise (common random numbers).") + parser.add_argument("--vae_model_path", type=str, default=None, + help="Override the VAE path from the training config (e.g. local dir for offline nodes).") + +@@ -282,7 +327,7 @@ if __name__ == "__main__": + + # Top-level / non-hierarchical CLI args + for key in ('ckpt', 'save_path', 'num_samples', 'batch_size', 'rollout_length', +- 'history_length', 'fps', 'eta', 'use_fp16', 'vae_model_path'): ++ 'history_length', 'fps', 'eta', 'use_fp16', 'vae_model_path', 'seed'): + val = cli_args.get(key) + if val is not None: + cli_overrides[key] = val diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py b/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py index f23dcabd..19d9f564 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py @@ -71,11 +71,14 @@ def _baseline(clips: int, run) -> dict: if cache.exists(): try: data = json.loads(cache.read_text()) - if data.get("clips", 0) >= clips: + # Deterministic seeding makes the cached baseline a valid common-random + # -numbers partner ONLY for the same clip set (== not >=) and same seed. + if data.get("clips") == clips and data.get("seed") == S.SEED: return data except Exception: pass m = run(None, clips) + m["seed"] = S.SEED try: cache.parent.mkdir(parents=True, exist_ok=True) cache.write_text(json.dumps(m)) diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py b/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py index 8108e4ce..9474ea5e 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py @@ -61,7 +61,7 @@ def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, st "--batch_size", str(S.BATCH_SIZE), "--rollout_length", str(S.ROLLOUT_LENGTH), "--history_length", str(S.HISTORY_LENGTH), "--scheduling_mode", S.SCHEDULING_MODE, "--num_sampling_steps", str(steps), "--history_stabilization_level", str(S.HISTORY_STAB), - "--fps", "8"] + "--fps", "8", "--seed", str(S.SEED)] t0 = time.time() r = subprocess.run(cmd, cwd=str(repo), env=env, capture_output=True, text=True) wall = time.time() - t0 diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py b/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py index 8c1948c8..4ecdef92 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py @@ -47,6 +47,11 @@ def _get(name: str, default): QUICK_CLIPS = int(_get("FRONTIER_NWM_QUICK_CLIPS", 8)) FINAL_CLIPS = int(_get("FRONTIER_NWM_FINAL_CLIPS", 24)) BATCH_SIZE = int(_get("FRONTIER_NWM_BATCH_SIZE", 2)) +# Deterministic RNG seed (judge infra): per-clip seed = SEED + clip_index, so the +# baseline (unpatched) and patched arms draw identical initial noise per clip +# (common random numbers). Makes a no-op patch score exactly 0 and the cached +# baseline a valid CRN partner. Vary only for multi-seed robustness runs. +SEED = int(_get("FRONTIER_NWM_SEED", 42)) # Wall-clock guardrail: the patched rollout must not exceed the baseline's # generation wall-clock by more than this (relative) — else drift is being # bought with more compute (the speedup task's axis), so penalize. From e06fed9fc42d2efd74cf5dd7a5f3036a598e86ae Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Sun, 14 Jun 2026 14:54:42 -0400 Subject: [PATCH 10/12] docs(2.0/nanowm): include the adversarial-audit report in each task Adds AUDIT_REPORT.md (the multi-agent adversarial audit + the "fixes applied + validated" table) to both task dirs so reviewers can see the provenance: the refuted strong attacks (fair fp32 baseline, no leakage, real frontier), the one HIGH finding (unseeded/cached metric) and its deterministic-seeding fix, and the post-fix validation numbers. Absolute working-repo paths rewritten to relative. PR body reference updated to point here. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../nanowm_rollout_speedup/AUDIT_REPORT.md | 123 ++++++++++++++++++ .../nanowm_rollout_stability/AUDIT_REPORT.md | 123 ++++++++++++++++++ 2 files changed, 246 insertions(+) create mode 100644 2.0/problems/nanowm_rollout_speedup/AUDIT_REPORT.md create mode 100644 2.0/problems/nanowm_rollout_stability/AUDIT_REPORT.md diff --git a/2.0/problems/nanowm_rollout_speedup/AUDIT_REPORT.md b/2.0/problems/nanowm_rollout_speedup/AUDIT_REPORT.md new file mode 100644 index 00000000..887f7498 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/AUDIT_REPORT.md @@ -0,0 +1,123 @@ +# NanoWM Frontier-CS 2.0 — Audit Report + +**Scope:** `nanowm_rollout_speedup` and `nanowm_rollout_stability` benchmark tasks. +**Date:** 2026-06-13. **Audit lead synthesis** over 5 independent auditors + adversarial verifiers. +**Bar applied:** these are benchmark tasks; a *modest* reference is fine (headroom for agents). Real risks are (a) unfair/strawman baseline, (b) reference that does not reliably beat baseline, (c) marginal statistics sold as robust, (d) gameable/leaky metric, (e) over-claim in docs/PR. + +This report uses **only findings whose verifier verdict was `real==true`**; refuted findings are excluded. Overlapping findings (many auditors hit the same root causes — unseeded sampling, cached baseline, the 7.2/5.54/3.56 doc spread) are de-duplicated into single problems. All numbers below were independently recomputed with the project's own env (`envs/nanowm/bin/python`, scipy) using the harness's exact tail definition (`groupby(frame_idx).mean()` then mean over `frame_idx>=60`, `is_context` excluded). + +--- + +## Summary + +Both tasks rest on a **sound domain choice and clean leakage controls** (C1, C6 survive). The reference solutions *do* beat baseline in every run measured, and the baselines are fair (symmetric harness, the model's own defaults, the in-scope sampling layer is the only lever). The strong claims that this is a strawman baseline, a non-viable task, or a fabricated 7.2% headline were **refuted** on the artifacts. + +The genuine residue is concentrated and mostly **low severity**, but one item is **high**: + +1. **(HIGH) The metric is unseeded and the baseline is cached, not re-measured paired.** Sampling draws unseeded `th.randn` and the harness pairs a *cached* baseline against a *freshly-run* patched arm. Same-config run-to-run tail-LPIPS noise (~2-3%) is the same order as both the stability effect (~3.6%) and the 3% speedup quality guardrail. Consequence: a no-op patch scores nonzero (random sign), and a true iso-quality speedup patch can trip the guardrail on an unlucky draw. This is the one item that touches *gameability* and *"reliably beats baseline."* + +The remaining real items are **documentation/robustness hygiene** (low): three-way numeric inconsistency across docs (stability 3.56 / 5.54 / 7.2; speedup 22.4 / 20.1), a dead per-region timer that silently falls back to full-process wall-clock, a single-seed marginal `t=2.41 (p=0.025)` sold as "reliably," the baseline-cache `>= clips` rule, and absolute-LPIPS tables that include context frames without saying so. None of these, alone, makes a task ill-posed — but the HIGH item plus the doc spread together mean the *magnitude* of the reference margin is overstated and noise-contaminated. + +**Bottom line:** ship-able after fixing the seeding/baseline-pairing (1 high) and reconciling the doc numbers. The tasks are well-posed and the references are real; the scoring is just noisier and the docs more optimistic than advertised. + +--- + +## Confirmed Problems (severity-ranked) + +### P1 — Unseeded sampling + cached-not-paired baseline makes the metric noisy and gameable `HIGH` +- **Evidence:** + - No seed anywhere on the eval path: `grep -rnE 'manual_seed|seed_everything|Generator\(' ` over `nano-world-model/src/sample/rollout.py`, `src/diffusion/df_sample.py`, `src/diffusion/gaussian_diffusion.py`, and both `*_eval/` dirs returns nothing. Initial latent is unseeded `th.randn(*shape, device=device)` (`gaussian_diffusion.py:808`, also `:522,:690`); history-stab `q_sample` noise (`df_sample.py:296`) is unseeded. + - Baseline is cached and reused, patched runs fresh: `speedup_eval/orchestrate.py:69-91` `_baseline()` reads `S.BASELINE_CACHE` and only reruns if missing/insufficient; `run_pair()` returns `(cached_baseline, fresh_patched)`. Identical pattern in `stability_eval/orchestrate.py`. + - Same-config noise floor (independent unpatched stab=0.02 runs): tail-LPIPS spread `0.6402 / 0.6518 / 0.6582 / 0.6539` = **0.018 abs / 2.8% rel**. Speedup baseline LPIPS `0.5479` (speedup_val) vs `0.5363` (e2e) = **2.1%** pure run-to-run noise on the same 4 clips. + - **Smoking gun:** the bf16-autocast reference (a pure precision change, `reference.patch:10`) shows patched LPIPS `0.5206` vs baseline `0.5479` = a **4.97% LPIPS *improvement*** — physically impossible for lowering precision; it is run-to-run noise. + - Gameability: stability score `= 100*(base_tail - patched_tail)/base_tail` (`scoring.py`); a no-op allowlisted whitespace edit re-runs fresh against the cached baseline and scores ~2-3 pts of random sign. Speedup guardrail tolerance is `0.03` (`settings.py`), within ~1.5× the observed ~2% baseline noise, so an iso-quality patch can be zeroed on an unlucky draw. +- **Why it matters:** hits (d) non-gameable metric and (b) "reliably beats baseline." Scores carry an unacknowledged noise/offset budget; marginal effects are confounded with RNG. +- **Fix:** seed the rollout deterministically keyed to clip id (`torch.manual_seed(hash(clip_id))` before the initial `th.randn`), **and** recompute the baseline on the exact same clips/seed *in the same job* (common random numbers) instead of serving a cached baseline. This is the standard paired/CRN remedy and shrinks the metric noise toward zero for the baseline-vs-patched delta. + +### P2 — Mutually inconsistent baselines and scores across docs; no single canonical number `LOW` +- **Evidence:** + - Stability reference reported as **three** values for the same intervention: `3.56%` (drift_ref, 22 clips, t=2.41 — reproduced), `5.54%` (stability_val, 16 clips — `DESIGN.md:48-50`, `PR_SUMMARY.md:25`), and `7.2%` / score 7.2 (`CALIBRATION_FINDINGS.md:274`, 24 clips). The `7.2` appears in exactly **one** place (a calibration log) and **is** backed by a real judge run (`calib/logs/eval_e2e_stab_9617933.log`: `score 7.204`, patched tail `0.6068`) — so it is not fabricated, but it is unreconciled and it is the high end of a noisy range. + - Speedup reference reported as `1.17×/22.4` (`DESIGN.md:78-79`, `PR_SUMMARY.md:14`, `CALIBRATION_FINDINGS.md:253`) vs `1.15×/20.1` (`CALIBRATION_FINDINGS.md:273`) — **two distinct 4-clip runs with different baselines** (299.95s vs 291.36s = 2.95% swing) quoted interchangeably. Cross-pairing the two baselines moves the score **4.2 pts** (18.3 ↔ 22.4), and only the `speedup_val` patched (256.7s/0.521) has a JSON; the `253s/0.535` run has no on-disk patched artifact. + - `DESIGN.md` mtime is **after** the e2e runs yet still carries the older val numbers — i.e. docs were not reconciled to the later runs. +- **Why it matters:** (e) docs should not over-claim. A reviewer cannot tell which baseline/score is canonical; the most-cited figures are the most flattering (slower baseline / higher reduction). +- **Fix:** pick **one** canonical config per task (the production `final_clips`: stability 24, speedup 16), run it once paired, and report that single number with its CI across `DESIGN.md` / `PR_SUMMARY.md` / `CALIBRATION_FINDINGS.md`. Restate stability as the artifact-backed range **3.6–7.2% (≈5% typical)**, not a bare 7.2. + +### P3 — Speedup timer is dead code; score is full-process wall-clock, and the cached baseline is a single un-repeated run `LOW` +- **Evidence:** + - `runner.py:58` sets `NWM_TIME_FILE` and the docstring says "rollout-region timed inside; falls back to total wall," but **nothing writes `gen_seconds.txt`**: `grep -rn 'gen_seconds.txt|NWM_TIME' nano-world-model/src/` returns empty. So `run_rollout` **always** returns `wall = time.time()-t0` (`runner.py:65-76`), timing model load + VAE load + dataset load + the full VAE decode + mp4 save — none of which the allowlisted sampling patch can touch. + - Linear fit of `outputs/csgo_frontier/summary.tsv` (wall vs steps) gives fixed overhead ≈ 36s of an 851s wall (~4.3%); the bf16 patch can only act on the ~95.7% sampling region, so wall-clock timing is **conservative** (under-credits the agent) — not unfair, but it dilutes the measured speedup and injects overhead variance. + - Baseline is a single un-repeated run (no median-of-N, no warmup: `grep` for `repeat|median|trials|per_clip` in `orchestrate.py`/`runner.py` is empty), so the cached value biases every agent's absolute score by the ~3% baseline swing (a shared offset, not per-agent noise). + - `evaluator.py:178` passes `{"all": gen_seconds}` — the advertised per-clip geomean (`scoring.py`) is vestigial (length-1). +- **Why it matters:** (e) misleading internal docstring + dead code; minor dilution of the speedup signal and a ~4-pt shared offset in the headline absolute score. Not gameable (NWM_TIME/gen_seconds are in `FORBIDDEN_TOKENS`). +- **Fix:** either (i) actually implement the per-region timer (write elapsed sampling time to `NWM_TIME_FILE` inside `rollout.py`'s sample loop) so the metric isolates the region the patch can affect, or (ii) drop the dead env-var/fallback and fix the docstring to say "full-process wall-clock." Measure the baseline as median-of-N (or paired in-job per P1). + +### P4 — Stability effect is marginal and single-seed; "reliably beats baseline" overstates a `p=0.025`, one-seed result `LOW` +- **Evidence:** recomputed on `outputs/drift_ref` (22 clips, tail≥60): mean reduction `0.0232`, `SE 0.0096`, **`t=2.410, p=0.0252`**, win `16/22=73%` — exactly matching `DESIGN.md:25`. But it is barely below α=0.05, on **one model seed** (no seed replication in `outputs/`), and window-fragile: over *all* generated frames (4–79) the same clips give only `+2.04%`, `t=1.16`, `p=0.26` (NS) — significance is concentrated in the last 20 frames. (Verifier note: a Wilcoxon signed-rank *confirms* it, `W=60, p=0.032`, and no single-clip drop flips significance, so the effect is real, just marginal.) +- **Why it matters:** (c) marginal statistics. The hidden-set 24-clip final could land NS by chance; "reliably" is stronger than a single-seed α-edge result supports. +- **Fix:** report **multi-seed** (≥3 model/sampling seeds) and the **Wilcoxon** alongside the `t`, and soften "reliably beats baseline" to "beats baseline (paired t=2.41, p=0.025, 73% win, 22 clips; Wilcoxon p=0.03)." Increasing `final_clips` or averaging seeds tightens the CI. + +### P5 — Baseline-cache `>= clips` rule pairs different-sized clip sets across roles `LOW` +- **Evidence:** `orchestrate.py:74` `if data.get('clips',0) >= clips: return data` — a cached 24-clip baseline is served to an 8-clip QUICK eval while patched runs only 8 clips. Per-clip tail-LPIPS std is `0.041` (6.3% rel). *However* clip selection is a deterministic seeded prefix (`rollout.py` iterates a fixed order, `random_seed=42`), so the 8-subset is **nested** in the 24-set; the actual prefix-mean offset (recomputed: first-8 vs all-22 = **−0.19%**, negative → clamps a no-op to 0) is ~1/19 of the effect, not "the same size." Real but small. +- **Why it matters:** (d) cleaner to compare like-for-like; scores not strictly comparable across QUICK (agent) and FINAL roles. +- **Fix:** require the cached baseline clip count to **equal** the requested count and be computed on the identical clip ids (subsumed by the P1 in-job paired-baseline fix). + +### P6 — Scoring/labeling hygiene: context-included LPIPS tables and the log2 cap `LOW` +- **Evidence:** + - Headline CSGO LPIPS (`0.517/0.543/0.579/0.676`, `DESIGN.md:23-27`, header "LPIPS-vs-GT") are **all-rows** means including the 48 near-zero context frames (LPIPS ≈ 0.006, 8% of rows); generated-only means are ~0.045 higher (`0.561/0.590/0.629/0.735`). **Scoring-neutral** (the relative guardrail cancels the constant context block — verified the rel-change differs by 0.004pp) and the **judge uses the same all-rows convention** (`runner.py:88` `mean()` over all rows), so it is a labeling nit, not a metric defect. + - `score_from_speedup = clip(100*log2(s),0,100)` caps the **primary** score at 2× (`scoring.py:33-36`). The "true 4× is indistinguishable from 2×" concern is rebutted: `scoring.py:66`/`evaluator.py:194` also return `score_unbounded` (uncapped) and `metrics` carry raw `geomean_speedup` — the repo-wide Frontier-CS 2.0 convention, matching `bboplace`. +- **Why it matters:** (e) minor; a reader calibrating the 3% tolerance against `0.517` is reasoning about a diluted absolute number. +- **Fix:** add a one-line note to the LPIPS tables: "(all frames incl. 4 context frames; same convention the judge scores on)." No scoring change needed. + +--- + +## Standing Conclusions (claims that SURVIVED scrutiny) + +| Claim | Verdict | Recomputed support | +|---|---|---| +| **C1 — CSGO has a real step↔quality frontier** | **SURVIVES** | Gen-only LPIPS strictly orders at the named corners: seq@2 `0.7345` > seq@5 `0.6285` > seq@20 `0.5901` > seq@50 `0.5612`; +30.9% seq2-vs-seq50. (Caveat: seq@10≈seq@20 flat-spot, `0.5903` vs `0.5901` — disclose, but the {2,5,20,50} corners are monotone.) | +| **C2 — speedup reference ~1.15–1.17× at iso-quality, fair baseline** | **SURVIVES** (modest, as intended) | `speedup_val` 299.95s→256.73s = **1.168×**, score 22.45; patched LPIPS `0.5206` < baseline `0.5479` (quality-neutral, guardrail not tripped). Baseline is symmetric (one rollout cmd for both arms, `runner.py:59-64`); the native `--use_fp16` flag is walled off by the denylist (`evaluator.py` denies `src/models/**`, `src/sample/rollout.py`), so fp32 is the genuine in-scope default — **not** a strawman. | +| **C3 — stability reference reliably beats baseline (modest headroom)** | **SURVIVES** (magnitude is noisy; see P2/P4) | Reference > baseline in **every** run: drift_ref 22-clip `+3.56%` (t=2.41, p=0.025, 73% win), stability_val 16-clip `+5.54%`, e2e 24-clip `+7.2%` — all positive, iso-wall-clock. Baseline pinned at the model's own default stab=0.02 (`default.yaml:58`) is fair. The "non-viable / worst-point baseline / 0.00 better" attacks were **refuted**: on the shipped 80-frame config the lever is monotonic (0.02→0.10→0.20 decreasing) and a 0.00 revert *increases* drift (scores 0). | +| **C4 — RT-1 / Rope correctly rejected as domains** | **SURVIVES** | RT-1 frontier is U-shaped (min at seq@5 under both conventions, never monotone) → speedup trivial, correctly rejected. Rope rejection rests on a documented **data-block** (RAW per-episode h5, 21 frames/episode) + the paper's own Table 7 LPIPS `0.056`; the empty `rope_frontier/` stub is a cosmetic leftover, not the cited evidence. | +| **C5 — patch policy blocks gaming** | **PARTIALLY SURVIVES** | The denylist + `FORBIDDEN_TOKENS` (`evaluator.py:51-57`) and the LPIPS-vs-GT guardrail block the *semantic* attacks tested: config-conditioned step-cutting can't earn quality_mult=1.0 (seq@20 is +5.2% > 3% tol), and a disk-cache attack can't fabricate iso-quality speedup (different clips each call; repo `rmtree`'d each run). The **residual** gap is P1 (unseeded single-replicate guardrail), not benchmark-detection. | +| **C6 — no train/test leakage** | **SURVIVES (clean)** | All 22 `data/csgo_subset/val_files.txt` ∈ `csgo_splits/test_split.txt` (22/22), **0** ∈ `train_split.txt`; train/test disjoint; val ids not git-tracked. (The C6 brief mis-names `gen_episodes.py`, a PushT utility, as provenance — but the real guarantee is the verified test_split membership, and the *substantive* artifact `settings.py VAL_FILES/VAL_STARTS` is correct.) | + +--- + +## Recommended Fixes (highest-leverage first) + +1. **Seed the rollout + measure baseline paired in-job (P1).** The single highest-leverage change: `torch.manual_seed(clip_id)` before the initial `th.randn`, and replace the cached baseline with a baseline computed on the same clips/seed in the same job (common random numbers). Removes the gameability (no-op → ~0) and shrinks the metric noise that contaminates both references' margins. +2. **Reconcile the docs to one canonical number per task (P2).** Run the production `final_clips` config once (stability 24, speedup 16), report that single value + CI everywhere; restate stability as 3.6–7.2% (≈5% typical) and pick one speedup baseline. +3. **Fix the timer + baseline measurement (P3).** Either implement the per-region sampling timer (so the metric isolates the patchable region) or delete the dead `NWM_TIME_FILE` path and correct the docstring; measure the baseline as median-of-N (or paired per #1). +4. **Report multi-seed + Wilcoxon for stability and soften "reliably" (P4).** ≥3 seeds, Wilcoxon alongside the t-test; word the claim to the α-edge it actually is. +5. **Require `== clips` on the cached baseline and label the LPIPS tables (P5, P6).** Equal nested clip sets; one-line "(all frames, judge convention)" note on the LPIPS-vs-GT tables. + +--- + +## Methodology & limits of this audit + +- **Inputs:** 5 independent auditor findings, each with an adversarial verifier verdict. I used **only** findings with `verdict.real==true` and merged duplicates (the unseeded-baseline root cause appeared in ~6 findings; the 7.2/5.54/3.56 spread in ~5). +- **Independent recomputation:** every load-bearing number above was re-derived from the raw artifacts with the project's own `envs/nanowm/bin/python` + scipy, using the harness's exact tail definition (`groupby(frame_idx).mean()`, mean over `frame_idx>=60`, `is_context` excluded). Confirmed: drift_ref 22-clip (3.56%, t=2.41, p=0.025, 16/22), speedup_val 1.168×/22.45, the 2.95% baseline swing → 4.2-pt score swing, the e2e stability log (score 7.204, patched tail 0.6068), CSGO frontier monotonicity, and C6 disjointness (22/22 test, 0 train). +- **What this audit did NOT do:** run the Modal-GPU judge backend end-to-end (validated the local path only); execute new multi-seed rollouts (the single-seed limitation is inferred from the absence of seed-named artifacts and the documented unseeded noise floor); audit tasks other than the two NanoWM rollout tasks. +- **Severity calibration:** applied the benchmark bar — a modest reference and a small speedup are *acceptable*, so several "high"-filed findings (strawman baseline, non-viable task, fabricated 7.2) were correctly **refuted/downgraded** by the verifiers and are excluded here. The one surviving HIGH (P1) is the only item touching gameability + reliable-beat; everything else is doc/robustness hygiene. + +--- + +## Fixes applied + validated (2026-06-14) + +All confirmed problems addressed and re-validated on Della H100 +(`calib/audit_fix.sbatch`, job 9647587; analysis `calib/audit_fix_analyze.py`; raw in +`outputs/audit_fix/`). Canonical numbers also recorded in `docs/CALIBRATION_FINDINGS.md`. + +| # | Fix | Validation result | +|---|---|---| +| **P1** `HIGH` | Deterministic clip-keyed seeding in `rollout.py` (`manual_seed(seed+clip_idx)` per batch, common random numbers); `*_eval` baseline cache keyed by `(== clips, seed)`. | Baseline vs baseline_rep vs **no-op**: per-clip LPIPS Δ = **0.00** → no-op scores **0.15 (speedup) / 0.000 (drift)** ≈ 0 (ungameable). The "impossible" −4.97% bf16 LPIPS improvement is gone → **+1.7%** (bf16 slightly worse, physically correct). | +| **P3** `LOW` | Implemented the real per-region sampling timer (writes `NWM_TIME_FILE`); docstring now accurate. | Speedup is sampling-region only (baseline 1102.1s); residual wall noise **0.24%** (was ~3%). | +| **P4** `LOW` | Multi-seed (42/43/44) + Wilcoxon; softened "reliably" to the actual stats in DESIGN/PR. | Drift reduction **6.8% ± 1.2%** across 3 seeds; pooled **paired t=5.15, p<1e-4, Wilcoxon p<1e-4**, 74% win — no longer marginal (was single-seed t=2.41/p=0.025). | +| **P2** `LOW` | Reconciled to ONE canonical value per task + CI across DESIGN×2 / PR_SUMMARY / CALIBRATION / PR body. | speedup **1.17× / score 22.3**; stability **6.8% / score 6.8**. | +| **P5** `LOW` | Baseline cache now requires `== clips` (+ seed); clip-aligned seeding makes QUICK a noise-identical prefix of FINAL. | Subsumed by P1 determinism. | +| **P6** `LOW` | Labeled the LPIPS tables "(all frames incl. context; judge convention)". | No scoring change (as the audit noted). | + +The fix lives in judge infrastructure (`rollout.py` + `*_eval/`, regenerated into +`infra_patches/0001-rollout-judge-infra.patch`) — **outside the agent's editable +sampling scope**, and the shared 2.0 adapter/template is untouched. Standing +conclusions C1–C6 are unaffected (and C3/C5's residual P1 caveat is now closed). diff --git a/2.0/problems/nanowm_rollout_stability/AUDIT_REPORT.md b/2.0/problems/nanowm_rollout_stability/AUDIT_REPORT.md new file mode 100644 index 00000000..887f7498 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/AUDIT_REPORT.md @@ -0,0 +1,123 @@ +# NanoWM Frontier-CS 2.0 — Audit Report + +**Scope:** `nanowm_rollout_speedup` and `nanowm_rollout_stability` benchmark tasks. +**Date:** 2026-06-13. **Audit lead synthesis** over 5 independent auditors + adversarial verifiers. +**Bar applied:** these are benchmark tasks; a *modest* reference is fine (headroom for agents). Real risks are (a) unfair/strawman baseline, (b) reference that does not reliably beat baseline, (c) marginal statistics sold as robust, (d) gameable/leaky metric, (e) over-claim in docs/PR. + +This report uses **only findings whose verifier verdict was `real==true`**; refuted findings are excluded. Overlapping findings (many auditors hit the same root causes — unseeded sampling, cached baseline, the 7.2/5.54/3.56 doc spread) are de-duplicated into single problems. All numbers below were independently recomputed with the project's own env (`envs/nanowm/bin/python`, scipy) using the harness's exact tail definition (`groupby(frame_idx).mean()` then mean over `frame_idx>=60`, `is_context` excluded). + +--- + +## Summary + +Both tasks rest on a **sound domain choice and clean leakage controls** (C1, C6 survive). The reference solutions *do* beat baseline in every run measured, and the baselines are fair (symmetric harness, the model's own defaults, the in-scope sampling layer is the only lever). The strong claims that this is a strawman baseline, a non-viable task, or a fabricated 7.2% headline were **refuted** on the artifacts. + +The genuine residue is concentrated and mostly **low severity**, but one item is **high**: + +1. **(HIGH) The metric is unseeded and the baseline is cached, not re-measured paired.** Sampling draws unseeded `th.randn` and the harness pairs a *cached* baseline against a *freshly-run* patched arm. Same-config run-to-run tail-LPIPS noise (~2-3%) is the same order as both the stability effect (~3.6%) and the 3% speedup quality guardrail. Consequence: a no-op patch scores nonzero (random sign), and a true iso-quality speedup patch can trip the guardrail on an unlucky draw. This is the one item that touches *gameability* and *"reliably beats baseline."* + +The remaining real items are **documentation/robustness hygiene** (low): three-way numeric inconsistency across docs (stability 3.56 / 5.54 / 7.2; speedup 22.4 / 20.1), a dead per-region timer that silently falls back to full-process wall-clock, a single-seed marginal `t=2.41 (p=0.025)` sold as "reliably," the baseline-cache `>= clips` rule, and absolute-LPIPS tables that include context frames without saying so. None of these, alone, makes a task ill-posed — but the HIGH item plus the doc spread together mean the *magnitude* of the reference margin is overstated and noise-contaminated. + +**Bottom line:** ship-able after fixing the seeding/baseline-pairing (1 high) and reconciling the doc numbers. The tasks are well-posed and the references are real; the scoring is just noisier and the docs more optimistic than advertised. + +--- + +## Confirmed Problems (severity-ranked) + +### P1 — Unseeded sampling + cached-not-paired baseline makes the metric noisy and gameable `HIGH` +- **Evidence:** + - No seed anywhere on the eval path: `grep -rnE 'manual_seed|seed_everything|Generator\(' ` over `nano-world-model/src/sample/rollout.py`, `src/diffusion/df_sample.py`, `src/diffusion/gaussian_diffusion.py`, and both `*_eval/` dirs returns nothing. Initial latent is unseeded `th.randn(*shape, device=device)` (`gaussian_diffusion.py:808`, also `:522,:690`); history-stab `q_sample` noise (`df_sample.py:296`) is unseeded. + - Baseline is cached and reused, patched runs fresh: `speedup_eval/orchestrate.py:69-91` `_baseline()` reads `S.BASELINE_CACHE` and only reruns if missing/insufficient; `run_pair()` returns `(cached_baseline, fresh_patched)`. Identical pattern in `stability_eval/orchestrate.py`. + - Same-config noise floor (independent unpatched stab=0.02 runs): tail-LPIPS spread `0.6402 / 0.6518 / 0.6582 / 0.6539` = **0.018 abs / 2.8% rel**. Speedup baseline LPIPS `0.5479` (speedup_val) vs `0.5363` (e2e) = **2.1%** pure run-to-run noise on the same 4 clips. + - **Smoking gun:** the bf16-autocast reference (a pure precision change, `reference.patch:10`) shows patched LPIPS `0.5206` vs baseline `0.5479` = a **4.97% LPIPS *improvement*** — physically impossible for lowering precision; it is run-to-run noise. + - Gameability: stability score `= 100*(base_tail - patched_tail)/base_tail` (`scoring.py`); a no-op allowlisted whitespace edit re-runs fresh against the cached baseline and scores ~2-3 pts of random sign. Speedup guardrail tolerance is `0.03` (`settings.py`), within ~1.5× the observed ~2% baseline noise, so an iso-quality patch can be zeroed on an unlucky draw. +- **Why it matters:** hits (d) non-gameable metric and (b) "reliably beats baseline." Scores carry an unacknowledged noise/offset budget; marginal effects are confounded with RNG. +- **Fix:** seed the rollout deterministically keyed to clip id (`torch.manual_seed(hash(clip_id))` before the initial `th.randn`), **and** recompute the baseline on the exact same clips/seed *in the same job* (common random numbers) instead of serving a cached baseline. This is the standard paired/CRN remedy and shrinks the metric noise toward zero for the baseline-vs-patched delta. + +### P2 — Mutually inconsistent baselines and scores across docs; no single canonical number `LOW` +- **Evidence:** + - Stability reference reported as **three** values for the same intervention: `3.56%` (drift_ref, 22 clips, t=2.41 — reproduced), `5.54%` (stability_val, 16 clips — `DESIGN.md:48-50`, `PR_SUMMARY.md:25`), and `7.2%` / score 7.2 (`CALIBRATION_FINDINGS.md:274`, 24 clips). The `7.2` appears in exactly **one** place (a calibration log) and **is** backed by a real judge run (`calib/logs/eval_e2e_stab_9617933.log`: `score 7.204`, patched tail `0.6068`) — so it is not fabricated, but it is unreconciled and it is the high end of a noisy range. + - Speedup reference reported as `1.17×/22.4` (`DESIGN.md:78-79`, `PR_SUMMARY.md:14`, `CALIBRATION_FINDINGS.md:253`) vs `1.15×/20.1` (`CALIBRATION_FINDINGS.md:273`) — **two distinct 4-clip runs with different baselines** (299.95s vs 291.36s = 2.95% swing) quoted interchangeably. Cross-pairing the two baselines moves the score **4.2 pts** (18.3 ↔ 22.4), and only the `speedup_val` patched (256.7s/0.521) has a JSON; the `253s/0.535` run has no on-disk patched artifact. + - `DESIGN.md` mtime is **after** the e2e runs yet still carries the older val numbers — i.e. docs were not reconciled to the later runs. +- **Why it matters:** (e) docs should not over-claim. A reviewer cannot tell which baseline/score is canonical; the most-cited figures are the most flattering (slower baseline / higher reduction). +- **Fix:** pick **one** canonical config per task (the production `final_clips`: stability 24, speedup 16), run it once paired, and report that single number with its CI across `DESIGN.md` / `PR_SUMMARY.md` / `CALIBRATION_FINDINGS.md`. Restate stability as the artifact-backed range **3.6–7.2% (≈5% typical)**, not a bare 7.2. + +### P3 — Speedup timer is dead code; score is full-process wall-clock, and the cached baseline is a single un-repeated run `LOW` +- **Evidence:** + - `runner.py:58` sets `NWM_TIME_FILE` and the docstring says "rollout-region timed inside; falls back to total wall," but **nothing writes `gen_seconds.txt`**: `grep -rn 'gen_seconds.txt|NWM_TIME' nano-world-model/src/` returns empty. So `run_rollout` **always** returns `wall = time.time()-t0` (`runner.py:65-76`), timing model load + VAE load + dataset load + the full VAE decode + mp4 save — none of which the allowlisted sampling patch can touch. + - Linear fit of `outputs/csgo_frontier/summary.tsv` (wall vs steps) gives fixed overhead ≈ 36s of an 851s wall (~4.3%); the bf16 patch can only act on the ~95.7% sampling region, so wall-clock timing is **conservative** (under-credits the agent) — not unfair, but it dilutes the measured speedup and injects overhead variance. + - Baseline is a single un-repeated run (no median-of-N, no warmup: `grep` for `repeat|median|trials|per_clip` in `orchestrate.py`/`runner.py` is empty), so the cached value biases every agent's absolute score by the ~3% baseline swing (a shared offset, not per-agent noise). + - `evaluator.py:178` passes `{"all": gen_seconds}` — the advertised per-clip geomean (`scoring.py`) is vestigial (length-1). +- **Why it matters:** (e) misleading internal docstring + dead code; minor dilution of the speedup signal and a ~4-pt shared offset in the headline absolute score. Not gameable (NWM_TIME/gen_seconds are in `FORBIDDEN_TOKENS`). +- **Fix:** either (i) actually implement the per-region timer (write elapsed sampling time to `NWM_TIME_FILE` inside `rollout.py`'s sample loop) so the metric isolates the region the patch can affect, or (ii) drop the dead env-var/fallback and fix the docstring to say "full-process wall-clock." Measure the baseline as median-of-N (or paired in-job per P1). + +### P4 — Stability effect is marginal and single-seed; "reliably beats baseline" overstates a `p=0.025`, one-seed result `LOW` +- **Evidence:** recomputed on `outputs/drift_ref` (22 clips, tail≥60): mean reduction `0.0232`, `SE 0.0096`, **`t=2.410, p=0.0252`**, win `16/22=73%` — exactly matching `DESIGN.md:25`. But it is barely below α=0.05, on **one model seed** (no seed replication in `outputs/`), and window-fragile: over *all* generated frames (4–79) the same clips give only `+2.04%`, `t=1.16`, `p=0.26` (NS) — significance is concentrated in the last 20 frames. (Verifier note: a Wilcoxon signed-rank *confirms* it, `W=60, p=0.032`, and no single-clip drop flips significance, so the effect is real, just marginal.) +- **Why it matters:** (c) marginal statistics. The hidden-set 24-clip final could land NS by chance; "reliably" is stronger than a single-seed α-edge result supports. +- **Fix:** report **multi-seed** (≥3 model/sampling seeds) and the **Wilcoxon** alongside the `t`, and soften "reliably beats baseline" to "beats baseline (paired t=2.41, p=0.025, 73% win, 22 clips; Wilcoxon p=0.03)." Increasing `final_clips` or averaging seeds tightens the CI. + +### P5 — Baseline-cache `>= clips` rule pairs different-sized clip sets across roles `LOW` +- **Evidence:** `orchestrate.py:74` `if data.get('clips',0) >= clips: return data` — a cached 24-clip baseline is served to an 8-clip QUICK eval while patched runs only 8 clips. Per-clip tail-LPIPS std is `0.041` (6.3% rel). *However* clip selection is a deterministic seeded prefix (`rollout.py` iterates a fixed order, `random_seed=42`), so the 8-subset is **nested** in the 24-set; the actual prefix-mean offset (recomputed: first-8 vs all-22 = **−0.19%**, negative → clamps a no-op to 0) is ~1/19 of the effect, not "the same size." Real but small. +- **Why it matters:** (d) cleaner to compare like-for-like; scores not strictly comparable across QUICK (agent) and FINAL roles. +- **Fix:** require the cached baseline clip count to **equal** the requested count and be computed on the identical clip ids (subsumed by the P1 in-job paired-baseline fix). + +### P6 — Scoring/labeling hygiene: context-included LPIPS tables and the log2 cap `LOW` +- **Evidence:** + - Headline CSGO LPIPS (`0.517/0.543/0.579/0.676`, `DESIGN.md:23-27`, header "LPIPS-vs-GT") are **all-rows** means including the 48 near-zero context frames (LPIPS ≈ 0.006, 8% of rows); generated-only means are ~0.045 higher (`0.561/0.590/0.629/0.735`). **Scoring-neutral** (the relative guardrail cancels the constant context block — verified the rel-change differs by 0.004pp) and the **judge uses the same all-rows convention** (`runner.py:88` `mean()` over all rows), so it is a labeling nit, not a metric defect. + - `score_from_speedup = clip(100*log2(s),0,100)` caps the **primary** score at 2× (`scoring.py:33-36`). The "true 4× is indistinguishable from 2×" concern is rebutted: `scoring.py:66`/`evaluator.py:194` also return `score_unbounded` (uncapped) and `metrics` carry raw `geomean_speedup` — the repo-wide Frontier-CS 2.0 convention, matching `bboplace`. +- **Why it matters:** (e) minor; a reader calibrating the 3% tolerance against `0.517` is reasoning about a diluted absolute number. +- **Fix:** add a one-line note to the LPIPS tables: "(all frames incl. 4 context frames; same convention the judge scores on)." No scoring change needed. + +--- + +## Standing Conclusions (claims that SURVIVED scrutiny) + +| Claim | Verdict | Recomputed support | +|---|---|---| +| **C1 — CSGO has a real step↔quality frontier** | **SURVIVES** | Gen-only LPIPS strictly orders at the named corners: seq@2 `0.7345` > seq@5 `0.6285` > seq@20 `0.5901` > seq@50 `0.5612`; +30.9% seq2-vs-seq50. (Caveat: seq@10≈seq@20 flat-spot, `0.5903` vs `0.5901` — disclose, but the {2,5,20,50} corners are monotone.) | +| **C2 — speedup reference ~1.15–1.17× at iso-quality, fair baseline** | **SURVIVES** (modest, as intended) | `speedup_val` 299.95s→256.73s = **1.168×**, score 22.45; patched LPIPS `0.5206` < baseline `0.5479` (quality-neutral, guardrail not tripped). Baseline is symmetric (one rollout cmd for both arms, `runner.py:59-64`); the native `--use_fp16` flag is walled off by the denylist (`evaluator.py` denies `src/models/**`, `src/sample/rollout.py`), so fp32 is the genuine in-scope default — **not** a strawman. | +| **C3 — stability reference reliably beats baseline (modest headroom)** | **SURVIVES** (magnitude is noisy; see P2/P4) | Reference > baseline in **every** run: drift_ref 22-clip `+3.56%` (t=2.41, p=0.025, 73% win), stability_val 16-clip `+5.54%`, e2e 24-clip `+7.2%` — all positive, iso-wall-clock. Baseline pinned at the model's own default stab=0.02 (`default.yaml:58`) is fair. The "non-viable / worst-point baseline / 0.00 better" attacks were **refuted**: on the shipped 80-frame config the lever is monotonic (0.02→0.10→0.20 decreasing) and a 0.00 revert *increases* drift (scores 0). | +| **C4 — RT-1 / Rope correctly rejected as domains** | **SURVIVES** | RT-1 frontier is U-shaped (min at seq@5 under both conventions, never monotone) → speedup trivial, correctly rejected. Rope rejection rests on a documented **data-block** (RAW per-episode h5, 21 frames/episode) + the paper's own Table 7 LPIPS `0.056`; the empty `rope_frontier/` stub is a cosmetic leftover, not the cited evidence. | +| **C5 — patch policy blocks gaming** | **PARTIALLY SURVIVES** | The denylist + `FORBIDDEN_TOKENS` (`evaluator.py:51-57`) and the LPIPS-vs-GT guardrail block the *semantic* attacks tested: config-conditioned step-cutting can't earn quality_mult=1.0 (seq@20 is +5.2% > 3% tol), and a disk-cache attack can't fabricate iso-quality speedup (different clips each call; repo `rmtree`'d each run). The **residual** gap is P1 (unseeded single-replicate guardrail), not benchmark-detection. | +| **C6 — no train/test leakage** | **SURVIVES (clean)** | All 22 `data/csgo_subset/val_files.txt` ∈ `csgo_splits/test_split.txt` (22/22), **0** ∈ `train_split.txt`; train/test disjoint; val ids not git-tracked. (The C6 brief mis-names `gen_episodes.py`, a PushT utility, as provenance — but the real guarantee is the verified test_split membership, and the *substantive* artifact `settings.py VAL_FILES/VAL_STARTS` is correct.) | + +--- + +## Recommended Fixes (highest-leverage first) + +1. **Seed the rollout + measure baseline paired in-job (P1).** The single highest-leverage change: `torch.manual_seed(clip_id)` before the initial `th.randn`, and replace the cached baseline with a baseline computed on the same clips/seed in the same job (common random numbers). Removes the gameability (no-op → ~0) and shrinks the metric noise that contaminates both references' margins. +2. **Reconcile the docs to one canonical number per task (P2).** Run the production `final_clips` config once (stability 24, speedup 16), report that single value + CI everywhere; restate stability as 3.6–7.2% (≈5% typical) and pick one speedup baseline. +3. **Fix the timer + baseline measurement (P3).** Either implement the per-region sampling timer (so the metric isolates the patchable region) or delete the dead `NWM_TIME_FILE` path and correct the docstring; measure the baseline as median-of-N (or paired per #1). +4. **Report multi-seed + Wilcoxon for stability and soften "reliably" (P4).** ≥3 seeds, Wilcoxon alongside the t-test; word the claim to the α-edge it actually is. +5. **Require `== clips` on the cached baseline and label the LPIPS tables (P5, P6).** Equal nested clip sets; one-line "(all frames, judge convention)" note on the LPIPS-vs-GT tables. + +--- + +## Methodology & limits of this audit + +- **Inputs:** 5 independent auditor findings, each with an adversarial verifier verdict. I used **only** findings with `verdict.real==true` and merged duplicates (the unseeded-baseline root cause appeared in ~6 findings; the 7.2/5.54/3.56 spread in ~5). +- **Independent recomputation:** every load-bearing number above was re-derived from the raw artifacts with the project's own `envs/nanowm/bin/python` + scipy, using the harness's exact tail definition (`groupby(frame_idx).mean()`, mean over `frame_idx>=60`, `is_context` excluded). Confirmed: drift_ref 22-clip (3.56%, t=2.41, p=0.025, 16/22), speedup_val 1.168×/22.45, the 2.95% baseline swing → 4.2-pt score swing, the e2e stability log (score 7.204, patched tail 0.6068), CSGO frontier monotonicity, and C6 disjointness (22/22 test, 0 train). +- **What this audit did NOT do:** run the Modal-GPU judge backend end-to-end (validated the local path only); execute new multi-seed rollouts (the single-seed limitation is inferred from the absence of seed-named artifacts and the documented unseeded noise floor); audit tasks other than the two NanoWM rollout tasks. +- **Severity calibration:** applied the benchmark bar — a modest reference and a small speedup are *acceptable*, so several "high"-filed findings (strawman baseline, non-viable task, fabricated 7.2) were correctly **refuted/downgraded** by the verifiers and are excluded here. The one surviving HIGH (P1) is the only item touching gameability + reliable-beat; everything else is doc/robustness hygiene. + +--- + +## Fixes applied + validated (2026-06-14) + +All confirmed problems addressed and re-validated on Della H100 +(`calib/audit_fix.sbatch`, job 9647587; analysis `calib/audit_fix_analyze.py`; raw in +`outputs/audit_fix/`). Canonical numbers also recorded in `docs/CALIBRATION_FINDINGS.md`. + +| # | Fix | Validation result | +|---|---|---| +| **P1** `HIGH` | Deterministic clip-keyed seeding in `rollout.py` (`manual_seed(seed+clip_idx)` per batch, common random numbers); `*_eval` baseline cache keyed by `(== clips, seed)`. | Baseline vs baseline_rep vs **no-op**: per-clip LPIPS Δ = **0.00** → no-op scores **0.15 (speedup) / 0.000 (drift)** ≈ 0 (ungameable). The "impossible" −4.97% bf16 LPIPS improvement is gone → **+1.7%** (bf16 slightly worse, physically correct). | +| **P3** `LOW` | Implemented the real per-region sampling timer (writes `NWM_TIME_FILE`); docstring now accurate. | Speedup is sampling-region only (baseline 1102.1s); residual wall noise **0.24%** (was ~3%). | +| **P4** `LOW` | Multi-seed (42/43/44) + Wilcoxon; softened "reliably" to the actual stats in DESIGN/PR. | Drift reduction **6.8% ± 1.2%** across 3 seeds; pooled **paired t=5.15, p<1e-4, Wilcoxon p<1e-4**, 74% win — no longer marginal (was single-seed t=2.41/p=0.025). | +| **P2** `LOW` | Reconciled to ONE canonical value per task + CI across DESIGN×2 / PR_SUMMARY / CALIBRATION / PR body. | speedup **1.17× / score 22.3**; stability **6.8% / score 6.8**. | +| **P5** `LOW` | Baseline cache now requires `== clips` (+ seed); clip-aligned seeding makes QUICK a noise-identical prefix of FINAL. | Subsumed by P1 determinism. | +| **P6** `LOW` | Labeled the LPIPS tables "(all frames incl. context; judge convention)". | No scoring change (as the audit noted). | + +The fix lives in judge infrastructure (`rollout.py` + `*_eval/`, regenerated into +`infra_patches/0001-rollout-judge-infra.patch`) — **outside the agent's editable +sampling scope**, and the shared 2.0 adapter/template is untouched. Standing +conclusions C1–C6 are unaffected (and C3/C5's residual P1 caveat is now closed). From 60da6981ec37da6ddafd539ba11552d1e1a6d26e Mon Sep 17 00:00:00 2001 From: Wenhao Chai Date: Wed, 17 Jun 2026 01:08:48 -0400 Subject: [PATCH 11/12] chore(2.0/nanowm): reconcile residual pre-audit numbers + harden build helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-audit cleanup a maintainer review would flag. No change to validated logic (rollout / scoring / infra-patch / evaluator untouched; canonical results stand: speedup 1.17x/22.3, stability 6.8%/6.8). Doc reconciliation (audit P2 — retire leftover pre-audit single-seed numbers to the canonical CRN values: t=5.15, p<1e-4, 74% win, 6.8% +/- 1.2%): - stability config.yaml + stability_eval/settings.py inline comments: t~2.5/73% win - stability readme "Reference & difficulty": 73% win / t~2.5 -> 74% / t=5.15 / p<1e-4 build_images.sh (both tasks; maintainer-only image-build helper): - drop hardcoded personal scratch path as the NWM_ASSETS default; require it via :? (clear error if unset instead of silently pointing at a nonexistent path) - correct the asset-layout doc to match what the script actually reads (ckpts/nanowm-l2-csgo-100k, csgo/, csgo_subset/ -- not the old data/ paths) - fail-fast if the required checkpoint / held-out data didn't land (was silently swallowed by `|| true`, producing an empty judge image that fails at runtime) - fix stability script header (had copy-pasted "speedup") Co-Authored-By: Claude Opus 4.8 (1M context) --- .../docker/build_images.sh | 17 +++++++++------ .../nanowm_rollout_stability/config.yaml | 3 ++- .../docker/build_images.sh | 21 ++++++++++++------- 2.0/problems/nanowm_rollout_stability/readme | 5 +++-- .../stability_eval/settings.py | 3 ++- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh b/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh index 298b4c98..815b802d 100755 --- a/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh +++ b/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash # Build the nanowm_rollout_speedup agent + judge images. # bash 2.0/problems/nanowm_rollout_speedup/docker/build_images.sh [tag] -# Requires the hidden assets staged under $NWM_ASSETS (default below): -# ckpts/nanowm-l2-csgo/model_state_dict.pt -# data/csgo/1-200/*.hdf5 (held-out CSGO episode subset) -# data/csgo_subset/{val_files.txt,val_starts.npy} -# baseline/baseline_metrics.json (optional; computed on first trial otherwise) +# Requires the hidden assets staged under $NWM_ASSETS (must be set; no default): +# $NWM_ASSETS/ckpts/nanowm-l2-csgo-100k/ (L/2 CSGO checkpoint dir -> baked as ckpts/nanowm-l2-csgo) +# $NWM_ASSETS/csgo/1-200/*.hdf5 (held-out CSGO episode subset -> data/csgo) +# $NWM_ASSETS/csgo_subset/{val_files.txt,val_starts.npy} (-> data/csgo_subset) +# (baseline is optional; computed on first trial otherwise) set -euo pipefail TAG="${1:-experimental-v0}" NANOWM_COMMIT="${NANOWM_COMMIT:-main}" SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PROB=$(cd "$SCRIPT_DIR/.." && pwd) -NWM_ASSETS="${NWM_ASSETS:-/scratch/gpfs/KARTHIKN/wc9403/project/fcs-2/data}" +NWM_ASSETS="${NWM_ASSETS:?set NWM_ASSETS to the dir holding the L/2 CSGO ckpt + held-out CSGO subset (see header)}" CTX=$(mktemp -d); trap 'rm -rf "$CTX"' EXIT mkdir -p "$CTX/task_ctx" @@ -25,6 +25,11 @@ cp -r "$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k" "$CTX/task_ctx/assets/ckpts/nanowm cp -r "$NWM_ASSETS/csgo" "$CTX/task_ctx/assets/data/csgo" 2>/dev/null || true cp -r "$NWM_ASSETS/csgo_subset" "$CTX/task_ctx/assets/data/csgo_subset" 2>/dev/null || true +# The copies above are best-effort (|| true); fail loudly if a REQUIRED asset is +# missing so we never build a judge image with no model / no held-out data. +[ -d "$CTX/task_ctx/assets/ckpts/nanowm-l2-csgo" ] || { echo "ERROR: checkpoint missing — expected \$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k" >&2; exit 1; } +[ -d "$CTX/task_ctx/assets/data/csgo" ] || { echo "ERROR: held-out CSGO data missing — expected \$NWM_ASSETS/csgo" >&2; exit 1; } + docker build --target "" --build-arg NANOWM_COMMIT="$NANOWM_COMMIT" \ -t "frontiercs/nanowm-rollout-speedup-agent:$TAG" -f "$SCRIPT_DIR/agent/Dockerfile" "$CTX" docker build --build-arg NANOWM_COMMIT="$NANOWM_COMMIT" \ diff --git a/2.0/problems/nanowm_rollout_stability/config.yaml b/2.0/problems/nanowm_rollout_stability/config.yaml index 85927251..7640ea84 100644 --- a/2.0/problems/nanowm_rollout_stability/config.yaml +++ b/2.0/problems/nanowm_rollout_stability/config.yaml @@ -38,7 +38,8 @@ evaluation: # else drift is being bought with compute (the speedup task's axis). wallclock_tolerance: 0.10 # Drift reductions are small; enough clips to resolve above per-clip noise - # (validated: stab=0.20 reference beats baseline at 22 clips, t~2.5, 73% win). + # (validated under common-random-numbers pairing: stab=0.20 reference beats + # baseline; 74% per-clip win, pooled paired t=5.15, p<1e-4 across 3 seeds x 22 clips). quick_clips: 8 final_clips: 24 batch_size: 2 diff --git a/2.0/problems/nanowm_rollout_stability/docker/build_images.sh b/2.0/problems/nanowm_rollout_stability/docker/build_images.sh index ec0df373..040ff0aa 100755 --- a/2.0/problems/nanowm_rollout_stability/docker/build_images.sh +++ b/2.0/problems/nanowm_rollout_stability/docker/build_images.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash -# Build the nanowm_rollout_speedup agent + judge images. -# bash 2.0/problems/nanowm_rollout_speedup/docker/build_images.sh [tag] -# Requires the hidden assets staged under $NWM_ASSETS (default below): -# ckpts/nanowm-l2-csgo/model_state_dict.pt -# data/csgo/1-200/*.hdf5 (held-out CSGO episode subset) -# data/csgo_subset/{val_files.txt,val_starts.npy} -# baseline/baseline_metrics.json (optional; computed on first trial otherwise) +# Build the nanowm_rollout_stability agent + judge images. +# bash 2.0/problems/nanowm_rollout_stability/docker/build_images.sh [tag] +# Requires the hidden assets staged under $NWM_ASSETS (must be set; no default): +# $NWM_ASSETS/ckpts/nanowm-l2-csgo-100k/ (L/2 CSGO checkpoint dir -> baked as ckpts/nanowm-l2-csgo) +# $NWM_ASSETS/csgo/1-200/*.hdf5 (held-out CSGO episode subset -> data/csgo) +# $NWM_ASSETS/csgo_subset/{val_files.txt,val_starts.npy} (-> data/csgo_subset) +# (baseline is optional; computed on first trial otherwise) set -euo pipefail TAG="${1:-experimental-v0}" NANOWM_COMMIT="${NANOWM_COMMIT:-main}" SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PROB=$(cd "$SCRIPT_DIR/.." && pwd) -NWM_ASSETS="${NWM_ASSETS:-/scratch/gpfs/KARTHIKN/wc9403/project/fcs-2/data}" +NWM_ASSETS="${NWM_ASSETS:?set NWM_ASSETS to the dir holding the L/2 CSGO ckpt + held-out CSGO subset (see header)}" CTX=$(mktemp -d); trap 'rm -rf "$CTX"' EXIT mkdir -p "$CTX/task_ctx" @@ -25,6 +25,11 @@ cp -r "$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k" "$CTX/task_ctx/assets/ckpts/nanowm cp -r "$NWM_ASSETS/csgo" "$CTX/task_ctx/assets/data/csgo" 2>/dev/null || true cp -r "$NWM_ASSETS/csgo_subset" "$CTX/task_ctx/assets/data/csgo_subset" 2>/dev/null || true +# The copies above are best-effort (|| true); fail loudly if a REQUIRED asset is +# missing so we never build a judge image with no model / no held-out data. +[ -d "$CTX/task_ctx/assets/ckpts/nanowm-l2-csgo" ] || { echo "ERROR: checkpoint missing — expected \$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k" >&2; exit 1; } +[ -d "$CTX/task_ctx/assets/data/csgo" ] || { echo "ERROR: held-out CSGO data missing — expected \$NWM_ASSETS/csgo" >&2; exit 1; } + docker build --target "" --build-arg NANOWM_COMMIT="$NANOWM_COMMIT" \ -t "frontiercs/nanowm-rollout-stability-agent:$TAG" -f "$SCRIPT_DIR/agent/Dockerfile" "$CTX" docker build --build-arg NANOWM_COMMIT="$NANOWM_COMMIT" \ diff --git a/2.0/problems/nanowm_rollout_stability/readme b/2.0/problems/nanowm_rollout_stability/readme index 5b16a64b..719480f1 100644 --- a/2.0/problems/nanowm_rollout_stability/readme +++ b/2.0/problems/nanowm_rollout_stability/readme @@ -50,8 +50,9 @@ score = clip(100 * (baseline_tail_drift - patched_tail_drift) / baseline_tail_dr ## Reference & difficulty `reference.patch` raises history stabilization (a one-line sampling change) — it -reliably reduces tail-drift a few percent over the baseline at iso-compute -(validated: 73% per-clip win, t≈2.5 over 22 clips), proving the task is solvable. +reliably reduces tail-drift ~6.8% (± 1.2%) over the baseline at iso-wall-clock +(validated under common-random-numbers pairing: 74% per-clip win, pooled paired +t=5.15, p<1e-4 across 3 seeds × 22 clips), proving the task is solvable. Substantially beating it is the open challenge. ## Resource budget diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py b/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py index 4ecdef92..a727513e 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py @@ -43,7 +43,8 @@ def _get(name: str, default): # Drift metric: mean LPIPS over the drifted tail (frames >= this index). DRIFT_TAIL_START = int(_get("FRONTIER_NWM_DRIFT_TAIL_START", 60)) # Eval set sizes — drift reductions are small, so enough clips to resolve them -# above per-clip noise (validated: 22 clips gives t~2.5 for the reference). +# above per-clip noise (validated under CRN pairing: 3 seeds x 22 clips give +# pooled paired t=5.15, p<1e-4 for the reference). QUICK_CLIPS = int(_get("FRONTIER_NWM_QUICK_CLIPS", 8)) FINAL_CLIPS = int(_get("FRONTIER_NWM_FINAL_CLIPS", 24)) BATCH_SIZE = int(_get("FRONTIER_NWM_BATCH_SIZE", 2)) From 97e30a8c699b3155387394801005fcdb5e084d7e Mon Sep 17 00:00:00 2001 From: momoway <3499622023@qq.com> Date: Sat, 27 Jun 2026 23:10:01 +0000 Subject: [PATCH 12/12] fix(2.0/nanowm): close two tail-targeting reward-hacks + the agent-side judge-config leaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hardens the paired nanowm_rollout_{speedup,stability} 2.0 tasks against the reward-hacks surfaced by the codex Harbor trials, validated end-to-end on H100. stability #7 (tail-targeting via a hardcoded module counter): - The scored (role=final) rollout horizon is now drawn at random per run from [ROLLOUT_LENGTH_MIN, ROLLOUT_LENGTH_MAX]=[64,72] (< the agent-measurable nominal 80), threaded once to both CRN arms and every fan-out chunk; the scored tail = horizon - TAIL_FRAMES. CRN pairing + chunk bit-identity preserved; agent-role QUICK + cache stay at nominal. A submission that hardcodes the horizon (codex's %76 / frame-60 ramp) now misfires off the tail. Proof: stability_eval/ test_antihack_horizon.py (no GPU) — expected targeting collapses +0.064 -> ~0. speedup temp_embed gray area + model-mutation: - frozen-model guard, actively injected into the copied rollout.py at apply-time ({speedup,stability}_eval.runner.inject_frozen_guard): a restored transient reshape (causal-prefix) passes, a persistently mutated model hard-errors -> 0. Fail-closed if the rollout.py anchors move. Unit-tested without torch. - speedup faithfulness fail-closed: role==final + faithfulness_lpips is None -> 0, not a silent faithfulness_mult=1.0 free pass (the primary model-mutation defense). agent-side judge-config LEAKS (how codex read 80/60 in the first place): - adapter agent template Dockerfile no longer copies config.yaml/task_config.json (full evaluation block: band, seed, drift_tail_start, val paths) into /app. - {speedup,stability}/docker/agent/Dockerfile no longer bakes the eval package (task_pkg/*_eval with settings.py) into /app/task. The judge keeps its own /judge/task_config.json + /opt/nanowm/task. De-leaked the agent readme/config environment string (no exact horizon/tail). Also: graduated wallclock-multiplier comment corrected; #6 (task collapses to the history_stabilization scalar) documented as a depth limitation (H100-sweep-gated), not a reward-hack. AUDIT_REPORT/DESIGN updated in both tasks. Validated on Modal H100 (stability, regenerated package + rebuilt images): scored at rollout_length=65 / tail>=45 (random, not 80/60), agent could not read the band, frozen guard clean, reward 0.0338 (22 clips, iso-wall-clock). Co-Authored-By: Claude Opus 4.8 --- .../nanowm_rollout_speedup/AUDIT_REPORT.md | 36 ++++ 2.0/problems/nanowm_rollout_speedup/DESIGN.md | 89 ++++++++- .../nanowm_rollout_speedup/config.yaml | 19 +- .../docker/agent/Dockerfile | 9 +- .../docker/build_images.sh | 28 ++- .../docker/judge/Dockerfile | 5 +- .../docker/prep_assets.py | 149 +++++++++++++++ .../nanowm_rollout_speedup/evaluator.py | 70 ++++++- .../harbor/app/README.md | 9 +- .../0001-rollout-judge-infra.patch | 78 +++++++- .../speedup_eval/frozen_model_guard.py | 87 +++++++++ .../speedup_eval/modal_app.py | 112 +++++++++-- .../speedup_eval/orchestrate.py | 75 +++++++- .../speedup_eval/runner.py | 135 +++++++++++++- .../speedup_eval/scoring.py | 36 +++- .../speedup_eval/settings.py | 43 +++++ .../speedup_eval/test_frozen_model_guard.py | 128 +++++++++++++ .../speedup_eval/test_inject_frozen_guard.py | 132 +++++++++++++ .../nanowm_rollout_stability/AUDIT_REPORT.md | 142 +++++++++++++- .../nanowm_rollout_stability/DESIGN.md | 54 +++++- .../nanowm_rollout_stability/config.yaml | 44 ++++- .../docker/agent/Dockerfile | 10 +- .../docker/build_images.sh | 29 ++- .../docker/judge/Dockerfile | 7 +- .../docker/prep_assets.py | 149 +++++++++++++++ .../nanowm_rollout_stability/evaluator.py | 68 ++++++- .../harbor/app/README.md | 33 +++- .../0001-rollout-judge-infra.patch | 131 +++++++++++-- 2.0/problems/nanowm_rollout_stability/readme | 33 ++-- .../stability_eval/__init__.py | 2 +- .../stability_eval/modal_app.py | 175 ++++++++++++++++-- .../stability_eval/orchestrate.py | 95 ++++++++-- .../stability_eval/runner.py | 168 ++++++++++++++--- .../stability_eval/scoring.py | 44 ++++- .../stability_eval/settings.py | 109 ++++++++++- .../stability_eval/test_antihack_horizon.py | 154 +++++++++++++++ .../test_inject_frozen_guard.py | 132 +++++++++++++ .../task-template/environment/Dockerfile | 9 +- 38 files changed, 2638 insertions(+), 190 deletions(-) create mode 100644 2.0/problems/nanowm_rollout_speedup/docker/prep_assets.py create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/frozen_model_guard.py create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/test_frozen_model_guard.py create mode 100644 2.0/problems/nanowm_rollout_speedup/speedup_eval/test_inject_frozen_guard.py create mode 100644 2.0/problems/nanowm_rollout_stability/docker/prep_assets.py create mode 100644 2.0/problems/nanowm_rollout_stability/stability_eval/test_antihack_horizon.py create mode 100644 2.0/problems/nanowm_rollout_stability/stability_eval/test_inject_frozen_guard.py diff --git a/2.0/problems/nanowm_rollout_speedup/AUDIT_REPORT.md b/2.0/problems/nanowm_rollout_speedup/AUDIT_REPORT.md index 887f7498..3e375785 100644 --- a/2.0/problems/nanowm_rollout_speedup/AUDIT_REPORT.md +++ b/2.0/problems/nanowm_rollout_speedup/AUDIT_REPORT.md @@ -121,3 +121,39 @@ The fix lives in judge infrastructure (`rollout.py` + `*_eval/`, regenerated int `infra_patches/0001-rollout-judge-infra.patch`) — **outside the agent's editable sampling scope**, and the shared 2.0 adapter/template is untouched. Standing conclusions C1–C6 are unaffected (and C3/C5's residual P1 caveat is now closed). + +--- + +## Update — P1 actually closed; H100; reviewer-honesty note (2026-06-21, branch `fix/nanowm-h100-crn`) + +A re-review found the 2026-06-14 "P1 fixed" claim above was only **half** true: +deterministic per-clip seeding was added, but `orchestrate._baseline()` still +**served a cached baseline** rather than recomputing it paired in-job, and the +infra patch enforced **no GPU determinism** (`use_deterministic_algorithms`, +TF32-off, `CUBLAS_WORKSPACE_CONFIG` were all absent — and upstream `rollout.py` +turns TF32 *on* at import). Seeding fixes the initial noise but not the +arithmetic, so "the cached baseline is a valid CRN partner" only held on the +H100 single node where determinism happened to apply — not the (then L40S, never +run e2e) production path. P1 was therefore not closed for the scored path. + +Now actually closed: +- **Determinism enforced** in the infra patch (`use_deterministic_algorithms(True, + warn_only=True)`, `cudnn.deterministic`, `benchmark=False`, TF32 disabled) + + `CUBLAS_WORKSPACE_CONFIG=:4096:8` exported by the launcher before CUDA init. +- **Scored (final) run measures baseline + patched as a true CRN pair** in one + job/GPU/process (`run_pair(role="final")` → `modal_app.run_pair_remote`), no + cache. Cached baseline kept only for cheap iterative feedback, now keyed by a + full **config fingerprint**. +- **GPU pinned to H100** (was L40S) so the production scoring path and every + calibrated number in this report share one SKU. + +**Reviewer-honesty note (closes audit risk (e), over-claim):** the calibration +artifacts this report cites as evidence — `CALIBRATION_FINDINGS.md`, `outputs/`, +`calib/…sbatch`, job numbers, `PR_SUMMARY.md` — **are not shipped in this repo** +(they live in the author's working tree), so the pre-fix numbers above are +**not independently reproducible from the PR** and should be read as provisional. +The code-level fixes here are CPU-verified (policy/scoring/smoke; no-op CRN pair → +exactly 0.0; reference reproduces 1.17×/22.3 and ~6.8); **GPU/Modal end-to-end on +H100 is still pending maintainer credentials** and must re-establish the no-op≈0 +and residual-noise numbers on the production path before the headline margins are +final. diff --git a/2.0/problems/nanowm_rollout_speedup/DESIGN.md b/2.0/problems/nanowm_rollout_speedup/DESIGN.md index 7d539ff9..a2a3ff64 100644 --- a/2.0/problems/nanowm_rollout_speedup/DESIGN.md +++ b/2.0/problems/nanowm_rollout_speedup/DESIGN.md @@ -38,8 +38,10 @@ frontier; CSGO was chosen precisely because the frontier is real.) Python-only (`.py`/`.pyi`), ≤256 KB, no file deletion, safe paths. -- **Allowed:** `src/diffusion/**.py`, `src/sample/sampling_utils.py` (the sampling - layer: scheduling matrices, the DDIM/diffusion-forcing loop, solvers, caches). +- **Allowed:** `src/diffusion/**.py` (the sampling layer: scheduling matrices, the + DDIM/diffusion-forcing loop, solvers, caches). `src/sample/sampling_utils.py` is + DENIED — it decodes+saves the frames the LPIPS guardrail reads (editing it could + blur the scored pixels to mask a quality regression). - **Denied:** model (`src/models/**`), VAE (`src/latent_codecs/**`), metric (`src/sample/evaluate_metrics.py`), harness (`src/sample/rollout.py`), data (`src/wm_datasets/**`), training/eval/utils, native/build/deps. @@ -54,10 +56,13 @@ config and patching the scheduler). ## 4. GPU on Modal (judge stays CPU) -`speedup_eval/modal_app.py` runs one rollout in a Modal GPU function from baked -assets (NanoWM checkout + L/2 CSGO ckpt + held-out CSGO episode subset); -`orchestrate.run_pair` computes the cached vanilla baseline once and the patched -run per submission. A `local` backend (`orchestrate._run_local`) runs the same +`speedup_eval/modal_app.py` runs rollouts in a Modal GPU function from baked +assets (NanoWM checkout + L/2 CSGO ckpt + held-out CSGO episode subset). For the +SCORED (final/verifier) run, `orchestrate.run_pair(role="final")` measures the +vanilla baseline and the patched arm back-to-back on one GPU in one job (no +cache) — see §8. The cached vanilla baseline is used ONLY for the cheap iterative +(agent-role) feedback path, keyed by `settings.config_fingerprint()`. A `local` +backend (`orchestrate._run_local` / `_run_pair_local`) runs the same `speedup_eval.runner` on a directly-visible GPU — this is the path validated on Della H100. End-to-end Modal execution awaits maintainer Modal credentials. @@ -95,3 +100,75 @@ end-to-end Modal execution (maintainer credentials). baseline metrics); the held-out episode ids are the only hidden component. - CI smoke (`FRONTIER_NWM_SMOKE=1`, CPU) validates the patch policy + empty reference; confirm this matches the repo's CI expectation for GPU/Modal tasks. + +## 8. Update — H100 + CRN hardening (this branch) + +Changes since the audit, all in judge infra (`infra_patches/0001-…`, `speedup_eval/`) +— the agent-editable sampling scope and the shared 2.0 template are untouched: + +- **GPU is now H100, not L40S** (`config.yaml`, `modal_app.py`). The reference, + noise floor, and OOM-decode tuning were all calibrated on H100, so the + production scoring path and the validated numbers now share one GPU SKU + (closes the hardware-mismatch risk). +- **Determinism is enforced, not assumed.** The infra patch now sets + `torch.use_deterministic_algorithms(True, warn_only=True)`, + `cudnn.deterministic=True`, `benchmark=False`, and **disables TF32** (the + upstream `rollout.py` enabled it at import — a real run-to-run noise source the + earlier patch left on); the launcher exports `CUBLAS_WORKSPACE_CONFIG=:4096:8` + before CUDA init. Per-clip seeding alone (the audit's P1 partial fix) fixes the + initial noise but **not** the arithmetic; this makes the kernels bit-stable so a + cached/separate baseline is actually a valid common-random-numbers partner. +- **The scored (final) run measures baseline + patched as a true CRN pair** in + one job/GPU/process (`orchestrate.run_pair(role="final")` → + `modal_app.run_pair_remote`), no cache — airtight even if residual + nondeterminism survives. The cheap cached-baseline path is kept only for the + iterative (agent) role, and its cache key is now a full **config fingerprint** + (model/dataset/rollout/steps/scheduling/stab/batch/seed/val-set), so a config + change invalidates rather than mispairs. + +**Validation status:** policy/scoring/smoke re-verified on CPU (no-op CRN pair → +exactly 0.0; reference reproduces 1.17×/22.3; hardened token list still accepts +the reference and `model.eval()`/`torch.compile()` while rejecting `/opt/nanowm`, +`*.hdf5`, and `os.getenv` peeks). **GPU/Modal end-to-end on H100 is still pending +maintainer credentials** — the determinism + paired-baseline numbers (no-op ≈ 0, +residual noise) must be re-measured on the production H100 path before the +headline margins are treated as final. + +## 9. Update — frozen-model guard for the causal-prefix `temp_embed` gray area + +The reference-class causal-prefix optimization (codex's 2.87×, reward 1.0) is a +LEGITIMATE, FP-exact inference optimization: NanoWM is causal (`causal: true`, +frame t attends only to ≤ t) and the sequential diffusion-forcing schedule denoises +one frame at a time, so the baseline harness wastefully runs the model on the full +window every DDIM step while only a short causal *prefix* is active. Cropping to that +prefix cannot change the kept frames' outputs (verified bit-exact: SDPA causal crop +diff 0.0, full temporal-path replication 3e-8 ≈ fp32 ε) — patched LPIPS 0.5234 vs +baseline 0.5235. It edits only `src/diffusion/gaussian_diffusion.py`, trips no +forbidden token, and never touches the metric or scored pixels. + +The one wrinkle: to run the unmodified forward on the cropped window it **temporarily +reshapes `module.temp_embed`** (a frozen-tree `src/models/**` nn.Parameter) from the +allowed sampler, then RESTORES it via try/finally. That is exact and reversible, but +the static policy (path allowlist + token scan) cannot see a runtime attribute +monkeypatch, so nothing structurally stops a *different* patch from **persistently** +mutating the frozen model (e.g. swapping in blurred/distilled weights to win +wall-clock). Hardening (`speedup_eval/frozen_model_guard.py`, unit-tested in +`speedup_eval/test_frozen_model_guard.py`): a cheap dynamic invariant — fingerprint +the model params after load, re-check after the rollout; a restored transient reshape +(causal-prefix) passes, a left-mutated model hard-errors (→ submission scored 0). The +residual (a transient swap restored before the check) stays backed by the faithfulness +check below. **Wiring is ACTIVE, not pending:** `speedup_eval.runner.inject_frozen_guard()` +inlines the guard into the copied `rollout.py` at apply-time (after the agent patch; +rollout.py is denied to the agent), so it runs on every rollout in both backends — +fail-closed if the anchors move, and unit-tested (`test_frozen_model_guard.py`, +`test_inject_frozen_guard.py`). The complete alternative (make the model natively slice +`temp_embed`/temporal-rope to the active window so causal-prefix needs NO model touch, +then statically forbid model mutation) is left as the deeper maintainer option. + +**The primary, always-active defense against model mutation is the faithfulness check** +(patched-vs-baseline frame LPIPS on the scored pair): any mutation that changes the +OUTPUT frames raises faithfulness and is penalized; an iso-output mutation is a +legitimate faster-equivalent. It is now **fail-closed** — `evaluator.py` returns 0 (not +`faithfulness_mult=1.0`) if faithfulness could not be computed on a `role=="final"` run, +so the backstop cannot be silently disabled. The frozen-model guard is defense-in-depth +on top of it (catching an iso-output-but-still-mutated model). diff --git a/2.0/problems/nanowm_rollout_speedup/config.yaml b/2.0/problems/nanowm_rollout_speedup/config.yaml index 80f330f4..9c3bc1bf 100644 --- a/2.0/problems/nanowm_rollout_speedup/config.yaml +++ b/2.0/problems/nanowm_rollout_speedup/config.yaml @@ -40,9 +40,11 @@ environment: build_timeout_seconds: 5400 evaluation: # GPU served on Modal (one per environment); judge container is CPU-only. + # H100 matches the hardware the reference + noise floor were calibrated on, so + # the production scoring path and the validated numbers share one GPU SKU. model: nanowm_l2_csgo dataset: game/csgo - gpu: L40S + gpu: H100 # FIXED rollout invocation (the agent's patch changes sampler internals, not these). rollout_length: 50 history_length: 4 @@ -54,10 +56,23 @@ evaluation: # Calibrated: seq@20 is already +5% over seq@50, so a 3% tolerance forces real # fast-sampling work (DPM-Solver++, caching, distillation), not naive step cuts. quality_tolerance: 0.03 + # (E) Speedup at which the latency score saturates to 100: score is + # 100*log2(speedup)/log2(target). The old bare 100*log2 capped everything >=2x + # at 100; 4x keeps a gradient across the achievable range (causal-prefix ~3x). + speedup_target: 4.0 + # (A) Faithfulness BACKSTOP: mean LPIPS between PATCHED and BASELINE rollout + # frames (paired final run), always reported; penalty only past this generous + # threshold so it catches an egregious rollout SUBSTITUTION, not legitimate + # iso-quality speedups. Calibrated on H100: bf16 reference drifts 0.206 from the + # fp32 baseline (iso-quality vs GT, different trajectory), so 0.30 clears it with + # margin while still flagging ~half-divergent substitutions; causal-prefix ~0. + faithfulness_tol: 0.30 quick_clips: 4 # iterative (agent-role) public feedback final_clips: 16 # final (verifier-role) evaluation batch_size: 4 - baseline_cache_path: /opt/nanowm/baseline/baseline_metrics.json + # Key MUST be `baseline_cache` (settings.py strips the FRONTIER_NWM_ prefix and + # looks up `baseline_cache`); `baseline_cache_path` was silently ignored. + baseline_cache: /opt/nanowm/baseline/baseline_metrics.json submission: kind: file path: /app/solution.patch diff --git a/2.0/problems/nanowm_rollout_speedup/docker/agent/Dockerfile b/2.0/problems/nanowm_rollout_speedup/docker/agent/Dockerfile index c4de13c8..d043659a 100644 --- a/2.0/problems/nanowm_rollout_speedup/docker/agent/Dockerfile +++ b/2.0/problems/nanowm_rollout_speedup/docker/agent/Dockerfile @@ -25,8 +25,15 @@ RUN git clone https://github.com/simchowitzlabpublic/nano-world-model /app/nano- COPY task_ctx/infra_patches/ /tmp/infra_patches/ RUN cd /app/nano-world-model && for p in /tmp/infra_patches/*.patch; do \ [ -e "$p" ] && git apply "$p" || true; done && \ + { grep -q 'use_deterministic_algorithms' src/sample/evaluate_metrics.py \ + && grep -q 'NWM_TIME_FILE' src/sample/rollout.py; } \ + || { echo 'FATAL: infra patch did not apply (determinism / timer markers missing)'; exit 1; } && \ git add -A && git -c user.email=t@e -c user.name=t commit -q -m base || true COPY task_ctx/harbor_app/ /app/ -COPY task_ctx/task_pkg/ /app/task/ +# NOTE: the eval package (task_pkg/speedup_eval + evaluator.py) is JUDGE-ONLY and is +# deliberately NOT shipped to the agent. It carries settings.py (scoring knobs, seed, +# val-file paths) and the scoring/orchestration source (incl. the frozen-model guard); +# baking it into /app/task let the agent read judge internals directly. The judge image +# keeps it under /opt/nanowm/task. The agent validates by submitting + reading feedback. RUN chmod +x /app/*.sh 2>/dev/null || true diff --git a/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh b/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh index 815b802d..83c1f58c 100755 --- a/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh +++ b/2.0/problems/nanowm_rollout_speedup/docker/build_images.sh @@ -1,26 +1,42 @@ #!/usr/bin/env bash # Build the nanowm_rollout_speedup agent + judge images. # bash 2.0/problems/nanowm_rollout_speedup/docker/build_images.sh [tag] -# Requires the hidden assets staged under $NWM_ASSETS (must be set; no default): -# $NWM_ASSETS/ckpts/nanowm-l2-csgo-100k/ (L/2 CSGO checkpoint dir -> baked as ckpts/nanowm-l2-csgo) +# Judge-only hidden assets are AUTO-STAGED from public sources (no manual step): +# $NWM_ASSETS/ckpts/nanowm-l2-csgo-100k/ (L/2 CSGO ckpt -> baked as ckpts/nanowm-l2-csgo) # $NWM_ASSETS/csgo/1-200/*.hdf5 (held-out CSGO episode subset -> data/csgo) # $NWM_ASSETS/csgo_subset/{val_files.txt,val_starts.npy} (-> data/csgo_subset) # (baseline is optional; computed on first trial otherwise) +# NWM_ASSETS defaults to a cache dir and is populated by docker/prep_assets.py +# (downloads the public HF checkpoint + the CSGO 1-200 chunk, derives the val +# subset). Pre-stage NWM_ASSETS yourself to skip the download. set -euo pipefail TAG="${1:-experimental-v0}" NANOWM_COMMIT="${NANOWM_COMMIT:-main}" SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PROB=$(cd "$SCRIPT_DIR/.." && pwd) -NWM_ASSETS="${NWM_ASSETS:?set NWM_ASSETS to the dir holding the L/2 CSGO ckpt + held-out CSGO subset (see header)}" +NWM_ASSETS="${NWM_ASSETS:-$HOME/.cache/frontier-cs/nwm_assets}" + +# Auto-stage the public assets if not already present (self-contained build). +if [ ! -f "$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k/model_state_dict.pt" ] || \ + ! ls "$NWM_ASSETS/csgo/1-200/"*.hdf5 >/dev/null 2>&1; then + echo "Staging NanoWM assets into $NWM_ASSETS (one-time public download via prep_assets.py)..." + uv run --no-project --with huggingface_hub --with safetensors --with torch --with numpy \ + python "$SCRIPT_DIR/prep_assets.py" --out "$NWM_ASSETS" +fi CTX=$(mktemp -d); trap 'rm -rf "$CTX"' EXIT mkdir -p "$CTX/task_ctx" -cp -r "$PROB/speedup_eval" "$CTX/task_ctx/task_pkg" +# Preserve the package DIRECTORY (task_pkg/speedup_eval/) so the baked layout is +# /opt/nanowm/task/speedup_eval/ + /opt/nanowm/task/evaluator.py. evaluator.py +# does `from speedup_eval import ...` and the eval modules use relative imports +# (`from . import settings`), so flattening the contents into task/ breaks both. +mkdir -p "$CTX/task_ctx/task_pkg" +cp -r "$PROB/speedup_eval" "$CTX/task_ctx/task_pkg/speedup_eval" cp "$PROB/evaluator.py" "$CTX/task_ctx/evaluator.py" cp -r "$PROB/harbor/app" "$CTX/task_ctx/harbor_app" cp -r "$PROB/infra_patches" "$CTX/task_ctx/infra_patches" 2>/dev/null || mkdir -p "$CTX/task_ctx/infra_patches" -# judge-only hidden assets -mkdir -p "$CTX/task_ctx/assets" +# judge-only hidden assets (create dest parents first so the renaming cp lands) +mkdir -p "$CTX/task_ctx/assets/ckpts" "$CTX/task_ctx/assets/data" cp -r "$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k" "$CTX/task_ctx/assets/ckpts/nanowm-l2-csgo" 2>/dev/null || true cp -r "$NWM_ASSETS/csgo" "$CTX/task_ctx/assets/data/csgo" 2>/dev/null || true cp -r "$NWM_ASSETS/csgo_subset" "$CTX/task_ctx/assets/data/csgo_subset" 2>/dev/null || true diff --git a/2.0/problems/nanowm_rollout_speedup/docker/judge/Dockerfile b/2.0/problems/nanowm_rollout_speedup/docker/judge/Dockerfile index f5846dd0..f8249cf3 100644 --- a/2.0/problems/nanowm_rollout_speedup/docker/judge/Dockerfile +++ b/2.0/problems/nanowm_rollout_speedup/docker/judge/Dockerfile @@ -14,7 +14,10 @@ RUN git clone https://github.com/simchowitzlabpublic/nano-world-model /opt/nanow cd /opt/nanowm/nano-world-model && git checkout "$NANOWM_COMMIT" COPY task_ctx/infra_patches/ /tmp/infra_patches/ RUN cd /opt/nanowm/nano-world-model && for p in /tmp/infra_patches/*.patch; do \ - [ -e "$p" ] && git apply "$p" || true; done + [ -e "$p" ] && git apply "$p" || true; done && \ + { grep -q 'use_deterministic_algorithms' src/sample/evaluate_metrics.py \ + && grep -q 'NWM_TIME_FILE' src/sample/rollout.py; } \ + || { echo 'FATAL: infra patch did not apply (determinism / timer markers missing)'; exit 1; } # Hidden assets baked by build_images.sh into the build context: # task_ctx/assets/ckpts/nanowm-l2-csgo/model_state_dict.pt diff --git a/2.0/problems/nanowm_rollout_speedup/docker/prep_assets.py b/2.0/problems/nanowm_rollout_speedup/docker/prep_assets.py new file mode 100644 index 00000000..df4a341d --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/docker/prep_assets.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Auto-prepare the (public) NanoWM rollout-task assets so the build is +self-contained -- no hand-staged NWM_ASSETS required. Invoked by build_images.sh +when the staging dir is empty; safe to run standalone. + +Produces, under --out (the NWM_ASSETS layout build_images.sh consumes): + ckpts/nanowm-l2-csgo-100k/model_state_dict.pt (converted from HF safetensors) + ckpts/nanowm-l2-csgo-100k/config.yaml + csgo/1-200/hdf5_dm_july2021_.hdf5 (ONLY the held-out val episodes) + csgo_subset/{val_files.txt,val_starts.npy} (derived from the repo split) + +Public sources (verified public, no token / no gating / no terms): + checkpoint : HF model knightnemo/nanowm-l2-csgo-100k (model.safetensors, 2.23 GB) + csgo data : HF dataset TeaPearce/CounterStrike_Deathmatch (hdf5_dm_july2021_1_to_200.tar, 25.7 GB) + val subset : DERIVED from nano-world-model/src/.../csgo_splits/{test_split.txt, + csgo_validation_start_indices.npy} -- the held-out subset is exactly + the test_split episodes with number <= 200 (22 clips), which is the + 1-200 data chunk the task stages. No download, no seed needed. + +Notes: + * The HF model ships `model.safetensors`, not the `model_state_dict.pt` the + upstream loader (`utils.nanowm_utils.find_model` -> torch.load) expects, so we + convert. find_model strips a leading `model.`/`_orig_mod.` prefix and unwraps a + `model` key, so a bare state_dict loads cleanly. + * The dataset is stored as 200-episode TAR chunks (no individual .hdf5), so we + download the single 1-200 chunk to a temp dir, extract ONLY the ~22 needed + members, then delete the tar -- the staged data is ~3 GB, not 25.7 GB. + * Idempotent: each step is skipped if its output already exists. + * Keep this judge/operator-side ONLY; the agent image never sees these assets. +""" +from __future__ import annotations + +import argparse +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + +CKPT_REPO = "knightnemo/nanowm-l2-csgo-100k" +DATA_REPO = "TeaPearce/CounterStrike_Deathmatch" +DATA_TAR = "hdf5_dm_july2021_1_to_200.tar" +NANOWM_REPO_URL = "https://github.com/simchowitzlabpublic/nano-world-model" +MAX_EPISODE = 200 # held-out subset = test_split episodes with number <= this + + +def _ep_num(fn: str) -> int: + m = re.search(r"_(\d+)\.hdf5$", fn) + return int(m.group(1)) if m else 10**9 + + +def _splits_dir(repo_hint: str) -> Path: + """Locate (or clone) the csgo_splits dir holding the public split files.""" + rel = "src/wm_datasets/data_source/game/csgo_splits" + if repo_hint: + cand = Path(repo_hint) / rel + if (cand / "test_split.txt").exists(): + return cand + cand = Path(repo_hint) # maybe repo_hint already points at the splits dir + if (cand / "test_split.txt").exists(): + return cand + tmp = Path(tempfile.mkdtemp(prefix="nwm_splits_")) / "repo" + print(f"[splits] cloning {NANOWM_REPO_URL} (shallow) for split files ...", flush=True) + subprocess.run(["git", "clone", "--depth", "1", NANOWM_REPO_URL, str(tmp)], check=True) + return tmp / rel + + +def derive_val_subset(splits: Path, out_dir: Path) -> list[str]: + import numpy as np + test = [l.strip() for l in (splits / "test_split.txt").read_text().splitlines() if l.strip()] + starts = np.load(splits / "csgo_validation_start_indices.npy") + if len(starts) != len(test): + raise SystemExit(f"split/start length mismatch: {len(starts)} starts vs {len(test)} files") + sel = [(i, f) for i, f in enumerate(test) if _ep_num(f) <= MAX_EPISODE] + files = [f for _, f in sel] + sub_starts = np.array([int(starts[i]) for i, _ in sel], dtype=np.int64) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "val_files.txt").write_text("\n".join(files) + "\n") + np.save(out_dir / "val_starts.npy", sub_starts) + print(f"[val] derived {len(files)} held-out clips -> {out_dir}", flush=True) + return files + + +def ensure_ckpt(out: Path) -> None: + dst = out / "ckpts" / "nanowm-l2-csgo-100k" + pt = dst / "model_state_dict.pt" + if pt.exists(): + print(f"[ckpt] exists, skip ({pt})", flush=True) + return + from huggingface_hub import hf_hub_download + from safetensors.torch import load_file + import torch + dst.mkdir(parents=True, exist_ok=True) + print(f"[ckpt] downloading {CKPT_REPO} model.safetensors (~2.23 GB) ...", flush=True) + st = hf_hub_download(CKPT_REPO, "model.safetensors") + cfg = hf_hub_download(CKPT_REPO, "config.yaml") + shutil.copy(cfg, dst / "config.yaml") + torch.save(load_file(st), pt) # torch-loadable .pt for upstream find_model() + print(f"[ckpt] wrote {pt} ({pt.stat().st_size / 1e9:.2f} GB)", flush=True) + + +def ensure_data(out: Path, files: list[str]) -> None: + dst = out / "csgo" / "1-200" + needed = set(files) + have = {p.name for p in dst.glob("*.hdf5")} if dst.exists() else set() + missing = needed - have + if not missing: + print(f"[data] all {len(needed)} episodes present, skip", flush=True) + return + from huggingface_hub import hf_hub_download + dst.mkdir(parents=True, exist_ok=True) + # Download into the HF cache (resumable across runs); a 25.7 GB pull is too + # fragile to restart from scratch on an interruption. The cached tar is left + # in ~/.cache/huggingface afterwards (purge with `huggingface-cli delete-cache` + # to reclaim ~25.7 GB once the 22 episodes are staged). + print(f"[data] downloading {DATA_TAR} (~25.7 GB, resumable HF cache) ...", flush=True) + tarp = hf_hub_download(DATA_REPO, DATA_TAR, repo_type="dataset") + with tarfile.open(tarp) as tf: + members = {Path(m.name).name: m for m in tf.getmembers() if m.name.endswith(".hdf5")} + for fn in sorted(missing): + m = members.get(fn) + if m is None: + raise SystemExit(f"{fn} not found in {DATA_TAR}; sample members={list(members)[:3]}") + m.name = fn # flatten into 1-200/.hdf5 + tf.extract(m, dst) + print(f"[data] extracted {fn}", flush=True) + print(f"[data] staged {len(needed)} episodes -> {dst}", flush=True) + + +def main() -> int: + ap = argparse.ArgumentParser(description="Auto-stage public NanoWM rollout assets.") + ap.add_argument("--out", required=True, help="NWM_ASSETS staging dir to populate") + ap.add_argument("--repo", default="", help="optional nano-world-model checkout (for split files)") + ap.add_argument("--skip-data", action="store_true", help="stage ckpt+val only (skip the 25.7 GB data)") + a = ap.parse_args() + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + files = derive_val_subset(_splits_dir(a.repo), out / "csgo_subset") + ensure_ckpt(out) + if not a.skip_data: + ensure_data(out, files) + print(f"[done] assets staged under {out}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/2.0/problems/nanowm_rollout_speedup/evaluator.py b/2.0/problems/nanowm_rollout_speedup/evaluator.py index 1cff1129..a485a5e1 100644 --- a/2.0/problems/nanowm_rollout_speedup/evaluator.py +++ b/2.0/problems/nanowm_rollout_speedup/evaluator.py @@ -35,24 +35,47 @@ ALLOWED = ( "src/diffusion/*.py", "src/diffusion/**/*.py", - "src/sample/sampling_utils.py", ) DENIED = ( "src/models/**", "src/latent_codecs/**", # frozen model + VAE "src/sample/evaluate_metrics.py", "src/sample/plot_metrics.py", # the metric "src/sample/rollout.py", # the judge harness + # sampling_utils.py is PLUMBING (VAE encode/decode, video IO, resize), NOT the + # diffusion sampler -- it decodes+saves the generated frames the LPIPS quality + # guardrail reads, so allowing it would let a submission blur ONLY the scored + # pixels and mask a quality regression (passing the guardrail unfairly). Frozen. + "src/sample/sampling_utils.py", "src/wm_datasets/**", # data loading / no bench detection "src/experiments/**", "src/main.py", "src/utils/**", "**/*.so", "**/*.pyx", "**/*.c", "**/*.cpp", "**/*.cu", # no native "**/setup.py", "**/pyproject.toml", "**/requirements*.txt", ) # Forbidden substrings in ADDED lines (no benchmark detection / env leakage / -# timing short-circuits / hardcoded ground truth). +# timing short-circuits / hardcoded ground truth / sandbox escape / reading the +# held-out data). Defense-in-depth only -- substring matching is necessarily +# imperfect, so the real guarantees are the path allowlist, the frozen metric, +# and the sampler never receiving future GT frames. Tokens are chosen to be +# high-value AND low-false-positive: e.g. we deliberately do NOT ban `eval(` +# (would hit `model.eval()`), `compile(` (would hit `torch.compile()`), or +# `open(` (legit solver caches), relying on the deeper guardrails for those. FORBIDDEN_TOKENS = ( + # benchmark / judge / env detection + leakage. `environ` (not just + # `os.environ`) also catches `from os import environ` / aliased reads; the + # rollout injects CSGO_DATA_DIR (the held-out GT dir) into the patched + # subprocess env, so that key is banned outright. "FRONTIER_", "JUDGE_", "HARBOR_", "MODAL_", "HF_TOKEN", "NWM_TIME", + "environ", "os.getenv", "getenv(", "CSGO_DATA_DIR", "sys.modules", + # the metric / ground truth / held-out selection (no hard-coding or peeking) "csgo_validation_start_indices", "val_starts", "val_files", "gen_seconds", "evaluate_metrics", "lpips", "ground_truth", "gt_frames", - "os.environ", "subprocess", "socket", "/judge", "/opt/nanowm/baseline", + # held-out asset / data paths (no reading the GT episodes off disk) + "/judge", "/opt/nanowm", "/proc/", ".hdf5", "csgo_subset", + "train_split", "test_split", + # process / network / dynamic-exec escapes + "subprocess", "socket", "os.system", "os.popen", "os.exec", "os.fork", + "__import__", "importlib", "__builtins__", "ctypes", "marshal", + "urllib", "requests", "http.client", + # timing short-circuits "time.sleep", "while True", ) @@ -172,11 +195,39 @@ def evaluate(solution_path: str): try: baseline, patched = orchestrate.run_pair(patch_path, clips=clips, role=role) except Exception as exc: # concise public error only - return 0.0, 0.0, f"evaluation failed: {type(exc).__name__}", {**pmetrics, "role": role} + # Attribute the failure: a patch that won't apply OR a patch whose code + # crashes the rollout/metric is the submission's fault (a legitimate 0); + # an orchestration/GPU error is judge-side infra. All score 0 per the 2.0 + # convention, but `infra_error` lets the operator spot/re-run the latter + # instead of mistaking a transient GPU failure for a bad submission. + ml = str(exc).lower() + if isinstance(exc, RuntimeError) and "patch failed" in ml: + kind, infra, pub = "patch_apply", 0, "patch failed to apply" + elif isinstance(exc, RuntimeError) and ("rollout failed" in ml or "metrics failed" in ml): + kind, infra, pub = "execution", 0, "submission crashed during rollout/metrics" + else: + kind, infra, pub = "infra", 1, f"evaluation infra error: {type(exc).__name__}" + return 0.0, 0.0, pub, {**pmetrics, "role": role, "error_class": type(exc).__name__, + "error_kind": kind, "infra_error": infra} + faith = patched.get("faithfulness_lpips") # set only on the paired (final) run + if role == "final" and faith is None: + # Fail CLOSED. Faithfulness (patched-vs-baseline frame LPIPS) is the scored + # run's active backstop against a model swap/mutation the static policy can't + # see: any mutation that changes the OUTPUT frames raises faithfulness and is + # penalized (an iso-output mutation is a legitimate faster-equivalent). If it + # could not be computed (frames_lpips raised -- e.g. a patch that produced + # unreadable frames, or a transient infra failure), do NOT silently score with + # faithfulness_mult=1.0 (a free pass that disables the defense). Treat as an + # infra error to re-run; a submission that consistently breaks it keeps scoring 0. + return 0.0, 0.0, "faithfulness backstop unavailable on the scored run", { + **pmetrics, "role": role, "error_kind": "infra", "infra_error": 1, + "error_class": "FaithfulnessUnavailable"} res = scoring.provisional_score( {"all": baseline["gen_seconds"]}, {"all": patched["gen_seconds"]}, baseline["lpips"], patched["lpips"], tolerance=S.QUALITY_TOLERANCE, + faithfulness_lpips=faith, faithfulness_tolerance=S.FAITHFULNESS_TOLERANCE, + speedup_target=S.SPEEDUP_SCORE_TARGET, ) metrics = { **pmetrics, "role": role, "clips": clips, @@ -186,11 +237,18 @@ def evaluate(solution_path: str): "patched_lpips": round(patched["lpips"], 4), "geomean_speedup": round(res["geomean_speedup"], 3), "quality_multiplier": round(res["quality_multiplier"], 3), - "score_formula": "100*log2(speedup)*quality_mult; quality_mult<1 if LPIPS rises >tol over baseline", + "faithfulness_lpips": round(faith, 4) if faith is not None else None, + "faithfulness_multiplier": round(res["faithfulness_multiplier"], 3), + "speedup_target": S.SPEEDUP_SCORE_TARGET, + "score_formula": ("100*log2(speedup)/log2(target) * quality_mult * faithfulness_mult; " + "quality_mult<1 if LPIPS-vs-GT rises >tol over baseline; " + "faithfulness_mult<1 if patched frames diverge >tol LPIPS from baseline frames"), } + _faith_str = (f", faith {faith:.3f} vs base-rollout (tol {S.FAITHFULNESS_TOLERANCE:.2f}" + f", mult {res['faithfulness_multiplier']:.2f})" if faith is not None else "") msg = (f"speedup {res['geomean_speedup']:.2f}x " f"(lpips {patched['lpips']:.3f} vs baseline {baseline['lpips']:.3f}, " - f"tol {S.QUALITY_TOLERANCE:.0%}); score {res['score']:.1f}") + f"tol {S.QUALITY_TOLERANCE:.0%}){_faith_str}; score {res['score']:.1f}") return res["score"], res["score_unbounded"], msg, metrics diff --git a/2.0/problems/nanowm_rollout_speedup/harbor/app/README.md b/2.0/problems/nanowm_rollout_speedup/harbor/app/README.md index 77e72eac..3a2f2687 100644 --- a/2.0/problems/nanowm_rollout_speedup/harbor/app/README.md +++ b/2.0/problems/nanowm_rollout_speedup/harbor/app/README.md @@ -5,14 +5,15 @@ Patch the diffusion **sampling** code of the NanoWM checkout at iso-quality, then submit the unified diff as `/app/solution.patch`. Workflow: -1. Edit `nano-world-model/src/diffusion/**` or `src/sample/sampling_utils.py`. +1. Edit the diffusion sampler under `nano-world-model/src/diffusion/**`. 2. `bash /app/public_test.sh` — static patch-policy check on your diff. 3. `bash /app/make_submission.sh` — writes `/app/solution.patch` from your edits. 4. `bash /app/submit.sh` — enqueue for the black-box judge (quick set). -Rules (see the task statement): Python-only; allowed = `src/diffusion/**.py`, -`src/sample/sampling_utils.py`; the model/VAE/metric/harness/data are frozen and -denied; no env-var/benchmark/timing tricks. The judge fixes the rollout call — +Rules (see the task statement): Python-only; allowed = `src/diffusion/**.py`; +the model/VAE/metric/harness/data AND the VAE-decode/video plumbing +(`src/sample/sampling_utils.py`) are frozen and denied; no env-var/benchmark/ +timing tricks. The judge fixes the rollout call — you change the sampler internals. Scored on wall-clock speedup vs the unpatched baseline, gated by an LPIPS-vs-GT diff --git a/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-judge-infra.patch b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-judge-infra.patch index 4c2b40b7..1f4344fd 100644 --- a/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-judge-infra.patch +++ b/2.0/problems/nanowm_rollout_speedup/infra_patches/0001-rollout-judge-infra.patch @@ -1,5 +1,32 @@ +diff --git a/src/sample/evaluate_metrics.py b/src/sample/evaluate_metrics.py +index cc6846d..6132034 100644 +--- a/src/sample/evaluate_metrics.py ++++ b/src/sample/evaluate_metrics.py +@@ -34,7 +34,21 @@ def main(): + parser.add_argument("--history_length", type=int, default=1, help="Number of history frames (context)") + + args = parser.parse_args() +- ++ ++ # Deterministic metric (judge infra): the scored LPIPS runs on GPU by default, ++ # so without this the AlexNet/VGG convs are cuDNN-nondeterministic and the ++ # baseline vs patched arms (separate metric processes) can differ even for a ++ # no-op patch -- reintroducing the run-to-run noise the rollout determinism ++ # is meant to remove. Mirror rollout.py: deterministic kernels + TF32 off. ++ # NWM_DETERMINISM_STRICT=1 makes a missing deterministic kernel raise (proof). ++ _strict = os.environ.get("NWM_DETERMINISM_STRICT", "0") == "1" ++ torch.use_deterministic_algorithms(True, warn_only=not _strict) ++ if args.device == "cuda": ++ torch.backends.cudnn.deterministic = True ++ torch.backends.cudnn.benchmark = False ++ torch.backends.cuda.matmul.allow_tf32 = False ++ torch.backends.cudnn.allow_tf32 = False ++ + video_dir = Path(args.video_dir) + if not video_dir.exists(): + print(f"Directory {video_dir} does not exist.") diff --git a/src/sample/rollout.py b/src/sample/rollout.py -index b83f0ee..d4bfd6a 100644 +index b83f0ee..d87ba9c 100644 --- a/src/sample/rollout.py +++ b/src/sample/rollout.py @@ -9,6 +9,7 @@ Uses the sliced dataset for evaluation consistency. @@ -10,7 +37,44 @@ index b83f0ee..d4bfd6a 100644 sys.path.append(os.path.split(sys.path[0])[0]) import torch -@@ -131,9 +132,20 @@ def main(args): +@@ -37,7 +38,35 @@ torch.backends.cudnn.allow_tf32 = True + def main(args): + torch.set_grad_enabled(False) + device = "cuda" if torch.cuda.is_available() else "cpu" +- ++ ++ # Deterministic numerics (judge infra, outside the agent's editable sampling ++ # scope). Goal: the baseline (unpatched) and patched arms share an identical ++ # numerical pipeline -- not just identical initial noise from the per-clip ++ # seed below, but identical arithmetic -- so a cached/separate-run baseline is ++ # a valid common-random-numbers partner. Otherwise GPU reduction/atomic ++ # nondeterminism (and TF32, which upstream turns ON at import) reintroduce ++ # ~2-3% run-to-run LPIPS noise, the same magnitude as the speedup quality ++ # guardrail and the stability drift effect. CUBLAS_WORKSPACE_CONFIG is ++ # exported by the launcher before the process starts (cuBLAS needs it set ++ # before the first CUDA call). ++ # ++ # CAVEAT: warn_only=True (the default) means any op WITHOUT a deterministic ++ # CUDA kernel (an atomicAdd scatter/index_add, some interpolate variants) ++ # runs nondeterministically and is only WARNED about, so bit-stability is not ++ # guaranteed until proven for this exact model+VAE+decode on the production ++ # GPU. Run ONCE during H100 calibration with NWM_DETERMINISM_STRICT=1 ++ # (warn_only=False) to PROVE no op falls back -- it raises instead of warning ++ # if one does -- and re-establish no-op-CRN-pair -> 0.0 there. Production may ++ # then stay strict (a fallback raises -> judged an infra error and re-run, ++ # never a silently corrupted score) or relax to warn_only once proven clean. ++ _strict = os.environ.get("NWM_DETERMINISM_STRICT", "0") == "1" ++ torch.use_deterministic_algorithms(True, warn_only=not _strict) ++ if device == "cuda": ++ torch.backends.cudnn.deterministic = True ++ torch.backends.cudnn.benchmark = False ++ torch.backends.cuda.matmul.allow_tf32 = False ++ torch.backends.cudnn.allow_tf32 = False ++ + save_dir = Path(args.save_path) + save_dir.mkdir(parents=True, exist_ok=True) + +@@ -131,9 +160,20 @@ def main(args): print(f" History length: {history_length}") print(f" Rollout length: {rollout_length}") @@ -31,7 +95,7 @@ index b83f0ee..d4bfd6a 100644 print(f"Processing batch {start_idx//batch_size + 1} (slices {start_idx} to {end_idx-1})...") -@@ -181,7 +193,12 @@ def main(args): +@@ -181,7 +221,12 @@ def main(args): gt_latents = encode_frames(vae, gt_visual, device, vae_precision=vae_precision) generated_latents = gt_latents[:, :history_length] @@ -45,7 +109,7 @@ index b83f0ee..d4bfd6a 100644 for t in tqdm(range(history_length, rollout_length), desc=f"Rollout progress"): context_latents = generated_latents[:, -history_length:] -@@ -224,9 +241,25 @@ def main(args): +@@ -224,9 +269,25 @@ def main(args): new_latent = pred_latents[:, history_length:history_length+1] generated_latents = torch.cat([generated_latents, new_latent], dim=1) @@ -73,7 +137,7 @@ index b83f0ee..d4bfd6a 100644 for i in range(current_batch_size): sample_id = start_idx + i -@@ -234,6 +267,15 @@ def main(args): +@@ -234,6 +295,15 @@ def main(args): save_video(gt_frames_batch[i], str(save_dir / f"sample_{sample_id:04d}_gt.mp4"), fps=args.fps) save_comparison_video(gt_frames_batch[i], gen_frames_batch[i], str(save_dir / f"sample_{sample_id:04d}_compare.mp4"), fps=args.fps) @@ -89,7 +153,7 @@ index b83f0ee..d4bfd6a 100644 print(f"Done! Processed {num_samples} samples.") if __name__ == "__main__": -@@ -259,6 +301,9 @@ if __name__ == "__main__": +@@ -259,6 +329,9 @@ if __name__ == "__main__": help="DFoT stabilization noise level in [0, 1) for context frames. " "0.0 disables; 0.02 is the DFoT default recommended starting point.") parser.add_argument("--use_fp16", action="store_true", help="Use fp16") @@ -99,7 +163,7 @@ index b83f0ee..d4bfd6a 100644 parser.add_argument("--vae_model_path", type=str, default=None, help="Override the VAE path from the training config (e.g. local dir for offline nodes).") -@@ -282,7 +327,7 @@ if __name__ == "__main__": +@@ -282,7 +355,7 @@ if __name__ == "__main__": # Top-level / non-hierarchical CLI args for key in ('ckpt', 'save_path', 'num_samples', 'batch_size', 'rollout_length', diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/frozen_model_guard.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/frozen_model_guard.py new file mode 100644 index 00000000..bd310157 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/frozen_model_guard.py @@ -0,0 +1,87 @@ +"""Frozen-model guard for the speedup judge (audit: temp_embed gray area). + +WHY. The agent's allowed scope is `src/diffusion/**` (the sampler), and the model +(`src/models/**`) is FROZEN. The reference-class causal-prefix optimization (codex's +2.87x) reaches into the model from the sampler at RUNTIME -- it temporarily reshapes +`module.temp_embed` (a frozen-tree nn.Parameter) to run the unmodified forward on a +shorter causal window, then RESTORES it via try/finally. That specific use is exact +and benign, and the static patch policy (path allowlist + token scan) cannot see a +runtime attribute monkeypatch. The gray area: nothing stops a DIFFERENT patch from +*persistently* mutating the frozen model (e.g. swapping in blurred/distilled weights +to win wall-clock) from inside the allowed sampler. + +WHAT THIS DOES. A cheap dynamic invariant -- "the model you were handed is the model +you must return." Fingerprint the model parameters right after load, re-check after +the rollout. A patch that restores any transient reshape (like causal-prefix) passes; +a patch that leaves the model mutated is a frozen-model violation -> hard error (the +runner attributes a rollout crash to the submission -> score 0). This does NOT catch a +transient swap that is restored before the check -- that residual is covered by the +LPIPS-vs-GT quality guardrail (a degrade trips it) and is documented in DESIGN/AUDIT. + +The fingerprint is a structure hash (name/shape/dtype) plus a float64 sum per tensor: +cheap (one reduction per param, OUTSIDE the timed sampling region) and catches any +value change with overwhelming probability. It is not meant to resist an adversary +crafting a sum-preserving mutation -- that is the quality/faithfulness guardrail's job. + +WIRING (ACTIVE). `speedup_eval.runner.inject_frozen_guard()` inlines this exact logic +into the copied `src/sample/rollout.py` at apply-time (after the agent patch; rollout.py +is denied to the agent) -- snapshot after `model.eval()`, verify before the final +`print("Done!...")`. So the guard runs on every rollout in BOTH backends WITHOUT +importing this module into the frozen subprocess; this file is the unit-tested +reference (test_frozen_model_guard.py) and the human-readable spec. Injection itself is +fail-closed (raises if rollout.py anchors are missing) and tested in +test_inject_frozen_guard.py. The stability task wires the identical guard in +stability_eval.runner. + +Self-contained inline equivalent (what the runner injects): + + import hashlib as _hl + def _fp(m): + h=_hl.sha256() + for n,p in sorted(m.state_dict().items()): + h.update(f"{n}|{tuple(p.shape)}|{p.dtype}".encode()) + h.update(repr(float(p.detach().to('cpu',dtype=__import__('torch').float64).sum())).encode()) + return h.hexdigest() +""" +from __future__ import annotations + +import hashlib + + +def model_param_fingerprint(model) -> str: + """SHA-256 over (name, shape, dtype, float64 sum) of every state_dict tensor. + + Works with any object exposing `.state_dict()` returning a name->tensor mapping + where each tensor has `.shape`, `.dtype`, and `.detach()/.to()/.sum()` (a real + torch model; a stub in tests). Order-independent (sorted by name).""" + h = hashlib.sha256() + for name, p in sorted(model.state_dict().items()): + h.update(f"{name}|{tuple(p.shape)}|{p.dtype}".encode()) + h.update(repr(_tensor_checksum(p)).encode()) + return h.hexdigest() + + +def _tensor_checksum(p) -> float: + """float64 sum of a tensor's values (cheap, catches value changes). Falls back + through a couple of access paths so it works on real torch tensors and stubs.""" + q = p.detach() if hasattr(p, "detach") else p + # Prefer a float64 reduction on CPU for cross-device/dtype stability. + try: + import torch # type: ignore + return float(q.to("cpu", dtype=torch.float64).sum()) + except Exception: + pass + try: + return float(q.double().sum()) + except Exception: + return float(q.sum()) + + +def assert_frozen(model, fingerprint0: str, where: str = "rollout") -> None: + """Raise if `model`'s parameters changed since `fingerprint0` was taken.""" + fp = model_param_fingerprint(model) + if fp != fingerprint0: + raise RuntimeError( + f"frozen-model violation: the sampling patch persistently mutated the " + f"model parameters ({where}); the model is frozen (denied)." + ) diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/modal_app.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/modal_app.py index b36b5515..1e958a45 100644 --- a/2.0/problems/nanowm_rollout_speedup/speedup_eval/modal_app.py +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/modal_app.py @@ -1,4 +1,4 @@ -"""Modal app: run one NanoWM CSGO rollout (patched or vanilla) on one GPU. +"""Modal app: run NanoWM CSGO rollouts (patched and/or vanilla) on one GPU. Mirrors the GPU-on-Modal pattern of vllm_llm_serving_optimization: the judge container stays CPU-only and calls into a Modal GPU function. The NanoWM @@ -6,61 +6,147 @@ the Modal image at build time; the agent's patch text travels in per call. Parametrized via env vars (set by the judge): - NWM_MODAL_GPU Modal GPU string (default "L40S") + NWM_MODAL_GPU Modal GPU string (default "H100") NWM_MODAL_APP Modal app name NWM_HF_SECRET Modal Secret name (unused once weights are baked) Deploy: modal deploy speedup_eval/modal_app.py -Call: run_rollout_remote(patch_text, clips, steps) -> metrics dict +Call: + run_pair_remote(patch_text, clips, steps) -> {"baseline":{...}, "patched":{...}} + Runs BOTH arms back-to-back in one container on one GPU (two rollout + subprocesses) so the scored (final) result shares hardware/driver/seed. + The no-op->0 CRN guarantee additionally relies on the determinism flags + holding for every op (audit P1; prove once on H100 with strict mode). + run_rollout_remote(patch_text, clips, steps) -> metrics dict + Single arm; used for cheap cached-baseline iterative (agent-role) feedback. NOTE: validated structurally against the #145 modal pattern; end-to-end Modal execution is pending maintainer Modal credentials (Della has no Modal access). -The LOCAL backend (orchestrate._run_local) is the path validated on H100. +The LOCAL backend (orchestrate._run_local / _run_pair_local) is the path +validated on H100 -- the same SKU now requested here, so the production path and +the calibrated numbers share one GPU type. """ from __future__ import annotations import os -GPU = os.environ.get("NWM_MODAL_GPU", "L40S") +GPU = os.environ.get("NWM_MODAL_GPU", "H100") APP_NAME = os.environ.get("NWM_MODAL_APP", "nanowm-rollout-speedup") REMOTE_ROOT = "/opt/nanowm" +# cuBLAS determinism must be set before the first CUDA call; bake it into the +# image env so the rollout subprocess inherits it (pairs with the deterministic +# kernels enabled in rollout.py via the infra patch). +_DETERMINISM_ENV = {"CUBLAS_WORKSPACE_CONFIG": ":4096:8", "PYTHONHASHSEED": "0"} + try: import modal image = ( modal.Image.from_registry("nvidia/cuda:12.1.0-devel-ubuntu22.04", add_python="3.11") + # runner.apply_patch shells out to `git apply` (fallback `patch -p1`); the + # CUDA base ships neither, so the patched arm died with FileNotFoundError + # ('git') before any rollout — the baseline arm (no patch) never hit it. + .apt_install("git", "patch") .pip_install( "torch==2.4.1", "torchvision==0.19.1", "numpy<2", "scipy==1.15.3", "lpips", "diffusers[torch]==0.24.0", "omegaconf", "hydra-core", "decord", "imageio", "imageio-ffmpeg", "opencv-python-headless", "scikit-image", "pandas", "einops", "timm", "pytorch-lightning==2.4.0", "huggingface_hub==0.25.2", "transformers==4.46.3", + # rollout import-chain deps the hand-curated list omitted (the Modal + # path had never run a rollout end-to-end before): h5py reads the CSGO + # .hdf5 episodes (wm_datasets), tensorboard backs the SummaryWriter + # import at the top of utils/nanowm_utils.py pulled in by find_model. + "h5py", "tensorboard", + # metric-path deps (utils.metrics, imported by evaluate_metrics.py): + # piqa backs PSNR/SSIM, pytorch-fid the InceptionV3 the Evaluator ctor + # builds, requests is a top-level import in utils.fvd. FVD/I3D is never + # computed (only LPIPS is scored) but these module/ctor imports must + # resolve or the metric subprocess dies before writing any LPIPS. + "piqa", "pytorch-fid", "requests", + ) + # Pre-fetch the metric model weights at BUILD time so scoring never depends + # on a flaky per-container runtime download: LPIPS' VGG16 backbone and the + # InceptionV3 the Evaluator ctor builds were fetched from download.pytorch.org + # on every cold container and intermittently hit "Connection reset by peer", + # which would zero out a whole 22-clip scored run. Baked into the torch hub + # cache, the runtime Evaluator finds them locally. + .run_commands( + "python -c 'import lpips; lpips.LPIPS(net=\"vgg\"); " + "from pytorch_fid.inception import InceptionV3; " + "InceptionV3([InceptionV3.BLOCK_INDEX_BY_DIM[2048]])'" ) + .env(_DETERMINISM_ENV) # bake the clean checkout + ckpt + CSGO subset + task package .add_local_dir(os.environ.get("NWM_BAKE_DIR", "/opt/nanowm"), REMOTE_ROOT, copy=True) ) app = modal.App(APP_NAME) - @app.function(gpu=GPU, image=image, timeout=3600) - def _rollout(patch_text: str, clips: int, steps: int) -> dict: + def _eval_arm(patch_text: str, clips: int, steps: int, strict: bool = False, + keep_gen=None) -> dict: import sys, tempfile from pathlib import Path + # Propagate the judge's strict-determinism choice INTO this Modal container + # BEFORE settings is imported: the flag is set on the CPU judge, not here, + # so without this S.STRICT_DETERMINISM (hence the rollout/metric subprocess + # NWM_DETERMINISM_STRICT) would always read 0 (warn_only) on the GPU and the + # H100 proof run would prove nothing about op-level determinism. + os.environ["FRONTIER_NWM_STRICT_DETERMINISM"] = "1" if strict else "0" sys.path.insert(0, f"{REMOTE_ROOT}/task") - from speedup_eval import runner, orchestrate # noqa + from speedup_eval import runner, orchestrate, settings as S # noqa patch = None if patch_text.strip(): p = Path(tempfile.mkstemp(suffix=".patch")[1]) p.write_text(patch_text) patch = p cfg = orchestrate._compose_config() - from speedup_eval import settings as S - return runner.evaluate(S.REPO, patch, cfg, S.CKPT, clips=clips, steps=steps, device="cuda") + return runner.evaluate(S.REPO, patch, cfg, S.CKPT, clips=clips, steps=steps, + device="cuda", keep_gen=keep_gen) + + # Timeouts are generous: strict determinism (TF32 off + deterministic kernels) + # roughly triples the sampling wall-clock, and the long 80-frame stability + # rollout runs ~1560s per 2-clip batch on H100 -> a 22-clip baseline+patched + # pair is ~9-10h. Single-arm (quick/agent-role) tops out near ~2h. + @app.function(gpu=GPU, image=image, timeout=14400) # 4h: single-arm quick run + def _rollout(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: + return _eval_arm(patch_text, clips, steps, strict) + + @app.function(gpu=GPU, image=image, timeout=43200) # 12h: scored baseline+patched pair + def _rollout_pair(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: + # Baseline and patched on the SAME container/GPU (each its own rollout + # subprocess): identical hardware, driver, image and (with the infra-patch + # seeding) identical initial noise. This removes cross-container/-hardware + # variance; bit-equality for a no-op still depends on the determinism + # flags holding for every op (prove once on H100 with strict mode). + import sys, tempfile + from pathlib import Path + base_dir = tempfile.mkdtemp(prefix="nwm_faith_base_") + patch_dir = tempfile.mkdtemp(prefix="nwm_faith_patch_") + baseline = _eval_arm("", clips, steps, strict, keep_gen=base_dir) + patched = _eval_arm(patch_text, clips, steps, strict, keep_gen=patch_dir) + # (A) Faithfulness: how far the patched rollout's frames drifted from the + # baseline rollout's frames (closeness of OUTPUTS, not just to GT). Carried + # on the patched dict so run_pair's (baseline, patched) arity is unchanged. + try: + sys.path.insert(0, f"{REMOTE_ROOT}/task") + from speedup_eval import runner as _r + patched["faithfulness_lpips"] = _r.frames_lpips(Path(base_dir), Path(patch_dir), "cuda") + except Exception as _e: + patched["faithfulness_lpips"] = None + return {"baseline": baseline, "patched": patched} + + def run_rollout_remote(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: + with app.run(): + return _rollout.remote(patch_text, clips, steps, strict) - def run_rollout_remote(patch_text: str, clips: int, steps: int) -> dict: + def run_pair_remote(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: with app.run(): - return _rollout.remote(patch_text, clips, steps) + return _rollout_pair.remote(patch_text, clips, steps, strict) except ImportError: # modal not installed (e.g. local validation) - def run_rollout_remote(patch_text: str, clips: int, steps: int) -> dict: # type: ignore + def run_rollout_remote(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: # type: ignore + raise RuntimeError("modal not available; use the LOCAL backend (FRONTIER_NWM_BACKEND=local)") + + def run_pair_remote(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: # type: ignore raise RuntimeError("modal not available; use the LOCAL backend (FRONTIER_NWM_BACKEND=local)") diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py index 5ba87dbe..33b01d67 100644 --- a/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/orchestrate.py @@ -10,8 +10,18 @@ - LOCAL (default when a CUDA device is visible): runs the runner directly on the local GPU. Used for Della validation and any GPU-equipped judge. -The unpatched baseline is computed once and cached (baseline_cache_path); the -patched run always re-applies the agent patch to a clean checkout. +Baseline measurement (audit P1, common random numbers): + - role="final" (the SCORED verifier run): baseline and patched are measured + back-to-back on ONE GPU in one job (no cache). They run as two separate + rollout subprocesses (not one process), so what makes a no-op score ~0 is + op-level bit-stability -- the determinism flags in rollout.py/evaluate_metrics.py + holding for EVERY op, plus the per-clip seed. Pairing removes cross-run, + cross-container, cross-cache and cross-hardware variance; it does NOT by + itself cancel op-level nondeterminism. Prove the flags actually hold once on + H100 with FRONTIER_NWM_STRICT_DETERMINISM=1 (a fallback then raises). + - role="agent" (cheap iterative feedback): the unpatched baseline is cached, + keyed by a full config fingerprint; deterministic kernels + per-clip seeding + keep that cached baseline a valid CRN partner on the fixed (now H100) SKU. """ from __future__ import annotations @@ -23,8 +33,12 @@ def _compose_config() -> Path: - """Write the fixed CSGO rollout config (subset val list) the runner consumes.""" - out = Path(os.environ.get("FRONTIER_NWM_WORKDIR", "/tmp/nwm_speedup")) / "csgo_config.yaml" + """Write the fixed CSGO rollout config (subset val list) the runner consumes. + + The filename carries the config fingerprint so a changed task config never + silently reuses a stale composed config from a long-lived container.""" + workdir = Path(os.environ.get("FRONTIER_NWM_WORKDIR", "/tmp/nwm_speedup")) + out = workdir / f"csgo_config_{S.config_fingerprint()}.yaml" out.parent.mkdir(parents=True, exist_ok=True) if out.exists(): return out @@ -54,7 +68,36 @@ def _run_modal(patch_path: Path | None, clips: int) -> dict: on the baked NanoWM checkout and returns the metrics dict.""" from .modal_app import run_rollout_remote # deploys/looks up the Modal app patch_text = "" if patch_path is None else Path(patch_path).read_text(errors="replace") - return run_rollout_remote(patch_text=patch_text, clips=clips, steps=S.NUM_SAMPLING_STEPS) + return run_rollout_remote(patch_text=patch_text, clips=clips, steps=S.NUM_SAMPLING_STEPS, + strict=S.STRICT_DETERMINISM) + + +def _run_pair_local(patch_path: Path, clips: int) -> tuple[dict, dict]: + """Baseline then patched, back-to-back on the same local GPU (two rollout + subprocesses; op-level determinism is what makes them comparable).""" + import tempfile + from . import runner + cfg = _compose_config() + base_dir = tempfile.mkdtemp(prefix="nwm_faith_base_") + patch_dir = tempfile.mkdtemp(prefix="nwm_faith_patch_") + baseline = runner.evaluate(S.REPO, None, cfg, S.CKPT, clips=clips, + steps=S.NUM_SAMPLING_STEPS, device="cuda", keep_gen=base_dir) + patched = runner.evaluate(S.REPO, Path(patch_path), cfg, S.CKPT, clips=clips, + steps=S.NUM_SAMPLING_STEPS, device="cuda", keep_gen=patch_dir) + try: # (A) faithfulness of the patched rollout to the baseline rollout + patched["faithfulness_lpips"] = runner.frames_lpips(Path(base_dir), Path(patch_dir), "cuda") + except Exception: + patched["faithfulness_lpips"] = None + return baseline, patched + + +def _run_pair_modal(patch_path: Path, clips: int) -> tuple[dict, dict]: + """Baseline + patched in ONE Modal GPU container (true CRN pair).""" + from .modal_app import run_pair_remote + patch_text = Path(patch_path).read_text(errors="replace") + res = run_pair_remote(patch_text=patch_text, clips=clips, steps=S.NUM_SAMPLING_STEPS, + strict=S.STRICT_DETERMINISM) + return res["baseline"], res["patched"] def _backend(): @@ -68,17 +111,20 @@ def _backend(): def _baseline(clips: int, run) -> dict: cache = S.BASELINE_CACHE + fp = S.config_fingerprint() if cache.exists(): try: data = json.loads(cache.read_text()) - # Deterministic seeding makes the cached baseline a valid common-random - # -numbers partner ONLY for the same clip set (== not >=) and same seed. - if data.get("clips") == clips and data.get("seed") == S.SEED: + # Valid cached CRN partner only for the IDENTICAL config (fingerprint + # covers model/dataset/rollout/steps/scheduling/stab/batch/seed/val set) + # and the same clip count; deterministic kernels keep it bit-stable. + if data.get("fingerprint") == fp and data.get("clips") == clips: return data except Exception: pass m = run(None, clips) m["seed"] = S.SEED + m["fingerprint"] = fp try: cache.parent.mkdir(parents=True, exist_ok=True) cache.write_text(json.dumps(m)) @@ -88,7 +134,18 @@ def _baseline(clips: int, run) -> dict: def run_pair(patch_path: Path, clips: int, role: str = "agent") -> tuple[dict, dict]: - run = _run_modal if _backend() == "modal" else _run_local + backend = _backend() + if role == "final": + # Scored run: baseline and patched measured back-to-back on one GPU in one + # job (no cache). Removes cross-run/-container/-cache/-hardware variance; + # the no-op->0 guarantee additionally needs the determinism flags to hold + # for every op (prove once on H100 with FRONTIER_NWM_STRICT_DETERMINISM=1). + if backend == "modal": + return _run_pair_modal(Path(patch_path), clips) + return _run_pair_local(Path(patch_path), clips) + # Iterative (agent) feedback: cached baseline for speed; determinism + seeding + # keep it a valid CRN partner on the fixed GPU SKU. + run = _run_modal if backend == "modal" else _run_local baseline = _baseline(clips, run) patched = run(Path(patch_path), clips) return baseline, patched diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py index 8525b2ea..097f7950 100644 --- a/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/runner.py @@ -37,17 +37,93 @@ def apply_patch(clean_repo: Path, patch_path: Path | None) -> Path: else: (repo / child.name).symlink_to(child) if patch_path is not None and str(patch_path) != "-" and Path(patch_path).stat().st_size > 0: + # The upstream checkout ships CRLF .py files, but a submitted/reference diff + # is usually LF -- and the patch text is round-tripped through universal- + # newline read_text() (orchestrate -> Modal) which strips CR. Mismatched + # endings make git/patch reject with "patch does not apply (different line + # endings)", so EVERY submission would score 0. Normalise BOTH the copied + # target tree and the patch to LF so application is line-ending-agnostic + # (LF .py executes identically; only the copy is touched, not the baseline). + for py in (repo / "src").rglob("*.py"): + b = py.read_bytes() + if b"\r" in b: + py.write_bytes(b.replace(b"\r\n", b"\n").replace(b"\r", b"\n")) + patch_lf = work / "patch.diff" + patch_lf.write_bytes(Path(patch_path).read_bytes().replace(b"\r\n", b"\n").replace(b"\r", b"\n")) r = subprocess.run(["git", "apply", "--unsafe-paths", "--directory", str(repo), - str(patch_path)], capture_output=True, text=True) + str(patch_lf)], capture_output=True, text=True) if r.returncode != 0: # fall back to patch(1) (git apply needs a git root / clean context) - r = subprocess.run(["patch", "-p1", "-d", str(repo), "-i", str(patch_path)], + r = subprocess.run(["patch", "-p1", "-d", str(repo), "-i", str(patch_lf)], capture_output=True, text=True) if r.returncode != 0: - raise RuntimeError(f"patch failed: {r.stderr[:2000]}") + # patch(1) reports to stdout, git to stderr -- surface both. + raise RuntimeError(f"patch failed: {(r.stdout + r.stderr)[:2000]}") + # Inject the frozen-model guard into the (denied, judge-owned) rollout.py AFTER the + # agent patch, so the model-frozen invariant is enforced at runtime even though the + # static policy can't see a sampler-side runtime monkeypatch (the temp_embed gray area). + inject_frozen_guard(repo) return repo +# The frozen-model guard, inlined into rollout.py at apply-time (self-contained: no +# import of speedup_eval from the rollout subprocess). Snapshot the params right after +# the model is loaded+eval'd; re-check after the rollout. A restored transient reshape +# (the causal-prefix temp_embed slice) passes; a persistently mutated model hard-errors +# -> nonzero exit -> scored as a submission execution failure (0). See +# speedup_eval/frozen_model_guard.py (unit-tested logic) and DESIGN.md s9. +_FG_SNAPSHOT = ( + " # _FROZEN_GUARD (judge infra, injected by speedup_eval.runner; outside the\n" + " # agent's editable scope): the model is FROZEN -- a sampling patch may READ it\n" + " # but not PERSISTENTLY mutate it. Snapshot its params now; re-check after the\n" + " # rollout. A restored transient reshape (causal-prefix temp_embed) passes.\n" + " import hashlib as _hl_fg\n" + " def _fg_fp(_m):\n" + " _h = _hl_fg.sha256()\n" + " for _n, _p in sorted(_m.state_dict().items()):\n" + " _h.update(('%s|%s|%s' % (_n, tuple(_p.shape), _p.dtype)).encode())\n" + " _h.update(repr(float(_p.detach().to('cpu', dtype=torch.float64).sum())).encode())\n" + " return _h.hexdigest()\n" + " _fg_frozen = _fg_fp(model)\n" +) +_FG_VERIFY = ( + " # _FROZEN_GUARD: enforce the frozen-model invariant after the rollout.\n" + " if _fg_fp(model) != _fg_frozen:\n" + " raise RuntimeError('frozen-model violation: the sampling patch persistently '\n" + " 'mutated the model parameters (the model is frozen; denied)')\n" +) + + +def inject_frozen_guard(repo: Path) -> None: + """Insert the frozen-model guard into the copied src/sample/rollout.py. Anchors: + snapshot just after `model.eval()`, verify just before the final `print("Done!...")` + (after the rollout loop). FAIL CLOSED: if the anchors are not found (rollout.py + structure changed) the guard cannot be placed, so RAISE rather than run unguarded.""" + f = repo / "src" / "sample" / "rollout.py" + if not f.exists(): + return + src = f.read_text() + if "_fg_frozen" in src: # idempotent + return + lines = src.splitlines(keepends=True) + snap_at = verify_at = None + for i, ln in enumerate(lines): + s = ln.strip() + if snap_at is None and s == "model.eval()": + snap_at = i + 1 + if verify_at is None and (s.startswith('print(f"Done! Processed') + or s.startswith("print(f'Done! Processed") + or s.startswith('print("Done! Processed')): + verify_at = i + if snap_at is None or verify_at is None or verify_at <= snap_at: + raise RuntimeError( + "frozen-model guard injection failed: rollout.py anchors not found " + f"(model.eval()@{snap_at}, Done@{verify_at}) -- refusing to run unguarded") + lines.insert(verify_at, _FG_VERIFY) # insert the later one first (indices stable) + lines.insert(snap_at, _FG_SNAPSHOT) + f.write_text("".join(lines)) + + def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, steps: int, device: str = "cuda") -> float: """Run rollout.py; return generation wall-clock seconds (rollout-region timed @@ -56,6 +132,7 @@ def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, st env = dict(os.environ) env["CSGO_DATA_DIR"] = str(S.CSGO_DATA) env["NWM_TIME_FILE"] = str(save / "gen_seconds.txt") + _apply_determinism_env(env) cmd = [sys.executable, "-u", "src/sample/rollout.py", "--config", str(config), "--ckpt", str(ckpt), "--save_path", str(save), "--num_samples", str(clips), "--batch_size", str(S.BATCH_SIZE), "--rollout_length", str(S.ROLLOUT_LENGTH), @@ -76,11 +153,26 @@ def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, st return wall +def _apply_determinism_env(env: dict) -> None: + """Determinism vars for the GPU subprocesses (rollout + the LPIPS metric). + cuBLAS needs CUBLAS_WORKSPACE_CONFIG set before the first CUDA call, so it + lives in the subprocess env, not toggled in-process. Pairs with the + deterministic kernels enabled in rollout.py / evaluate_metrics.py so the + baseline and patched arms share a bit-stable pipeline (common random numbers). + NWM_DETERMINISM_STRICT=1 makes a missing deterministic kernel RAISE (proof + mode) instead of silently running nondeterministically.""" + env.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + env.setdefault("PYTHONHASHSEED", "0") + env["NWM_DETERMINISM_STRICT"] = "1" if S.STRICT_DETERMINISM else "0" + + def lpips_vs_gt(repo: Path, save: Path) -> float: out_csv = save / "metrics.csv" + env = {**os.environ, "CSGO_DATA_DIR": str(S.CSGO_DATA)} + _apply_determinism_env(env) r = subprocess.run([sys.executable, "src/sample/evaluate_metrics.py", "--video_dir", str(save), "--history_length", str(S.HISTORY_LENGTH), "--output_csv", str(out_csv)], - cwd=str(repo), env={**os.environ, "CSGO_DATA_DIR": str(S.CSGO_DATA)}, + cwd=str(repo), env=env, capture_output=True, text=True) if not out_csv.exists(): raise RuntimeError(f"metrics failed: {r.stderr[-2000:]}") @@ -88,13 +180,46 @@ def lpips_vs_gt(repo: Path, save: Path) -> float: return float(pd.read_csv(out_csv)["lpips"].mean()) +def frames_lpips(dir_a: Path, dir_b: Path, device: str = "cuda") -> float | None: + """Mean LPIPS between matching `sample_*_gen.mp4` in two rollout-output dirs -- + the FAITHFULNESS of the patched rollout to the baseline rollout (closeness of + OUTPUTS, vs the quality metric's closeness to GROUND TRUTH). Returns None if + nothing comparable is found. GPU; reuses the baked VGG weights.""" + import lpips + import numpy as np + import torch + import imageio.v2 as imageio + + def _load(p: Path) -> "torch.Tensor": + frames = [f for f in imageio.get_reader(str(p))] + v = torch.from_numpy(np.stack(frames)).permute(0, 3, 1, 2).float() / 255.0 + return (v * 2.0 - 1.0).to(device) # LPIPS expects [-1, 1] + + net = lpips.LPIPS(net="vgg").to(device).eval() + vals: list[float] = [] + for ga in sorted(Path(dir_a).glob("*_gen.mp4")): + gb = Path(dir_b) / ga.name + if not gb.exists(): + continue + a, b = _load(ga), _load(gb) + t = min(a.shape[0], b.shape[0]) + with torch.no_grad(): + for i in range(t): + vals.append(float(net(a[i:i + 1], b[i:i + 1]).mean().item())) + return (sum(vals) / len(vals)) if vals else None + + def evaluate(clean_repo: Path, patch_path: Path | None, config: Path, ckpt: Path, - clips: int, steps: int, device: str = "cuda") -> dict: + clips: int, steps: int, device: str = "cuda", keep_gen: Path | None = None) -> dict: repo = apply_patch(clean_repo, patch_path) try: save = repo / "_rollout_out" secs = run_rollout(repo, config, ckpt, save, clips, steps, device) lp = lpips_vs_gt(repo, save) + if keep_gen is not None: # stash generated frames for cross-arm faithfulness + kp = Path(keep_gen); kp.mkdir(parents=True, exist_ok=True) + for f in save.glob("*_gen.mp4"): + shutil.copy(f, kp / f.name) return {"gen_seconds": secs, "lpips": lp, "clips": clips, "steps": steps} finally: shutil.rmtree(repo.parent, ignore_errors=True) diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/scoring.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/scoring.py index 1074d65f..eb4fa197 100644 --- a/2.0/problems/nanowm_rollout_speedup/speedup_eval/scoring.py +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/scoring.py @@ -30,10 +30,29 @@ def paired_speedups(baseline_s: dict[str, float], patched_s: dict[str, float]) - return out -def score_from_speedup(speedup: float) -> float: +def score_from_speedup(speedup: float, target: float = 2.0) -> float: + """Map geomean speedup -> [0, 100], reaching 100 at `target`x. target=2.0 + reproduces the old bare 100*log2(speedup) (saturated at 2x, so everything + >=2x capped at 100 and became indistinguishable); a larger target keeps a + gradient across the achievable range.""" if speedup <= 0: return 0.0 - return max(0.0, min(100.0, 100.0 * math.log(speedup, 2))) + denom = math.log(max(target, 1.0 + 1e-9), 2) + return max(0.0, min(100.0, 100.0 * math.log(speedup, 2) / denom)) + + +def faithfulness_multiplier(faithfulness_lpips, tolerance: float) -> float: + """1.0 while the patched rollout frames stay within `tolerance` mean LPIPS of + the BASELINE rollout frames (a faithful acceleration); decays inverse- + proportionally beyond that (a different rollout substituted for the baseline). + `None` (no paired baseline frames, e.g. agent-role cached-baseline path) means + 'not measured' -> no penalty.""" + if faithfulness_lpips is None: + return 1.0 + f = max(0.0, float(faithfulness_lpips)) + if f <= tolerance: + return 1.0 + return max(0.0, min(1.0, tolerance / f)) def quality_multiplier(baseline_lpips: float, patched_lpips: float, tolerance: float) -> float: @@ -53,16 +72,23 @@ def provisional_score( baseline_lpips: float, patched_lpips: float, tolerance: float, + faithfulness_lpips=None, + faithfulness_tolerance: float = 0.10, + speedup_target: float = 2.0, ) -> dict[str, float]: speedups = paired_speedups(baseline_seconds, patched_seconds) gm = geometric_mean(speedups) if speedups else 0.0 - latency_score = score_from_speedup(gm) + latency_score = score_from_speedup(gm, target=speedup_target) qmult = quality_multiplier(baseline_lpips, patched_lpips, tolerance) + fmult = faithfulness_multiplier(faithfulness_lpips, faithfulness_tolerance) + gate = qmult * fmult return { "geomean_speedup": gm, "latency_score": latency_score, "quality_multiplier": qmult, - "score": max(0.0, min(100.0, latency_score * qmult)), - "score_unbounded": max(0.0, 100.0 * math.log(max(gm, 1e-9), 2)) * qmult, + "faithfulness_multiplier": fmult, + "faithfulness_lpips": (float(faithfulness_lpips) if faithfulness_lpips is not None else -1.0), + "score": max(0.0, min(100.0, latency_score * gate)), + "score_unbounded": max(0.0, 100.0 * math.log(max(gm, 1e-9), 2)) * gate, "clips_scored": float(len(speedups)), } diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py index 0218506d..f53b291c 100644 --- a/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/settings.py @@ -51,6 +51,22 @@ def _get(name: str, default): # Quality guardrail: patched rollout LPIPS may rise at most this (relative) # above the baseline (unpatched seq@50) LPIPS before the score is penalized. QUALITY_TOLERANCE = float(_get("FRONTIER_NWM_QUALITY_TOLERANCE", 0.03)) +# (E) Speedup at which the latency score saturates to 100. The score is +# 100*log2(speedup)/log2(target), so the OLD behaviour (bare 100*log2) was an +# implicit target of 2x -- every solution >=2x capped at 100 and became +# indistinguishable. Raised to 4x so the score keeps a gradient across the +# realistically achievable ~1-4x+ range (causal-prefix ~3x, +bf16 stacks higher). +SPEEDUP_SCORE_TARGET = float(_get("FRONTIER_NWM_SPEEDUP_TARGET", 4.0)) +# (A) Faithfulness BACKSTOP: mean LPIPS between the PATCHED and BASELINE rollout +# frames (paired, role=final). Always REPORTED (faithfulness_lpips in metrics); a +# penalty only kicks in past this generous threshold, so it catches an egregious +# rollout SUBSTITUTION (a different sampler dressed up as a speedup) without +# rejecting legitimate iso-quality optimizations. Calibrated on H100: the bf16 +# reference drifts 0.206 LPIPS from the fp32 baseline over the 50-frame +# autoregressive rollout (still iso-quality vs GT, but a different trajectory), so +# 0.10 would wrongly punish it -- 0.30 clears bf16 with margin while still flagging +# wildly substituted rollouts (~half-divergent). codex's causal-prefix is ~0. +FAITHFULNESS_TOLERANCE = float(_get("FRONTIER_NWM_FAITHFULNESS_TOL", 0.30)) # Image-asset layout (baked into the images; overridable locally). REPO = Path(_get("FRONTIER_NWM_REPO", "/opt/nanowm/nano-world-model")) CKPT = Path(_get("FRONTIER_NWM_CKPT", "/opt/nanowm/ckpts/nanowm-l2-csgo/model_state_dict.pt")) @@ -59,3 +75,30 @@ def _get(name: str, default): VAL_STARTS = Path(_get("FRONTIER_NWM_VAL_STARTS", "/opt/nanowm/data/csgo_subset/val_starts.npy")) BASELINE_CACHE = Path(_get("FRONTIER_NWM_BASELINE_CACHE", "/opt/nanowm/baseline/baseline_metrics.json")) SMOKE = os.environ.get("FRONTIER_NWM_SMOKE", "0") == "1" +# Determinism proof switch (judge infra). 0 (default): a missing deterministic +# kernel WARNS and runs nondeterministically. 1: it RAISES instead -- run once on +# H100 to PROVE the model+VAE+metric have no nondeterministic op (and re-establish +# no-op CRN pair -> 0.0 there); production may then stay strict (a fallback -> an +# infra error, never a silently corrupted score) or relax once proven clean. +def _as_bool(v) -> bool: + if isinstance(v, str): + return v.strip().lower() in ("1", "true", "yes", "on") + return bool(v) +STRICT_DETERMINISM = _as_bool(_get("FRONTIER_NWM_STRICT_DETERMINISM", 0)) + + +def config_fingerprint() -> str: + """Stable hash of every knob that affects the unpatched baseline. Used to key + the baseline cache and the composed-config filename so a cached baseline is + only ever reused as a common-random-numbers partner for the IDENTICAL config + (changing any of these invalidates the cache instead of silently mispairing).""" + import hashlib + keys = { + "model": MODEL, "dataset": DATASET, "rollout_length": ROLLOUT_LENGTH, + "history_length": HISTORY_LENGTH, "num_steps": NUM_SAMPLING_STEPS, + "scheduling": SCHEDULING_MODE, "history_stab": HISTORY_STAB, + "batch_size": BATCH_SIZE, "seed": SEED, + "val_files": str(VAL_FILES), "val_starts": str(VAL_STARTS), + } + blob = json.dumps(keys, sort_keys=True) + return hashlib.sha256(blob.encode()).hexdigest()[:16] diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/test_frozen_model_guard.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/test_frozen_model_guard.py new file mode 100644 index 00000000..0f0c8285 --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/test_frozen_model_guard.py @@ -0,0 +1,128 @@ +"""Unit test for the frozen-model guard (speedup temp_embed gray area). + +Runs WITHOUT torch (a tiny pure-Python tensor stub) so it is CI-checkable on the +CPU judge; if torch is present it additionally checks a real nn.Module. Verifies: + 1. no mutation -> fingerprint unchanged (assert_frozen passes) + 2. persistent value mutation -> CAUGHT (assert_frozen raises) + 3. transient reshape THEN RESTORE -> passes (the causal-prefix temp_embed case) + 4. a different-but-sum-preserving... -> documented residual (NOT a goal; quality gate) + +Run: python -m speedup_eval.test_frozen_model_guard (exit non-zero on regression) +""" +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from speedup_eval.frozen_model_guard import model_param_fingerprint, assert_frozen # noqa: E402 + + +class _Stub: + """Minimal tensor stub: a value list with .shape/.dtype/.detach()/.sum().""" + def __init__(self, vals, shape=None, dtype="float32"): + self.vals = list(vals) + self.shape = tuple(shape) if shape is not None else (len(self.vals),) + self.dtype = dtype + + def detach(self): + return self + + def to(self, *a, **k): + return self + + def double(self): + return self + + def sum(self): + return sum(self.vals) + + +class _Model: + def __init__(self, sd): + self._sd = sd + + def state_dict(self): + return self._sd + + +def main() -> int: + ok = True + + def check(cond, msg): + nonlocal ok + if not cond: + print("FAIL:", msg); ok = False + + # Build a frozen model and snapshot it. + sd = { + "blocks.0.weight": _Stub([0.1, 0.2, 0.3], shape=(3,)), + "temp_embed": _Stub([1.0, 2.0, 3.0, 4.0], shape=(1, 4, 1)), + "pos_embed": _Stub([0.5, 0.5], shape=(1, 2, 1)), + } + model = _Model(sd) + fp0 = model_param_fingerprint(model) + + # 1) No mutation -> passes. + try: + assert_frozen(model, fp0) + except RuntimeError: + check(False, "no-op model wrongly flagged as mutated") + + # 2) Persistent value mutation -> caught. + sd["blocks.0.weight"] = _Stub([0.1, 0.2, 0.9], shape=(3,)) # changed a value + caught = False + try: + assert_frozen(model, fp0) + except RuntimeError: + caught = True + check(caught, "persistent weight mutation NOT caught") + sd["blocks.0.weight"] = _Stub([0.1, 0.2, 0.3], shape=(3,)) # restore for next case + + # 3) Transient reshape THEN restore (the causal-prefix temp_embed pattern): + # slice temp_embed during the rollout, then put the original back -> passes. + original = sd["temp_embed"] + sd["temp_embed"] = _Stub(original.vals[:2], shape=(1, 2, 1)) # sliced prefix (transient) + # ... (model used on the cropped window here) ... + sd["temp_embed"] = original # restored in finally + try: + assert_frozen(model, fp0) + except RuntimeError: + check(False, "restored transient reshape wrongly flagged (would block causal-prefix)") + + # 4) Shape change alone (e.g. left sliced, not restored) -> caught (structure hash). + sd["temp_embed"] = _Stub(original.vals[:2], shape=(1, 2, 1)) + caught = False + try: + assert_frozen(model, fp0) + except RuntimeError: + caught = True + check(caught, "unrestored temp_embed reshape NOT caught") + sd["temp_embed"] = original + + # Optional: same checks against a real torch nn.Module if torch is installed. + try: + import torch # type: ignore + import torch.nn as nn # type: ignore + m = nn.Linear(4, 4) + f0 = model_param_fingerprint(m) + assert_frozen(m, f0) # no-op passes + with torch.no_grad(): + m.weight[0, 0] += 1.0 # persistent mutation + hit = False + try: + assert_frozen(m, f0) + except RuntimeError: + hit = True + check(hit, "torch nn.Module persistent mutation NOT caught") + print("torch path: checked a real nn.Linear") + except ImportError: + print("torch not installed -> stub-only check (CI path)") + + print("RESULT:", "PASS - frozen-model guard catches persistent mutation, allows restored reshape" + if ok else "FAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/2.0/problems/nanowm_rollout_speedup/speedup_eval/test_inject_frozen_guard.py b/2.0/problems/nanowm_rollout_speedup/speedup_eval/test_inject_frozen_guard.py new file mode 100644 index 00000000..bdbac8ab --- /dev/null +++ b/2.0/problems/nanowm_rollout_speedup/speedup_eval/test_inject_frozen_guard.py @@ -0,0 +1,132 @@ +"""Test runner.inject_frozen_guard: it must place a working frozen-model guard into +a copied rollout.py (snapshot after model.eval(), verify before the final print), +catch a persistent model mutation, allow a restored transient reshape (causal-prefix), +fail CLOSED when anchors are missing, and be idempotent. Runs without torch/GPU by +injecting into a synthetic rollout.py with a fake torch + stub model and exec-ing it. + +Run: python -m speedup_eval.test_inject_frozen_guard (exit non-zero on regression) +""" +from __future__ import annotations + +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from speedup_eval import runner # noqa: E402 + + +# A synthetic rollout.py: the loaded `model` is eval'd, the SAMPLER runs (and may +# mutate the model), then the final "Done!" print. {SAMPLER} is filled per-case. +SYNTH = ''' +import torch +def main(): + model = MODEL + model.eval() + for _b in range(2): + SAMPLER(model) + print(f"Done! Processed {{N}} samples.") +main() +''' + + +class _Stub: + def __init__(self, vals, shape=None, dtype="float32"): + self.vals = list(vals); self.shape = tuple(shape or (len(self.vals),)); self.dtype = dtype + def detach(self): return self + def to(self, *a, **k): return self + def sum(self): return sum(self.vals) + + +class _Model: + def __init__(self): self._sd = {"w": _Stub([0.1, 0.2, 0.3]), "temp_embed": _Stub([1.0, 2.0, 3.0, 4.0], (1, 4, 1))} + def state_dict(self): return self._sd + def eval(self): return self + + +class _FakeTorch: + float64 = "float64" + + +def _write_synth(tmp: Path) -> Path: + f = tmp / "src" / "sample" / "rollout.py" + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text(SYNTH) + return f + + +def _run_injected(repo: Path, sampler) -> tuple[bool, str]: + """Inject, then exec the resulting rollout.py with a fake torch/model/sampler. + Returns (raised, message).""" + runner.inject_frozen_guard(repo) + src = (repo / "src" / "sample" / "rollout.py").read_text() + compile(src, "rollout.py", "exec") # syntactic validity + g = {"MODEL": _Model(), "SAMPLER": sampler, "N": 2, "torch": _FakeTorch(), + "print": lambda *a, **k: None} + # shadow `import torch` inside the synthetic module with our fake + sys.modules.setdefault("torch", _FakeTorch()) + try: + exec(compile(src, "rollout.py", "exec"), g) + return False, "ok" + except RuntimeError as e: + return True, str(e) + + +def main() -> int: + ok = True + + def check(cond, msg): + nonlocal ok + if not cond: + print("FAIL:", msg); ok = False + + # 1) no-op sampler -> guard passes (no raise). + with tempfile.TemporaryDirectory() as d: + repo = Path(d); _write_synth(repo) + raised, msg = _run_injected(repo, lambda m: None) + check(not raised, f"no-op rollout wrongly flagged: {msg}") + + # 2) restored transient reshape (causal-prefix temp_embed) -> passes. + def transient(m): + sd = m.state_dict(); orig = sd["temp_embed"] + sd["temp_embed"] = _Stub(orig.vals[:2], (1, 2, 1)) # sliced during forward + sd["temp_embed"] = orig # restored in finally + with tempfile.TemporaryDirectory() as d: + repo = Path(d); _write_synth(repo) + raised, msg = _run_injected(repo, transient) + check(not raised, f"restored transient reshape wrongly flagged (blocks causal-prefix): {msg}") + + # 3) persistent mutation -> CAUGHT (raise). + def persistent(m): + m.state_dict()["w"] = _Stub([0.1, 0.2, 0.9]) # never restored + with tempfile.TemporaryDirectory() as d: + repo = Path(d); _write_synth(repo) + raised, msg = _run_injected(repo, persistent) + check(raised and "frozen-model violation" in msg, f"persistent mutation NOT caught: {msg}") + + # 4) idempotent: a second injection does not double-insert. + with tempfile.TemporaryDirectory() as d: + repo = Path(d); f = _write_synth(repo) + runner.inject_frozen_guard(repo) + once = f.read_text() + runner.inject_frozen_guard(repo) + check(f.read_text() == once and once.count("_fg_frozen = _fg_fp(model)") == 1, "injection not idempotent") + + # 5) FAIL CLOSED: missing anchor (no model.eval()) -> raise, do not run unguarded. + with tempfile.TemporaryDirectory() as d: + repo = Path(d); f = _write_synth(repo) + f.write_text('def main():\n x = 1\n print(f"Done! Processed 0 samples.")\nmain()\n') + caught = False + try: + runner.inject_frozen_guard(repo) + except RuntimeError as e: + caught = "anchors not found" in str(e) + check(caught, "missing-anchor injection did NOT fail closed") + + print("RESULT:", "PASS - frozen guard injects, catches persistent mutation, allows " + "restored reshape, fails closed" if ok else "FAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/2.0/problems/nanowm_rollout_stability/AUDIT_REPORT.md b/2.0/problems/nanowm_rollout_stability/AUDIT_REPORT.md index 887f7498..bc467320 100644 --- a/2.0/problems/nanowm_rollout_stability/AUDIT_REPORT.md +++ b/2.0/problems/nanowm_rollout_stability/AUDIT_REPORT.md @@ -40,7 +40,7 @@ The remaining real items are **documentation/robustness hygiene** (low): three-w - Speedup reference reported as `1.17×/22.4` (`DESIGN.md:78-79`, `PR_SUMMARY.md:14`, `CALIBRATION_FINDINGS.md:253`) vs `1.15×/20.1` (`CALIBRATION_FINDINGS.md:273`) — **two distinct 4-clip runs with different baselines** (299.95s vs 291.36s = 2.95% swing) quoted interchangeably. Cross-pairing the two baselines moves the score **4.2 pts** (18.3 ↔ 22.4), and only the `speedup_val` patched (256.7s/0.521) has a JSON; the `253s/0.535` run has no on-disk patched artifact. - `DESIGN.md` mtime is **after** the e2e runs yet still carries the older val numbers — i.e. docs were not reconciled to the later runs. - **Why it matters:** (e) docs should not over-claim. A reviewer cannot tell which baseline/score is canonical; the most-cited figures are the most flattering (slower baseline / higher reduction). -- **Fix:** pick **one** canonical config per task (the production `final_clips`: stability 24, speedup 16), run it once paired, and report that single number with its CI across `DESIGN.md` / `PR_SUMMARY.md` / `CALIBRATION_FINDINGS.md`. Restate stability as the artifact-backed range **3.6–7.2% (≈5% typical)**, not a bare 7.2. +- **Fix:** pick **one** canonical config per task (the production `final_clips`: stability **22**, speedup 16), run it once paired, and report that single number with its CI across `DESIGN.md` / `PR_SUMMARY.md` / `CALIBRATION_FINDINGS.md`. Restate stability as the artifact-backed range **3.6–7.2% (≈5% typical)**, not a bare 7.2. (The old `7.2`/`24-clip` log predates the shipped asset set: the held-out subset bound to the staged 1-200 data chunk is exactly the 22 test_split episodes ≤200, so `final_clips` is capped at 22 — `24` indexed past the sliced dataset and could only have run on a fuller local data tree.) ### P3 — Speedup timer is dead code; score is full-process wall-clock, and the cached baseline is a single un-repeated run `LOW` - **Evidence:** @@ -86,7 +86,7 @@ The remaining real items are **documentation/robustness hygiene** (low): three-w ## Recommended Fixes (highest-leverage first) 1. **Seed the rollout + measure baseline paired in-job (P1).** The single highest-leverage change: `torch.manual_seed(clip_id)` before the initial `th.randn`, and replace the cached baseline with a baseline computed on the same clips/seed in the same job (common random numbers). Removes the gameability (no-op → ~0) and shrinks the metric noise that contaminates both references' margins. -2. **Reconcile the docs to one canonical number per task (P2).** Run the production `final_clips` config once (stability 24, speedup 16), report that single value + CI everywhere; restate stability as 3.6–7.2% (≈5% typical) and pick one speedup baseline. +2. **Reconcile the docs to one canonical number per task (P2).** Run the production `final_clips` config once (stability 22, speedup 16), report that single value + CI everywhere; restate stability as 3.6–7.2% (≈5% typical) and pick one speedup baseline. 3. **Fix the timer + baseline measurement (P3).** Either implement the per-region sampling timer (so the metric isolates the patchable region) or delete the dead `NWM_TIME_FILE` path and correct the docstring; measure the baseline as median-of-N (or paired per #1). 4. **Report multi-seed + Wilcoxon for stability and soften "reliably" (P4).** ≥3 seeds, Wilcoxon alongside the t-test; word the claim to the α-edge it actually is. 5. **Require `== clips` on the cached baseline and label the LPIPS tables (P5, P6).** Equal nested clip sets; one-line "(all frames, judge convention)" note on the LPIPS-vs-GT tables. @@ -121,3 +121,141 @@ The fix lives in judge infrastructure (`rollout.py` + `*_eval/`, regenerated int `infra_patches/0001-rollout-judge-infra.patch`) — **outside the agent's editable sampling scope**, and the shared 2.0 adapter/template is untouched. Standing conclusions C1–C6 are unaffected (and C3/C5's residual P1 caveat is now closed). + +--- + +## Update — P1 actually closed; H100; reviewer-honesty note (2026-06-21, branch `fix/nanowm-h100-crn`) + +A re-review found the 2026-06-14 "P1 fixed" claim above was only **half** true: +deterministic per-clip seeding was added, but `orchestrate._baseline()` still +**served a cached baseline** rather than recomputing it paired in-job, and the +infra patch enforced **no GPU determinism** (`use_deterministic_algorithms`, +TF32-off, `CUBLAS_WORKSPACE_CONFIG` were all absent — and upstream `rollout.py` +turns TF32 *on* at import). Seeding fixes the initial noise but not the +arithmetic, so "the cached baseline is a valid CRN partner" only held on the +H100 single node where determinism happened to apply — not the (then L40S, never +run e2e) production path. P1 was therefore not closed for the scored path. + +Now actually closed: +- **Determinism enforced** in the infra patch (`use_deterministic_algorithms(True, + warn_only=True)`, `cudnn.deterministic`, `benchmark=False`, TF32 disabled) + + `CUBLAS_WORKSPACE_CONFIG=:4096:8` exported by the launcher before CUDA init. +- **Scored (final) run measures baseline + patched as a true CRN pair** in one + job/GPU/process (`run_pair(role="final")` → `modal_app.run_pair_remote`), no + cache. Cached baseline kept only for cheap iterative feedback, now keyed by a + full **config fingerprint** (incl. `drift_tail_start`). +- **GPU pinned to H100** (was L40S) so the production scoring path and every + calibrated number in this report share one SKU. + +**Reviewer-honesty note (closes audit risk (e), over-claim):** the calibration +artifacts this report cites as evidence — `CALIBRATION_FINDINGS.md`, `outputs/`, +`calib/…sbatch`, job numbers, `PR_SUMMARY.md` — **are not shipped in this repo** +(they live in the author's working tree), so the pre-fix numbers above are +**not independently reproducible from the PR** and should be read as provisional. +The code-level fixes here are CPU-verified (policy/scoring/smoke; no-op CRN pair → +exactly 0.0; reference reproduces 1.17×/22.3 and ~6.8); **GPU/Modal end-to-end on +H100 is still pending maintainer credentials** and must re-establish the no-op≈0 +and residual-noise numbers on the production path before the headline margins are +final. + +--- + +## Update — #7 tail-targeting reward-hack closed; #6 scoped (2026-06-25, branch `fix/nanowm-h100-crn`) + +A post-mortem of the codex Harbor trials surfaced a concrete reward-hack that the +earlier audit had not (the scored run had not yet been exercised by a real agent): + +### #7 — tail window targetable via a hardcoded module counter `HIGH (closed)` +- **The hack (as shipped, codex 7.83% run).** The winning stability submission added + a **module-global call counter** in `df_sample` and ramped EXTRA history-stabilization + onto the rollout's late frames: + `_STABILITY_ROLLOUT_PERIOD = 76` (== `ROLLOUT_LENGTH 80 − HISTORY_LENGTH 4`), + `rollout_phase = _stability_rollout_call % 76`, + `extra = 0.08·clamp((rollout_phase−32)/28)`. `df_sample` is called once per generated + frame, so at the **disclosed** nominal horizon 80 the counter aligns to the + generated-frame index and the ramp **saturates exactly at frame 64**, i.e. squarely + on the scored tail (`DRIFT_TAIL_START = 60`). It contributed ≈ +1 pp over the flat-scalar + reference (7.83 vs ≈6.8), earned by dumping the intervention on the scored window + rather than reducing drift generally. Slips the static policy (a module counter is + not a forbidden token). +- **Root enabler.** `df_sample` (the agent's editable scope) never receives + `rollout_length`, so a submission can only HARDCODE the horizon — which it learned + from the **README ("80-frame rollout")** and could measure during the agent-role QUICK + loop. +- **Fix (this branch).** The SCORED (role=final) horizon is now **drawn at random per run** + from `[ROLLOUT_LENGTH_MIN, ROLLOUT_LENGTH_MAX] = [64, 72]` (well below the + agent-measurable nominal 80), decided ONCE on the judge and threaded to every arm / + fan-out chunk (`settings.draw_final_rollout_length`; `orchestrate.run_pair`; + `runner.evaluate(rollout_length=…)`; `modal_app` spec). The scored tail is derived + from it (`tail_start = horizon − TAIL_FRAMES`). CRN pairing and chunk bit-identity are + preserved (both arms + all chunks share one horizon; the per-batch global-index seed is + unchanged; `MAX ≤ nominal` so GT headroom and the 22-clip set are unchanged). The + README/DESIGN no longer disclose the horizon or tail index. +- **Verification (no GPU; `stability_eval/test_antihack_horizon.py`).** Replaying the + exact shipped counter, its **expected** tail-targeting over the `[64,72]` band collapses + from +0.064 (100% at the disclosed horizon 80) to **+0.00002 (≈0%)** — and is *negative* + for horizons ≤68, because the fixed period-76/frame-64 ramp misfires onto non-tail frames + at the shorter horizons. So the concrete shipped overfit is neutralized. +- **What deliberately survives (not a hack).** A **horizon-robust** "stabilize deeper + frames more" schedule (keyed on a runtime signal — per-clip call index reset at the RNG + reseed, or context staleness — rather than a hardcoded period) can still raise tail + quality (~70% of the no-defense ceiling in the diagnostic). That is the **legitimate** + behaviour the hack imitated: it responds to genuine accumulated drift, generalizes across + horizons/content, and faithfully cleans the tail via less-contaminated context (LPIPS-vs-GT + rewards real fidelity, not cosmetic blur — `sampling_utils.py`, the decode/save plumbing, + is already DENIED). Randomization converts the overfit into *either-legitimate-or-broken*, + which is the goal. **Caveat:** lowering the horizon moves the tail earlier than the + calibrated `tail≥60`; the relative drift metric is ~horizon-robust but the band's SNR must + be re-confirmed on H100. `MIN/MAX` are env-tunable (widen for stronger anti-hack, raise + toward 80 for the calibrated SNR). +- **Re-verification note (the "partly" survivor is not a practical hack).** A second audit + suggested a counter with a period grid-searched to `[64,72]` (≈58-62) recovers ~5% of the + targeting ceiling. Two reasons this is not a meaningful reward-hack: (1) **the band is NOT + disclosed or observable.** `[64,72]`/`MIN/MAX` live ONLY in the judge-internal `config.yaml` + `evaluation` block (read via `task_config.json` on the judge); they are absent from + `harbor/` (the agent workspace) and the agent-facing `environment` string, and the + agent-role QUICK loop runs at the nominal 80 — so the agent can only calibrate to 80 (period + 76), which collapses to ≈0 on the real band. The grid-search premise ("calibrate on the + disclosed band during dev") is false. (2) Even granting a leaked band, the residual is ~5% + of ceiling and **weaker than the flat-`0.20` reference**, with a mechanism (more stabilization + at greater frame depth) that responds to genuine accumulated drift — i.e. the legitimate + survivor, not a metric exploit. **Control:** keep the band judge-internal (never ship the + `evaluation` block to the agent); for belt-and-suspenders the bounds can be jittered per run + via the nonce. No code change needed; documented as the accepted residual. + +### #6 — task collapses to tuning the `history_stabilization_level` scalar `MEDIUM (scoped, not a reward-hack)` +- The dominant lever is a single exposed scalar: the reference IS `stab→0.20` and yields + ≈6.8% by itself; codex's per-frame age-profile + the (now-defeated) counter added only + ~1 pp. This is a **depth/ceiling** limitation (the headline difficulty is shallow), **not** + a reward-hack — bumping the exposed knob is the intended lever, just a shallow use of it. +- The #7 randomization partially helps (it forces any position-dependent gain to be earned + *generally*, not on a fixed window), but does not raise the ceiling. **A real fix is a + recalibrated redesign** (gated on an H100 sweep, so deferred here): set the *baseline* + `history_stab` to the best-constant value found by a sweep so that merely raising the + scalar yields ≈0, and ship a NEW reference that is a genuinely better *schedule/mechanism* + (e.g. drift-keyed adaptive re-anchoring). Until that sweep, the task remains solvable-but- + shallow; documented here rather than silently shipped as "hard." + +### Cross-task hardening shipped alongside #7 (frozen-model guard + faithfulness fail-closed) +A reward-hack audit of the FIXED branch (4 parallel auditors + synthesis) confirmed the +#7 fix is CLEAN with zero regressions (CRN pairing, chunk bit-identity, cached-baseline +validity all preserved) and the scoring math of BOTH tasks is clean (only cosmetic +notes). It surfaced one cross-cutting gap — the model-frozen invariant was not +structurally enforced (the speedup causal-prefix `temp_embed` gray area): a sampler +patch could persistently mutate the frozen model and the static policy can't see a +runtime monkeypatch. Closed in BOTH tasks: +- **Frozen-model guard, actively wired:** `{speedup,stability}_eval.runner.inject_frozen_guard()` + inlines a param fingerprint into the copied `rollout.py` at apply-time (after the + agent patch; rollout.py is denied to the agent) — snapshot after `model.eval()`, + verify before the final `print`. A restored transient reshape (causal-prefix) passes; + a persistently-mutated model hard-errors → scored 0. Fail-closed if the anchors move. + Unit-tested (`test_inject_frozen_guard.py`, `speedup_eval/test_frozen_model_guard.py`). + Matters more for stability (no faithfulness backstop) than speedup. +- **Speedup faithfulness fail-closed:** `evaluator.py` now returns 0 (not + `faithfulness_mult=1.0`) when `role=="final"` and `faithfulness_lpips` is None, so the + active model-mutation defense (patched-vs-baseline frame LPIPS) can't be silently + disabled. +- Cosmetic: corrected the wallclock-multiplier comment (≈0.556 at +tol, 0.5 at + grace+tol). Net: no exploitable reward-hack or scoring overfit remains on the + authoritative scored path of either task (remaining items are #6 depth + the + acknowledged sum-preserving-fingerprint residual, both backstopped). diff --git a/2.0/problems/nanowm_rollout_stability/DESIGN.md b/2.0/problems/nanowm_rollout_stability/DESIGN.md index debd0e48..84f03640 100644 --- a/2.0/problems/nanowm_rollout_stability/DESIGN.md +++ b/2.0/problems/nanowm_rollout_stability/DESIGN.md @@ -40,8 +40,10 @@ compute*: stabilization, scheduling-matrix design, drift-aware caching that free time for re-grounding, periodic context re-anchoring, error correction, solvers. ## 4. Patch policy / GPU / scoring -Identical to nanowm_rollout_speedup: Python-only allowlist (`src/diffusion/**`, -`src/sample/sampling_utils.py`), deny model/VAE/metric/harness/data; Modal GPU +Identical to nanowm_rollout_speedup: Python-only allowlist (`src/diffusion/**` +only) — `src/sample/sampling_utils.py` is DENIED (it decodes+saves the generated +frames the LPIPS metric reads, so editing it could blur the scored pixels); +deny model/VAE/metric/harness/data; Modal GPU from a CPU judge (`stability_eval/{runner,orchestrate,modal_app}.py`); smoke score 1.0 when GPU/Modal unconfigured (local CI). Score = `clip(100·(base_tail−patched_tail)/base_tail,0,100)·wallclock_mult`. @@ -58,3 +60,51 @@ p<1e-4; a no-op patch scores 0.000). Patch policy + smoke pass. Modal end-to-end run; bake-asset provenance (ckpt + held-out CSGO subset + cached baseline); confirm final_clips gives stable scoring on the judge's hidden set (the small effect needs ~20+ clips). + +## 7. Update — H100 + CRN hardening (this branch) + +All in judge infra (`infra_patches/0001-…`, `stability_eval/`); the agent-editable +sampling scope and the shared 2.0 template are untouched. This matters more here +than for speedup: the drift effect (~6.8%) is the same order as the run-to-run +noise the audit measured, so a noisy baseline is more likely to swamp the signal. + +- **GPU is now H100, not L40S** (`config.yaml`, `modal_app.py`) — production + scoring and the calibrated numbers share one SKU. +- **Determinism enforced** (deterministic kernels + TF32 off in `rollout.py`, + `CUBLAS_WORKSPACE_CONFIG` exported by the launcher) so per-clip seeding yields a + bit-stable pipeline, not just identical initial noise. +- **Scored (final) run pairs baseline + patched in one job/GPU/process** (true + CRN, no cache); the cached path is iterative-only and keyed by a full config + fingerprint (incl. `drift_tail_start`). +- Also fixed the `stability_eval/settings.py` docstring (was "speedup") and the + `modal_app` app-name default (was `nanowm-rollout-speedup`). + +**Validation status:** scoring/smoke re-verified on CPU (no-op CRN pair → 0.0; +reference reproduces ~6.8; `score_unbounded == score` by construction for a +relative reduction). **GPU/Modal end-to-end on H100 pending maintainer +credentials** — the multi-seed drift numbers must be re-measured on the +production path before being treated as final. + +## 8. Update — #7 tail-targeting hack closed (randomized scored horizon, this branch) + +The codex 7.83% submission added a module-global call counter (`% 76`, == +`ROLLOUT_LENGTH − HISTORY_LENGTH`) and ramped extra history-stabilization onto the +late frames, saturating exactly at frame 64 == the scored tail (`drift_tail_start +60`). `df_sample` never receives the horizon, so the only way it could target the +tail was to HARDCODE the disclosed length (the README said "80-frame rollout"). + +Fix: the **scored horizon is drawn at random per run** from `[64, 72]` (well below +the agent-measurable nominal 80), decided once on the judge and threaded to both CRN +arms and every fan-out chunk; the scored tail is derived from it (`tail = horizon − +TAIL_FRAMES`). CRN + chunk bit-identity preserved (one horizon shared; per-batch +global-index seed unchanged; `MAX ≤ nominal` keeps GT headroom and the 22-clip set). +The README/DESIGN no longer disclose the horizon or the tail index. A no-GPU proof +(`stability_eval/test_antihack_horizon.py`) shows the shipped counter's expected +tail-targeting collapses from 100% (at the disclosed 80) to ≈0% over the band (even +negative at the shorter horizons — its fixed ramp misfires). What survives is a +**horizon-robust** "stabilize deeper frames more" schedule, which is the legitimate +adaptive-stabilization behaviour the hack imitated (responds to real accumulated +drift, generalizes, faithfully cleans the tail). The single-scalar shallowness (#6) +is a separate depth limitation, not a reward-hack — see AUDIT_REPORT for the +H100-sweep-gated redesign. **Pending:** re-confirm the band's SNR on H100 (the +calibrated effect was measured at `tail≥60`); `MIN/MAX` are env-tunable. diff --git a/2.0/problems/nanowm_rollout_stability/config.yaml b/2.0/problems/nanowm_rollout_stability/config.yaml index 7640ea84..3cc19dca 100644 --- a/2.0/problems/nanowm_rollout_stability/config.yaml +++ b/2.0/problems/nanowm_rollout_stability/config.yaml @@ -5,10 +5,16 @@ runtime: # standard (mirrors vllm_llm_serving_optimization, #145); there is no separate # "patch" language in the framework. language: python - timeout_seconds: 21600 + # 12h. The scored final is a 22->12-clip baseline+patched PAIR of 80-frame + # rollouts under strict determinism (TF32 off ~3x slower): ~5-7h on H100. The + # old 6h verifier timeout was SHORTER than the final run, so the verifier raised + # VerifierTimeoutError -> reward 0 even though the agent submissions scored fine. + # Matches the Modal _rollout_pair function timeout (43200s). + timeout_seconds: 43200 environment: >- Python-only patch against a clean NanoWM checkout (Nano World Models, - arXiv:2605.23993); Modal GPU runs the NanoWM-L/2 CSGO 80-frame long-rollout; + arXiv:2605.23993); Modal GPU runs a NanoWM-L/2 CSGO long-horizon rollout (the + exact length and scored tail are fixed by the judge and not disclosed); minimize long-horizon drift (tail-frame LPIPS) at iso-wall-clock apt_packages: [bash, ca-certificates, curl, git, python3, python3-pip] judge_apt_packages: [bash, ca-certificates, curl, git, python3, python3-pip] @@ -22,18 +28,30 @@ environment: storage_mb: 32768 build_timeout_seconds: 5400 evaluation: + # H100 matches the hardware the reference + noise floor were calibrated on, so + # the production scoring path and the validated numbers share one GPU SKU. model: nanowm_l2_csgo dataset: game/csgo - gpu: L40S + gpu: H100 # LONG rollout so error accumulates into a drifted tail; FIXED steps + a # wall-clock budget => the agent improves the rollout PROCEDURE at iso-compute # (stabilization / scheduling / drift-aware caching), not by adding steps. - rollout_length: 80 + rollout_length: 80 # NOMINAL: agent-role QUICK loop + cache fingerprint history_length: 4 num_steps: 50 # fixed compute budget scheduling: sequential history_stab: 0.02 # baseline default (repo long_rollout setting) - drift_tail_start: 60 # drift = mean LPIPS over frames >= this + drift_tail_start: 60 # NOMINAL tail (cached agent path); scored tail derives from the randomized horizon + # Anti-overfit (audit #7): the SCORED (role=final) horizon is drawn at random per + # run from [rollout_length_min, rollout_length_max] (MAX < nominal so the agent's + # dev-measured horizon never scores, and GT headroom/clip-count are unchanged), and + # the scored tail = horizon - tail_frames. This neutralizes the codex module-counter + # tail-targeting hack (its period 76 / frame-64 ramp misfire off the tail at <=72; + # see stability_eval/test_antihack_horizon.py). Tune to trade anti-hack margin vs + # SNR (lower max = stronger anti-hack; raise toward 80 = closer to calibrated tail>=60). + rollout_length_min: 64 + rollout_length_max: 72 + tail_frames: 20 # Wall-clock guardrail: patched gen time may rise at most this over baseline, # else drift is being bought with compute (the speedup task's axis). wallclock_tolerance: 0.10 @@ -41,9 +59,21 @@ evaluation: # (validated under common-random-numbers pairing: stab=0.20 reference beats # baseline; 74% per-clip win, pooled paired t=5.15, p<1e-4 across 3 seeds x 22 clips). quick_clips: 8 - final_clips: 24 + # Full held-out set = the 22 test_split episodes number<=200 staged from the + # 1-200 chunk (>22 indexes past the sliced dataset and crashes). The scored final + # uses all 22 for SNR (validated headline). The 80-frame paired rollout is ~10h + # sequentially under strict determinism, so the judge FANS the clips out across + # Modal containers (chunk_size each) -- bit-identical to the sequential run since + # the per-batch seed keys on the global clip index -- finishing in ~one chunk's + # wall-time. batch_size=2 => QUICK(8) is a noise-identical prefix of FINAL(22). + final_clips: 22 batch_size: 2 - baseline_cache_path: /opt/nanowm/baseline/stability_baseline.json + # Clips per Modal container in the fanned-out scored pair (rounded up to a + # multiple of batch_size for global batch alignment). 22/4 => 6 parallel chunks. + chunk_size: 4 + # Key MUST be `baseline_cache` (settings.py strips the FRONTIER_NWM_ prefix and + # looks up `baseline_cache`); `baseline_cache_path` was silently ignored. + baseline_cache: /opt/nanowm/baseline/stability_baseline.json submission: kind: file path: /app/solution.patch diff --git a/2.0/problems/nanowm_rollout_stability/docker/agent/Dockerfile b/2.0/problems/nanowm_rollout_stability/docker/agent/Dockerfile index c4de13c8..d2f8f4df 100644 --- a/2.0/problems/nanowm_rollout_stability/docker/agent/Dockerfile +++ b/2.0/problems/nanowm_rollout_stability/docker/agent/Dockerfile @@ -25,8 +25,16 @@ RUN git clone https://github.com/simchowitzlabpublic/nano-world-model /app/nano- COPY task_ctx/infra_patches/ /tmp/infra_patches/ RUN cd /app/nano-world-model && for p in /tmp/infra_patches/*.patch; do \ [ -e "$p" ] && git apply "$p" || true; done && \ + { grep -q 'clip_offset' src/sample/rollout.py \ + && grep -q 'use_deterministic_algorithms' src/sample/evaluate_metrics.py; } \ + || { echo 'FATAL: infra patch did not apply (clip_offset / determinism markers missing)'; exit 1; } && \ git add -A && git -c user.email=t@e -c user.name=t commit -q -m base || true COPY task_ctx/harbor_app/ /app/ -COPY task_ctx/task_pkg/ /app/task/ +# NOTE: the eval package (task_pkg/stability_eval + evaluator.py) is JUDGE-ONLY and is +# deliberately NOT shipped to the agent. It carries settings.py (the scored rollout +# horizon band, seed, val-file paths, drift-tail index) and the scoring/orchestration +# source; baking it into /app/task let the agent read the judge internals directly +# (e.g. the exact rollout length / tail window). The judge image keeps it under +# /opt/nanowm/task. The agent validates patches by submitting and reading judge feedback. RUN chmod +x /app/*.sh 2>/dev/null || true diff --git a/2.0/problems/nanowm_rollout_stability/docker/build_images.sh b/2.0/problems/nanowm_rollout_stability/docker/build_images.sh index 040ff0aa..8bfa505d 100755 --- a/2.0/problems/nanowm_rollout_stability/docker/build_images.sh +++ b/2.0/problems/nanowm_rollout_stability/docker/build_images.sh @@ -1,26 +1,43 @@ #!/usr/bin/env bash # Build the nanowm_rollout_stability agent + judge images. # bash 2.0/problems/nanowm_rollout_stability/docker/build_images.sh [tag] -# Requires the hidden assets staged under $NWM_ASSETS (must be set; no default): -# $NWM_ASSETS/ckpts/nanowm-l2-csgo-100k/ (L/2 CSGO checkpoint dir -> baked as ckpts/nanowm-l2-csgo) +# Judge-only hidden assets are AUTO-STAGED from public sources (no manual step): +# $NWM_ASSETS/ckpts/nanowm-l2-csgo-100k/ (L/2 CSGO ckpt -> baked as ckpts/nanowm-l2-csgo) # $NWM_ASSETS/csgo/1-200/*.hdf5 (held-out CSGO episode subset -> data/csgo) # $NWM_ASSETS/csgo_subset/{val_files.txt,val_starts.npy} (-> data/csgo_subset) # (baseline is optional; computed on first trial otherwise) +# NWM_ASSETS defaults to a cache dir and is populated by docker/prep_assets.py +# (downloads the public HF checkpoint + the CSGO 1-200 chunk, derives the val +# subset). Pre-stage NWM_ASSETS yourself to skip the download. Both nanowm tasks +# share an identical asset set, so a single staged dir serves both builds. set -euo pipefail TAG="${1:-experimental-v0}" NANOWM_COMMIT="${NANOWM_COMMIT:-main}" SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PROB=$(cd "$SCRIPT_DIR/.." && pwd) -NWM_ASSETS="${NWM_ASSETS:?set NWM_ASSETS to the dir holding the L/2 CSGO ckpt + held-out CSGO subset (see header)}" +NWM_ASSETS="${NWM_ASSETS:-$HOME/.cache/frontier-cs/nwm_assets}" + +# Auto-stage the public assets if not already present (self-contained build). +if [ ! -f "$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k/model_state_dict.pt" ] || \ + ! ls "$NWM_ASSETS/csgo/1-200/"*.hdf5 >/dev/null 2>&1; then + echo "Staging NanoWM assets into $NWM_ASSETS (one-time public download via prep_assets.py)..." + uv run --no-project --with huggingface_hub --with safetensors --with torch --with numpy \ + python "$SCRIPT_DIR/prep_assets.py" --out "$NWM_ASSETS" +fi CTX=$(mktemp -d); trap 'rm -rf "$CTX"' EXIT mkdir -p "$CTX/task_ctx" -cp -r "$PROB/stability_eval" "$CTX/task_ctx/task_pkg" +# Preserve the package DIRECTORY (task_pkg/stability_eval/) so the baked layout is +# /opt/nanowm/task/stability_eval/ + /opt/nanowm/task/evaluator.py. evaluator.py +# does `from stability_eval import ...` and the eval modules use relative imports +# (`from . import settings`), so flattening the contents into task/ breaks both. +mkdir -p "$CTX/task_ctx/task_pkg" +cp -r "$PROB/stability_eval" "$CTX/task_ctx/task_pkg/stability_eval" cp "$PROB/evaluator.py" "$CTX/task_ctx/evaluator.py" cp -r "$PROB/harbor/app" "$CTX/task_ctx/harbor_app" cp -r "$PROB/infra_patches" "$CTX/task_ctx/infra_patches" 2>/dev/null || mkdir -p "$CTX/task_ctx/infra_patches" -# judge-only hidden assets -mkdir -p "$CTX/task_ctx/assets" +# judge-only hidden assets (create dest parents first so the renaming cp lands) +mkdir -p "$CTX/task_ctx/assets/ckpts" "$CTX/task_ctx/assets/data" cp -r "$NWM_ASSETS/ckpts/nanowm-l2-csgo-100k" "$CTX/task_ctx/assets/ckpts/nanowm-l2-csgo" 2>/dev/null || true cp -r "$NWM_ASSETS/csgo" "$CTX/task_ctx/assets/data/csgo" 2>/dev/null || true cp -r "$NWM_ASSETS/csgo_subset" "$CTX/task_ctx/assets/data/csgo_subset" 2>/dev/null || true diff --git a/2.0/problems/nanowm_rollout_stability/docker/judge/Dockerfile b/2.0/problems/nanowm_rollout_stability/docker/judge/Dockerfile index f5846dd0..be96e56c 100644 --- a/2.0/problems/nanowm_rollout_stability/docker/judge/Dockerfile +++ b/2.0/problems/nanowm_rollout_stability/docker/judge/Dockerfile @@ -1,4 +1,4 @@ -# Judge image for nanowm_rollout_speedup (CPU-only; drives a Modal GPU). +# Judge image for nanowm_rollout_stability (CPU-only; drives a Modal GPU). # Bakes the task package + clean NanoWM checkout + L/2 CSGO ckpt + held-out CSGO # episode subset + cached vanilla baseline; the GPU rollout runs on Modal. FROM python:3.11-slim @@ -14,7 +14,10 @@ RUN git clone https://github.com/simchowitzlabpublic/nano-world-model /opt/nanow cd /opt/nanowm/nano-world-model && git checkout "$NANOWM_COMMIT" COPY task_ctx/infra_patches/ /tmp/infra_patches/ RUN cd /opt/nanowm/nano-world-model && for p in /tmp/infra_patches/*.patch; do \ - [ -e "$p" ] && git apply "$p" || true; done + [ -e "$p" ] && git apply "$p" || true; done && \ + { grep -q 'clip_offset' src/sample/rollout.py \ + && grep -q 'use_deterministic_algorithms' src/sample/evaluate_metrics.py; } \ + || { echo 'FATAL: infra patch did not apply (clip_offset / determinism markers missing)'; exit 1; } # Hidden assets baked by build_images.sh into the build context: # task_ctx/assets/ckpts/nanowm-l2-csgo/model_state_dict.pt diff --git a/2.0/problems/nanowm_rollout_stability/docker/prep_assets.py b/2.0/problems/nanowm_rollout_stability/docker/prep_assets.py new file mode 100644 index 00000000..df4a341d --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/docker/prep_assets.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Auto-prepare the (public) NanoWM rollout-task assets so the build is +self-contained -- no hand-staged NWM_ASSETS required. Invoked by build_images.sh +when the staging dir is empty; safe to run standalone. + +Produces, under --out (the NWM_ASSETS layout build_images.sh consumes): + ckpts/nanowm-l2-csgo-100k/model_state_dict.pt (converted from HF safetensors) + ckpts/nanowm-l2-csgo-100k/config.yaml + csgo/1-200/hdf5_dm_july2021_.hdf5 (ONLY the held-out val episodes) + csgo_subset/{val_files.txt,val_starts.npy} (derived from the repo split) + +Public sources (verified public, no token / no gating / no terms): + checkpoint : HF model knightnemo/nanowm-l2-csgo-100k (model.safetensors, 2.23 GB) + csgo data : HF dataset TeaPearce/CounterStrike_Deathmatch (hdf5_dm_july2021_1_to_200.tar, 25.7 GB) + val subset : DERIVED from nano-world-model/src/.../csgo_splits/{test_split.txt, + csgo_validation_start_indices.npy} -- the held-out subset is exactly + the test_split episodes with number <= 200 (22 clips), which is the + 1-200 data chunk the task stages. No download, no seed needed. + +Notes: + * The HF model ships `model.safetensors`, not the `model_state_dict.pt` the + upstream loader (`utils.nanowm_utils.find_model` -> torch.load) expects, so we + convert. find_model strips a leading `model.`/`_orig_mod.` prefix and unwraps a + `model` key, so a bare state_dict loads cleanly. + * The dataset is stored as 200-episode TAR chunks (no individual .hdf5), so we + download the single 1-200 chunk to a temp dir, extract ONLY the ~22 needed + members, then delete the tar -- the staged data is ~3 GB, not 25.7 GB. + * Idempotent: each step is skipped if its output already exists. + * Keep this judge/operator-side ONLY; the agent image never sees these assets. +""" +from __future__ import annotations + +import argparse +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + +CKPT_REPO = "knightnemo/nanowm-l2-csgo-100k" +DATA_REPO = "TeaPearce/CounterStrike_Deathmatch" +DATA_TAR = "hdf5_dm_july2021_1_to_200.tar" +NANOWM_REPO_URL = "https://github.com/simchowitzlabpublic/nano-world-model" +MAX_EPISODE = 200 # held-out subset = test_split episodes with number <= this + + +def _ep_num(fn: str) -> int: + m = re.search(r"_(\d+)\.hdf5$", fn) + return int(m.group(1)) if m else 10**9 + + +def _splits_dir(repo_hint: str) -> Path: + """Locate (or clone) the csgo_splits dir holding the public split files.""" + rel = "src/wm_datasets/data_source/game/csgo_splits" + if repo_hint: + cand = Path(repo_hint) / rel + if (cand / "test_split.txt").exists(): + return cand + cand = Path(repo_hint) # maybe repo_hint already points at the splits dir + if (cand / "test_split.txt").exists(): + return cand + tmp = Path(tempfile.mkdtemp(prefix="nwm_splits_")) / "repo" + print(f"[splits] cloning {NANOWM_REPO_URL} (shallow) for split files ...", flush=True) + subprocess.run(["git", "clone", "--depth", "1", NANOWM_REPO_URL, str(tmp)], check=True) + return tmp / rel + + +def derive_val_subset(splits: Path, out_dir: Path) -> list[str]: + import numpy as np + test = [l.strip() for l in (splits / "test_split.txt").read_text().splitlines() if l.strip()] + starts = np.load(splits / "csgo_validation_start_indices.npy") + if len(starts) != len(test): + raise SystemExit(f"split/start length mismatch: {len(starts)} starts vs {len(test)} files") + sel = [(i, f) for i, f in enumerate(test) if _ep_num(f) <= MAX_EPISODE] + files = [f for _, f in sel] + sub_starts = np.array([int(starts[i]) for i, _ in sel], dtype=np.int64) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "val_files.txt").write_text("\n".join(files) + "\n") + np.save(out_dir / "val_starts.npy", sub_starts) + print(f"[val] derived {len(files)} held-out clips -> {out_dir}", flush=True) + return files + + +def ensure_ckpt(out: Path) -> None: + dst = out / "ckpts" / "nanowm-l2-csgo-100k" + pt = dst / "model_state_dict.pt" + if pt.exists(): + print(f"[ckpt] exists, skip ({pt})", flush=True) + return + from huggingface_hub import hf_hub_download + from safetensors.torch import load_file + import torch + dst.mkdir(parents=True, exist_ok=True) + print(f"[ckpt] downloading {CKPT_REPO} model.safetensors (~2.23 GB) ...", flush=True) + st = hf_hub_download(CKPT_REPO, "model.safetensors") + cfg = hf_hub_download(CKPT_REPO, "config.yaml") + shutil.copy(cfg, dst / "config.yaml") + torch.save(load_file(st), pt) # torch-loadable .pt for upstream find_model() + print(f"[ckpt] wrote {pt} ({pt.stat().st_size / 1e9:.2f} GB)", flush=True) + + +def ensure_data(out: Path, files: list[str]) -> None: + dst = out / "csgo" / "1-200" + needed = set(files) + have = {p.name for p in dst.glob("*.hdf5")} if dst.exists() else set() + missing = needed - have + if not missing: + print(f"[data] all {len(needed)} episodes present, skip", flush=True) + return + from huggingface_hub import hf_hub_download + dst.mkdir(parents=True, exist_ok=True) + # Download into the HF cache (resumable across runs); a 25.7 GB pull is too + # fragile to restart from scratch on an interruption. The cached tar is left + # in ~/.cache/huggingface afterwards (purge with `huggingface-cli delete-cache` + # to reclaim ~25.7 GB once the 22 episodes are staged). + print(f"[data] downloading {DATA_TAR} (~25.7 GB, resumable HF cache) ...", flush=True) + tarp = hf_hub_download(DATA_REPO, DATA_TAR, repo_type="dataset") + with tarfile.open(tarp) as tf: + members = {Path(m.name).name: m for m in tf.getmembers() if m.name.endswith(".hdf5")} + for fn in sorted(missing): + m = members.get(fn) + if m is None: + raise SystemExit(f"{fn} not found in {DATA_TAR}; sample members={list(members)[:3]}") + m.name = fn # flatten into 1-200/.hdf5 + tf.extract(m, dst) + print(f"[data] extracted {fn}", flush=True) + print(f"[data] staged {len(needed)} episodes -> {dst}", flush=True) + + +def main() -> int: + ap = argparse.ArgumentParser(description="Auto-stage public NanoWM rollout assets.") + ap.add_argument("--out", required=True, help="NWM_ASSETS staging dir to populate") + ap.add_argument("--repo", default="", help="optional nano-world-model checkout (for split files)") + ap.add_argument("--skip-data", action="store_true", help="stage ckpt+val only (skip the 25.7 GB data)") + a = ap.parse_args() + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + files = derive_val_subset(_splits_dir(a.repo), out / "csgo_subset") + ensure_ckpt(out) + if not a.skip_data: + ensure_data(out, files) + print(f"[done] assets staged under {out}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/2.0/problems/nanowm_rollout_stability/evaluator.py b/2.0/problems/nanowm_rollout_stability/evaluator.py index b21a5171..b6c47add 100644 --- a/2.0/problems/nanowm_rollout_stability/evaluator.py +++ b/2.0/problems/nanowm_rollout_stability/evaluator.py @@ -1,10 +1,11 @@ -"""nanowm_rollout_speedup evaluator (Frontier-CS 2.0 contract). +"""nanowm_rollout_stability evaluator (Frontier-CS 2.0 contract). The agent submits a Python-only patch against a clean NanoWM checkout that -speeds up the diffusion SAMPLING for a fixed CSGO long-rollout invocation -(NanoWM-L/2, 50 frames, 4 context, nominal 50 DDIM steps). The judge applies the -patch, runs the rollout on a Modal GPU, and scores wall-clock speedup over the -unpatched baseline, gated by a rollout-quality (LPIPS-vs-GT) guardrail. +reduces long-horizon DRIFT of a fixed CSGO long-rollout invocation (NanoWM-L/2, +80 frames, 4 context, fixed 50 DDIM steps) at iso-wall-clock. The judge applies +the patch, runs the rollout on a Modal GPU, and scores the relative reduction in +tail-frame LPIPS-vs-GT over the unpatched baseline, gated by a wall-clock +guardrail (so drift can't simply be bought with more compute). This file holds the self-contained static patch policy + scoring. Heavy orchestration (Modal sandbox, baseline caching) lives in `stability_eval/`. When @@ -35,24 +36,48 @@ ALLOWED = ( "src/diffusion/*.py", "src/diffusion/**/*.py", - "src/sample/sampling_utils.py", ) DENIED = ( "src/models/**", "src/latent_codecs/**", # frozen model + VAE "src/sample/evaluate_metrics.py", "src/sample/plot_metrics.py", # the metric "src/sample/rollout.py", # the judge harness + # sampling_utils.py is PLUMBING (VAE encode/decode, video IO, resize), NOT the + # diffusion sampler -- it decodes+saves the generated frames the LPIPS metric + # reads (gen is decoded/saved BEFORE gt), so allowing it would let a submission + # blur/low-pass ONLY the scored pixels (a stateful counter in decode_latents/ + # save_video) and cut LPIPS-vs-GT with zero faithful improvement. Frozen. + "src/sample/sampling_utils.py", "src/wm_datasets/**", # data loading / no bench detection "src/experiments/**", "src/main.py", "src/utils/**", "**/*.so", "**/*.pyx", "**/*.c", "**/*.cpp", "**/*.cu", # no native "**/setup.py", "**/pyproject.toml", "**/requirements*.txt", ) # Forbidden substrings in ADDED lines (no benchmark detection / env leakage / -# timing short-circuits / hardcoded ground truth). +# timing short-circuits / hardcoded ground truth / sandbox escape / reading the +# held-out data). Defense-in-depth only -- substring matching is necessarily +# imperfect, so the real guarantees are the path allowlist, the frozen metric, +# and the sampler never receiving future GT frames. Tokens are chosen to be +# high-value AND low-false-positive: e.g. we deliberately do NOT ban `eval(` +# (would hit `model.eval()`), `compile(` (would hit `torch.compile()`), or +# `open(` (legit solver caches), relying on the deeper guardrails for those. FORBIDDEN_TOKENS = ( + # benchmark / judge / env detection + leakage. `environ` (not just + # `os.environ`) also catches `from os import environ` / aliased reads; the + # rollout injects CSGO_DATA_DIR (the held-out GT dir) into the patched + # subprocess env, so that key is banned outright. "FRONTIER_", "JUDGE_", "HARBOR_", "MODAL_", "HF_TOKEN", "NWM_TIME", + "environ", "os.getenv", "getenv(", "CSGO_DATA_DIR", "sys.modules", + # the metric / ground truth / held-out selection (no hard-coding or peeking) "csgo_validation_start_indices", "val_starts", "val_files", "gen_seconds", "evaluate_metrics", "lpips", "ground_truth", "gt_frames", - "os.environ", "subprocess", "socket", "/judge", "/opt/nanowm/baseline", + # held-out asset / data paths (no reading the GT episodes off disk) + "/judge", "/opt/nanowm", "/proc/", ".hdf5", "csgo_subset", + "train_split", "test_split", + # process / network / dynamic-exec escapes + "subprocess", "socket", "os.system", "os.popen", "os.exec", "os.fork", + "__import__", "importlib", "__builtins__", "ctypes", "marshal", + "urllib", "requests", "http.client", + # timing short-circuits "time.sleep", "while True", ) @@ -173,11 +198,26 @@ def evaluate(solution_path: str): try: baseline, patched = orchestrate.run_pair(patch_path, clips=clips, role=role) except Exception as exc: # concise public error only - return 0.0, 0.0, f"evaluation failed: {type(exc).__name__}", {**pmetrics, "role": role} + # Attribute the failure: a patch that won't apply OR a patch whose code + # crashes the rollout/metric is the submission's fault (a legitimate 0); + # an orchestration/GPU error is judge-side infra. All score 0 per the 2.0 + # convention, but `infra_error` lets the operator spot/re-run the latter + # instead of mistaking a transient GPU failure for a bad submission. + ml = str(exc).lower() + if isinstance(exc, RuntimeError) and "patch failed" in ml: + kind, infra, pub = "patch_apply", 0, "patch failed to apply" + elif isinstance(exc, RuntimeError) and ("rollout failed" in ml or "metrics failed" in ml): + kind, infra, pub = "execution", 0, "submission crashed during rollout/metrics" + else: + kind, infra, pub = "infra", 1, f"evaluation infra error: {type(exc).__name__}" + return 0.0, 0.0, pub, {**pmetrics, "role": role, "error_class": type(exc).__name__, + "error_kind": kind, "infra_error": infra} res = scoring.provisional_score( baseline["tail_lpips"], patched["tail_lpips"], baseline["gen_seconds"], patched["gen_seconds"], tolerance=S.WALLCLOCK_TOLERANCE, + # fan-out provides the per-chunk-fair over-budget; None on the agent path. + wallclock_rel_over=patched.get("wallclock_rel_over"), ) metrics = { **pmetrics, "role": role, "clips": clips, @@ -186,12 +226,20 @@ def evaluate(solution_path: str): "rel_reduction_pct": round(res["rel_reduction_pct"], 2), "baseline_seconds": round(baseline["gen_seconds"], 2), "patched_seconds": round(patched["gen_seconds"], 2), + "wallclock_rel_over": round(res["wallclock_rel_over"], 4), "wallclock_multiplier": round(res["wallclock_multiplier"], 3), + # The SCORED horizon is drawn at random per run (audit #7) so a patch that + # hardcodes the rollout length to target the tail window cannot; surface it + # (and the derived tail) so operators can see/reproduce which horizon scored. + "rollout_length": patched.get("rollout_length") or baseline.get("rollout_length"), + "tail_start": patched.get("tail_start") or baseline.get("tail_start"), "score_formula": ("100*(baseline_tail-patched_tail)/baseline_tail * wallclock_mult; " - "wallclock_mult<1 if gen time rises >tol over baseline (no buying drift with compute)"), + "wallclock_mult decays from parity (grace 2%), ~0.556 at +tol and 0.5 at " + "grace+tol, so drift cannot be bought with compute (no flat free-budget plateau)"), } msg = (f"tail-drift {patched['tail_lpips']:.3f} vs baseline {baseline['tail_lpips']:.3f} " f"({res['rel_reduction_pct']:+.1f}%, {patched['gen_seconds']:.0f}s/budget {baseline['gen_seconds']:.0f}s); " + f"horizon={metrics['rollout_length']} tail>={metrics['tail_start']}; " f"score {res['score']:.1f}") return res["score"], res["score_unbounded"], msg, metrics diff --git a/2.0/problems/nanowm_rollout_stability/harbor/app/README.md b/2.0/problems/nanowm_rollout_stability/harbor/app/README.md index d8bab948..2b06b27d 100644 --- a/2.0/problems/nanowm_rollout_stability/harbor/app/README.md +++ b/2.0/problems/nanowm_rollout_stability/harbor/app/README.md @@ -1,19 +1,32 @@ -# nanowm_rollout_speedup — agent workspace +# nanowm_rollout_stability — agent workspace Patch the diffusion **sampling** code of the NanoWM checkout at -`/app/nano-world-model` to reduce drift in the fixed CSGO 50-frame rollout at -iso-quality, then submit the unified diff as `/app/solution.patch`. +`/app/nano-world-model` to reduce long-horizon **drift** in a fixed CSGO +long-horizon rollout **at iso-wall-clock** (no extra compute), then submit the +unified diff as `/app/solution.patch`. + +The exact rollout length and which frames are scored are fixed by the judge and +**not disclosed** (the scored horizon is drawn per run), so a solution must reduce +drift *generally* — keying behaviour off an assumed rollout length or a hardcoded +frame index will not transfer to the scored run. Workflow: -1. Edit `nano-world-model/src/diffusion/**` or `src/sample/sampling_utils.py`. +1. Edit the diffusion sampler under `nano-world-model/src/diffusion/**`. 2. `bash /app/public_test.sh` — static patch-policy check on your diff. 3. `bash /app/make_submission.sh` — writes `/app/solution.patch` from your edits. 4. `bash /app/submit.sh` — enqueue for the black-box judge (quick set). -Rules (see the task statement): Python-only; allowed = `src/diffusion/**.py`, -`src/sample/sampling_utils.py`; the model/VAE/metric/harness/data are frozen and -denied; no env-var/benchmark/timing tricks. The judge fixes the rollout call — -you change the sampler internals. +Rules (see the task statement): Python-only; allowed = `src/diffusion/**.py`; +the model/VAE/metric/harness/data AND the VAE-decode/video plumbing +(`src/sample/sampling_utils.py`) are frozen and denied; no env-var/benchmark/ +timing tricks. The judge fixes the rollout call +(length / context / steps / scheduling) — you change the sampler internals to +make the long rollout drift less per step. -Scored on wall-clock speedup vs the unpatched baseline, gated by an LPIPS-vs-GT -quality guardrail (≤3% rise). See `../readme` and `../DESIGN.md`. +Scored on the relative reduction in **tail-frame LPIPS-vs-GT** drift (the drifted +tail of the long rollout) vs the unpatched baseline, gated by a **wall-clock +guardrail** (patched generation time may rise at most ~10% over baseline — you +cannot buy lower drift with more compute; that is the speedup task's axis). The +length of the rollout and the exact tail window scored are not disclosed and vary +per scored run, so target *real* drift reduction, not a fixed frame range. See +`../readme`. diff --git a/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-judge-infra.patch b/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-judge-infra.patch index 4c2b40b7..9d3a57ec 100644 --- a/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-judge-infra.patch +++ b/2.0/problems/nanowm_rollout_stability/infra_patches/0001-rollout-judge-infra.patch @@ -1,5 +1,32 @@ +diff --git a/src/sample/evaluate_metrics.py b/src/sample/evaluate_metrics.py +index cc6846d..6132034 100644 +--- a/src/sample/evaluate_metrics.py ++++ b/src/sample/evaluate_metrics.py +@@ -34,7 +34,21 @@ def main(): + parser.add_argument("--history_length", type=int, default=1, help="Number of history frames (context)") + + args = parser.parse_args() +- ++ ++ # Deterministic metric (judge infra): the scored LPIPS runs on GPU by default, ++ # so without this the AlexNet/VGG convs are cuDNN-nondeterministic and the ++ # baseline vs patched arms (separate metric processes) can differ even for a ++ # no-op patch -- reintroducing the run-to-run noise the rollout determinism ++ # is meant to remove. Mirror rollout.py: deterministic kernels + TF32 off. ++ # NWM_DETERMINISM_STRICT=1 makes a missing deterministic kernel raise (proof). ++ _strict = os.environ.get("NWM_DETERMINISM_STRICT", "0") == "1" ++ torch.use_deterministic_algorithms(True, warn_only=not _strict) ++ if args.device == "cuda": ++ torch.backends.cudnn.deterministic = True ++ torch.backends.cudnn.benchmark = False ++ torch.backends.cuda.matmul.allow_tf32 = False ++ torch.backends.cudnn.allow_tf32 = False ++ + video_dir = Path(args.video_dir) + if not video_dir.exists(): + print(f"Directory {video_dir} does not exist.") diff --git a/src/sample/rollout.py b/src/sample/rollout.py -index b83f0ee..d4bfd6a 100644 +index b83f0ee..6ea2d7b 100644 --- a/src/sample/rollout.py +++ b/src/sample/rollout.py @@ -9,6 +9,7 @@ Uses the sliced dataset for evaluation consistency. @@ -10,13 +37,90 @@ index b83f0ee..d4bfd6a 100644 sys.path.append(os.path.split(sys.path[0])[0]) import torch -@@ -131,9 +132,20 @@ def main(args): +@@ -37,7 +38,35 @@ torch.backends.cudnn.allow_tf32 = True + def main(args): + torch.set_grad_enabled(False) + device = "cuda" if torch.cuda.is_available() else "cpu" +- ++ ++ # Deterministic numerics (judge infra, outside the agent's editable sampling ++ # scope). Goal: the baseline (unpatched) and patched arms share an identical ++ # numerical pipeline -- not just identical initial noise from the per-clip ++ # seed below, but identical arithmetic -- so a cached/separate-run baseline is ++ # a valid common-random-numbers partner. Otherwise GPU reduction/atomic ++ # nondeterminism (and TF32, which upstream turns ON at import) reintroduce ++ # ~2-3% run-to-run LPIPS noise, the same magnitude as the speedup quality ++ # guardrail and the stability drift effect. CUBLAS_WORKSPACE_CONFIG is ++ # exported by the launcher before the process starts (cuBLAS needs it set ++ # before the first CUDA call). ++ # ++ # CAVEAT: warn_only=True (the default) means any op WITHOUT a deterministic ++ # CUDA kernel (an atomicAdd scatter/index_add, some interpolate variants) ++ # runs nondeterministically and is only WARNED about, so bit-stability is not ++ # guaranteed until proven for this exact model+VAE+decode on the production ++ # GPU. Run ONCE during H100 calibration with NWM_DETERMINISM_STRICT=1 ++ # (warn_only=False) to PROVE no op falls back -- it raises instead of warning ++ # if one does -- and re-establish no-op-CRN-pair -> 0.0 there. Production may ++ # then stay strict (a fallback raises -> judged an infra error and re-run, ++ # never a silently corrupted score) or relax to warn_only once proven clean. ++ _strict = os.environ.get("NWM_DETERMINISM_STRICT", "0") == "1" ++ torch.use_deterministic_algorithms(True, warn_only=not _strict) ++ if device == "cuda": ++ torch.backends.cudnn.deterministic = True ++ torch.backends.cudnn.benchmark = False ++ torch.backends.cuda.matmul.allow_tf32 = False ++ torch.backends.cudnn.allow_tf32 = False ++ + save_dir = Path(args.save_path) + save_dir.mkdir(parents=True, exist_ok=True) + +@@ -109,6 +138,13 @@ def main(args): + if dataset.slice_mode != "exhaustive": + raise ValueError(f"Rollout requires exhaustive slice_mode, got '{dataset.slice_mode}'") + ++ # Clip parallelism (judge infra): this process handles the GLOBAL clip window ++ # [clip_offset, clip_offset+num_samples) of the filtered valid-slice list, so ++ # the scored pair can be fanned out across Modal containers (one chunk each) ++ # yet stay bit-identical to one sequential run -- the per-batch reseed below ++ # keys on the GLOBAL start_idx, so aligned chunks reproduce the same batches. ++ # Search far enough to cover the window's far end. ++ _clip_offset = int(getattr(args, "clip_offset", 0)) + valid_slice_indices = [] + for i in range(len(dataset)): + slice_idx = dataset.slice_indices[i] +@@ -118,22 +154,38 @@ def main(args): + total_len = dataset.data_source.get_seq_length(traj_idx) + if (total_len - start_frame) >= rollout_length * frame_interval: + valid_slice_indices.append(i) +- if len(valid_slice_indices) >= args.num_samples: ++ if len(valid_slice_indices) >= _clip_offset + args.num_samples: + break + + num_samples = len(valid_slice_indices) +- if num_samples < args.num_samples: +- print(f"Warning: Only found {num_samples} valid slices with enough length (requested {args.num_samples})") +- ++ # Window [clip_offset, _window_hi) of the GLOBAL valid-slice list is this ++ # chunk's work; the seed and sample_id below use the global start_idx, so a ++ # chunk is bit-identical to the matching slice of one sequential run. ++ _window_hi = min(_clip_offset + args.num_samples, num_samples) ++ if _window_hi - _clip_offset < args.num_samples: ++ print(f"Warning: only {_window_hi - _clip_offset} of requested {args.num_samples} valid " ++ f"slices available from clip_offset {_clip_offset} (total valid {num_samples})") ++ + print(f"\nRollout configuration:") +- print(f" Total samples to process: {num_samples}") ++ print(f" Clip window: [{_clip_offset}, {_window_hi}) of {num_samples} valid slices") + print(f" Batch size: {batch_size}") print(f" History length: {history_length}") print(f" Rollout length: {rollout_length}") - +- +- for start_idx in range(0, num_samples, batch_size): +- end_idx = min(start_idx + batch_size, num_samples) ++ + gen_seconds = 0.0 # accumulated sampling-region wall-clock (excludes load/decode) - for start_idx in range(0, num_samples, batch_size): - end_idx = min(start_idx + batch_size, num_samples) ++ for start_idx in range(_clip_offset, _window_hi, batch_size): ++ end_idx = min(start_idx + batch_size, _window_hi) current_batch_size = end_idx - start_idx + + # Deterministic, clip-keyed RNG (judge infra, outside the agent's editable @@ -31,7 +135,7 @@ index b83f0ee..d4bfd6a 100644 print(f"Processing batch {start_idx//batch_size + 1} (slices {start_idx} to {end_idx-1})...") -@@ -181,7 +193,12 @@ def main(args): +@@ -181,7 +233,12 @@ def main(args): gt_latents = encode_frames(vae, gt_visual, device, vae_precision=vae_precision) generated_latents = gt_latents[:, :history_length] @@ -45,7 +149,7 @@ index b83f0ee..d4bfd6a 100644 for t in tqdm(range(history_length, rollout_length), desc=f"Rollout progress"): context_latents = generated_latents[:, -history_length:] -@@ -224,9 +241,25 @@ def main(args): +@@ -224,9 +281,25 @@ def main(args): new_latent = pred_latents[:, history_length:history_length+1] generated_latents = torch.cat([generated_latents, new_latent], dim=1) @@ -73,7 +177,7 @@ index b83f0ee..d4bfd6a 100644 for i in range(current_batch_size): sample_id = start_idx + i -@@ -234,6 +267,15 @@ def main(args): +@@ -234,6 +307,15 @@ def main(args): save_video(gt_frames_batch[i], str(save_dir / f"sample_{sample_id:04d}_gt.mp4"), fps=args.fps) save_comparison_video(gt_frames_batch[i], gen_frames_batch[i], str(save_dir / f"sample_{sample_id:04d}_compare.mp4"), fps=args.fps) @@ -89,22 +193,27 @@ index b83f0ee..d4bfd6a 100644 print(f"Done! Processed {num_samples} samples.") if __name__ == "__main__": -@@ -259,6 +301,9 @@ if __name__ == "__main__": +@@ -259,6 +341,14 @@ if __name__ == "__main__": help="DFoT stabilization noise level in [0, 1) for context frames. " "0.0 disables; 0.02 is the DFoT default recommended starting point.") parser.add_argument("--use_fp16", action="store_true", help="Use fp16") + parser.add_argument("--seed", type=int, default=42, + help="Deterministic RNG seed (judge infra). Per-clip seed = seed + clip_index " + "so the baseline and patched arms share initial noise (common random numbers).") ++ parser.add_argument("--clip_offset", type=int, default=0, ++ help="Judge infra: process only the GLOBAL clip window " ++ "[clip_offset, clip_offset+num_samples) of the filtered valid-slice list, " ++ "so the scored pair can fan out across Modal containers bit-identically " ++ "(the per-batch seed keys on the global index). Must be batch_size-aligned.") parser.add_argument("--vae_model_path", type=str, default=None, help="Override the VAE path from the training config (e.g. local dir for offline nodes).") -@@ -282,7 +327,7 @@ if __name__ == "__main__": +@@ -282,7 +372,7 @@ if __name__ == "__main__": # Top-level / non-hierarchical CLI args for key in ('ckpt', 'save_path', 'num_samples', 'batch_size', 'rollout_length', - 'history_length', 'fps', 'eta', 'use_fp16', 'vae_model_path'): -+ 'history_length', 'fps', 'eta', 'use_fp16', 'vae_model_path', 'seed'): ++ 'history_length', 'fps', 'eta', 'use_fp16', 'vae_model_path', 'seed', 'clip_offset'): val = cli_args.get(key) if val is not None: cli_overrides[key] = val diff --git a/2.0/problems/nanowm_rollout_stability/readme b/2.0/problems/nanowm_rollout_stability/readme index 719480f1..6faf8599 100644 --- a/2.0/problems/nanowm_rollout_stability/readme +++ b/2.0/problems/nanowm_rollout_stability/readme @@ -3,15 +3,21 @@ ## Problem You are given a clean checkout of Nano World Models (arXiv:2605.23993) and its -frozen NanoWM-L/2 CSGO checkpoint. The judge runs a **fixed 80-frame** -autoregressive rollout (4 context frames, sequential, **50 DDIM steps**). Long -autoregressive rollouts accumulate perceptual error — by the tail of the rollout -the prediction has drifted into a "plausible but wrong" state (paper Finding #5). +frozen NanoWM-L/2 CSGO checkpoint. The judge runs a **fixed long-horizon** +autoregressive rollout (sequential, **50 DDIM steps**). Long autoregressive +rollouts accumulate perceptual error — by the tail of the rollout the prediction +has drifted into a "plausible but wrong" state (paper Finding #5). Your job: **minimize that drift** — the mean LPIPS-vs-ground-truth over the -**tail frames (index ≥ 60)** — by submitting a **Python-only patch** to the -diffusion **sampling** code, **without using more compute** (a wall-clock -budget = the unpatched baseline's generation time is enforced). +**drifted tail frames** (the late portion of the rollout) — by submitting a +**Python-only patch** to the diffusion **sampling** code, **without using more +compute** (a wall-clock budget = the unpatched baseline's generation time is +enforced). + +The exact rollout length and which frames are scored as the "tail" are fixed by +the judge and **not disclosed** — the scored horizon is drawn per run — so a +solution must reduce drift **generally**; keying behaviour off an assumed rollout +length or a hardcoded frame index will not transfer to the scored run. This is a hard, open problem: simply adding denoising steps reduces drift but is disallowed (it costs compute — that's the *speedup* task). At fixed compute you @@ -30,11 +36,12 @@ files. No env-var/benchmark/timing tricks. Validated before running. ## Evaluation & scoring -- Judge applies your patch, runs the fixed 80-frame CSGO rollout on hidden - episodes (Modal GPU), measures **tail-drift** (mean LPIPS-vs-GT, frames ≥ 60) - and **generation wall-clock**. Quick set for iterative `submit.sh`; a larger - disjoint set for the final verifier (enough clips to resolve small drift - reductions above per-clip noise). +- Judge applies your patch, runs the fixed long-horizon CSGO rollout on hidden + episodes (Modal GPU), measures **tail-drift** (mean LPIPS-vs-GT over the late / + tail frames) and **generation wall-clock**. Quick set for iterative `submit.sh`; + a larger disjoint set for the final verifier (enough clips to resolve small drift + reductions above per-clip noise). The exact rollout length and tail window are + not disclosed and vary per scored run. - **Score:** ``` @@ -58,4 +65,4 @@ Substantially beating it is the open challenge. ## Resource budget CPU agent + judge; one Modal GPU per evaluation. Evaluation timeout 21600 s. -See `AGENT.md`, `harbor/app/README.md`, and `DESIGN.md`. +See `AGENT.md` and `harbor/app/README.md`. diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/__init__.py b/2.0/problems/nanowm_rollout_stability/stability_eval/__init__.py index 9c5b1828..a443875f 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/__init__.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/__init__.py @@ -1 +1 @@ -"""nanowm_rollout_speedup evaluation package (shared by judge + public test).""" +"""nanowm_rollout_stability evaluation package (shared by judge + public test).""" diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/modal_app.py b/2.0/problems/nanowm_rollout_stability/stability_eval/modal_app.py index 2482352e..50bd1fa5 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/modal_app.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/modal_app.py @@ -1,4 +1,4 @@ -"""Modal app: run one NanoWM CSGO rollout (patched or vanilla) on one GPU. +"""Modal app: run NanoWM CSGO rollouts (patched and/or vanilla) on one GPU. Mirrors the GPU-on-Modal pattern of vllm_llm_serving_optimization: the judge container stays CPU-only and calls into a Modal GPU function. The NanoWM @@ -6,61 +6,208 @@ the Modal image at build time; the agent's patch text travels in per call. Parametrized via env vars (set by the judge): - NWM_MODAL_GPU Modal GPU string (default "L40S") + NWM_MODAL_GPU Modal GPU string (default "H100") NWM_MODAL_APP Modal app name NWM_HF_SECRET Modal Secret name (unused once weights are baked) Deploy: modal deploy stability_eval/modal_app.py -Call: run_rollout_remote(patch_text, clips, steps) -> metrics dict +Call: + run_pair_remote(patch_text, clips, steps) -> {"baseline":{...}, "patched":{...}} + Runs BOTH arms back-to-back in one container on one GPU (two rollout + subprocesses) so the scored (final) result shares hardware/driver/seed. + The no-op->0 CRN guarantee additionally relies on the determinism flags + holding for every op (audit P1; prove once on H100 with strict mode). + run_rollout_remote(patch_text, clips, steps) -> metrics dict + Single arm; used for cheap cached-baseline iterative (agent-role) feedback. NOTE: validated structurally against the #145 modal pattern; end-to-end Modal execution is pending maintainer Modal credentials (Della has no Modal access). -The LOCAL backend (orchestrate._run_local) is the path validated on H100. +The LOCAL backend (orchestrate._run_local / _run_pair_local) is the path +validated on H100 -- the same SKU now requested here, so the production path and +the calibrated numbers share one GPU type. """ from __future__ import annotations import os -GPU = os.environ.get("NWM_MODAL_GPU", "L40S") -APP_NAME = os.environ.get("NWM_MODAL_APP", "nanowm-rollout-speedup") +GPU = os.environ.get("NWM_MODAL_GPU", "H100") +APP_NAME = os.environ.get("NWM_MODAL_APP", "nanowm-rollout-stability") REMOTE_ROOT = "/opt/nanowm" +# cuBLAS determinism must be set before the first CUDA call; bake it into the +# image env so the rollout subprocess inherits it (pairs with the deterministic +# kernels enabled in rollout.py via the infra patch). +_DETERMINISM_ENV = {"CUBLAS_WORKSPACE_CONFIG": ":4096:8", "PYTHONHASHSEED": "0"} + try: import modal image = ( modal.Image.from_registry("nvidia/cuda:12.1.0-devel-ubuntu22.04", add_python="3.11") + # runner.apply_patch shells out to `git apply` (fallback `patch -p1`); the + # CUDA base ships neither, so the patched arm died with FileNotFoundError + # ('git') before any rollout — the baseline arm (no patch) never hit it. + .apt_install("git", "patch") .pip_install( "torch==2.4.1", "torchvision==0.19.1", "numpy<2", "scipy==1.15.3", "lpips", "diffusers[torch]==0.24.0", "omegaconf", "hydra-core", "decord", "imageio", "imageio-ffmpeg", "opencv-python-headless", "scikit-image", "pandas", "einops", "timm", "pytorch-lightning==2.4.0", "huggingface_hub==0.25.2", "transformers==4.46.3", + # rollout import-chain deps the hand-curated list omitted (the Modal + # path had never run a rollout end-to-end before): h5py reads the CSGO + # .hdf5 episodes (wm_datasets), tensorboard backs the SummaryWriter + # import at the top of utils/nanowm_utils.py pulled in by find_model. + "h5py", "tensorboard", + # metric-path deps (utils.metrics, imported by evaluate_metrics.py): + # piqa backs PSNR/SSIM, pytorch-fid the InceptionV3 the Evaluator ctor + # builds, requests is a top-level import in utils.fvd. FVD/I3D is never + # computed (only LPIPS is scored) but these module/ctor imports must + # resolve or the metric subprocess dies before writing any LPIPS. + "piqa", "pytorch-fid", "requests", + ) + # Pre-fetch the metric model weights at BUILD time so scoring never depends + # on a flaky per-container runtime download: LPIPS' VGG16 backbone and the + # InceptionV3 the Evaluator ctor builds were fetched from download.pytorch.org + # on every cold container and intermittently hit "Connection reset by peer", + # which would zero out a whole scored run. Baked into the torch hub cache, + # the runtime Evaluator finds them locally. + .run_commands( + "python -c 'import lpips; lpips.LPIPS(net=\"vgg\"); " + "from pytorch_fid.inception import InceptionV3; " + "InceptionV3([InceptionV3.BLOCK_INDEX_BY_DIM[2048]])'" ) + .env(_DETERMINISM_ENV) # bake the clean checkout + ckpt + CSGO subset + task package .add_local_dir(os.environ.get("NWM_BAKE_DIR", "/opt/nanowm"), REMOTE_ROOT, copy=True) ) app = modal.App(APP_NAME) - @app.function(gpu=GPU, image=image, timeout=3600) - def _rollout(patch_text: str, clips: int, steps: int) -> dict: + def _eval_arm(patch_text: str, clips: int, steps: int, strict: bool = False, + clip_offset: int = 0, rollout_length: int | None = None) -> dict: import sys, tempfile from pathlib import Path + # Propagate the judge's strict-determinism choice INTO this Modal container + # BEFORE settings is imported: the flag is set on the CPU judge, not here, + # so without this S.STRICT_DETERMINISM (hence the rollout/metric subprocess + # NWM_DETERMINISM_STRICT) would always read 0 (warn_only) on the GPU and the + # H100 proof run would prove nothing about op-level determinism. + os.environ["FRONTIER_NWM_STRICT_DETERMINISM"] = "1" if strict else "0" sys.path.insert(0, f"{REMOTE_ROOT}/task") - from stability_eval import runner, orchestrate # noqa + from stability_eval import runner, orchestrate, settings as S # noqa patch = None if patch_text.strip(): p = Path(tempfile.mkstemp(suffix=".patch")[1]) p.write_text(patch_text) patch = p cfg = orchestrate._compose_config() - from stability_eval import settings as S - return runner.evaluate(S.REPO, patch, cfg, S.CKPT, clips=clips, steps=steps, device="cuda") + # rollout_length is decided ONCE on the judge (audit #7 randomized horizon) + # and passed in identically for every chunk + both arms, so chunk + # bit-identity and CRN pairing are unaffected. + return runner.evaluate(S.REPO, patch, cfg, S.CKPT, clips=clips, steps=steps, + device="cuda", clip_offset=clip_offset, + rollout_length=rollout_length) + + # Strict determinism (TF32 off + deterministic kernels) ~triples the sampling + # wall-clock and the 80-frame stability rollout is slow, so the scored 22-clip + # pair would be ~10h sequentially. Instead run_pair_remote FANS it out across + # containers, one batch_size-aligned clip CHUNK each -- the per-batch seed keys + # on the GLOBAL clip index, so chunks are bit-identical to one sequential run. + @app.function(gpu=GPU, image=image, timeout=14400) # 4h: single-arm quick run + def _rollout(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: + return _eval_arm(patch_text, clips, steps, strict) + + @app.function(gpu=GPU, image=image, timeout=21600) # 6h: one chunk's baseline+patched pair + def _rollout_pair_chunk(spec) -> dict: + # spec = (clip_offset, clip_count, patch_text, steps, strict, rollout_length). + # Runs BOTH arms for the GLOBAL clip window [clip_offset, clip_offset+clip_count) + # on ONE container, so the pair shares hardware AND the global per-clip seed + # (CRN). rollout_length is the per-run randomized horizon (audit #7), identical + # across all chunks (decided on the judge), so the fan-out stays bit-identical. + clip_offset, clip_count, patch_text, steps, strict, rollout_length = spec + baseline = _eval_arm("", clip_count, steps, strict, clip_offset, rollout_length) + patched = _eval_arm(patch_text, clip_count, steps, strict, clip_offset, rollout_length) + # Use the ACTUAL processed count (the 80-frame headroom filter can drop an + # episode): both arms see the SAME clips, so they must agree -- and the real + # count, not the requested one, drives the downstream weighting/divisor. + nb, npp = int(baseline.get("clips", 0)), int(patched.get("clips", 0)) + if nb != npp: + raise RuntimeError(f"chunk@{clip_offset}: baseline/patched clip-count mismatch {nb} != {npp}") + return {"clip_offset": clip_offset, "requested": clip_count, "clip_count": nb, + "baseline": baseline, "patched": patched} + + def run_rollout_remote(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: + with app.run(): + return _rollout.remote(patch_text, clips, steps, strict) - def run_rollout_remote(patch_text: str, clips: int, steps: int) -> dict: + def run_pair_remote(patch_text: str, clips: int, steps: int, strict: bool = False, + chunk_size: int = 4, rollout_length: int | None = None) -> dict: + # Fan the scored pair out across Modal containers, one batch_size-aligned + # clip chunk each (chunk_size must be a multiple of the rollout batch_size, + # enforced by the caller, so chunk offsets land on global batch boundaries + # and reproduce the sequential run's batches bit-for-bit). Aggregate: + # tail/lpips are per-clip-weighted means; gen_seconds is the SUM (total + # sampling compute, for the wall-clock guardrail -- each chunk's two arms + # ran on the same container so their ratio is a fair paired comparison). + # rollout_length (audit #7) is the SAME value in every spec so the fan-out is + # bit-identical to one sequential run at that horizon. + specs, off = [], 0 + while off < clips: + cnt = min(chunk_size, clips - off) + specs.append((off, cnt, patch_text, steps, strict, rollout_length)) + off += cnt + # Modal .map raises (return_exceptions=False) if any chunk's rollout/metric + # errors -> the whole scored run hard-fails (correct: never score a partial + # result on a real error). Empty-window chunks instead return clip_count=0 + # and are dropped here; a clip SHORTFALL (the headroom filter dropping an + # episode) is scored on the actual set, weighted correctly, but logged. with app.run(): - return _rollout.remote(patch_text, clips, steps) + results = [r for r in _rollout_pair_chunk.map(specs) if r and r.get("clip_count", 0) > 0] + tot = sum(r["clip_count"] for r in results) + if tot == 0: + raise RuntimeError("no clips processed across any chunk (empty valid-slice set?)") + if tot != clips: + print(f"[run_pair_remote] WARNING: requested {clips} clips but processed {tot} " + f"(some episodes lack long-rollout headroom); scoring on {tot}, " + f"weighted correctly -- NOT silently mis-divided.") + + # The randomized scored horizon (audit #7) is identical across chunks/arms; + # surface it (and the derived tail) so the evaluator/operator can see/repro it. + _roll = next((r["baseline"].get("rollout_length") for r in results), rollout_length) + _tail = next((r["baseline"].get("tail_start") for r in results), None) + + def _agg(arm: str) -> dict: + # Per-clip-weighted grand mean: sum(chunk_mean * chunk_clips) / total + # equals the sequential mean-over-all-clips (the chunk-size factor + # cancels) up to FP summation order. gen_seconds is the SUM (total + # sampling compute) -- a fair paired ratio vs baseline, not a + # bit-reproducible sequential time. + return { + "tail_lpips": sum(r[arm]["tail_lpips"] * r["clip_count"] for r in results) / tot, + "lpips": sum(r[arm]["lpips"] * r["clip_count"] for r in results) / tot, + "gen_seconds": sum(r[arm]["gen_seconds"] for r in results), + "clips": tot, "steps": steps, "rollout_length": _roll, "tail_start": _tail, + } + # Wall-clock over-budget for the guardrail: the clip-weighted mean of each + # chunk's OWN patched/baseline ratio (both arms ran on ONE board, so each + # ratio is a fair paired comparison), NOT the ratio of cross-board summed + # times. Carried on the patched dict so the (baseline, patched) return arity + # is unchanged and the evaluator can prefer it over the summed-time ratio. + def _chunk_rel(r): + b = max(r["baseline"]["gen_seconds"], 1e-9) + return max(0.0, (r["patched"]["gen_seconds"] - b) / b) + wallclock_rel_over = sum(_chunk_rel(r) * r["clip_count"] for r in results) / tot + baseline, patched = _agg("baseline"), _agg("patched") + patched["wallclock_rel_over"] = wallclock_rel_over + return {"baseline": baseline, "patched": patched, + "chunks": len(results), "requested_clips": clips, "scored_clips": tot, + "wallclock_rel_over": wallclock_rel_over, + "rollout_length": _roll, "tail_start": _tail} except ImportError: # modal not installed (e.g. local validation) - def run_rollout_remote(patch_text: str, clips: int, steps: int) -> dict: # type: ignore + def run_rollout_remote(patch_text: str, clips: int, steps: int, strict: bool = False) -> dict: # type: ignore + raise RuntimeError("modal not available; use the LOCAL backend (FRONTIER_NWM_BACKEND=local)") + + def run_pair_remote(patch_text: str, clips: int, steps: int, strict: bool = False, + chunk_size: int = 4, rollout_length: int | None = None) -> dict: # type: ignore raise RuntimeError("modal not available; use the LOCAL backend (FRONTIER_NWM_BACKEND=local)") diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py b/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py index 19d9f564..88a47edc 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/orchestrate.py @@ -1,7 +1,7 @@ -"""GPU orchestration for nanowm_rollout_speedup. +"""GPU orchestration for nanowm_rollout_stability. `run_pair(patch_path, clips, role)` returns (baseline_metrics, patched_metrics), -each {"gen_seconds", "lpips", "clips", "steps"}. +each {"gen_seconds", "lpips", "tail_lpips", "clips", "steps"}. Two backends, selected automatically: - MODAL (FRONTIER_NWM_BACKEND=modal or a Modal token is present): runs the @@ -10,8 +10,18 @@ - LOCAL (default when a CUDA device is visible): runs the runner directly on the local GPU. Used for Della validation and any GPU-equipped judge. -The unpatched baseline is computed once and cached (baseline_cache_path); the -patched run always re-applies the agent patch to a clean checkout. +Baseline measurement (audit P1, common random numbers): + - role="final" (the SCORED verifier run): baseline and patched are measured + back-to-back on ONE GPU in one job (no cache). They run as two separate + rollout subprocesses (not one process), so what makes a no-op score ~0 is + op-level bit-stability -- the determinism flags in rollout.py/evaluate_metrics.py + holding for EVERY op, plus the per-clip seed. Pairing removes cross-run, + cross-container, cross-cache and cross-hardware variance; it does NOT by + itself cancel op-level nondeterminism. Prove the flags actually hold once on + H100 with FRONTIER_NWM_STRICT_DETERMINISM=1 (a fallback then raises). + - role="agent" (cheap iterative feedback): the unpatched baseline is cached, + keyed by a full config fingerprint; deterministic kernels + per-clip seeding + keep that cached baseline a valid CRN partner on the fixed (now H100) SKU. """ from __future__ import annotations @@ -23,8 +33,12 @@ def _compose_config() -> Path: - """Write the fixed CSGO rollout config (subset val list) the runner consumes.""" - out = Path(os.environ.get("FRONTIER_NWM_WORKDIR", "/tmp/nwm_speedup")) / "csgo_config.yaml" + """Write the fixed CSGO rollout config (subset val list) the runner consumes. + + The filename carries the config fingerprint so a changed task config never + silently reuses a stale composed config from a long-lived container.""" + workdir = Path(os.environ.get("FRONTIER_NWM_WORKDIR", "/tmp/nwm_stability")) + out = workdir / f"csgo_config_{S.config_fingerprint()}.yaml" out.parent.mkdir(parents=True, exist_ok=True) if out.exists(): return out @@ -54,7 +68,44 @@ def _run_modal(patch_path: Path | None, clips: int) -> dict: on the baked NanoWM checkout and returns the metrics dict.""" from .modal_app import run_rollout_remote # deploys/looks up the Modal app patch_text = "" if patch_path is None else Path(patch_path).read_text(errors="replace") - return run_rollout_remote(patch_text=patch_text, clips=clips, steps=S.NUM_SAMPLING_STEPS) + return run_rollout_remote(patch_text=patch_text, clips=clips, steps=S.NUM_SAMPLING_STEPS, + strict=S.STRICT_DETERMINISM) + + +def _run_pair_local(patch_path: Path, clips: int, rollout_length: int | None = None) -> tuple[dict, dict]: + """Baseline then patched, back-to-back on the same local GPU (two rollout + subprocesses; op-level determinism is what makes them comparable). Both arms + receive the SAME rollout_length (the per-run randomized horizon, audit #7) so + the CRN pair is measured at one identical, agent-unpredictable horizon.""" + from . import runner + cfg = _compose_config() + baseline = runner.evaluate(S.REPO, None, cfg, S.CKPT, clips=clips, + steps=S.NUM_SAMPLING_STEPS, device="cuda", + rollout_length=rollout_length) + patched = runner.evaluate(S.REPO, Path(patch_path), cfg, S.CKPT, clips=clips, + steps=S.NUM_SAMPLING_STEPS, device="cuda", + rollout_length=rollout_length) + return baseline, patched + + +def _run_pair_modal(patch_path: Path, clips: int, rollout_length: int | None = None) -> tuple[dict, dict]: + """Scored pair, FANNED OUT across Modal containers one clip chunk each (the + per-batch seed keys on the global clip index, so chunks are bit-identical to a + single sequential run -- see modal_app.run_pair_remote). Lets the 22-clip pair + finish in ~one chunk's wall-time instead of ~10h sequentially. The randomized + horizon (audit #7) is decided HERE on the judge and threaded into every chunk + spec so all chunks AND both arms share one identical, agent-unpredictable + horizon (chunk bit-identity + CRN preserved).""" + from .modal_app import run_pair_remote + patch_text = Path(patch_path).read_text(errors="replace") + # chunk_size MUST be a multiple of the rollout batch_size so chunk offsets land + # on global batch boundaries (bit-identity); round up if a non-multiple is set. + bs = max(1, S.BATCH_SIZE) + chunk_size = ((max(bs, S.CHUNK_SIZE) + bs - 1) // bs) * bs + res = run_pair_remote(patch_text=patch_text, clips=clips, steps=S.NUM_SAMPLING_STEPS, + strict=S.STRICT_DETERMINISM, chunk_size=chunk_size, + rollout_length=rollout_length) + return res["baseline"], res["patched"] def _backend(): @@ -68,17 +119,20 @@ def _backend(): def _baseline(clips: int, run) -> dict: cache = S.BASELINE_CACHE + fp = S.config_fingerprint() if cache.exists(): try: data = json.loads(cache.read_text()) - # Deterministic seeding makes the cached baseline a valid common-random - # -numbers partner ONLY for the same clip set (== not >=) and same seed. - if data.get("clips") == clips and data.get("seed") == S.SEED: + # Valid cached CRN partner only for the IDENTICAL config (fingerprint + # covers model/dataset/rollout/steps/scheduling/stab/batch/seed/val set) + # and the same clip count; deterministic kernels keep it bit-stable. + if data.get("fingerprint") == fp and data.get("clips") == clips: return data except Exception: pass m = run(None, clips) m["seed"] = S.SEED + m["fingerprint"] = fp try: cache.parent.mkdir(parents=True, exist_ok=True) cache.write_text(json.dumps(m)) @@ -88,7 +142,26 @@ def _baseline(clips: int, run) -> dict: def run_pair(patch_path: Path, clips: int, role: str = "agent") -> tuple[dict, dict]: - run = _run_modal if _backend() == "modal" else _run_local + backend = _backend() + if role == "final": + # Scored run: baseline and patched measured back-to-back on one GPU in one + # job (no cache). Removes cross-run/-container/-cache/-hardware variance; + # the no-op->0 guarantee additionally needs the determinism flags to hold + # for every op (prove once on H100 with FRONTIER_NWM_STRICT_DETERMINISM=1). + # Draw the SCORED horizon ONCE here (audit #7) and thread the SAME value to + # both arms / all chunks: a hardcoded frame-position counter (the codex + # tail-targeting hack) cannot have predicted it (the agent-role QUICK loop + # ran at the nominal horizon), so it can no longer dump stabilization on the + # scored tail. Logged for operator reproducibility (pin via the nonce env). + roll_len = S.draw_final_rollout_length() + print(f"[run_pair] scored horizon (audit #7 randomized) rollout_length={roll_len} " + f"tail_start={S.tail_start_for(roll_len)} (nominal {S.ROLLOUT_LENGTH}/{S.DRIFT_TAIL_START})") + if backend == "modal": + return _run_pair_modal(Path(patch_path), clips, rollout_length=roll_len) + return _run_pair_local(Path(patch_path), clips, rollout_length=roll_len) + # Iterative (agent) feedback: cached baseline for speed; determinism + seeding + # keep it a valid CRN partner on the fixed GPU SKU. + run = _run_modal if backend == "modal" else _run_local baseline = _baseline(clips, run) patched = run(Path(patch_path), clips) return baseline, patched diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py b/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py index 9474ea5e..bb738bf7 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/runner.py @@ -4,8 +4,9 @@ The judge fixes the rollout invocation (rollout_length / history / num_steps / scheduling); the agent's patch may only change the *sampling internals* (`src/diffusion/**`, `src/sample/sampling_utils.py`) to make that invocation -reduce drift at iso-wall-clock. Speedup = baseline_seconds / patched_seconds; quality = -LPIPS vs ground truth, guardrailed against the unpatched baseline. +reduce drift at iso-wall-clock. Score = relative reduction in tail-frame LPIPS-vs-GT +over the unpatched baseline, gated by a wall-clock guardrail (patched gen time may +not exceed the baseline's by more than the tolerance). Runnable standalone for local/Modal validation: python -m stability_eval.runner --repo --patch \ @@ -27,7 +28,7 @@ def apply_patch(clean_repo: Path, patch_path: Path | None) -> Path: - work = Path(tempfile.mkdtemp(prefix="nwm_speedup_")) + work = Path(tempfile.mkdtemp(prefix="nwm_stability_")) repo = work / "repo" # symlink the heavy/immutable parts, copy only src/ so patches are cheap repo.mkdir(parents=True) @@ -37,31 +38,115 @@ def apply_patch(clean_repo: Path, patch_path: Path | None) -> Path: else: (repo / child.name).symlink_to(child) if patch_path is not None and str(patch_path) != "-" and Path(patch_path).stat().st_size > 0: + # The upstream checkout ships CRLF .py files, but a submitted/reference diff + # is usually LF -- and the patch text is round-tripped through universal- + # newline read_text() (orchestrate -> Modal) which strips CR. Mismatched + # endings make git/patch reject with "patch does not apply (different line + # endings)", so EVERY submission would score 0. Normalise BOTH the copied + # target tree and the patch to LF so application is line-ending-agnostic + # (LF .py executes identically; only the copy is touched, not the baseline). + for py in (repo / "src").rglob("*.py"): + b = py.read_bytes() + if b"\r" in b: + py.write_bytes(b.replace(b"\r\n", b"\n").replace(b"\r", b"\n")) + patch_lf = work / "patch.diff" + patch_lf.write_bytes(Path(patch_path).read_bytes().replace(b"\r\n", b"\n").replace(b"\r", b"\n")) r = subprocess.run(["git", "apply", "--unsafe-paths", "--directory", str(repo), - str(patch_path)], capture_output=True, text=True) + str(patch_lf)], capture_output=True, text=True) if r.returncode != 0: # fall back to patch(1) (git apply needs a git root / clean context) - r = subprocess.run(["patch", "-p1", "-d", str(repo), "-i", str(patch_path)], + r = subprocess.run(["patch", "-p1", "-d", str(repo), "-i", str(patch_lf)], capture_output=True, text=True) if r.returncode != 0: - raise RuntimeError(f"patch failed: {r.stderr[:2000]}") + # patch(1) reports to stdout, git to stderr -- surface both. + raise RuntimeError(f"patch failed: {(r.stdout + r.stderr)[:2000]}") + # Inject the frozen-model guard into the (denied, judge-owned) rollout.py AFTER the + # agent patch, enforcing the model-frozen invariant at runtime (the static policy + # cannot see a sampler-side runtime model monkeypatch). Matters more here than for + # speedup: stability has NO patched-vs-baseline faithfulness backstop, so this is + # the main guard against a persistent model mutation winning drift reduction. + inject_frozen_guard(repo) return repo +# The frozen-model guard, inlined into rollout.py at apply-time (self-contained: no +# import of stability_eval from the rollout subprocess). Snapshot the params right +# after the model is loaded+eval'd; re-check after the rollout. A restored transient +# reshape (a causal-prefix-style temp_embed slice) passes; a persistently mutated model +# hard-errors -> nonzero exit -> scored as a submission execution failure (0). +_FG_SNAPSHOT = ( + " # _FROZEN_GUARD (judge infra, injected by stability_eval.runner; outside the\n" + " # agent's editable scope): the model is FROZEN -- a sampling patch may READ it\n" + " # but not PERSISTENTLY mutate it. Snapshot its params now; re-check after the\n" + " # rollout. A restored transient reshape (causal-prefix temp_embed) passes.\n" + " import hashlib as _hl_fg\n" + " def _fg_fp(_m):\n" + " _h = _hl_fg.sha256()\n" + " for _n, _p in sorted(_m.state_dict().items()):\n" + " _h.update(('%s|%s|%s' % (_n, tuple(_p.shape), _p.dtype)).encode())\n" + " _h.update(repr(float(_p.detach().to('cpu', dtype=torch.float64).sum())).encode())\n" + " return _h.hexdigest()\n" + " _fg_frozen = _fg_fp(model)\n" +) +_FG_VERIFY = ( + " # _FROZEN_GUARD: enforce the frozen-model invariant after the rollout.\n" + " if _fg_fp(model) != _fg_frozen:\n" + " raise RuntimeError('frozen-model violation: the sampling patch persistently '\n" + " 'mutated the model parameters (the model is frozen; denied)')\n" +) + + +def inject_frozen_guard(repo: Path) -> None: + """Insert the frozen-model guard into the copied src/sample/rollout.py. Anchors: + snapshot just after `model.eval()`, verify just before the final `print("Done!...")` + (after the rollout loop). FAIL CLOSED: if the anchors are not found (rollout.py + structure changed) the guard cannot be placed, so RAISE rather than run unguarded.""" + f = repo / "src" / "sample" / "rollout.py" + if not f.exists(): + return + src = f.read_text() + if "_fg_frozen" in src: # idempotent + return + lines = src.splitlines(keepends=True) + snap_at = verify_at = None + for i, ln in enumerate(lines): + s = ln.strip() + if snap_at is None and s == "model.eval()": + snap_at = i + 1 + if verify_at is None and (s.startswith('print(f"Done! Processed') + or s.startswith("print(f'Done! Processed") + or s.startswith('print("Done! Processed')): + verify_at = i + if snap_at is None or verify_at is None or verify_at <= snap_at: + raise RuntimeError( + "frozen-model guard injection failed: rollout.py anchors not found " + f"(model.eval()@{snap_at}, Done@{verify_at}) -- refusing to run unguarded") + lines.insert(verify_at, _FG_VERIFY) # insert the later one first (indices stable) + lines.insert(snap_at, _FG_SNAPSHOT) + f.write_text("".join(lines)) + + def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, steps: int, - device: str = "cuda") -> float: + device: str = "cuda", clip_offset: int = 0, rollout_length: int | None = None) -> float: """Run rollout.py; return generation wall-clock seconds (rollout-region timed - inside; falls back to total wall).""" + inside; falls back to total wall). clip_offset processes only the GLOBAL clip + window [clip_offset, clip_offset+clips) so a scored pair can fan out across + Modal containers bit-identically (the per-batch seed keys on the global index). + rollout_length defaults to the nominal S.ROLLOUT_LENGTH; the scored run passes a + per-run RANDOM horizon (audit #7) so a hardcoded frame-position counter cannot + target the tail. Both arms of a CRN pair receive the SAME value.""" save.mkdir(parents=True, exist_ok=True) + roll_len = int(rollout_length) if rollout_length is not None else S.ROLLOUT_LENGTH env = dict(os.environ) env["CSGO_DATA_DIR"] = str(S.CSGO_DATA) env["NWM_TIME_FILE"] = str(save / "gen_seconds.txt") + _apply_determinism_env(env) cmd = [sys.executable, "-u", "src/sample/rollout.py", "--config", str(config), "--ckpt", str(ckpt), "--save_path", str(save), "--num_samples", str(clips), - "--batch_size", str(S.BATCH_SIZE), "--rollout_length", str(S.ROLLOUT_LENGTH), + "--batch_size", str(S.BATCH_SIZE), "--rollout_length", str(roll_len), "--history_length", str(S.HISTORY_LENGTH), "--scheduling_mode", S.SCHEDULING_MODE, "--num_sampling_steps", str(steps), "--history_stabilization_level", str(S.HISTORY_STAB), - "--fps", "8", "--seed", str(S.SEED)] + "--fps", "8", "--seed", str(S.SEED), "--clip_offset", str(clip_offset)] t0 = time.time() r = subprocess.run(cmd, cwd=str(repo), env=env, capture_output=True, text=True) wall = time.time() - t0 @@ -76,33 +161,72 @@ def run_rollout(repo: Path, config: Path, ckpt: Path, save: Path, clips: int, st return wall -def lpips_vs_gt(repo: Path, save: Path) -> tuple[float, float]: - """Returns (mean_lpips, tail_lpips) where tail = frames >= S.DRIFT_TAIL_START - (the drifted region of a long rollout).""" +def _apply_determinism_env(env: dict) -> None: + """Determinism vars for the GPU subprocesses (rollout + the LPIPS metric). + cuBLAS needs CUBLAS_WORKSPACE_CONFIG set before the first CUDA call, so it + lives in the subprocess env, not toggled in-process. Pairs with the + deterministic kernels enabled in rollout.py / evaluate_metrics.py so the + baseline and patched arms share a bit-stable pipeline (common random numbers). + NWM_DETERMINISM_STRICT=1 makes a missing deterministic kernel RAISE (proof + mode) instead of silently running nondeterministically.""" + env.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + env.setdefault("PYTHONHASHSEED", "0") + env["NWM_DETERMINISM_STRICT"] = "1" if S.STRICT_DETERMINISM else "0" + + +def lpips_vs_gt(repo: Path, save: Path, tail_start: int | None = None) -> tuple[float, float, int]: + """Returns (mean_lpips, tail_lpips, n_clips) where tail = frames >= tail_start + (the drifted region of a long rollout; defaults to the nominal S.DRIFT_TAIL_START, + but the scored run passes the tail derived from its randomized horizon, audit #7). + n_clips is the count of clips ACTUALLY produced (unique sample_id), so a fanned-out + chunk is weighted by its REAL count -- the long-rollout headroom filter can drop an + episode, leaving a chunk short, and weighting by the requested count would corrupt + the grand mean.""" out_csv = save / "metrics.csv" + env = {**os.environ, "CSGO_DATA_DIR": str(S.CSGO_DATA)} + _apply_determinism_env(env) r = subprocess.run([sys.executable, "src/sample/evaluate_metrics.py", "--video_dir", str(save), "--history_length", str(S.HISTORY_LENGTH), "--output_csv", str(out_csv)], - cwd=str(repo), env={**os.environ, "CSGO_DATA_DIR": str(S.CSGO_DATA)}, + cwd=str(repo), env=env, capture_output=True, text=True) if not out_csv.exists(): raise RuntimeError(f"metrics failed: {r.stderr[-2000:]}") import pandas as pd - df = pd.read_csv(out_csv) + try: + df = pd.read_csv(out_csv) + except Exception as e: # column-less/empty CSV from an empty video dir + raise RuntimeError(f"metrics produced an unreadable CSV ({e}); stderr: {r.stderr[-1000:]}") + if df.empty or "lpips" not in df.columns or "sample_id" not in df.columns: + raise RuntimeError(f"metrics CSV has no clips/rows (empty rollout?); stderr: {r.stderr[-1000:]}") + n_clips = int(df["sample_id"].nunique()) mean = float(df["lpips"].mean()) - tail_start = getattr(S, "DRIFT_TAIL_START", 60) + ts = int(tail_start) if tail_start is not None else getattr(S, "DRIFT_TAIL_START", 60) g = df.groupby("frame_idx")["lpips"].mean() - tail = float(g[g.index >= tail_start].mean()) if (g.index >= tail_start).any() else mean - return mean, tail + tail = float(g[g.index >= ts].mean()) if (g.index >= ts).any() else mean + return mean, tail, n_clips def evaluate(clean_repo: Path, patch_path: Path | None, config: Path, ckpt: Path, - clips: int, steps: int, device: str = "cuda") -> dict: + clips: int, steps: int, device: str = "cuda", clip_offset: int = 0, + rollout_length: int | None = None) -> dict: repo = apply_patch(clean_repo, patch_path) + roll_len = int(rollout_length) if rollout_length is not None else S.ROLLOUT_LENGTH + tail_start = S.tail_start_for(roll_len) try: save = repo / "_rollout_out" - secs = run_rollout(repo, config, ckpt, save, clips, steps, device) - lp, tail = lpips_vs_gt(repo, save) - return {"gen_seconds": secs, "lpips": lp, "tail_lpips": tail, "clips": clips, "steps": steps} + secs = run_rollout(repo, config, ckpt, save, clips, steps, device, + clip_offset=clip_offset, rollout_length=roll_len) + # An empty clip window (clip_offset at/over the valid-slice count) produces + # no videos; report 0 clips so the fan-out aggregator skips it cleanly + # instead of crashing the metric subprocess on a column-less CSV. + if not list(save.glob("*_gen.mp4")): + return {"gen_seconds": secs, "lpips": None, "tail_lpips": None, "clips": 0, + "steps": steps, "rollout_length": roll_len, "tail_start": tail_start} + lp, tail, n_clips = lpips_vs_gt(repo, save, tail_start=tail_start) + # 'clips' is the ACTUAL processed count (not the requested arg), so a chunk + # short of its request is weighted/divided correctly downstream. + return {"gen_seconds": secs, "lpips": lp, "tail_lpips": tail, "clips": n_clips, + "steps": steps, "rollout_length": roll_len, "tail_start": tail_start} finally: shutil.rmtree(repo.parent, ignore_errors=True) diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/scoring.py b/2.0/problems/nanowm_rollout_stability/stability_eval/scoring.py index 6b8e5b22..9eff56a3 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/scoring.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/scoring.py @@ -16,24 +16,50 @@ def drift_reduction_score(baseline_tail: float, patched_tail: float) -> float: return max(0.0, min(100.0, 100.0 * rel)) +def wallclock_multiplier_from_rel(rel_over: float, tolerance: float, grace: float = 0.02) -> float: + """Wall-clock guardrail credit given the RELATIVE over-budget (patched vs + baseline gen time). The task is ISO-WALL-CLOCK drift reduction, so credit DECAYS + as gen time rises -- NOT a flat 1.0 plateau up to +tolerance (which let drift be + bought with up to `tolerance` extra compute for free, contradicting the premise). + A small `grace` absorbs sub-effect timing jitter; past it the credit decays + smoothly as mult = 1/(1 + (rel_over-grace)/tolerance): ~0.556 at `tolerance` over + parity, exactly 0.5 at `grace+tolerance`, and ->0 beyond. A speedup (rel_over<=0) + keeps full credit.""" + rel_over = max(0.0, rel_over) + if rel_over <= grace: + return 1.0 + x = (rel_over - grace) / max(tolerance, 1e-9) + return max(0.0, min(1.0, 1.0 / (1.0 + x))) + + def wallclock_multiplier(baseline_seconds: float, patched_seconds: float, tolerance: float) -> float: - """1.0 while patched wall-clock stays within `tolerance` of baseline; decays - inverse-proportionally beyond (can't buy drift reduction with more compute).""" base = max(baseline_seconds, 1e-9) - rel_over = max(0.0, (patched_seconds - baseline_seconds) / base) - if rel_over <= tolerance: - return 1.0 - return max(0.0, min(1.0, tolerance / rel_over)) + return wallclock_multiplier_from_rel((patched_seconds - baseline_seconds) / base, tolerance) -def provisional_score(baseline_tail, patched_tail, baseline_seconds, patched_seconds, tolerance): +def provisional_score(baseline_tail, patched_tail, baseline_seconds, patched_seconds, tolerance, + wallclock_rel_over=None): drift_score = drift_reduction_score(baseline_tail, patched_tail) - wmult = wallclock_multiplier(baseline_seconds, patched_seconds, tolerance) + # Prefer the clip-weighted mean of PER-CHUNK over-budget ratios (each measured on + # ONE board) when the fan-out provides it -- fairer than a ratio of gen_seconds + # SUMMED across heterogeneous H100 boards; else fall back to the summed-time ratio. + if wallclock_rel_over is not None: + rel_over = max(0.0, float(wallclock_rel_over)) + else: + base = max(baseline_seconds, 1e-9) + rel_over = max(0.0, (patched_seconds - baseline_seconds) / base) + wmult = wallclock_multiplier_from_rel(rel_over, tolerance) return { "tail_drift_reduction": (baseline_tail - patched_tail), "rel_reduction_pct": (100.0 * (baseline_tail - patched_tail) / max(baseline_tail, 1e-9)), "drift_score": drift_score, + "wallclock_rel_over": rel_over, "wallclock_multiplier": wmult, "score": max(0.0, min(100.0, drift_score * wmult)), - "score_unbounded": max(0.0, drift_score * wmult), + # Unlike the speedup task (whose 100*log2(speedup) is genuinely uncapped + # beyond 2x), a RELATIVE drift reduction is intrinsically in [0, 100] + # (patched_tail in [0, baseline_tail]), so score_unbounded == score by + # construction. Kept for a uniform (score, score_unbounded) return shape + # across the two tasks. + "score_unbounded": max(0.0, min(100.0, drift_score * wmult)), } diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py b/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py index a727513e..d4f900f4 100644 --- a/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/settings.py @@ -1,4 +1,4 @@ -"""Task settings for nanowm_rollout_speedup, overridable via config.yaml +"""Task settings for nanowm_rollout_stability, overridable via config.yaml `evaluation` block (read by the judge) or env vars (local runs).""" from __future__ import annotations @@ -40,14 +40,90 @@ def _get(name: str, default): NUM_SAMPLING_STEPS = int(_get("FRONTIER_NWM_NUM_STEPS", 50)) # fixed compute budget SCHEDULING_MODE = _get("FRONTIER_NWM_SCHEDULING", "sequential") HISTORY_STAB = float(_get("FRONTIER_NWM_HISTORY_STAB", 0.02)) # baseline default -# Drift metric: mean LPIPS over the drifted tail (frames >= this index). +# Drift metric: mean LPIPS over the drifted tail (frames >= this index). This is +# the NOMINAL tail used by the cached agent-role path; the SCORED (role=final) run +# derives its tail from the randomized horizon below (tail_start_for). DRIFT_TAIL_START = int(_get("FRONTIER_NWM_DRIFT_TAIL_START", 60)) + +# --- Anti-overfit: randomized SCORED horizon (audit #7) ---------------------- +# A patch can only target the scored tail window if it knows the rollout's +# absolute frame positions. `df_sample` (the agent's editable scope) never +# receives rollout_length -- only the fixed model window -- so a submission can +# only HARDCODE it: the codex 7.83% run hardcoded a module counter with period 76 +# (== ROLLOUT_LENGTH-HISTORY_LENGTH) and ramped EXTRA history-stabilization onto +# frames >= DRIFT_TAIL_START, i.e. it overfit the fixed judge horizon to dump its +# intervention exactly on the scored tail. Defeat that by drawing the SCORED +# horizon at random per run from [ROLLOUT_LENGTH_MIN, ROLLOUT_LENGTH] and deriving +# the tail from it. Drawn DOWNWARD only (<= nominal) so no extra GT headroom is +# needed and the held-out clip count is unchanged; decided ONCE on the judge and +# threaded to every arm/chunk so CRN pairing and chunk bit-identity hold (both +# arms + all chunks share one horizon; the per-batch global-index seed is +# untouched). A hardcoded-period counter then no longer maps to frame position +# (L-HISTORY != 76, so the phase drifts across clips and the ramp sprays at random +# frames), collapsing position-targeting back to a general -- legitimate -- drift +# reducer (e.g. an input-staleness-keyed schedule), which is exactly the task. +# The SCORED horizon is drawn from [MIN, MAX] with MAX well BELOW the nominal +# ROLLOUT_LENGTH (the value the agent-role QUICK loop runs at and can therefore +# measure during dev). What this defeats: the codex 7.83% submission HARDCODED a +# counter period 76 (== nominal 80 - HISTORY 4) and a frame-64 ramp tuned to the +# disclosed horizon 80, dumping extra stabilization exactly on the scored tail. The +# margin matters -- empirically (test_antihack_horizon) that fixed ramp stays +# ~aligned for horizons within ~6 of 80, but for horizons <=72 it MISFIRES (its +# expected tail-targeting over a [64,72] band is ~0, even slightly NEGATIVE), so +# the concrete shipped overfit is neutralized. `df_sample` never receives the +# horizon, so a submission can only hardcode it; randomizing (undisclosed, < the +# measurable nominal) leaves only a HORIZON-ROBUST "stabilize deeper frames more" +# schedule, which responds to genuine accumulated drift and generalizes -- i.e. the +# legitimate behaviour the hack was imitating, not the overfit. MAX <= nominal keeps +# the held-out clip count and GT headroom unchanged (shorter rollouts need <= +# headroom). NOTE: lowering the horizon moves the scored tail earlier; the relative +# drift metric is ~horizon-robust but the band's SNR must be re-confirmed on H100 +# (the calibrated effect was measured at tail>=60); MIN/MAX are env-tunable to widen +# the margin (stronger anti-hack) or raise it (closer to the calibrated SNR). +ROLLOUT_LENGTH_MIN = int(_get("FRONTIER_NWM_ROLLOUT_LENGTH_MIN", 64)) +ROLLOUT_LENGTH_MAX = int(_get("FRONTIER_NWM_ROLLOUT_LENGTH_MAX", 72)) +TAIL_FRAMES = int(_get("FRONTIER_NWM_TAIL_FRAMES", 20)) # scored tail = last N frames + + +def draw_final_rollout_length() -> int: + """Pick the SCORED rollout horizon for one run. Random per run (fresh entropy) + so the agent cannot have measured/hardcoded it during the agent-role QUICK loop + (which stays at the nominal ROLLOUT_LENGTH); pin via FRONTIER_NWM_ROLLOUT_NONCE + for reproducible validation. Bounded to [ROLLOUT_LENGTH_MIN, ROLLOUT_LENGTH_MAX] + with MAX < nominal (so GT headroom and the held-out clip count are unchanged AND + the scored horizon never equals the agent-measurable nominal). A band of width 0 + (MIN >= MAX) disables randomization and pins to MAX (e.g. CI / pinned validation).""" + import random + lo = min(ROLLOUT_LENGTH_MIN, ROLLOUT_LENGTH_MAX) + hi = min(ROLLOUT_LENGTH_MAX, ROLLOUT_LENGTH) # never exceed the nominal headroom + if lo >= hi: + return hi + nonce = (os.environ.get("FRONTIER_NWM_ROLLOUT_NONCE") or "").strip() + rng = random.Random(nonce) if nonce else random.Random() + return rng.randint(lo, hi) + + +def tail_start_for(rollout_length: int) -> int: + """Scored tail window = the last TAIL_FRAMES frames of the (possibly + randomized) horizon, floored at 1 so the filter always selects >=1 frame.""" + return max(1, int(rollout_length) - TAIL_FRAMES) # Eval set sizes — drift reductions are small, so enough clips to resolve them # above per-clip noise (validated under CRN pairing: 3 seeds x 22 clips give # pooled paired t=5.15, p<1e-4 for the reference). QUICK_CLIPS = int(_get("FRONTIER_NWM_QUICK_CLIPS", 8)) -FINAL_CLIPS = int(_get("FRONTIER_NWM_FINAL_CLIPS", 24)) +# Full held-out set = the 22 test_split episodes <=200 staged from the 1-200 chunk +# (>22 indexes past the sliced dataset). The scored final uses all 22 for SNR; the +# 80-frame paired rollout (~10h sequentially under strict determinism) is made +# affordable by fanning the clips out across Modal containers (CHUNK_SIZE each), +# bit-identically to the sequential run (per-batch seed keys on the global clip +# index). batch_size=2 => QUICK(8) stays a noise-identical prefix of FINAL. +FINAL_CLIPS = int(_get("FRONTIER_NWM_FINAL_CLIPS", 22)) BATCH_SIZE = int(_get("FRONTIER_NWM_BATCH_SIZE", 2)) +# Clips per Modal container when the scored (role=final) pair is fanned out (see +# modal_app.run_pair_remote). Rounded up to a multiple of BATCH_SIZE in orchestrate +# so chunk offsets land on global batch boundaries (-> bit-identical to sequential). +# 22 clips / 4 => 6 parallel chunks; smaller = more parallelism, more load overhead. +CHUNK_SIZE = int(_get("FRONTIER_NWM_CHUNK_SIZE", 4)) # Deterministic RNG seed (judge infra): per-clip seed = SEED + clip_index, so the # baseline (unpatched) and patched arms draw identical initial noise per clip # (common random numbers). Makes a no-op patch score exactly 0 and the cached @@ -65,3 +141,30 @@ def _get(name: str, default): VAL_STARTS = Path(_get("FRONTIER_NWM_VAL_STARTS", "/opt/nanowm/data/csgo_subset/val_starts.npy")) BASELINE_CACHE = Path(_get("FRONTIER_NWM_BASELINE_CACHE", "/opt/nanowm/baseline/baseline_metrics.json")) SMOKE = os.environ.get("FRONTIER_NWM_SMOKE", "0") == "1" +# Determinism proof switch (judge infra). 0 (default): a missing deterministic +# kernel WARNS and runs nondeterministically. 1: it RAISES instead -- run once on +# H100 to PROVE the model+VAE+metric have no nondeterministic op (and re-establish +# no-op CRN pair -> 0.0 there); production may then stay strict (a fallback -> an +# infra error, never a silently corrupted score) or relax once proven clean. +def _as_bool(v) -> bool: + if isinstance(v, str): + return v.strip().lower() in ("1", "true", "yes", "on") + return bool(v) +STRICT_DETERMINISM = _as_bool(_get("FRONTIER_NWM_STRICT_DETERMINISM", 0)) + + +def config_fingerprint() -> str: + """Stable hash of every knob that affects the unpatched baseline. Used to key + the baseline cache and the composed-config filename so a cached baseline is + only ever reused as a common-random-numbers partner for the IDENTICAL config + (changing any of these invalidates the cache instead of silently mispairing).""" + import hashlib + keys = { + "model": MODEL, "dataset": DATASET, "rollout_length": ROLLOUT_LENGTH, + "history_length": HISTORY_LENGTH, "num_steps": NUM_SAMPLING_STEPS, + "scheduling": SCHEDULING_MODE, "history_stab": HISTORY_STAB, + "batch_size": BATCH_SIZE, "seed": SEED, "drift_tail_start": DRIFT_TAIL_START, + "val_files": str(VAL_FILES), "val_starts": str(VAL_STARTS), + } + blob = json.dumps(keys, sort_keys=True) + return hashlib.sha256(blob.encode()).hexdigest()[:16] diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/test_antihack_horizon.py b/2.0/problems/nanowm_rollout_stability/stability_eval/test_antihack_horizon.py new file mode 100644 index 00000000..ba206f11 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/test_antihack_horizon.py @@ -0,0 +1,154 @@ +"""Proof that the randomized SCORED horizon (audit #7) neutralizes the codex +tail-targeting reward-hack as shipped, and a characterization of what survives. + +Background. The codex 7.83% stability submission added a MODULE-GLOBAL call counter +and ramped EXTRA history-stabilization onto the rollout's late frames: + + _STABILITY_ROLLOUT_PERIOD = 76 # == ROLLOUT_LENGTH(80) - HISTORY_LENGTH(4) + rollout_phase = _stability_rollout_call % 76 + extra = 0.08 * clamp((rollout_phase - 32) / 28, 0, 1) # ramps onto late frames + +`df_sample` is called once per generated frame, so at the FIXED, DISCLOSED nominal +horizon 80 every clip generates exactly 76 frames => the counter aligns to the +generated-frame index => the ramp lands squarely and repeatably on the scored tail +(frames >= DRIFT_TAIL_START). That hardcoded period/ramp is overfit to the disclosed +judge horizon. + +The fix (settings.draw_final_rollout_length) draws the scored horizon at random per +run from [MIN, MAX], MAX well below the nominal the agent measures during dev, and +`df_sample` NEVER receives the horizon. So the hardcoded period 76 and the frame-64 +ramp no longer match the run: this test shows their EXPECTED tail-targeting over the +band collapses to ~0 (even slightly negative -- the fixed ramp misfires onto +non-tail frames at the shorter horizons). + +What SURVIVES (reported, not failed-on): a HORIZON-ROBUST schedule -- "apply more +history-stabilization to deeper frames" keyed on a runtime signal (per-clip call +index, context staleness) rather than a hardcoded period -- can still raise tail +quality. But that is the LEGITIMATE behaviour the hack imitated: it responds to +genuine accumulated drift, generalizes across horizons/content, and faithfully +cleans the tail via less-contaminated context (LPIPS-vs-GT rewards real fidelity). +Randomization converts the overfit into either-legit-or-broken, which is the point. +The remaining single-knob shallowness is audit #6 (a task-depth issue, not a +reward-hack); see AUDIT_REPORT. + +Run: python -m stability_eval.test_antihack_horizon (exits non-zero on regression) +""" +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from stability_eval import settings as S # noqa: E402 + +HISTORY = 4 +N_CLIPS = 22 +CHUNK = 4 # fan-out chunk (clips per Modal process); the module-global + # counter persists within a chunk and resets between chunks. +TAIL_FRAMES = S.TAIL_FRAMES +PEAK = 0.08 # the codex ramp peak; absolute scale is irrelevant (gaps compared) + + +def _clamp(x, lo=0.0, hi=1.0): + return max(lo, min(hi, x)) + + +def _phase_hist(horizon: int, period: int): + """Replay the module-global counter for one run at `horizon` (resetting every + CHUNK clips, as the fan-out does); return (scored_mass, nonscored_mass) over the + `period` phases, each normalized to sum 1.""" + gen = horizon - HISTORY + tail_start_abs = horizon - TAIL_FRAMES + sc = [0] * period + nc = [0] * period + for c in range(N_CLIPS): + if c % CHUNK == 0: + call = 0 + for k in range(gen): + ph = call % period + (sc if HISTORY + k >= tail_start_abs else nc)[ph] += 1 + call += 1 + st, nt = sum(sc), sum(nc) + return ([x / st for x in sc] if st else sc, [x / nt for x in nc] if nt else nc) + + +def _gap(extra_fn, horizon: int, period: int) -> float: + """Mean `extra` on SCORED tail frames minus mean `extra` on NON-scored frames. + >0 == the attack concentrates intervention on the scored frames.""" + sc, nc = _phase_hist(horizon, period) + return sum(extra_fn(ph) * (sc[ph] - nc[ph]) for ph in range(period)) + + +def codex_extra(phase: int) -> float: + return PEAK * _clamp((phase - 32) / 28) + + +def main() -> int: + band = list(range(min(S.ROLLOUT_LENGTH_MIN, S.ROLLOUT_LENGTH_MAX), + min(S.ROLLOUT_LENGTH_MAX, S.ROLLOUT_LENGTH) + 1)) + nominal = S.ROLLOUT_LENGTH + + # Ceiling: if the horizon is KNOWN, perfect targeting is trivial (period = + # frames-per-clip => phase == frame index; ramp full on the tail). + def known_gap(L): + return _gap(lambda ph: PEAK if ph >= (L - HISTORY - TAIL_FRAMES) else 0.0, + L, period=L - HISTORY) + ceiling = sum(known_gap(L) for L in band) / len(band) + + # The codex hack AS SHIPPED (hardcoded period 76, ramp 32->60). + codex_nominal = _gap(codex_extra, nominal, period=76) # on the old fixed judge + codex_band = {L: _gap(codex_extra, L, period=76) for L in band} # under randomization + codex_exp = sum(codex_band.values()) / len(band) + codex_worst = max(codex_band.values()) + + # Diagnostic only: the strongest HORIZON-ROBUST survivor (best fixed period + + # optimal phase set over the band). This is the legitimate adaptive-stabilization + # ceiling, NOT a pass/fail gate -- it responds to depth/drift and generalizes. + def adaptive_expected(period: int) -> float: + Sm = [0.0] * period + Nm = [0.0] * period + for L in band: + sc, nc = _phase_hist(L, period) + for ph in range(period): + Sm[ph] += sc[ph] / len(band) + Nm[ph] += nc[ph] / len(band) + return PEAK * sum(max(0.0, Sm[ph] - Nm[ph]) for ph in range(period)) + survivor = max(adaptive_expected(p) for p in range(20, 161)) + + print(f"band (scored horizon) : {band[0]}..{band[-1]} " + f"(nominal {nominal}; agent measures {nominal}; tail = last {TAIL_FRAMES})") + print(f"known-horizon ceiling : {ceiling:+.5f} (no defense -> perfect targeting)") + print(f"codex hack @ disclosed {nominal} : {codex_nominal:+.5f} (what it got on the old fixed judge)") + print(f"codex hack EXPECTED on band : {codex_exp:+.5f} ({100*codex_exp/ceiling:+.0f}% of ceiling)") + print(f"codex hack WORST draw on band : {codex_worst:+.5f} ({100*codex_worst/ceiling:+.0f}% of ceiling)") + print(f" per-horizon: " + ", ".join(f"{L}:{g:+.4f}" for L, g in codex_band.items())) + print(f"horizon-ROBUST survivor (legit): {survivor:+.5f} " + f"(adaptive 'stabilize deeper frames more' -- responds to drift, generalizes)") + + ok = True + # 1) Sanity: the hack genuinely worked at the disclosed nominal horizon. + if not (codex_nominal > 0.5 * ceiling): + print("FAIL: sanity -- codex hack should be strong at the disclosed nominal"); ok = False + # 2) Core claim: the shipped hardcoded hack's EXPECTED targeting collapses to a + # small fraction of the ceiling once the horizon is randomized below nominal. + if not (codex_exp < 0.15 * ceiling): + print(f"FAIL: codex hack still targets in expectation ({codex_exp:.4f} >= {0.15*ceiling:.4f}); " + f"widen the margin (lower ROLLOUT_LENGTH_MAX)"); ok = False + # 3) The draw is randomized over the documented band and excludes the measurable + # nominal and its neighbour (so the agent's dev-measured horizon never scores). + draws = {S.draw_final_rollout_length() for _ in range(6000)} + if draws != set(band): + print(f"FAIL: draws {sorted(draws)} != band {band}"); ok = False + if nominal in band or nominal - 1 in band: + print(f"FAIL: band must exclude the agent-measurable nominal {nominal} and its neighbour"); ok = False + # 4) df_sample provably cannot read the horizon -> a submission can only hardcode + # it (structural; enforced by the runner passing it only on the CLI to the + # frozen rollout.py, never into the sampler). Asserted as documentation here. + + print("\nRESULT:", "PASS - randomized horizon neutralizes the shipped tail-targeting hack; " + "only the legit horizon-robust schedule survives" if ok else "FAIL - regression") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/2.0/problems/nanowm_rollout_stability/stability_eval/test_inject_frozen_guard.py b/2.0/problems/nanowm_rollout_stability/stability_eval/test_inject_frozen_guard.py new file mode 100644 index 00000000..56add454 --- /dev/null +++ b/2.0/problems/nanowm_rollout_stability/stability_eval/test_inject_frozen_guard.py @@ -0,0 +1,132 @@ +"""Test runner.inject_frozen_guard: it must place a working frozen-model guard into +a copied rollout.py (snapshot after model.eval(), verify before the final print), +catch a persistent model mutation, allow a restored transient reshape (causal-prefix), +fail CLOSED when anchors are missing, and be idempotent. Runs without torch/GPU by +injecting into a synthetic rollout.py with a fake torch + stub model and exec-ing it. + +Run: python -m stability_eval.test_inject_frozen_guard (exit non-zero on regression) +""" +from __future__ import annotations + +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from stability_eval import runner # noqa: E402 + + +# A synthetic rollout.py: the loaded `model` is eval'd, the SAMPLER runs (and may +# mutate the model), then the final "Done!" print. {SAMPLER} is filled per-case. +SYNTH = ''' +import torch +def main(): + model = MODEL + model.eval() + for _b in range(2): + SAMPLER(model) + print(f"Done! Processed {{N}} samples.") +main() +''' + + +class _Stub: + def __init__(self, vals, shape=None, dtype="float32"): + self.vals = list(vals); self.shape = tuple(shape or (len(self.vals),)); self.dtype = dtype + def detach(self): return self + def to(self, *a, **k): return self + def sum(self): return sum(self.vals) + + +class _Model: + def __init__(self): self._sd = {"w": _Stub([0.1, 0.2, 0.3]), "temp_embed": _Stub([1.0, 2.0, 3.0, 4.0], (1, 4, 1))} + def state_dict(self): return self._sd + def eval(self): return self + + +class _FakeTorch: + float64 = "float64" + + +def _write_synth(tmp: Path) -> Path: + f = tmp / "src" / "sample" / "rollout.py" + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text(SYNTH) + return f + + +def _run_injected(repo: Path, sampler) -> tuple[bool, str]: + """Inject, then exec the resulting rollout.py with a fake torch/model/sampler. + Returns (raised, message).""" + runner.inject_frozen_guard(repo) + src = (repo / "src" / "sample" / "rollout.py").read_text() + compile(src, "rollout.py", "exec") # syntactic validity + g = {"MODEL": _Model(), "SAMPLER": sampler, "N": 2, "torch": _FakeTorch(), + "print": lambda *a, **k: None} + # shadow `import torch` inside the synthetic module with our fake + sys.modules.setdefault("torch", _FakeTorch()) + try: + exec(compile(src, "rollout.py", "exec"), g) + return False, "ok" + except RuntimeError as e: + return True, str(e) + + +def main() -> int: + ok = True + + def check(cond, msg): + nonlocal ok + if not cond: + print("FAIL:", msg); ok = False + + # 1) no-op sampler -> guard passes (no raise). + with tempfile.TemporaryDirectory() as d: + repo = Path(d); _write_synth(repo) + raised, msg = _run_injected(repo, lambda m: None) + check(not raised, f"no-op rollout wrongly flagged: {msg}") + + # 2) restored transient reshape (causal-prefix temp_embed) -> passes. + def transient(m): + sd = m.state_dict(); orig = sd["temp_embed"] + sd["temp_embed"] = _Stub(orig.vals[:2], (1, 2, 1)) # sliced during forward + sd["temp_embed"] = orig # restored in finally + with tempfile.TemporaryDirectory() as d: + repo = Path(d); _write_synth(repo) + raised, msg = _run_injected(repo, transient) + check(not raised, f"restored transient reshape wrongly flagged (blocks causal-prefix): {msg}") + + # 3) persistent mutation -> CAUGHT (raise). + def persistent(m): + m.state_dict()["w"] = _Stub([0.1, 0.2, 0.9]) # never restored + with tempfile.TemporaryDirectory() as d: + repo = Path(d); _write_synth(repo) + raised, msg = _run_injected(repo, persistent) + check(raised and "frozen-model violation" in msg, f"persistent mutation NOT caught: {msg}") + + # 4) idempotent: a second injection does not double-insert. + with tempfile.TemporaryDirectory() as d: + repo = Path(d); f = _write_synth(repo) + runner.inject_frozen_guard(repo) + once = f.read_text() + runner.inject_frozen_guard(repo) + check(f.read_text() == once and once.count("_fg_frozen = _fg_fp(model)") == 1, "injection not idempotent") + + # 5) FAIL CLOSED: missing anchor (no model.eval()) -> raise, do not run unguarded. + with tempfile.TemporaryDirectory() as d: + repo = Path(d); f = _write_synth(repo) + f.write_text('def main():\n x = 1\n print(f"Done! Processed 0 samples.")\nmain()\n') + caught = False + try: + runner.inject_frozen_guard(repo) + except RuntimeError as e: + caught = "anchors not found" in str(e) + check(caught, "missing-anchor injection did NOT fail closed") + + print("RESULT:", "PASS - frozen guard injects, catches persistent mutation, allows " + "restored reshape, fails closed" if ok else "FAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/adapters/frontier-cs-2.0/src/frontier_cs_2_0/task-template/environment/Dockerfile b/adapters/frontier-cs-2.0/src/frontier_cs_2_0/task-template/environment/Dockerfile index 2cc631d4..fee5c33d 100644 --- a/adapters/frontier-cs-2.0/src/frontier_cs_2_0/task-template/environment/Dockerfile +++ b/adapters/frontier-cs-2.0/src/frontier_cs_2_0/task-template/environment/Dockerfile @@ -25,7 +25,14 @@ ENV CLAUDE_CODE_MAX_OUTPUT_TOKENS=128000 WORKDIR /app -COPY readme config.yaml task_config.json submission_config.json AGENT.md \ +# NOTE: config.yaml and task_config.json are JUDGE/operator internals (they carry +# the full `evaluation` block — seed, val-file paths, scoring knobs, and any +# undisclosed parameters like a randomized scored horizon). They are deliberately +# NOT copied into the agent workspace: the agent's task description is `readme`, and +# the submit scripts use `submission_config.json` + JUDGE_URL only. The judge reads +# its own `/judge/task_config.json`. Shipping them to /app leaked judge internals to +# the agent (e.g. the exact rollout length / tail index for nanowm_rollout_stability). +COPY readme submission_config.json AGENT.md \ submit.py submit.sh submissions.py submissions.sh \ wait_submission.py wait_submission.sh cancel_submission.py cancel_submission.sh /app/ COPY harbor_app/ /app/