From f558821e1f767a235b462de224696618fb371c28 Mon Sep 17 00:00:00 2001
From: rsasaki0109 <rsasaki0109@gmail.com>
Date: Fri, 5 Jun 2026 07:41:07 +0900
Subject: [PATCH] Add "beat the robot" challenge mode to the playground
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add a "Score 10 seeds" button to the pick_and_retry scenario that scores the
edited agent against the shipped one across 10 seeds and shows a baseline-vs-you
scoreboard (success rate, reward, steps, retries, grasp misses) with a win/lose
verdict.

Scoring across many seeds is the point: a single-seed score rewards overfitting
(dropping the retry schedule solves seed 3 in 2 steps), but robustness across
seeds does not — an agent that ignores its belief drops to a 0% success rate.
This turns "edit the brain" into a sticky, shareable challenge and teaches why
the retry/belief logic exists.

- pir/viz/playground_trace.py: score_pick_and_retry(run_agent, agent_factory,
  seeds) aggregates trace summaries; run_agent is passed in to keep the module
  decoupled from examples/.
- docs/playground.{html,js,css}: challenge panel (pick_and_retry only), driver
  that scores baseline + edited agent and returns plain JSON, scoreboard with
  per-metric "better" highlighting and verdict.
- tests/test_playground_trace.py: scorer shape, shipped agent is 100% robust,
  and a belief-ignoring agent scores strictly worse.

Verified the exact driver in an unpacked-bundle sim: shipped agent 100% /
reward 0.65 vs a belief-ignoring agent 0% / reward -1.2. Full suite green
(124 tests).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 docs/playground.css                 |  81 +++++++++++++++
 docs/playground.html                |   8 ++
 docs/playground.js                  | 155 ++++++++++++++++++++++++++++
 docs/pyodide/pir_bundle.zip         | Bin 32024 -> 32601 bytes
 docs/pyodide_playground_strategy.md |  11 ++
 pir/viz/playground_trace.py         |  43 ++++++++
 tests/test_playground_trace.py      |  42 ++++++++
 7 files changed, 340 insertions(+)

diff --git a/docs/playground.css b/docs/playground.css
index 15a6093..0d396e8 100644
--- a/docs/playground.css
+++ b/docs/playground.css
@@ -198,6 +198,87 @@ button:disabled {
   width: 100%;
 }
 
+.challenge-panel {
+  border: 1px solid #d7dcd6;
+  border-radius: 10px;
+  margin-top: 12px;
+  overflow: hidden;
+}
+
+.challenge-head {
+  align-items: center;
+  background: #eef1ec;
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  justify-content: space-between;
+  padding: 8px 12px;
+}
+
+.challenge-head span {
+  color: var(--ink-soft, #5a6b67);
+  font-size: 0.82rem;
+}
+
+.challenge-board {
+  padding: 12px;
+}
+
+.challenge-empty {
+  color: var(--ink-soft, #5a6b67);
+  font-size: 0.85rem;
+}
+
+.challenge-verdict {
+  border-radius: 8px;
+  font-weight: 800;
+  margin-bottom: 10px;
+  padding: 8px 12px;
+}
+
+.challenge-verdict.win {
+  background: #e3f3dc;
+  color: #2f6b1f;
+}
+
+.challenge-verdict.lose {
+  background: #f7e6df;
+  color: #9a3d22;
+}
+
+.challenge-verdict.tie {
+  background: #eef1ec;
+  color: #4a5b57;
+}
+
+.challenge-table {
+  border-collapse: collapse;
+  font-size: 0.85rem;
+  width: 100%;
+}
+
+.challenge-table th,
+.challenge-table td {
+  border-bottom: 1px solid #e4e8e2;
+  padding: 5px 8px;
+  text-align: right;
+}
+
+.challenge-table th:first-child,
+.challenge-table td:first-child {
+  text-align: left;
+}
+
+.challenge-table thead th {
+  color: var(--ink-soft, #5a6b67);
+  font-weight: 700;
+}
+
+.challenge-table td.better {
+  background: #e3f3dc;
+  font-weight: 800;
+}
+
 .status-strip {
   display: grid;
   gap: 10px;
diff --git a/docs/playground.html b/docs/playground.html
index 0d3a6fa..d52e18e 100644
--- a/docs/playground.html
+++ b/docs/playground.html
@@ -124,6 +124,14 @@ <h1 id="playground-title">Playground</h1>
             </div>
             <textarea id="codeCell" class="code-cell" spellcheck="false" rows="20" aria-label="Agent source code"></textarea>
           </section>
+
+          <section id="challengePanel" class="challenge-panel" aria-label="Challenge" hidden>
+            <div class="challenge-head">
+              <span>Challenge — your edited agent vs. the shipped one, across 10 seeds</span>
+              <button id="scoreButton" type="button">Score 10 seeds</button>
+            </div>
+            <div id="challengeBoard" class="challenge-board" aria-live="polite"></div>
+          </section>
         </div>
 
         <aside class="trace-panel" aria-labelledby="trace-title">
diff --git a/docs/playground.js b/docs/playground.js
index a26c480..6e7c9e4 100644
--- a/docs/playground.js
+++ b/docs/playground.js
@@ -53,6 +53,9 @@
     codeCell: document.getElementById("codeCell"),
     runCode: document.getElementById("runCodeButton"),
     resetCode: document.getElementById("resetCodeButton"),
+    challengePanel: document.getElementById("challengePanel"),
+    challengeBoard: document.getElementById("challengeBoard"),
+    scoreButton: document.getElementById("scoreButton"),
     reset: document.getElementById("resetButton"),
     step: document.getElementById("stepButton"),
     run: document.getElementById("runButton"),
@@ -105,6 +108,9 @@
   elements.resetCode.addEventListener("click", () => {
     resetEditedAgent();
   });
+  elements.scoreButton.addEventListener("click", () => {
+    runChallenge();
+  });
   elements.replay.addEventListener("input", () => {
     state.replayIndex = clampReplayIndex(elements.replay.value);
     render();
@@ -197,6 +203,11 @@
         setRealStatus("running real Python: " + sourcePath(scenario));
         if (scenario === "pickretry") {
           populateDefaultAgentSource();
+          if (!elements.challengeBoard.textContent.trim()) {
+            renderChallengeMessage(
+              'Edit the agent above, then "Score ' + CHALLENGE_SEEDS + ' seeds" to see if you beat it.'
+            );
+          }
         }
       } catch (error) {
         if (scenario === "pickretry") {
@@ -460,6 +471,149 @@
     }
   }
 
+  // --- "Beat the robot" challenge ---------------------------------------------
+  // Scores the edited agent vs. the shipped one across many seeds. Single-seed
+  // scoring would reward overfitting; robustness across seeds is the real test.
+  const CHALLENGE_SEEDS = 10;
+  const CHALLENGE_DRIVER = [
+    "import json, os, sys, importlib.util",
+    "cwd = os.getcwd()",
+    "if cwd not in sys.path:",
+    "    sys.path.insert(0, cwd)",
+    "class _NoMatplotlib:",
+    "    def find_spec(self, name, path=None, target=None):",
+    "        if name == 'matplotlib' or name.startswith('matplotlib.'):",
+    "            raise ImportError('matplotlib is intentionally unavailable on the headless browser path')",
+    "        return None",
+    "sys.meta_path.insert(0, _NoMatplotlib())",
+    "path = os.path.join(cwd, 'examples', 'manipulation', '01_pick_and_retry.py')",
+    "spec = importlib.util.spec_from_file_location('pick_and_retry', path)",
+    "mod = importlib.util.module_from_spec(spec)",
+    "spec.loader.exec_module(mod)",
+    "from pir.viz.playground_trace import score_pick_and_retry",
+    "seeds = list(range(" + CHALLENGE_SEEDS + "))",
+    "baseline = score_pick_and_retry(mod.run_agent, mod.PickAndRetryAgent, seeds=seeds)",
+    "src = USER_SRC",
+    "if src and src.strip():",
+    "    ns = {}",
+    "    exec(src, ns)",
+    "    Agent = ns.get('PickAndRetryAgent')",
+    "    if Agent is None:",
+    "        raise ValueError('Your code must define a class named PickAndRetryAgent')",
+    "    you = score_pick_and_retry(mod.run_agent, Agent, seeds=seeds)",
+    "else:",
+    "    you = baseline",
+    "json.dumps({'seeds': seeds, 'baseline': baseline, 'you': you})",
+  ].join("\n");
+
+  async function runChallenge() {
+    stopRun();
+    elements.scoreButton.disabled = true;
+    renderChallengeMessage("scoring " + CHALLENGE_SEEDS + " seeds for both agents…");
+    setRealStatus("running the challenge…");
+    try {
+      const pyodide = await ensurePyodide();
+      pyodide.globals.set("USER_SRC", elements.codeCell.value || "");
+      const result = JSON.parse(await pyodide.runPythonAsync(CHALLENGE_DRIVER));
+      renderScoreboard(result);
+      setRealStatus("challenge scored across " + CHALLENGE_SEEDS + " seeds");
+    } catch (error) {
+      renderChallengeMessage("");
+      setRealStatus("challenge failed: " + error, true);
+    } finally {
+      elements.scoreButton.disabled = false;
+    }
+  }
+
+  function renderChallengeMessage(message) {
+    elements.challengeBoard.textContent = "";
+    if (!message) {
+      return;
+    }
+    const note = document.createElement("p");
+    note.className = "challenge-empty";
+    note.textContent = message;
+    elements.challengeBoard.appendChild(note);
+  }
+
+  function challengeVerdict(you, base) {
+    const same =
+      you.success_rate === base.success_rate &&
+      you.mean_reward === base.mean_reward &&
+      you.mean_steps === base.mean_steps;
+    if (same) {
+      return { cls: "tie", text: "You're running the shipped agent — edit it and score again to try to beat it." };
+    }
+    // Higher success rate wins; then higher reward; then fewer steps.
+    let youWin;
+    if (you.success_rate !== base.success_rate) {
+      youWin = you.success_rate > base.success_rate;
+    } else if (you.mean_reward !== base.mean_reward) {
+      youWin = you.mean_reward > base.mean_reward;
+    } else {
+      youWin = you.mean_steps < base.mean_steps;
+    }
+    return youWin
+      ? { cls: "win", text: "🏆 You beat the shipped agent across " + CHALLENGE_SEEDS + " seeds!" }
+      : { cls: "lose", text: "Shipped agent still wins — make your policy more robust across seeds." };
+  }
+
+  function renderScoreboard(result) {
+    const you = result.you;
+    const base = result.baseline;
+    elements.challengeBoard.textContent = "";
+
+    const verdict = challengeVerdict(you, base);
+    const banner = document.createElement("div");
+    banner.className = "challenge-verdict " + verdict.cls;
+    banner.textContent = verdict.text;
+    elements.challengeBoard.appendChild(banner);
+
+    // metric label, key, and whether higher is better
+    const rows = [
+      ["success rate", "success_rate", true, (v) => Math.round(v * 100) + "%"],
+      ["mean reward", "mean_reward", true, (v) => v.toFixed(2)],
+      ["mean steps", "mean_steps", false, (v) => v.toFixed(1)],
+      ["mean retries", "mean_retries", false, (v) => v.toFixed(1)],
+      ["mean grasp_miss", "mean_grasp_miss", false, (v) => v.toFixed(1)],
+    ];
+
+    const table = document.createElement("table");
+    table.className = "challenge-table";
+    const thead = document.createElement("thead");
+    const headRow = document.createElement("tr");
+    ["metric", "baseline", "you"].forEach((label) => {
+      const th = document.createElement("th");
+      th.textContent = label;
+      headRow.appendChild(th);
+    });
+    thead.appendChild(headRow);
+    table.appendChild(thead);
+
+    const tbody = document.createElement("tbody");
+    rows.forEach(([label, key, higherBetter, fmt]) => {
+      const tr = document.createElement("tr");
+      const name = document.createElement("td");
+      name.textContent = label;
+      tr.appendChild(name);
+
+      const baseCell = document.createElement("td");
+      baseCell.textContent = fmt(base[key]);
+      const youCell = document.createElement("td");
+      youCell.textContent = fmt(you[key]);
+
+      if (base[key] !== you[key]) {
+        const youBetter = higherBetter ? you[key] > base[key] : you[key] < base[key];
+        (youBetter ? youCell : baseCell).classList.add("better");
+      }
+      tr.appendChild(baseCell);
+      tr.appendChild(youCell);
+      tbody.appendChild(tr);
+    });
+    table.appendChild(tbody);
+    elements.challengeBoard.appendChild(table);
+  }
+
   function stepOnce() {
     if (state.index >= state.config.steps.length) {
       render();
@@ -955,6 +1109,7 @@
     }
     elements.answer.disabled = state.scenario === "pickretry";
     elements.codePanel.hidden = state.scenario !== "pickretry";
+    elements.challengePanel.hidden = state.scenario !== "pickretry";
 
     renderReplay(replayIndex);
     renderCompare();
diff --git a/docs/pyodide/pir_bundle.zip b/docs/pyodide/pir_bundle.zip
index 7ea0d9afe01b21c3628de8c4e3373363bc515513..fee1270b3d58c246b537fd1880dba2b0ed314657 100644
GIT binary patch
delta 3848
zcmV+j5BKnx`T^Pg0kD-P5+ydpTxtQsu0Rd|09q{o02=_4&3YYw#ajJt<hBw2f1iTb
z`y<k2ckR1tg2I6VT+=i~P^4{q{eof0w8Y(2tQA#JvTx%W=tJ}g`y~C%kd!E`*5_-R
z)(CPEIWwFY&i86QpI@jtmqq@g5>ljvtn>9gFE`7-w#qcQF2(h`TBhn+<Yl8pvsI!h
z<bG4@w#+^;qE=;prfOlfvQn$r>>c)RYJG38*29S?)2TV>7gj%*SY)-lH^QW<R7R|8
zy#oUP$xD&ya-DBx`sN3fHfKVDiR-+~8}NC3ElepZv(*iNTLYXTH;vG1QK@=q8dV8(
zr^?2x#ILcT>pYc3u?Od}nOVZ(UK+8LWwyMp^QKWHWE0<i$~$?H)_K(|K(ki)t}4_H
zgd}SL^hx&L?VBy$3au+ou`M!T@&b@JC4>4JD-3}*(4oqNdM}CE3g|k^GbIYu7zXgV
zs@sy@XD@3di<c1Bp_mi3y8({M5}BX8`7G&gDDgM6s`k(dvGfIK<VNgt))qLttxGeb
zQ)*?}q5;l-;br2Sv=QH4{PsJ3{r1A$D0q?un925Z8>ZbUvk*qlLZ<Bn631RIx^1eq
z0d<HiGg6u)k~^K=!Z^OZQ#Ig-k1b9RV6|%NPN|<V8_RW_g9GT&;NoO$L{w@~w!3P-
z5O~_jrYdw(<TvqRX0*r~Ql1fQnc_+}&^9jfD@$^JLC<st;2qaZZ|(+Gf-IG-mMBy;
zRF*0;BbM^A1eKe;#XO<^C%x$=4QUWM7ytbGKiCEsj)WoISOeI!vd;KIGiCBkDRye7
zy%$&xLXrkF8@1W5c3CXQT#c9$F=(DnYv!xjd_JGe*shXf-QsFhk_gzSt{aF}YHOy(
z!9YoWTfbZA&1<{b>?_i}TYp>b9dwn~t5nx&1pv6B+y9l!3mh|>y<lDSh9??%0|SN!
zK<0J+US(%60%)wjH4Eq?I$gbay%49X&pwYASZ)zo8j!*I=RbGr_5wn4v(^MVkm7Xt
z>hx4x{P7zoqr8)Bd7{!LXBmYAS1;H+>^%X0n8=ak=XI@NGGBGf{@yvspLAVhD>#B`
zqIfk+Ks>ql>g{j77UztAI-B9H)*^u)*SgwA8G=Wh-?W5$CbB$jF0rr>>q5)sDi+Hx
z?c>=@;IF|26d(mhi7N4ul^#{GSZl;Vvb&Jn>P|u@CW=Lls3pFHMowcUQ!8>>v9#oW
zuxlSwVH5!g@j}Quoo8Z6Z$um+Q^T2;J86u#QAMuSUK0BjL1tatA%LX1(Dj)BAL4J~
zJ6$RkqL&N?ab=}}vAw(3OM9V~%o!Xk$ALTi+vJ5a2gIcXnR)5?5X8rdjg<&Ng5mLG
zGIX>(hyFsvK!Fc)n9}@=jAS9^WIOhM<*`F>7X;YJyNWwNr_6_Lfkm_ALySvayIKt4
z+|ffoUK~R@E&%MFM*Gs-euJ!bbidL?o)S7i&ihR(@>oCJwv*%)4B=4C=jV36Go7BS
zm+Uwq1B@^qL;E~n;z@NrPnCfhr|&fY?KL`-EYKu^rf1x_!`jBP<Tfv}C@<H4`fOzT
z><)Odw&}p3a)MoL5I5#-Z9Z^Tx40s@q|n?@`jcg<BDb~hu{5?yCGW62flY{(6cSDr
z*_WreatIWt!I^!tywTDeE>ofoa14T(0sOhOu(Pp;&}wwQ&C_kdE+5Ys!WqQi6khAe
z+mJG*x1K))=RQUc4EE^AI-mf5!fT_-)7wNg$1jH}y-OQ_T?n&h`_wSFET-mndX_ub
zgGl5Uqs5TX%N~G|b;RdKp3LPbSbHh@e6kQf3ruwq<<(J?E3XPy?Iym~jZ(eap@iY^
zeKc+#x+hx&{K2Q}AiT^SNo>2PV6Txilt`Fu=_5plD>v}b3ltqpy+WCPrV>vt0P_xW
zMs``MaLc9@@^+V}I3tD5jQiu@LK~y?x6R@DTBof!T=rG+;f^fLy{ZT6U9g}&Vp{nN
zF=d_$p>NEj%}(Ew)#$)pyFk3rvY5vLzW4Ara6}Jr=!!qV#oI&i-*YUWmx7m}K3YdK
zn#YsYa2SkP-VYlIb2VvyCD4EG_dcD?$DnLDgFq8cm{3QH7bZvrM%BR<5`99q!SeKK
z>DQ+Wa7e=~$3v408b9tpl12wDL}aT_+BhRav$!u8&VG9Hg&DCTZ1+jg-s#((+QHO(
z+fP-9U+EJ4T=Vf$1J~XQ+upBirTC89RI*~Y1)WGeOu9uoa!)OP*5Uj>+_ZU-p<5{#
z|N5HW`@;VsRVAJ^T4QR|uCE`wvD>2h0v>M2nK{s+!=U#^K;Khu$~2Wsx4cfd0t9Tz
zT7b`cw08^9rT5+!7Tt8>(H(5GYpk84x^4%J4R<`$ectEj*3jD82G*>^AIc)XRlXso
z?s-8&je3uEOBI=a@1AXasOp?TNjG!gew?NOpBOSMRo_Y1HgN~M)~sH$^7m-<pHQM_
z6W}JpK0$0%raH4tH4ilCR=*Ips%l*OVN|hRqRUNDs8MrB?W;hdvnph*p4r4bw4*B=
zczI<@Eh041EiuYLl}`RKucFRvZ2xWHae?;LqIIu?-Mxf=N?<B+pYxf&m8>e|eaQa}
z+Xm~esq^`%x-D?*4#9E-gs=>-i}A2d=tIy`F>(U_<HK-=>MfPdn#^~>EsXH{ZtPh8
zc{0az{)g>Cl@Z@U7#W>c5@*LMp%C0STKsPxHeZP4aw%-EFS7(p!h<3js^n#>Thk3P
zVgmfo-|Pi{PeR}mJpJ2BLq3Z*dBPKpuvhs4UTwPUMQ$X;>tFVzC&fsFH=1HZJ0)rK
zWwv!*-jV~=GRs>NyV26*ki=AybW-eBg2J2<8>M$79T9A5aARXDt}acB7N}jIgMaQ+
zEn{NJg<~Kf{?JKmHKL?Y1HV>K<qIFb$gOFni}s#>Mnp8@8-<*APMHw2mRPMxf-;;)
zg4!0QzeGeKLGNiK!_!4{OEj+y=7VZSgW0T~6Y^Y`=7TF=qP^coJaXWCH*OCG_3-7e
z3J1<~)h-U<rb_jzVZY)n%X-^n^RggoN&Fz?M(bkO)lt{jxvpu`R2egm5Bs!?$|H|4
zRYOsKi?^wzsghj|5{DGpDg%>_m7e#GXcUByzCp9q45A1h0P~+}A`Mbnf>54V+Z!}{
zPf_KD^@vJsL;t_iX{no#?4*qU*#!jZ?z(SMmX&`b{>K)2R`c+TV9Q5<V9ADnaN$D*
z;`8djQKWtPha(?*U;BE{6@u5-ofN2`PUzi#N#I^B_jr-Dnv6a^82ZwXPr@O(M%aP%
z8yZBfUd8lliP|$6H%oSW`e+L-3GnpPjiE;+6ZoVdB;ruVaqLx2#Z4GZ?heBUmpykB
zN4<bo{&r3sT^Fm}EY$_@^QYRp*V)H+_+b~oTf^pl?AOn0?;SApHa~>RPE^`R(~wnv
z7L_=K_`!On8kJ7>@5crKTU}6o!N#v-VZxoRoqU*6+WqriJTm-}vu^(`k$!enLo25X
z>$bV_Z~dH+1Hxlha?n4|DtWol>ts8dm}~*In?(H|3*?SR1BSXjwO_SC+)2e|x(;x9
zI)_3#8}EUUlrQJq!0yVKJ^i_K#idezI5G*==l$HDUP0L@kQfT_-Pq4gVu^uZGajF4
ztzzm-XD`984ZM(;yA8mHt;C28M|8kQ7%K)~<GAsrKe2xrj5<F!1nn?TU-mj0(m2r4
zV7sT|d<}Fc14mJ3A)7i!Q&sypR%}<jYsS3HC=*G#C-1$GWop9W#JlH0oW_HHpM1zo
z55R3YO`+swgwc=rC(?P($6)OKL^booL_YM8@mPLTQdwt#9SFA<QJit}bNYjY91@``
zIznX4n~~F+c<}0b9MCjs8hbFR_nX~U80X_(*q+Qu4Dm!>vJ|@9v{gkmCyjYSvp=AW
zQx;<l($?|fe9;An+;G7+8ay+9tv<7P1DP04GURlkvs*`!oz;O)c(#Brmwx4ne-`c>
z$*smAw^j?oOvyyY+44Yh!w@|FA<`#Q7ad01eO`4><6lg>?0PtB=+dpz{nW6;XW%}P
zPR%>|yzCz*2QNNdyQc}5E{Q+UbUepa9>>4f(%-)~S7L|zZ_knZl=1_Avm@`%m*Mi&
z1XeIM@{h-dLqG9!$WL#dISQKpLb;hrt#2R!O<W5cv7rt?*W@M77zDXd(A(0@OLD{5
z=pSTSYhwfi+SB}m$}KV)t|*P>ZzRaH>}W@$Yg*HsNYa!AXQN3e+Vs|<ib$<CE0NW@
z;*WN0UqW4tNw>;=-=zhATFo2VsIl*{wSro{+o>$a1r~db8|pnWxBXT|ibCHDmuhKH
zqui%6?)#;C_%?l?W(<|3^aaxkd+v!lqR^W>1w|CBh6_`uvNcT!3@a_Iag3^Rqci2_
zHLRkMt3`ENCagr;j9P^v5vPr=T@#Dm*6OmK^BieJzi;U~!2yYXC`O-1ipB;Lp|-Ka
zMQt{<+Ccbzs7;^l>?FmCt`Y6H1C~>d`z((}>-X+6z=&sJc0#Vjty0xs=0&73h0>!c
z{FJ`?wht}nh+%sY58E<m5o}d<7MIy3z_tUmGFz*wmH3{W%T36Z{2^a29c8_7+sb-C
z)~1#t;ul-zIGf;q>glxaWD@^r6MZA)yqUB#u}<;mV43D12a6;N3k9Xw%k3j4UjtR|
zo%*BEomzu9r~fTnvwqI4Uc7SHQD3?t1G>ePlW<t%^kRQ{E?x}-xa0fN^PU7PkIT&-
zjH*r#%GIW-&2$Ay^~7mdxCY7#&f|r|?)D8%uhIb2m#=(aMpuP*qbNGm8US?&99Q}7
ziIG=F!nAYYE)X+97;L@nDB$|%!~LYTDaK~6Ug0OA9A+;r6A~Z79e8jNjPT|FA;|WO
zGQ5_fNQ0a@xPT)l;b7091y?aZdYt_WP)h>@lkj>Pv*9ybDgtXRvp-&L91bNm#awCu
z!>&LM003Gole>CZ0t7ab-!mAKE_*5h0ymT0G#rz5dny7nK$8zX8k4+xDgu^GlW|KQ
zlLve%0;g7!GkhlkYc7*qd>jIFV3SW?ER&{uDgqs7llWyY7EntC1^@s602crk09ghA
K0P1)E00002O@AW*

delta 3263
zcmV;w3_$bQ{sEZ!0kD-P5(>Y>T!!qYQU4170OlqD02=_44t*Ve#aerh+qe<`-=Bg|
z{x~VM9Vh7>3I{nLNbiB5xVG5zg~dY4m1vu{vZ#vEZhYGX`XTxW`$_s6J|s$(ouuux
zMsO<)hr^lSyme6&U5HxpLjEKeXIa5(xjD#kyZCD>bR$*CuHV-@6W2_ZjbhDCu&UsP
zZLQie|3b4`l)0#XncneAEN8R#Sih;&y#}nuV{xuB{j3Kpf7A)fYksep&O|9R+tg|g
z27n?<mZ@?hw=;F~qsW>y<6z=ima+k#*Vjy!ywW?>0J+uJQ%Kz~wPBU07rGG@6L+F)
z^pbs#1y#$87sUab^JZoVvwN=Dj+go3zLrfRO323E@jHHhk=3$l=Ac;%xvvVb2O(~>
z0Qw~R?+(olZ-r8or`Q%b)3N|2PRXIZ#tK8|4Rk1SCO&YYw#0Uo%UrNRG}<6|UDa(#
z@3W`1;Kfsj>$d1;Vt<1@B2T$|_VUZLzoFFM(6Tx}E5y<((2$z#Ro)gjysb+;qf=_3
z+oHjq<IBW<Ia$NrUHtY3^ZM?>-6(kC1(?YXbQ`+e3q5CA%|fQ_1ro<zFRE>-wgGjB
z&2v&3Cz5-W-NHD&xf3<;h>smkU|_Xs>rSb^Fl;O~wFC#yrN+haTC=!RtZetyVb1Wh
z=S@|prjR$ue5MtX4Jl8vw#;y)TWA}X`PP!0(=*+F0eHuC)0?}2B_m5UR!bDB8Y;_#
zo)Js3EJ5YwV0S*F|E9g^rVVKjI%og<`#+2g7(5b&bYl%*+wwX$7n%z#w<X()z4BgQ
zIS5H=&}_u^u-xYfBXiX(B4W@yoz_IlSrkRHnX#)h-L$w`k){kbs_F)!mCBl_b~sSd
z&aW1KYWu=Mn?psqcknmm!C_ajUS_HmOCZ1%-TJp&7C2@$dt!9e8y;);4Gb6_0GT)P
zgUHt~0%)wjH8bcVUM*j~n6uUL%de6-z%4>c12TYr^(zOr7Z948wI<jBXRF2Y)rwvG
z@jED^yyM35Sf#WyGBOUXo*476_XK1jN0u*t>srBN-geCX-Z{yiR9)mtID%;6WI0Pg
zJiU1P<~QH4*Jl4}Hp5+QSPDO`RdtAS1dm$Yw6uB6a+x)k0L<B@P`tTH*y3yZxSlcm
z8eBjDQgD>GVo#0I<0@erg*Zrd7m{1vap*+H36qFg>}zObl^8O$Ag5&uOAf2{K@?hl
z5E5fg7{608XA62G;s}`v&b-)jt=Wwzq}X^#>{|?(RdI&^lBq(~YX&~p-`EeT6h??%
zG8n{_l?KN4{$4HYg<3LeIGDtNd;D!=;miSXX-;NddOjHOu{6d?gdoB2crqC}+MYwd
zP%%*8Lj+Tb)?_4e7Lo1Pm&a~{yCC3yPTp10Av$F~bO0loB_Hm%G_b4r5YHVwL^QxL
zrsD$0?rF5n_3d}aYA4q#RmhCC6K2#eT9L>4>6V=&&tVA1YJUCNu6L%>ll79F-e?d;
zJ0C;)JYwRkIzLU7ff}dpH9+k(I+QHXB!Z@A+_}TX#<TQRmU%48jarXv-?#&R-mGmp
zbf}zgms`Y*$ibq4vpV3C=#oOEru4_lOvG+!?qg|Ul}g@Wc><ddEiE{lEVeJ_xpEsQ
zP=ho3W_cq}1eYmM2RH`7%mDwjwXpTrLkJq*?_{=1jmsyI!EgpKIK{wv@;0Q5>20Ei
z;M~Xf!Js`lvJNT0@Y<+kcAN5l=Je%IrFUrqunS@KY@Zqimu0B=m7e7e_b3uM#t0ZP
zdf6jTvX0I9u_tr60&7oMpHJrOmw~BHZh3xk%avD!t9Dag>&BVd?@_{-@O?6FAG@cq
z3iyLh*+F=T97!y@r(mt&HIzs@+tNpfa4+A&M=wxx80r<uG?99G0h)Jzm^ImDDZ(Y2
zR><33o-!FJbY|S22`;oSg1>2w;Tx5;`WWo1<l_}y>U&WS;9anwK4Mz=b2eq3bEa<e
zq|Hv>lhx?JUb{fNRlJB22H$&l95|xKICjOK;o|L~`1dIm&`V*!P#?h&jiO}I8jgdp
zk@w?9Vz`>L66nA8d!NpKqA{v5oI#*TCQPWKWdIYT0;B413ywaaTQKtUYU$xq1~{bQ
zz{$`ggT~KvAZeq579zG)C@rkX(Cpr)b7w!j`NE8hB5e0b(cbCXp4#Em=C;pOh;LPi
zey;iasex<nnQiZvwo-g=+ElV)xCNa^JxsPkJ91Ag*75v+-L$fQ$kDBoX8-lIdG8DV
zi%gVwR%nf>QM<l=@WyV7>I-<dCTBK*799q?KLY!ndQ(GF@N8$`lq*2MrmQ*myhnRC
zXI*;lePP~BCm!9wTDivBIjZY+(Ab!chq}*uiEa(8t!-fSlKr79<gM@xId#u-8fw%B
zv|FOcefMnZLs3hA3MJjl!Sv%a4fsNnVTt;VySB-6z#C=MtC#*Bt^QL=^u`2ClVP7A
zwkR`|+oswKH0V}ev0G6!uKm!W*euZHW+>FCIi&WLA<<bC8m+Ev;vU-36%M?-vZWRg
z8tIl8H9?h5{z+DG=Qg(gw)nU}`)ap!ue7^+36)@|Bz?|*H~cMmRSEAy{x@tJtiNW?
z=cnqnz_GgxmMd%s%K!^ZhIK+8g3iUr3H;9w!`)VIsdUz4z6)+)w7>7hPUWAoIi~YJ
zEEl4T%`JqH(Rn3tcB&Ez(Y2#x{`O(>30o`{%m({1Pr)QSD5jx`EL+v;Zjg}>;>Uio
zCuR}?pD@#Zza7`)vxt+=%!DKCRbIiXb(g)!jW~Pp>%R1)7>V#kQ;c{oIE}vau9oF3
zIZ(}W+3LiNmL`WJy5gjh;;>{C<^<acwI}I_U{iw|8(VR8Syr^z+66lJ=U&u2A*Nh7
z#s<V6I>}fKD=E~#uLV^3#K$jkYnthzy{8co&G<%tA?LkQCIqcHpfyQQh7(Cp+rjh~
zh$tlJJ&k0{bP?SW&1-}CpxD!3Ht*+zW-d(g!KE+JJ{)2*axnRB+#U?-;mctajy=;=
zJ3GdkD%G!s{ff6N>ur<G%bctw^@EffrHWx!M_ps*y1LCWp>>oT_h~tmM}Egt4Mi>9
zw&uEjO7{{Zjw!TN4kjHdJ?|aSI0zwqgJ!E4L=kfU%zvth)JSP5LV0R!Z_w<WqiPz~
z6DqY0{r^s<rD{U5lQRBi7Z9ks>%K`@R{oLrpIhiz&BHT-B_9C-kPQJL;9~{ir`3U@
zNPGUnk&nGEd_Cw2!RzZz3RF<f=-o-+UM=^3c;U5}j6OaX`qI#xghNz~umkJYG>Be4
zPv~ojI2bZ+mhAZS(H2}1;%eo_(4&$Ge9{mSaj4@s@hYd{CX6O`$6>^jJ$DpGy?|H#
zc2<tAi`8zH>H_%db8X&>{PR2funRCt!{&bM*PqtjJ7nr@ehk4*R9eWgkX0ipaSHK&
z19&bPkxlpSrv?E#RZxDx!neH8;Y!y|K17su|NIxf8GdQ9ZvW1ae)d&Ekk#C}trY&P
zpEH_(@Yt0c^yiaGUeoAxvaKg3TfprmQUAvRx#LlTp{`Hup*Dy+sf?L!f;~N*W1*dm
z_sB@fmr*ycyK-hvKbNjdsT7V(0(jJa&;98Yl$`>Jp>V$&``Jk>F%T>!;}fk_Or7cM
zCHUH40I9y)0)5y@jM#8Q2aJTVVt_V|8!!74`=`OE^P@x1ZU^eiUPnV3M_L*z_jH`E
zfevNhIPNTDTT3)mwVz`p7V2FyF<`T0B1!k;y%!ppny@(W?m1_xWbny{-1G>4-KNtF
zN^V9w`Z51RI`8=yjNQ*vGiN69p@)pe@}rW<It%Q8xwV+$jGLd+4;FGvgs$iWk&Ri5
zoz~ccSKs4+rWVt?2cvqw*!^D^ug725&gLY8aV9TW3SDm6i6W0kW8Tp02b6KjGP?$8
z>vVBG?*fF>rr;Y5o(-)&vv~u5nHWzR%;`jDmrf)*s{^0#YyqJ!J><%K7VaF$L6eYM
ztA$~vG(^YQ@<?;T5Ip`6=@Y7p4x{Bhuezu47t=1g9?u%ObnA3KH7xNNxR0b$^Nv0*
z`^U+_i%-|?X#%H9;%AzUPqCH9@fTbA{d;pI<8c4&Ig+1Jeq?s!{rNH*1Yb?;3dTnM
z@$_)$XPyrE>Fq~GLH`0!O9KRxF?|}dA~swq0-XV~fM9MM4hp}-T!!qYQU4170Olr>
z{Crsg!!wg0HW-s{eJTRMG?N}T9Fw<wDgyI8lTbh!ll^@v0#!?sDSk-;W>k}OekTH*
x0h67690D_5laOF6ljnXa0^DVjIcP8zP)h{{000007XTLkSq1<AsdfMW000&BZR7v|

diff --git a/docs/pyodide_playground_strategy.md b/docs/pyodide_playground_strategy.md
index c9c622b..70c8cc8 100644
--- a/docs/pyodide_playground_strategy.md
+++ b/docs/pyodide_playground_strategy.md
@@ -183,6 +183,17 @@ All three phases are built and Python-verified; the remaining work is a single
 browser pass over Phases 0–3 and (optionally) deleting the JS `clarifying`
 preview once the real path is the trusted default.
 
+**Bonus — "beat the robot" challenge. ✅ built (Python path verified).** A
+**Score 10 seeds** button scores the edited agent against the shipped one across
+10 seeds and shows a baseline-vs-you scoreboard (success rate, reward, steps,
+retries, grasp misses) with a win/lose verdict. Scoring across *many* seeds is
+deliberate: a single-seed score rewards overfitting (drop the retry schedule and
+seed 3 solves in 2 steps), but robustness across seeds does not — an agent that
+ignores its belief drops to a 0% success rate. `score_pick_and_retry` (pinned by
+`tests/test_playground_trace.py`) does the aggregation; verified locally the
+shipped agent scores 100% / reward 0.65 while a belief-ignoring agent scores 0% /
+reward -1.2.
+
 ## Risks / watch-list
 
 - **First-load latency.** Pyodide core is a few MB. Lazy-load it only when the
diff --git a/pir/viz/playground_trace.py b/pir/viz/playground_trace.py
index 9dd8d5e..e6b7bf4 100644
--- a/pir/viz/playground_trace.py
+++ b/pir/viz/playground_trace.py
@@ -286,3 +286,46 @@ def pick_and_retry_trace_to_playground(
         "initial": initial,
         "steps": steps,
     }
+
+
+# --- "Beat the robot" challenge scoring -------------------------------------
+#
+# An agent is scored across many seeds, not one, so a policy that overfits a
+# single lucky seed (e.g. dropping the retry schedule to grab the belief mean
+# immediately) is exposed by a low success rate. This is the whole lesson: the
+# retry/belief logic exists for robustness, not for one episode.
+
+
+def score_pick_and_retry(run_agent: Any, agent_factory: Any, *, seeds: Any) -> dict[str, Any]:
+    """Run ``agent_factory()`` over ``seeds`` and aggregate the trace summaries.
+
+    ``run_agent`` is the example's own loop (passed in to keep this module
+    decoupled from examples/); ``agent_factory`` is called once per seed for a
+    fresh agent. Returns plain-JSON aggregate stats.
+    """
+
+    seeds = list(seeds)
+    successes = 0
+    steps_total = 0.0
+    retries_total = 0.0
+    reward_total = 0.0
+    miss_total = 0.0
+    for seed in seeds:
+        summary = run_agent(agent_factory(), seed=seed, render=False).summary()
+        if summary.success:
+            successes += 1
+        steps_total += summary.steps
+        retries_total += int(summary.counters.get("retry_count", 0) or 0)
+        reward_total += summary.total_reward
+        miss_total += summary.failure_counts.get("grasp_miss", 0)
+
+    n = len(seeds) or 1
+    return {
+        "episodes": len(seeds),
+        "successes": successes,
+        "success_rate": round(successes / n, 4),
+        "mean_steps": round(steps_total / n, 3),
+        "mean_retries": round(retries_total / n, 3),
+        "mean_reward": round(reward_total / n, 3),
+        "mean_grasp_miss": round(miss_total / n, 3),
+    }
diff --git a/tests/test_playground_trace.py b/tests/test_playground_trace.py
index 3708b41..aae8d08 100644
--- a/tests/test_playground_trace.py
+++ b/tests/test_playground_trace.py
@@ -16,6 +16,7 @@
 from pir.viz.playground_trace import (
     clarifying_trace_to_playground,
     pick_and_retry_trace_to_playground,
+    score_pick_and_retry,
 )
 from pir.worlds.tabletop_2d import Tabletop2D
 
@@ -222,3 +223,44 @@ def update(self, obs, reward, info):
     assert json.loads(json.dumps(config)) == config
     # No belief attributes -> belief radius is unknown for every step.
     assert all(step["snapshot"]["belief"]["radius"] is None for step in config["steps"])
+
+
+SEEDS = list(range(10))
+
+
+def test_challenge_scoring_shape_and_baseline_is_robust() -> None:
+    module = _load_pick_and_retry()
+    baseline = score_pick_and_retry(module.run_agent, module.PickAndRetryAgent, seeds=SEEDS)
+
+    assert set(baseline) == {
+        "episodes",
+        "successes",
+        "success_rate",
+        "mean_steps",
+        "mean_retries",
+        "mean_reward",
+        "mean_grasp_miss",
+    }
+    assert baseline["episodes"] == len(SEEDS)
+    # The shipped agent should solve every seed in this small set.
+    assert baseline["success_rate"] == 1.0
+    assert json.loads(json.dumps(baseline)) == baseline
+
+
+def test_challenge_exposes_an_agent_that_ignores_belief() -> None:
+    """An agent that ignores its belief and always picks the same wrong spot
+    must score a lower success rate than the shipped, belief-driven agent."""
+    import numpy as np
+
+    module = _load_pick_and_retry()
+
+    class StubbornAgent(module.PickAndRetryAgent):
+        def act(self, obs):
+            # Never use the belief — stab a fixed corner far from the object.
+            return {"type": "pick", "position": np.array([0.1, 0.1])}
+
+    baseline = score_pick_and_retry(module.run_agent, module.PickAndRetryAgent, seeds=SEEDS)
+    stubborn = score_pick_and_retry(module.run_agent, StubbornAgent, seeds=SEEDS)
+
+    assert stubborn["success_rate"] < baseline["success_rate"]
+    assert stubborn["mean_reward"] < baseline["mean_reward"]