From f68a6b6e834fae0c64b866524868bb7e9fd96c61 Mon Sep 17 00:00:00 2001 From: rsasaki0109 Date: Thu, 4 Jun 2026 23:53:15 +0900 Subject: [PATCH] Pyodide Phase 2: run the real pick_and_retry loop in the playground MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a "Pick and retry (real Python)" scenario — the browser twin of the README hero GIF. It is real-Python-only: selecting it boots Pyodide and runs the unmodified examples/manipulation/01_pick_and_retry.py run(seed=3) loop headless. There is deliberately no JS mock: the dynamics are stochastic and belief-driven, so a hand-faked version would reintroduce the exact drift Pyodide removes. A continuous tabletop2d renderer draws the real scene — true object vs. the agent's spatial belief (mean + shrinking uncertainty radius), occluder, camera, last detection, each pick attempt — mirroring the matplotlib render in tabletop_2d.py. The belief panel switches to a spatial layout (uncertainty bar + attempts/retries/policy). - examples/manipulation/01_pick_and_retry.py: record belief_mean/belief_radius/ retry_count into the trace info each step, so belief is inspectable without the live agent object (belief becomes first-class in the Trace). Backward compatible; the matplotlib render and GIFs read the agent, not info. - pir/viz/playground_trace.py: pick_and_retry_trace_to_playground serializer. Scene geometry (true object, occluder, camera) is ground truth the agent never sees, so the caller passes it from the real Tabletop2D — no world constants hidden in the serializer. - tests/test_playground_trace.py: pins the tabletop2d render contract and the seed=3 hero outcome (grasp_miss then pick, belief appears, retries>=1). - build_pyodide_bundle.py output rebuilt. Verified the exact browser driver's Python path in an unpacked-bundle sim: seed=3 yields scan -> pick(miss) -> pick(miss) -> pick(done), belief radius shrinking 10 -> 9.8 -> 9.5 -> 2.5, holding=True, retries=2. Full suite green (121 tests). Browser check still pending. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/playground.html | 1 + docs/playground.js | 286 +++++++++++++++++++-- docs/pyodide/pir_bundle.zip | Bin 30562 -> 31786 bytes docs/pyodide_playground_strategy.md | 30 ++- examples/manipulation/01_pick_and_retry.py | 7 + pir/viz/playground_trace.py | 134 ++++++++++ tests/test_playground_trace.py | 83 +++++- 7 files changed, 522 insertions(+), 19 deletions(-) diff --git a/docs/playground.html b/docs/playground.html index be5ab46..f4027b4 100644 --- a/docs/playground.html +++ b/docs/playground.html @@ -41,6 +41,7 @@

Playground

Scenario diff --git a/docs/playground.js b/docs/playground.js index e4da4f4..fc64a51 100644 --- a/docs/playground.js +++ b/docs/playground.js @@ -16,10 +16,12 @@ const sourceLinks = { clarifying: "https://github.com/rsasaki0109/PythonInteractiveRobotics/blob/main/examples/embodied_ai/35_clarifying_question.py", + pickretry: + "https://github.com/rsasaki0109/PythonInteractiveRobotics/blob/main/examples/manipulation/01_pick_and_retry.py", household: "https://github.com/rsasaki0109/PythonInteractiveRobotics/blob/main/examples/embodied_ai/36_household_task_agent.py", }; - const scenarioValues = new Set(["clarifying", "household"]); + const scenarioValues = new Set(["clarifying", "pickretry", "household"]); const answerValues = new Set(["red", "blue"]); const failureValues = new Set([ "all", @@ -74,7 +76,7 @@ let copyStatusTimer = null; elements.scenario.addEventListener("change", () => { - if (elements.scenario.value !== "clarifying") { + if (elements.scenario.value === "household") { elements.realPython.checked = false; } rebuild(); @@ -118,7 +120,15 @@ }); render(); - if (readAutoplayParam()) { + if (elements.scenario.value === "pickretry") { + // Real-Python-only scenario: boot Pyodide and run it on load so shared + // ?scenario=pickretry links show the real loop, not the static placeholder. + rebuild().then(() => { + if (readAutoplayParam()) { + startRun(); + } + }); + } else if (readAutoplayParam()) { window.setTimeout(startRun, 180); } @@ -130,11 +140,16 @@ if (realConfig) { config = realConfig; source = "python"; + } else if (scenario === "household") { + config = buildHouseholdScenario(answer); + source = "js"; + } else if (scenario === "pickretry") { + // No JS dynamics for pick_and_retry; show the static tabletop until the + // real Python config (from Pyodide) is available. + config = buildPickRetryPlaceholder(); + source = "pending"; } else { - config = - scenario === "household" - ? buildHouseholdScenario(answer) - : buildClarifyingScenario(answer); + config = buildClarifyingScenario(answer); source = "js"; } return { @@ -154,20 +169,31 @@ // Otherwise we use the instant JS preview so first paint never waits on Pyodide. async function rebuild() { stopRun(); + const scenario = elements.scenario.value; + // pick_and_retry is real-Python-only: its dynamics are stochastic and + // belief-driven, so a hand-written JS mock would be exactly the drift we are + // removing. clarifying keeps a JS preview and opts into real Python. const useReal = - elements.realPython.checked && elements.scenario.value === "clarifying"; + scenario === "pickretry" || + (elements.realPython.checked && scenario === "clarifying"); if (useReal) { setRealStatus("booting Pyodide + running real Python…"); try { - const config = await fetchRealClarifyingConfig(elements.answer.value); + const config = + scenario === "pickretry" + ? await fetchRealPickRetryConfig() + : await fetchRealClarifyingConfig(elements.answer.value); state = buildState(config); - setRealStatus( - "running real Python: examples/embodied_ai/35_clarifying_question.py" - ); + setRealStatus("running real Python: " + sourcePath(scenario)); } catch (error) { - elements.realPython.checked = false; - state = buildState(); - setRealStatus("Pyodide failed (" + error + ") — showing JS preview", true); + if (scenario === "pickretry") { + state = buildState(); + setRealStatus("Pyodide failed (" + error + ") — pick_and_retry needs it", true); + } else { + elements.realPython.checked = false; + state = buildState(); + setRealStatus("Pyodide failed (" + error + ") — showing JS preview", true); + } } } else { state = buildState(); @@ -177,6 +203,12 @@ render(); } + function sourcePath(scenario) { + return scenario === "pickretry" + ? "examples/manipulation/01_pick_and_retry.py" + : "examples/embodied_ai/35_clarifying_question.py"; + } + function setRealStatus(message, isError) { if (!elements.realStatus) { return; @@ -243,6 +275,66 @@ return config; } + // Reads the real Tabletop2D geometry (true object, occluder, camera) and runs + // the unmodified pick_and_retry loop, then serializes with the same helper + // pinned by tests/test_playground_trace.py. + const PICKRETRY_DRIVER = [ + "import json, os, sys, importlib.util", + "cwd = os.getcwd()", + "if cwd not in sys.path:", + " sys.path.insert(0, cwd)", + "class _NoMatplotlib:", + " def find_spec(self, name, path=None, target=None):", + " if name == 'matplotlib' or name.startswith('matplotlib.'):", + " raise ImportError('matplotlib is intentionally unavailable on the headless browser path')", + " return None", + "sys.meta_path.insert(0, _NoMatplotlib())", + "path = os.path.join(cwd, 'examples', 'manipulation', '01_pick_and_retry.py')", + "spec = importlib.util.spec_from_file_location('pick_and_retry', path)", + "mod = importlib.util.module_from_spec(spec)", + "spec.loader.exec_module(mod)", + "from pir.viz.playground_trace import pick_and_retry_trace_to_playground", + "from pir.worlds.tabletop_2d import Tabletop2D", + "geom = Tabletop2D(seed=3)", + "trace = mod.run(seed=3, render=False)", + "json.dumps(pick_and_retry_trace_to_playground(", + " trace,", + " object_xy=[float(geom.obj.position[0]), float(geom.obj.position[1])],", + " occluder=[float(v) for v in geom.occluder],", + " camera=[float(geom.camera_pos[0]), float(geom.camera_pos[1])],", + "))", + ].join("\n"); + + async function fetchRealPickRetryConfig() { + const cacheKey = "pickretry:3"; + if (realConfigCache[cacheKey]) { + return realConfigCache[cacheKey]; + } + const pyodide = await ensurePyodide(); + const json = await pyodide.runPythonAsync(PICKRETRY_DRIVER); + const config = JSON.parse(json); + realConfigCache[cacheKey] = config; + return config; + } + + function buildPickRetryPlaceholder() { + const initial = { + type: "tabletop2d", + command: "pick the block", + target: "block", + agentState: "scan_for_object", + failure: "none", + object: [64, 54], + occluder: [43, 42, 57, 68], + camera: [16, 50], + detection: null, + pickAt: null, + holding: false, + belief: { meanXY: null, radius: null, attempts: 0, retries: 0, policy: "scan" }, + }; + return { command: "pick the block", totalSteps: 0, initial, steps: [] }; + } + function stepOnce() { if (state.index >= state.config.steps.length) { render(); @@ -729,7 +821,14 @@ elements.run.disabled = state.index >= state.config.steps.length && !timer; elements.copyTrace.disabled = state.trace.length === 0; elements.compare.disabled = state.scenario !== "household"; + // clarifying opts in; pick_and_retry is always real Python; household is JS. elements.realPython.disabled = state.scenario !== "clarifying"; + if (state.scenario === "pickretry") { + elements.realPython.checked = true; + } else if (state.scenario === "household") { + elements.realPython.checked = false; + } + elements.answer.disabled = state.scenario === "pickretry"; renderReplay(replayIndex); renderCompare(); @@ -819,6 +918,12 @@ } elements.beliefPanel.hidden = false; + // Spatial belief (pick_and_retry): position estimate + shrinking uncertainty. + if (typeof belief.red !== "number") { + renderSpatialBelief(belief); + return; + } + const bars = document.createElement("div"); bars.className = "belief-bars"; [ @@ -847,6 +952,51 @@ elements.beliefPanel.appendChild(metrics); } + function renderSpatialBelief(belief) { + // One "uncertainty" bar (belief radius, smaller = more confident) plus the + // attempt/retry counters and the current policy. + const bars = document.createElement("div"); + bars.className = "belief-bars"; + const hasRadius = typeof belief.radius === "number"; + // radius is on the 0..100 canvas; ~14 is the agent's initial uncertainty. + const fraction = hasRadius ? Math.min(1, belief.radius / 14) : 0; + const row = document.createElement("div"); + row.className = "belief-row"; + const name = document.createElement("span"); + name.textContent = "uncertainty"; + row.appendChild(name); + const track = document.createElement("div"); + track.className = "belief-track"; + const fill = document.createElement("div"); + fill.className = "belief-fill belief-red"; + fill.style.width = Math.round(fraction * 100) + "%"; + track.appendChild(fill); + row.appendChild(track); + const value = document.createElement("strong"); + value.className = "belief-value"; + value.textContent = hasRadius ? belief.radius.toFixed(1) : "—"; + row.appendChild(value); + bars.appendChild(row); + + const metrics = document.createElement("div"); + metrics.className = "belief-metrics"; + [ + ["attempts", String(belief.attempts || 0)], + ["retries", String(belief.retries || 0)], + ["policy", belief.policy || "scan"], + ].forEach(([label, value]) => { + const metric = document.createElement("span"); + metric.textContent = label; + const strong = document.createElement("strong"); + strong.textContent = value; + metric.appendChild(strong); + metrics.appendChild(metric); + }); + + elements.beliefPanel.appendChild(bars); + elements.beliefPanel.appendChild(metrics); + } + function renderBeliefRow(label, probability) { const row = document.createElement("div"); row.className = "belief-row"; @@ -928,11 +1078,117 @@ elements.scene.textContent = ""; if (snapshot.type === "household") { renderHousehold(snapshot); + } else if (snapshot.type === "tabletop2d") { + renderTabletop2D(snapshot); } else { renderTabletop(snapshot); } } + // Continuous tabletop for pick_and_retry: true object vs. the agent's spatial + // belief (mean + uncertainty radius), the occluder, camera, last detection, + // and the current pick attempt. Mirrors the matplotlib render in tabletop_2d.py. + function renderTabletop2D(snapshot) { + const svg = createSvg("svg", { + class: "tabletop-svg", + viewBox: "0 0 100 100", + "aria-label": "Pick and retry tabletop", + }); + svg.appendChild(createSvg("rect", { x: 4, y: 4, width: 92, height: 92, rx: 2, fill: "#fbfaf7" })); + for (let i = 10; i < 100; i += 10) { + svg.appendChild(createSvg("line", { x1: i, y1: 5, x2: i, y2: 95, class: "tabletop-grid" })); + svg.appendChild(createSvg("line", { x1: 5, y1: i, x2: 95, y2: i, class: "tabletop-grid" })); + } + + const occ = snapshot.occluder; + if (occ) { + svg.appendChild( + createSvg("rect", { + x: occ[0], + y: occ[1], + width: occ[2] - occ[0], + height: occ[3] - occ[1], + fill: "#2a2a2a", + opacity: 0.16, + }) + ); + } + + // true object (ground truth the agent never sees directly) + if (snapshot.object && !snapshot.holding) { + svg.appendChild( + createSvg("circle", { cx: snapshot.object[0], cy: snapshot.object[1], r: 4.5, fill: "#d94b3d", opacity: 0.85 }) + ); + } + + // camera + if (snapshot.camera) { + svg.appendChild( + createSvg("rect", { + x: snapshot.camera[0] - 2, + y: snapshot.camera[1] - 2, + width: 4, + height: 4, + fill: "#2b6cb0", + }) + ); + } + + // last detection (orange x) + if (snapshot.detection) { + svg.appendChild(drawCross(snapshot.detection, 3.5, "#dd7711", 1.6)); + } + + // agent belief: mean + uncertainty radius (green dashed circle) + const belief = snapshot.belief || {}; + if (belief.meanXY) { + const r = Math.max(2, belief.radius || 0); + svg.appendChild( + createSvg("circle", { + cx: belief.meanXY[0], + cy: belief.meanXY[1], + r, + fill: "none", + stroke: "#47743a", + "stroke-width": 1.6, + "stroke-dasharray": "3 2", + }) + ); + svg.appendChild( + createSvg("circle", { cx: belief.meanXY[0], cy: belief.meanXY[1], r: 1.1, fill: "#47743a" }) + ); + } + + // pick attempt (black +) + if (snapshot.pickAt) { + svg.appendChild(drawCross(snapshot.pickAt, 4, "#17201f", 1.8)); + } + + const caption = createSvg("text", { x: 7, y: 11, class: "svg-small" }); + const bits = ["attempts: " + (belief.attempts || 0)]; + if (snapshot.holding) { + bits.push("holding block"); + } else if (snapshot.failure && snapshot.failure !== "none") { + bits.push(snapshot.failure); + } + caption.textContent = bits.join(" "); + svg.appendChild(caption); + elements.scene.appendChild(svg); + } + + function drawCross(point, size, color, width) { + return createSvg("path", { + d: + "M " + (point[0] - size) + " " + point[1] + + " L " + (point[0] + size) + " " + point[1] + + " M " + point[0] + " " + (point[1] - size) + + " L " + point[0] + " " + (point[1] + size), + stroke: color, + "stroke-width": width, + "stroke-linecap": "round", + }); + } + function renderTabletop(snapshot) { const svg = createSvg("svg", { class: "tabletop-svg", diff --git a/docs/pyodide/pir_bundle.zip b/docs/pyodide/pir_bundle.zip index 6dcbcb189bdc9ceabebfb0ae13ce0f8d1971fcd5..93752d9a5c4c7e65b11bab13a3ce6becbd03743b 100644 GIT binary patch delta 4954 zcmV-g6Q%6p?g6U!0kD-P4q&{*T!!qYQU4170Olr>qbDJM#aerh+qe<`-=Bg|{x~VM z9Vh7>3I{nLNbiB5xVG5zg~dY4m1vu{vZ#vEZhYGX`XTxW`$_s6J|s$(ouuuxMsO<) zhr^lSyme6&U5HxpLjEKeXIa5(xjD#kyZCD>bR$*CuHV-@6W2_ZjbhDCu&UsPZLQie z|3b4`l)0#XncneAEN8R#Sih;&y#}nuV{xuB{j3Kpf7A)fYksep&O|9R+tg|g27n?< zmZ@?hw=;F~qsW>y<6z=ima+k#*Vjy!ywW?>0J+uJQ%Kz~wPBU07rGG@6L+F)^pbs# z1y#$87sUab^JZoVvwN=Dj+go3zLrfRO323E@jHHhk=3$l=Ac;%xvvVb2O(~>0Qw~R z?+(olZ-r8or`Q%b)3N|2PRXIZ#tK8|4Rk1SCO&YYw#0Uo%UrNRG}<6|UDa(#@3W`1 z;Kfsj>$d1;Vt<1@B2T$|_VUZLzoFFM(6Tx}E5y<((2$z#Ro)gjysb+;qf=_3+oHjq z-6(kC1(?YXbQ`+e3q5CA%|fQ_1ro-wgGjB&2v&3 zCz5-W-NHD&xf3<;h>smkU|_Xs>rSb^Fl;O~wFC#yrN+haTC=!RtZetyVb1Wh=S@|p zrjR$ue5MtX4Jl8vw#;y)TWA}X`PP!0(=*+F0eHuC)0?}2B_m5UR!bDB8Y;_#o)Js3 zEJ5YwV0S*F|E9g^rVVKjI%og<`#+2g7(5b&bYl%*+wwX$7n%z#wc12TYr^(zOr7Z948wIsrBN-geCX-Z{yiR9)mtID%;6WI0PgJiU1P z<~QH4*Jl4}Hp5+QSPDO`RdtAS1dm$Yw6uB6a+x)k0LGVo#0I<0@erg*Zrd7m{1vap*+H36qFg>}zObl^8O$Ag5&uOAf2{K@?hl5E5fg z7{608XA62G;s}`v&b-)jt=Wwzq}X^#>{|?(RdI&^lBq(~YX&~p-`EeT6h??%G8n{_ zl?KN4{$4HYg<3LeIGDtNd;D!=;miSXX-;NddOjHOu{6d?gdoB2crqC}+MYwdP%%*8 zLj+Tb)?_4e7Lo1Pm&a~{yCC3yPTp10Av$F~bO0loB_Hm%G_b4r5YHVwL^QxLrsD$0 z?rF5n_3d}aYA4q#RmhCC6K2#eT9L>4>6V=&&tVA1YJUCNu6L%>ll79F-e?d;J0C;) zJYwRkIzLU7ff}dpH9+k(I+QHXB!Z@A+_}TX#20Ei;M~Xf z!Js`lvJNT0@Y<+kcAN5l=Je%IrFUrqunS@KY@Zqimu0B=m7e7e_b3uM#t0ZPdf6jT zvX0I9u_tr60&7oMpHJrOmw~BHZh3xk%avD!t9Dag>&BVd?@_{-@O?6FAG@cq3iyLh z*+F=T97!y@r(mt&HIzs@+tNpfa4+A&M=wxx80r<33 zo-!FJbY|S22`;oSg1>2w;Tx5;`WWo1U&WS;9anwK4Mz=b2eq3bEa zlhx?JUb{fNRlJB22H$&l95|xKICjOK;o|L~`1dIm&`V*!P#?h&jiO}I8jgdpk@w?9 zVz`>L66nA8d!NpKqA{v5oI#*TCQPWKWdIYT0;B413ywaaTQKtUYU$xq1~{bQz{$`g zgT~KvAZeq579zG)C@rkX(Cpr)b7w!j`NE8hB5e0b(cbCXp4#Em=C;pOh;LPiey;ia zsexQMI-FCIi&WLA<7hPUWAoIi~YJEEl4T z%`JqH(Rn3tcB&Ez(Y2#x{`O(>30o`{%m({1Pr)QSD5jx`EL+v;Zjg}>;>UioCuR}? zpD@#Zza7`)vxt+=%!DKCRbIiXb(g)!jW~Pp>%R1)7>V#kQ;c{oIE}vau9oF3IZ(}W z+3LiNmL`WJy5gjh;;>{C<^}fKD=E~#uLV^3#K$jkYnthzy{8co&G<%tA?LkQCIqcHpfyQQh7(Cp+rjh~h$tlJ zJ&k0{bP?SW&1-}CpxD!3Ht*+zW-d(g!KE+JJ{)2*axnRB+#U?-;mctajy=;=J3Gdk zD%G!s{ff6N>ur>oT_h~tmM}Egt4Mi>9w&uEj zO7{{Zjw!TN4kjHdJ?|aSI0zwqgJ!E4L=kfU%zvth)JSP5LV0R!Z_w%K`@R{oLrpIhiz&BHT-B_9C-kPQJL;9~{ir`3U@NPGUn zk&nGEd_Cw2!RzZz3RFBe4Pv~oj zI2bZ+mhAZS(H2}1;%eo_(4&$Ge9{mSaj4@s@hYd{CX6O`$6>^jJ$DpGy?|H#c2H_%db8X&>{PR2funRCt!{&bM*PqtjJ7nr@ehk4*R9eWgkX0ipaSHK&19&bP zkxlpSrv?E#RZxDx!neH8;Y!y|K17su|NIxf8GdQ9ZvW1ae)d&Ekk#C}trY&PpEH_( z@Yt0c^yiaGUeoAxvaKg3TfprmQUAvRx#LlTp{`Hup*Dy+sf?L!f;~N*W1*dm_sB@f zmr*ycyK-hvKbNjdsT7V(0(jJa&;98Yl$`>Jp>V$&``Jk>F%T>!;}fk_Or7cMCHUH4 z0I9y)0)5y@jM#8Q2aJTVVt_V|8!!74`=`OE^P@x1ZU^eiUPnV3M_L*z_jH`EfevNh zIPNTDTT3)mwVz`p7V2FyF<`T0B1!k;y%!ppny@(W?m1_xWbny{-1G>4-KNtFN^V9w z`Z51RI`8=yjNQ*vGiN69p@)pe@}rWvVBG z?*fF>rr;Y5o(-)&vv~u5nHWzR%;`jDmrf)*s{^0#YyqJ!J><%K7VaF$L6eYMtA$~v zG(^YQ@Fq~GLH`0!O9KRxI&~Vepf6l14ph9vT;inK6>|mv0O}Hx3R*IM+iv4F5PjEI z5GsmF8(VOk1iLVx0s)FXwu?pH3`3wLDq$m&0!bw`()@d8c(E?ldV}?YFTfP>g2??ptTYQ!F$15Zro8DCTad1;s9|K!8qVqkXjViqqmHEFbD|%eBH;q#$S{OWMBo*EWn5^cOtM<^7OBs-vpmEgE*6Qt(hrw`NXQO zVNl%dNdaG?&8Sj;^q6dytL*yAf~>ODYL_m)qlBv;(7FAQ&hjHlr_FAEq_dnV?Z#Dj z;q0}Q9H4Wd0ITea4e4%uX}h%z>C45CKHEC3*B2rJOf!?D{b@SErK5CsDGrQ^4Q?kY zyCM0{DnnxUF>9h|#XC$Lj`(&O2f(u^We4bk_3>gO<@5J{)(QZ*0R$TMsDNRr!tevz zKo^=xK&K38>GE3uS?LWBP-(W%kPD=PolSZynxx9L^9DzcOz$0o2#5$j})NFof70NMw*E?Bu3?UL1GsMNZgnA zJ`>ltA;3w08;>D;;6A{?VJ`ZxC$>;&a-GiL5=o|LrRG>y*cc#IHoQ&F3VP*(v%R?> z*V(Ffca|x7#Y}@^kc~Ea?*ZcOs_$cD0p4G!sy3uX1eD?@bT9sB#*8$ zIFTgwvN#=rjBB`53>76yPFJnJHBx)YdT9ohU%=#`JxxtjS@nENT7vMDz&X|mR_eLY}$qa<`pnAG1O9nZoGfAJb6U6+p z`}f!4{6Sx4o-trR%hH*}fkBM8WKafJUS6joF!bvAC2pKygbz?E#1Q=VICWDV&SY-c zh1W}eA;|4+6eCVZ{rYSy=ib42Z?jzusA9wtRJ)t+q`U3A9h$a6D#&fi_92y0)VyNQ zK;Xl!_2w>{UL&@*a{4ngf2Nh6Vu!h}JtpY@xO!qtoA!wgh;WtN+}YN;y(6Cn8DMxf zcXljTr9axYbb{Pj#g|Ao+27{Sc?CkX5F8fq8WDWk(q zV3oV5b8>^6Dk`QnWG`g{k_Xj6Lg2H|hP65<-mYw)*n!wpiHR3fKMlUtZ;o%S25zFw z&*xxYbATbBSY!W&Iw1 z{<>?7j|GhrXORy1Aly9wX8@VIdceflLI)J4Ya2FzliSQ+X(I<{+Uxy7zA<inj&iD$HJp%dIuIRH$LY)C)8Yeq$ISVp;vFSqZ`IzM+&ow3zk zG5YLu{2XON^SyhZaOF>KEBhF(FgyQ9p53d*?f~0T=%`@es7o8yUA>S1$9{h!!wg$bwUEcG?Sb#9FxFxDgyI8lg>FBlLmGw0#!?sFh?Jg zG<6V@VRjkk|wP#cq&b_^g?yu@7Mq}mm81^@u+5&!@y00000000000002Afiqr{ Y4q7ae@OCN!?qri3cR&WabpQYW0Hr^_%K!iX delta 3724 zcmV;74s-FU_yOYX0kD-P5~jJtT(?J*lD-E30NWG*02=_4-+3K>wOHGZBexNK*H;vJ zAKaR>dS`t@AY%*(1x70rC;~!hA_ik?e~uv#}pC zFucuTu~o83BsbD1QLA4h2`X!0)cT;B&Eg-Ov`*=U?mrq)%6n3c)8w|2+FBeo zMt4p1nI$8eN*c0%ThYpVHv5SAmC<_(*{DZprAzyI6y!hIIaNmNE!k2w($d=K9UOp) zYN*uBT5V=}^^+{!k_4Q%SB-M;^ZuS}BU-!F4$7Uynp#;$dQGh~3+rS{@=-R<=JZEQ zXroF|*9Ukm+$oVAXG*a!_TJB&-coblt&;FN# z+hSB}-S!r{x+1G;XkwQN@VRgpR(HTqRwREGY%RxfU8zb^EuHlWJyxb`cs#o@Qq)&C zZd_tt%iRiVWK{_D`irlMF+xQjp}ai+6?W+x*ie>sy6S3d-kHYEco!pWS39gZ4<`00 z9eud@^ACQ1_~9l-6h4U>&Qu4EhV6FJW@Pnj>U0Pod)(uqJJ)s&)^Ti6F=&EK?sWM8 za(wqF4fNQLEp{Nd+ID6D^>^Nl#o8!%04yy6Cyb@E(bROi_K*?2cEYu_cC}i~vzgVT z9D`@6Yf6N41GEv$_rYYw-y8$@j&SoRMj$6Hwr|KU?d~=RxWYEn^?$@0vr@ zZk5S@OKoHh1*Bq}|ASC9wwcXdd8|g_krOKr7#V;wuhq}8T7m>XtVWm#c%;|)7oTTz zoqzT9JcGDHX>l+E`PaXX@(_S2-2$7y13}k|H`mv6^SAE-M)N2<^QhHU`7@G$SFb!h zLY$z?=9u})7!At2ADsO$CduEmsjD1`AYD3t&u0aU7dP+U{pmY;>(^h;W{B0A3gofT z?IEpDJd9d(ym(2KD%~vv8LewA+})fO--gfSjPU0WfEuQdC}~Sq9`v-G(^{hrGViA5 z@<#zoY&s`}szu)dk?XlPV893@jDnHM`@Ev;mK zt>n5tNf@O#v#uXeKuTR}vn2RI|D+#uBmE)9lYtO-;WQxI$Gu*JfCe*5B$&d6JN)ZZ z9qE9&lrhbl-Vb6wa!*z^gcBTZ&y?Yc~}qw1l^0V znh%N*%%?%%j~2{NSKNBpU3RP&w?0;X^uklk=mnJH*U7xF58vZfdvU(iwJLczYbN7l zIPxhz!*o2!8xY|f=C^Od{6L-2Y=5$s7kY)0mCsXto;mSJIDZ|K6BsYYb)q^VdVI18 zk|>(J<1ro9p`H~Fs;N@dto8DQ`#v2o3fzq9fSh&n4eCY`Wyw*pMq$o&8Bk?^`T7%0 zDbqNW^|f>!K;`TRKEX{Kt*8Z(EDeKCdleUOp@wH+6ntY*f|P0S4u}rInIrwTfnm$j z3}I=y->Pz3_>`X~ULz9XXcsRV&4-e5YHyu9P4<0GpS;?0G#iyB8a*1VDjy2rUJeew z^ln1|jwLMY-Cqqy!SdGnnbGBcLHDc@MaNh;b@X;r;LAGo`=`B`+iSRXMdN*v(U%jd zUR?6##U*z=6!EoN^l#m?)Vm#?Fur`hShvs9)3XBk&|le;@{&Z8n2ukQIVTK2qPI6u>l0?lylmujDqJg^?eU)SH_|kj5kFslTzKL{{?47t z*ShTNxwwBNpU;TK?xi`B556N|6*~@VXh#v8ED8w5r>SRvHy{G4+ zMC>)rQ}O>0_^>Jd`!x(0PvOOYkL9RF$^0T4&Wo`>@Bb!=_v#``fd82H{dJa{s(Rua z6`J`43Jtcr;DV_MsRrGDTA)uDCj5CuSVs9J0v_9l;`uQpCyk%)KnkZPEkqh#p**o< zqJ_0rS>(@1U(m<{5xP&V_7U%}>!4$Nv_JJie6JhybMCkQY7pCd3hjL!UWy-mo60Ok zTJVnC!<1XJBYSSK&VN6kRj29--Ad!v-{1T3`0&3eWrJ^x)|eZAwfp<0p6uaKeS-|Q zoXoz^qQl_vGqm^In|hm4lv^+7y8;Hnm6gHIJ=(jBhU3kB;yG??b!@C-qQ-6q zjg9Ykxcl5IbZcmBLj!Ab`fF3G2iZ5|+&yRfqlP)4-I8_HchBK{D2?J$GW_P?`*Hp? z;4{m_lIBsww#j#Yz-#T{wRs=MfPcXky{CY0GQt;*Et^tTp{e#iXmC`&p$FNz*nU`9 zuNUZYOFY!LIpp@0aM1+_J*vxaaUb*O4jbOyg{Kx9I>DA5^+lC;{zbLvkT#+J4*CdS z7=~rTkXH|bDG1)me7y6$f4OK|+2>IIcic9Fd@WI=`$b*=g04a zu>^gpMqbeW?PYja+B22+x_IABqHwZ)?8aWYe?ED~4EcxYS~jVV!n88RtYpt#dI?R{ zZM5b8?aStW0kgw1dMXY*e8gO%Ml*8k1pokN5tB<^GJor9<2Vri?!Q797VK?e(zIol zfDa1CeY{4SYl4-zm*o%4_WdKc?Yx3q-xDL`(br_TXPTFUAqS}%MHZhytI{0P&@*zK=)mQ$wPI14X=zqXA4=v+v^D*I|( zx?5i|x3(^Qz39>xo5%I~f=7U9Zj!XGrXyS`rNaw7a3(isPL|dn`Oh*#VE8dBqF5<9 z3>`{*JM{zLNsFcf^uhV~ux^^?pPf+vxqkr!8vdwY!j#mA1K&Uwnu$ZF2wCCcTfnf= z8yG-^*+N4ua2)Jx&|}gh{N5G?d9GgUP)gvt&mz8XDok=kgq~y*Uj}9v$FL^&*iHlg zL~1UR30JxsDVxe%<>Y$A09o#oByTZ}nRr8DWR4dkc5VR0eR=OAag7@soV4*6!hZ+h z9UK&M(Yrm7g-ny{bOx6=WN6!Hfq4bP0I{MLZE}{-EAO1`%>}v6R=v8jNKq?B8WaO< zWazyIh`XzPj*$gue`TuNkg9GNm>05YW`Q)*Js*OPzcwLwRGmRZlGxMYbO4$6C(?P@S8MpJ@mck`Wiw|%!m)0Q|2a@(@KOGP6qQF3S?&|%MdaTiU`5!+ii z{hcv?#+9FZhrO>oCg}jUdLqo~_K6J!;VQeiv#oV|M?McCK=*F$>{_r&e`Ve&{d|6O zMG}~#Fq|z%3M|3XD4NsPPk#dTFSxcs4`$9kgIizLkh{ok869>4FWp6*lN+3=vgB$* z_Dxd*CLzV3ZR;U?hSdo-W{SJNV)BtEC%*^th6OA;mAHo3so%q2cYn3o3XHd271J)ey#V{Hp%aD2}Clz)hQU$PO@Tq2R4 zSkt+EchdQ}it?;g_KLG-$Kx+?>zW77fx&$<8I|@KP+?;I^I&$b9=ijW{ccZuHXK{V z%91jFp2W)~mKr+S4u1zB?W1eOAes5iMKP%G(pGp@_I7nqST;+Sqd25FkF|m4KWYH> z{)z_yM5b3^KTwGdkPA2ry_Dl&hTlRn{>{RA&?(>~E*WYlf z^pJJeLi$ePfmrY?e+%Ej_g*8rL4_E)tePE`o z8&ivY)*kU2{*BPqrUyFw$OJ5_gT6$PerjH^n)oLIrIH=+{|;QzC`Fu8>Q^)e`sja9 zO9KQH000080JHdH#}p2xxx`$zN0gGj2LJ%u6qC()SptGDlixEKlR$bZ0)Q}+-83AN zhZ04e|g q00000000000JMSfS(8s*ER(o(L0002)em&3t diff --git a/docs/pyodide_playground_strategy.md b/docs/pyodide_playground_strategy.md index 42888a2..ef7d298 100644 --- a/docs/pyodide_playground_strategy.md +++ b/docs/pyodide_playground_strategy.md @@ -129,9 +129,33 @@ on a multi-MB download. Full deletion waits until the real path is the verified default — at which point the JS mock can be dropped and the contract test becomes the single source of truth. -**Phase 2 — tabletop renderer + hero loop (1–2 days).** Add the continuous -tabletop renderer and wire `pick_and_retry`. Now the README hero GIF has a -"run it yourself" twin. +**Phase 2 — tabletop renderer + hero loop. ✅ built (Python path verified; needs +a browser check).** The playground has a **"Pick and retry (real Python)"** +scenario. It is real-Python-only: selecting it boots Pyodide and runs the +**unmodified** `examples/manipulation/01_pick_and_retry.py` `run(seed=3)` loop — +there is deliberately no JS mock, because the dynamics are stochastic and +belief-driven and a hand-faked version would reintroduce the exact drift Pyodide +removes. The README hero GIF now has a "run it yourself" twin. + +A continuous `tabletop2d` renderer draws the real scene: the true object vs. the +agent's **spatial belief** (mean + a shrinking uncertainty radius), the occluder, +the camera, the last detection, and each pick attempt — mirroring the matplotlib +render in `pir/worlds/tabletop_2d.py`. The belief panel switches to a spatial +layout (uncertainty bar + attempts/retries/policy). + +To make the loop's belief inspectable without the live agent object, +`01_pick_and_retry.py` now records `belief_mean`/`belief_radius`/`retry_count` +into the trace `info` each step (belief becomes first-class in the `Trace`). +Scene geometry (true object, occluder, camera) is ground truth the agent never +sees, so the driver reads it from a real `Tabletop2D` and passes it to +`pick_and_retry_trace_to_playground` rather than hard-coding world constants. +`tests/test_playground_trace.py` pins this second contract too. + +Verified locally (unpacked-bundle sim, the exact `playground.js` driver): +`seed=3` yields the hero story — `scan → pick(miss) → pick(miss) → pick(done)`, +belief radius shrinking 10 → 9.8 → 9.5 → 2.5, `holding=True`, `retries=2`. +**Still to confirm in a real browser:** selecting the scenario boots Pyodide and +the tabletop/belief/timeline redraw from the real trace. **Phase 3 — editable code cell (1–2 days).** Expose the agent's `act()` in a small editor so visitors can tweak the retry/belief logic and re-run. This is diff --git a/examples/manipulation/01_pick_and_retry.py b/examples/manipulation/01_pick_and_retry.py index 074df49..a1f4750 100644 --- a/examples/manipulation/01_pick_and_retry.py +++ b/examples/manipulation/01_pick_and_retry.py @@ -105,6 +105,13 @@ def run(seed: int = 3, render: bool = True, max_steps: int = 40) -> Trace: result = env.step(action) obs, reward, done, info = result.as_tuple() agent.update(obs, reward, info) + # Record the agent's belief in the trace so it is inspectable without the + # live agent object (used by the browser playground and trace tooling). + info["belief_mean"] = ( + None if agent.belief_mean is None else agent.belief_mean.copy() + ) + info["belief_radius"] = float(agent.belief_radius) + info["retry_count"] = agent.retry_count trace.append(obs, action, reward, info) if render: diff --git a/pir/viz/playground_trace.py b/pir/viz/playground_trace.py index 73bcbc3..9dd8d5e 100644 --- a/pir/viz/playground_trace.py +++ b/pir/viz/playground_trace.py @@ -152,3 +152,137 @@ def clarifying_trace_to_playground( "initial": _initial_snapshot(command), "steps": steps, } + + +# --- pick_and_retry (continuous tabletop) ----------------------------------- +# +# This loop has no red/blue distribution; its "belief" is a 2D position estimate +# (mean + shrinking radius) that the JS renderer draws spatially. Scene geometry +# (true object, occluder, initial camera) is ground truth the agent never sees, +# so the caller passes it in from the real Tabletop2D rather than this module +# hard-coding world constants. Everything emitted here is plain JSON. + + +def _xy(point: Any) -> list[float]: + return [round(float(point[0]) * _SVG_SCALE, 4), round(float(point[1]) * _SVG_SCALE, 4)] + + +def _pick_and_retry_state(info: dict[str, Any], has_belief: bool) -> str: + if info.get("success"): + return "done" + if _failure_kind(info) == "grasp_miss": + return "update_belief_and_retry" + if (info.get("action_type") or "") == "look": + return "scan_for_object" + return info.get("action_type") or "noop" + + +def _pick_and_retry_policy(info: dict[str, Any], has_belief: bool) -> str: + if info.get("success"): + return "done" + if not has_belief: + return "scan" + if _failure_kind(info) == "grasp_miss": + return "retry" + return "act" + + +def pick_and_retry_trace_to_playground( + trace: Any, + *, + object_xy: Any, + occluder: Any, + camera: Any, + command: str = "pick the block", +) -> dict[str, Any]: + """Convert a pick_and_retry `Trace` into the playground's tabletop2d config. + + ``object_xy`` / ``occluder`` / ``camera`` are the real Tabletop2D geometry in + table coordinates (0..1); they are scaled to the renderer's 0..100 canvas. + """ + + obj = _xy(object_xy) + occ = [round(float(v) * _SVG_SCALE, 4) for v in occluder] + cam0 = _xy(camera) + + initial = { + "type": "tabletop2d", + "command": command, + "target": "block", + "agentState": "scan_for_object", + "failure": "none", + "object": obj, + "occluder": occ, + "camera": cam0, + "detection": None, + "pickAt": None, + "holding": False, + "belief": {"meanXY": None, "radius": None, "attempts": 0, "retries": 0, "policy": "scan"}, + } + + steps: list[dict[str, Any]] = [] + last_detection: list[float] | None = None + for action, reward, info, obs in zip( + trace.actions, trace.rewards, trace.infos, trace.observations + ): + detections = obs.get("detections") or [] + if detections: + last_detection = _xy(detections[0]["position"]) + + belief_mean = info.get("belief_mean") + mean_xy = None if belief_mean is None else _xy(belief_mean) + belief_radius = info.get("belief_radius") + radius_svg = ( + None if belief_radius is None else round(float(belief_radius) * _SVG_SCALE, 3) + ) + pick_position = info.get("pick_position") + pick_at = None if pick_position is None else _xy(pick_position) + holding = bool((obs.get("gripper") or {}).get("holding")) or bool(info.get("success")) + attempts = int(info.get("attempts", 0)) + retries = int(info.get("retry_count", 0)) + failure = _failure_kind(info) + action_type = info.get("action_type") or action.get("type", "noop") + + if action_type == "look": + label = "look(scan)" + elif action_type == "pick": + label = f"pick(attempt {attempts})" + else: + label = action_type + + snapshot = { + "type": "tabletop2d", + "command": command, + "target": "held" if holding else "block", + "agentState": _pick_and_retry_state(info, mean_xy is not None), + "failure": failure or "none", + "object": obj, + "occluder": occ, + "camera": _xy(obs["camera"]) if obs.get("camera") is not None else cam0, + "detection": None if holding else last_detection, + "pickAt": pick_at, + "holding": holding, + "belief": { + "meanXY": mean_xy, + "radius": radius_svg, + "attempts": attempts, + "retries": retries, + "policy": _pick_and_retry_policy(info, mean_xy is not None), + }, + } + steps.append( + { + "action": label, + "reward": round(float(reward), 4), + "failure": failure, + "agentState": snapshot["agentState"], + "snapshot": snapshot, + } + ) + + return { + "command": command, + "totalSteps": len(steps), + "initial": initial, + "steps": steps, + } diff --git a/tests/test_playground_trace.py b/tests/test_playground_trace.py index 0c3fcea..fabeae4 100644 --- a/tests/test_playground_trace.py +++ b/tests/test_playground_trace.py @@ -13,7 +13,11 @@ import json from pathlib import Path -from pir.viz.playground_trace import clarifying_trace_to_playground +from pir.viz.playground_trace import ( + clarifying_trace_to_playground, + pick_and_retry_trace_to_playground, +) +from pir.worlds.tabletop_2d import Tabletop2D ROOT = Path(__file__).resolve().parents[1] @@ -98,3 +102,80 @@ def test_config_is_plain_json() -> None: config = clarifying_trace_to_playground(_run("red"), answer="red") reparsed = json.loads(json.dumps(config)) assert reparsed == config + + +# --- pick_and_retry (continuous tabletop) ----------------------------------- + +TABLETOP2D_FIELDS = { + "type", + "command", + "target", + "agentState", + "failure", + "object", + "occluder", + "camera", + "detection", + "pickAt", + "holding", + "belief", +} +SPATIAL_BELIEF_FIELDS = {"meanXY", "radius", "attempts", "retries", "policy"} + + +def _run_pick_and_retry(seed: int = 3): + path = ROOT / "examples" / "manipulation" / "01_pick_and_retry.py" + spec = importlib.util.spec_from_file_location("pick_and_retry_contract", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.run(seed=seed, render=False) + + +def _pick_and_retry_config(seed: int = 3): + geom = Tabletop2D(seed=seed) + return pick_and_retry_trace_to_playground( + _run_pick_and_retry(seed), + object_xy=list(map(float, geom.obj.position)), + occluder=[float(v) for v in geom.occluder], + camera=list(map(float, geom.camera_pos)), + ) + + +def test_pick_and_retry_shape_matches_tabletop2d_contract() -> None: + config = _pick_and_retry_config() + + assert config["command"] == "pick the block" + assert config["totalSteps"] == len(config["steps"]) >= 1 + assert set(config["initial"]) == TABLETOP2D_FIELDS + assert config["initial"]["type"] == "tabletop2d" + # occluder is the real Tabletop2D rectangle scaled to the 0..100 canvas. + assert config["initial"]["occluder"] == [43.0, 42.0, 57.0, 68.0] + + for step in config["steps"]: + assert set(step) == STEP_FIELDS + snapshot = step["snapshot"] + assert set(snapshot) == TABLETOP2D_FIELDS + assert set(snapshot["belief"]) == SPATIAL_BELIEF_FIELDS + + +def test_pick_and_retry_misses_then_picks_and_belief_appears() -> None: + config = _pick_and_retry_config(seed=3) + steps = config["steps"] + + # seed=3 is the hero seed: at least one grasp_miss before the pick succeeds. + assert any(step["failure"] == "grasp_miss" for step in steps) + + final = steps[-1] + assert final["agentState"] == "done" + assert final["snapshot"]["holding"] is True + + # Belief becomes a concrete spatial estimate once the object is detected. + assert any(step["snapshot"]["belief"]["meanXY"] is not None for step in steps) + # Retry count is non-decreasing and ends at >=1 (it missed at least once). + assert final["snapshot"]["belief"]["retries"] >= 1 + + +def test_pick_and_retry_config_is_plain_json() -> None: + config = _pick_and_retry_config() + assert json.loads(json.dumps(config)) == config