From 577082a9ba423d15146b439c6ef0c32aec3c9ef9 Mon Sep 17 00:00:00 2001 From: Nitin Khanna Date: Fri, 19 Jun 2026 07:42:54 -0700 Subject: [PATCH] Add Rodin (Hyper3D) text-to-3D generation provider Wire a new BYOK-gated Rodin/Hyper3D provider through the full asset generation stack, plus related copilot and dashboard fixes. Backend (Go AI proxy): - RodinClient with create/status/download over the three-call task lifecycle; OSS-gated POST /api/AI/ObjectGeneration/Rodin/Generate handler and rodin case in the task-status router. - BYOK resolution chain (RODIN_API_KEY / HYPER3D_API_KEY) in handle_capabilities.go; api_clients wiring; unit tests. Frontend (editor-oss): - RodinDirectClient for browser-direct polling in playground mode. - Provider routing in ModelGeneratorProvider via a centralized capability table; wired StemAI, AiWorldController, ObjectHandlers, CommandsRegistry, helpData, Create/PromptStep UI, BYOKKeysPanel, and CSP (hyperhuman.deemos.com). OSS asset fetch via uploadModelFromUrl. - Copilot playground knowledge cards; OpenFolderBanner in-place refresh. Docs: byok.md, runtime-api.md, README, CONTRIBUTING, CLAUDE.md, .env.example, and planning doc. Excludes the 69MB compiled server/ai-server binary (now gitignored). Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 5 + .gitignore | 2 + CLAUDE.md | 4 +- CONTRIBUTING.md | 5 +- README.md | 2 +- .../editor-oss/src/agent/CommandsRegistry.ts | 6 + .../src/agent/handlers/ObjectHandlers.ts | 67 +- .../agent/handlers/SettingsHandlers.test.ts | 16 +- .../src/agent/handlers/SettingsHandlers.ts | 1 + .../src/agent/script-tool/helpData.ts | 1 + .../src/ai/RodinDirectClient.test.ts | 53 + .../editor-oss/src/ai/RodinDirectClient.ts | 163 +++ client/packages/editor-oss/src/ai/types.ts | 3 +- .../src/behaviors/stem/ai/StemAI.ts | 2 +- .../behaviors/stem/ai/createAIInterface.ts | 2 + .../AiWorldController/AiWorldController.ts | 11 +- .../src/copilot/DirectCopilotProvider.test.ts | 548 +++++++- .../src/copilot/DirectCopilotProvider.ts | 1202 ++++++++++++++++- .../packages/editor-oss/src/copilot/index.ts | 2 + .../src/copilot/playgroundCopilotKeys.test.ts | 36 +- .../src/copilot/playgroundCopilotKeys.ts | 176 ++- .../copilot/playgroundKnowledgeCards.test.ts | 58 + .../src/copilot/playgroundKnowledgeCards.ts | 252 ++++ .../src/copilot/playgroundLLMClient.test.ts | 174 ++- .../src/copilot/playgroundLLMClient.ts | 164 ++- .../copilot/playgroundStemscriptKnowledge.ts | 6 +- .../copilot/playgroundStemscriptPlan.test.ts | 88 ++ .../src/copilot/playgroundStemscriptPlan.ts | 266 +++- .../AiCopilot/AiCopilot.sessionMode.test.tsx | 75 + .../assets/v2/AiCopilot/AiCopilot.styles.ts | 13 + .../editor/assets/v2/AiCopilot/AiCopilot.tsx | 59 +- .../assets/v2/AiCopilot/AiKeysModal.test.tsx | 8 +- .../assets/v2/AiCopilot/AiKeysModal.tsx | 2 +- .../AiCopilot/workspaceChatSnapshot.test.ts | 58 +- .../v2/AiCopilot/workspaceChatSnapshot.ts | 214 ++- .../assets/v2/ContextMenu/Create/Create.tsx | 42 +- .../v2/ContextMenu/Create/PromptStep.tsx | 22 +- .../CreateDashboardView.tsx | 3 + .../v2/CreateDashboard/OpenFolderBanner.tsx | 73 +- .../BYOKKeysPanel/BYOKKeysPanel.tsx | 33 +- .../ModelUpload/utils/render.worker.ts | 29 +- .../src/model/uploadModelFromUrl.ts | 12 +- .../editor-oss/src/utils/CSPMetaTag.tsx | 8 + .../src/utils/ModelGeneratorProvider.ts | 94 +- docs/byok.md | 8 + ...6-06-16-rodin-asset-generation-provider.md | 330 +++++ docs/runtime-api.md | 6 +- .../tools/ai/byok/handle_capabilities.go | 1 + .../tools/ai/helpers/api_clients.go | 89 ++ .../tools/ai/helpers/rodin_client.go | 269 ++++ .../tools/ai/helpers/rodin_client_test.go | 195 +++ .../handle_rodin_generate_oss.go | 108 ++ .../tools/ai/object_generation/handle_task.go | 7 + 53 files changed, 4777 insertions(+), 296 deletions(-) create mode 100644 client/packages/editor-oss/src/ai/RodinDirectClient.test.ts create mode 100644 client/packages/editor-oss/src/ai/RodinDirectClient.ts create mode 100644 client/packages/editor-oss/src/copilot/playgroundKnowledgeCards.test.ts create mode 100644 client/packages/editor-oss/src/copilot/playgroundKnowledgeCards.ts create mode 100644 docs/planning/2026-06-16-rodin-asset-generation-provider.md create mode 100644 server/server/controllers/tools/ai/helpers/rodin_client.go create mode 100644 server/server/controllers/tools/ai/helpers/rodin_client_test.go create mode 100644 server/server/controllers/tools/ai/object_generation/handle_rodin_generate_oss.go diff --git a/.env.example b/.env.example index 3d51a8a9..5c13c1fd 100644 --- a/.env.example +++ b/.env.example @@ -91,6 +91,11 @@ MULTIPLAYER_PORT=2567 # Multiplayer sidecar — ws://localhost: # Tripo — alternate text-to-3D + character rigging. # TRIPO_API_KEY= +# Rodin (Hyper3D) — alternate text-to-3D model generation. +# RODIN_API_KEY= +# Optionally override the API base URL (staging / self-hosted): +# RODIN_API_BASE_URL= + # ElevenLabs — text-to-speech, NPC voices. # ELEVEN_LABS_API_KEY= diff --git a/.gitignore b/.gitignore index a899bf74..a37a1a7a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ bin/ **/.memsearch/ lib/ +docs/marketing node-compile-cache/ **/node-compile-cache/ pyvenv.cfg server/build/ +server/ai-server web/build firebase-json-dev.json server/.env.b* diff --git a/CLAUDE.md b/CLAUDE.md index d91475e8..f285efed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -160,8 +160,8 @@ proxy that fronts the BYOK keys. - The Go AI server (`server/cmd/ai-server`) is what the editor's `network/adapters/remote-go` calls. It accepts BYOK keys from env or the dashboard's BYOK panel and proxies to Anthropic / OpenAI / Meshy / - ElevenLabs / Anything World. **It is not a hosted service** — it runs - on the user's machine. + Tripo / Rodin / ElevenLabs / Anything World. **It is not a hosted + service** — it runs on the user's machine. - Inline `exec` and the script-tool import pipeline (`editor-oss/src/agent/script-tool/`) work without any AI provider. - The dashboard exposes "Import stemscript folder" which stages a folder diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 735038c4..17c324a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,8 +44,9 @@ stemstudio-multiplayer/ ← Colyseus server. Runs as a sidecar in dev server/ cmd/ai-server/ ← Go AI proxy entry point. Forwards calls to - Anthropic, OpenAI, Meshy, ElevenLabs, AnythingWorld - using env keys or BYOK keys forwarded by the editor. + Anthropic, OpenAI, Meshy, Tripo, Rodin, ElevenLabs, + AnythingWorld using env keys or BYOK keys forwarded + by the editor. server/controllers/tools/ai/ ← AI handler implementations. server/controllers/tools/ai/byok/ ← BYOK key resolution. diff --git a/README.md b/README.md index a9d38078..fd124338 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ If StemStudio is useful to you or your organization, please consider sponsoring - **In-editor code editor** — Monaco for behavior and script authoring with full TypeScript-style assist. - **Physics** — Ammo.js / Rapier integration with rigid bodies, joints, raycasting. - **Local multiplayer** — Colyseus sidecar auto-spawned on `bun run dev`. Two browser tabs on the same machine join a real room. -- **AI copilot (BYOK)** — bring your own keys for Anthropic, OpenAI, Meshy (3D model gen), ElevenLabs (TTS), and AnythingWorld. Configure once, use everywhere. +- **AI copilot (BYOK)** — bring your own keys for Anthropic, OpenAI, Meshy / Tripo / Rodin (3D model gen), ElevenLabs (TTS), and AnythingWorld. Configure once, use everywhere. - **Local-first persistence** — IndexedDB for seamless auto-save, or open a real folder via File System Access API (Chromium) for git-friendly workflows. - **Export & share** — package any project as a standalone static site (Player-only build) you can host anywhere. diff --git a/client/packages/editor-oss/src/agent/CommandsRegistry.ts b/client/packages/editor-oss/src/agent/CommandsRegistry.ts index 2b578491..8d7f5cbe 100644 --- a/client/packages/editor-oss/src/agent/CommandsRegistry.ts +++ b/client/packages/editor-oss/src/agent/CommandsRegistry.ts @@ -812,6 +812,12 @@ export class CommandsRegistry { description: "Parent object name or UUID to attach the model", required: false, }, + { + name: "provider", + type: "string", + description: "Generation provider: meshy, tripo, or rodin (defaults to meshy)", + required: false, + }, ], handler: async params => this.objectHandlers.handleGenerate3DModel(params), }); diff --git a/client/packages/editor-oss/src/agent/handlers/ObjectHandlers.ts b/client/packages/editor-oss/src/agent/handlers/ObjectHandlers.ts index 3d40cc82..e45243e5 100644 --- a/client/packages/editor-oss/src/agent/handlers/ObjectHandlers.ts +++ b/client/packages/editor-oss/src/agent/handlers/ObjectHandlers.ts @@ -17,12 +17,27 @@ import { } from "../../editor/assets/v2/materials/materialUtils"; import {IMaterialSettingsTextures} from "../../editor/assets/v2/RightPanel/sections/MaterialRenderingSection/types"; import EngineRuntime from "../../EngineRuntime"; +import {uploadModelFromUrl} from "../../model/uploadModelFromUrl"; import {GENERATOR_TYPES} from "../../utils/ModelGeneratorProvider"; import TagUtil from "../../utils/TagUtil"; import {SupportedCommands} from "../CommandsRegistry"; import {CommandResult} from "../types/ACPTypes"; import {getObjectBaseMetaData, serializeObjectForAI} from "../utils/serialization"; +/** Map an agent-supplied provider string to a generator, defaulting to Meshy. */ +function resolveGenerator(provider?: string): GENERATOR_TYPES { + switch ((provider ?? "").trim().toLowerCase()) { + case "tripo": + return GENERATOR_TYPES.TRIPO; + case "rodin": + return GENERATOR_TYPES.RODIN; + case "meshy": + return GENERATOR_TYPES.MESHY; + default: + return GENERATOR_TYPES.MESHY; + } +} + type BaseObjectData = { uuid: string; name: string; @@ -1194,28 +1209,58 @@ export class ObjectHandlers { async handleGenerate3DModel({ prompt, name, + position, + provider, }: { prompt: string; name?: string; position?: {x: number; y: number; z: number}; parent?: string; + provider?: string; }): Promise { const sceneId = this.engine.editor?.sceneID; if (!sceneId) { return {status: "failed", message: "No scene is currently open.", data: null}; } - const {jobId} = await this.aiWorldController.modelGeneratorProvider!.submitGenerationJob({ - generator: GENERATOR_TYPES.MESHY, - sceneId, - name: name || prompt, - prompt, - }); - return { - status: "success", - message: `Generation job started (jobId: ${jobId}). The model will be added to the scene automatically when ready — no further action needed.`, - data: {jobId}, - }; + // This OSS build has no server-side generation jobs, so generate + // synchronously: create the provider task, poll to completion, fetch + // the GLB URL, import it, and place it in the scene. + const generator = resolveGenerator(provider); + const modelName = name || prompt; + + try { + const res = await this.aiWorldController.generate3dObject({ + generationType: "text_to_model", + prompt, + generator, + }); + if (!res?.model) { + return {status: "failed", message: "Model generation returned no model URL.", data: null}; + } + + const uploaded = await uploadModelFromUrl({url: res.model, name: modelName}); + const point = position ? new THREE.Vector3(position.x, position.y, position.z) : undefined; + const placed = this.aiWorldController.addObjectToScene( + uploaded.object, + false, + undefined, + undefined, + point, + ); + + return { + status: "success", + message: `Generated and placed "${modelName}" with ${generator}.`, + data: {assetId: uploaded.assetId, uuid: placed.uuid}, + }; + } catch (error) { + return { + status: "failed", + message: `Failed to generate 3D model: ${error instanceof Error ? error.message : String(error)}`, + data: null, + }; + } } handleGetPlayer(): CommandResult { diff --git a/client/packages/editor-oss/src/agent/handlers/SettingsHandlers.test.ts b/client/packages/editor-oss/src/agent/handlers/SettingsHandlers.test.ts index c9a0e818..4d40cad6 100644 --- a/client/packages/editor-oss/src/agent/handlers/SettingsHandlers.test.ts +++ b/client/packages/editor-oss/src/agent/handlers/SettingsHandlers.test.ts @@ -19,7 +19,7 @@ function createHarness() { }; const app = { camera, - editor: {scene}, + editor: {scene, sceneName: "Untitled"}, environmentManager: { updateEnvironmentSettings: vi.fn(async () => {}), }, @@ -131,6 +131,20 @@ describe("SettingsHandlers normalization", () => { expect(result.data.isGame).toBe(true); }); + it("updates project title, notifies scene-name listeners, and supports project readback", () => { + const {handlers, app} = createHarness(); + + const result = handlers.handleSetProjectTitle({title: "Crystal Dash"}); + const readback = handlers.handleGetSceneSetting({category: "project"}); + + expect(result.status).toBe("success"); + expect(app.editor.sceneName).toBe("Crystal Dash"); + expect(app.call).toHaveBeenCalledWith("sceneNameUpdated"); + expect(app.call).toHaveBeenCalledWith("objectChanged", app.editor, app.editor.scene); + expect(readback.status).toBe("success"); + expect(readback.data).toEqual({title: "Crystal Dash"}); + }); + it("does not turn rendering-only settings into a game", async () => { const {handlers, scene} = createHarness(); diff --git a/client/packages/editor-oss/src/agent/handlers/SettingsHandlers.ts b/client/packages/editor-oss/src/agent/handlers/SettingsHandlers.ts index 17c39478..1e5718a4 100644 --- a/client/packages/editor-oss/src/agent/handlers/SettingsHandlers.ts +++ b/client/packages/editor-oss/src/agent/handlers/SettingsHandlers.ts @@ -603,6 +603,7 @@ export class SettingsHandlers { } editor.sceneName = title; + this.engine.call("sceneNameUpdated"); this.engine.call("objectChanged", editor, editor.scene); return { diff --git a/client/packages/editor-oss/src/agent/script-tool/helpData.ts b/client/packages/editor-oss/src/agent/script-tool/helpData.ts index 17075163..c8d44323 100644 --- a/client/packages/editor-oss/src/agent/script-tool/helpData.ts +++ b/client/packages/editor-oss/src/agent/script-tool/helpData.ts @@ -939,6 +939,7 @@ const COMMAND_PARAMS: Record = { {name: "name", type: "string", required: false, description: "Name for the generated model"}, {name: "position", type: "x,y,z", required: false, description: "Position to place the model"}, {name: "parent", type: "string", required: false, description: "Parent object name or UUID"}, + {name: "provider", type: "string", required: false, description: "'meshy', 'tripo', or 'rodin' (default meshy)"}, ], examples: ['generate model prompt="a medieval wooden barrel" name="Barrel" position=3,0,0'], }, diff --git a/client/packages/editor-oss/src/ai/RodinDirectClient.test.ts b/client/packages/editor-oss/src/ai/RodinDirectClient.test.ts new file mode 100644 index 00000000..97a4fd31 --- /dev/null +++ b/client/packages/editor-oss/src/ai/RodinDirectClient.test.ts @@ -0,0 +1,53 @@ +import {describe, expect, it} from "vitest"; + +import {decodeRodinTaskId, encodeRodinTaskId} from "./RodinDirectClient"; +import { + GENERATOR_TYPES, + getGeneratorCapability, + MODEL_GENERATOR_CAPABILITIES, +} from "../utils/ModelGeneratorProvider"; + +describe("Rodin composite task id codec", () => { + it("round-trips a task uuid and subscription key", () => { + const id = encodeRodinTaskId("task-uuid", "sub-key"); + expect(id).toBe("task-uuid|sub-key"); + expect(decodeRodinTaskId(id)).toEqual({taskUUID: "task-uuid", subscriptionKey: "sub-key"}); + }); + + it("preserves subscription keys that themselves contain the separator", () => { + const id = encodeRodinTaskId("uuid", "a|b|c"); + expect(decodeRodinTaskId(id)).toEqual({taskUUID: "uuid", subscriptionKey: "a|b|c"}); + }); + + it("yields an empty subscription key when the separator is missing", () => { + expect(decodeRodinTaskId("bare-uuid")).toEqual({taskUUID: "bare-uuid", subscriptionKey: ""}); + }); +}); + +describe("model generator capabilities", () => { + it("includes every generator enum value", () => { + for (const value of Object.values(GENERATOR_TYPES)) { + expect(MODEL_GENERATOR_CAPABILITIES[value]).toBeDefined(); + } + }); + + it("marks Rodin as text-only, browser-direct, no rig/refine", () => { + const cap = getGeneratorCapability(GENERATOR_TYPES.RODIN); + expect(cap.byokProvider).toBe("rodin"); + expect(cap.supportsTextToModel).toBe(true); + expect(cap.supportsImageToModel).toBe(false); + expect(cap.supportsRefine).toBe(false); + expect(cap.supportsAutoRig).toBe(false); + expect(cap.supportsBrowserDirectPlayground).toBe(true); + }); + + it("only offers browser-direct providers in the playground filter", () => { + const playgroundProviders = Object.values(GENERATOR_TYPES).filter( + v => getGeneratorCapability(v).supportsBrowserDirectPlayground, + ); + expect(playgroundProviders).toContain(GENERATOR_TYPES.MESHY); + expect(playgroundProviders).toContain(GENERATOR_TYPES.RODIN); + expect(playgroundProviders).not.toContain(GENERATOR_TYPES.TRIPO); + expect(playgroundProviders).not.toContain(GENERATOR_TYPES.ERTH); + }); +}); diff --git a/client/packages/editor-oss/src/ai/RodinDirectClient.ts b/client/packages/editor-oss/src/ai/RodinDirectClient.ts new file mode 100644 index 00000000..5b22aaf3 --- /dev/null +++ b/client/packages/editor-oss/src/ai/RodinDirectClient.ts @@ -0,0 +1,163 @@ +// Browser-direct Hyper3D Rodin client. +// +// In the public-site playground there is no Go `ai-server` to proxy model +// generation, so — as with Meshy — the browser calls the Rodin API directly +// with a BYOK key. This mirrors the subset of the Go server's Rodin wrapper +// that `ModelGeneratorProvider` needs. +// +// Rodin's task lifecycle uses three calls keyed off two identifiers (the task +// uuid for downloads, the subscription key for status), so this client packs +// both into the same composite `task_id` the Go path uses: +// "|" +// +// NOTE: unlike Meshy, Rodin's CORS posture for browser origins is not formally +// documented. If the playground origin is blocked, generation surfaces a clear +// network error; desktop builds route through the Go server instead. + +import {getBYOKKeyStore} from "./aiBackendFactory"; + +const RODIN_BASE_URL = "https://hyperhuman.deemos.com/api/v2"; +const TASK_ID_SEPARATOR = "|"; + +/** Task shape consumed by `ModelGeneratorProvider.pollTaskStatus`. */ +export type RodinTask = { + id: string; + status: string; + progress: number; + model?: string; + error?: string; +}; + +export function encodeRodinTaskId(taskUUID: string, subscriptionKey: string): string { + return `${taskUUID}${TASK_ID_SEPARATOR}${subscriptionKey}`; +} + +export function decodeRodinTaskId(taskId: string): {taskUUID: string; subscriptionKey: string} { + const idx = taskId.indexOf(TASK_ID_SEPARATOR); + if (idx === -1) { + return {taskUUID: taskId, subscriptionKey: ""}; + } + return {taskUUID: taskId.slice(0, idx), subscriptionKey: taskId.slice(idx + 1)}; +} + +async function getRodinKey(): Promise { + const store = getBYOKKeyStore(); + const key = (await store?.get("rodin"))?.trim(); + if (!key) { + throw new Error( + "No Rodin API key configured. Add one via the AI provider key panel " + + "to generate 3D models with Rodin in the playground.", + ); + } + return key; +} + +/** Map Rodin's per-job status vocabulary to the unified poller vocabulary. */ +function summarizeJobs(jobs: Array<{status?: string}>): {status: string; progress: number} { + if (!jobs.length) return {status: "processing", progress: 0}; + let done = 0; + for (const job of jobs) { + const s = (job.status ?? "").trim().toLowerCase(); + if (s === "done" || s === "succeeded" || s === "success" || s === "completed") { + done++; + } else if (s === "failed" || s === "error" || s === "canceled" || s === "cancelled") { + return {status: "failed", progress: 0}; + } + } + if (done === jobs.length) return {status: "completed", progress: 100}; + return {status: "processing", progress: Math.floor((done * 100) / jobs.length)}; +} + +function selectModelUrl(list: Array<{name?: string; url?: string}>): string | undefined { + let gltf: string | undefined; + for (const f of list) { + const name = (f.name ?? "").toLowerCase(); + if (name.endsWith(".glb")) return f.url; + if (name.endsWith(".gltf") && !gltf) gltf = f.url; + } + return gltf; +} + +/** + * Browser-direct equivalent of the Go server's Rodin object-generation + * endpoints. Every method resolves the BYOK key fresh so a key added after the + * editor loaded is picked up without a reload. + */ +export const RodinDirectClient = { + /** Create a text-to-3D task. `payload` carries at least `{prompt}`. */ + async generate(payload: Record): Promise<{task_id: string}> { + const apiKey = await getRodinKey(); + const prompt = String(payload.prompt ?? "").trim(); + if (!prompt) throw new Error("A text prompt is required for Rodin generation."); + + const form = new FormData(); + form.append("prompt", prompt); + form.append("tier", String(payload.tier ?? "Regular")); + form.append("quality", String(payload.quality ?? "medium")); + form.append("material", String(payload.material ?? "PBR")); + form.append("geometry_file_format", "glb"); + + const res = await fetch(`${RODIN_BASE_URL}/rodin`, { + method: "POST", + headers: {Authorization: `Bearer ${apiKey}`, Accept: "application/json"}, + body: form, + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`Rodin generate failed (HTTP ${res.status}): ${text.slice(0, 300)}`); + } + const body = (await res.json()) as { + uuid?: string; + jobs?: {subscription_key?: string}; + error?: string; + }; + if (body.error) throw new Error(`Rodin generate error: ${body.error}`); + const taskUUID = body.uuid; + const subscriptionKey = body.jobs?.subscription_key; + if (!taskUUID || !subscriptionKey) { + throw new Error("Rodin generate returned no task uuid / subscription key"); + } + return {task_id: encodeRodinTaskId(taskUUID, subscriptionKey)}; + }, + + /** Poll a Rodin task; resolves the GLB url once every job is done. */ + async fetchTask(taskId: string): Promise { + const apiKey = await getRodinKey(); + const {taskUUID, subscriptionKey} = decodeRodinTaskId(taskId); + if (!subscriptionKey) { + throw new Error(`Invalid Rodin task id (missing subscription key): ${taskId}`); + } + + const statusRes = await fetch(`${RODIN_BASE_URL}/status`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify({subscription_key: subscriptionKey}), + }); + if (!statusRes.ok) throw new Error(`Rodin status fetch failed (HTTP ${statusRes.status})`); + const statusBody = (await statusRes.json()) as {jobs?: Array<{status?: string}>; error?: string}; + const {status, progress} = summarizeJobs(statusBody.jobs ?? []); + + const task: RodinTask = {id: taskId, status, progress, error: statusBody.error}; + if (status !== "completed") return task; + + const downloadRes = await fetch(`${RODIN_BASE_URL}/download`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify({task_uuid: taskUUID}), + }); + if (!downloadRes.ok) throw new Error(`Rodin download fetch failed (HTTP ${downloadRes.status})`); + const downloadBody = (await downloadRes.json()) as {list?: Array<{name?: string; url?: string}>}; + const model = selectModelUrl(downloadBody.list ?? []); + if (!model) throw new Error("Rodin task produced no GLB/GLTF output"); + task.model = model; + return task; + }, +}; diff --git a/client/packages/editor-oss/src/ai/types.ts b/client/packages/editor-oss/src/ai/types.ts index bfe48c26..0a429906 100644 --- a/client/packages/editor-oss/src/ai/types.ts +++ b/client/packages/editor-oss/src/ai/types.ts @@ -9,7 +9,8 @@ export type AIProvider = | "elevenlabs" | "anythingworld" | "gemini" - | "tripo"; + | "tripo" + | "rodin"; export type ProviderStatus = "ready" | "missing-key"; export type ProviderSource = "env" | "byok-session" | ""; diff --git a/client/packages/editor-oss/src/behaviors/stem/ai/StemAI.ts b/client/packages/editor-oss/src/behaviors/stem/ai/StemAI.ts index 4f860720..265205f5 100644 --- a/client/packages/editor-oss/src/behaviors/stem/ai/StemAI.ts +++ b/client/packages/editor-oss/src/behaviors/stem/ai/StemAI.ts @@ -15,7 +15,7 @@ export interface Generate3dModelParams { /** Model version to use for generation. */ modelVersion?: string; /** Which generation provider to use. */ - generator?: "meshy" | "tripo"; + generator?: "meshy" | "tripo" | "rodin"; /** Target polygon count for the generated model. */ targetPolygonCount?: number; /** Whether to automatically rig the model for animation. */ diff --git a/client/packages/editor-oss/src/behaviors/stem/ai/createAIInterface.ts b/client/packages/editor-oss/src/behaviors/stem/ai/createAIInterface.ts index 4ef1da2e..4ff46d65 100644 --- a/client/packages/editor-oss/src/behaviors/stem/ai/createAIInterface.ts +++ b/client/packages/editor-oss/src/behaviors/stem/ai/createAIInterface.ts @@ -8,6 +8,8 @@ const mapGenerator = (generator: string | undefined): GENERATOR_TYPES | undefine return GENERATOR_TYPES.MESHY; case "tripo": return GENERATOR_TYPES.TRIPO; + case "rodin": + return GENERATOR_TYPES.RODIN; default: return undefined; } diff --git a/client/packages/editor-oss/src/controls/AiWorldController/AiWorldController.ts b/client/packages/editor-oss/src/controls/AiWorldController/AiWorldController.ts index a5311478..dbfba207 100644 --- a/client/packages/editor-oss/src/controls/AiWorldController/AiWorldController.ts +++ b/client/packages/editor-oss/src/controls/AiWorldController/AiWorldController.ts @@ -50,7 +50,7 @@ import {GenerateImageRequest} from "@stem/editor-oss/types/imageGenerator"; import {getAIBackend} from "@stem/editor-oss/ai"; import Ajax from "@stem/editor-oss/utils/Ajax"; import ImageGeneratorProvider from "@stem/editor-oss/utils/ImageGeneratorProvider"; -import ModelGeneratorProvider, {GENERATOR_TYPES} from "@stem/editor-oss/utils/ModelGeneratorProvider"; +import ModelGeneratorProvider, {GENERATOR_TYPES, getGeneratorCapability} from "@stem/editor-oss/utils/ModelGeneratorProvider"; import {ModelUtils} from "@stem/editor-oss/utils/ModelUtils"; import {backendUrlFromPath} from "@stem/editor-oss/utils/UrlUtils"; @@ -551,9 +551,12 @@ class AIWorldController { const rendered_image = res.thumbnail; const composition = res.composition; const intermediateImage = res.intermediateImage; - // Include rigging metadata if auto-rig was requested - // Use riggingFailed flag to determine if rigging actually succeeded - const riggingMetadata: RiggingMetadata | undefined = autoRig + // Include rigging metadata only when auto-rig was requested AND + // the provider has a proven rigging path (Meshy/Tripo). Rodin + // exposes no rigging yet, so it never reports a rigged model. + // Use riggingFailed flag to determine if rigging actually succeeded. + const providerSupportsRig = generator ? getGeneratorCapability(generator).supportsAutoRig : false; + const riggingMetadata: RiggingMetadata | undefined = autoRig && providerSupportsRig ? { isRigged: !res.riggingFailed, riggedWith: res.riggingFailed ? undefined : generator, diff --git a/client/packages/editor-oss/src/copilot/DirectCopilotProvider.test.ts b/client/packages/editor-oss/src/copilot/DirectCopilotProvider.test.ts index 30124712..a99eee1e 100644 --- a/client/packages/editor-oss/src/copilot/DirectCopilotProvider.test.ts +++ b/client/packages/editor-oss/src/copilot/DirectCopilotProvider.test.ts @@ -5,7 +5,7 @@ import {DirectCopilotProvider} from "./DirectCopilotProvider"; import type {CopilotChatKey} from "./playgroundCopilotKeys"; import type {PlaygroundLLMClient} from "./playgroundLLMClient"; -const openAIKey: CopilotChatKey = {provider: "openai", apiKey: "sk-test", model: "gpt-5.2-codex"}; +const openAIKey: CopilotChatKey = {provider: "openai", apiKey: "sk-test", model: "gpt-5.5"}; const anthropicKey: CopilotChatKey = { provider: "anthropic", apiKey: "sk-test", @@ -20,16 +20,126 @@ const makeLLMClient = (...responses: Array>): Playground }); const makeExecutor = () => { - const executeCommand = vi.fn().mockImplementation(async (command: string, parameters: Record) => ({ - success: true, - step: { - id: "step-1", - command, - parameters, - status: "completed", - }, - result: {message: "ok"}, - })); + const objects = new Map>(); + const behaviors = new Map>(); + let projectTitle = "Untitled"; + let gameSettings: Record = { + isGame: false, + lives: 0, + maxScore: 0, + showHUD: false, + }; + + const executeCommand = vi.fn().mockImplementation(async (command: string, parameters: Record) => { + const successResult = (data?: unknown) => ({ + success: true, + step: { + id: "step-1", + command, + parameters, + status: "completed", + }, + result: {message: "ok", data}, + }); + + if (command === "create_primitive") { + const name = String(parameters.name); + objects.set(name, { + name, + kind: parameters.type, + transform: { + position: parameters.position, + rotation: parameters.rotation, + scale: parameters.scale, + }, + material: { + color: parameters.color, + }, + geometry: {parameters: {}}, + }); + return successResult(); + } + + if (command === "create_group") { + const name = String(parameters.name); + objects.set(name, { + name, + kind: "group", + transform: { + position: parameters.position, + rotation: parameters.rotation, + scale: parameters.scale, + }, + }); + return successResult(); + } + + if (command === "modify_object") { + const target = String(parameters.target ?? parameters.name); + objects.set(target, { + ...(objects.get(target) ?? {name: target}), + name: parameters.name ?? target, + transform: { + ...(objects.get(target)?.transform ?? {}), + position: parameters.position ?? objects.get(target)?.transform?.position, + rotation: parameters.rotation ?? objects.get(target)?.transform?.rotation, + scale: parameters.scale ?? objects.get(target)?.transform?.scale, + }, + material: { + ...(objects.get(target)?.material ?? {}), + color: parameters.color ?? objects.get(target)?.material?.color, + }, + }); + return successResult(); + } + + if (command === "set_project_title") { + projectTitle = String(parameters.title); + return successResult({title: projectTitle}); + } + + if (command === "set_game_settings") { + gameSettings = {...gameSettings, ...parameters}; + if (parameters.enabled !== undefined && parameters.isGame === undefined) { + gameSettings.isGame = parameters.enabled; + } + return successResult(gameSettings); + } + + if (command === "attach_behavior" || command === "set_behavior_config") { + const target = String(parameters.target); + const behaviorId = String(parameters.behaviorId); + const key = `${target}:${behaviorId}`; + const current = behaviors.get(key) ?? {behavior: {attributesData: {}, enabled: true}}; + behaviors.set(key, { + behavior: { + ...current.behavior, + attributesData: parameters.config ?? parameters.attributesData ?? current.behavior.attributesData, + enabled: parameters.enabled ?? current.behavior.enabled, + }, + }); + return successResult(behaviors.get(key)); + } + + if (command === "get_object_settings") { + const target = String(parameters.target); + return successResult(objects.get(target) ?? {name: target, kind: parameters.kind}); + } + + if (command === "get_scene_setting") { + if (parameters.category === "project") return successResult({title: projectTitle}); + if (parameters.category === "game") return successResult(gameSettings); + return successResult({}); + } + + if (command === "get_behavior_settings") { + const target = String(parameters.target); + const behaviorId = String(parameters.behaviorId); + return successResult(behaviors.get(`${target}:${behaviorId}`) ?? {behavior: {attributesData: {}, enabled: true}}); + } + + return successResult(); + }); return { executeCommand, @@ -40,6 +150,76 @@ const makeExecutor = () => { }; }; +const withFakeAssetApp = async (run: (assetSource: any) => Promise) => { + const previousApp = global.app; + const assets: any[] = []; + let assetCounter = 0; + const assetSource = { + kind: "scene", + id: "scene-1", + getAssets: vi.fn(async ({types}: {types?: string[]} = {}) => ({ + assets: types?.length ? assets.filter(asset => types.includes(asset.type)) : assets, + })), + addDependencies: vi.fn(async () => {}), + removeDependencies: vi.fn(async () => {}), + createAsset: vi.fn(async ({type, name}: {type: string; name: string}) => { + assetCounter += 1; + const asset = { + id: `asset-${assetCounter}`, + name, + type, + headRevisionId: `rev-${assetCounter}`, + revisionId: `rev-${assetCounter}`, + }; + assets.push(asset); + return asset; + }), + createAssetRevision: vi.fn(async ({assetId}: {assetId: string}) => { + assetCounter += 1; + const revision = { + id: `rev-${assetCounter}`, + assetId, + createTime: "2026-01-01T00:00:00Z", + }; + const asset = assets.find(item => item.id === assetId); + if (asset) { + asset.headRevisionId = revision.id; + asset.revisionId = revision.id; + } + return revision; + }), + }; + const lambdaConfigs = new Map(); + const lambdaAssetMeta = new Map(); + const scene = { + children: [], + traverse: vi.fn(), + userData: {}, + }; + const fakeApp = { + scene, + editor: { + sceneID: "scene-1", + scene, + assetSource, + lambdaConfigRegistry: { + getConfig: vi.fn((id: string) => lambdaConfigs.get(id) ?? null), + registerConfig: vi.fn((id: string, config: any) => lambdaConfigs.set(id, config)), + updateConfig: vi.fn((id: string, config: any) => lambdaConfigs.set(id, config)), + setAssetMeta: vi.fn((id: string, meta: any) => lambdaAssetMeta.set(id, meta)), + getAssetMeta: vi.fn((id: string) => lambdaAssetMeta.get(id) ?? null), + }, + }, + call: vi.fn(), + }; + global.app = fakeApp as any; + try { + await run(assetSource); + } finally { + global.app = previousApp; + } +}; + describe("DirectCopilotProvider", () => { it("generates StemScript through the provider and executes it in the registry path", async () => { const executor = makeExecutor(); @@ -67,12 +247,12 @@ describe("DirectCopilotProvider", () => { expect(llmRequest?.knowledgePrompt).toContain("StemStudio playground knowledge base"); expect(llmRequest?.knowledgePrompt).toContain("1 unit = 1 meter"); expect(llmRequest?.systemPrompt).toContain("complete playable changes"); + expect(llmRequest?.systemPrompt).toContain("designBrief"); + expect(llmRequest?.systemPrompt).toContain("coreLoop"); expect(llmRequest?.systemPrompt).toContain("Prefer existing built-in behavior components"); - expect(llmRequest?.knowledgePrompt).toContain("Behavior registry"); - expect(llmRequest?.knowledgePrompt).toContain("Built-in behavior catalog"); - expect(llmRequest?.knowledgePrompt).toContain("behaviorId=character"); - expect(llmRequest?.knowledgePrompt).toContain("behaviorId=consumable"); - expect(llmRequest?.knowledgePrompt).toContain("Import API reference"); + expect(llmRequest?.knowledgePrompt).toContain("Dynamic Asset and Registry Inspection"); + expect(llmRequest?.knowledgePrompt).not.toContain("Racing Recipe"); + expect(llmRequest?.knowledgePrompt).not.toContain("Script Imports and Shared Helpers"); expect(llmRequest?.systemPrompt).toContain("set a project title"); expect(llmRequest?.systemPrompt).toContain('project title "Arena Runner"'); expect(llmRequest?.systemPrompt).toContain('description="Copilot generated for:'); @@ -82,22 +262,69 @@ describe("DirectCopilotProvider", () => { expect(llmRequest?.systemPrompt).toContain("list assets"); expect(llmRequest?.systemPrompt).toContain("list imports"); expect(llmRequest?.systemPrompt).toContain("list files"); - expect(llmRequest?.knowledgePrompt).toContain("Asset/import inspection"); - expect(llmRequest?.knowledgePrompt).toContain("Descriptions are searchable metadata"); + expect(llmRequest?.knowledgePrompt).toContain("behavior list/behavior get"); expect(llmRequest?.knowledgePrompt).toContain("list behavior packs"); expect(llmRequest?.knowledgePrompt).toContain("list lambda packs"); expect(llmRequest?.promptCacheKey).toBe("stemstudio-playground-copilot-v5"); - expect(llmRequest?.maxOutputTokens).toBe(4096); - expect(llmRequest?.key).toMatchObject({provider: "openai", model: "gpt-5.2-codex"}); + expect(llmRequest?.maxOutputTokens).toBe(128000); + expect(llmRequest?.key).toMatchObject({provider: "openai", model: "gpt-5.5"}); expect(executor.executeCommand).toHaveBeenCalledWith("create_primitive", expect.objectContaining({ type: "box", name: "TestBox", })); - expect(events).toEqual(["will:create_primitive", "done:create_primitive"]); + expect(events).toEqual([ + "will:create_primitive", + "done:create_primitive", + "will:get_object_settings", + "done:get_object_settings", + ]); expect(response).toContain("Added a box."); expect(response).toContain("Applied 1/1 command"); }); + it("emits workflow progress while OpenAI response chunks stream", async () => { + const llmClient: PlaygroundLLMClient = { + generateText: vi.fn(async request => { + request.onStreamProgress?.({type: "raw"}); + request.onStreamProgress?.({type: "reasoning", delta: "thinking"}); + request.onStreamProgress?.({type: "text", delta: "{\"reply\":\"No changes.\",\"stemscript\":\"\"}"}); + return JSON.stringify({reply: "No changes.", stemscript: ""}); + }), + }; + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: makeExecutor, + }); + const progressLines: string[] = []; + provider.on("toolCallUpdate", event => { + if (typeof event.data.line === "string") progressLines.push(event.data.line); + }); + + await provider.prompt("think for a while before changing the scene"); + + expect(progressLines.some(line => line.includes("OpenAI stream active"))).toBe(true); + expect(progressLines.some(line => line.includes("OpenAI stream complete"))).toBe(true); + }); + + it("selects game, porting, and custom-code cards for complex game requests", async () => { + const llmClient = makeLLMClient({reply: "No changes.", stemscript: ""}); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: makeExecutor, + }); + + await provider.prompt("port this kart racing game with custom vehicle handling and checkpoints"); + const llmRequest = vi.mocked(llmClient.generateText).mock.calls[0]?.[0]; + + expect(llmRequest?.knowledgePrompt).toContain("Full Game Build Flow"); + expect(llmRequest?.knowledgePrompt).toContain("Game Porting and Source Mapping"); + expect(llmRequest?.knowledgePrompt).toContain("Racing Recipe"); + expect(llmRequest?.knowledgePrompt).toContain("Custom Behavior Authoring"); + expect(llmRequest?.knowledgePrompt).toContain("Built-In Behavior Catalog"); + }); + it("includes available behavior registry details in the dynamic provider prompt", async () => { const previousApp = global.app; global.app = { @@ -158,12 +385,36 @@ describe("DirectCopilotProvider", () => { }); it("runs read-only inspection and replans before mutating the scene", async () => { + const objects = new Map>([ + ["Player", { + name: "Player", + kind: "capsule", + transform: {position: {x: 0, y: 1, z: 0}}, + }], + ]); const executeCommand = vi.fn().mockImplementation(async (command: string, parameters: Record) => { + if (command === "modify_object") { + const target = String(parameters.target); + objects.set(target, { + ...(objects.get(target) ?? {name: target}), + transform: { + ...(objects.get(target)?.transform ?? {}), + position: parameters.position ?? objects.get(target)?.transform?.position, + }, + }); + return { + success: true, + step: {id: "step-1", command, parameters, status: "completed"}, + result: {message: "ok"}, + }; + } const result = command === "get_scene_objects" ? {message: "Found objects", data: [{name: "Player", type: "Mesh"}]} : command === "get_object" ? {message: "Retrieved Player", data: {name: "Player", position: {x: 0, y: 1, z: 0}}} + : command === "get_object_settings" + ? {message: "Retrieved Player", data: objects.get(String(parameters.target))} : {message: "ok"}; return { success: true, @@ -297,6 +548,15 @@ describe("DirectCopilotProvider", () => { const executor = makeExecutor(); const llmClient = makeLLMClient({ reply: "Created a playable arena game.", + designBrief: { + coreLoop: "Collect crystals while avoiding hazards.", + controlsCamera: "Third-person character controls.", + goalsFailState: "Win at five crystals, lose when lives run out.", + challengeCurve: "More hazards near the goal.", + feedbackProgression: "HUD score and pickup feedback.", + reusePlan: "Use character and consumable built-ins.", + implementationStrategy: "Set metadata first, then objects and mechanics.", + }, stemscript: [ 'project title "Crystal Dash"', "game settings isGame=true lives=3 maxScore=5 showHUD=true", @@ -319,10 +579,252 @@ describe("DirectCopilotProvider", () => { maxScore: 5, showHUD: true, })); + expect(response).toContain("Design brief:"); + expect(response).toContain("Collect crystals"); expect(response).toContain("Applied 2/2 command"); }); - it("passes Anthropic key and static knowledge to the LLM client", async () => { + it("executes phased plans and materializes reusable behavior artifacts", async () => { + const executor = makeExecutor(); + const llmClient = makeLLMClient({ + reply: "Built the staged game loop.", + phases: [ + { + name: "Environment", + stemscript: "add group name=Arena", + }, + { + name: "Controller", + artifacts: [ + { + type: "behavior", + name: "CheckpointController", + description: "Copilot generated for: checkpoint racing loop; purpose: track laps; inspected/reused: built-in trigger; target: Player", + code: "this.update = function(dt) {}", + metadata: { + attributes: { + maxLaps: {type: "number", default: 3}, + }, + }, + }, + ], + stemscript: "behavior attach Player behaviorId=CheckpointController config={maxLaps:3}", + }, + ], + }); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("make a checkpoint racing game"); + + expect(executor.executeCommand).toHaveBeenCalledWith("create_group", expect.objectContaining({ + name: "Arena", + })); + expect(executor.executeCommand).toHaveBeenCalledWith("add_behavior", expect.objectContaining({ + name: "CheckpointController", + code: "this.update = function(dt) {}", + description: expect.stringContaining("checkpoint racing loop"), + metadata: expect.objectContaining({ + attributes: expect.objectContaining({ + maxLaps: expect.objectContaining({default: 3}), + }), + }), + })); + expect(executor.executeCommand).toHaveBeenCalledWith("attach_behavior", expect.objectContaining({ + target: "Player", + behaviorId: "CheckpointController", + })); + expect(response).toContain("Materialized 1/1 reusable artifact"); + expect(response).toContain("Applied 2/2 command(s) across 2 phase(s)"); + expect(response).toContain("Environment: 1/1 command"); + expect(response).toContain("Controller: 1/1 command(s), artifacts 1/1"); + }); + + it("repairs a non-empty StemScript response with no executable commands exactly once", async () => { + const executor = makeExecutor(); + const llmClient = makeLLMClient( + { + reply: "Generated a script.", + stemscript: "# I forgot the command", + }, + { + reply: "Repaired the empty script.", + stemscript: "add box name=RepairBox position=0,1,0 color=#00ff00", + }, + ); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("add a repair test box"); + + expect(llmClient.generateText).toHaveBeenCalledTimes(2); + expect(executor.executeCommand).toHaveBeenCalledWith("create_primitive", expect.objectContaining({ + name: "RepairBox", + })); + expect(response).toContain("Repair pass:"); + expect(response).toContain("Repaired the empty script."); + }); + + it("repairs failed readback verification once and does not loop", async () => { + let projectTitle = "Untitled"; + let mismatchFirstReadback = true; + const executeCommand = vi.fn().mockImplementation(async (command: string, parameters: Record) => { + if (command === "set_project_title") { + projectTitle = String(parameters.title); + } + const data = command === "get_scene_setting" && parameters.category === "project" + ? {title: mismatchFirstReadback ? "Wrong Title" : projectTitle} + : undefined; + if (command === "get_scene_setting") { + mismatchFirstReadback = false; + } + return { + success: true, + step: {id: "step-1", command, parameters, status: "completed"}, + result: {message: "ok", data}, + }; + }); + const executor = { + executeCommand, + hasPendingInteractiveResults: () => false, + getPendingInteractiveResults: () => [], + handleUserSelectionResult: () => false, + on: vi.fn(), + }; + const llmClient = makeLLMClient( + { + reply: "Set the title.", + stemscript: 'project title "Crystal Dash"', + }, + { + reply: "Re-applied the title.", + stemscript: 'project title "Crystal Dash"', + }, + { + reply: "Should not be used.", + stemscript: "add box name=Unexpected", + }, + ); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("set the game title"); + + expect(llmClient.generateText).toHaveBeenCalledTimes(2); + expect(executeCommand).toHaveBeenCalledWith("set_project_title", expect.objectContaining({ + title: "Crystal Dash", + })); + expect(response).toContain("Verification failed"); + expect(response).toContain("Repair pass:"); + }); + + it("materializes lambda, script import, and file artifacts before assembly StemScript", async () => { + await withFakeAssetApp(async assetSource => { + const executor = makeExecutor(); + const llmClient = makeLLMClient({ + reply: "Created reusable assets and assembled the scene.", + phases: [ + { + name: "Reusable systems", + artifacts: [ + { + type: "lambda", + name: "Enemy State", + code: "export default class EnemyState {}", + config: { + id: "enemy-state", + name: "Enemy State", + version: "1.0.0", + main: "index.js", + attributes: {}, + componentSchema: {}, + }, + }, + { + type: "scriptImport", + name: "wave-math", + code: "export const clamp = (v, min, max) => Math.max(min, Math.min(max, v));", + }, + { + type: "file", + name: "waves.json", + content: "{\"waves\":[1,2,3]}", + format: "json", + contentType: "application/json", + }, + ], + stemscript: "add group name=ArtifactScene", + }, + ], + }); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("make reusable wave artifacts"); + + expect(assetSource.createAsset, response).toHaveBeenCalledTimes(3); + expect(assetSource.createAsset).toHaveBeenCalledWith(expect.objectContaining({ + name: "Enemy State", + format: "json", + contentType: "application/json", + })); + expect(assetSource.createAsset).toHaveBeenCalledWith(expect.objectContaining({ + name: "wave-math", + })); + expect(assetSource.createAsset).toHaveBeenCalledWith(expect.objectContaining({ + name: "waves.json", + format: "json", + contentType: "application/json", + })); + expect(executor.executeCommand).toHaveBeenCalledWith("create_group", expect.objectContaining({ + name: "ArtifactScene", + })); + expect(response).toContain("Materialized 3/3 reusable artifact"); + expect(response).toContain("Applied 1/1 command"); + }); + }); + + it("asks before essential external asset generation instead of mutating", async () => { + const executor = makeExecutor(); + const llmClient = makeLLMClient({ + reply: "A generated hero model is essential for this request.", + assetRequests: [ + { + type: "model", + name: "HeroShip", + prompt: "sleek hero spaceship", + essential: true, + reason: "The user asked for an exact generated model.", + }, + ], + stemscript: "add box name=ShouldNotRun", + }); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("generate a hero spaceship model"); + + expect(executor.executeCommand).not.toHaveBeenCalled(); + expect(response).toContain("Asset generation approval needed"); + expect(response).toContain("Should I generate these assets"); + }); + + it("passes Anthropic key and dynamic knowledge to the LLM client", async () => { const executor = makeExecutor(); const llmClient = makeLLMClient({ reply: "Added behavior.", @@ -380,7 +882,7 @@ describe("DirectCopilotProvider", () => { expect(llmClient.generateText).not.toHaveBeenCalled(); expect(response).toContain("Multiple AI provider keys"); - expect(response).toContain("openai: gpt-5.2-codex"); + expect(response).toContain("openai: gpt-5.5"); expect(response).toContain("gemini: gemini-2.5-flash"); }); }); diff --git a/client/packages/editor-oss/src/copilot/DirectCopilotProvider.ts b/client/packages/editor-oss/src/copilot/DirectCopilotProvider.ts index e3fde1ad..1aac807f 100644 --- a/client/packages/editor-oss/src/copilot/DirectCopilotProvider.ts +++ b/client/packages/editor-oss/src/copilot/DirectCopilotProvider.ts @@ -6,17 +6,27 @@ // applies that script through the same CommandsRegistry used by the terminal. import type {RequestPermissionResponse} from "@agentclientprotocol/sdk"; +import {AssetType, type Asset} from "@stem/network/api/asset"; +import {queryClient} from "@web-shared/queryClient"; import {CommandsExecutor, type CommandExecutionResult} from "../agent/CommandsExecutor"; import {CommandsRegistry} from "../agent/CommandsRegistry"; +import {runScriptCheck, type ScriptCheckReport} from "../agent/script-tool/checkScript"; import {ScriptExecutor} from "../agent/script-tool/ScriptExecutor"; import type {ACPEvent, ACPEventType, InteractiveResult, InteractiveSelectionResolution} from "../agent/types/ACPTypes"; import {ConnectionState} from "../agent/types/ACPTypes"; +import {getAssetResolutionContext, setAssetRevision} from "../asset-management/AssetResolutionContext"; import { buildBehaviorRegistrySummary, buildLambdaRegistrySummary, buildStructuredSceneSummary, } from "../editor/assets/v2/AiCopilot/utils/prompt"; +import {createAsset, createAssetRevision, seedAssetRevisionData} from "../editor/asset-management/hooks/assets"; +import {updateSceneLambdaRevision} from "../editor/lambdas/util"; +import {updateSceneScriptRevision} from "../editor/scripts/util"; +import global from "../global"; +import type {LambdaConfig} from "../lambdas/Lambda"; +import {buildNameAwareScriptImportContext, getScriptImportDependencyMap} from "../script-runtime/scriptImports"; import type {CopilotEventHandler, ICopilotProvider} from "./ICopilotProvider"; import { resolveCopilotChatKeyChoice, @@ -25,15 +35,18 @@ import { } from "./playgroundCopilotKeys"; import { createPlaygroundLLMClient, - PLAYGROUND_MAX_OUTPUT_TOKENS, + getPlaygroundMaxOutputTokens, PLAYGROUND_PROMPT_CACHE_KEY, type PlaygroundLLMClient, + type PlaygroundLLMStreamProgress, } from "./playgroundLLMClient"; -import {PLAYGROUND_STEMSCRIPT_KNOWLEDGE} from "./playgroundStemscriptKnowledge"; +import {selectPlaygroundKnowledgeCards} from "./playgroundKnowledgeCards"; import { parseProviderStemscriptPlan, validateGeneratedStemscript, validateInspectionStemscript, + type PlaygroundPlanArtifact, + type PlaygroundPlanPhase, type PlaygroundStemscriptPlan, } from "./playgroundStemscriptPlan"; @@ -53,17 +66,20 @@ You are the StemStudio playground copilot. You run inside a browser-based 3D edi Your job: - Convert the user's request into live StemScript commands that create or edit the current scene. -- Use the cached StemScript/API knowledge base when choosing scale, physics, cameras, VFX, behaviors, game rules, and scene structure. +- Use the cached StemScript/API knowledge base and dynamically selected prompt cards when choosing scale, physics, cameras, VFX, behaviors, game rules, and scene structure. - Build complete playable changes, not static mockups. When a request implies gameplay, set a project title, attach/configure behaviors, physics, camera, game settings, triggers, feedback, and any needed custom controller behavior in the same script. +- For full games, first produce a designBrief with coreLoop, controlsCamera, goalsFailState, challengeCurve, feedbackProgression, reusePlan, and implementationStrategy. Then execute a compact playable MVP in phases. - Prefer existing built-in behavior components and behavior IDs from the available behavior registry before writing custom behavior code. - Use the available lambda registry when debugging or extending ECS-style runtime systems. Query lambda metadata with lambda list/lambda get before assuming schema. - Query imported scene assets before referencing models, behavior/lambda packs, script imports, generic files, media, VFX assets, or prefabs. Use list assets/list imports/list files/list models/list behavior packs/list lambda packs and get asset/get import/get file. Use names, descriptions, tags, and formats from those results to decide which existing asset can be reused. - Prefer commands that can execute immediately in the browser. If the user asks for local file imports, explain the exact import StemScript they can run in the terminal instead of emitting direct import commands here. +- External prompt-to-image/model/audio generation requires user approval. If generated assets are essential, set assetRequests and ask before any mutation. If optional, build a playable primitive/code fallback first and mention the optional upgrade after execution. - Return only JSON with this exact shape: - {"reply":"short user-facing summary","inspectionStemscript":"optional read-only query commands","stemscript":"multi-line mutation commands","notes":["optional note"]} + {"reply":"short user-facing summary","designBrief":{"title":"game title","coreLoop":"what the player repeats","controlsCamera":"input and camera plan","goalsFailState":"win/lose/reset plan","challengeCurve":"how difficulty ramps","feedbackProgression":"HUD, VFX, scoring, unlocks","reusePlan":"built-ins/assets/lambdas/imports reused","implementationStrategy":"phase and code strategy"},"assetRequests":[{"type":"image|model|audio","name":"optional asset name","prompt":"generation prompt","essential":true,"reason":"why primitives cannot satisfy this","fallback":"primitive/code fallback if optional"}],"inspectionStemscript":"optional read-only query commands","stemscript":"multi-line mutation commands","phases":[{"name":"optional phase name","goal":"optional goal","inspectionStemscript":"optional read-only phase query commands","artifacts":[{"type":"behavior|lambda|scriptImport|file","name":"ReusableName","description":"why it exists and where it attaches","code":"source code or text","content":"file text when not code","config":{"id":"machine-id","name":"Display Name","version":"1.0.0","main":"index.js","attributes":{},"componentSchema":{}},"format":"js|json|txt","contentType":"text/javascript|application/json|text/plain","metadata":{}}],"stemscript":"multi-line phase mutation commands"}],"artifacts":[{"type":"behavior|lambda|scriptImport|file","name":"ReusableName","description":"why it exists and where it attaches","code":"source code or text","content":"file text when not code","config":{},"format":"js|json|txt","contentType":"...","metadata":{}}],"notes":["optional note"]} When the user asks a question or does not want a mutation, set "stemscript" to "" and answer in "reply". When you need more scene context before editing, set "inspectionStemscript" to read-only query commands and leave "stemscript" empty. The editor will run those queries and call you again with the results. +Use phases for full games, ports, or complex repairs so environment, player/camera, mechanics, and polish can execute independently. Use artifacts for generated reusable behavior/lambda/script/file code; attach or use created artifacts in a later phase stemscript after the artifact is created. Allowed live patterns: - add group name="Arena" @@ -99,10 +115,11 @@ Allowed live patterns: Rules: - Do not use exec, export, save, require, add_model_to_scene, search_external_assets, search_local_assets, get_library_asset, or generate_3d_model. -- Do not create files, folders, bundles, YAML files, or external asset dependencies. +- Do not create local folders, bundles, YAML files, or external asset dependencies. Browser artifact objects may create behavior, lambda, scriptImport, and file assets when they are explicitly included in JSON artifacts. - Behavior code is allowed when built-ins are insufficient. Before adding or updating custom behavior code, inspect existing behavior/lambda registries or packs when relevant; if a listed asset fits, reuse it. If you add or update custom behavior code, include a description summarizing the user request, runtime purpose, inspected/reused assets, and expected attachment target, then attach it to the right scene object in the same stemscript. - Existing behavior IDs are exact and case-sensitive. Use behaviorId=character, behaviorId=trigger, etc.; do not invent behavior IDs when a listed behavior fits. - Inspection commands must be read-only: list/get objects, settings, materials, physics, lights, camera, scene settings, behavior settings/code, VFX, prefabs, lambdas, and scene assets/imports/files/models. Never put mutating commands in "inspectionStemscript". +- Prefer richer primitive compositions over single-shape placeholders: combine supported primitives, materials, VFX, lights, labels/markers, waypoints, navmesh, and runtime behaviors to communicate gameplay. - Keep most plans between 5 and 40 commands. Name important objects and group related objects. - Use "size" for primitive dimensions. Use "parent" to organize children. - For floors and walls, mark static colliders with physics commands when relevant. @@ -147,6 +164,58 @@ type InspectionCommandResult = { error?: string; }; +type StemscriptExecution = Awaited>; + +type MutationValidationFailure = { + script: string; + error: string; +}; + +type VerifiedStemscriptExecution = { + script: string; + execution: StemscriptExecution; + verification?: ScriptCheckReport; +}; + +type ArtifactExecutionSummary = { + artifact: PlaygroundPlanArtifact; + success: boolean; + message?: string; + error?: string; +}; + +type PhaseExecutionSummary = { + phase: PlaygroundPlanPhase; + label: string; + artifacts: ArtifactExecutionSummary[]; + inspections: InspectionCommandResult[]; + script: string; + validationFailure?: MutationValidationFailure; + execution?: StemscriptExecution; + verification?: ScriptCheckReport; + skippedReason?: string; +}; + +type StructuredPlanExecutionSummary = { + artifacts: ArtifactExecutionSummary[]; + phases: PhaseExecutionSummary[]; + executedCommands: number; + successCount: number; + failureCount: number; + artifactSuccessCount: number; + artifactFailureCount: number; + verificationProbes: number; + verificationPassed: number; + verificationFailed: number; +}; + +type PlanRunResult = { + message: string; + mutated: boolean; + needsRepair: boolean; + repairContext?: Record; +}; + export class DirectCopilotProvider implements ICopilotProvider { readonly isSuppressingSessionUpdates = false; @@ -291,7 +360,8 @@ export class DirectCopilotProvider implements ICopilotProvider { this.emit("agentThinking", {message: "Generating StemScript for the live scene..."}); let providerPrompt = this.buildProviderPrompt(promptText, context); - let rawPlan = await this.requestPlan(key, providerPrompt, controller.signal); + let knowledgePrompt = this.buildKnowledgePrompt(promptText, context); + let rawPlan = await this.requestPlan(key, providerPrompt, knowledgePrompt, controller.signal); let plan = parseProviderStemscriptPlan(rawPlan); const inspections: InspectionRound[] = []; @@ -308,19 +378,37 @@ export class DirectCopilotProvider implements ICopilotProvider { inspections, previousPlan: plan, }); - rawPlan = await this.requestPlan(key, providerPrompt, controller.signal); + knowledgePrompt = this.buildKnowledgePrompt(promptText, context, { + inspections, + previousPlan: plan, + }); + rawPlan = await this.requestPlan(key, providerPrompt, knowledgePrompt, controller.signal); plan = parseProviderStemscriptPlan(rawPlan); } - let finalMessage = plan.reply || "Done."; + let runResult = await this.runPlan(plan, controller.signal); + let finalMessage = runResult.message; - if (plan.stemscript.trim()) { - const validated = validateGeneratedStemscript(plan.stemscript); - if (validated.executableCommands > 0) { - this.emit("toolCall", {toolCall: {title: "Apply StemScript commands"}}); - const execution = await this.executeStemscript(validated.script, controller.signal); - finalMessage = this.formatExecutionSummary(finalMessage, validated.script, execution); - } + if (runResult.needsRepair) { + this.emit("agentThinking", {message: "Repairing generated scene changes..."}); + const repairPrompt = this.buildRepairPrompt(promptText, context, plan, runResult.repairContext); + const repairKnowledge = this.buildKnowledgePrompt(promptText, { + ...context, + repair: runResult.repairContext, + }); + const repairRawPlan = await this.requestPlan(key, repairPrompt, repairKnowledge, controller.signal); + const repairPlan = parseProviderStemscriptPlan(repairRawPlan); + const repairResult = await this.runPlan(repairPlan, controller.signal, {isRepair: true}); + finalMessage = this.formatRepairSummary(finalMessage, repairResult.message); + runResult = { + ...repairResult, + mutated: runResult.mutated || repairResult.mutated, + needsRepair: false, + }; + } + + if (runResult.mutated) { + finalMessage = this.appendSatisfactionPrompt(finalMessage); } this.emit("agentMessage", {message: finalMessage, replayStartNewMessage: true}); @@ -370,6 +458,8 @@ export class DirectCopilotProvider implements ICopilotProvider { inspectionContext ? JSON.stringify({ inspectionStemscript: inspectionContext.previousPlan.inspectionStemscript, reply: inspectionContext.previousPlan.reply, + designBrief: inspectionContext.previousPlan.designBrief, + assetRequests: inspectionContext.previousPlan.assetRequests, notes: inspectionContext.previousPlan.notes, }, null, 2) : "", inspectionContext ? "" : "", @@ -388,20 +478,699 @@ export class DirectCopilotProvider implements ICopilotProvider { ].filter(part => part !== "").join("\n"); } + private buildKnowledgePrompt( + promptText: string, + context: Record, + inspectionContext?: {inspections: InspectionRound[]; previousPlan: PlaygroundStemscriptPlan}, + ): string { + return selectPlaygroundKnowledgeCards({ + promptText, + context, + inspectionText: inspectionContext ? safeJsonStringify({ + inspections: inspectionContext.inspections, + previousPlan: { + reply: inspectionContext.previousPlan.reply, + designBrief: inspectionContext.previousPlan.designBrief, + assetRequests: inspectionContext.previousPlan.assetRequests, + notes: inspectionContext.previousPlan.notes, + phases: inspectionContext.previousPlan.phases.map(phase => ({ + name: phase.name, + goal: phase.goal, + artifacts: phase.artifacts.map(artifact => ({ + type: artifact.type, + name: artifact.name, + description: artifact.description, + })), + })), + }, + }) : undefined, + }).prompt; + } + private async requestPlan( key: CopilotChatKey, prompt: string, + knowledgePrompt: string, signal: AbortSignal, ): Promise { - return this.llmClient.generateText({ - key, - prompt, - signal, - systemPrompt: SYSTEM_PROMPT, - knowledgePrompt: PLAYGROUND_STEMSCRIPT_KNOWLEDGE, - promptCacheKey: PLAYGROUND_PROMPT_CACHE_KEY, - maxOutputTokens: PLAYGROUND_MAX_OUTPUT_TOKENS, + const streamReporter = this.createStreamProgressReporter(key); + try { + const text = await this.llmClient.generateText({ + key, + prompt, + signal, + systemPrompt: SYSTEM_PROMPT, + knowledgePrompt, + promptCacheKey: PLAYGROUND_PROMPT_CACHE_KEY, + maxOutputTokens: getPlaygroundMaxOutputTokens(key), + onStreamProgress: streamReporter?.onProgress, + }); + streamReporter?.finish(); + return text; + } catch (error) { + streamReporter?.finish({failed: true}); + throw error; + } + } + + private createStreamProgressReporter(key: CopilotChatKey): { + onProgress: (progress: PlaygroundLLMStreamProgress) => void; + finish: (options?: {failed?: boolean}) => void; + } | null { + if (key.provider !== "openai") return null; + + let textChars = 0; + let rawTextChars = 0; + let reasoningChars = 0; + let rawEvents = 0; + let lastEmitAt = 0; + let lastTextChars = 0; + let emitted = false; + + this.emit("toolCall", {toolCall: {title: "Stream OpenAI response"}}); + + const effectiveTextChars = () => Math.max(textChars, rawTextChars); + const describe = () => [ + `text=${formatCount(effectiveTextChars())} chars`, + rawTextChars > 0 && textChars === 0 ? "from raw event stream" : "", + reasoningChars > 0 ? `reasoning=${formatCount(reasoningChars)} chars` : "", + rawEvents > 0 ? `events=${rawEvents}` : "", + ].filter(Boolean).join(", "); + + const emitProgress = (force = false) => { + const now = Date.now(); + const currentTextChars = effectiveTextChars(); + if (!force && emitted && now - lastEmitAt < 1500 && currentTextChars - lastTextChars < 2048) { + return; + } + + emitted = true; + lastEmitAt = now; + lastTextChars = currentTextChars; + this.emit("toolCallUpdate", { + line: `OpenAI stream active (${describe() || "waiting for first chunk"})`, + }); + }; + + return { + onProgress: progress => { + const textCharsBefore = effectiveTextChars(); + if (progress.type === "text") { + if (progress.source === "openai-raw") { + rawTextChars += progress.delta.length; + } else { + textChars += progress.delta.length; + } + } else if (progress.type === "reasoning") { + reasoningChars += progress.delta.length; + } else { + rawEvents += 1; + } + const hasFirstText = textCharsBefore === 0 && effectiveTextChars() > 0; + emitProgress(hasFirstText || (rawEvents === 1 && effectiveTextChars() === 0 && reasoningChars === 0)); + }, + finish: options => { + if (!emitted && effectiveTextChars() === 0 && reasoningChars === 0 && rawEvents === 0) return; + this.emit("toolCallUpdate", { + line: options?.failed + ? `OpenAI stream ended before a usable plan was received (${describe() || "no chunks"})` + : `OpenAI stream complete (${describe()}); parsing plan`, + }); + }, + }; + } + + private async runPlan( + plan: PlaygroundStemscriptPlan, + signal: AbortSignal, + options: {isRepair?: boolean} = {}, + ): Promise { + const reply = plan.reply || (options.isRepair ? "Repair attempted." : "Done."); + + if (this.hasEssentialAssetRequests(plan)) { + return { + message: this.formatAssetRequestMessage(reply, plan), + mutated: false, + needsRepair: false, + }; + } + + if (this.hasStructuredPlan(plan)) { + const execution = await this.executeStructuredPlan(plan, signal); + const message = this.formatStructuredPlanSummary(reply, execution, plan); + return { + message, + mutated: this.structuredPlanMutated(execution), + needsRepair: this.structuredPlanNeedsRepair(execution), + repairContext: this.buildStructuredRepairContext(plan, execution), + }; + } + + if (plan.stemscript.trim()) { + const validation = this.validateMutationStemscript(plan.stemscript); + if (validation.failure) { + const message = this.formatValidationFailureSummary(reply, validation.failure); + return { + message, + mutated: false, + needsRepair: true, + repairContext: { + kind: "validation", + failure: validation.failure, + plan: this.compactPlanForPrompt(plan), + }, + }; + } + + const validated = validation.validated; + if (!validated || validated.executableCommands <= 0) { + const failure: MutationValidationFailure = { + script: plan.stemscript, + error: "Generated StemScript was non-empty but contained no executable commands.", + }; + return { + message: this.formatValidationFailureSummary(reply, failure), + mutated: false, + needsRepair: true, + repairContext: { + kind: "empty-generation", + failure, + plan: this.compactPlanForPrompt(plan), + }, + }; + } + + this.emit("toolCall", {toolCall: {title: options.isRepair ? "Apply repair StemScript commands" : "Apply StemScript commands"}}); + const execution = await this.executeAndVerifyStemscript(validated.script, signal); + const message = this.formatExecutionSummary(reply, execution.script, execution.execution, execution.verification, plan); + return { + message, + mutated: execution.execution.executedCommands > 0, + needsRepair: execution.execution.failCount > 0 || (execution.verification?.failed ?? 0) > 0, + repairContext: this.buildSimpleRepairContext(plan, execution), + }; + } + + return { + message: this.formatNoMutationSummary(reply, plan), + mutated: false, + needsRepair: false, + }; + } + + private hasStructuredPlan(plan: PlaygroundStemscriptPlan): boolean { + return plan.artifacts.length > 0 || plan.phases.length > 0; + } + + private hasEssentialAssetRequests(plan: PlaygroundStemscriptPlan): boolean { + return plan.assetRequests.some(request => request.essential === true); + } + + private validateMutationStemscript(script: string): { + validated?: ReturnType; + failure?: MutationValidationFailure; + } { + try { + return {validated: validateGeneratedStemscript(script)}; + } catch (error) { + return { + failure: { + script, + error: error instanceof Error ? error.message : String(error), + }, + }; + } + } + + private structuredPlanMutated(execution: StructuredPlanExecutionSummary): boolean { + return execution.executedCommands > 0 || execution.artifactSuccessCount > 0; + } + + private structuredPlanNeedsRepair(execution: StructuredPlanExecutionSummary): boolean { + return ( + execution.artifactFailureCount > 0 || + execution.failureCount > 0 || + execution.verificationFailed > 0 || + execution.phases.some(phase => Boolean(phase.validationFailure)) + ); + } + + private async executeStructuredPlan( + plan: PlaygroundStemscriptPlan, + signal: AbortSignal, + ): Promise { + const summary: StructuredPlanExecutionSummary = { + artifacts: [], + phases: [], + executedCommands: 0, + successCount: 0, + failureCount: 0, + artifactSuccessCount: 0, + artifactFailureCount: 0, + verificationProbes: 0, + verificationPassed: 0, + verificationFailed: 0, + }; + + if (plan.artifacts.length > 0) { + this.emit("toolCall", {toolCall: {title: "Create reusable copilot artifacts"}}); + const artifacts = await this.materializeArtifacts(plan.artifacts, signal); + summary.artifacts.push(...artifacts); + this.countArtifacts(summary, artifacts); + if (artifacts.some(result => !result.success)) return summary; + } + + const phases: PlaygroundPlanPhase[] = []; + if (plan.stemscript.trim()) { + phases.push({ + name: "StemScript", + goal: "Apply top-level mutation script", + inspectionStemscript: "", + stemscript: plan.stemscript, + artifacts: [], + }); + } + phases.push(...plan.phases); + + for (let i = 0; i < phases.length; i++) { + const phase = phases[i]!; + const label = phase.name || phase.id || phase.goal || `Phase ${i + 1}`; + this.emit("toolCall", {toolCall: {title: `Apply ${label}`}}); + const phaseSummary = await this.executePlanPhase(phase, label, signal); + summary.phases.push(phaseSummary); + this.countArtifacts(summary, phaseSummary.artifacts); + + if (phaseSummary.execution) { + summary.executedCommands += phaseSummary.execution.executedCommands; + summary.successCount += phaseSummary.execution.successCount; + summary.failureCount += phaseSummary.execution.failCount; + } + if (phaseSummary.verification) { + summary.verificationProbes += phaseSummary.verification.probes; + summary.verificationPassed += phaseSummary.verification.passed; + summary.verificationFailed += phaseSummary.verification.failed; + } + + const phaseFailed = + phaseSummary.artifacts.some(result => !result.success) || + Boolean(phaseSummary.validationFailure) || + (phaseSummary.execution?.failCount ?? 0) > 0 || + (phaseSummary.verification?.failed ?? 0) > 0; + if (phaseFailed) break; + } + + return summary; + } + + private async executePlanPhase( + phase: PlaygroundPlanPhase, + label: string, + signal: AbortSignal, + ): Promise { + const artifacts = await this.materializeArtifacts(phase.artifacts, signal); + if (artifacts.some(result => !result.success)) { + return { + phase, + label, + artifacts, + inspections: [], + script: phase.stemscript, + skippedReason: "Skipped phase commands because a required reusable artifact could not be created.", + }; + } + + let inspections: InspectionCommandResult[] = []; + if (phase.inspectionStemscript.trim()) { + const validatedInspection = validateInspectionStemscript(phase.inspectionStemscript); + if (validatedInspection.executableCommands > 0) { + inspections = await this.executeInspectionStemscript(validatedInspection.script, signal); + } + } + if (inspections.some(result => !result.success)) { + return { + phase, + label, + artifacts, + inspections, + script: phase.stemscript, + skippedReason: "Skipped phase commands because a read-only phase inspection failed.", + }; + } + + const validation = this.validateMutationStemscript(phase.stemscript); + if (validation.failure) { + return { + phase, + label, + artifacts, + inspections, + script: phase.stemscript, + validationFailure: validation.failure, + }; + } + + const validated = validation.validated; + if (!validated || validated.executableCommands <= 0) { + if (phase.stemscript.trim()) { + return { + phase, + label, + artifacts, + inspections, + script: validated?.script ?? "", + validationFailure: { + script: phase.stemscript, + error: "Generated StemScript was non-empty but contained no executable commands.", + }, + }; + } + return { + phase, + label, + artifacts, + inspections, + script: validated?.script ?? "", + }; + } + + const execution = await this.executeAndVerifyStemscript(validated.script, signal); + return { + phase, + label, + artifacts, + inspections, + script: validated.script, + execution: execution.execution, + verification: execution.verification, + }; + } + + private async materializeArtifacts( + artifacts: PlaygroundPlanArtifact[], + signal: AbortSignal, + ): Promise { + const results: ArtifactExecutionSummary[] = []; + for (let i = 0; i < artifacts.length; i++) { + if (signal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + const artifact = artifacts[i]!; + this.emit("toolCallUpdate", { + line: `${artifact.type} artifact ${artifact.name}`, + index: i, + total: artifacts.length, + }); + results.push(await this.materializeArtifact(artifact, {index: i, total: artifacts.length})); + } + return results; + } + + private async materializeArtifact( + artifact: PlaygroundPlanArtifact, + meta: CommandEventMeta = {}, + ): Promise { + try { + if (artifact.type === "behavior") { + return await this.materializeBehaviorArtifact(artifact, meta); + } + if (artifact.type === "lambda") { + return await this.materializeLambdaArtifact(artifact); + } + if (artifact.type === "scriptImport") { + return await this.materializeScriptImportArtifact(artifact); + } + if (artifact.type === "file") { + return await this.materializeFileArtifact(artifact); + } + return { + artifact, + success: false, + error: `Unsupported artifact type "${artifact.type}".`, + }; + } catch (error) { + return { + artifact, + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } + } + + private async materializeBehaviorArtifact( + artifact: PlaygroundPlanArtifact, + meta: CommandEventMeta = {}, + ): Promise { + if (!artifact.code?.trim()) { + return { + artifact, + success: false, + error: `Behavior artifact "${artifact.name}" is missing code.`, + }; + } + + const params = removeUndefined({ + name: artifact.name, + code: artifact.code, + description: artifact.description, + metadata: artifact.metadata, + version: artifact.version, + author: artifact.author, }); + const result = await this.executeRegistryCommand("add_behavior", params, meta); + return { + artifact, + success: result.success, + message: stringifyForPrompt(result.result?.message, 800), + error: result.error, + }; + } + + private async materializeLambdaArtifact(artifact: PlaygroundPlanArtifact): Promise { + const code = artifact.code?.trim(); + if (!code) { + return { + artifact, + success: false, + error: `Lambda artifact "${artifact.name}" is missing code.`, + }; + } + + const config = this.buildLambdaConfig(artifact); + const configStr = JSON.stringify(config); + const data = JSON.stringify({config: configStr, code}); + const asset = await this.createOrUpdateArtifactAsset(artifact, { + type: AssetType.Lambda, + data, + format: artifact.format || "json", + contentType: artifact.contentType || "application/json", + dependencies: await this.getScriptDependencies(code), + }); + + seedAssetRevisionData(queryClient, asset.assetId, asset.revisionId, "json", {config: configStr, code}); + this.pinAssetRevision(asset.assetId, asset.revisionId); + await updateSceneLambdaRevision({ + assetId: asset.assetId, + revisionId: asset.revisionId, + code, + configStr, + }); + + return { + artifact, + success: true, + message: `${asset.created ? "Created" : "Updated"} lambda ${artifact.name} (${asset.assetId}).`, + }; + } + + private async materializeScriptImportArtifact(artifact: PlaygroundPlanArtifact): Promise { + const code = (artifact.code ?? artifact.content)?.trim(); + if (!code) { + return { + artifact, + success: false, + error: `Script import artifact "${artifact.name}" is missing code.`, + }; + } + + const data = JSON.stringify({code}); + const asset = await this.createOrUpdateArtifactAsset(artifact, { + type: AssetType.Script, + data, + format: artifact.format || "json", + contentType: artifact.contentType || "application/json", + dependencies: await this.getScriptDependencies(code), + }); + + seedAssetRevisionData(queryClient, asset.assetId, asset.revisionId, "json", {code}); + this.pinAssetRevision(asset.assetId, asset.revisionId); + await updateSceneScriptRevision({ + assetId: asset.assetId, + revisionId: asset.revisionId, + code, + }); + + return { + artifact, + success: true, + message: `${asset.created ? "Created" : "Updated"} script import ${artifact.name} (${asset.assetId}).`, + }; + } + + private async materializeFileArtifact(artifact: PlaygroundPlanArtifact): Promise { + const content = artifact.content ?? artifact.code; + if (content === undefined) { + return { + artifact, + success: false, + error: `File artifact "${artifact.name}" is missing content.`, + }; + } + + const format = artifact.format || inferFileFormat(artifact.name); + const asset = await this.createOrUpdateArtifactAsset(artifact, { + type: AssetType.File, + data: content, + format, + contentType: artifact.contentType || contentTypeForFormat(format), + }); + + this.pinAssetRevision(asset.assetId, asset.revisionId); + + return { + artifact, + success: true, + message: `${asset.created ? "Created" : "Updated"} file ${artifact.name} (${asset.assetId}).`, + }; + } + + private async createOrUpdateArtifactAsset( + artifact: PlaygroundPlanArtifact, + params: { + type: typeof AssetType[keyof typeof AssetType]; + data: string | ArrayBuffer | Blob | ReadableStream; + format: string; + contentType: string; + dependencies?: Record; + }, + ): Promise<{assetId: string; revisionId: string; created: boolean}> { + const assetSource = global.app?.editor?.assetSource; + if (!assetSource) { + throw new Error(`${artifact.type} artifact "${artifact.name}" requires an active scene asset source.`); + } + + const existing = await this.findExistingArtifactAsset(artifact, params.type); + const options = removeUndefined({ + description: artifact.description, + metadata: artifact.metadata, + dependencies: params.dependencies && Object.keys(params.dependencies).length > 0 + ? params.dependencies + : undefined, + }) as {description?: string; metadata?: Record; dependencies?: Record}; + + if (existing) { + const parentRevisionId = existing.headRevisionId || existing.revisionId; + if (!parentRevisionId) { + throw new Error(`Existing asset "${existing.name}" has no revision to update.`); + } + const revision = await createAssetRevision({ + assetId: existing.id, + parentRevisionId, + data: params.data, + format: params.format, + contentType: params.contentType, + options, + }); + return {assetId: existing.id, revisionId: revision.id, created: false}; + } + + const asset = await createAsset({ + assetSource, + type: params.type, + name: artifact.name, + data: params.data, + format: params.format, + contentType: params.contentType, + options, + }); + return {assetId: asset.id, revisionId: asset.headRevisionId, created: true}; + } + + private async findExistingArtifactAsset( + artifact: PlaygroundPlanArtifact, + type: typeof AssetType[keyof typeof AssetType], + ): Promise { + const assetSource = global.app?.editor?.assetSource; + if (!assetSource) return undefined; + const {assets} = await assetSource.getAssets({types: [type]}); + const targetId = artifact.assetId?.trim(); + const normalizedName = artifact.name.trim().toLowerCase(); + return assets.find(asset => + (targetId && asset.id === targetId) || + asset.name === artifact.name || + asset.name?.trim().toLowerCase() === normalizedName); + } + + private async getScriptDependencies(code: string): Promise> { + try { + const scene = global.app?.scene; + const editor = global.app?.editor; + const sceneContext = scene ? getAssetResolutionContext(scene) || undefined : undefined; + const importContext = await buildNameAwareScriptImportContext(editor?.sceneID, sceneContext); + return getScriptImportDependencyMap(code, importContext); + } catch (error) { + console.warn("[DirectCopilotProvider] Failed to resolve script dependencies for artifact:", error); + return {}; + } + } + + private pinAssetRevision(assetId: string, revisionId: string): void { + const app = global.app; + const scene = app?.scene; + if (!app || !scene) return; + setAssetRevision(scene, assetId, revisionId); + app.call("objectChanged", null, scene); + } + + private buildLambdaConfig(artifact: PlaygroundPlanArtifact): LambdaConfig { + const parsedConfig = typeof artifact.config === "string" + ? tryParseJsonObject(artifact.config) + : artifact.config; + const configRecord = parsedConfig && typeof parsedConfig === "object" && !Array.isArray(parsedConfig) + ? parsedConfig as Partial + : {}; + const metadataConfig = artifact.metadata?.config; + const metadataRecord = metadataConfig && typeof metadataConfig === "object" && !Array.isArray(metadataConfig) + ? metadataConfig as Partial + : {}; + const merged = {...metadataRecord, ...configRecord}; + + return { + id: typeof merged.id === "string" && merged.id.trim() ? merged.id.trim() : slugifyArtifactName(artifact.name), + name: typeof merged.name === "string" && merged.name.trim() ? merged.name.trim() : artifact.name, + description: typeof merged.description === "string" ? merged.description : artifact.description || "", + author: typeof merged.author === "string" ? merged.author : artifact.author || "", + version: typeof merged.version === "string" && merged.version.trim() ? merged.version.trim() : artifact.version || "1.0.0", + main: typeof merged.main === "string" && merged.main.trim() ? merged.main.trim() : "index.js", + tags: Array.isArray(merged.tags) ? merged.tags.filter((tag): tag is string => typeof tag === "string") : [], + attributes: isRecord(merged.attributes) ? merged.attributes as LambdaConfig["attributes"] : {}, + componentSchema: isRecord(merged.componentSchema) ? merged.componentSchema as LambdaConfig["componentSchema"] : {}, + ...(typeof merged.isCritical === "boolean" ? {isCritical: merged.isCritical} : {}), + ...(Array.isArray(merged.readComponents) ? {readComponents: merged.readComponents.filter((item): item is string => typeof item === "string")} : {}), + ...(Array.isArray(merged.writeComponents) ? {writeComponents: merged.writeComponents.filter((item): item is string => typeof item === "string")} : {}), + }; + } + + private countArtifacts( + summary: StructuredPlanExecutionSummary, + artifacts: ArtifactExecutionSummary[], + ): void { + for (const artifact of artifacts) { + if (artifact.success) { + summary.artifactSuccessCount++; + } else { + summary.artifactFailureCount++; + } + } } private async executeInspectionStemscript( @@ -476,15 +1245,52 @@ export class DirectCopilotProvider implements ICopilotProvider { ); } + private async executeAndVerifyStemscript( + script: string, + signal: AbortSignal, + ): Promise { + const execution = await this.executeStemscript(script, signal); + const verification = await this.verifyStemscript(script, signal); + return {script, execution, verification}; + } + + private async verifyStemscript( + script: string, + signal: AbortSignal, + ): Promise { + this.emit("toolCall", {toolCall: {title: "Verify scene updates"}}); + return runScriptCheck(script, async (command, params) => { + if (signal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + const result = await this.executeRegistryCommand(command, params); + return { + success: result.success, + data: result.result?.data, + message: stringifyForPrompt(result.result?.message, 800), + error: result.error || (!result.success ? stringifyForPrompt(result.result?.message, 800) : undefined), + }; + }); + } + private async executeRegistryCommand( command: string, params: Record, meta: CommandEventMeta = {}, ): Promise { this.emit("commandWillExecute", {command, parameters: params, ...meta}); - const result = await this.getExecutor().executeCommand(command, params); - if (result.success) { - this.emit("commandExecuted", {command, parameters: params, result: result.result, ...meta}); + const rawResult = await this.getExecutor().executeCommand(command, params); + const commandStatus = rawResult.result?.status; + const success = rawResult.success && commandStatus !== "failed" && commandStatus !== "error"; + const result: CommandExecutionResult = success + ? rawResult + : { + ...rawResult, + success: false, + error: rawResult.error || stringifyForPrompt(rawResult.result?.message, 800) || "Command failed", + }; + if (success) { + this.emit("commandExecuted", {command, parameters: params, result: rawResult.result, ...meta}); } else { this.emit("commandExecutionFailed", {command, parameters: params, error: result.error, ...meta}); } @@ -495,10 +1301,13 @@ export class DirectCopilotProvider implements ICopilotProvider { reply: string, script: string, execution: Awaited>, + verification?: ScriptCheckReport, + plan?: PlaygroundStemscriptPlan, ): string { const failures = execution.results.filter(result => !result.success); const lines = [ reply, + ...this.formatDesignBriefLines(plan), "", "```stemscript", script, @@ -506,6 +1315,8 @@ export class DirectCopilotProvider implements ICopilotProvider { "", `Applied ${execution.successCount}/${execution.executedCommands} command(s).`, ]; + this.appendVerificationLines(lines, verification); + this.appendOptionalAssetRequestLines(lines, plan); if (failures.length > 0) { lines.push(""); @@ -518,6 +1329,284 @@ export class DirectCopilotProvider implements ICopilotProvider { return lines.join("\n").trim(); } + private formatValidationFailureSummary(reply: string, failure: MutationValidationFailure): string { + return [ + reply, + "", + "The generated StemScript could not be applied.", + `Validation: ${failure.error}`, + ].join("\n").trim(); + } + + private formatNoMutationSummary(reply: string, plan: PlaygroundStemscriptPlan): string { + const lines = [reply, ...this.formatDesignBriefLines(plan)]; + this.appendOptionalAssetRequestLines(lines, plan); + return lines.join("\n").trim(); + } + + private formatAssetRequestMessage(reply: string, plan: PlaygroundStemscriptPlan): string { + const lines = [reply || "I need approval before generating external assets.", ...this.formatDesignBriefLines(plan)]; + const essentialRequests = plan.assetRequests.filter(request => request.essential === true); + if (essentialRequests.length > 0) { + lines.push(""); + lines.push("Asset generation approval needed:"); + for (const request of essentialRequests.slice(0, 5)) { + lines.push(`- ${request.name || request.type || "Asset"}: ${request.reason || request.prompt || "required for this request"}`); + } + } + lines.push(""); + lines.push("Should I generate these assets, or build a primitive/code fallback instead?"); + return lines.join("\n").trim(); + } + + private formatRepairSummary(initialMessage: string, repairMessage: string): string { + return [ + initialMessage, + "", + "Repair pass:", + repairMessage, + ].join("\n").trim(); + } + + private appendSatisfactionPrompt(message: string): string { + return [ + message, + "", + "Are you satisfied with this, or what would you like changed next?", + ].join("\n").trim(); + } + + private appendVerificationLines(lines: string[], verification?: ScriptCheckReport): void { + if (!verification) return; + lines.push(`Verified ${verification.passed}/${verification.probes} readback probe(s).`); + if (verification.failed > 0) { + const failed = verification.results.filter(result => !result.success); + lines.push(""); + lines.push("Verification failed:"); + for (const result of failed.slice(0, 5)) { + const mismatch = result.mismatches[0]; + lines.push(`- Line ${mismatch?.lineNumber ?? result.probe.lineNumber}: ${mismatch?.path || "(getter)"} ${mismatch?.reason || "readback mismatch"}`); + } + } + } + + private appendOptionalAssetRequestLines(lines: string[], plan?: PlaygroundStemscriptPlan): void { + const optionalRequests = plan?.assetRequests.filter(request => request.essential !== true) ?? []; + if (optionalRequests.length === 0) return; + lines.push(""); + lines.push("Optional asset upgrade available:"); + for (const request of optionalRequests.slice(0, 3)) { + lines.push(`- ${request.name || request.type || "Asset"}: ${request.reason || request.prompt || "generate a richer asset"}`); + } + } + + private formatDesignBriefLines(plan?: PlaygroundStemscriptPlan): string[] { + const brief = plan?.designBrief; + if (!brief) return []; + const parts = [ + brief.coreLoop ? `core loop: ${brief.coreLoop}` : "", + brief.controlsCamera ? `controls/camera: ${brief.controlsCamera}` : "", + brief.goalsFailState ? `goals/fail: ${brief.goalsFailState}` : "", + brief.challengeCurve ? `challenge: ${brief.challengeCurve}` : "", + brief.feedbackProgression ? `feedback: ${brief.feedbackProgression}` : "", + brief.reusePlan ? `reuse: ${brief.reusePlan}` : "", + brief.implementationStrategy ? `implementation: ${brief.implementationStrategy}` : "", + ].filter(Boolean); + if (parts.length === 0) return []; + return ["", `Design brief: ${parts.join("; ")}`]; + } + + private buildSimpleRepairContext( + plan: PlaygroundStemscriptPlan, + execution: VerifiedStemscriptExecution, + ): Record { + return { + kind: "stemscript", + plan: this.compactPlanForPrompt(plan), + script: execution.script, + commandFailures: execution.execution.results + .filter(result => !result.success) + .map(result => ({ + lineNumber: result.lineNumber, + command: result.command, + error: result.error, + })), + verification: this.compactVerificationForPrompt(execution.verification), + }; + } + + private buildStructuredRepairContext( + plan: PlaygroundStemscriptPlan, + execution: StructuredPlanExecutionSummary, + ): Record { + return { + kind: "structured", + plan: this.compactPlanForPrompt(plan), + artifacts: [...execution.artifacts, ...execution.phases.flatMap(phase => phase.artifacts)] + .filter(result => !result.success) + .map(result => ({ + type: result.artifact.type, + name: result.artifact.name, + error: result.error, + })), + phases: execution.phases.map(phase => ({ + label: phase.label, + skippedReason: phase.skippedReason, + validationFailure: phase.validationFailure, + commandFailures: phase.execution?.results + .filter(result => !result.success) + .map(result => ({ + lineNumber: result.lineNumber, + command: result.command, + error: result.error, + })) ?? [], + verification: this.compactVerificationForPrompt(phase.verification), + })), + }; + } + + private compactVerificationForPrompt(verification?: ScriptCheckReport): Record | undefined { + if (!verification) return undefined; + return { + probes: verification.probes, + passed: verification.passed, + failed: verification.failed, + mismatches: verification.results + .filter(result => !result.success) + .flatMap(result => result.mismatches) + .slice(0, 12), + skipped: verification.skipped.slice(0, 12), + }; + } + + private compactPlanForPrompt(plan: PlaygroundStemscriptPlan): Record { + return { + reply: plan.reply, + designBrief: plan.designBrief, + assetRequests: plan.assetRequests, + inspectionStemscript: plan.inspectionStemscript, + stemscript: plan.stemscript, + notes: plan.notes, + artifacts: plan.artifacts.map(artifact => ({ + type: artifact.type, + name: artifact.name, + description: artifact.description, + })), + phases: plan.phases.map(phase => ({ + name: phase.name, + goal: phase.goal, + inspectionStemscript: phase.inspectionStemscript, + stemscript: phase.stemscript, + artifacts: phase.artifacts.map(artifact => ({ + type: artifact.type, + name: artifact.name, + description: artifact.description, + })), + })), + }; + } + + private buildRepairPrompt( + promptText: string, + context: Record, + previousPlan: PlaygroundStemscriptPlan, + repairContext: Record | undefined, + ): string { + return [ + this.buildProviderPrompt(promptText, context), + "", + "Repair pass required.", + "Return JSON only. Do not request another inspectionStemscript. Produce the smallest repair artifacts/stemscript needed to fix the failed lines or readback mismatches. Stop after this repair.", + "", + "Previous plan JSON:", + safeJsonStringify(this.compactPlanForPrompt(previousPlan)), + "", + "Failure and verification context JSON:", + safeJsonStringify(repairContext ?? {}), + ].join("\n"); + } + + private formatStructuredPlanSummary( + reply: string, + execution: StructuredPlanExecutionSummary, + plan?: PlaygroundStemscriptPlan, + ): string { + const lines = [reply, ...this.formatDesignBriefLines(plan)]; + const artifactTotal = execution.artifactSuccessCount + execution.artifactFailureCount; + const phasesWithCommands = execution.phases.filter(phase => (phase.execution?.executedCommands ?? 0) > 0); + + if (artifactTotal > 0) { + lines.push(""); + lines.push(`Materialized ${execution.artifactSuccessCount}/${artifactTotal} reusable artifact(s).`); + } + + if (execution.executedCommands > 0) { + lines.push(`Applied ${execution.successCount}/${execution.executedCommands} command(s) across ${phasesWithCommands.length} phase(s).`); + } + if (execution.verificationProbes > 0) { + lines.push(`Verified ${execution.verificationPassed}/${execution.verificationProbes} readback probe(s).`); + } + this.appendOptionalAssetRequestLines(lines, plan); + + if (execution.phases.length > 0) { + lines.push(""); + lines.push("Phase results:"); + for (const phase of execution.phases) { + const commandText = phase.execution + ? `${phase.execution.successCount}/${phase.execution.executedCommands} command(s)` + : "no mutation commands"; + const artifactText = phase.artifacts.length > 0 + ? `, artifacts ${phase.artifacts.filter(result => result.success).length}/${phase.artifacts.length}` + : ""; + const verificationText = phase.verification && phase.verification.probes > 0 + ? `, verified ${phase.verification.passed}/${phase.verification.probes}` + : ""; + lines.push(`- ${phase.label}: ${commandText}${artifactText}${verificationText}`); + if (phase.skippedReason) { + lines.push(` ${phase.skippedReason}`); + } + if (phase.validationFailure) { + lines.push(` Validation failed: ${phase.validationFailure.error}`); + } + } + } + + const artifactFailures = [...execution.artifacts, ...execution.phases.flatMap(phase => phase.artifacts)] + .filter(result => !result.success); + const commandFailures = execution.phases + .flatMap(phase => phase.execution?.results ?? []) + .filter(result => !result.success); + const inspectionFailures = execution.phases + .flatMap(phase => phase.inspections) + .filter(result => !result.success); + const verificationFailures = execution.phases + .flatMap(phase => phase.verification?.results ?? []) + .filter(result => !result.success); + + if (artifactFailures.length > 0 || commandFailures.length > 0 || inspectionFailures.length > 0 || verificationFailures.length > 0) { + lines.push(""); + lines.push("Some steps failed:"); + for (const failure of artifactFailures.slice(0, 5)) { + lines.push(`- ${failure.artifact.type} ${failure.artifact.name}: ${failure.error || "Unknown error"}`); + } + const remainingSlotsAfterArtifacts = Math.max(0, 5 - artifactFailures.length); + for (const failure of inspectionFailures.slice(0, remainingSlotsAfterArtifacts)) { + lines.push(`- Inspection line ${failure.lineNumber}: ${failure.error || "Unknown error"}`); + } + const remainingSlots = Math.max(0, remainingSlotsAfterArtifacts - inspectionFailures.length); + for (const failure of commandFailures.slice(0, remainingSlots)) { + lines.push(`- Line ${failure.lineNumber}: ${failure.error || "Unknown error"}`); + } + const verificationSlots = Math.max(0, 5 - artifactFailures.length - inspectionFailures.length - commandFailures.length); + for (const failure of verificationFailures.slice(0, verificationSlots)) { + const mismatch = failure.mismatches[0]; + lines.push(`- Verification line ${mismatch?.lineNumber ?? failure.probe.lineNumber}: ${mismatch?.reason || "readback mismatch"}`); + } + } + + return lines.join("\n").trim(); + } + async executeCommand(method: string, params: Record): Promise { return this.executeRegistryCommand(method, params); } @@ -549,6 +1638,11 @@ function stringifyForPrompt(value: unknown, maxChars: number): string | undefine return text.length > maxChars ? `${text.slice(0, maxChars - 24)}... [truncated ${text.length} chars]` : text; } +function formatCount(value: number): string { + if (value < 1000) return String(value); + return `${(value / 1000).toFixed(value < 10_000 ? 1 : 0)}k`; +} + function compactForPrompt(value: unknown, maxChars = 6000): unknown { if (value === undefined || value === null) return undefined; const text = safeJsonStringify(value); @@ -556,6 +1650,62 @@ function compactForPrompt(value: unknown, maxChars = 6000): unknown { return `${text.slice(0, maxChars - 24)}... [truncated ${text.length} chars]`; } +function removeUndefined(value: Record): Record { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function tryParseJsonObject(value: string): unknown | null { + try { + return JSON.parse(value); + } catch { + return null; + } +} + +function slugifyArtifactName(value: string): string { + const slug = value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + return slug || "copilot-artifact"; +} + +function inferFileFormat(name: string): string { + const ext = name.trim().split(".").pop()?.toLowerCase(); + return ext && ext !== name.toLowerCase() ? ext : "txt"; +} + +function contentTypeForFormat(format: string): string { + const normalized = format.toLowerCase(); + const map: Record = { + cjs: "text/javascript", + css: "text/css", + frag: "text/plain", + glsl: "text/plain", + html: "text/html", + js: "text/javascript", + json: "application/json", + jsx: "text/javascript", + md: "text/markdown", + mjs: "text/javascript", + sh: "text/x-shellscript", + svg: "image/svg+xml", + ts: "text/typescript", + tsx: "text/typescript", + txt: "text/plain", + vert: "text/plain", + xml: "application/xml", + yaml: "text/yaml", + yml: "text/yaml", + }; + return map[normalized] || "text/plain"; +} + function safeJsonStringify(value: unknown): string { try { return JSON.stringify(value); diff --git a/client/packages/editor-oss/src/copilot/index.ts b/client/packages/editor-oss/src/copilot/index.ts index 4cb218f0..251453be 100644 --- a/client/packages/editor-oss/src/copilot/index.ts +++ b/client/packages/editor-oss/src/copilot/index.ts @@ -11,7 +11,9 @@ export { COPILOT_DEFAULT_MODELS, COPILOT_KEYS_CHANGED_EVENT, COPILOT_MODEL_OPTIONS, + clearCopilotChatKeyHandoff, hasCopilotKeysSync, + prepareCopilotChatKeyHandoff, refreshCopilotKeysMarker, resolveCopilotChatKey, resolveCopilotChatKeyChoice, diff --git a/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.test.ts b/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.test.ts index 7d9f474c..f0f4e192 100644 --- a/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.test.ts +++ b/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.test.ts @@ -11,8 +11,10 @@ vi.mock("../ai", () => ({ })); import { + clearCopilotChatKeyHandoff, getCopilotModelSelectionSync, hasCopilotKeysSync, + prepareCopilotChatKeyHandoff, refreshCopilotKeysMarker, resolveCopilotChatKeyChoice, resolveCopilotChatKeys, @@ -22,7 +24,9 @@ import { describe("playgroundCopilotKeys", () => { beforeEach(() => { vi.clearAllMocks(); + clearCopilotChatKeyHandoff(); window.localStorage.clear(); + window.sessionStorage.clear(); }); it("resolves a single chat key with the provider default model", async () => { @@ -31,7 +35,7 @@ describe("playgroundCopilotKeys", () => { const keys = await resolveCopilotChatKeys(); const choice = await resolveCopilotChatKeyChoice(); - expect(keys).toEqual([{provider: "openai", apiKey: "sk-openai", model: "gpt-5.2-codex"}]); + expect(keys).toEqual([{provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}]); expect(choice).toEqual({kind: "ready", key: keys[0], keys}); }); @@ -57,10 +61,10 @@ describe("playgroundCopilotKeys", () => { const choice = await resolveCopilotChatKeyChoice(); - expect(getCopilotModelSelectionSync()).toEqual({provider: "openai", model: "gpt-5.1-codex"}); + expect(getCopilotModelSelectionSync()).toEqual({provider: "openai", model: "gpt-5.5"}); expect(choice.kind).toBe("ready"); if (choice.kind === "ready") { - expect(choice.key).toEqual({provider: "openai", apiKey: "sk-openai", model: "gpt-5.1-codex"}); + expect(choice.key).toEqual({provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}); } }); @@ -87,4 +91,30 @@ describe("playgroundCopilotKeys", () => { expect(ready).toBe(false); expect(hasCopilotKeysSync()).toBe(false); }); + + it("uses a prepared route handoff when the encrypted store is locked after navigation", async () => { + const handoffStorageKey = "stem.playground.copilot.routeKeyHandoff"; + mocks.store.all.mockResolvedValueOnce({openai: "sk-openai"}); + + const prepared = await prepareCopilotChatKeyHandoff(); + + expect(prepared).toBe(true); + expect(window.sessionStorage.getItem(handoffStorageKey)).toContain("sk-openai"); + + mocks.store.all.mockResolvedValueOnce({}); + const ready = await refreshCopilotKeysMarker(); + + expect(ready).toBe(true); + expect(hasCopilotKeysSync()).toBe(true); + expect(window.sessionStorage.getItem(handoffStorageKey)).toContain("sk-openai"); + + mocks.store.all.mockResolvedValueOnce({}); + const choice = await resolveCopilotChatKeyChoice(); + + expect(choice.kind).toBe("ready"); + if (choice.kind === "ready") { + expect(choice.key).toEqual({provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}); + } + expect(window.sessionStorage.getItem(handoffStorageKey)).toBeNull(); + }); }); diff --git a/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.ts b/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.ts index 739ef2ec..93c84c66 100644 --- a/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.ts +++ b/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.ts @@ -10,7 +10,9 @@ // marker instead. The marker is refreshed asynchronously at bootstrap and // whenever the BYOK panel saves/clears a key. // 2. `resolveCopilotChatKey()` — the actual decrypted key + provider used to -// make requests, read from the BYOK key store on demand. +// make requests, read from the BYOK key store on demand. Prompt-created +// projects use a one-navigation session handoff because `openEditorRoute` +// reloads the app and loses the encrypted store's in-memory unlock. import {getBYOKKeyStore} from "../ai"; import type {AIProvider} from "../ai"; @@ -29,11 +31,15 @@ export const CHAT_PROVIDERS: ReadonlyArray = [ const COPILOT_READY_MARKER = "stem.playground.copilotReady"; const COPILOT_SELECTED_PROVIDER = "stem.playground.copilot.selectedProvider"; +const COPILOT_ROUTE_KEY_HANDOFF = "stem.playground.copilot.routeKeyHandoff"; +const COPILOT_ROUTE_KEY_HANDOFF_TTL_MS = 15 * 60 * 1000; export const COPILOT_KEYS_CHANGED_EVENT = "stem:playground-copilot-keys-changed"; +export const OPENAI_COPILOT_MODEL = "gpt-5.5"; +export const OPENAI_COPILOT_REASONING_EFFORT = "high"; export const COPILOT_DEFAULT_MODELS: Record = { anthropic: "claude-sonnet-4-5-20250929", - openai: "gpt-5.2-codex", + openai: OPENAI_COPILOT_MODEL, gemini: "gemini-2.5-flash", }; @@ -45,10 +51,7 @@ export const COPILOT_MODEL_OPTIONS: Record; + if (record.version !== 1) return null; + if (!isCopilotChatProvider(record.provider)) return null; + if (typeof record.apiKey !== "string" || !record.apiKey.trim()) return null; + if (typeof record.model !== "string" || !record.model.trim()) return null; + if (typeof record.expiresAt !== "number" || record.expiresAt <= Date.now()) return null; + return { + provider: record.provider, + apiKey: record.apiKey.trim(), + model: normalizeProviderModel(record.provider, record.model), + }; +} + +function readRouteHandoffKey(consume = true): CopilotChatKey | null { + if (activeRouteHandoffKey) return activeRouteHandoffKey; + + const storage = getSessionStorage(); + if (!storage) return null; + + try { + const raw = storage.getItem(COPILOT_ROUTE_KEY_HANDOFF); + if (!raw) return null; + + const handoffKey = normalizeRouteHandoff(JSON.parse(raw)); + if (!handoffKey) { + storage.removeItem(COPILOT_ROUTE_KEY_HANDOFF); + return null; + } + + if (consume) { + // Consume the plaintext handoff into process memory. It should only + // bridge the dashboard -> editor reload, not remain in storage. + storage.removeItem(COPILOT_ROUTE_KEY_HANDOFF); + activeRouteHandoffKey = handoffKey; + } + return handoffKey; + } catch { + try { + storage.removeItem(COPILOT_ROUTE_KEY_HANDOFF); + } catch { + // Ignore storage failures. + } + return null; + } +} + +function writeRouteHandoffKey(key: CopilotChatKey): boolean { + activeRouteHandoffKey = null; + const storage = getSessionStorage(); + if (!storage) { + activeRouteHandoffKey = key; + return false; + } + + try { + const payload: CopilotRouteKeyHandoff = { + ...key, + expiresAt: Date.now() + COPILOT_ROUTE_KEY_HANDOFF_TTL_MS, + version: 1, + }; + storage.setItem(COPILOT_ROUTE_KEY_HANDOFF, JSON.stringify(payload)); + return true; + } catch { + activeRouteHandoffKey = key; + return false; + } +} + +export function clearCopilotChatKeyHandoff(): void { + activeRouteHandoffKey = null; + const storage = getSessionStorage(); + if (!storage) return; + try { + storage.removeItem(COPILOT_ROUTE_KEY_HANDOFF); + } catch { + // Ignore storage failures. + } +} + /** * Synchronous best-effort answer to "can the playground copilot run?". Reads * the localStorage marker written by `refreshCopilotKeysMarker()`. When the @@ -109,12 +208,19 @@ function modelStorageKey(provider: CopilotChatProvider): string { return `stem.playground.copilot.${provider}Model`; } +function normalizeProviderModel(provider: CopilotChatProvider, model: string | null | undefined): string { + if (provider === "openai") { + return OPENAI_COPILOT_MODEL; + } + return model?.trim() || COPILOT_DEFAULT_MODELS[provider]; +} + function readProviderModel(provider: CopilotChatProvider): string { const storage = getLocalStorage(); if (!storage) return COPILOT_DEFAULT_MODELS[provider]; try { const override = storage.getItem(modelStorageKey(provider))?.trim(); - return override || COPILOT_DEFAULT_MODELS[provider]; + return normalizeProviderModel(provider, override); } catch { return COPILOT_DEFAULT_MODELS[provider]; } @@ -143,8 +249,9 @@ export function setCopilotModelSelection(provider: CopilotChatProvider, model?: if (!storage) return; try { storage.setItem(COPILOT_SELECTED_PROVIDER, provider); - if (model?.trim()) { - storage.setItem(modelStorageKey(provider), model.trim()); + const normalizedModel = normalizeProviderModel(provider, model); + if (normalizedModel) { + storage.setItem(modelStorageKey(provider), normalizedModel); } } catch { // Ignore storage failures. @@ -156,29 +263,45 @@ export type CopilotChatKeyChoice = | {kind: "ready"; key: CopilotChatKey; keys: CopilotChatKey[]} | {kind: "needs-selection"; keys: CopilotChatKey[]}; +type ResolveCopilotChatKeysOptions = { + includeHandoff?: boolean; + consumeHandoff?: boolean; +}; + /** * Resolve every chat-capable BYOK key currently available to the direct * playground copilot. */ -export async function resolveCopilotChatKeys(): Promise { +export async function resolveCopilotChatKeys( + options: ResolveCopilotChatKeysOptions = {}, +): Promise { const store = getBYOKKeyStore(); - if (!store) return []; let keys: Partial>; - try { - keys = await store.all(); - } catch { - return []; + if (store) { + try { + keys = await store.all(); + } catch { + keys = {}; + } + } else { + keys = {}; } const available: CopilotChatKey[] = []; for (const provider of CHAT_PROVIDERS) { const apiKey = keys[provider]?.trim(); if (apiKey) available.push({provider, apiKey, model: readProviderModel(provider)}); } + if (available.length === 0 && options.includeHandoff !== false) { + const handoffKey = readRouteHandoffKey(options.consumeHandoff !== false); + if (handoffKey) available.push(handoffKey); + } return available; } -export async function resolveCopilotChatKeyChoice(): Promise { - const keys = await resolveCopilotChatKeys(); +export async function resolveCopilotChatKeyChoice( + options: ResolveCopilotChatKeysOptions = {}, +): Promise { + const keys = await resolveCopilotChatKeys(options); if (keys.length === 0) return {kind: "none", keys: []}; if (keys.length === 1) return {kind: "ready", key: keys[0]!, keys}; @@ -193,21 +316,34 @@ export async function resolveCopilotChatKeyChoice(): Promise { const choice = await resolveCopilotChatKeyChoice(); return choice.kind === "ready" ? choice.key : null; } +/** + * Stage the selected chat key for a dashboard -> editor hard navigation. This + * is intentionally short-lived and consumed into memory on the editor page. + */ +export async function prepareCopilotChatKeyHandoff(): Promise { + const choice = await resolveCopilotChatKeyChoice({includeHandoff: false}); + if (choice.kind !== "ready") return false; + writeMarker(true); + return writeRouteHandoffKey(choice.key); +} + /** * Re-read the BYOK store and update the synchronous marker. Returns whether a * chat key is currently configured. Call at bootstrap and after any key * mutation. */ export async function refreshCopilotKeysMarker(): Promise { - const ready = (await resolveCopilotChatKeys()).length > 0; + const ready = (await resolveCopilotChatKeys({consumeHandoff: false})).length > 0; + if (!ready) clearCopilotChatKeyHandoff(); writeMarker(ready); notifyKeysChanged(); return ready; diff --git a/client/packages/editor-oss/src/copilot/playgroundKnowledgeCards.test.ts b/client/packages/editor-oss/src/copilot/playgroundKnowledgeCards.test.ts new file mode 100644 index 00000000..fec6e1d9 --- /dev/null +++ b/client/packages/editor-oss/src/copilot/playgroundKnowledgeCards.test.ts @@ -0,0 +1,58 @@ +import {describe, expect, it} from "vitest"; + +import {selectPlaygroundKnowledgeCards} from "./playgroundKnowledgeCards"; + +describe("playgroundKnowledgeCards", () => { + it("keeps simple scene edits focused on always-on browser knowledge", () => { + const selection = selectPlaygroundKnowledgeCards({ + promptText: "make a red test box", + context: {}, + }); + + expect(selection.prompt).toContain("StemStudio playground knowledge base"); + expect(selection.selectedCards.map(card => card.id)).toEqual([ + "core.browser-contract", + "core.scale-and-scene", + "commands.live-patterns", + "inspection.assets-and-registries", + ]); + expect(selection.prompt).not.toContain("Racing Recipe"); + expect(selection.prompt).not.toContain("Script Imports and Shared Helpers"); + }); + + it("selects game, porting, genre, and behavior cards for full-game requests", () => { + const selection = selectPlaygroundKnowledgeCards({ + promptText: "port a kart racing game with custom vehicle controls, checkpoints, boosts, and laps", + context: {sourceKind: "paste"}, + }); + const ids = selection.selectedCards.map(card => card.id); + + expect(ids).toContain("game.full-build-flow"); + expect(ids).toContain("game.porting"); + expect(ids).toContain("genre.racing"); + expect(ids).toContain("behaviors.custom-code"); + expect(ids).toContain("behaviors.built-in-first"); + }); + + it("selects lambda and import cards only when the prompt or context asks for them", () => { + const selection = selectPlaygroundKnowledgeCards({ + promptText: "create reusable lambda schema and shared script imports for enemy wave math", + context: { + requestedArtifacts: ["lambda", "scriptImport"], + }, + }); + const ids = selection.selectedCards.map(card => card.id); + + expect(ids).toContain("lambdas.data-systems"); + expect(ids).toContain("imports.script-helpers"); + }); + + it("honors the character budget for optional cards", () => { + const selection = selectPlaygroundKnowledgeCards({ + promptText: "racing platformer shooter import lambda behavior port game", + maxChars: 1500, + }); + + expect(selection.selectedCards.every(card => card.always || ["game.full-build-flow", "game.porting", "genre.racing"].includes(card.id))).toBe(true); + }); +}); diff --git a/client/packages/editor-oss/src/copilot/playgroundKnowledgeCards.ts b/client/packages/editor-oss/src/copilot/playgroundKnowledgeCards.ts new file mode 100644 index 00000000..6c1db9d8 --- /dev/null +++ b/client/packages/editor-oss/src/copilot/playgroundKnowledgeCards.ts @@ -0,0 +1,252 @@ +export interface PlaygroundKnowledgeCard { + id: string; + title: string; + tags: string[]; + always?: boolean; + body: string; +} + +export interface PlaygroundKnowledgeSelection { + prompt: string; + selectedCards: PlaygroundKnowledgeCard[]; +} + +export interface PlaygroundKnowledgeSelectionOptions { + promptText: string; + context?: Record; + inspectionText?: string; + maxChars?: number; +} + +const DEFAULT_MAX_CHARS = 18000; + +const CARDS: PlaygroundKnowledgeCard[] = [ + { + id: "core.browser-contract", + title: "Browser Direct Contract", + tags: ["core", "browser", "stemscript", "commands"], + always: true, + body: [ + "Work live in the browser editor. Use StemScript commands, behavior assets, scene settings, physics, cameras, VFX, game settings, navmesh/waypoints, and existing registries.", + "Do not use filesystem-only commands, external search/generation, bundled file creation, or server-only workflows in direct playground mode.", + "Prompt-to-image/model/audio generation is never automatic. If generated assets are essential, ask first with assetRequests. If optional, build a playable fallback and propose the upgrade after execution.", + "Keep plans compact and executable. Use inspectionStemscript for read-only queries before risky edits.", + ].join("\n"), + }, + { + id: "core.scale-and-scene", + title: "Scale, Scene, and Object Rules", + tags: ["core", "objects", "scene", "scale", "physics"], + always: true, + body: [ + "Scale: 1 unit = 1 meter. A human player is usually a capsule around size=0.5,1.8,0.5 at position=0,0.9,0.", + "Use size for primitive dimensions. Floors can be boxes such as size=50,0.1,50 at position=0,-0.05,0.", + "Name important objects and group related children. Use parent=Group during creation instead of flat scene roots.", + "Use richer primitive compositions for gameplay readability: combine primitives with materials, lights, VFX, signs/markers, gates, hazards, collectibles, and camera framing instead of leaving a static single-shape mockup.", + "Player-controlled objects must be tagged with tag=Player for character, camera, trigger, and touch systems.", + ].join("\n"), + }, + { + id: "commands.live-patterns", + title: "Live StemScript Command Patterns", + tags: ["core", "commands", "stemscript", "objects"], + always: true, + body: [ + 'Examples: add group name="Arena"; add box name="Ground" position=0,-0.05,0 size=30,0.1,30 color=#334455 parent="Arena".', + 'Use update "Object" position=x,y,z rotation=x,y,z scale=x,y,z color=#rrggbb tag=Player.', + 'Use material "Object" color=#rrggbb roughness=0.5 metalness=0.1 opacity=1.', + 'Use scene background, scene lighting, scene fog, scene tonemapping, scene postprocessing, light, render settings, camera "DefaultCamera", project title, and game settings.', + ].join("\n"), + }, + { + id: "inspection.assets-and-registries", + title: "Dynamic Asset and Registry Inspection", + tags: ["core", "inspect", "assets", "behaviors", "lambdas", "imports", "reuse"], + always: true, + body: [ + "Before referencing existing resources, inspect live project truth. Use list objects/get object, behavior list/behavior get, lambda list/lambda get, list assets/list imports/list files/list models/list behavior packs/list lambda packs, and get asset/get import/get file.", + "Use exact behaviorId/lambdaId/asset names from live inspection. Do not invent IDs.", + "Use names, descriptions, tags, formats, attributes, component schemas, and revision IDs from inspection results to choose reusable components.", + ].join("\n"), + }, + { + id: "behaviors.built-in-first", + title: "Built-In Behavior Catalog", + tags: ["behavior", "behaviors", "reuse", "game", "port", "player", "pickup", "trigger", "ai", "audio", "mobile"], + body: [ + "Prefer built-in behavior IDs before custom code. Existing IDs are exact and case-sensitive.", + "Key built-ins: character, consumable, trigger, tween, platform, enemy, projectile, npc, aiNpc, follow, jumppad, teleport, objectInteractions, enableDisable, visualEffect, genericSound, cinematicCamera, animation, dayNightCycle, touchControls, spawnpoint, navmesh, navmesh-connection.", + "Playable character recipe: add capsule Player; update Player tag=Player; behavior attach Player behaviorId=character config={isDefault:true,walkSpeed:3,runSpeed:8,jumpHeight:1.2}; camera DefaultCamera THIRD_PERSON; game settings isGame=true showHUD=true.", + ].join("\n"), + }, + { + id: "behaviors.custom-code", + title: "Custom Behavior Authoring", + tags: ["behavior", "custom", "code", "controller", "gameplay", "repair"], + body: [ + "Write custom behavior code only when built-ins cannot preserve the mechanic. Generated behavior descriptions must include request, runtime purpose, inspected/reused assets or behavior IDs, and expected attachment target.", + "Lifecycle methods: init(game), onStart(), update(deltaTime), fixedUpdate(fixedDeltaTime), onStop(), dispose(), onEvent(msg,data).", + "Use let/const, clamp deltaTime for motion integrators, guard missing objects/subsystems, and clean up listeners, timers, geometry, materials, textures, and runtime objects in dispose().", + "Expose tunable values through metadata/config attributes when possible, including debugLogs default false for generated systems.", + ].join("\n"), + }, + { + id: "lambdas.data-systems", + title: "Lambda Data Systems", + tags: ["lambda", "lambdas", "ecs", "data", "schema"], + body: [ + "Lambdas are ECS-style systems for shared object data and batched logic. Use lambda list/lambda get before assuming schema or code.", + "Use lambdas for per-object data such as health, velocity, team, inventory, tags, cooldowns, and state flags. Use behavior config for attached-system tuning.", + "Direct browser chat can inspect and design lambda schemas. It should not claim lambda asset creation unless a browser command for that asset type is available.", + ].join("\n"), + }, + { + id: "imports.script-helpers", + title: "Script Imports and Shared Helpers", + tags: ["import", "imports", "script", "helper", "reuse", "code"], + body: [ + "Script import assets are reusable JavaScript modules consumed with @import \"name\" as alias; at the top of behavior or lambda code.", + "Use imports for shared pure helpers reused across 2+ behaviors/lambdas when they reduce meaningful duplication. Helpers cannot touch this, this.erth, this.gameObject, or closed-over behavior state; pass state as arguments.", + "In browser direct mode, inspect existing imports with list imports/get import. When returning a scriptImport artifact, include source code and use it from generated behavior/lambda code after the artifact is created.", + ].join("\n"), + }, + { + id: "game.full-build-flow", + title: "Full Game Build Flow", + tags: ["game", "create", "build", "genre", "playable", "mvp"], + body: [ + "For full games, include a designBrief before StemScript: core loop, controls/camera, goals/fail state, challenge curve, feedback/progression, reuse plan, and implementation strategy.", + "Build an MVP loop, not a static mockup. Include project title, game settings, environment, player, camera, physics, win/lose/scoring rules, challenge objects, feedback, and a clear next tuning step.", + "Recommended phases: environment -> player/camera -> core objects -> mechanics -> polish. Execute and verify in small phases rather than one giant script.", + "Ask at most 1-2 questions only when missing details materially change camera/control feel, non-negotiable mechanics, or MVP boundary.", + ].join("\n"), + }, + { + id: "game.porting", + title: "Game Porting and Source Mapping", + tags: ["port", "convert", "source", "paste", "github", "mapping", "fidelity"], + body: [ + "For ports, preserve source gameplay first: entrypoint, 2D vs 3D, player object, input model, camera model, physics/collision, UI flow, audio/VFX, assets, progression, scale, and tuning values.", + "Map source systems to Stem systems using built-ins first; write custom behaviors only for source-faithful movement, advanced camera, custom UI flow, procedural systems, or mechanics with no close built-in.", + "If browser mode cannot ingest files or backend services directly, produce a playable local substitute and exact import/manual follow-up instructions.", + ].join("\n"), + }, + { + id: "genre.platformer", + title: "Platformer Recipe", + tags: ["platformer", "jump", "side-scroller", "collectible", "goal"], + body: [ + "A small platformer needs Player capsule, Ground, 3-5 platforms, hazards or gaps, collectibles, a goal trigger, physics, and SIDE_SCROLLER or THIRD_PERSON camera.", + "Use character for movement when source feel is not custom. Use consumable for pickups and trigger/tween/platform for doors, lifts, and moving hazards.", + ].join("\n"), + }, + { + id: "genre.racing", + title: "Racing Recipe", + tags: ["racing", "vehicle", "kart", "checkpoint", "lap", "boost"], + body: [ + "A racing sketch needs a kart/player body, readable track pieces, 3 checkpoints, start/finish markers, THIRD_PERSON camera, game settings, and lap/checkpoint state.", + "Built-ins cover touchControls, genericSound, triggers, and VFX. Source-faithful vehicle control usually needs a custom behavior with exposed maxSpeed, acceleration, steering, brake, boost, drag, and checkpoint tuning.", + ].join("\n"), + }, + { + id: "genre.top-down-shooter", + title: "Top-Down Shooter Recipe", + tags: ["shooter", "top-down", "arena", "enemy", "projectile", "combat"], + body: [ + "A top-down arena needs Player marker, boundaries, cover, enemies, projectiles, pickups, TOP_DOWN camera, health/score state, and feedback for fire/hit/collect events.", + "Reuse projectile, enemy, consumable, visualEffect, and genericSound where possible. Use a thin custom game controller when score, waves, and win/lose state need orchestration.", + ].join("\n"), + }, + { + id: "runtime.validation-repair", + title: "Validation and Repair", + tags: ["validate", "validation", "repair", "smoke", "errors"], + body: [ + "Never assume success. After execution, inspect changed objects and react to command failures or codeValidation payloads.", + "If a phase partially fails, stop after that phase, summarize failing lines, and produce a focused repair script rather than continuing with dependent phases.", + "Readback verification should use deterministic getters such as get project, get game settings, get object/settings/material/physics/behavior, get camera, get vfx, and get scene settings.", + "For behavior/lambda code, validation errors are blocking. Warnings about lifecycle, API use, async calls, and cleanup should be fixed unless clearly intentional.", + ].join("\n"), + }, +]; + +const ALWAYS_IDS = new Set(CARDS.filter(card => card.always).map(card => card.id)); + +export function selectPlaygroundKnowledgeCards({ + promptText, + context, + inspectionText, + maxChars = DEFAULT_MAX_CHARS, +}: PlaygroundKnowledgeSelectionOptions): PlaygroundKnowledgeSelection { + const query = normalizeQuery([ + promptText, + inspectionText, + context ? safeJsonStringify(context) : "", + ].join(" ")); + + const scored = CARDS.filter(card => !ALWAYS_IDS.has(card.id)).map(card => ({ + card, + score: scoreCard(card, query), + })).sort((a, b) => b.score - a.score || a.card.id.localeCompare(b.card.id)); + + const selected: PlaygroundKnowledgeCard[] = CARDS.filter(card => ALWAYS_IDS.has(card.id)); + let charCount = headerText().length + selected.reduce((total, card) => total + renderCard(card).length, 0); + + for (const {card, score} of scored) { + if (score <= 0) continue; + const rendered = renderCard(card); + if (charCount + rendered.length > maxChars) continue; + selected.push(card); + charCount += rendered.length; + } + + const deduped = selected.filter((card, index, array) => array.findIndex(item => item.id === card.id) === index); + + return { + selectedCards: deduped, + prompt: [ + headerText(), + `Selected cards: ${deduped.map(card => card.id).join(", ")}`, + "", + ...deduped.map(renderCard), + ].join("\n"), + }; +} + +export function getAllPlaygroundKnowledgeCards(): PlaygroundKnowledgeCard[] { + return CARDS.slice(); +} + +function headerText(): string { + return "StemStudio playground knowledge base, dynamically selected from browser-safe API, behavior, lambda, import, genre, and validation cards."; +} + +function renderCard(card: PlaygroundKnowledgeCard): string { + return [`## ${card.title} (${card.id})`, card.body, ""].join("\n"); +} + +function scoreCard(card: PlaygroundKnowledgeCard, query: string): number { + if (card.always) return Number.POSITIVE_INFINITY; + let score = 0; + for (const tag of card.tags) { + const normalized = normalizeQuery(tag); + if (query.includes(normalized)) score += 3; + } + for (const word of card.title.toLowerCase().split(/\W+/)) { + if (word.length >= 4 && query.includes(word)) score += 1; + } + return score; +} + +function normalizeQuery(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9_.-]+/g, " ").trim(); +} + +function safeJsonStringify(value: unknown): string { + try { + return JSON.stringify(value); + } catch { + return ""; + } +} diff --git a/client/packages/editor-oss/src/copilot/playgroundLLMClient.test.ts b/client/packages/editor-oss/src/copilot/playgroundLLMClient.test.ts index fd0e0d09..ff54d228 100644 --- a/client/packages/editor-oss/src/copilot/playgroundLLMClient.test.ts +++ b/client/packages/editor-oss/src/copilot/playgroundLLMClient.test.ts @@ -2,6 +2,7 @@ import {beforeEach, describe, expect, it, vi} from "vitest"; const mocks = vi.hoisted(() => ({ generateText: vi.fn(), + streamText: vi.fn(), createOpenAI: vi.fn(), createAnthropic: vi.fn(), createGoogleGenerativeAI: vi.fn(), @@ -12,6 +13,7 @@ const mocks = vi.hoisted(() => ({ vi.mock("ai", () => ({ generateText: mocks.generateText, + streamText: mocks.streamText, })); vi.mock("@ai-sdk/openai", () => ({ @@ -28,10 +30,24 @@ vi.mock("@ai-sdk/google", () => ({ import {createPlaygroundLLMClient} from "./playgroundLLMClient"; +const streamFromParts = (parts: unknown[]) => ({ + async *[Symbol.asyncIterator]() { + for (const part of parts) { + yield part; + } + }, +}); + describe("createPlaygroundLLMClient", () => { beforeEach(() => { vi.clearAllMocks(); mocks.generateText.mockResolvedValue({text: "{\"reply\":\"ok\",\"stemscript\":\"\"}"}); + mocks.streamText.mockReturnValue({ + fullStream: streamFromParts([ + {type: "text-delta", id: "text-1", text: "{\"reply\":\"ok\","}, + {type: "text-delta", id: "text-1", text: "\"stemscript\":\"\"}"}, + ]), + }); mocks.openAIResponses.mockReturnValue({provider: "openai"}); mocks.anthropicModel.mockReturnValue({provider: "anthropic"}); mocks.googleModel.mockReturnValue({provider: "google"}); @@ -40,12 +56,12 @@ describe("createPlaygroundLLMClient", () => { mocks.createGoogleGenerativeAI.mockReturnValue(mocks.googleModel); }); - it("uses OpenAI Responses with prompt caching provider options", async () => { + it("streams OpenAI Responses with prompt caching provider options", async () => { const fetchImpl = vi.fn() as unknown as typeof fetch; const client = createPlaygroundLLMClient(fetchImpl); const text = await client.generateText({ - key: {provider: "openai", apiKey: "sk-openai", model: "gpt-5.2-codex"}, + key: {provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}, prompt: "User request", systemPrompt: "System prompt", knowledgePrompt: "Knowledge prompt", @@ -55,21 +71,169 @@ describe("createPlaygroundLLMClient", () => { expect(text).toBe("{\"reply\":\"ok\",\"stemscript\":\"\"}"); expect(mocks.createOpenAI).toHaveBeenCalledWith({apiKey: "sk-openai", fetch: fetchImpl}); - expect(mocks.openAIResponses).toHaveBeenCalledWith("gpt-5.2-codex"); - expect(mocks.generateText).toHaveBeenCalledWith(expect.objectContaining({ + expect(mocks.openAIResponses).toHaveBeenCalledWith("gpt-5.5"); + expect(mocks.generateText).not.toHaveBeenCalled(); + expect(mocks.streamText).toHaveBeenCalledWith(expect.objectContaining({ model: {provider: "openai"}, system: "System prompt\n\nKnowledge prompt", prompt: "User request", maxOutputTokens: 1234, + includeRawChunks: true, providerOptions: { openai: { promptCacheKey: "cache-key", - promptCacheRetention: "24h", + reasoningEffort: "high", }, }, })); }); + it("reports OpenAI stream progress without exposing raw chunks as final text", async () => { + const fetchImpl = vi.fn() as unknown as typeof fetch; + const onStreamProgress = vi.fn(); + const client = createPlaygroundLLMClient(fetchImpl); + mocks.streamText.mockReturnValue({ + fullStream: streamFromParts([ + {type: "raw", rawValue: {type: "response.created"}}, + {type: "reasoning-delta", id: "reasoning-1", text: "thinking"}, + {type: "text-delta", id: "text-1", text: "{\"reply\":\"ok\",\"stemscript\":\"\"}"}, + ]), + }); + + const text = await client.generateText({ + key: {provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}, + prompt: "User request", + systemPrompt: "System prompt", + knowledgePrompt: "Knowledge prompt", + onStreamProgress, + }); + + expect(text).toBe("{\"reply\":\"ok\",\"stemscript\":\"\"}"); + expect(onStreamProgress).toHaveBeenCalledWith({type: "raw"}); + expect(onStreamProgress).toHaveBeenCalledWith({type: "reasoning", delta: "thinking"}); + expect(onStreamProgress).toHaveBeenCalledWith({ + type: "text", + delta: "{\"reply\":\"ok\",\"stemscript\":\"\"}", + }); + }); + + it("reconstructs OpenAI Responses text from raw event-stream deltas when normalized text is missing", async () => { + const fetchImpl = vi.fn() as unknown as typeof fetch; + const onStreamProgress = vi.fn(); + const client = createPlaygroundLLMClient(fetchImpl); + mocks.streamText.mockReturnValue({ + fullStream: streamFromParts([ + {type: "raw", rawValue: {type: "response.created"}}, + { + type: "raw", + rawValue: { + type: "response.output_text.delta", + delta: "{\"reply\":\"ok\",", + }, + }, + { + type: "raw", + rawValue: { + type: "response.output_text.delta", + delta: "\"stemscript\":\"\"}", + }, + }, + { + type: "raw", + rawValue: { + type: "response.completed", + response: { + output: [ + { + type: "message", + content: [ + { + type: "output_text", + text: "{\"reply\":\"ok\",\"stemscript\":\"\"}", + }, + ], + }, + ], + }, + }, + }, + ]), + }); + + const text = await client.generateText({ + key: {provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}, + prompt: "User request", + systemPrompt: "System prompt", + knowledgePrompt: "Knowledge prompt", + onStreamProgress, + }); + + expect(text).toBe("{\"reply\":\"ok\",\"stemscript\":\"\"}"); + expect(onStreamProgress).toHaveBeenCalledWith({ + type: "text", + delta: "{\"reply\":\"ok\",", + source: "openai-raw", + }); + expect(onStreamProgress).toHaveBeenCalledWith({ + type: "text", + delta: "\"stemscript\":\"\"}", + source: "openai-raw", + }); + }); + + it("uses completed OpenAI raw response output as a final text fallback", async () => { + const fetchImpl = vi.fn() as unknown as typeof fetch; + const client = createPlaygroundLLMClient(fetchImpl); + mocks.streamText.mockReturnValue({ + fullStream: streamFromParts([ + { + type: "raw", + rawValue: { + type: "response.completed", + response: { + output: [ + { + type: "message", + content: [ + { + type: "output_text", + text: "{\"reply\":\"done\",\"stemscript\":\"\"}", + }, + ], + }, + ], + }, + }, + }, + ]), + }); + + const text = await client.generateText({ + key: {provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}, + prompt: "User request", + systemPrompt: "System prompt", + knowledgePrompt: "Knowledge prompt", + }); + + expect(text).toBe("{\"reply\":\"done\",\"stemscript\":\"\"}"); + }); + + it("uses a larger default output budget for OpenAI high-reasoning scene plans", async () => { + const fetchImpl = vi.fn() as unknown as typeof fetch; + const client = createPlaygroundLLMClient(fetchImpl); + + await client.generateText({ + key: {provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}, + prompt: "User request", + systemPrompt: "System prompt", + knowledgePrompt: "Knowledge prompt", + }); + + expect(mocks.streamText).toHaveBeenCalledWith(expect.objectContaining({ + maxOutputTokens: 128000, + })); + }); + it("marks Anthropic knowledge prompt as cacheable and uses browser direct-access headers", async () => { const fetchImpl = vi.fn() as unknown as typeof fetch; const client = createPlaygroundLLMClient(fetchImpl); diff --git a/client/packages/editor-oss/src/copilot/playgroundLLMClient.ts b/client/packages/editor-oss/src/copilot/playgroundLLMClient.ts index 84739eef..f026876c 100644 --- a/client/packages/editor-oss/src/copilot/playgroundLLMClient.ts +++ b/client/packages/editor-oss/src/copilot/playgroundLLMClient.ts @@ -1,9 +1,18 @@ -import type {generateText as generateTextType, LanguageModel, SystemModelMessage} from "ai"; +import type { + generateText as generateTextType, + LanguageModel, + streamText as streamTextType, + SystemModelMessage, +} from "ai"; -import type {CopilotChatKey} from "./playgroundCopilotKeys"; +import { + OPENAI_COPILOT_REASONING_EFFORT, + type CopilotChatKey, +} from "./playgroundCopilotKeys"; export const PLAYGROUND_PROMPT_CACHE_KEY = "stemstudio-playground-copilot-v5"; export const PLAYGROUND_MAX_OUTPUT_TOKENS = 4096; +export const PLAYGROUND_OPENAI_MAX_OUTPUT_TOKENS = 128000; export type PlaygroundLLMGenerateRequest = { key: CopilotChatKey; @@ -13,35 +22,176 @@ export type PlaygroundLLMGenerateRequest = { promptCacheKey?: string; maxOutputTokens?: number; signal?: AbortSignal; + onStreamProgress?: (progress: PlaygroundLLMStreamProgress) => void; }; export type PlaygroundLLMClient = { generateText(request: PlaygroundLLMGenerateRequest): Promise; }; +export type PlaygroundLLMStreamProgress = + | {type: "raw"} + | {type: "reasoning"; delta: string} + | {type: "text"; delta: string; source?: "openai-raw"}; + type ProviderOptions = NonNullable[0]["providerOptions"]>; export function createPlaygroundLLMClient(fetchImpl: typeof fetch = fetch.bind(globalThis)): PlaygroundLLMClient { return { async generateText(request: PlaygroundLLMGenerateRequest): Promise { - const {generateText} = await import("ai"); + const {generateText, streamText} = await import("ai"); const model = await createLanguageModel(request.key, fetchImpl); - const result = await generateText({ + const baseOptions = { model, system: buildSystemPrompt(request), prompt: request.prompt, - maxOutputTokens: request.maxOutputTokens ?? PLAYGROUND_MAX_OUTPUT_TOKENS, + maxOutputTokens: request.maxOutputTokens ?? getPlaygroundMaxOutputTokens(request.key), abortSignal: request.signal, maxRetries: 1, providerOptions: buildProviderOptions(request), + }; + + if (request.key.provider === "openai") { + return streamOpenAIText(streamText, baseOptions, request); + } + + const result = await generateText({ + ...baseOptions, }); if (result.text.trim()) return result.text; - throw new Error(`${request.key.provider} response did not include text content.`); + throw new Error( + `${request.key.provider} response did not include text content. ` + + "The model may have exhausted its output budget during reasoning.", + ); }, }; } +async function streamOpenAIText( + streamText: typeof streamTextType, + options: Parameters[0], + request: PlaygroundLLMGenerateRequest, +): Promise { + const result = streamText({ + ...options, + includeRawChunks: true, + }); + let text = ""; + let rawText = ""; + let rawFinishReason: string | undefined; + let rawFailureMessage: string | undefined; + + for await (const part of result.fullStream) { + if (part.type === "text-delta") { + text += part.text; + request.onStreamProgress?.({type: "text", delta: part.text}); + } else if (part.type === "reasoning-delta") { + request.onStreamProgress?.({type: "reasoning", delta: part.text}); + } else if (part.type === "raw") { + request.onStreamProgress?.({type: "raw"}); + const raw = getRecord(part.rawValue); + const rawDelta = getOpenAIResponseTextDelta(raw); + if (rawDelta) { + rawText += rawDelta; + request.onStreamProgress?.({type: "text", delta: rawDelta, source: "openai-raw"}); + } + + const completedText = getOpenAICompletedResponseText(raw); + if (completedText) { + const delta = completedText.startsWith(rawText) + ? completedText.slice(rawText.length) + : rawText.trim() + ? "" + : completedText; + if (delta) { + rawText += delta; + request.onStreamProgress?.({type: "text", delta, source: "openai-raw"}); + } + } + + rawFinishReason = getOpenAIResponseFinishReason(raw) ?? rawFinishReason; + rawFailureMessage = getOpenAIResponseFailureMessage(raw) ?? rawFailureMessage; + } else if (part.type === "error") { + throw part.error instanceof Error ? part.error : new Error(String(part.error)); + } + } + + if (text.trim()) return text; + if (rawText.trim()) return rawText; + if (rawFailureMessage) { + throw new Error(`OpenAI response failed: ${rawFailureMessage}`); + } + throw new Error( + `${request.key.provider} response did not include text content. ` + + "The model may have exhausted its output budget during reasoning." + + (rawFinishReason ? ` Finish reason: ${rawFinishReason}.` : ""), + ); +} + +function getRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? value as Record + : null; +} + +function getNestedRecord(value: Record | null, key: string): Record | null { + return getRecord(value?.[key]); +} + +function getOpenAIResponseTextDelta(raw: Record | null): string { + if (raw?.type === "response.output_text.delta" && typeof raw.delta === "string") { + return raw.delta; + } + return ""; +} + +function getOpenAICompletedResponseText(raw: Record | null): string { + if (raw?.type !== "response.completed" && raw?.type !== "response.incomplete") return ""; + + const response = getNestedRecord(raw, "response"); + const output = Array.isArray(response?.output) ? response.output : []; + const parts: string[] = []; + + for (const item of output) { + const itemRecord = getRecord(item); + const content = Array.isArray(itemRecord?.content) ? itemRecord.content : []; + for (const contentItem of content) { + const contentRecord = getRecord(contentItem); + if (contentRecord?.type === "output_text" && typeof contentRecord.text === "string") { + parts.push(contentRecord.text); + } + } + } + + return parts.join(""); +} + +function getOpenAIResponseFinishReason(raw: Record | null): string | undefined { + if (raw?.type !== "response.completed" && raw?.type !== "response.incomplete" && raw?.type !== "response.failed") { + return undefined; + } + + const response = getNestedRecord(raw, "response"); + const incompleteDetails = getNestedRecord(response, "incomplete_details"); + return typeof incompleteDetails?.reason === "string" ? incompleteDetails.reason : undefined; +} + +function getOpenAIResponseFailureMessage(raw: Record | null): string | undefined { + if (raw?.type !== "response.failed") return undefined; + + const response = getNestedRecord(raw, "response"); + const error = getNestedRecord(response, "error"); + if (typeof error?.message === "string") return error.message; + return getOpenAIResponseFinishReason(raw); +} + +export function getPlaygroundMaxOutputTokens(key: CopilotChatKey): number { + return key.provider === "openai" + ? PLAYGROUND_OPENAI_MAX_OUTPUT_TOKENS + : PLAYGROUND_MAX_OUTPUT_TOKENS; +} + async function createLanguageModel(key: CopilotChatKey, fetchImpl: typeof fetch): Promise { switch (key.provider) { case "anthropic": { @@ -102,7 +252,7 @@ function buildProviderOptions(request: PlaygroundLLMGenerateRequest): ProviderOp return { openai: { promptCacheKey: request.promptCacheKey ?? PLAYGROUND_PROMPT_CACHE_KEY, - promptCacheRetention: "24h", + reasoningEffort: OPENAI_COPILOT_REASONING_EFFORT, }, }; } diff --git a/client/packages/editor-oss/src/copilot/playgroundStemscriptKnowledge.ts b/client/packages/editor-oss/src/copilot/playgroundStemscriptKnowledge.ts index cdc7c6bc..30002a8e 100644 --- a/client/packages/editor-oss/src/copilot/playgroundStemscriptKnowledge.ts +++ b/client/packages/editor-oss/src/copilot/playgroundStemscriptKnowledge.ts @@ -5,7 +5,7 @@ StemStudio playground knowledge base, distilled from stemstudio-importer docs: - Primitive dimensions: prefer size over scale for new geometry. For capsules and cylinders, size.x is diameter and size.y is height or length. For spheres, size.x is diameter. For boxes, size is width,height,depth. - Object commands: add group name=Arena; add box|sphere|cylinder|cone|plane|torus|torusKnot|triangle|capsule|icosahedron|octahedron|dodecahedron|ring name=Object position=x,y,z size=x,y,z color=#rrggbb parent=Group; clone Target name=Copy position=x,y,z; delete Target; move Target parent=Group keepLocalSpace=true; update Target position=x,y,z rotation=x,y,z scale=x,y,z color=#rrggbb tag=Player. - Object settings: use objectSettings={isBatchable:true,isStatic:true} for repeated or static level pieces when appropriate. Use update Target tag=Player or tag=Goal after creation; tags are an update operation. Use inspectionStemscript with get/list commands before risky edits: list objects, get Target, get settings Target, get material Target, get physics Target, get behavior Target behaviorId=, get selected, get player, get camera DefaultCamera, get game settings, get scene settings. -- Asset/import inspection: use read-only scene asset commands to see imported resources before referencing them. Commands: list assets type=all|models|imports|files|behaviors|lambdas|packs|media filter= limit=80; get asset assetId=; list models; list imports; list files; list behavior packs; list lambda packs; get import ; get file . Models are 3D model assets, imports are script import assets, files are generic data files, behavior/lambda packs are asset-backed registered packs, media includes image/audio/video, vfx maps to Quarks assets, and prefab/stem maps to Prefab assets. Use asset names, descriptions, tags, formats, and revision IDs from these results to choose reusable existing assets before generating replacement code. +- Asset/import inspection: use read-only scene asset commands to see imported resources before referencing them. Commands: list assets type=all|models|imports|files|behaviors|lambdas|packs|media filter= limit=80; get asset assetId=; list models; list imports; list files; list behavior packs; list lambda packs; get import ; get file . Models are 3D model assets, imports are script import assets, files are generic data files, behavior/lambda packs are asset-backed registered packs, media includes image/audio/video, vfx maps to Quarks assets, and prefab/stem maps to Prefab assets. Use asset names, descriptions, tags, formats, and revision IDs from these results to choose reusable existing assets before generating replacement code. Prompt-to-image/model/audio generation is never automatic in direct chat: ask first if essential; if optional, build a primitive/code fallback and propose the upgrade after execution. - Physics: enable physics before detailed physics config. Use shape values box, sphere, capsule, convexHull, or concaveHull. Use concaveHull only for static terrain, tracks, or buildings. Use ctype Static, Dynamic, or Kinematic. Static level geometry should usually have mass 0; player-like bodies are usually Dynamic or Kinematic. - Camera: configure "DefaultCamera". Use cameraType=THIRD_PERSON for 3D character or kart scenes, FIRST_PERSON for first-person scenes, TOP_DOWN for board or arena games, SIDE_SCROLLER for side-view platformers, and NONE when only projection settings are needed. - Materials and atmosphere: use material Target color=#rrggbb roughness=0.5 metalness=0.1 opacity=1, texture Target textureUrl=|imageAsset= textureType=map, scene background, scene lighting, scene fog, scene tonemapping, scene postprocessing, light, render settings, and game settings to make primitives readable. Keep colors high-contrast enough to understand the play space. @@ -18,8 +18,8 @@ StemStudio playground knowledge base, distilled from stemstudio-importer docs: - Behavior code lifecycle: valid behavior methods include this.init = function(game) {}, this.update = function(deltaTime) {}, this.fixedUpdate = function(fixedDeltaTime) {}, this.dispose = function() {}, this.onEvent = function(msg,data) {}, this.onStart = function() {}, and this.onStop = function() {}. Prefer let/const, avoid remote dependencies, keep code short, and avoid hallucinated APIs. Use game/runtime APIs exposed by the editor; inspect existing behavior docs when available. Descriptions are searchable metadata, so generated or revised behavior code should always describe the gameplay purpose and any existing assets/packs it was matched against. - Lambda API reference: imported lambda YAML/assets can use runtime lambdas APIs such as lambdas.getInstance, lambdas.getInstancesByType, lambdas.registerObject, lambdas.deregisterObject, and lambdas.getObjectLambdas. Lambda code should not use this.erth, this.target, or this.gameObject; lambdas process many objects through this.processObjects(...) and object arguments. The Available lambda registry JSON is the compact list of all currently registered lambda packs/configs. Use lambda list filter=, lambda get lambdaId= includeCode=true, list lambda packs, and get asset assetId= as read-only inspection commands when debugging existing lambda config, component bindings, scene instances, assets, or code. Direct playground chat can explain/import lambda assets but does not auto-open a file picker. - Navmesh and waypoints: navmesh add target="Default Scene" autoGenerate=true agentHeight=1.8 agentRadius=0.45 debugVisualization=false; navmesh rebuild; navmesh connection add Start target=End bidirectional=false radius=0.75; waypoint path add name=PatrolPath loop=true; waypoint add path=PatrolPath position=x,y,z order=0 waitTime=1 arrivalRadius=0.5. -- Import API reference: terminal StemScript supports import model, behavior, lambda, vfx, image, audio/sound, video, prefab, script/import, and file. Syntax: import [filepath] ["comment"], import name= filepath=, or import name= url=. Examples: import model Kart kart.glb; import behavior NpcController behaviors/npc.yaml; import lambda PatrolBrain lambdas/patrol.yaml; import script name="math-helpers" filepath="imports/math-helpers.js"; import file name="level-data" filepath="data/level.json". Direct playground copilot should not emit import commands unless an import executor is available; instead inspect existing assets with list imports/list files/list models, provide exact import lines in the reply, and use primitives/behaviors for immediate edits. +- Import API reference: terminal StemScript supports import model, behavior, lambda, vfx, image, audio/sound, video, prefab, script/import, and file. Syntax: import [filepath] ["comment"], import name= filepath=, or import name= url=. Examples: import model Kart kart.glb; import behavior NpcController behaviors/npc.yaml; import lambda PatrolBrain lambdas/patrol.yaml; import script name="math-helpers" filepath="imports/math-helpers.js"; import file name="level-data" filepath="data/level.json". Direct playground copilot should not emit import commands unless an import executor is available; instead inspect existing assets with list imports/list files/list models, provide exact import lines in the reply, and use primitives/behaviors for immediate edits. Direct JSON artifacts may create browser-safe behavior, lambda, scriptImport, and file assets, then use them from later phase StemScript. - Genre recipes: a small platformer needs a Player capsule, Ground, 3 to 5 platforms, a collectible, a goal trigger, physics, and SIDE_SCROLLER or THIRD_PERSON camera. A racing sketch needs a kart body, ground or track blocks, 3 checkpoint triggers, start/finish markers, physics, and THIRD_PERSON camera. A top-down arena needs a player marker, boundaries, obstacles, pickups, and TOP_DOWN camera. -- Gameplay recipes: for an arcade loop, start with project title "" and game settings isGame=true showHUD=true, then create clear spawn, goal, hazards, collectibles, scoring variables, reset rules, camera, lights, and player feedback. Use behavior scripts for motion, scoring, health, timers, AI patrol, pickups, checkpoints, or win/lose logic when primitives alone are not enough. +- Gameplay recipes: for an arcade loop, start with a design brief (core loop, controls/camera, goals/fail state, challenge curve, feedback/progression, reuse plan, implementation strategy), project title "", and game settings isGame=true showHUD=true. Then create clear spawn, goal, hazards, collectibles, scoring variables, reset rules, camera, lights, and player feedback. Use built-in behaviors first and behavior/lambda/script artifacts for motion, scoring, health, timers, AI patrol, pickups, checkpoints, or win/lose logic when primitives alone are not enough. Compose multiple supported primitives, VFX, materials, textures, waypoints, navmesh, and runtime code rather than leaving a primitive-only static mockup. - Performance: keep live plans compact. Prefer grouped pieces and repeated simple primitives over hundreds of unique meshes. Avoid huge command counts unless the user explicitly asks for a large build. `.trim(); diff --git a/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.test.ts b/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.test.ts index cf12db37..b1ed60d5 100644 --- a/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.test.ts +++ b/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.test.ts @@ -53,6 +53,94 @@ describe("playgroundStemscriptPlan", () => { expect(plan.stemscript).toContain('position={"x":0,"y":0,"z":0}'); }); + it("parses phased plans with reusable artifacts", () => { + const plan = parseProviderStemscriptPlan(JSON.stringify({ + reply: "Built a phased racing loop.", + designBrief: { + coreLoop: "Drive through checkpoints and finish laps.", + controlsCamera: "Third-person kart camera.", + goalsFailState: "Win at three laps, reset on missed checkpoint.", + challengeCurve: "Add tighter turns each lap.", + feedbackProgression: "HUD lap counter and boost VFX.", + reusePlan: "Use trigger built-ins and one custom controller.", + implementationStrategy: "Create track, then controller, then polish.", + }, + assetRequests: [ + { + type: "model", + name: "Kart", + prompt: "small stylized kart", + essential: false, + reason: "would improve visuals", + }, + ], + artifacts: [ + { + type: "behavior", + name: "RaceHudController", + description: "Tracks HUD state.", + code: "this.update = function(dt) {}", + }, + ], + phases: [ + { + name: "Inspect", + inspectionCommands: [ + {command: "list_lambdas", params: {filter: "race"}}, + ], + }, + { + title: "Mechanics", + goal: "Create checkpoints and attach controller.", + artifacts: [ + { + kind: "import", + name: "race-math", + content: "export const clamp = (v, min, max) => Math.max(min, Math.min(max, v));", + format: "js", + contentType: "text/javascript", + }, + ], + commands: [ + {command: "create_group", params: {name: "Checkpoints"}}, + ], + }, + ], + })); + + expect(plan.artifacts).toEqual([ + expect.objectContaining({ + type: "behavior", + name: "RaceHudController", + code: "this.update = function(dt) {}", + }), + ]); + expect(plan.designBrief?.coreLoop).toContain("checkpoints"); + expect(plan.assetRequests[0]).toEqual(expect.objectContaining({ + type: "model", + name: "Kart", + essential: false, + })); + expect(plan.phases).toHaveLength(2); + expect(plan.phases[0]).toEqual(expect.objectContaining({ + name: "Inspect", + inspectionStemscript: "list_lambdas filter=race", + stemscript: "", + })); + expect(plan.phases[1]).toEqual(expect.objectContaining({ + name: "Mechanics", + goal: "Create checkpoints and attach controller.", + stemscript: "create_group name=Checkpoints", + })); + expect(plan.phases[1]?.artifacts[0]).toEqual(expect.objectContaining({ + type: "scriptImport", + name: "race-math", + content: "export const clamp = (v, min, max) => Math.max(min, Math.min(max, v));", + format: "js", + contentType: "text/javascript", + })); + }); + it("validates primitive-only live scripts", () => { const result = validateGeneratedStemscript([ "# Generated in browser", diff --git a/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.ts b/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.ts index e2f50909..1f7ff630 100644 --- a/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.ts +++ b/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.ts @@ -2,10 +2,61 @@ import {isReadOnlyCommand} from "../agent/script-tool/checkScript"; import {ScriptExecutor} from "../agent/script-tool/ScriptExecutor"; export interface PlaygroundStemscriptPlan { + designBrief?: PlaygroundDesignBrief; + assetRequests: PlaygroundAssetRequest[]; inspectionStemscript: string; reply: string; stemscript: string; notes: string[]; + phases: PlaygroundPlanPhase[]; + artifacts: PlaygroundPlanArtifact[]; +} + +export type PlaygroundPlanArtifactType = "behavior" | "lambda" | "scriptImport" | "file"; + +export interface PlaygroundPlanArtifact { + type: PlaygroundPlanArtifactType; + name: string; + assetId?: string; + description?: string; + code?: string; + content?: string; + format?: string; + contentType?: string; + config?: Record | string; + metadata?: Record; + version?: string; + author?: string; +} + +export interface PlaygroundDesignBrief { + title?: string; + coreLoop?: string; + controlsCamera?: string; + goalsFailState?: string; + challengeCurve?: string; + feedbackProgression?: string; + reusePlan?: string; + implementationStrategy?: string; + notes?: string[]; +} + +export type PlaygroundAssetRequest = { + type?: string; + name?: string; + prompt?: string; + essential?: boolean; + reason?: string; + fallback?: string; +}; + +export interface PlaygroundPlanPhase { + id?: string; + name?: string; + goal?: string; + inspectionStemscript: string; + stemscript: string; + artifacts: PlaygroundPlanArtifact[]; } export interface ValidatedStemscript { @@ -64,6 +115,16 @@ const stringArray = (value: unknown): string[] => { return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0); }; +const objectArray = (value: unknown): Record[] => { + if (!Array.isArray(value)) return []; + return value.filter((item): item is Record => Boolean(item) && typeof item === "object" && !Array.isArray(item)); +}; + +const objectRecord = (value: unknown): Record | undefined => { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + return value as Record; +}; + const commandArrayToScript = (value: unknown): string => { if (!Array.isArray(value)) return ""; return value @@ -92,49 +153,220 @@ const formatParamValue = (value: unknown): string => { return JSON.stringify(value); }; +const parseArtifactType = (value: unknown): PlaygroundPlanArtifactType | null => { + if (typeof value !== "string") return null; + switch (value.trim().toLowerCase()) { + case "behavior": + case "behaviour": + return "behavior"; + case "lambda": + return "lambda"; + case "scriptimport": + case "script-import": + case "script_import": + case "import": + return "scriptImport"; + case "file": + return "file"; + default: + return null; + } +}; + +const parseStringField = (record: Record, ...keys: string[]): string | undefined => { + for (const key of keys) { + const value = record[key]; + if (typeof value === "string" && value.trim()) return value.trim(); + } + return undefined; +}; + +const parseBooleanField = (record: Record, ...keys: string[]): boolean | undefined => { + for (const key of keys) { + const value = record[key]; + if (typeof value === "boolean") return value; + if (typeof value === "string") { + const normalized = value.trim().toLowerCase(); + if (["true", "yes", "required", "essential"].includes(normalized)) return true; + if (["false", "no", "optional"].includes(normalized)) return false; + } + } + return undefined; +}; + +const parseDesignBrief = (value: unknown): PlaygroundDesignBrief | undefined => { + if (typeof value === "string" && value.trim()) return {notes: [value.trim()]}; + const record = objectRecord(value); + if (!record) return undefined; + + const brief: PlaygroundDesignBrief = {}; + const fields: Array<[keyof PlaygroundDesignBrief, string[]]> = [ + ["title", ["title", "name"]], + ["coreLoop", ["coreLoop", "core_loop", "loop"]], + ["controlsCamera", ["controlsCamera", "controls_camera", "controls", "camera"]], + ["goalsFailState", ["goalsFailState", "goals_fail_state", "goals", "failState", "fail_state"]], + ["challengeCurve", ["challengeCurve", "challenge_curve", "challenge"]], + ["feedbackProgression", ["feedbackProgression", "feedback_progression", "feedback", "progression"]], + ["reusePlan", ["reusePlan", "reuse_plan", "reuse"]], + ["implementationStrategy", ["implementationStrategy", "implementation_strategy", "implementation"]], + ]; + + for (const [field, keys] of fields) { + const parsed = parseStringField(record, ...keys); + if (parsed) { + (brief[field] as string | undefined) = parsed; + } + } + + const notes = stringArray(record.notes); + if (notes.length > 0) brief.notes = notes; + + return Object.keys(brief).length > 0 ? brief : undefined; +}; + +const parseAssetRequests = (value: unknown): PlaygroundAssetRequest[] => + objectArray(value) + .map(record => { + const request: PlaygroundAssetRequest = {}; + const type = parseStringField(record, "type", "kind"); + const name = parseStringField(record, "name", "id"); + const prompt = parseStringField(record, "prompt", "description"); + const reason = parseStringField(record, "reason", "why"); + const fallback = parseStringField(record, "fallback", "fallbackPlan", "fallback_plan"); + const essential = parseBooleanField(record, "essential", "required"); + + if (type) request.type = type; + if (name) request.name = name; + if (prompt) request.prompt = prompt; + if (reason) request.reason = reason; + if (fallback) request.fallback = fallback; + if (essential !== undefined) request.essential = essential; + + return Object.keys(request).length > 0 ? request : null; + }) + .filter((request): request is PlaygroundAssetRequest => Boolean(request)); + +const parseArtifacts = (value: unknown): PlaygroundPlanArtifact[] => + objectArray(value) + .map(record => { + const type = parseArtifactType(record.type ?? record.kind); + const name = parseStringField(record, "name", "id", "behaviorId", "lambdaId", "importName", "fileName"); + if (!type || !name) return null; + + const artifact: PlaygroundPlanArtifact = { + type, + name, + }; + const assetId = parseStringField(record, "assetId", "id"); + const description = parseStringField(record, "description", "summary"); + const code = parseStringField(record, "code", "source"); + const content = parseStringField(record, "content", "text", "data"); + const format = parseStringField(record, "format", "extension"); + const contentType = parseStringField(record, "contentType", "mimeType", "mime"); + const config = objectRecord(record.config) ?? parseStringField(record, "config"); + const metadata = objectRecord(record.metadata); + const version = parseStringField(record, "version"); + const author = parseStringField(record, "author"); + + if (assetId && assetId !== name) artifact.assetId = assetId; + if (description) artifact.description = description; + if (code) artifact.code = code; + if (content) artifact.content = content; + if (format) artifact.format = format; + if (contentType) artifact.contentType = contentType; + if (config) artifact.config = config; + if (metadata) artifact.metadata = metadata; + if (version) artifact.version = version; + if (author) artifact.author = author; + return artifact; + }) + .filter((artifact): artifact is PlaygroundPlanArtifact => Boolean(artifact)); + +const parseInspectionScript = (record: Record): string => { + const inspectionStemscript = + typeof record.inspectionStemscript === "string" + ? record.inspectionStemscript + : typeof record.inspectionScript === "string" + ? record.inspectionScript + : typeof record.inspectStemscript === "string" + ? record.inspectStemscript + : commandArrayToScript(record.inspectionCommands ?? record.queries); + return stripCodeFence(inspectionStemscript || ""); +}; + +const parseMutationScript = (record: Record): string => { + const commands = commandArrayToScript(record.commands); + const stemscript = + typeof record.stemscript === "string" + ? record.stemscript + : typeof record.script === "string" + ? record.script + : commands; + return stripCodeFence(stemscript || ""); +}; + +const parsePhases = (value: unknown): PlaygroundPlanPhase[] => + objectArray(value) + .map(record => { + const phase: PlaygroundPlanPhase = { + inspectionStemscript: parseInspectionScript(record), + stemscript: parseMutationScript(record), + artifacts: parseArtifacts(record.artifacts), + }; + const id = parseStringField(record, "id"); + const name = parseStringField(record, "name", "title"); + const goal = parseStringField(record, "goal", "description"); + + if (id) phase.id = id; + if (name) phase.name = name; + if (goal) phase.goal = goal; + if (!phase.id && !phase.name && !phase.goal && !phase.inspectionStemscript && !phase.stemscript && phase.artifacts.length === 0) { + return null; + } + return phase; + }) + .filter((phase): phase is PlaygroundPlanPhase => Boolean(phase)); + export function parseProviderStemscriptPlan(rawText: string): PlaygroundStemscriptPlan { const parsed = tryParseJsonObject(rawText); if (parsed && typeof parsed === "object") { const record = parsed as Record; - const commands = commandArrayToScript(record.commands); - const stemscript = - typeof record.stemscript === "string" - ? record.stemscript - : typeof record.script === "string" - ? record.script - : commands; - const inspectionStemscript = - typeof record.inspectionStemscript === "string" - ? record.inspectionStemscript - : typeof record.inspectionScript === "string" - ? record.inspectionScript - : typeof record.inspectStemscript === "string" - ? record.inspectStemscript - : commandArrayToScript(record.inspectionCommands ?? record.queries); return { - inspectionStemscript: stripCodeFence(inspectionStemscript || ""), + designBrief: parseDesignBrief(record.designBrief ?? record.design_brief ?? record.brief), + assetRequests: parseAssetRequests(record.assetRequests ?? record.asset_requests), + inspectionStemscript: parseInspectionScript(record), reply: typeof record.reply === "string" ? record.reply.trim() : "", - stemscript: stripCodeFence(stemscript || ""), + stemscript: parseMutationScript(record), notes: stringArray(record.notes), + phases: parsePhases(record.phases), + artifacts: parseArtifacts(record.artifacts), }; } const fenced = rawText.match(STEMSCRIPT_FENCE_RE); if (fenced?.[1]) { return { + designBrief: undefined, + assetRequests: [], inspectionStemscript: "", reply: rawText.replace(fenced[0], "").trim(), stemscript: stripCodeFence(fenced[1]), notes: [], + phases: [], + artifacts: [], }; } return { + designBrief: undefined, + assetRequests: [], inspectionStemscript: "", reply: rawText.trim(), stemscript: "", notes: [], + phases: [], + artifacts: [], }; } diff --git a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.sessionMode.test.tsx b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.sessionMode.test.tsx index 90c2f15a..3af52606 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.sessionMode.test.tsx +++ b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.sessionMode.test.tsx @@ -33,6 +33,7 @@ vi.mock("./AiCopilot.styles", () => { PermissionButtons: div, PermissionContainer: div, PermissionMessage: div, + PlaygroundProcessingBanner: div, ProcessingMainText: div, ProcessingStatusContainer: div, ProcessingSubText: div, @@ -295,6 +296,80 @@ describe("AiCopilot session mode startup", () => { }); }); + it("does not render blank transcript rows for empty agent chunks", async () => { + mocks.isPlayground = true; + + const {container} = renderCopilot(); + + await waitFor(() => expect(mocks.provider.createSession).toHaveBeenCalledOnce()); + + act(() => { + mocks.provider.emitTestEvent("agentMessage", { + message: " ", + }); + mocks.provider.emitTestEvent("agentThinking", { + message: " ", + }); + }); + + expect(container.querySelector(".message-agent")).not.toBeInTheDocument(); + expect(screen.queryByTestId("copilot-process-details")).not.toBeInTheDocument(); + }); + + it("shows the playground browser-running banner only while chat prompt is processing", async () => { + mocks.isPlayground = true; + + renderCopilot(); + + await waitFor(() => expect(mocks.provider.createSession).toHaveBeenCalledOnce()); + + expect(screen.queryByTestId("copilot-playground-processing-banner")).not.toBeInTheDocument(); + + act(() => { + mocks.provider.emitTestEvent("promptStarted", { + prompt: "make a game", + }); + }); + + await waitFor(() => { + expect(screen.getByTestId("copilot-playground-processing-banner").textContent).toContain( + "COPILOT IS RUNNING IN THE BROWSER", + ); + }); + + act(() => { + mocks.provider.emitTestEvent("promptCompleted"); + }); + + await waitFor(() => { + expect(screen.queryByTestId("copilot-playground-processing-banner")).not.toBeInTheDocument(); + }); + }); + + it("surfaces OpenAI stream progress in the playground processing status", async () => { + mocks.isPlayground = true; + + renderCopilot(); + + await waitFor(() => expect(mocks.provider.createSession).toHaveBeenCalledOnce()); + + act(() => { + mocks.provider.emitTestEvent("promptStarted", { + prompt: "make a game", + }); + mocks.provider.emitTestEvent("toolCall", {toolCall: {title: "Stream OpenAI response"}}); + mocks.provider.emitTestEvent("toolCallUpdate", { + line: "OpenAI stream active (text=2.1K chars, reasoning=8.3K chars, events=4)", + }); + }); + + await waitFor(() => { + expect(screen.getByText( + "OpenAI stream active (text=2.1K chars, reasoning=8.3K chars, events=4)", + )).toBeInTheDocument(); + }); + }); + it("starts a fresh local session in non-playground OSS mode without server history", async () => { mocks.isPlayground = false; diff --git a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.styles.ts b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.styles.ts index 63aa132c..303400bb 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.styles.ts +++ b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.styles.ts @@ -195,6 +195,19 @@ export const LoadingWrapper = styled.div` } `; +export const PlaygroundProcessingBanner = styled.div` + margin: 10px 0 0; + padding: 12px; + border-radius: 6px; + border: 1px solid #ff4d4f; + background: #b00020; + color: #fff; + font-size: 13px; + font-weight: 800; + line-height: 1.25; + text-transform: uppercase; +`; + export const CloseBtn = styled.img` cursor: pointer; width: 26px; diff --git a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.tsx b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.tsx index 8628ee12..5a1faddc 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.tsx +++ b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.tsx @@ -25,6 +25,7 @@ import { PermissionButtons, PermissionContainer, PermissionMessage, + PlaygroundProcessingBanner, ProcessingMainText, ProcessingStatusContainer, ProcessingSubText, @@ -82,6 +83,21 @@ const stripCopilotSuggestionsBlocks = (content: string): string => { return content.replace(COPILOT_SUGGESTIONS_BLOCK_REGEX, "").trim(); }; +const getRenderableMessageContent = (message: Message, stripSuggestions: boolean): string => { + if (message.type === "agent" && stripSuggestions) { + return stripCopilotSuggestionsBlocks(message.content); + } + return message.content.trim(); +}; + +const shouldRenderChatMessage = (message: Message, stripSuggestions: boolean): boolean => { + if (message.type === "interactive") { + return Boolean(message.interactiveResult) || Boolean(message.content.trim()); + } + if (message.attachedObjects?.length) return true; + return getRenderableMessageContent(message, stripSuggestions).length > 0; +}; + const hasDashboardCopilotBootstrapIntent = (bootstrap: DashboardCopilotBootstrap | null): bootstrap is DashboardCopilotBootstrap => Boolean(bootstrap?.prompt || bootstrap?.entryMode); @@ -172,6 +188,9 @@ const compactWorkflowLine = (value: string): string => { : normalized; }; +const isOpenAIStreamWorkflowLine = (value: string): boolean => + /^OpenAI stream (?:active|complete|ended)\b/.test(value); + const parseModeCommand = (value: string): ParsedModeCommand | null => { const match = value.trim().match(/^\/mode(?:\s+(.+))?$/i); if (!match) return null; @@ -314,7 +333,7 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o if (!isWorkspaceMode || !sceneID || aiMessages.length === 0) return; if (messageSceneIDRef.current !== sceneID) return; - saveWorkspaceChatSnapshot({ + void saveWorkspaceChatSnapshot({ sceneID, sessionID: acpClientRef.current?.getSessionId() || sessionSeqCounterRef.current.sessionId, messages: aiMessages, @@ -332,8 +351,8 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o const restoreWorkspaceChatSnapshotForScene = useCallback(( targetSceneID: string | null | undefined, sessionID?: string | null, - ): boolean => { - const snapshot = readWorkspaceChatSnapshot(targetSceneID, sessionID); + ): Promise => (async () => { + const snapshot = await readWorkspaceChatSnapshot(targetSceneID, sessionID); if (!snapshot) return false; messageSceneIDRef.current = snapshot.sceneID; @@ -342,7 +361,7 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o setProcessingStatus({main: "", subTasks: []}); processingEventRef.current = null; return true; - }, []); + })(), []); // In AI-focused layout (advancedMode === false), the copilot stays as a // right-anchored rail over the full scene and starts wider than the @@ -595,6 +614,7 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o }; const handleAgentMessage = (event: any) => { + const content = typeof event?.data?.message === "string" ? event.data.message : ""; markMessagesForCurrentScene(); if (!acpClientRef.current?.isSuppressingSessionUpdates) { setCopilotState(AI_COPILOT_STATE.PROCESSING); @@ -606,11 +626,13 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o lastMsg && lastMsg.type === "agent" && processingEventRef.current === "agentMessage" && - !event.data.replayStartNewMessage + !event?.data?.replayStartNewMessage ) { - return [...prev.slice(0, -1), {...lastMsg, content: lastMsg.content + event.data.message}]; + if (!content) return prev; + return [...prev.slice(0, -1), {...lastMsg, content: lastMsg.content + content}]; } + if (!content.trim()) return prev; processingEventRef.current = "agentMessage"; const sessionId = sessionSeqCounterRef.current.sessionId || acpClientRef.current?.getSessionId() || null; @@ -621,7 +643,7 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o { id: msgId, type: "agent", - content: event.data.message, + content, timestamp: Date.now(), }, ]; @@ -735,6 +757,9 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o const index = typeof event?.data?.index === "number" ? event.data.index + 1 : undefined; const total = typeof event?.data?.total === "number" ? event.data.total : undefined; const prefix = index && total ? `${index}/${total}` : "step"; + if (isPlayground && mode === "chat" && line && isOpenAIStreamWorkflowLine(line)) { + setProcessingStatus(prev => ({...prev, main: line})); + } appendProcessMessage(line ? `- ${prefix}: \`${line}\`` : "- Tool progress updated", "workflow"); }; @@ -1044,7 +1069,7 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o await handleResetThread(); if (sceneLoadGeneration.current === generation) { if (shouldStartIdleWorkspace && !pendingDashboardPromptRef.current?.autoSubmit) { - restoreWorkspaceChatSnapshotForScene(sceneID); + await restoreWorkspaceChatSnapshotForScene(sceneID); } postPendingEntryGreeting(); } @@ -1494,6 +1519,11 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o }, [prompt]); const selectedObjectsToDisplay = selectedObjects.filter(obj => !attachedObjects.find(o => o.uuid === obj.uuid)); + const visibleAiMessages = aiMessages.filter(message => shouldRenderChatMessage(message, isWorkspaceMode)); + const visibleProcessingMain = processingStatus.main.trim(); + const visibleProcessingSubTasks = processingStatus.subTasks + .map(task => task.trim()) + .filter(Boolean); const showContent = (connectionState === ConnectionState.CONNECTED && !isLoadingSession) || insufficientCredits; return ( - {aiMessages?.map(message => { + {visibleAiMessages.map(message => { // Check if this interactive result is currently pending const isPending = message.type === "interactive" && message.interactiveResult && acpClientRef.current?.checkPendingInteractiveResult(message.interactiveResult.id); - const isLatestMessage = aiMessages.length > 0 && aiMessages[aiMessages.length - 1]?.id === message.id; + const isLatestMessage = visibleAiMessages.length > 0 && visibleAiMessages[visibleAiMessages.length - 1]?.id === message.id; const openProcessDetails = message.type === "thought" && copilotState === AI_COPILOT_STATE.PROCESSING && @@ -1781,8 +1811,13 @@ export const AiCopilot = ({isOpen, setIsOpen, pinnedCodeEditorWidth, onResize, o !insufficientCredits && copilotState === AI_COPILOT_STATE.PROCESSING && ( - {processingStatus.main && {processingStatus.main}} - {processingStatus.subTasks.map((task, index) => ( + {isPlayground && mode === "chat" && ( + + COPILOT IS RUNNING IN THE BROWSER. DON'T SWITCH TABS IN THE COPILOT WINDOW. + + )} + {visibleProcessingMain && {visibleProcessingMain}} + {visibleProcessingSubTasks.map((task, index) => ( {task} ))} diff --git a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.test.tsx b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.test.tsx index 94132185..eeec120d 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.test.tsx +++ b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.test.tsx @@ -12,7 +12,7 @@ vi.mock("../../../../copilot", () => ({ COPILOT_KEYS_CHANGED_EVENT: "stem:playground-copilot-keys-changed", COPILOT_MODEL_OPTIONS: { anthropic: [{label: "Claude Sonnet 4.5", model: "claude-sonnet-4-5-20250929"}], - openai: [{label: "GPT-5.2 Codex", model: "gpt-5.2-codex"}], + openai: [{label: "GPT-5.5 High", model: "gpt-5.5"}], gemini: [{label: "Gemini 2.5 Flash", model: "gemini-2.5-flash"}], }, getCopilotModelSelectionSync: mocks.getCopilotModelSelectionSync, @@ -48,15 +48,15 @@ describe("AiKeysModal", () => { it("lets the user choose a copilot model when multiple chat keys are configured", async () => { mocks.getCopilotModelSelectionSync.mockReturnValue(null); mocks.resolveCopilotChatKeys.mockResolvedValue([ - {provider: "openai", apiKey: "sk-openai", model: "gpt-5.2-codex"}, + {provider: "openai", apiKey: "sk-openai", model: "gpt-5.5"}, {provider: "gemini", apiKey: "sk-gemini", model: "gemini-2.5-flash"}, ]); render(); const select = await screen.findByLabelText("Copilot model"); - fireEvent.change(select, {target: {value: "openai:gpt-5.2-codex"}}); + fireEvent.change(select, {target: {value: "openai:gpt-5.5"}}); - expect(mocks.setCopilotModelSelection).toHaveBeenCalledWith("openai", "gpt-5.2-codex"); + expect(mocks.setCopilotModelSelection).toHaveBeenCalledWith("openai", "gpt-5.5"); }); }); diff --git a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.tsx b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.tsx index be76e9d1..ca611745 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.tsx +++ b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.tsx @@ -97,7 +97,7 @@ export const AiKeysModal = ({onClose}: {onClose: () => void}) => { const [selection, setSelection] = useState(getCopilotModelSelectionSync); const refreshChatKeys = useCallback(async () => { - const keys = await resolveCopilotChatKeys(); + const keys = await resolveCopilotChatKeys({consumeHandoff: false}); setChatKeys(keys); setSelection(getCopilotModelSelectionSync()); }, []); diff --git a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workspaceChatSnapshot.test.ts b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workspaceChatSnapshot.test.ts index 88898e9e..34f7dcff 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workspaceChatSnapshot.test.ts +++ b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workspaceChatSnapshot.test.ts @@ -1,4 +1,7 @@ -import {beforeEach, describe, expect, it} from "vitest"; +import "fake-indexeddb/auto"; + +import {IDBFactory} from "fake-indexeddb"; +import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; import { readWorkspaceChatSnapshot, @@ -8,10 +11,16 @@ import { describe("workspaceChatSnapshot", () => { beforeEach(() => { window.localStorage.clear(); + vi.stubGlobal("indexedDB", new IDBFactory()); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); - it("stores and restores latest default-workspace chat state by scene", () => { - saveWorkspaceChatSnapshot({ + it("stores and restores latest default-workspace chat state by scene", async () => { + await saveWorkspaceChatSnapshot({ sceneID: "scene-1", sessionID: "session-1", messages: [ @@ -20,7 +29,7 @@ describe("workspaceChatSnapshot", () => { ], }); - const snapshot = readWorkspaceChatSnapshot("scene-1"); + const snapshot = await readWorkspaceChatSnapshot("scene-1"); expect(snapshot?.sceneID).toBe("scene-1"); expect(snapshot?.sessionID).toBe("session-1"); @@ -30,25 +39,25 @@ describe("workspaceChatSnapshot", () => { ]); }); - it("can restore a specific session snapshot", () => { - saveWorkspaceChatSnapshot({ + it("can restore a specific session snapshot", async () => { + await saveWorkspaceChatSnapshot({ sceneID: "scene-1", sessionID: "session-a", messages: [{id: "a", type: "user", content: "First", timestamp: 1}], }); - saveWorkspaceChatSnapshot({ + await saveWorkspaceChatSnapshot({ sceneID: "scene-1", sessionID: "session-b", messages: [{id: "b", type: "user", content: "Second", timestamp: 2}], }); - expect(readWorkspaceChatSnapshot("scene-1", "session-a")?.messages[0]?.content).toBe("First"); - expect(readWorkspaceChatSnapshot("scene-1", "session-b")?.messages[0]?.content).toBe("Second"); - expect(readWorkspaceChatSnapshot("scene-1")?.messages[0]?.content).toBe("Second"); + expect((await readWorkspaceChatSnapshot("scene-1", "session-a"))?.messages[0]?.content).toBe("First"); + expect((await readWorkspaceChatSnapshot("scene-1", "session-b"))?.messages[0]?.content).toBe("Second"); + expect((await readWorkspaceChatSnapshot("scene-1"))?.messages[0]?.content).toBe("Second"); }); - it("stores stale interactive result messages as inert agent transcript entries", () => { - saveWorkspaceChatSnapshot({ + it("stores stale interactive result messages as inert agent transcript entries", async () => { + await saveWorkspaceChatSnapshot({ sceneID: "scene-1", messages: [ { @@ -60,9 +69,32 @@ describe("workspaceChatSnapshot", () => { ], }); - const message = readWorkspaceChatSnapshot("scene-1")?.messages[0]; + const message = (await readWorkspaceChatSnapshot("scene-1"))?.messages[0]; expect(message?.type).toBe("agent"); expect(message?.content).toBe("Choose an asset"); }); + + it("stores large snapshots in IndexedDB without touching localStorage", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const localStorageSetSpy = vi.spyOn(Storage.prototype, "setItem"); + + await saveWorkspaceChatSnapshot({ + sceneID: "scene-1", + sessionID: "session-large", + messages: Array.from({length: 20}, (_, index) => ({ + id: `m${index}`, + type: index % 2 === 0 ? "user" : "agent", + content: `message ${index} ${"x".repeat(3_000)}`, + timestamp: index, + })), + }); + + const snapshot = await readWorkspaceChatSnapshot("scene-1", "session-large"); + + expect(snapshot?.messages.length).toBe(20); + expect(snapshot?.messages[19]?.content).toContain("message 19"); + expect(localStorageSetSpy).not.toHaveBeenCalled(); + expect(warnSpy).not.toHaveBeenCalled(); + }); }); diff --git a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workspaceChatSnapshot.ts b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workspaceChatSnapshot.ts index 26e486fd..9d3b0b4c 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workspaceChatSnapshot.ts +++ b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workspaceChatSnapshot.ts @@ -1,5 +1,8 @@ import type {Message} from "./utils/history"; +const DB_NAME = "stemstudio-ai-copilot-chat"; +const DB_VERSION = 1; +const STORE_NAME = "workspaceChatSnapshots"; const WORKSPACE_CHAT_SNAPSHOT_PREFIX = "ai_copilot_workspace_chat_snapshot"; const MAX_STORED_MESSAGES = 80; const MAX_MESSAGE_CHARS = 20_000; @@ -25,6 +28,15 @@ type StoredWorkspaceChatSnapshot = { messages: StoredWorkspaceMessage[]; }; +type StoredLatestPointer = { + sceneID: string; + latestSessionID: string; + updatedAt: number; +}; + +const memoryStore = new Map(); +const warnedSnapshotFailures = new Set(); + const storageKey = (sceneID: string, sessionID?: string | null): string => { const safeSceneID = encodeURIComponent(sceneID); if (!sessionID) return `${WORKSPACE_CHAT_SNAPSHOT_PREFIX}:${safeSceneID}:latest`; @@ -74,33 +86,148 @@ const deserializeMessage = (message: unknown): Message | null => { }; }; -const writeSnapshot = (snapshot: StoredWorkspaceChatSnapshot) => { +const isSnapshot = (value: unknown): value is StoredWorkspaceChatSnapshot => { + if (!value || typeof value !== "object") return false; + const snapshot = value as Partial; + return typeof snapshot.sceneID === "string" && Array.isArray(snapshot.messages); +}; + +const isLatestPointer = (value: unknown): value is StoredLatestPointer => { + if (!value || typeof value !== "object") return false; + const pointer = value as Partial; + return typeof pointer.sceneID === "string" && typeof pointer.latestSessionID === "string"; +}; + +const toWorkspaceSnapshot = ( + value: unknown, + sceneID: string, +): WorkspaceChatSnapshot | null => { + if (!isSnapshot(value) || value.sceneID !== sceneID) return null; + + const messages = value.messages + .map(deserializeMessage) + .filter((message): message is Message => Boolean(message)); + if (messages.length === 0) return null; + + return { + sceneID, + sessionID: typeof value.sessionID === "string" ? value.sessionID : null, + updatedAt: typeof value.updatedAt === "number" ? value.updatedAt : 0, + messages, + }; +}; + +const purgeLegacyLocalSnapshots = (): void => { if (typeof window === "undefined") return; try { - const serialized = JSON.stringify(snapshot); + const keys: string[] = []; + for (let i = 0; i < window.localStorage.length; i += 1) { + const key = window.localStorage.key(i); + if (key?.startsWith(`${WORKSPACE_CHAT_SNAPSHOT_PREFIX}:`)) keys.push(key); + } + keys.forEach(key => window.localStorage.removeItem(key)); + } catch { + // localStorage can be disabled or already over quota; snapshot storage + // no longer depends on it, so cleanup is best-effort. + } +}; +purgeLegacyLocalSnapshots(); + +const openDb = (): Promise => new Promise((resolve, reject) => { + if (typeof indexedDB === "undefined") { + reject(new Error("IndexedDB is unavailable.")); + return; + } + + const request = indexedDB.open(DB_NAME, DB_VERSION); + request.onerror = () => reject(request.error ?? new Error("Failed to open workspace chat snapshot database.")); + request.onupgradeneeded = () => { + const db = request.result; + if (!db.objectStoreNames.contains(STORE_NAME)) { + db.createObjectStore(STORE_NAME); + } + }; + request.onsuccess = () => resolve(request.result); +}); + +const runStoreRequest = async ( + mode: IDBTransactionMode, + createRequest: (store: IDBObjectStore) => IDBRequest, +): Promise => { + const db = await openDb(); + return await new Promise((resolve, reject) => { + const transaction = db.transaction(STORE_NAME, mode); + const store = transaction.objectStore(STORE_NAME); + const request = createRequest(store); + + request.onerror = () => reject(request.error ?? new Error("Workspace chat snapshot request failed.")); + request.onsuccess = () => resolve(request.result); + transaction.oncomplete = () => db.close(); + transaction.onerror = () => { + db.close(); + reject(transaction.error ?? new Error("Workspace chat snapshot transaction failed.")); + }; + transaction.onabort = () => { + db.close(); + reject(transaction.error ?? new Error("Workspace chat snapshot transaction aborted.")); + }; + }); +}; + +const putSnapshot = async (snapshot: StoredWorkspaceChatSnapshot) => { + const sessionKey = snapshot.sessionID + ? storageKey(snapshot.sceneID, snapshot.sessionID) + : storageKey(snapshot.sceneID); + const latestKey = storageKey(snapshot.sceneID); + + memoryStore.set(sessionKey, snapshot); + if (snapshot.sessionID) { + memoryStore.set(latestKey, { + sceneID: snapshot.sceneID, + latestSessionID: snapshot.sessionID, + updatedAt: snapshot.updatedAt, + }); + } else { + memoryStore.set(latestKey, snapshot); + } + + await runStoreRequest("readwrite", store => { if (snapshot.sessionID) { - // Full snapshot under the session key; `:latest` holds only a tiny - // pointer to the most recent session — not a second copy of the - // (up to ~1.6MB) blob, which is what previously doubled this cache. - window.localStorage.setItem(storageKey(snapshot.sceneID, snapshot.sessionID), serialized); - window.localStorage.setItem( - storageKey(snapshot.sceneID), - JSON.stringify({latestSessionID: snapshot.sessionID}), + store.put(snapshot, sessionKey); + return store.put( + { + sceneID: snapshot.sceneID, + latestSessionID: snapshot.sessionID, + updatedAt: snapshot.updatedAt, + } satisfies StoredLatestPointer, + latestKey, ); - } else { - // No session: the `:latest` key holds the snapshot itself. - window.localStorage.setItem(storageKey(snapshot.sceneID), serialized); } + return store.put(snapshot, latestKey); + }); +}; + +const getStoredValue = async (key: string): Promise => { + try { + const value = await runStoreRequest( + "readonly", + store => store.get(key), + ); + return value ?? null; } catch (error) { - console.warn("[workspaceChatSnapshot] Failed to write workspace chat snapshot:", error); + if (!warnedSnapshotFailures.has(key)) { + warnedSnapshotFailures.add(key); + console.warn("[workspaceChatSnapshot] Failed to read IndexedDB workspace chat snapshot:", error); + } + return memoryStore.get(key) ?? null; } }; -export const saveWorkspaceChatSnapshot = (input: { +export const saveWorkspaceChatSnapshot = async (input: { sceneID: string | null | undefined; sessionID?: string | null; messages: Message[]; -}) => { +}): Promise => { const sceneID = input.sceneID?.trim(); if (!sceneID || input.messages.length === 0 || typeof window === "undefined") return; @@ -110,62 +237,41 @@ export const saveWorkspaceChatSnapshot = (input: { .filter((message): message is StoredWorkspaceMessage => Boolean(message)); if (messages.length === 0) return; - writeSnapshot({ + const snapshot: StoredWorkspaceChatSnapshot = { sceneID, sessionID: input.sessionID || null, updatedAt: Date.now(), messages, - }); -}; + }; -const parseSnapshot = (raw: string, sceneID: string): WorkspaceChatSnapshot | null => { try { - const parsed = JSON.parse(raw) as Partial; - if (parsed.sceneID !== sceneID) return null; - - const messages = Array.isArray(parsed.messages) - ? parsed.messages - .map(deserializeMessage) - .filter((message): message is Message => Boolean(message)) - : []; - if (messages.length === 0) return null; - - return { - sceneID, - sessionID: typeof parsed.sessionID === "string" ? parsed.sessionID : null, - updatedAt: typeof parsed.updatedAt === "number" ? parsed.updatedAt : 0, - messages, - }; + await putSnapshot(snapshot); + warnedSnapshotFailures.delete(storageKey(sceneID, input.sessionID || null)); } catch (error) { - console.warn("[workspaceChatSnapshot] Failed to read workspace chat snapshot:", error); - return null; + if (!warnedSnapshotFailures.has(sceneID)) { + warnedSnapshotFailures.add(sceneID); + console.warn("[workspaceChatSnapshot] Failed to write IndexedDB workspace chat snapshot:", error); + } } }; -export const readWorkspaceChatSnapshot = ( +export const readWorkspaceChatSnapshot = async ( sceneID: string | null | undefined, sessionID?: string | null, -): WorkspaceChatSnapshot | null => { +): Promise => { const normalizedSceneID = sceneID?.trim(); if (!normalizedSceneID || typeof window === "undefined") return null; if (sessionID) { - const raw = window.localStorage.getItem(storageKey(normalizedSceneID, sessionID)); - return raw ? parseSnapshot(raw, normalizedSceneID) : null; + return toWorkspaceSnapshot(await getStoredValue(storageKey(normalizedSceneID, sessionID)), normalizedSceneID); } - // No session requested: `:latest` is either a small pointer to the most - // recent session, or (for session-less saves) the snapshot itself. - const latestRaw = window.localStorage.getItem(storageKey(normalizedSceneID)); - if (!latestRaw) return null; - try { - const maybePointer = JSON.parse(latestRaw) as {latestSessionID?: unknown}; - if (typeof maybePointer.latestSessionID === "string") { - const raw = window.localStorage.getItem(storageKey(normalizedSceneID, maybePointer.latestSessionID)); - return raw ? parseSnapshot(raw, normalizedSceneID) : null; - } - } catch { - return null; + const latest = await getStoredValue(storageKey(normalizedSceneID)); + if (isLatestPointer(latest)) { + return toWorkspaceSnapshot( + await getStoredValue(storageKey(normalizedSceneID, latest.latestSessionID)), + normalizedSceneID, + ); } - return parseSnapshot(latestRaw, normalizedSceneID); + return toWorkspaceSnapshot(latest, normalizedSceneID); }; diff --git a/client/packages/editor-oss/src/editor/assets/v2/ContextMenu/Create/Create.tsx b/client/packages/editor-oss/src/editor/assets/v2/ContextMenu/Create/Create.tsx index b294c5fd..3d145d86 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/ContextMenu/Create/Create.tsx +++ b/client/packages/editor-oss/src/editor/assets/v2/ContextMenu/Create/Create.tsx @@ -59,7 +59,6 @@ export const Create = ({ onMenuClose, setIsOpen, position, - sceneID, objectToReplace, replaceObject, onGenerationStart, @@ -203,27 +202,10 @@ export const Create = ({ imageToken = uploadRes.image_token; } - // For Meshy/Tripo: submit a background job and return immediately. - // The playground has no Go server to run jobs, so Meshy falls through - // to the polling flow below and is imported browser-direct. - if (!isPlaygroundMode() && (generator === GENERATOR_TYPES.MESHY || generator === GENERATOR_TYPES.TRIPO)) { - const {jobId} = await aiWorldController.modelGeneratorProvider!.submitGenerationJob({ - generator, - sceneId: sceneID || app.editor?.sceneID || "", - name: name || prompt, - prompt, - negative_prompt: "", - doRefine: refine, - doRig: autoRig, - target_polycount: 3000, - type: imageFile ? "image_to_model" : "text_to_model", - file_token: imageToken, - quality, - model_version: modelVersion, - }); - return {jobId}; - } - + // This OSS build has no server-side generation jobs (see + // handle_jobs_oss.go), so every provider — desktop and playground — + // runs the synchronous polling flow below and imports the resulting + // GLB URL directly. The provider task endpoints return a `task_id`. const res = await aiWorldController.generate3dObject( { generationType: imageFile ? "image_to_model" : "text_to_model", @@ -274,11 +256,10 @@ export const Create = ({ }; } - // Playground Meshy: the polling flow above produced a GLB URL. There - // is no Go server, so import it browser-direct (uploadModelFromUrl - // fetches the CDN URL itself in playground mode) and hand back a - // ready-to-place object. - if (isPlaygroundMode() && res.model) { + // The polling flow produced a GLB URL. Import it and hand back a + // ready-to-place object. uploadModelFromUrl fetches the provider CDN + // directly in this OSS build (no asset-download proxy ships here). + if (res.model) { const uploaded = await uploadModelFromUrl({ url: res.model, name: name || prompt || "Generated Model", @@ -376,13 +357,6 @@ export const Create = ({ setLoadingDescription(generateStep?.description || "Generating model..."); const objData = await generateAndUploadModel(currentPrompt, uuid, modelName, tags); - // For Meshy/Tripo: job was submitted; close dialog and let monitor handle completion - if ("jobId" in objData) { - showToast({type: "success", title: "Generation started! The model will appear in your scene when ready."}); - handleClose(); - return; - } - // Add the generated object to the scene. Both the Erth // composition and the playground browser-direct Meshy import // hand back a ready Object3D. diff --git a/client/packages/editor-oss/src/editor/assets/v2/ContextMenu/Create/PromptStep.tsx b/client/packages/editor-oss/src/editor/assets/v2/ContextMenu/Create/PromptStep.tsx index 50ddba2a..099aa5c0 100644 --- a/client/packages/editor-oss/src/editor/assets/v2/ContextMenu/Create/PromptStep.tsx +++ b/client/packages/editor-oss/src/editor/assets/v2/ContextMenu/Create/PromptStep.tsx @@ -3,7 +3,7 @@ import {useRef} from "react"; import {useAuthorizationContext} from "@stem/editor-oss/context"; import {MODEL_VERSION, TEXTURE_QUALITY} from "../../../../../controls/AiWorldController/AiWorldController.types"; import CheckBox from "../../../../../ui/form/v2/CheckBox"; -import {GENERATOR_TYPES} from "@stem/editor-oss/utils/ModelGeneratorProvider"; +import {GENERATOR_TYPES, getGeneratorCapability} from "@stem/editor-oss/utils/ModelGeneratorProvider"; import {isPlaygroundMode} from "@web-shared/playgroundMode"; import {BasicCombobox} from "../../common/BasicCombobox/BasicCombobox"; import {CreditsBar} from "../../CreditsBar/CreditsBar"; @@ -81,18 +81,14 @@ export const PromptStep = ({ const qualityOptions = Object.values(TEXTURE_QUALITY).map(value => ({key: value, value})); const modelVersionOptions = Object.values(MODEL_VERSION).map(value => ({key: value, value})); - const generatorLabels: Record = { - [GENERATOR_TYPES.MESHY]: "meshy", - [GENERATOR_TYPES.TRIPO]: "tripo", - [GENERATOR_TYPES.ERTH]: "Erth (experimental)", - }; - // The playground has no Go server; only Meshy can run browser-direct, so - // Tripo and Erth are hidden there. + const capability = getGeneratorCapability(generator); + // The playground has no Go server, so only providers that can run fully + // browser-direct (Meshy, Rodin) are offered there; desktop shows all. const generatorOptions = Object.values(GENERATOR_TYPES) - .filter(value => !isPlaygroundMode() || value === GENERATOR_TYPES.MESHY) + .filter(value => !isPlaygroundMode() || getGeneratorCapability(value).supportsBrowserDirectPlayground) .map(value => ({ key: value, - value: generatorLabels[value], + value: getGeneratorCapability(value).label, })); if (!isOpen) return null; @@ -136,7 +132,7 @@ export const PromptStep = ({ } - {(generator === GENERATOR_TYPES.MESHY || generator === GENERATOR_TYPES.TRIPO) && + {capability.supportsAutoRig &&