Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,11 @@ MULTIPLAYER_PORT=2567 # Multiplayer sidecar — ws://localhost:<PORT>
# Tripo — alternate text-to-3D + character rigging.
# TRIPO_API_KEY=

# Rodin (Hyper3D) — alternate text-to-3D model generation.
# RODIN_API_KEY=
# Optionally override the API base URL (staging / self-hosted):
# RODIN_API_BASE_URL=

# ElevenLabs — text-to-speech, NPC voices.
# ELEVEN_LABS_API_KEY=

Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
bin/
**/.memsearch/
lib/
docs/marketing
node-compile-cache/
**/node-compile-cache/
pyvenv.cfg
server/build/
server/ai-server
web/build
firebase-json-dev.json
server/.env.b*
Expand Down
4 changes: 2 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,8 @@ proxy that fronts the BYOK keys.
- The Go AI server (`server/cmd/ai-server`) is what the editor's
`network/adapters/remote-go` calls. It accepts BYOK keys from env or
the dashboard's BYOK panel and proxies to Anthropic / OpenAI / Meshy /
ElevenLabs / Anything World. **It is not a hosted service** — it runs
on the user's machine.
Tripo / Rodin / ElevenLabs / Anything World. **It is not a hosted
service** — it runs on the user's machine.
- Inline `exec` and the script-tool import pipeline
(`editor-oss/src/agent/script-tool/`) work without any AI provider.
- The dashboard exposes "Import stemscript folder" which stages a folder
Expand Down
5 changes: 3 additions & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ stemstudio-multiplayer/ ← Colyseus server. Runs as a sidecar in dev

server/
cmd/ai-server/ ← Go AI proxy entry point. Forwards calls to
Anthropic, OpenAI, Meshy, ElevenLabs, AnythingWorld
using env keys or BYOK keys forwarded by the editor.
Anthropic, OpenAI, Meshy, Tripo, Rodin, ElevenLabs,
AnythingWorld using env keys or BYOK keys forwarded
by the editor.
server/controllers/tools/ai/ ← AI handler implementations.
server/controllers/tools/ai/byok/ ← BYOK key resolution.

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ If StemStudio is useful to you or your organization, please consider sponsoring
- **In-editor code editor** — Monaco for behavior and script authoring with full TypeScript-style assist.
- **Physics** — Ammo.js / Rapier integration with rigid bodies, joints, raycasting.
- **Local multiplayer** — Colyseus sidecar auto-spawned on `bun run dev`. Two browser tabs on the same machine join a real room.
- **AI copilot (BYOK)** — bring your own keys for Anthropic, OpenAI, Meshy (3D model gen), ElevenLabs (TTS), and AnythingWorld. Configure once, use everywhere.
- **AI copilot (BYOK)** — bring your own keys for Anthropic, OpenAI, Meshy / Tripo / Rodin (3D model gen), ElevenLabs (TTS), and AnythingWorld. Configure once, use everywhere.
- **Local-first persistence** — IndexedDB for seamless auto-save, or open a real folder via File System Access API (Chromium) for git-friendly workflows.
- **Export & share** — package any project as a standalone static site (Player-only build) you can host anywhere.

Expand Down
6 changes: 6 additions & 0 deletions client/packages/editor-oss/src/agent/CommandsRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -812,6 +812,12 @@ export class CommandsRegistry {
description: "Parent object name or UUID to attach the model",
required: false,
},
{
name: "provider",
type: "string",
description: "Generation provider: meshy, tripo, or rodin (defaults to meshy)",
required: false,
},
],
handler: async params => this.objectHandlers.handleGenerate3DModel(params),
});
Expand Down
67 changes: 56 additions & 11 deletions client/packages/editor-oss/src/agent/handlers/ObjectHandlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,27 @@ import {
} from "../../editor/assets/v2/materials/materialUtils";
import {IMaterialSettingsTextures} from "../../editor/assets/v2/RightPanel/sections/MaterialRenderingSection/types";
import EngineRuntime from "../../EngineRuntime";
import {uploadModelFromUrl} from "../../model/uploadModelFromUrl";
import {GENERATOR_TYPES} from "../../utils/ModelGeneratorProvider";
import TagUtil from "../../utils/TagUtil";
import {SupportedCommands} from "../CommandsRegistry";
import {CommandResult} from "../types/ACPTypes";
import {getObjectBaseMetaData, serializeObjectForAI} from "../utils/serialization";

/** Map an agent-supplied provider string to a generator, defaulting to Meshy. */
function resolveGenerator(provider?: string): GENERATOR_TYPES {
switch ((provider ?? "").trim().toLowerCase()) {
case "tripo":
return GENERATOR_TYPES.TRIPO;
case "rodin":
return GENERATOR_TYPES.RODIN;
case "meshy":
return GENERATOR_TYPES.MESHY;
default:
return GENERATOR_TYPES.MESHY;
}
}

type BaseObjectData = {
uuid: string;
name: string;
Expand Down Expand Up @@ -1194,28 +1209,58 @@ export class ObjectHandlers {
async handleGenerate3DModel({
prompt,
name,
position,
provider,
}: {
prompt: string;
name?: string;
position?: {x: number; y: number; z: number};
parent?: string;
provider?: string;
}): Promise<CommandResult> {
const sceneId = this.engine.editor?.sceneID;
if (!sceneId) {
return {status: "failed", message: "No scene is currently open.", data: null};
}
const {jobId} = await this.aiWorldController.modelGeneratorProvider!.submitGenerationJob({
generator: GENERATOR_TYPES.MESHY,
sceneId,
name: name || prompt,
prompt,
});

return {
status: "success",
message: `Generation job started (jobId: ${jobId}). The model will be added to the scene automatically when ready — no further action needed.`,
data: {jobId},
};
// This OSS build has no server-side generation jobs, so generate
// synchronously: create the provider task, poll to completion, fetch
// the GLB URL, import it, and place it in the scene.
const generator = resolveGenerator(provider);
const modelName = name || prompt;

try {
const res = await this.aiWorldController.generate3dObject({
generationType: "text_to_model",
prompt,
generator,
});
if (!res?.model) {
return {status: "failed", message: "Model generation returned no model URL.", data: null};
}

const uploaded = await uploadModelFromUrl({url: res.model, name: modelName});
const point = position ? new THREE.Vector3(position.x, position.y, position.z) : undefined;
const placed = this.aiWorldController.addObjectToScene(
uploaded.object,
false,
undefined,
undefined,
point,
);

return {
status: "success",
message: `Generated and placed "${modelName}" with ${generator}.`,
data: {assetId: uploaded.assetId, uuid: placed.uuid},
};
} catch (error) {
return {
status: "failed",
message: `Failed to generate 3D model: ${error instanceof Error ? error.message : String(error)}`,
data: null,
};
}
}

handleGetPlayer(): CommandResult {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ function createHarness() {
};
const app = {
camera,
editor: {scene},
editor: {scene, sceneName: "Untitled"},
environmentManager: {
updateEnvironmentSettings: vi.fn(async () => {}),
},
Expand Down Expand Up @@ -131,6 +131,20 @@ describe("SettingsHandlers normalization", () => {
expect(result.data.isGame).toBe(true);
});

it("updates project title, notifies scene-name listeners, and supports project readback", () => {
const {handlers, app} = createHarness();

const result = handlers.handleSetProjectTitle({title: "Crystal Dash"});
const readback = handlers.handleGetSceneSetting({category: "project"});

expect(result.status).toBe("success");
expect(app.editor.sceneName).toBe("Crystal Dash");
expect(app.call).toHaveBeenCalledWith("sceneNameUpdated");
expect(app.call).toHaveBeenCalledWith("objectChanged", app.editor, app.editor.scene);
expect(readback.status).toBe("success");
expect(readback.data).toEqual({title: "Crystal Dash"});
});

it("does not turn rendering-only settings into a game", async () => {
const {handlers, scene} = createHarness();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,7 @@ export class SettingsHandlers {
}

editor.sceneName = title;
this.engine.call("sceneNameUpdated");
this.engine.call("objectChanged", editor, editor.scene);

return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -939,6 +939,7 @@ const COMMAND_PARAMS: Record<string, CommandHelp> = {
{name: "name", type: "string", required: false, description: "Name for the generated model"},
{name: "position", type: "x,y,z", required: false, description: "Position to place the model"},
{name: "parent", type: "string", required: false, description: "Parent object name or UUID"},
{name: "provider", type: "string", required: false, description: "'meshy', 'tripo', or 'rodin' (default meshy)"},
],
examples: ['generate model prompt="a medieval wooden barrel" name="Barrel" position=3,0,0'],
},
Expand Down
53 changes: 53 additions & 0 deletions client/packages/editor-oss/src/ai/RodinDirectClient.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import {describe, expect, it} from "vitest";

import {decodeRodinTaskId, encodeRodinTaskId} from "./RodinDirectClient";
import {
GENERATOR_TYPES,
getGeneratorCapability,
MODEL_GENERATOR_CAPABILITIES,
} from "../utils/ModelGeneratorProvider";

describe("Rodin composite task id codec", () => {
it("round-trips a task uuid and subscription key", () => {
const id = encodeRodinTaskId("task-uuid", "sub-key");
expect(id).toBe("task-uuid|sub-key");
expect(decodeRodinTaskId(id)).toEqual({taskUUID: "task-uuid", subscriptionKey: "sub-key"});
});

it("preserves subscription keys that themselves contain the separator", () => {
const id = encodeRodinTaskId("uuid", "a|b|c");
expect(decodeRodinTaskId(id)).toEqual({taskUUID: "uuid", subscriptionKey: "a|b|c"});
});

it("yields an empty subscription key when the separator is missing", () => {
expect(decodeRodinTaskId("bare-uuid")).toEqual({taskUUID: "bare-uuid", subscriptionKey: ""});
});
});

describe("model generator capabilities", () => {
it("includes every generator enum value", () => {
for (const value of Object.values(GENERATOR_TYPES)) {
expect(MODEL_GENERATOR_CAPABILITIES[value]).toBeDefined();
}
});

it("marks Rodin as text-only, browser-direct, no rig/refine", () => {
const cap = getGeneratorCapability(GENERATOR_TYPES.RODIN);
expect(cap.byokProvider).toBe("rodin");
expect(cap.supportsTextToModel).toBe(true);
expect(cap.supportsImageToModel).toBe(false);
expect(cap.supportsRefine).toBe(false);
expect(cap.supportsAutoRig).toBe(false);
expect(cap.supportsBrowserDirectPlayground).toBe(true);
});

it("only offers browser-direct providers in the playground filter", () => {
const playgroundProviders = Object.values(GENERATOR_TYPES).filter(
v => getGeneratorCapability(v).supportsBrowserDirectPlayground,
);
expect(playgroundProviders).toContain(GENERATOR_TYPES.MESHY);
expect(playgroundProviders).toContain(GENERATOR_TYPES.RODIN);
expect(playgroundProviders).not.toContain(GENERATOR_TYPES.TRIPO);
expect(playgroundProviders).not.toContain(GENERATOR_TYPES.ERTH);
});
});
Loading
Loading