Interior views in view_scene, paint and search_materials with native scene materials

packages/core/src/agent-operations/apply-changes.ts

@@ -1,4 +1,4 @@
-import type { AnyNode, Collection } from '../schema'
+import type { AnyNode, Collection, SceneMaterial } from '../schema'
 import type { SceneChanges, SceneNodes } from './types'
 
 const childrenOf = (node: AnyNode | undefined): string[] | null =>
@@ -54,18 +54,22 @@ export function applySceneChanges(
  * step. Updates to a node the batch created are folded into its creation, and a node created then
  * deleted never appears; updates only reach nodes that existed before, deletes come last.
  */
-export function mergeSceneChanges(sets: readonly SceneChanges[]): Required<SceneChanges> {
+export function mergeSceneChanges(
+  sets: readonly SceneChanges[],
+): Required<Omit<SceneChanges, 'materials'>> & Pick<SceneChanges, 'materials'> {
   const created = new Map<string, { node: AnyNode; parentId?: string }>()
   const updated = new Map<string, Partial<AnyNode>>()
   const deleted = new Set<string>()
   const collections: Record<string, Collection | null> = {}
+  const materials = new Map<string, SceneMaterial>()
   const dropCreated = (id: string) => {
     created.delete(id)
     for (const [childId, entry] of [...created])
       if ((entry.parentId ?? entry.node.parentId) === id) dropCreated(childId)
   }
   for (const set of sets) {
     Object.assign(collections, set.collections)
+    for (const material of set.materials ?? []) materials.set(material.id, material)
     for (const { id, data } of set.update ?? []) {
       const entry = created.get(id)
       if (entry)
@@ -86,6 +90,7 @@ export function mergeSceneChanges(sets: readonly SceneChanges[]): Required<Scene
     }
   }
   return {
+    ...(materials.size ? { materials: [...materials.values()] } : {}),
     create: [...created.values()],
     update: [...updated].map(([id, data]) => ({ id, data })),
     delete: [...deleted],

packages/core/src/agent-operations/create-stair.ts

@@ -121,7 +121,12 @@ export const createStair: AgentOperation<CreateStairInput> = (nodes, input, cont
     (level) => level.parentId === building.id || building.children.includes(level.id),
   )
 
-  const changes: Required<SceneChanges> = { create: [], update: [], delete: [], collections: {} }
+  const changes: Required<Omit<SceneChanges, 'materials'>> = {
+    create: [],
+    update: [],
+    delete: [],
+    collections: {},
+  }
   let upper = input.toLevelId
     ? requireLevel(nodes, input.toLevelId)
     : floors.find((level) => level.level > from.level)

packages/core/src/agent-operations/get-node.ts

@@ -1,10 +1,14 @@
 import { refuse } from '../agent-tools/refusal'
 import { isFloorAnchoredOpening } from '../lib/floor-opening-footprints'
 import { getOpeningFloorDatum } from '../lib/opening-floor-datum'
+import { referencedSceneMaterialIds } from './scene-materials'
 import type { AgentOperation } from './types'
 
-/** `get_node`: a node, whole; a door or window also says where its floor is and what it hangs from. */
-export const getNode: AgentOperation<{ id: string }> = (nodes, { id }) => {
+/**
+ * `get_node`: a node, whole; a door or window also says where its floor is and what it hangs from;
+ * with the scene's materials lent, the ones its finishes use.
+ */
+export const getNode: AgentOperation<{ id: string }> = (nodes, { id }, { materials }) => {
   const node = nodes[id]
   if (!node) refuse('node_not_found', `Node not found: ${id}.`, { id })
   const wall = node.parentId ? nodes[node.parentId] : undefined
@@ -15,5 +19,19 @@ export const getNode: AgentOperation<{ id: string }> = (nodes, { id }) => {
           anchor: isFloorAnchoredOpening(node) ? 'floor' : 'wall',
         }
       : undefined
-  return { result: { node: resolvedOpening ? { ...node, resolvedOpening } : node } }
+  const used = referencedSceneMaterialIds(node)
+  return {
+    result: {
+      node: resolvedOpening ? { ...node, resolvedOpening } : node,
+      ...(materials
+        ? {
+            materials: Object.fromEntries(
+              Object.values(materials)
+                .filter((material) => used.has(material.id))
+                .map((material) => [material.id, material]),
+            ),
+          }
+        : {}),
+    },
+  }
 }

packages/core/src/agent-operations/index.ts

@@ -9,9 +9,11 @@ import { furnishRoom } from './furnish-room'
 import { getNode } from './get-node'
 import { getLevelSummary, getWalls, getZones } from './level-reads'
 import { listLevels } from './list-levels'
+import { paint } from './paint'
 import { placeItems } from './place-items'
 import { ROOM_OPERATIONS } from './room-structure'
 import { searchAssets } from './search-assets'
+import { searchMaterials } from './search-materials'
 import { fitStair, measureStairOperation } from './stairs'
 import type { AgentOperations } from './types'
 import { verifyScene } from './verify-scene'
@@ -40,13 +42,16 @@ export * from './list-levels'
 export * from './material-preset'
 export * from './material-refs'
 export * from './node-patch'
+export * from './paint'
 export * from './place-items'
 export * from './plan-geometry'
 export * from './room-structure'
+export * from './scene-materials'
 export * from './scene-measure'
 export * from './scene-queries'
 export * from './scene-view'
 export * from './search-assets'
+export * from './search-materials'
 export * from './stairs'
 export * from './types'
 export * from './verify-scene'
@@ -72,5 +77,7 @@ export const AGENT_OPERATIONS = {
   create_room: createRoom,
   furnish_room: furnishRoom,
   search_assets: searchAssets,
+  paint,
+  search_materials: searchMaterials,
   ...ROOM_OPERATIONS,
 } as const satisfies AgentOperations

packages/core/src/agent-operations/material-refs.ts

@@ -5,6 +5,7 @@ import {
   type MaterialSurface,
   parseMaterialRef,
 } from '../material-library'
+import type { SceneMaterial, SceneMaterialId } from '../schema/scene-material'
 
 const SURFACE_WORDS: Record<MaterialSurface, string> = {
   roof: 'roofing',
@@ -74,20 +75,35 @@ export function nearestLibraryMaterials(
 
 /**
  * A material ref that renders: a library id the catalog holds (`library:<id>`, or the bare id), or
- * a scene material (`scene:<id>`, which only the host's store can check). Anything else would fall
- * back to a default with no word (an unknown preset came out grey), so it is refused, naming
- * the nearest library ids for the surface; when the library has nothing of the kind like it, the
- * refusal says so, and that a colour is the nearest (a corrugated roof was once offered bricks).
+ * a scene material (`scene:<id>`, checked against the scene's materials when the host lends them).
+ * Anything else would fall back to a default with no word (an unknown preset came out grey), so it
+ * is refused, naming the nearest library ids for the surface; when the library has nothing of the
+ * kind like it, the refusal says so, and that a colour is the nearest (a corrugated roof was once
+ * offered bricks).
  */
 export function requireMaterialRef(
   asked: string,
   field?: string,
   surface?: MaterialSurface,
-  /** Whether to point to `paint` for a colour (paint passes it); else a flat library colour. */
-  { paint = false } = {},
+  {
+    paint = false,
+    materials,
+  }: {
+    /** Whether to point to `paint` for a colour (paint passes it); else a flat library colour. */
+    paint?: boolean
+    /** The scene's materials, when the host has them. */
+    materials?: Readonly<Record<SceneMaterialId, SceneMaterial>>
+  } = {},
 ): string {
   const parsed = parseMaterialRef(asked.includes(':') ? asked : `library:${asked}`)
-  if (parsed?.kind === 'scene') return asked
+  if (parsed?.kind === 'scene') {
+    if (materials && !Object.hasOwn(materials, parsed.id))
+      refuse('unknown_material', `${asked} is not one of this scene's materials.`, {
+        material: asked,
+        ...(field ? { field } : {}),
+      })
+    return asked
+  }
   // A community material (`library:mtl_…`): the host checks it against its catalog.
   if (parsed?.kind === 'library' && parsed.id.startsWith('mtl_')) return asked
   if (parsed?.kind === 'library' && getCatalogMaterialById(parsed.id)) return `library:${parsed.id}`
@@ -108,7 +124,7 @@ export function requireMaterialRef(
           kind.length ? kind.slice(0, 5).map(named).join(', ') : 'none'
         }.${byName.length ? ` Nearest by name: ${byName.join(', ')}.` : ''} Or ${colour} is the nearest.`
       : `${head}. Nearest: ${nearest.join(', ')}; or give ${colour}.`
-  return refuse('unknown_material', message, {
+  return refuse('unknown_material', `${message} search_materials finds one by words.`, {
     material: asked,
     ...(field ? { field } : {}),
     nearest,