From de2ce8de30b4d0a3ebbee4c120fc3e2c1570117f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 09:29:11 +0000 Subject: [PATCH 1/3] fix(scenarios): the seed loads a user-handle field with a user id, and both author loops run verify The seed loop left a persona-scoped owner field holding the handle text (`owner: "u_sam"`), which a build that models the field as a user lookup refuses, and sent a handle no persona holds (`u_tia`) as-is. A user-handle field (a direct scope field the card declares as neither lookup nor select, and not `id`) now carries the persona's user id, or the admin's for an unheld handle, in the field and in `owner_id`; the record lists the unheld handles. Scope fields naming a row or an option are sent verbatim, as before. The build and iterate prompts now name the same `objectstack verify` invocation `buildHalf` scores with (`verifyArgs`), next to validate and build, and the data contract says the handle field is loaded with a user id. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2ebbZkr2QAEG1x7UC8 --- .../qa/scenarios/src/runner/orchestrate.ts | 30 ++++++-- packages/qa/scenarios/src/runner/prompts.ts | 76 +++++++++++++++++-- .../scenarios/test/runner/orchestrate.test.ts | 62 +++++++++++++-- .../qa/scenarios/test/runner/units.test.ts | 23 +++++- 4 files changed, 172 insertions(+), 19 deletions(-) diff --git a/packages/qa/scenarios/src/runner/orchestrate.ts b/packages/qa/scenarios/src/runner/orchestrate.ts index 33006555b24..fc77ede74c1 100644 --- a/packages/qa/scenarios/src/runner/orchestrate.ts +++ b/packages/qa/scenarios/src/runner/orchestrate.ts @@ -41,7 +41,7 @@ import type { AgentDriver, StepRef } from './driver.js'; import { citationLabels, namespaceOfArtifact, namespacedScenario, permissionSetName, templateHostName } from './namespace.js'; import { observeOperation, touchedRowId, type Row } from './observe.js'; import type { CommandResult } from './project.js'; -import { buildPrompt, iteratePrompt, operationPrompt, personaHandles, questionPrompt, templateIteratePrompt, verifyFlags } from './prompts.js'; +import { buildPrompt, iteratePrompt, operationPrompt, personaHandles, questionPrompt, templateIteratePrompt, userHandleFields, verifyArgs } from './prompts.js'; import type { ParsedTurn } from './transcript.js'; // ── What the orchestrator needs from the outside world ────────────────────── @@ -113,7 +113,12 @@ export interface OperationResult { export interface RunHalf { personas: Array>; - seed: { inserted: number; refused: Array<{ object: string; id: string; code?: string; message?: string }> }; + seed: { + inserted: number; + refused: Array<{ object: string; id: string; code?: string; message?: string }>; + /** Handles a user-handle field named that no persona holds, each loaded as the admin user (`u_tia`). */ + handlesAsAdmin?: string[]; + }; questions: QuestionResult[]; operations: OperationResult[]; error?: string; @@ -276,7 +281,7 @@ async function buildHalf( } half.commands.push(await ops.os(dir, ['validate'])); half.commands.push(await ops.os(dir, ['build'])); - half.commands.push(await ops.os(dir, ['verify', ...verifyFlags(scenario.card)])); + half.commands.push(await ops.os(dir, verifyArgs(scenario.card))); // the invocation every authoring prompt names const artifact = ops.readArtifact(dir); if (artifact === undefined) { half.error = half.error ?? 'no dist/objectstack.json after `objectstack build`'; @@ -311,16 +316,31 @@ export async function runHalf( } const adminKey = await app.adminApiKey(); - // Seed, with row ownership mapped from the persona handles. + // Seed, with row ownership mapped from the persona handles. A row whose + // direct scope field names a persona's handle is owned (`owner_id`, the + // platform's record owner) by that persona's user. A user-handle field + // (`account.owner`, `userHandleFields`) is itself loaded with a user id — + // the persona's, or the admin's for a handle no persona holds (`u_tia`) — + // because a faithful build models it as a user lookup, which refuses the + // handle text. Any other scope field names a row or an option and is sent + // verbatim. const handles = personaHandles(card); const ownerFields = new Set(card.run.personas.flatMap((p) => (p.scope ?? []).filter((s) => !s.field.includes('.')).map((s) => `${s.object}.${s.field}`))); + const userFields = userHandleFields(card); for (const object of seedOrder(scenario)) { for (const row of card.run.seed[object] ?? []) { const body: Record = { ...row }; for (const [k, v] of Object.entries(row)) { - const persona = typeof v === 'string' && ownerFields.has(`${object}.${k}`) ? handles.get(v) : undefined; + if (typeof v !== 'string' || !ownerFields.has(`${object}.${k}`)) continue; + const persona = handles.get(v); const acct = persona ? accounts.find((a) => a.persona === persona) : undefined; if (acct) body.owner_id = acct.userId; + if (userFields.has(`${object}.${k}`)) { + const userId = acct?.userId ?? app.adminUserId(); + body[k] = userId; + body.owner_id = userId; + if (!acct) half.seed.handlesAsAdmin = [...new Set([...(half.seed.handlesAsAdmin ?? []), v])].sort(); + } } const r = await app.insertRow(object, body); if (r.ok) half.seed.inserted++; diff --git a/packages/qa/scenarios/src/runner/prompts.ts b/packages/qa/scenarios/src/runner/prompts.ts index 82a7a680cb7..0eb5129d62a 100644 --- a/packages/qa/scenarios/src/runner/prompts.ts +++ b/packages/qa/scenarios/src/runner/prompts.ts @@ -50,6 +50,43 @@ export function personaHandles(card: Card): Map { return out; } +/** + * The scope fields whose seed values are user handles, `object.field`: a + * field a persona's direct (non-path) `scope` rule names that the card does + * not declare as a lookup or a select, and that is not `id`. In the crm card + * that is `account.owner` (`u_sam`, `u_tia`). Every other direct scope field + * holds a row id or an option value (`asset.custodian` = `e06`, an employee + * row; `ticket_comment.visibility` = `public`), which the seed loads verbatim. + */ +export function userHandleFields(card: Card): Set { + const declared = new Set([ + ...(card.expect.lookups ?? []).map((l) => `${l.object}.${l.field}`), + ...(card.expect.selectFields ?? []).map((s) => `${s.object}.${s.field}`), + ]); + const out = new Set(); + for (const p of card.run.personas) { + for (const sc of p.scope ?? []) { + const key = `${sc.object}.${sc.field}`; + if (!sc.field.includes('.') && sc.field !== 'id' && !declared.has(key)) out.add(key); + } + } + return out; +} + +/** The handles a seed row names in a {@link userHandleFields} field that no persona signs in as (`u_tia`): the seed loads each as the admin user. */ +export function unheldHandles(card: Card): string[] { + const held = personaHandles(card); + const out = new Set(); + for (const key of userHandleFields(card)) { + const [object, field] = key.split('.') as [string, string]; + for (const row of card.run.seed[object] ?? []) { + const v = row[field]; + if (typeof v === 'string' && !held.has(v)) out.add(v); + } + } + return [...out].sort(); +} + /** The every-item-prefixed rule, in every authoring prompt. */ export function namingRule(card: Card): string { const ns = scenarioNamespace(card); @@ -102,6 +139,15 @@ export function dataContract(card: Card): string { `Row ownership: a row whose ${[...new Set(card.run.personas.flatMap((p) => (p.scope ?? []).filter((s) => !s.field.includes('.')).map((s) => `\`${s.field}\``)))].join(' / ')} names a handle ` + `(${[...handles].map(([h, id]) => `\`${h}\` = \`${id}\``).join(', ')}) is loaded as owned by that person’s user.`, ); + const userFields = [...userHandleFields(card)]; + if (userFields.length) { + const unheld = unheldHandles(card); + lines.push( + `In ${userFields.map((k) => `\`${k}\``).join(' / ')} the handle stands for a user: the runner loads that field with the person’s user id, ` + + 'not the handle text, so a rule on it matches that user' + + (unheld.length ? `; a handle no person below holds (${unheld.map((h) => `\`${h}\``).join(', ')}) is loaded as the admin user.` : '.'), + ); + } } if (card.run.agentActions?.length) { lines.push(''); @@ -111,19 +157,35 @@ export function dataContract(card: Card): string { return lines.join('\n'); } -const BUILD_PREAMBLE = - 'You are building an ObjectStack app in this project (scaffolded by `create-objectstack`; follow the project’s AGENTS.md ' + - 'and its objectstack skills). Author the metadata, then run `npx objectstack validate` and `npx objectstack build` ' + - 'and fix what they report until both pass. Work without asking questions.'; +/** + * The `objectstack verify` arguments the runner scores a build with + * (`buildHalf` in `orchestrate.ts` runs exactly these), so every authoring + * loop can be told to run the same proof before it stops. + */ +export function verifyArgs(card: Card): string[] { + return ['verify', ...verifyFlags(card)]; +} + +/** The three commands a scratch build is scored on, as the agent runs them: `validate`, `build`, then {@link verifyArgs}. */ +function authoringChecks(card: Card): string { + return `run \`npx objectstack validate\`, \`npx objectstack build\` and \`npx objectstack ${verifyArgs(card).join(' ')}\` and fix what they report until all three pass`; +} + +function buildPreamble(card: Card): string { + return ( + 'You are building an ObjectStack app in this project (scaffolded by `create-objectstack`; follow the project’s AGENTS.md ' + + `and its objectstack skills). Author the metadata, then ${authoringChecks(card)}. Work without asking questions.` + ); +} export function buildPrompt(card: Card): string { - return [BUILD_PREAMBLE, '', '## The app', '', text(card.prompt.detailed), '', dataContract(card)].join('\n'); + return [buildPreamble(card), '', '## The app', '', text(card.prompt.detailed), '', dataContract(card)].join('\n'); } export function iteratePrompt(card: Card): string { return [ - 'This project holds a working ObjectStack app. Apply the change below, then run `npx objectstack validate` and ' + - '`npx objectstack build` and fix what they report until both pass. Keep everything else working. Work without asking questions.', + `This project holds a working ObjectStack app. Apply the change below, then ${authoringChecks(card)}. ` + + 'Keep everything else working. Work without asking questions.', '', namingRule(card), '', diff --git a/packages/qa/scenarios/test/runner/orchestrate.test.ts b/packages/qa/scenarios/test/runner/orchestrate.test.ts index 314aef1bdd3..2b01f688813 100644 --- a/packages/qa/scenarios/test/runner/orchestrate.test.ts +++ b/packages/qa/scenarios/test/runner/orchestrate.test.ts @@ -221,15 +221,18 @@ class ScriptedDriver implements AgentDriver { } } -function setup(opts: { degradedBuild?: boolean; leak?: { question: string; persona: string }; bareItems?: boolean; declares?: string } = {}) { +function setup( + opts: { degradedBuild?: boolean; leak?: { question: string; persona: string }; bareItems?: boolean; declares?: string; app?: () => A } = {}, +) { const scenario = load(); - let current = new FakeApp(); - const apps: FakeApp[] = []; + const fresh = opts.app ?? (() => new FakeApp() as A); + let current = fresh(); + const apps: A[] = []; const getApp = () => current; const ops = fakeOps(getApp, opts); const boot = ops.boot.bind(ops); ops.boot = async (dir) => { - current = new FakeApp(); + current = fresh(); apps.push(current); return boot(dir); }; @@ -273,12 +276,59 @@ describe('runCard — the crm card, both modes, a clean agent', () => { expect(renderScorecard(sc)).toContain('| Run-问数 | 9/9 | — |'); }); - it('seeds ownership from the persona handles: Acme and Bitworks are owned by the sdr user', async () => { + it('seeds ownership from the persona handles: Acme and Bitworks are owned by the sdr user, Shopwell by the admin', async () => { const { scenario, ops, driver, apps } = setup(); await runCard(scenario, driver, ops, { modes: ['scratch'], admin: ADMIN, iterateRun: false }); const accounts = apps[0]!.rows.democrm_account!; expect(accounts.filter((r) => r.owner_id === 'u-sdr').map((r) => r.id)).toEqual(['ac1', 'ac3']); - expect(accounts.find((r) => r.id === 'ac2')!.owner_id).toBeUndefined(); + expect(accounts.find((r) => r.id === 'ac2')!.owner_id).toBe('u-admin'); + }); +}); + +/** + * The data door of a faithful build: `account.owner` is a user lookup and + * `contact.account` a lookup to account, and the door refuses a value naming + * no such record, as the first live run's app did (`VALIDATION_FAILED · + * Account Owner: no sys_user record has id "u_sam"`). Built from the card in + * the repo, not from that run's recording, which this container does not hold. + */ +class UserLookupApp extends FakeApp { + readonly users = new Set(['u-admin']); + override async createPersona(persona: string, permissionSet: string): Promise { + const acct = await super.createPersona(persona, permissionSet); + this.users.add(acct.userId); + return acct; + } + override async insertRow(object: string, row: Record): Promise { + const refuse = (message: string): HttpAnswer => ({ ok: false, status: 400, body: { error: message, code: 'VALIDATION_FAILED' }, code: 'VALIDATION_FAILED', message }); + for (const userField of ['owner', 'owner_id']) { + const v = row[userField]; + if (object === 'democrm_account' && v !== undefined && !this.users.has(String(v))) return refuse(`Account Owner: no sys_user record has id "${String(v)}"`); + } + if (object === 'democrm_contact' && row.account !== null && !(this.rows.democrm_account ?? []).some((a) => a.id === row.account)) { + return refuse(`Account: no democrm_account record has id "${String(row.account)}"`); + } + return super.insertRow(object, row); + } +} + +describe('runCard — the seed against a build whose owner field is a user lookup', () => { + it('the double refuses the handle the old seed sent, so it can fail', async () => { + const app = new UserLookupApp(); + expect(await app.insertRow('democrm_account', { id: 'ac1', name: 'Acme', industry: 'tech', owner: 'u_sam', owner_id: 'u-admin' })).toMatchObject({ ok: false, code: 'VALIDATION_FAILED' }); + expect(await app.insertRow('democrm_contact', { id: 'ct1', account: 'ac1' })).toMatchObject({ ok: false, code: 'VALIDATION_FAILED' }); + }); + + it('loads all 8 rows: the owner field carries the persona user id, and a handle no persona holds is the admin', async () => { + const { scenario, ops, driver, apps } = setup({ app: () => new UserLookupApp() }); + const run = await runCard(scenario, driver, ops, { modes: ['scratch'], admin: ADMIN, iterateRun: false }); + const seed = run.modes[0]!.run!.seed; + expect(seed).toEqual({ inserted: 8, refused: [], handlesAsAdmin: ['u_tia'] }); + const owners = Object.fromEntries(apps[0]!.rows.democrm_account!.map((r) => [r.id, [r.owner, r.owner_id]])); + expect(owners).toEqual({ ac1: ['u-sdr', 'u-sdr'], ac2: ['u-admin', 'u-admin'], ac3: ['u-sdr', 'u-sdr'] }); + expect(apps[0]!.rows.democrm_contact!.map((r) => r.id)).toEqual(['ct1', 'ct2', 'ct3', 'ct4', 'ct5']); + expect(apps[0]!.rows.democrm_contact!.find((r) => r.id === 'ct1')!.owner_id).toBeUndefined(); // a path scope (`account.owner`) owns nothing + expect(cardScorecard(run).modes[0]!.first.operations).toEqual({ passed: 5, total: 5 }); // every write found its seeded target }); }); diff --git a/packages/qa/scenarios/test/runner/units.test.ts b/packages/qa/scenarios/test/runner/units.test.ts index 00149639175..73a8697b5f9 100644 --- a/packages/qa/scenarios/test/runner/units.test.ts +++ b/packages/qa/scenarios/test/runner/units.test.ts @@ -24,7 +24,7 @@ import { import { asksToConfirm, countWrites, noWriteAttempted, observedAnswer, observeOperation, uncodedToolError } from '../../src/runner/observe.js'; import { citationLabels, namespaceOfConfig, namespacedScenario, permissionSetName, templateHostName } from '../../src/runner/namespace.js'; import { changeSize, hiddenValues, installableManifest, seedOrder, withoutItems } from '../../src/runner/orchestrate.js'; -import { buildPrompt, dataContract, iteratePrompt, questionPrompt, templateIteratePrompt, verifyFlags } from '../../src/runner/prompts.js'; +import { buildPrompt, dataContract, iteratePrompt, questionPrompt, templateIteratePrompt, unheldHandles, userHandleFields, verifyArgs, verifyFlags } from '../../src/runner/prompts.js'; import { parseStreamJson, refusalCodeOf, toolNameOf, type ParsedTurn } from '../../src/runner/transcript.js'; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -217,6 +217,27 @@ describe('prompts, seed order, template manifest and change size', () => { expect(contract).not.toMatch(/named exactly by the id/); }); + it('both authoring loops name the verify invocation the runner scores with, next to validate and build', () => { + const card = namespacedScenario(crm).card; + expect(verifyArgs(card)).toEqual(['verify', '--reference-shapes']); + for (const p of [buildPrompt(card), iteratePrompt(card)]) { + expect(p).toContain('run `npx objectstack validate`, `npx objectstack build` and `npx objectstack verify --reference-shapes` and fix what they report until all three pass'); + } + }); + + it('a user-handle field is a scope field the card declares as neither lookup nor select; the contract says it is loaded with a user id', () => { + const card = namespacedScenario(crm).card; + expect([...userHandleFields(card)]).toEqual(['democrm_account.owner']); + expect(unheldHandles(card)).toEqual(['u_tia']); + expect(dataContract(card)).toContain( + 'In `democrm_account.owner` the handle stands for a user: the runner loads that field with the person’s user id, not the handle text, ' + + 'so a rule on it matches that user; a handle no person below holds (`u_tia`) is loaded as the admin user.', + ); + // A scope field that names a row (`ticket.assignee` → agent, `customer.id`) or an option (`ticket_comment.visibility`) is not one. + const desk = loadScenarioFile(resolve(HERE, '../../../../../docs/qa/scenarios/service_desk.json'), { areas: readCapabilityAreas(AREAS_DIR) }).card!; + expect([...userHandleFields(desk.card)]).toEqual([]); + }); + it('chooses verify proofs from the tags, and seeds a lookup target before its referrer', () => { expect(verifyFlags(crm.card)).toEqual(['--reference-shapes']); expect(seedOrder(crm)).toEqual(['account', 'contact']); From c6fee128ee40e7793be6c98212ef074e7dbd4efb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 09:31:50 +0000 Subject: [PATCH 2/3] test(scenarios): the user-lookup seed pin counts the sdr's added contact Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2ebbZkr2QAEG1x7UC8 --- packages/qa/scenarios/test/runner/orchestrate.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/qa/scenarios/test/runner/orchestrate.test.ts b/packages/qa/scenarios/test/runner/orchestrate.test.ts index 2b01f688813..ba1d1187fed 100644 --- a/packages/qa/scenarios/test/runner/orchestrate.test.ts +++ b/packages/qa/scenarios/test/runner/orchestrate.test.ts @@ -326,7 +326,7 @@ describe('runCard — the seed against a build whose owner field is a user looku expect(seed).toEqual({ inserted: 8, refused: [], handlesAsAdmin: ['u_tia'] }); const owners = Object.fromEntries(apps[0]!.rows.democrm_account!.map((r) => [r.id, [r.owner, r.owner_id]])); expect(owners).toEqual({ ac1: ['u-sdr', 'u-sdr'], ac2: ['u-admin', 'u-admin'], ac3: ['u-sdr', 'u-sdr'] }); - expect(apps[0]!.rows.democrm_contact!.map((r) => r.id)).toEqual(['ct1', 'ct2', 'ct3', 'ct4', 'ct5']); + expect(apps[0]!.rows.democrm_contact!.map((r) => r.id)).toEqual(['ct1', 'ct2', 'ct3', 'ct4', 'ct5', 'new-add_contact']); // the five seeded, then the sdr's add expect(apps[0]!.rows.democrm_contact!.find((r) => r.id === 'ct1')!.owner_id).toBeUndefined(); // a path scope (`account.owner`) owns nothing expect(cardScorecard(run).modes[0]!.first.operations).toEqual({ passed: 5, total: 5 }); // every write found its seeded target }); From b1f06284dc614f299377ceb493c2cb7a6bb4ed0e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 10:14:50 +0000 Subject: [PATCH 3/3] fix(scenarios): a refused seed row makes what reads it not measured, and the scorecard says so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A refused seed row and, through the card's declared lookups, every seed row under it are out. A question on an object holding such a row, and an operation whose target, lookup values or post-state name one, are not asked: each is recorded with the reason, and counts in neither passed nor total. The scorecard gains a Seed row (`7/8 ⚠ seed: 1 refused`), the ask and operation cells name how many were not measured, and a half with nothing measured stays `not-run` in the record. The build prompt's ownership paragraph now states the seed contract exactly: the handle field and `owner_id` both carry the user's id, the persona's user for a held handle and the admin the runner signs in as for an unheld one. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2ebbZkr2QAEG1x7UC8 --- .../qa/scenarios/src/runner/orchestrate.ts | 88 +++++++++++++++++++ packages/qa/scenarios/src/runner/prompts.ts | 13 ++- packages/qa/scenarios/src/runner/scorecard.ts | 37 ++++++-- .../scenarios/test/runner/orchestrate.test.ts | 76 ++++++++++++++-- .../qa/scenarios/test/runner/units.test.ts | 23 ++++- 5 files changed, 214 insertions(+), 23 deletions(-) diff --git a/packages/qa/scenarios/src/runner/orchestrate.ts b/packages/qa/scenarios/src/runner/orchestrate.ts index fc77ede74c1..3e1ac32b479 100644 --- a/packages/qa/scenarios/src/runner/orchestrate.ts +++ b/packages/qa/scenarios/src/runner/orchestrate.ts @@ -97,6 +97,8 @@ export interface BuildHalf { export interface QuestionResult { question: string; persona: string | null; + /** Why this turn was not asked: it reads a row the seed could not load ({@link seedDependence}). Not measured, never a fail. */ + notMeasured?: string; grade?: PersonaRunGrade; turn?: TurnSummary; error?: string; @@ -105,6 +107,8 @@ export interface QuestionResult { export interface OperationResult { operation: string; persona: string; + /** Why this operation was not run: it acts on a row the seed could not load ({@link seedDependence}). Not measured, never a fail. */ + notMeasured?: string; observation?: OperationObservation; grade?: OperationGrade; turn?: TurnSummary; @@ -241,6 +245,78 @@ export function hiddenValues(scenario: LoadedScenario, personaId: string): Array return [...out]; } +type Card = LoadedScenario['card']; +type Question = Card['run']['questions'][number]; +type Operation = Card['run']['operations'][number]; + +/** + * What a refused seed row takes out of the Run half, read from what the card + * names. The rows out are the refused ones plus, transitively, every seed row + * whose declared lookup (`expect.lookups`) names a row that is out — a contact + * of a refused account, whether or not the app refused it too. Then: + * + * - a question is not measured when its `object` has a row that is out: its + * truth was computed over rows the app does not hold; + * - an operation is not measured when its `target`, a lookup value it writes, + * or a post-state row (`id`, or a lookup value in its `values`) names a row + * that is out. + * + * Everything else grades as usual. A not-measured turn is not asked and counts + * in neither passed nor total. + */ +export function seedDependence( + card: Card, + refused: ReadonlyArray<{ object: string; id: string }>, +): { out: Map; question(q: Question): string | undefined; operation(op: Operation): string | undefined } { + /** `object/id` → the refused row it hangs off, as `object id`. */ + const out = new Map(refused.map((r) => [`${r.object}/${r.id}`, `${r.object} ${r.id}`])); + const lookups = card.expect.lookups ?? []; + for (let grew = true; grew; ) { + grew = false; + for (const l of lookups) { + for (const row of card.run.seed[l.object] ?? []) { + const parent = out.get(`${l.to}/${String(row[l.field])}`); + if (parent !== undefined && !out.has(`${l.object}/${row.id}`)) { + out.set(`${l.object}/${row.id}`, parent); + grew = true; + } + } + } + } + const why = (object: string, id: unknown): string | undefined => { + const root = typeof id === 'string' ? out.get(`${object}/${id}`) : undefined; + if (root === undefined) return undefined; + return root === `${object} ${String(id)}` ? `seed: needs ${root}, which the app refused` : `seed: needs ${object} ${String(id)}, a child of ${root}, which the app refused`; + }; + const viaLookups = (object: string, values: Record | undefined): string | undefined => { + for (const l of lookups) { + const w = l.object === object && values && l.field in values ? why(l.to, values[l.field]) : undefined; + if (w) return w; + } + return undefined; + }; + return { + out, + question: (q) => { + for (const row of card.run.seed[q.object] ?? []) { + const w = why(q.object, row.id); + if (w) return w; + } + return undefined; + }, + operation: (op) => { + const direct = (op.target !== undefined ? why(op.object, op.target) : undefined) ?? viaLookups(op.object, op.values); + if (direct) return direct; + if (op.outcome.kind !== 'applied') return undefined; + for (const st of op.outcome.state) { + const w = (st.id !== undefined ? why(st.object, st.id) : undefined) ?? viaLookups(st.object, st.values); + if (w) return w; + } + return undefined; + }, + }; +} + /** The persona (not a card artifact) a scenario's audit trail names, from the user id the app recorded. */ function personaOfUser(userId: string | null, accounts: readonly PersonaAccount[], adminId: string): string | null { if (userId === null) return null; @@ -349,11 +425,18 @@ export async function runHalf( } const keyOf = (persona: string | null) => (persona === null ? adminKey : accounts.find((a) => a.persona === persona)!.apiKey); + // A refused seed row makes what reads it not measured — never a fail of the app's answer. + const dependence = seedDependence(card, half.seed.refused); for (const q of card.run.questions) { + const notMeasured = dependence.question(q); for (const persona of [null, ...(q.personas ?? [])]) { if (persona === null && q.personas?.length) continue; // a persona-scoped question is asked as its personas const res: QuestionResult = { question: q.id, persona }; + if (notMeasured) { + half.questions.push({ ...res, notMeasured }); + continue; + } try { const turn = await driver.ask({ ...ref, step: q.id, ...(persona ? { persona } : {}), prompt: questionPrompt(scenario, q.id), mcp: { url: app.mcpUrl, apiKey: keyOf(persona) } }); res.turn = summarize(turn); @@ -372,6 +455,11 @@ export async function runHalf( for (const op of card.run.operations) { const res: OperationResult = { operation: op.id, persona: op.actingAs }; + const notMeasured = dependence.operation(op); + if (notMeasured) { + half.operations.push({ ...res, notMeasured }); + continue; + } try { const before = (await app.listRows(op.object)).rows as Row[]; const turn = await driver.ask({ ...ref, step: op.id, persona: op.actingAs, prompt: operationPrompt(card, op.id), mcp: { url: app.mcpUrl, apiKey: keyOf(op.actingAs) } }); diff --git a/packages/qa/scenarios/src/runner/prompts.ts b/packages/qa/scenarios/src/runner/prompts.ts index 0eb5129d62a..c317bb1746d 100644 --- a/packages/qa/scenarios/src/runner/prompts.ts +++ b/packages/qa/scenarios/src/runner/prompts.ts @@ -142,10 +142,17 @@ export function dataContract(card: Card): string { const userFields = [...userHandleFields(card)]; if (userFields.length) { const unheld = unheldHandles(card); + const held = [ + ...new Set( + card.run.personas.flatMap((p) => (p.scope ?? []).filter((sc) => userHandleFields(card).has(`${sc.object}.${sc.field}`)).map((sc) => `\`${String(sc.value)}\` → the \`${p.id}\` user`)), + ), + ]; lines.push( - `In ${userFields.map((k) => `\`${k}\``).join(' / ')} the handle stands for a user: the runner loads that field with the person’s user id, ` + - 'not the handle text, so a rule on it matches that user' + - (unheld.length ? `; a handle no person below holds (${unheld.map((h) => `\`${h}\``).join(', ')}) is loaded as the admin user.` : '.'), + `In ${userFields.map((k) => `\`${k}\``).join(' / ')} the value is a handle that stands for a user, and the handle text is never stored: ` + + `the runner loads that field, and the row’s owner (\`owner_id\`), with that user’s id (${[ + ...held, + ...(unheld.length ? [`${unheld.map((h) => `\`${h}\``).join(', ')}, which no person below holds → the admin user the runner signs in as`] : []), + ].join('; ')}). Make that field accept a user id; a rule on it matches that user.`, ); } } diff --git a/packages/qa/scenarios/src/runner/scorecard.ts b/packages/qa/scenarios/src/runner/scorecard.ts index d91d96edb37..7b6c35d5bdd 100644 --- a/packages/qa/scenarios/src/runner/scorecard.ts +++ b/packages/qa/scenarios/src/runner/scorecard.ts @@ -17,6 +17,11 @@ * leaking answer shows up even when it is otherwise right. * - Run-operations: one per operation; it passes when its grade passes. * - A turn that errored counts in the total and not in passed. + * - Seed: rows loaded of rows sent. A refused row is shown as + * `⚠ seed: N refused`, and every question turn and operation that depends + * on it (`seedDependence` in `orchestrate.ts`) is NOT MEASURED: it counts in + * neither passed nor total, and the cell says how many were left out. A + * half with nothing measured is `not-run` in the record, never `fail`. */ import type { CardRun, ModeRun, RunHalf, BuildHalf } from './orchestrate.js'; @@ -29,8 +34,10 @@ export interface Tally { export interface HalfScores { build?: Tally & { degraded: boolean; failedCommands: string[] }; - ask?: Tally & { leaks: number }; - operations?: Tally; + /** Rows the app took of the rows the runner sent. */ + seed?: { inserted: number; total: number; refused: number }; + ask?: Tally & { leaks: number; notMeasured: number }; + operations?: Tally & { notMeasured: number }; } export interface ModeScorecard { @@ -58,21 +65,28 @@ export function buildTally(b: BuildHalf | undefined, expectCount: number): HalfS return { passed, total, degraded: !b.score || failedCommands.length > 0 || passed < total, failedCommands }; } +export function seedTally(r: RunHalf | undefined): HalfScores['seed'] | undefined { + if (!r) return undefined; + return { inserted: r.seed.inserted, total: r.seed.inserted + r.seed.refused.length, refused: r.seed.refused.length }; +} + export function askTally(r: RunHalf | undefined): HalfScores['ask'] | undefined { if (!r) return undefined; let passed = 0; let leaks = 0; - for (const q of r.questions) { + const measured = r.questions.filter((q) => !q.notMeasured); + for (const q of measured) { const g = q.grade; if (g && g.leaks.length) leaks++; if (g && g.correct && g.cited && g.leaks.length === 0) passed++; } - return { passed, total: r.questions.length, leaks }; + return { passed, total: measured.length, leaks, notMeasured: r.questions.length - measured.length }; } -export function opsTally(r: RunHalf | undefined): Tally | undefined { +export function opsTally(r: RunHalf | undefined): HalfScores['operations'] | undefined { if (!r) return undefined; - return { passed: r.operations.filter((o) => o.grade?.pass).length, total: r.operations.length }; + const measured = r.operations.filter((o) => !o.notMeasured); + return { passed: measured.filter((o) => o.grade?.pass).length, total: measured.length, notMeasured: r.operations.length - measured.length }; } /** An artifact-less build scores 0 of 1 — a degraded build is never "nothing to score". */ @@ -87,6 +101,8 @@ export function modeScorecard(m: ModeRun): ModeScorecard { const first: HalfScores = {}; const b = buildTally(m.build, NO_ARTIFACT_TOTAL); if (b) first.build = b; + const sd = seedTally(m.run); + if (sd) first.seed = sd; const a = askTally(m.run); if (a) first.ask = a; const o = opsTally(m.run); @@ -102,6 +118,8 @@ export function modeScorecard(m: ModeRun): ModeScorecard { const it: HalfScores & { changeSize?: number } = {}; const ib = buildTally(m.iterate.build, NO_ARTIFACT_TOTAL); if (ib) it.build = ib; + const isd = seedTally(m.iterate.run); + if (isd) it.seed = isd; const ia = askTally(m.iterate.run); if (ia) it.ask = ia; const io = opsTally(m.iterate.run); @@ -116,7 +134,8 @@ export function cardScorecard(run: CardRun): CardScorecard { return { card: run.card, counts: run.counts, modes: run.modes.map((m) => modeScorecard(m)) }; } -const cell = (t: Tally | undefined) => (t ? `${t.passed}/${t.total}` : '—'); +const cell = (t: (Tally & { notMeasured?: number }) | undefined) => (t ? `${t.passed}/${t.total}${t.notMeasured ? ` · ${t.notMeasured} not measured` : ''}` : '—'); +const seedCell = (s: HalfScores['seed']) => (s ? `${s.inserted}/${s.total}${s.refused ? ` ⚠ seed: ${s.refused} refused` : ''}` : '—'); /** The scorecard as a markdown table: the modes side by side, one row per half. */ export function renderScorecard(sc: CardScorecard): string { @@ -128,9 +147,11 @@ export function renderScorecard(sc: CardScorecard): string { ]; const row = (label: string, f: (m: ModeScorecard) => string) => lines.push(`| ${label} | ${sc.modes.map((m) => (m.skipped ? `skipped` : f(m))).join(' | ')} |`); row('Build', (m) => (m.first.build ? `${cell(m.first.build)}${m.first.build.degraded ? ' ⚠ degraded' : ''}${m.first.build.failedCommands.length ? ` (failed: ${m.first.build.failedCommands.join(', ')})` : ''}` : '—')); + row('Seed', (m) => seedCell(m.first.seed)); row('Run-问数', (m) => (m.first.ask ? `${cell(m.first.ask)}${m.first.ask.leaks ? ` ⚠ ${m.first.ask.leaks} leaking` : ''}` : '—')); row('Run-operations', (m) => cell(m.first.operations)); row('Iterate · Build', (m) => (m.iterate?.build ? `${cell(m.iterate.build)}${m.iterate.build.degraded ? ' ⚠ degraded' : ''}` : '—')); + row('Iterate · Seed', (m) => seedCell(m.iterate?.seed)); row('Iterate · Run-问数', (m) => (m.iterate?.ask ? `${cell(m.iterate.ask)}${m.iterate.ask.leaks ? ` ⚠ ${m.iterate.ask.leaks} leaking` : ''}` : '—')); row('Iterate · Run-operations', (m) => cell(m.iterate?.operations)); row('AI change size (items)', (m) => (m.iterate?.changeSize !== undefined ? String(m.iterate.changeSize) : '—')); @@ -181,7 +202,7 @@ export function runRecord(meta: RunRecordMeta, cardRun: CardRun): Record ({ clause: i, evidence: e })), issues: [], - notes: [tally ? `${tally.passed}/${tally.total}` : '', error ?? '', m.skipped ?? ''].filter(Boolean).join(' · '), + notes: [tally ? cell(tally) : '', error ?? '', m.skipped ?? ''].filter(Boolean).join(' · '), }); } } diff --git a/packages/qa/scenarios/test/runner/orchestrate.test.ts b/packages/qa/scenarios/test/runner/orchestrate.test.ts index ba1d1187fed..16ad9f8f2a5 100644 --- a/packages/qa/scenarios/test/runner/orchestrate.test.ts +++ b/packages/qa/scenarios/test/runner/orchestrate.test.ts @@ -252,17 +252,17 @@ describe('runCard — the crm card, both modes, a clean agent', () => { const [scratch, template] = sc.modes; expect(scratch!.first.build).toMatchObject({ degraded: false, failedCommands: [] }); expect(scratch!.first.build!.passed).toBe(scratch!.first.build!.total); - expect(scratch!.first.ask).toEqual({ passed: 9, total: 9, leaks: 0 }); - expect(scratch!.first.operations).toEqual({ passed: 5, total: 5 }); + expect(scratch!.first.ask).toEqual({ passed: 9, total: 9, leaks: 0, notMeasured: 0 }); + expect(scratch!.first.operations).toEqual({ passed: 5, total: 5, notMeasured: 0 }); expect(scratch!.iterate!.build!.degraded).toBe(false); expect(scratch!.iterate!.changeSize).toBe(1); // the opportunity object - expect(scratch!.iterate!.ask).toEqual({ passed: 9, total: 9, leaks: 0 }); + expect(scratch!.iterate!.ask).toEqual({ passed: 9, total: 9, leaks: 0, notMeasured: 0 }); expect(template!.skipped).toBeUndefined(); expect(run.modes[1]!.template).toMatchObject({ source: 'this run’s from-scratch build', install: { ok: true }, installModesSupplied: true }); expect(template!.iterate!.changeSize).toBe(1); - expect(template!.iterate!.ask).toEqual({ passed: 9, total: 9, leaks: 0 }); - expect(template!.iterate!.operations).toEqual({ passed: 5, total: 5 }); + expect(template!.iterate!.ask).toEqual({ passed: 9, total: 9, leaks: 0, notMeasured: 0 }); + expect(template!.iterate!.operations).toEqual({ passed: 5, total: 5, notMeasured: 0 }); // Every step went through the driver under its own replay key. expect(driver.seen).toContain('crm_account_with_contacts/scratch/run/account_count/sdr'); @@ -274,6 +274,7 @@ describe('runCard — the crm card, both modes, a clean agent', () => { expect(results.find((r) => r.id === 'crm_account_with_contacts.scratch.run-operations')!.verdict).toBe('pass'); expect(results.find((r) => r.id === 'crm_account_with_contacts.template.build')!.verdict).toBe('not-run'); // template mode builds nothing first expect(renderScorecard(sc)).toContain('| Run-问数 | 9/9 | — |'); + expect(renderScorecard(sc)).toContain('| Seed | 8/8 | — |'); // every row loaded: no warning }); it('seeds ownership from the persona handles: Acme and Bitworks are owned by the sdr user, Shopwell by the admin', async () => { @@ -328,7 +329,7 @@ describe('runCard — the seed against a build whose owner field is a user looku expect(owners).toEqual({ ac1: ['u-sdr', 'u-sdr'], ac2: ['u-admin', 'u-admin'], ac3: ['u-sdr', 'u-sdr'] }); expect(apps[0]!.rows.democrm_contact!.map((r) => r.id)).toEqual(['ct1', 'ct2', 'ct3', 'ct4', 'ct5', 'new-add_contact']); // the five seeded, then the sdr's add expect(apps[0]!.rows.democrm_contact!.find((r) => r.id === 'ct1')!.owner_id).toBeUndefined(); // a path scope (`account.owner`) owns nothing - expect(cardScorecard(run).modes[0]!.first.operations).toEqual({ passed: 5, total: 5 }); // every write found its seeded target + expect(cardScorecard(run).modes[0]!.first.operations).toEqual({ passed: 5, total: 5, notMeasured: 0 }); // every write found its seeded target }); }); @@ -343,8 +344,8 @@ describe('runCard — the card’s own namespace, demo + code', () => { expect(run.modes[0]!.run!.personas.map((p) => p.permissionSet.name)).toEqual(['democrm_sdr', 'democrm_partner_user']); expect(sc.modes[0]!.first.build!.degraded).toBe(false); expect(run.modes[0]!.build!.score!.naming).toEqual([]); - expect(sc.modes[0]!.first.ask).toEqual({ passed: 9, total: 9, leaks: 0 }); - expect(sc.modes[0]!.first.operations).toEqual({ passed: 5, total: 5 }); + expect(sc.modes[0]!.first.ask).toEqual({ passed: 9, total: 9, leaks: 0, notMeasured: 0 }); + expect(sc.modes[0]!.first.operations).toEqual({ passed: 5, total: 5, notMeasured: 0 }); // The installed copy keeps `democrm`; the environment's own unprefixed items (`setup`, `admin_full_access`) are not the app's. expect(apps[1]!.installed!.namespace).toBe('democrm'); expect(sc.modes[1]!.iterate!.build!.degraded).toBe(false); @@ -400,7 +401,7 @@ describe('runCard — the acceptance: a degraded build and a leaking answer each const { scenario, ops, driver } = setup({ leak: { question: 'contacts_per_account', persona: 'sdr' } }); const run = await runCard(scenario, driver, ops, { modes: ['scratch'], admin: ADMIN, iterateRun: false }); const sc = cardScorecard(run); - expect(sc.modes[0]!.first.ask).toEqual({ passed: 8, total: 9, leaks: 1 }); + expect(sc.modes[0]!.first.ask).toEqual({ passed: 8, total: 9, leaks: 1, notMeasured: 0 }); const q = run.modes[0]!.run!.questions.find((x) => x.question === 'contacts_per_account' && x.persona === 'sdr')!; expect(q.grade!.correct).toBe(true); expect(q.grade!.leaks.length).toBeGreaterThan(0); @@ -422,3 +423,60 @@ describe('runCard — the acceptance: a degraded build and a leaking answer each expect(template.iterate).toBeUndefined(); }); }); + +/** A data door that refuses the seed rows it is told to, as a build whose validation rejects them would. */ +class RefusingApp extends FakeApp { + constructor(private readonly refuse: ReadonlySet) { + super(); + } + override async insertRow(object: string, row: Record): Promise { + if (this.refuse.has(`${object}/${String(row.id)}`)) { + return { ok: false, status: 400, body: {}, code: 'VALIDATION_FAILED', message: `${object} ${String(row.id)} refused` }; + } + return super.insertRow(object, row); + } +} + +describe('runCard — a refused seed row makes what reads it not measured, never a fail', () => { + it('shows ⚠ seed: 1 refused, leaves the questions on that object unasked and unscored, and grades the rest as usual', async () => { + const { scenario, ops, driver } = setup({ app: () => new RefusingApp(new Set(['democrm_contact/ct4'])) }); + const run = await runCard(scenario, driver, ops, { modes: ['scratch'], admin: ADMIN, iterateRun: false }); + const half = run.modes[0]!.run!; + expect(half.seed).toMatchObject({ inserted: 7, refused: [{ object: 'democrm_contact', id: 'ct4', code: 'VALIDATION_FAILED' }] }); + + // Every contact question is out (its object lost ct4); the account questions are asked and graded. + const out = half.questions.filter((q) => q.notMeasured).map((q) => `${q.question}/${q.persona ?? '-'}`); + expect(out).toEqual(['contacts_per_account/sdr', 'unattached_contacts/-', 'best_covered_account/-', 'tech_contacts/-', 'contacts_by_industry/-']); + expect(half.questions.find((q) => q.notMeasured)!.notMeasured).toBe('seed: needs democrm_contact ct4, which the app refused'); + expect(half.questions.filter((q) => q.notMeasured).every((q) => q.grade === undefined && q.turn === undefined)).toBe(true); + expect(driver.seen).not.toContain('crm_account_with_contacts/scratch/run/tech_contacts/-'); + expect(half.operations.filter((o) => o.notMeasured)).toEqual([]); // no operation names ct4 + + const sc = cardScorecard(run); + expect(sc.modes[0]!.first.seed).toEqual({ inserted: 7, total: 8, refused: 1 }); + expect(sc.modes[0]!.first.ask).toEqual({ passed: 4, total: 4, leaks: 0, notMeasured: 5 }); + expect(sc.modes[0]!.first.operations).toEqual({ passed: 5, total: 5, notMeasured: 0 }); + const table = renderScorecard(sc); + expect(table).toContain('| Seed | 7/8 ⚠ seed: 1 refused |'); + expect(table).toContain('| Run-问数 | 4/4 · 5 not measured |'); + const record = runRecord({ run: 'r', date: '2026-11-06', driver: driver.info(), runner: 'test', evidenceDir: '/tmp' }, run); + const ask = (record.results as Array<{ id: string; verdict: string; notes: string }>).find((r) => r.id === 'crm_account_with_contacts.scratch.run-ask')!; + expect(ask).toMatchObject({ verdict: 'pass', notes: '4/4 · 5 not measured' }); + }); + + it('an operation whose target hangs off a refused row is not run; a half with nothing measured is not-run, not fail', async () => { + const { scenario, ops, driver } = setup({ app: () => new RefusingApp(new Set(['democrm_account/ac1', 'democrm_account/ac2', 'democrm_account/ac3'])) }); + const run = await runCard(scenario, driver, ops, { modes: ['scratch'], admin: ADMIN, iterateRun: false }); + const half = run.modes[0]!.run!; + expect(half.seed.inserted).toBe(5); // the double does not cascade; the runner still takes the contacts out + expect(half.operations.filter((o) => !o.notMeasured).map((o) => o.operation)).toEqual(['contact_bad_email']); + expect(half.questions.every((q) => q.notMeasured)).toBe(true); + const sc = cardScorecard(run); + expect(sc.modes[0]!.first.ask).toEqual({ passed: 0, total: 0, leaks: 0, notMeasured: 9 }); + expect(renderScorecard(sc)).toContain('| Seed | 5/8 ⚠ seed: 3 refused |'); + const record = runRecord({ run: 'r', date: '2026-11-06', driver: driver.info(), runner: 'test', evidenceDir: '/tmp' }, run); + const verdict = (id: string) => (record.results as Array<{ id: string; verdict: string }>).find((r) => r.id === id)!.verdict; + expect(verdict('crm_account_with_contacts.scratch.run-ask')).toBe('not-run'); + expect(verdict('crm_account_with_contacts.scratch.run-operations')).toBe('pass'); + }); +}); diff --git a/packages/qa/scenarios/test/runner/units.test.ts b/packages/qa/scenarios/test/runner/units.test.ts index 73a8697b5f9..e544af75380 100644 --- a/packages/qa/scenarios/test/runner/units.test.ts +++ b/packages/qa/scenarios/test/runner/units.test.ts @@ -23,7 +23,7 @@ import { } from '../../src/runner/driver.js'; import { asksToConfirm, countWrites, noWriteAttempted, observedAnswer, observeOperation, uncodedToolError } from '../../src/runner/observe.js'; import { citationLabels, namespaceOfConfig, namespacedScenario, permissionSetName, templateHostName } from '../../src/runner/namespace.js'; -import { changeSize, hiddenValues, installableManifest, seedOrder, withoutItems } from '../../src/runner/orchestrate.js'; +import { changeSize, hiddenValues, installableManifest, seedDependence, seedOrder, withoutItems } from '../../src/runner/orchestrate.js'; import { buildPrompt, dataContract, iteratePrompt, questionPrompt, templateIteratePrompt, unheldHandles, userHandleFields, verifyArgs, verifyFlags } from '../../src/runner/prompts.js'; import { parseStreamJson, refusalCodeOf, toolNameOf, type ParsedTurn } from '../../src/runner/transcript.js'; @@ -230,14 +230,31 @@ describe('prompts, seed order, template manifest and change size', () => { expect([...userHandleFields(card)]).toEqual(['democrm_account.owner']); expect(unheldHandles(card)).toEqual(['u_tia']); expect(dataContract(card)).toContain( - 'In `democrm_account.owner` the handle stands for a user: the runner loads that field with the person’s user id, not the handle text, ' + - 'so a rule on it matches that user; a handle no person below holds (`u_tia`) is loaded as the admin user.', + 'In `democrm_account.owner` the value is a handle that stands for a user, and the handle text is never stored: the runner loads that field, ' + + 'and the row’s owner (`owner_id`), with that user’s id (`u_sam` → the `sdr` user; `u_tia`, which no person below holds → the admin user ' + + 'the runner signs in as). Make that field accept a user id; a rule on it matches that user.', ); // A scope field that names a row (`ticket.assignee` → agent, `customer.id`) or an option (`ticket_comment.visibility`) is not one. const desk = loadScenarioFile(resolve(HERE, '../../../../../docs/qa/scenarios/service_desk.json'), { areas: readCapabilityAreas(AREAS_DIR) }).card!; expect([...userHandleFields(desk.card)]).toEqual([]); }); + it('a refused seed row takes out its children and what the card names on them, and nothing else', () => { + const card = namespacedScenario(crm).card; + const dep = seedDependence(card, [{ object: 'democrm_account', id: 'ac1' }]); + expect([...dep.out.keys()].sort()).toEqual(['democrm_account/ac1', 'democrm_contact/ct1', 'democrm_contact/ct2']); + const op = (id: string) => dep.operation(card.run.operations.find((o) => o.id === id)!); + expect(op('partner_edits_account')).toBe('seed: needs democrm_account ac1, which the app refused'); // its target + expect(op('remove_ben')).toBe('seed: needs democrm_contact ct2, a child of democrm_account ac1, which the app refused'); // a child + expect(op('move_eli')).toBe('seed: needs democrm_account ac1, which the app refused'); // the account it writes + expect(op('add_contact')).toBeUndefined(); + expect(op('contact_bad_email')).toBeUndefined(); + const q = (id: string) => dep.question(card.run.questions.find((x) => x.id === id)!); + expect(q('tech_accounts')).toBe('seed: needs democrm_account ac1, which the app refused'); + expect(q('unattached_contacts')).toBe('seed: needs democrm_contact ct1, a child of democrm_account ac1, which the app refused'); + expect(seedDependence(card, []).out.size).toBe(0); + }); + it('chooses verify proofs from the tags, and seeds a lookup target before its referrer', () => { expect(verifyFlags(crm.card)).toEqual(['--reference-shapes']); expect(seedOrder(crm)).toEqual(['account', 'contact']);