From c9e0c3627b2982c6b3fb67edfd39fdac103e0006 Mon Sep 17 00:00:00 2001 From: anandpant <109482096+anandpant@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:25:47 +0000 Subject: [PATCH] test(generation): add structural reliability battery (#314) ## Scope - Register eight reliability scenarios spanning production regressions, interacting loops, near-limit flowcharts, and deep 25-35 topic mindmaps. - Expose them through the eval-harness generation registry. - Add an Effect/jiti endpoint battery that sends fresh generation requests, repeats every scenario at least three times, supports targeted scenario runs, and derives its 302.25-second outer timeout from the complete generation-policy budget plus a 30-second margin. - Keep scenario prompts organic by removing generic self-loop policy text; system prompts and prioritized repair now own that invariant. The manuscript scenario only adds the business requirement that author revision is followed by a separate resubmission process. - Model required path waypoints as alternative-label groups and require every group on the same qualifying path. Expense rejection must cycle through both expense submission and resubmission; manuscript revision must cycle through both author revision and resubmission; manuscript ethics must cycle through both ethics investigation and editorial triage. - Bind every required branch label to an expected source-node label group. In particular, only finance approval/audit can satisfy the expense `approved` terminal assertion; a manager's approved branch cannot. - Stop a required-cycle search the first time it returns to its branch source without collecting every waypoint group, preventing adjacent cycles from being stitched into one false qualifying cycle. - **Extended renderer scope:** add local exterior routing lanes so same-rank branch arrows do not cross nodes in intervening ranks during Excalidraw export. ## Proof - `pnpm test:tools` - 93/93 passed at `fa0d38d`. - `pnpm nx run-many -t typecheck,test,build --skip-nx-cache` - passed at `b4d1487`. - `pnpm nx build-storybook diagram-ui --skip-nx-cache` - passed at `b4d1487`. - Added regressions proving that a cycle missing expense resubmission fails, adjacent short and resubmission cycles cannot be stitched into one qualifying path, an ethics cycle missing Editorial Triage fails, and a manager-sourced `approved` branch cannot satisfy the finance terminal requirement, alongside inverse-semantics, one-cycle/two-decision, timeout-budget, organic-scenario, registry, selection, structural-fidelity, and reproduced ecommerce export-route coverage. - Corrected the structural-fidelity fixture so every padding edge references an existing node while preserving its intended cycle/path/count assertions; refreshed required CI passed at `fa0d38d` in workflow run `33560282033`. - Runtime preview deployment passed at `b4d14878428c` in workflow run `33558720216`; the only subsequent change at `fa0d38d` is the test-fixture correction. - Ecommerce returns 18-step scenario: 8/8 passed (100%) against the real preview with the scenario-level self-loop instruction removed. - Expense resubmission scenario: 8/8 passed (100%) against the real preview. - Full fresh preview battery: 24/24 passed (100%, three runs across all eight scenarios) against `https://sketchi-studio-pr-314.dimethyl.workers.dev/api/v1/generate`. This is PR 6 of 6 in the generation-reliability stack. --- .../eval-harness/src/lib/generate-scenario.ts | 4 +- .../src/server/generation/api.server.test.ts | 44 +- .../src/server/generation/api.server.ts | 19 +- .../src/server/generation/service.server.ts | 14 +- package.json | 1 + .../excalidraw/src/lib/convert.test.ts | 109 ++++ packages/diagram/renderer/src/scene.ts | 51 ++ packages/diagram/scenarios/src/index.ts | 2 + .../scenarios/src/lib/generation-registry.ts | 22 + .../src/lib/generation-reliability.test.ts | 88 +++ .../src/lib/generation-reliability.ts | 261 ++++++++ packages/diagram/scenarios/src/lib/prompt.ts | 21 +- tools/generation-reliability-probe.test.ts | 519 ++++++++++++++++ tools/generation-reliability-probe.ts | 564 ++++++++++++++++++ tools/project-graph.test.ts | 2 + 15 files changed, 1697 insertions(+), 24 deletions(-) create mode 100644 packages/diagram/scenarios/src/lib/generation-registry.ts create mode 100644 packages/diagram/scenarios/src/lib/generation-reliability.test.ts create mode 100644 packages/diagram/scenarios/src/lib/generation-reliability.ts create mode 100644 tools/generation-reliability-probe.test.ts create mode 100644 tools/generation-reliability-probe.ts diff --git a/apps/eval-harness/src/lib/generate-scenario.ts b/apps/eval-harness/src/lib/generate-scenario.ts index 9e5c6d59..90fc929b 100644 --- a/apps/eval-harness/src/lib/generate-scenario.ts +++ b/apps/eval-harness/src/lib/generate-scenario.ts @@ -19,7 +19,7 @@ import { summarizeGenerationCandidate, } from "@sketchi/diagram-generation"; import { - getScenario, + getGenerationScenario, toDiagramGenerationPrompt, } from "@sketchi/diagram-scenarios"; import { @@ -242,7 +242,7 @@ const runClient = Effect.fn("evalHarness.generateScenario.runClient")( return yield* Effect.gen(function* () { const scenario = yield* Effect.try({ - try: () => getScenario(scenarioId), + try: () => getGenerationScenario(scenarioId), catch: (cause) => DiagramGenerationInputError.make({ cause, diff --git a/apps/playground/src/server/generation/api.server.test.ts b/apps/playground/src/server/generation/api.server.test.ts index ae55f52e..4dd4728a 100644 --- a/apps/playground/src/server/generation/api.server.test.ts +++ b/apps/playground/src/server/generation/api.server.test.ts @@ -1,4 +1,7 @@ -import type { CloudflareAiGatewayProvider } from "@sketchi/diagram-generation"; +import type { + CloudflareAiGateway, + CloudflareAiGatewayProvider, +} from "@sketchi/diagram-generation"; import { describe, expect, it } from "vitest"; import type { StudioEnv } from "../bindings/studio-env.server"; @@ -29,18 +32,23 @@ const flowchartIr = { style: { accentColor: "#0f766e", backgroundColor: "#ffffff" }, }; -function fakeAiGateway(text: string): CloudflareAiGatewayProvider { +function fakeAiGateway( + text: string, + observeRun?: (input: Parameters[0]) => void, +): CloudflareAiGatewayProvider { return { gateway: () => ({ - run: () => - Promise.resolve( + run: (input) => { + observeRun?.(input); + return Promise.resolve( new Response( JSON.stringify({ candidates: [{ content: { parts: [{ text }] } }], }), { status: 200, headers: { "content-type": "application/json" } }, ), - ), + ); + }, getUrl: () => Promise.resolve("https://gateway.invalid"), }), }; @@ -94,6 +102,32 @@ describe("public generate endpoint", () => { expect(generation.provider).toBe("cloudflare-google-ai-studio"); }); + it("requests fresh provider output for reliability probes", async () => { + const observedRuns: Array[0]> = []; + const env: StudioEnv = { + AI: fakeAiGateway(JSON.stringify(flowchartIr), (input) => { + observedRuns.push(input); + }), + }; + const response = await generateRequest(env, { + cacheMode: "fresh", + prompt: "Map release approval with pass and revise branches", + type: "flowchart", + }); + + expect(response.status).toBe(200); + expect(observedRuns).toHaveLength(1); + expect(observedRuns[0]).toEqual( + expect.objectContaining({ + headers: expect.objectContaining({ + "Cache-Control": "no-store", + "cf-aig-skip-cache": "true", + Pragma: "no-cache", + }), + }), + ); + }); + it("rejects an empty prompt with a typed invalid-input contract", async () => { const env: StudioEnv = { AI: fakeAiGateway(JSON.stringify(flowchartIr)) }; const response = await generateRequest(env, { prompt: " " }); diff --git a/apps/playground/src/server/generation/api.server.ts b/apps/playground/src/server/generation/api.server.ts index fbb6f021..5ea504cc 100644 --- a/apps/playground/src/server/generation/api.server.ts +++ b/apps/playground/src/server/generation/api.server.ts @@ -73,6 +73,7 @@ interface GenerateSuccess { type GenerateResult = GenerateSuccess | GenerateFailure; const GenerateRequestSchema = Schema.Struct({ + cacheMode: Schema.optional(Schema.Literals(["default", "fresh"])), prompt: Schema.String, type: Schema.optional(Schema.Literals(["flowchart", "mindmap"])), model: Schema.optional(Schema.String), @@ -381,13 +382,15 @@ export const handleGenerateDiagramRequest = Effect.fn( ); } const type: DiagramGenerationType = input.type ?? "flowchart"; + const generationInput = { + ...(input.cacheMode ? { cacheMode: input.cacheMode } : {}), + prompt, + type, + ...(input.model ? { model: input.model } : {}), + }; const candidateResult = yield* withTelemetryCorrelation( - generation.generate({ - prompt, - type, - ...(input.model ? { model: input.model } : {}), - }), + generation.generate(generationInput), { attemptId: usageContext.attemptId, runId: usageContext.runId }, ).pipe( Effect.match({ @@ -397,7 +400,7 @@ export const handleGenerateDiagramRequest = Effect.fn( ); if (!candidateResult.ok) { return yield* finish( - { prompt, type, ...(input.model ? { model: input.model } : {}) }, + generationInput, generationErrorFailure(candidateResult.error), ); } @@ -405,7 +408,7 @@ export const handleGenerateDiagramRequest = Effect.fn( const malformed = malformedCandidateFailure(candidate); if (malformed || !candidate.diagram) { return yield* finish( - { prompt, type, ...(input.model ? { model: input.model } : {}) }, + generationInput, malformed ?? failure("malformed_output", [ issue( @@ -419,7 +422,7 @@ export const handleGenerateDiagramRequest = Effect.fn( } if (candidate.diagram.type !== type) { return yield* finish( - { prompt, type, ...(input.model ? { model: input.model } : {}) }, + generationInput, failure("invalid_generated_document", [ issue( "invalid_generated_document", diff --git a/apps/playground/src/server/generation/service.server.ts b/apps/playground/src/server/generation/service.server.ts index bdfecc71..8e67b71c 100644 --- a/apps/playground/src/server/generation/service.server.ts +++ b/apps/playground/src/server/generation/service.server.ts @@ -1,13 +1,11 @@ import "@tanstack/react-start/server-only"; -import type { - FlowchartDiagram, - MindmapDiagram, -} from "@sketchi/diagram-core"; +import type { FlowchartDiagram, MindmapDiagram } from "@sketchi/diagram-core"; import { CloudflareAiGatewayBinding, CloudflareGoogleAiStudioClientLive, CloudflareGoogleAiStudioConfig, + type DiagramGenerationCacheMode, DiagramGenerationClient, DiagramGenerationConfigurationError, type DiagramGenerationCandidate, @@ -25,6 +23,7 @@ const DEFAULT_GATEWAY_ID = "google-ai-studio"; const DEFAULT_MODEL = "google/gemini-3.1-flash-lite"; export interface GenerateDiagramServiceInput { + readonly cacheMode?: DiagramGenerationCacheMode; readonly model?: string; readonly prompt: string; readonly type: DiagramGenerationType; @@ -33,7 +32,11 @@ export interface GenerateDiagramServiceInput { export interface PlaygroundGenerationShape { readonly generate: ( input: GenerateDiagramServiceInput, - ) => Effect.Effect; + ) => Effect.Effect< + DiagramGenerationCandidate, + DiagramGenerationError, + PlaygroundBindings + >; readonly defaultModel: (env: StudioEnv) => string; } @@ -154,6 +157,7 @@ export const PlaygroundGenerationLive = Layer.succeed(PlaygroundGeneration, { ); const request = { + ...(input.cacheMode ? { cacheMode: input.cacheMode } : {}), model, prompt: { id: "sketchi-generate", diff --git a/package.json b/package.json index fe2dbb49..49f35ce2 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "eval:harness": "jiti tools/harness-eval.ts", "lint": "nx run-many -t lint", "preview": "nx run-many -t preview --parallel", + "probe:generation-reliability": "jiti tools/generation-reliability-probe.ts", "r2sql:codemode:resources": "node scripts/pipelines/codemode-r2-catalog.mjs resources", "r2sql:codemode:verify-run": "node scripts/pipelines/codemode-r2-catalog.mjs verify-run", "test:tools": "vitest run --config tools/vitest.config.mts", diff --git a/packages/diagram/excalidraw/src/lib/convert.test.ts b/packages/diagram/excalidraw/src/lib/convert.test.ts index 5455944f..02cd256f 100644 --- a/packages/diagram/excalidraw/src/lib/convert.test.ts +++ b/packages/diagram/excalidraw/src/lib/convert.test.ts @@ -484,6 +484,115 @@ describe("convertSceneToExcalidraw", () => { } }); + it("routes same-rank branch arrows without crossing nodes in other ranks", () => { + const scene = convertSceneToExcalidraw( + renderIntermediateDiagram({ + id: "ecommerce-return-process", + title: "Ecommerce returns with 18 steps", + type: "flowchart", + nodes: [ + { id: "end-reject", label: "Closure: Rejected", kind: "end" }, + { + id: "notify-final", + label: "Notify Customer: Finalized", + kind: "process", + }, + { + id: "restock-dec", + label: "Restocking Decision", + kind: "decision", + }, + { + id: "partial-refund", + label: "Process Partial Refund", + kind: "process", + }, + { + id: "notify-label", + label: "Notify Customer: Label Ready", + kind: "process", + }, + { + id: "refund-method", + label: "Refund Method Decision", + kind: "decision", + }, + { + id: "notify-receipt", + label: "Notify Customer: Received", + kind: "process", + }, + { id: "restock", label: "Restock Item", kind: "process" }, + { id: "pickup", label: "Carrier Pickup", kind: "process" }, + { + id: "start", + label: "Initiate Return Request", + kind: "start", + }, + { id: "end-success", label: "Closure: Success", kind: "end" }, + { + id: "notify-reject", + label: "Notify Customer: Rejected", + kind: "process", + }, + { + id: "eligibility", + label: "Eligibility Check", + kind: "decision", + }, + { id: "receipt", label: "Warehouse Receipt", kind: "process" }, + { id: "fraud-check", label: "Fraud Review", kind: "decision" }, + { + id: "label-gen", + label: "Generate Return Label", + kind: "process", + }, + ], + edges: [ + { id: "e19", source: "notify-final", target: "end-success" }, + { + id: "e15", + source: "refund-method", + target: "partial-refund", + label: "refund", + }, + { + id: "e2", + source: "eligibility", + target: "label-gen", + label: "eligible", + }, + { + id: "e12", + source: "restock-dec", + target: "restock", + label: "restockable", + }, + { id: "e17", source: "partial-refund", target: "notify-final" }, + { id: "e6", source: "pickup", target: "receipt" }, + { id: "e20", source: "notify-reject", target: "end-reject" }, + { id: "e1", source: "start", target: "eligibility" }, + { + id: "e10", + source: "fraud-check", + target: "restock-dec", + label: "pass", + }, + { id: "e5", source: "notify-label", target: "pickup" }, + { + id: "e13", + source: "restock-dec", + target: "refund-method", + label: "damaged", + }, + ], + layout: { direction: "TB", edgeRouting: "orthogonal" }, + }), + ); + + expect(validateExcalidrawScene(scene)).toEqual({ ok: true, issues: [] }); + }); + it("routes Agy left-to-right skip edges around intervening row nodes", () => { expectFlowchartExportValid({ id: "enterprise-vendor-onboarding-flow", diff --git a/packages/diagram/renderer/src/scene.ts b/packages/diagram/renderer/src/scene.ts index b78e2ca9..a5554d94 100644 --- a/packages/diagram/renderer/src/scene.ts +++ b/packages/diagram/renderer/src/scene.ts @@ -1469,10 +1469,37 @@ function exteriorLaneRoute( const rightLaneX = maxX + HORIZONTAL_GAP / 2 + laneOffset; const upperLaneY = minY - VERTICAL_GAP / 2 - laneOffset; const lowerLaneY = maxY + VERTICAL_GAP / 2 + laneOffset; + const localLaneOffset = ((route.index % 4) * PORT_SPACING) / 2; + const localLeftLaneX = + Math.min(route.source.x, route.target.x) - + HORIZONTAL_GAP / 2 - + localLaneOffset; + const localRightLaneX = + Math.max( + route.source.x + route.source.width, + route.target.x + route.target.width, + ) + + HORIZONTAL_GAP / 2 + + localLaneOffset; + const localUpperLaneY = + Math.min(route.source.y, route.target.y) - + VERTICAL_GAP / 2 - + localLaneOffset; + const localLowerLaneY = + Math.max( + route.source.y + route.source.height, + route.target.y + route.target.height, + ) + + VERTICAL_GAP / 2 + + localLaneOffset; const preferredX = useLeftLane ? leftLaneX : rightLaneX; const alternateX = useLeftLane ? rightLaneX : leftLaneX; const preferredY = useUpperLane ? upperLaneY : lowerLaneY; const alternateY = useUpperLane ? lowerLaneY : upperLaneY; + const preferredLocalX = useLeftLane ? localLeftLaneX : localRightLaneX; + const alternateLocalX = useLeftLane ? localRightLaneX : localLeftLaneX; + const preferredLocalY = useUpperLane ? localUpperLaneY : localLowerLaneY; + const alternateLocalY = useUpperLane ? localLowerLaneY : localUpperLaneY; const localStubDistance = ROUTE_STUB_LENGTH + (route.index % 4) * PORT_SPACING; const stubDistances = [0, localStubDistance]; @@ -1515,6 +1542,18 @@ function exteriorLaneRoute( ]); }; const horizontalCandidates = [ + ...stubDistances.map((stubDistance) => + routeForHorizontalLane(preferredLocalY, stubDistance), + ), + ...stubDistances.map((stubDistance) => + routeForHorizontalLane(alternateLocalY, stubDistance), + ), + ...stubDistances.map((stubDistance) => + routeForVerticalLane(preferredLocalX, stubDistance), + ), + ...stubDistances.map((stubDistance) => + routeForVerticalLane(alternateLocalX, stubDistance), + ), ...stubDistances.map((stubDistance) => routeForHorizontalLane(preferredY, stubDistance), ), @@ -1529,6 +1568,18 @@ function exteriorLaneRoute( ), ]; const verticalCandidates = [ + ...stubDistances.map((stubDistance) => + routeForVerticalLane(preferredLocalX, stubDistance), + ), + ...stubDistances.map((stubDistance) => + routeForVerticalLane(alternateLocalX, stubDistance), + ), + ...stubDistances.map((stubDistance) => + routeForHorizontalLane(preferredLocalY, stubDistance), + ), + ...stubDistances.map((stubDistance) => + routeForHorizontalLane(alternateLocalY, stubDistance), + ), ...stubDistances.map((stubDistance) => routeForVerticalLane(preferredX, stubDistance), ), diff --git a/packages/diagram/scenarios/src/index.ts b/packages/diagram/scenarios/src/index.ts index c14238a4..9865ee88 100644 --- a/packages/diagram/scenarios/src/index.ts +++ b/packages/diagram/scenarios/src/index.ts @@ -1,4 +1,6 @@ export * from "./lib/evaluate.js"; export * from "./lib/fixture-client.js"; +export * from "./lib/generation-reliability.js"; +export * from "./lib/generation-registry.js"; export * from "./lib/prompt.js"; export * from "./lib/scenarios.js"; diff --git a/packages/diagram/scenarios/src/lib/generation-registry.ts b/packages/diagram/scenarios/src/lib/generation-registry.ts new file mode 100644 index 00000000..26009107 --- /dev/null +++ b/packages/diagram/scenarios/src/lib/generation-registry.ts @@ -0,0 +1,22 @@ +import { + generationReliabilityScenarios, + type GenerationReliabilityScenario, +} from "./generation-reliability.js"; +import { flowchartScenarios, type DiagramScenario } from "./scenarios.js"; + +export type RegisteredGenerationScenario = + | DiagramScenario + | GenerationReliabilityScenario; + +export const generationScenarioRegistry: readonly RegisteredGenerationScenario[] = + [...flowchartScenarios, ...generationReliabilityScenarios]; + +export function getGenerationScenario( + id: string, +): RegisteredGenerationScenario { + const scenario = generationScenarioRegistry.find( + (candidate) => candidate.id === id, + ); + if (!scenario) throw new Error(`Unknown scenario "${id}".`); + return scenario; +} diff --git a/packages/diagram/scenarios/src/lib/generation-reliability.test.ts b/packages/diagram/scenarios/src/lib/generation-reliability.test.ts new file mode 100644 index 00000000..61aa3297 --- /dev/null +++ b/packages/diagram/scenarios/src/lib/generation-reliability.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; + +import { getGenerationScenario } from "./generation-registry"; +import { generationReliabilityScenarios } from "./generation-reliability"; +import { toDiagramGenerationPrompt } from "./prompt"; + +describe("generation reliability scenario registry", () => { + it("keeps the production regression prompts and explicit outcome labels", () => { + expect( + getGenerationScenario("reliability-expense-resubmission-loop").prompt, + ).toContain( + "Employee expense approval workflow with at least 6 nodes, 7 edges, and 2 decisions", + ); + expect( + getGenerationScenario("reliability-expense-resubmission-loop").prompt, + ).toContain('edge labeled "rejected"'); + expect( + getGenerationScenario("reliability-expense-resubmission-loop").prompt, + ).toContain( + "back to the expense submission process, never to the start node", + ); + expect(getGenerationScenario("reliability-wedding-richness").prompt).toBe( + "Planning a wedding", + ); + }); + + it("keeps self-loop policy out of organic scenarios", () => { + const returns = getGenerationScenario( + "reliability-ecommerce-return-18-step", + ); + const manuscript = getGenerationScenario( + "reliability-manuscript-interacting-loops", + ); + + expect(returns.prompt).not.toContain("self-loop"); + expect(manuscript.prompt).not.toContain("self-loop"); + expect(manuscript.prompt).toContain("a separate resubmission process"); + expect(manuscript.assertions).toMatchObject({ + requiredCyclePaths: [ + { + branchLabels: ["revision requested"], + branchSourceNodeLabels: expect.arrayContaining(["reviews complete"]), + cycleNodeLabelGroups: expect.arrayContaining([ + expect.arrayContaining(["author revision"]), + expect.arrayContaining(["resubmission"]), + ]), + }, + { + branchLabels: ["plagiarism flagged"], + branchSourceNodeLabels: expect.arrayContaining(["plagiarism flag"]), + cycleNodeLabelGroups: expect.arrayContaining([ + expect.arrayContaining(["ethics investigation"]), + expect.arrayContaining(["editorial triage"]), + ]), + }, + ], + requiredTerminalPaths: [ + { + branchLabels: ["desk reject"], + branchSourceNodeLabels: expect.arrayContaining(["editorial triage"]), + }, + { + branchLabels: ["accepted"], + branchSourceNodeLabels: expect.arrayContaining(["editorial triage"]), + }, + ], + }); + }); + + it("registers both large diagram types for eval-harness generation", () => { + expect(generationReliabilityScenarios.length).toBeGreaterThanOrEqual(7); + expect( + toDiagramGenerationPrompt( + getGenerationScenario("reliability-curriculum-depth-four"), + ), + ).toMatchObject({ + id: "reliability-curriculum-depth-four", + requiredBranchLabels: [], + requiredNodeLabels: [], + type: "mindmap", + }); + expect( + toDiagramGenerationPrompt( + getGenerationScenario("reliability-release-train-brutal"), + ).type, + ).toBe("flowchart"); + }); +}); diff --git a/packages/diagram/scenarios/src/lib/generation-reliability.ts b/packages/diagram/scenarios/src/lib/generation-reliability.ts new file mode 100644 index 00000000..f0076635 --- /dev/null +++ b/packages/diagram/scenarios/src/lib/generation-reliability.ts @@ -0,0 +1,261 @@ +export interface FlowchartReliabilityAssertions { + readonly minDistinctCycleCount?: number; + readonly minCycleDecisionCount: number; + readonly minDecisionCount: number; + readonly minEdgeCount: number; + readonly minEndCount: number; + readonly minNodeCount: number; + readonly requiredCyclePaths?: readonly FlowchartRequiredCyclePath[]; + readonly requiredTerminalPaths?: readonly FlowchartRequiredTerminalPath[]; +} + +export interface FlowchartRequiredCyclePath { + readonly branchLabels: readonly string[]; + readonly branchSourceNodeLabels: readonly string[]; + readonly cycleNodeLabelGroups: readonly (readonly string[])[]; +} + +export interface FlowchartRequiredTerminalPath { + readonly branchLabels: readonly string[]; + readonly branchSourceNodeLabels: readonly string[]; + readonly terminalNodeLabelGroups: readonly (readonly string[])[]; +} + +export interface MindmapReliabilityAssertions { + readonly minDepth: number; + readonly minTopicCount: number; +} + +interface ReliabilityScenarioBase { + readonly description: string; + readonly id: string; + readonly prompt: string; + readonly tags: readonly string[]; + readonly title: string; +} + +export interface FlowchartReliabilityScenario extends ReliabilityScenarioBase { + readonly assertions: FlowchartReliabilityAssertions; + readonly diagramType: "flowchart"; +} + +export interface MindmapReliabilityScenario extends ReliabilityScenarioBase { + readonly assertions: MindmapReliabilityAssertions; + readonly diagramType: "mindmap"; +} + +export type GenerationReliabilityScenario = + | FlowchartReliabilityScenario + | MindmapReliabilityScenario; + +export const generationReliabilityScenarios = [ + { + id: "reliability-expense-resubmission-loop", + title: "Expense resubmission loop", + diagramType: "flowchart", + description: + "The verbatim production failure that requires rejection to remain retryable while reimbursement terminates.", + prompt: + 'Employee expense approval workflow with at least 6 nodes, 7 edges, and 2 decisions: a start node, then a separate expense submission process, manager approval decision, finance audit decision, reimbursement, or rejection with a resubmission loop. Every rejection path must use an edge labeled "rejected" and loop through a resubmission process back to the expense submission process, never to the start node. The successful finance-audit edge must be labeled "approved" and reach a distinct end node labeled "reimbursed".', + tags: ["reliability", "production-regression", "loop"], + assertions: { + minCycleDecisionCount: 1, + minDecisionCount: 2, + minEdgeCount: 7, + minEndCount: 1, + minNodeCount: 6, + requiredCyclePaths: [ + { + branchLabels: ["rejected"], + branchSourceNodeLabels: [ + "manager approval", + "manager approved", + "manager decision", + "manager review", + "finance approval", + "finance approved", + "finance audit", + "finance decision", + ], + cycleNodeLabelGroups: [ + [ + "expense submission", + "submit expense", + "submits expense", + "expense submitted", + ], + ["resubmission", "resubmit"], + ], + }, + ], + requiredTerminalPaths: [ + { + branchLabels: ["approved"], + branchSourceNodeLabels: [ + "finance approval", + "finance approved", + "finance audit", + "finance decision", + "finance review", + ], + terminalNodeLabelGroups: [["reimbursement", "reimbursed"]], + }, + ], + }, + }, + { + id: "reliability-ecommerce-return-18-step", + title: "Ecommerce returns with 18 steps", + diagramType: "flowchart", + description: + "A production-scale return workflow with an explicit minimum step count and a fraud-review loop.", + prompt: + "End-to-end e-commerce return and refund process with at least 18 distinct steps: initiation, eligibility check, label generation, carrier pickup, warehouse receipt, inspection, restocking decision, refund method decision, partial refund path, exchange path, fraud review loop, customer notification at each stage, and final closure. The fraud review loop must traverse a distinct manual-review process before returning to the fraud decision.", + tags: ["reliability", "production-regression", "large", "loop"], + assertions: { + minCycleDecisionCount: 1, + minDecisionCount: 3, + minEdgeCount: 18, + minEndCount: 1, + minNodeCount: 18, + }, + }, + { + id: "reliability-manuscript-interacting-loops", + title: "Manuscript review interacting loops", + diagramType: "flowchart", + description: + "Two interacting review and ethics loops with several distinct terminal outcomes.", + prompt: + 'Manuscript peer review pipeline with at least 18 distinct steps and two interacting loops: submission, editorial triage decision (desk reject to end, or send to review), three parallel reviewer assignments, reviews-complete decision, revision-requested loop through author revision followed by a separate resubmission process (max two rounds tracked by a rounds-exhausted decision), plagiarism-flag decision that routes to an ethics investigation loop which can rejoin editorial triage or terminate in retraction, acceptance path with copyediting, typesetting, proof approval decision that can loop to typesetting, and final publication; include distinct end states for desk rejection, final rejection, retraction, and publication. The edge labeled "plagiarism flagged" must target the Ethics Investigation process, which must have a path back to Editorial Triage; never put that label on the retraction edge. Label the revision loop branch "revision requested", the desk-rejection branch "desk reject", and the publication branch "accepted".', + tags: ["reliability", "nested-loops", "multiple-ends"], + assertions: { + minCycleDecisionCount: 2, + minDecisionCount: 4, + minEdgeCount: 18, + minEndCount: 3, + minNodeCount: 15, + minDistinctCycleCount: 2, + requiredCyclePaths: [ + { + branchLabels: ["revision requested"], + branchSourceNodeLabels: [ + "reviews complete", + "review complete", + "review decision", + "revision needed", + "revision decision", + "revision requested", + ], + cycleNodeLabelGroups: [ + ["author revision", "revise manuscript", "author revises"], + ["resubmission", "resubmit"], + ], + }, + { + branchLabels: ["plagiarism flagged"], + branchSourceNodeLabels: ["plagiarism flag", "plagiarism detected"], + cycleNodeLabelGroups: [ + ["ethics investigation", "ethics review"], + ["editorial triage", "editor triage"], + ], + }, + ], + requiredTerminalPaths: [ + { + branchLabels: ["desk reject"], + branchSourceNodeLabels: [ + "editorial triage", + "editor triage", + "triage decision", + "desk reject", + ], + terminalNodeLabelGroups: [["desk rejection", "desk reject"]], + }, + { + branchLabels: ["accepted"], + branchSourceNodeLabels: [ + "editorial triage", + "editor triage", + "accepted", + "acceptance decision", + "final decision", + "editorial decision", + "manuscript decision", + "accept manuscript", + ], + terminalNodeLabelGroups: [["publication", "published"]], + }, + ], + }, + }, + { + id: "reliability-release-train-brutal", + title: "Global release train", + diagramType: "flowchart", + description: + "A near-limit flowchart with two recovery loops, five decisions, and independent stop states.", + prompt: + "Create a 22 to 24 node global software release train flowchart. Include intake, scope review, dependency analysis, build, unit tests, integration tests, security scan, change approval, canary deployment, regional rollout, observability checks, customer communication, and closure. Use at least five labeled decision nodes. Failed tests loop through remediation and rebuild; an unhealthy canary loops through rollback, incident review, and a new canary. Include separate end states for cancelled change, security rejection, rollback without retry, and successful release.", + tags: ["reliability", "brutal", "near-limit", "nested-loops"], + assertions: { + minCycleDecisionCount: 2, + minDecisionCount: 5, + minEdgeCount: 24, + minEndCount: 3, + minNodeCount: 22, + }, + }, + { + id: "reliability-curriculum-depth-four", + title: "Software engineering curriculum", + diagramType: "mindmap", + description: + "The production truncation regression with 25 topics distributed across four hierarchy levels.", + prompt: + "Complete software engineering curriculum with at least 25 topics across 4 levels: fundamentals, languages, systems, and practices, each broken into concrete subtopics and sub-subtopics", + tags: ["reliability", "production-regression", "large", "deep"], + assertions: { minDepth: 3, minTopicCount: 25 }, + }, + { + id: "reliability-immune-system-brutal", + title: "Human immune system", + diagramType: "mindmap", + description: + "A dense long-label hierarchy well beyond ordinary generation complexity.", + prompt: + "Exhaustive map of the human immune system with at least 35 topics and four levels of depth: innate branch (physical barriers, cellular components with neutrophils, macrophages, dendritic cells, NK cells, and their activation mechanisms), adaptive branch (humoral with B cell development, antibody classes and functions; cell-mediated with T cell subsets, MHC restriction, memory formation), complement system pathways, cytokine signaling families, and clinical topics covering hypersensitivity types, immunodeficiencies, and vaccines", + tags: ["reliability", "brutal", "large", "deep", "long-labels"], + assertions: { minDepth: 3, minTopicCount: 35 }, + }, + { + id: "reliability-wedding-richness", + title: "Wedding planning richness", + diagramType: "mindmap", + description: "The verbatim sparse-output production regression.", + prompt: "Planning a wedding", + tags: ["reliability", "production-regression", "sparse-guard"], + assertions: { minDepth: 2, minTopicCount: 10 }, + }, + { + id: "reliability-kubernetes-depth", + title: "Kubernetes depth", + diagramType: "mindmap", + description: + "The production depth regression with a hard three-level minimum.", + prompt: + "Kubernetes architecture, workloads, networking, storage, security, and operations with at least 18 topics across four hierarchy levels total: root, category, subtopic, and concrete detail (maximum depth at least 3)", + tags: ["reliability", "production-regression", "depth"], + assertions: { minDepth: 3, minTopicCount: 18 }, + }, +] as const satisfies readonly GenerationReliabilityScenario[]; + +export function getGenerationReliabilityScenario( + id: string, +): GenerationReliabilityScenario { + const scenario = generationReliabilityScenarios.find( + (candidate) => candidate.id === id, + ); + if (!scenario) throw new Error(`Unknown reliability scenario "${id}".`); + return scenario; +} diff --git a/packages/diagram/scenarios/src/lib/prompt.ts b/packages/diagram/scenarios/src/lib/prompt.ts index d13bb42d..89f19186 100644 --- a/packages/diagram/scenarios/src/lib/prompt.ts +++ b/packages/diagram/scenarios/src/lib/prompt.ts @@ -7,21 +7,34 @@ import { } from "@sketchi/diagram-generation"; import type { DiagramScenario } from "./scenarios.js"; +import type { GenerationReliabilityScenario } from "./generation-reliability.js"; + +type GenerationPromptScenario = DiagramScenario | GenerationReliabilityScenario; export type ScenarioPromptRole = DiagramGenerationRole; export type ScenarioPromptMessage = DiagramGenerationMessage; export type ScenarioPromptParts = DiagramGenerationMessages; export function toDiagramGenerationPrompt( - scenario: DiagramScenario, + scenario: GenerationPromptScenario, ): DiagramGenerationPrompt { + const requiredBranchLabels = + scenario.diagramType === "flowchart" && + "requiredBranchLabels" in scenario.assertions + ? scenario.assertions.requiredBranchLabels + : []; + const requiredNodeLabels = + scenario.diagramType === "flowchart" && + "requiredNodeLabels" in scenario.assertions + ? scenario.assertions.requiredNodeLabels + : []; return { id: scenario.id, request: scenario.prompt, - requiredBranchLabels: scenario.assertions.requiredBranchLabels, - requiredNodeLabels: scenario.assertions.requiredNodeLabels, + requiredBranchLabels, + requiredNodeLabels, title: scenario.title, - type: "flowchart", + type: scenario.diagramType, }; } diff --git a/tools/generation-reliability-probe.test.ts b/tools/generation-reliability-probe.test.ts new file mode 100644 index 00000000..d87f6295 --- /dev/null +++ b/tools/generation-reliability-probe.test.ts @@ -0,0 +1,519 @@ +import { + generationReliabilityScenarios, + type GenerationReliabilityScenario, +} from "@sketchi/diagram-scenarios"; +import { describe, expect, it } from "vitest"; + +import { + evaluateStructuralFidelity, + generationProbeRequestTimeoutMs, + selectProbeScenarios, +} from "./generation-reliability-probe"; + +function scenario(id: string): GenerationReliabilityScenario { + const value = generationReliabilityScenarios.find( + (candidate) => candidate.id === id, + ); + if (!value) throw new Error(`Missing test scenario ${id}.`); + return value; +} + +describe("generation reliability structural assertions", () => { + it("selects one scenario for an intensified regression probe", () => { + expect( + selectProbeScenarios("reliability-expense-resubmission-loop").map( + (selected) => selected.id, + ), + ).toEqual(["reliability-expense-resubmission-loop"]); + expect(selectProbeScenarios("missing-scenario")).toEqual([]); + expect(selectProbeScenarios(undefined)).toEqual( + generationReliabilityScenarios, + ); + }); + + it("budgets the outer request timeout beyond generation and repair policy", () => { + const policyBudgetMs = 3 * (3 * 30_000 + 250 + 500); + + expect(generationProbeRequestTimeoutMs()).toBeGreaterThan(policyBudgetMs); + }); + + it("requires distinct labeled cycles and labeled terminal outcomes", () => { + const result = evaluateStructuralFidelity( + scenario("reliability-manuscript-interacting-loops"), + { + type: "flowchart", + spec: { + nodes: [ + { id: "start", label: "Submission", kind: "start" }, + { + id: "decision-a", + label: "Reviews complete?", + kind: "decision", + }, + { + id: "work-a", + label: "Author revision and resubmission", + kind: "process", + }, + { + id: "decision-b", + label: "Plagiarism flag?", + kind: "decision", + }, + { + id: "work-b", + label: "Ethics investigation", + kind: "process", + }, + { + id: "decision-c", + label: "Editorial triage", + kind: "decision", + }, + { + id: "decision-d", + label: "Proof approved?", + kind: "decision", + }, + { id: "end-a", label: "Desk rejection", kind: "end" }, + { id: "end-b", label: "Final rejection", kind: "end" }, + { id: "end-c", label: "Publication", kind: "end" }, + ...Array.from({ length: 5 }, (_, index) => ({ + id: `extra-${index}`, + label: `Review work ${index}`, + kind: "process", + })), + ], + edges: [ + { source: "start", target: "decision-a" }, + { + source: "decision-a", + target: "work-a", + label: "revision requested", + }, + { source: "work-a", target: "decision-a" }, + { + source: "decision-a", + target: "decision-b", + label: "reviews complete", + }, + { + source: "decision-b", + target: "work-b", + label: "plagiarism flagged", + }, + { source: "work-b", target: "decision-c" }, + { source: "decision-b", target: "decision-c", label: "clear" }, + { + source: "decision-c", + target: "decision-b", + label: "ethics cleared", + }, + { source: "decision-c", target: "end-a", label: "desk reject" }, + { source: "decision-c", target: "decision-d", label: "accepted" }, + { source: "decision-d", target: "end-b", label: "retract" }, + { source: "decision-d", target: "end-c", label: "publish" }, + { + source: "decision-d", + target: "extra-0", + label: "extended review", + }, + { source: "extra-0", target: "extra-1" }, + { source: "extra-1", target: "extra-2" }, + { source: "extra-2", target: "extra-3" }, + { source: "extra-3", target: "extra-4" }, + { source: "extra-4", target: "end-c" }, + { source: "extra-1", target: "extra-3" }, + ], + }, + }, + ); + + expect(result.details.cycleDecisionCount).toBe(3); + expect(result.details.distinctCycleCount).toBe(2); + expect(result.details.requiredCyclePathCount).toBe(2); + expect(result.details.requiredTerminalPathCount).toBe(2); + expect(result.passed).toBe(true); + }); + + it("requires every expense-loop waypoint group", () => { + const selected = scenario("reliability-expense-resubmission-loop"); + if (selected.diagramType !== "flowchart") { + throw new Error("Expected an expense flowchart scenario."); + } + const requiredPath = selected.assertions.requiredCyclePaths?.[0]; + if (!requiredPath) throw new Error("Missing expense cycle assertion."); + + const result = evaluateStructuralFidelity( + { + ...selected, + assertions: { + minCycleDecisionCount: 0, + minDecisionCount: 0, + minEdgeCount: 0, + minEndCount: 0, + minNodeCount: 0, + requiredCyclePaths: [requiredPath], + }, + }, + { + type: "flowchart", + spec: { + nodes: [ + { id: "finance", label: "Finance audit", kind: "decision" }, + { id: "rejection", label: "Rejection", kind: "process" }, + { + id: "submission", + label: "Expense submission", + kind: "process", + }, + ], + edges: [ + { source: "finance", target: "rejection", label: "rejected" }, + { source: "rejection", target: "submission" }, + { source: "submission", target: "finance" }, + ], + }, + }, + ); + + expect(result.details.requiredCyclePathCount).toBe(0); + expect(result.passed).toBe(false); + }); + + it("does not stitch adjacent cycles to satisfy expense-loop waypoints", () => { + const selected = scenario("reliability-expense-resubmission-loop"); + if (selected.diagramType !== "flowchart") { + throw new Error("Expected an expense flowchart scenario."); + } + const requiredPath = selected.assertions.requiredCyclePaths?.[0]; + if (!requiredPath) throw new Error("Missing expense cycle assertion."); + + const result = evaluateStructuralFidelity( + { + ...selected, + assertions: { + minCycleDecisionCount: 0, + minDecisionCount: 0, + minEdgeCount: 0, + minEndCount: 0, + minNodeCount: 0, + requiredCyclePaths: [requiredPath], + }, + }, + { + type: "flowchart", + spec: { + nodes: [ + { id: "finance", label: "Finance audit", kind: "decision" }, + { id: "short-loop", label: "Correction", kind: "process" }, + { + id: "submission", + label: "Expense submission", + kind: "process", + }, + { + id: "resubmission", + label: "Resubmission", + kind: "process", + }, + ], + edges: [ + { source: "finance", target: "short-loop", label: "rejected" }, + { source: "short-loop", target: "finance" }, + { source: "finance", target: "submission", label: "approved" }, + { source: "submission", target: "resubmission" }, + { source: "resubmission", target: "finance" }, + ], + }, + }, + ); + + expect(result.details.requiredCyclePathCount).toBe(0); + expect(result.passed).toBe(false); + }); + + it("requires Editorial Triage to participate in the ethics cycle", () => { + const selected = scenario("reliability-manuscript-interacting-loops"); + if (selected.diagramType !== "flowchart") { + throw new Error("Expected a manuscript flowchart scenario."); + } + const requiredPath = selected.assertions.requiredCyclePaths?.[1]; + if (!requiredPath) throw new Error("Missing ethics cycle assertion."); + + const result = evaluateStructuralFidelity( + { + ...selected, + assertions: { + minCycleDecisionCount: 0, + minDecisionCount: 0, + minEdgeCount: 0, + minEndCount: 0, + minNodeCount: 0, + requiredCyclePaths: [requiredPath], + }, + }, + { + type: "flowchart", + spec: { + nodes: [ + { + id: "plagiarism", + label: "Plagiarism flagged?", + kind: "decision", + }, + { + id: "ethics", + label: "Ethics investigation", + kind: "process", + }, + { + id: "triage", + label: "Editorial triage", + kind: "process", + }, + ], + edges: [ + { + source: "plagiarism", + target: "ethics", + label: "plagiarism flagged", + }, + { source: "ethics", target: "plagiarism" }, + { source: "plagiarism", target: "triage", label: "clear" }, + ], + }, + }, + ); + + expect(result.details.requiredCyclePathCount).toBe(0); + expect(result.passed).toBe(false); + }); + + it("binds terminal branch labels to the required source node", () => { + const selected = scenario("reliability-expense-resubmission-loop"); + if (selected.diagramType !== "flowchart") { + throw new Error("Expected an expense flowchart scenario."); + } + const requiredPath = selected.assertions.requiredTerminalPaths?.[0]; + if (!requiredPath) throw new Error("Missing expense terminal assertion."); + + const result = evaluateStructuralFidelity( + { + ...selected, + assertions: { + minCycleDecisionCount: 0, + minDecisionCount: 0, + minEdgeCount: 0, + minEndCount: 0, + minNodeCount: 0, + requiredTerminalPaths: [requiredPath], + }, + }, + { + type: "flowchart", + spec: { + nodes: [ + { id: "manager", label: "Manager approval", kind: "decision" }, + { id: "finance", label: "Finance audit", kind: "process" }, + { + id: "reimbursement", + label: "Reimbursement", + kind: "process", + }, + { id: "end", label: "Reimbursed", kind: "end" }, + ], + edges: [ + { source: "manager", target: "finance", label: "approved" }, + { source: "finance", target: "reimbursement" }, + { source: "reimbursement", target: "end" }, + ], + }, + }, + ); + + expect(result.details.requiredTerminalPathCount).toBe(0); + expect(result.passed).toBe(false); + }); + + it("rejects inverse expense semantics even when counts and a cycle pass", () => { + const result = evaluateStructuralFidelity( + scenario("reliability-expense-resubmission-loop"), + { + type: "flowchart", + spec: { + nodes: [ + { id: "start", label: "Submission", kind: "start" }, + { id: "manager", label: "Manager review", kind: "decision" }, + { id: "finance", label: "Finance audit", kind: "decision" }, + { id: "rejection", label: "Rejection", kind: "process" }, + { + id: "reimbursement", + label: "Reimbursement", + kind: "process", + }, + { + id: "resubmit", + label: "Resubmission", + kind: "process", + }, + { id: "end", label: "Rejected", kind: "end" }, + ], + edges: [ + { source: "start", target: "finance" }, + { source: "finance", target: "manager", label: "audited" }, + { source: "finance", target: "rejection", label: "rejected" }, + { source: "manager", target: "reimbursement", label: "approved" }, + { source: "manager", target: "rejection", label: "rejected" }, + { source: "reimbursement", target: "resubmit" }, + { source: "resubmit", target: "manager" }, + { source: "rejection", target: "end" }, + ], + }, + }, + ); + + expect(result.details.cycleDecisionCount).toBe(1); + expect(result.details.requiredCyclePathCount).toBe(0); + expect(result.details.requiredTerminalPathCount).toBe(0); + expect(result.passed).toBe(false); + }); + + it("rejects one labeled cycle containing two manuscript decisions", () => { + const result = evaluateStructuralFidelity( + scenario("reliability-manuscript-interacting-loops"), + { + type: "flowchart", + spec: { + nodes: [ + { id: "start", label: "Submission", kind: "start" }, + { + id: "revision-decision", + label: "Revision needed?", + kind: "decision", + }, + { + id: "revision", + label: "Author revision and resubmission", + kind: "process", + }, + { + id: "ethics-decision", + label: "Plagiarism flag?", + kind: "decision", + }, + { + id: "ethics", + label: "Ethics investigation", + kind: "process", + }, + { id: "triage", label: "Editorial triage", kind: "decision" }, + { id: "proof", label: "Proof approved?", kind: "decision" }, + { id: "desk-end", label: "Desk rejection", kind: "end" }, + { id: "reject-end", label: "Final rejection", kind: "end" }, + { id: "publish-end", label: "Publication", kind: "end" }, + ...Array.from({ length: 5 }, (_, index) => ({ + id: `extra-${index}`, + label: `Review work ${index}`, + kind: "process", + })), + ], + edges: [ + { source: "start", target: "revision-decision" }, + { + source: "revision-decision", + target: "revision", + label: "revision requested", + }, + { source: "revision", target: "ethics-decision" }, + { + source: "ethics-decision", + target: "ethics", + label: "plagiarism flagged", + }, + { source: "ethics", target: "revision-decision" }, + { + source: "revision-decision", + target: "triage", + label: "reviews complete", + }, + { source: "ethics-decision", target: "triage", label: "clear" }, + { source: "triage", target: "desk-end", label: "desk reject" }, + { source: "triage", target: "proof", label: "accepted" }, + { source: "proof", target: "reject-end", label: "retract" }, + { source: "proof", target: "publish-end", label: "publish" }, + ...Array.from({ length: 7 }, (_, index) => ({ + source: `extra-${Math.max(0, index - 1)}`, + target: `extra-${Math.min(4, index)}`, + })), + ], + }, + }, + ); + + expect(result.details.cycleDecisionCount).toBe(2); + expect(result.details.requiredCyclePathCount).toBe(1); + expect(result.details.distinctCycleCount).toBe(1); + expect(result.passed).toBe(false); + }); + + it("rejects unlabeled decision branches even when counts pass", () => { + const selected = scenario("reliability-expense-resubmission-loop"); + const result = evaluateStructuralFidelity(selected, { + type: "flowchart", + spec: { + nodes: [ + { id: "start", kind: "start" }, + { id: "decision", kind: "decision" }, + { id: "loop", kind: "process" }, + { id: "decision-2", kind: "decision" }, + { id: "work", kind: "process" }, + { id: "work-2", kind: "process" }, + { id: "end", kind: "end" }, + ], + edges: [ + { source: "start", target: "decision" }, + { source: "decision", target: "loop" }, + { source: "loop", target: "decision" }, + { source: "decision", target: "decision-2", label: "approved" }, + { source: "decision-2", target: "work", label: "yes" }, + { source: "decision-2", target: "work-2", label: "no" }, + { source: "work", target: "end" }, + ], + }, + }); + + expect(result.details.unlabeledDecisionBranches).toBe(1); + expect(result.passed).toBe(false); + }); + + it("measures nested mindmap topic count and depth", () => { + const selected = scenario("reliability-wedding-richness"); + const leaf = (label: string) => ({ label, children: [] }); + const result = evaluateStructuralFidelity(selected, { + type: "mindmap", + spec: { + root: { + label: "Wedding", + children: [ + { + label: "Venue", + children: [leaf("Search"), leaf("Visit"), leaf("Book")], + }, + { + label: "Guests", + children: [leaf("List"), leaf("Invites"), leaf("RSVP")], + }, + { + label: "Day", + children: [leaf("Schedule")], + }, + ], + }, + }, + }); + + expect(result.details).toEqual({ maxDepth: 2, topicCount: 11 }); + expect(result.passed).toBe(true); + }); +}); diff --git a/tools/generation-reliability-probe.ts b/tools/generation-reliability-probe.ts new file mode 100644 index 00000000..f12e384d --- /dev/null +++ b/tools/generation-reliability-probe.ts @@ -0,0 +1,564 @@ +import { pathToFileURL } from "node:url"; + +import { NodeRuntime } from "@effect/platform-node"; +import { diagramGenerationPolicyDefaults } from "@sketchi/diagram-generation"; +import { + generationReliabilityScenarios, + type GenerationReliabilityScenario, +} from "@sketchi/diagram-scenarios"; +import { Clock, Effect, Schema } from "effect"; + +const DEFAULT_ENDPOINT = "https://playground.sketchi.app/api/v1/generate"; +const DEFAULT_REPEATS = 3; +const REQUEST_TIMEOUT_MARGIN_MS = 30_000; + +export function generationProbeRequestTimeoutMs(): number { + const policy = diagramGenerationPolicyDefaults; + const retryDelayBudgetMs = Array.from( + { length: policy.maxRetries }, + (_, retryIndex) => policy.retryDelayMs * 2 ** retryIndex, + ).reduce((total, delayMs) => total + delayMs, 0); + const modelCallBudgetMs = + (policy.maxRetries + 1) * policy.requestTimeoutMs + retryDelayBudgetMs; + return ( + (policy.maxRepairAttempts + 1) * modelCallBudgetMs + + REQUEST_TIMEOUT_MARGIN_MS + ); +} + +const REQUEST_TIMEOUT_MS = generationProbeRequestTimeoutMs(); + +interface UnknownRecord { + readonly [key: string]: unknown; +} + +export interface StructuralFidelityResult { + readonly details: Record; + readonly failures: readonly string[]; + readonly passed: boolean; +} + +interface ProbeRunResult extends StructuralFidelityResult { + readonly durationMs: number; + readonly runNumber: number; + readonly scenarioId: string; + readonly statusCode: number; +} + +export class GenerationProbeRequestError extends Schema.TaggedErrorClass()( + "GenerationProbeRequestError", + { + cause: Schema.Defect(), + message: Schema.String, + }, +) {} + +function isUnknownRecord(value: unknown): value is UnknownRecord { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function stringValue(record: UnknownRecord, key: string): string | undefined { + const value = record[key]; + return typeof value === "string" ? value : undefined; +} + +function normalizedLabel(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/gu, " ") + .trim(); +} + +function labelMatches( + value: string | undefined, + expectedLabels: readonly string[], +): boolean { + if (!value) return false; + const normalizedValue = normalizedLabel(value); + return expectedLabels.some((expected) => + normalizedValue.includes(normalizedLabel(expected)), + ); +} + +function flowchartFidelity( + scenario: Extract< + GenerationReliabilityScenario, + { diagramType: "flowchart" } + >, + spec: UnknownRecord, +): StructuralFidelityResult { + const rawNodes = spec["nodes"]; + const rawEdges = spec["edges"]; + if (!Array.isArray(rawNodes) || !Array.isArray(rawEdges)) { + return { + details: {}, + failures: ["Flowchart document omitted nodes or edges."], + passed: false, + }; + } + const nodes = rawNodes.filter(isUnknownRecord); + const edges = rawEdges.filter(isUnknownRecord); + const decisions = nodes.filter((node) => node["kind"] === "decision"); + const ends = nodes.filter((node) => node["kind"] === "end"); + const nodesById = new Map( + nodes.flatMap((node) => { + const id = stringValue(node, "id"); + return id ? [[id, node] as const] : []; + }), + ); + const flowEdges = edges.flatMap((edge) => { + const source = stringValue(edge, "source"); + const target = stringValue(edge, "target"); + return source && target + ? [{ label: stringValue(edge, "label"), source, target }] + : []; + }); + const adjacency = new Map(); + for (const edge of flowEdges) { + adjacency.set(edge.source, [ + ...(adjacency.get(edge.source) ?? []), + edge.target, + ]); + } + const findPath = ( + start: string, + destination: string, + ): readonly string[] | undefined => { + const pending: Array<{ id: string; path: readonly string[] }> = [ + { id: start, path: [start] }, + ]; + const visited = new Set(); + while (pending.length > 0) { + const current = pending.pop(); + if (!current) continue; + if (current.id === destination) return current.path; + if (visited.has(current.id)) continue; + visited.add(current.id); + pending.push( + ...(adjacency.get(current.id) ?? []).map((id) => ({ + id, + path: [...current.path, id], + })), + ); + } + return undefined; + }; + const pathExists = (start: string, destination: string): boolean => + findPath(start, destination) !== undefined; + const findPathIncludingLabelGroups = ( + start: string, + destination: string, + labelGroups: readonly (readonly string[])[], + seedNodeIds: readonly string[] = [], + ): readonly string[] | undefined => { + const matchGroups = ( + nodeIds: readonly string[], + matchedGroups: ReadonlySet, + ): ReadonlySet => { + const nextMatchedGroups = new Set(matchedGroups); + for (const nodeId of nodeIds) { + const label = stringValue(nodesById.get(nodeId) ?? {}, "label"); + labelGroups.forEach((group, index) => { + if (labelMatches(label, group)) nextMatchedGroups.add(index); + }); + } + return nextMatchedGroups; + }; + const initialMatchedGroups = matchGroups(seedNodeIds, new Set()); + const pending: Array<{ + id: string; + matchedGroups: ReadonlySet; + path: readonly string[]; + }> = [ + { + id: start, + matchedGroups: matchGroups([start], initialMatchedGroups), + path: [start], + }, + ]; + const visited = new Set(); + while (pending.length > 0) { + const current = pending.shift(); + if (!current) continue; + const stateKey = `${current.id}|${[...current.matchedGroups].sort().join(",")}`; + if (visited.has(stateKey)) continue; + visited.add(stateKey); + if (current.id === destination) { + if (current.matchedGroups.size === labelGroups.length) { + return current.path; + } + continue; + } + for (const id of adjacency.get(current.id) ?? []) { + pending.push({ + id, + matchedGroups: matchGroups([id], current.matchedGroups), + path: [...current.path, id], + }); + } + } + return undefined; + }; + const cycleDecisionCount = decisions.filter((decision) => { + const id = stringValue(decision, "id"); + return id + ? (adjacency.get(id) ?? []).some((target) => pathExists(target, id)) + : false; + }).length; + const requiredCyclePaths = scenario.assertions.requiredCyclePaths ?? []; + const requiredCycleFingerprints = requiredCyclePaths.map((required) => { + for (const edge of flowEdges) { + if (!labelMatches(edge.label, required.branchLabels)) continue; + const branchSourceLabel = stringValue( + nodesById.get(edge.source) ?? {}, + "label", + ); + if (!labelMatches(branchSourceLabel, required.branchSourceNodeLabels)) { + continue; + } + const returnPath = findPathIncludingLabelGroups( + edge.target, + edge.source, + required.cycleNodeLabelGroups, + [edge.source], + ); + if (!returnPath) continue; + const cycleNodeIds = new Set([edge.source, ...returnPath]); + return [...cycleNodeIds].sort().join("|"); + } + return undefined; + }); + const requiredCyclePathCount = requiredCycleFingerprints.filter( + (fingerprint) => fingerprint !== undefined, + ).length; + const distinctCycleCount = new Set( + requiredCycleFingerprints.filter( + (fingerprint) => fingerprint !== undefined, + ), + ).size; + const endIds = ends.flatMap((node) => { + const id = stringValue(node, "id"); + return id ? [id] : []; + }); + const requiredTerminalPaths = scenario.assertions.requiredTerminalPaths ?? []; + const requiredTerminalPathCount = requiredTerminalPaths.filter((required) => + flowEdges.some((edge) => { + if (!labelMatches(edge.label, required.branchLabels)) return false; + const branchSourceLabel = stringValue( + nodesById.get(edge.source) ?? {}, + "label", + ); + if (!labelMatches(branchSourceLabel, required.branchSourceNodeLabels)) { + return false; + } + if (findPath(edge.target, edge.source)) return false; + return endIds.some( + (endId) => + findPathIncludingLabelGroups( + edge.target, + endId, + required.terminalNodeLabelGroups, + ) !== undefined, + ); + }), + ).length; + const unlabeledDecisionBranches = decisions.reduce((count, decision) => { + const id = stringValue(decision, "id"); + if (!id) return count + 1; + return ( + count + + edges + .filter((edge) => edge["source"] === id) + .filter((edge) => { + const label = stringValue(edge, "label"); + return !label?.trim(); + }).length + ); + }, 0); + const details = { + cycleDecisionCount, + decisionCount: decisions.length, + distinctCycleCount, + edgeCount: edges.length, + endCount: ends.length, + nodeCount: nodes.length, + requiredCyclePathCount, + requiredTerminalPathCount, + unlabeledDecisionBranches, + }; + const failures = [ + ...(nodes.length < scenario.assertions.minNodeCount + ? [`Expected >=${scenario.assertions.minNodeCount} nodes.`] + : []), + ...(edges.length < scenario.assertions.minEdgeCount + ? [`Expected >=${scenario.assertions.minEdgeCount} edges.`] + : []), + ...(decisions.length < scenario.assertions.minDecisionCount + ? [`Expected >=${scenario.assertions.minDecisionCount} decisions.`] + : []), + ...(ends.length < scenario.assertions.minEndCount + ? [`Expected >=${scenario.assertions.minEndCount} ends.`] + : []), + ...(cycleDecisionCount < scenario.assertions.minCycleDecisionCount + ? [ + `Expected >=${scenario.assertions.minCycleDecisionCount} decisions participating in cycles.`, + ] + : []), + ...(requiredCyclePathCount < requiredCyclePaths.length + ? [ + `Expected ${requiredCyclePaths.length} labeled cycle paths; found ${requiredCyclePathCount}.`, + ] + : []), + ...(distinctCycleCount < (scenario.assertions.minDistinctCycleCount ?? 0) + ? [ + `Expected >=${scenario.assertions.minDistinctCycleCount ?? 0} distinct labeled cycles; found ${distinctCycleCount}.`, + ] + : []), + ...(requiredTerminalPathCount < requiredTerminalPaths.length + ? [ + `Expected ${requiredTerminalPaths.length} labeled terminal paths; found ${requiredTerminalPathCount}.`, + ] + : []), + ...(unlabeledDecisionBranches > 0 + ? [`Found ${unlabeledDecisionBranches} unlabeled decision branches.`] + : []), + ]; + return { details, failures, passed: failures.length === 0 }; +} + +function mindmapFidelity( + scenario: Extract, + spec: UnknownRecord, +): StructuralFidelityResult { + const root = spec["root"]; + if (!isUnknownRecord(root)) { + return { + details: {}, + failures: ["Mindmap document omitted its nested root."], + passed: false, + }; + } + let maxDepth = 0; + let topicCount = 0; + const pending: Array<{ depth: number; topic: UnknownRecord }> = [ + { depth: 0, topic: root }, + ]; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) continue; + topicCount += 1; + maxDepth = Math.max(maxDepth, current.depth); + const children = current.topic["children"]; + if (Array.isArray(children)) { + for (const child of children) { + if (isUnknownRecord(child)) { + pending.push({ depth: current.depth + 1, topic: child }); + } + } + } + } + const details = { maxDepth, topicCount }; + const failures = [ + ...(topicCount < scenario.assertions.minTopicCount + ? [`Expected >=${scenario.assertions.minTopicCount} topics.`] + : []), + ...(maxDepth < scenario.assertions.minDepth + ? [`Expected depth >=${scenario.assertions.minDepth}.`] + : []), + ]; + return { details, failures, passed: failures.length === 0 }; +} + +export function evaluateStructuralFidelity( + scenario: GenerationReliabilityScenario, + document: unknown, +): StructuralFidelityResult { + if (!isUnknownRecord(document) || document["type"] !== scenario.diagramType) { + return { + details: {}, + failures: [ + `Response did not contain a ${scenario.diagramType} document.`, + ], + passed: false, + }; + } + const spec = document["spec"]; + if (!isUnknownRecord(spec)) { + return { + details: {}, + failures: ["Response document omitted its spec."], + passed: false, + }; + } + return scenario.diagramType === "flowchart" + ? flowchartFidelity(scenario, spec) + : mindmapFidelity(scenario, spec); +} + +export function selectProbeScenarios( + scenarioId: string | undefined, +): readonly GenerationReliabilityScenario[] { + const selectedId = scenarioId?.trim(); + return selectedId + ? generationReliabilityScenarios.filter( + (scenario) => scenario.id === selectedId, + ) + : generationReliabilityScenarios; +} + +const runProbe = Effect.fn("generationReliabilityProbe.run")(function* ( + endpoint: string, + scenario: GenerationReliabilityScenario, + runNumber: number, +) { + const startedAt = yield* Clock.currentTimeMillis; + const response = yield* Effect.tryPromise({ + try: (signal) => + fetch(endpoint, { + body: JSON.stringify({ + cacheMode: "fresh", + prompt: scenario.prompt, + type: scenario.diagramType, + }), + headers: { + "Cache-Control": "no-store", + "Content-Type": "application/json", + "X-Sketchi-Client": "generation-reliability-probe", + }, + method: "POST", + signal, + }), + catch: (cause) => + GenerationProbeRequestError.make({ + cause, + message: `Request for ${scenario.id} failed.`, + }), + }).pipe( + Effect.timeoutOrElse({ + duration: REQUEST_TIMEOUT_MS, + orElse: () => + Effect.fail( + GenerationProbeRequestError.make({ + cause: new Error("Request timed out."), + message: `Request for ${scenario.id} timed out after ${REQUEST_TIMEOUT_MS} ms.`, + }), + ), + }), + ); + const body = yield* Effect.tryPromise({ + try: () => response.json(), + catch: (cause) => + GenerationProbeRequestError.make({ + cause, + message: `Response for ${scenario.id} was not JSON.`, + }), + }); + const finishedAt = yield* Clock.currentTimeMillis; + const durationMs = Math.round(finishedAt - startedAt); + if (!response.ok || !isUnknownRecord(body) || body["ok"] !== true) { + const status = isUnknownRecord(body) ? body["status"] : undefined; + const issues = isUnknownRecord(body) ? body["issues"] : undefined; + const message = Array.isArray(issues) + ? issues + .filter(isUnknownRecord) + .map((issue) => stringValue(issue, "message")) + .filter((value): value is string => Boolean(value)) + .slice(0, 3) + .join(" | ") + : ""; + return { + details: {}, + durationMs, + failures: [ + `HTTP ${response.status}; status=${String(status ?? "unknown")} ${message}`.trim(), + ], + passed: false, + runNumber, + scenarioId: scenario.id, + statusCode: response.status, + } satisfies ProbeRunResult; + } + const diagram = body["diagram"]; + const document = isUnknownRecord(diagram) ? diagram["document"] : undefined; + return { + ...evaluateStructuralFidelity(scenario, document), + durationMs, + runNumber, + scenarioId: scenario.id, + statusCode: response.status, + } satisfies ProbeRunResult; +}); + +function repeatCount(): number { + const configured = Number.parseInt( + process.env["SKETCHI_PROBE_REPEATS"] ?? String(DEFAULT_REPEATS), + 10, + ); + return Number.isInteger(configured) && configured >= DEFAULT_REPEATS + ? configured + : DEFAULT_REPEATS; +} + +const main = Effect.gen(function* () { + const endpoint = + process.env["SKETCHI_GENERATE_ENDPOINT"]?.trim() || DEFAULT_ENDPOINT; + const repeats = repeatCount(); + const scenarios = selectProbeScenarios(process.env["SKETCHI_PROBE_SCENARIO"]); + if (scenarios.length === 0) { + return yield* GenerationProbeRequestError.make({ + cause: new Error("Unknown generation reliability scenario."), + message: `No generation reliability scenario matched ${process.env["SKETCHI_PROBE_SCENARIO"] ?? "the configured id"}.`, + }); + } + const inputs = scenarios.flatMap((scenario) => + Array.from({ length: repeats }, (_, index) => ({ + runNumber: index + 1, + scenario, + })), + ); + const results = yield* Effect.forEach( + inputs, + ({ runNumber, scenario }) => + runProbe(endpoint, scenario, runNumber).pipe( + Effect.catch((error) => + Effect.succeed({ + details: {}, + durationMs: 0, + failures: [error.message], + passed: false, + runNumber, + scenarioId: scenario.id, + statusCode: 0, + } satisfies ProbeRunResult), + ), + ), + { concurrency: 1 }, + ); + for (const result of results) { + console.log( + `${result.scenarioId} run ${result.runNumber}: ${result.passed ? "PASS" : "FAIL"} (${result.durationMs} ms) ${JSON.stringify(result.passed ? result.details : result.failures)}`, + ); + } + console.log("\n| Scenario | Passed | Runs | Pass rate |"); + console.log("| --- | ---: | ---: | ---: |"); + for (const scenario of scenarios) { + const matching = results.filter( + (result) => result.scenarioId === scenario.id, + ); + const passed = matching.filter((result) => result.passed).length; + console.log( + `| ${scenario.id} | ${passed} | ${matching.length} | ${Math.round((passed / matching.length) * 100)}% |`, + ); + } + const passed = results.filter((result) => result.passed).length; + console.log( + `| **Total** | **${passed}** | **${results.length}** | **${Math.round((passed / results.length) * 100)}%** |`, + ); + console.log(`\nEndpoint: ${endpoint}`); + if (passed !== results.length) process.exitCode = 1; +}); + +const entryPointPath = process.argv[1]; +if (entryPointPath && import.meta.url === pathToFileURL(entryPointPath).href) { + NodeRuntime.runMain(main); +} diff --git a/tools/project-graph.test.ts b/tools/project-graph.test.ts index 39970908..fe6a5e85 100644 --- a/tools/project-graph.test.ts +++ b/tools/project-graph.test.ts @@ -106,6 +106,7 @@ const approvedRuntimeBoundaryFiles = [ "packages/diagram/scenarios/src/cli.ts", "packages/diagram/scenarios/src/live-generator.ts", "scripts/pipelines/r2-catalog-smoke.mjs", + "tools/generation-reliability-probe.ts", "tools/harness-eval.ts", ]; const approvedManagedPromiseSiteCounts: Record = { @@ -189,6 +190,7 @@ const approvedManagedPromiseSiteCounts: Record = { "packages/studio/projects/src/server/bucket.ts": 23, "packages/studio/projects/src/server/http.ts": 2, "scripts/pipelines/r2-catalog-smoke.mjs": 10, + "tools/generation-reliability-probe.ts": 4, "tools/harness-eval.ts": 8, "tools/sketchi-generators/src/generators/diagram-type/diagram-type.spec.ts": 6, "tools/sketchi-generators/src/generators/diagram-type/diagram-type.ts": 3,