test(generation): add structural reliability battery

apps/eval-harness/src/lib/generate-scenario.ts

@@ -19,7 +19,7 @@ import {
   summarizeGenerationCandidate,
 } from "@sketchi/diagram-generation";
 import {
-  getScenario,
+  getGenerationScenario,
   toDiagramGenerationPrompt,
 } from "@sketchi/diagram-scenarios";
 import {
@@ -242,7 +242,7 @@ const runClient = Effect.fn("evalHarness.generateScenario.runClient")(
 
     return yield* Effect.gen(function* () {
       const scenario = yield* Effect.try({
-        try: () => getScenario(scenarioId),
+        try: () => getGenerationScenario(scenarioId),
         catch: (cause) =>
           DiagramGenerationInputError.make({
             cause,

apps/playground/src/server/generation/api.server.test.ts

@@ -1,4 +1,7 @@
-import type { CloudflareAiGatewayProvider } from "@sketchi/diagram-generation";
+import type {
+  CloudflareAiGateway,
+  CloudflareAiGatewayProvider,
+} from "@sketchi/diagram-generation";
 import { describe, expect, it } from "vitest";
 
 import type { StudioEnv } from "../bindings/studio-env.server";
@@ -29,18 +32,23 @@ const flowchartIr = {
   style: { accentColor: "#0f766e", backgroundColor: "#ffffff" },
 };
 
-function fakeAiGateway(text: string): CloudflareAiGatewayProvider {
+function fakeAiGateway(
+  text: string,
+  observeRun?: (input: Parameters<CloudflareAiGateway["run"]>[0]) => void,
+): CloudflareAiGatewayProvider {
   return {
     gateway: () => ({
-      run: () =>
-        Promise.resolve(
+      run: (input) => {
+        observeRun?.(input);
+        return Promise.resolve(
           new Response(
             JSON.stringify({
               candidates: [{ content: { parts: [{ text }] } }],
             }),
             { status: 200, headers: { "content-type": "application/json" } },
           ),
-        ),
+        );
+      },
       getUrl: () => Promise.resolve("https://gateway.invalid"),
     }),
   };
@@ -94,6 +102,32 @@ describe("public generate endpoint", () => {
     expect(generation.provider).toBe("cloudflare-google-ai-studio");
   });
 
+  it("requests fresh provider output for reliability probes", async () => {
+    const observedRuns: Array<Parameters<CloudflareAiGateway["run"]>[0]> = [];
+    const env: StudioEnv = {
+      AI: fakeAiGateway(JSON.stringify(flowchartIr), (input) => {
+        observedRuns.push(input);
+      }),
+    };
+    const response = await generateRequest(env, {
+      cacheMode: "fresh",
+      prompt: "Map release approval with pass and revise branches",
+      type: "flowchart",
+    });
+
+    expect(response.status).toBe(200);
+    expect(observedRuns).toHaveLength(1);
+    expect(observedRuns[0]).toEqual(
+      expect.objectContaining({
+        headers: expect.objectContaining({
+          "Cache-Control": "no-store",
+          "cf-aig-skip-cache": "true",
+          Pragma: "no-cache",
+        }),
+      }),
+    );
+  });
+
   it("rejects an empty prompt with a typed invalid-input contract", async () => {
     const env: StudioEnv = { AI: fakeAiGateway(JSON.stringify(flowchartIr)) };
     const response = await generateRequest(env, { prompt: "   " });

apps/playground/src/server/generation/api.server.ts

@@ -73,6 +73,7 @@ interface GenerateSuccess {
 type GenerateResult = GenerateSuccess | GenerateFailure;
 
 const GenerateRequestSchema = Schema.Struct({
+  cacheMode: Schema.optional(Schema.Literals(["default", "fresh"])),
   prompt: Schema.String,
   type: Schema.optional(Schema.Literals(["flowchart", "mindmap"])),
   model: Schema.optional(Schema.String),
@@ -381,13 +382,15 @@ export const handleGenerateDiagramRequest = Effect.fn(
     );
   }
   const type: DiagramGenerationType = input.type ?? "flowchart";
+  const generationInput = {
+    ...(input.cacheMode ? { cacheMode: input.cacheMode } : {}),
+    prompt,
+    type,
+    ...(input.model ? { model: input.model } : {}),
+  };
 
   const candidateResult = yield* withTelemetryCorrelation(
-    generation.generate({
-      prompt,
-      type,
-      ...(input.model ? { model: input.model } : {}),
-    }),
+    generation.generate(generationInput),
     { attemptId: usageContext.attemptId, runId: usageContext.runId },
   ).pipe(
     Effect.match({
@@ -397,15 +400,15 @@ export const handleGenerateDiagramRequest = Effect.fn(
   );
   if (!candidateResult.ok) {
     return yield* finish(
-      { prompt, type, ...(input.model ? { model: input.model } : {}) },
+      generationInput,
       generationErrorFailure(candidateResult.error),
     );
   }
   const candidate = candidateResult.candidate;
   const malformed = malformedCandidateFailure(candidate);
   if (malformed || !candidate.diagram) {
     return yield* finish(
-      { prompt, type, ...(input.model ? { model: input.model } : {}) },
+      generationInput,
       malformed ??
         failure("malformed_output", [
           issue(
@@ -419,7 +422,7 @@ export const handleGenerateDiagramRequest = Effect.fn(
   }
   if (candidate.diagram.type !== type) {
     return yield* finish(
-      { prompt, type, ...(input.model ? { model: input.model } : {}) },
+      generationInput,
       failure("invalid_generated_document", [
         issue(
           "invalid_generated_document",

apps/playground/src/server/generation/service.server.ts

@@ -1,13 +1,11 @@
 import "@tanstack/react-start/server-only";
 
-import type {
-  FlowchartDiagram,
-  MindmapDiagram,
-} from "@sketchi/diagram-core";
+import type { FlowchartDiagram, MindmapDiagram } from "@sketchi/diagram-core";
 import {
   CloudflareAiGatewayBinding,
   CloudflareGoogleAiStudioClientLive,
   CloudflareGoogleAiStudioConfig,
+  type DiagramGenerationCacheMode,
   DiagramGenerationClient,
   DiagramGenerationConfigurationError,
   type DiagramGenerationCandidate,
@@ -25,6 +23,7 @@ const DEFAULT_GATEWAY_ID = "google-ai-studio";
 const DEFAULT_MODEL = "google/gemini-3.1-flash-lite";
 
 export interface GenerateDiagramServiceInput {
+  readonly cacheMode?: DiagramGenerationCacheMode;
   readonly model?: string;
   readonly prompt: string;
   readonly type: DiagramGenerationType;
@@ -33,7 +32,11 @@ export interface GenerateDiagramServiceInput {
 export interface PlaygroundGenerationShape {
   readonly generate: (
     input: GenerateDiagramServiceInput,
-  ) => Effect.Effect<DiagramGenerationCandidate, DiagramGenerationError, PlaygroundBindings>;
+  ) => Effect.Effect<
+    DiagramGenerationCandidate,
+    DiagramGenerationError,
+    PlaygroundBindings
+  >;
   readonly defaultModel: (env: StudioEnv) => string;
 }
 
@@ -154,6 +157,7 @@ export const PlaygroundGenerationLive = Layer.succeed(PlaygroundGeneration, {
     );
 
     const request = {
+      ...(input.cacheMode ? { cacheMode: input.cacheMode } : {}),
       model,
       prompt: {
         id: "sketchi-generate",

package.json

@@ -24,6 +24,7 @@
     "eval:harness": "jiti tools/harness-eval.ts",
     "lint": "nx run-many -t lint",
     "preview": "nx run-many -t preview --parallel",
+    "probe:generation-reliability": "jiti tools/generation-reliability-probe.ts",
     "r2sql:codemode:resources": "node scripts/pipelines/codemode-r2-catalog.mjs resources",
     "r2sql:codemode:verify-run": "node scripts/pipelines/codemode-r2-catalog.mjs verify-run",
     "test:tools": "vitest run --config tools/vitest.config.mts",

packages/diagram/excalidraw/src/lib/convert.test.ts

@@ -484,6 +484,115 @@ describe("convertSceneToExcalidraw", () => {
     }
   });
 
+  it("routes same-rank branch arrows without crossing nodes in other ranks", () => {
+    const scene = convertSceneToExcalidraw(
+      renderIntermediateDiagram({
+        id: "ecommerce-return-process",
+        title: "Ecommerce returns with 18 steps",
+        type: "flowchart",
+        nodes: [
+          { id: "end-reject", label: "Closure: Rejected", kind: "end" },
+          {
+            id: "notify-final",
+            label: "Notify Customer: Finalized",
+            kind: "process",
+          },
+          {
+            id: "restock-dec",
+            label: "Restocking Decision",
+            kind: "decision",
+          },
+          {
+            id: "partial-refund",
+            label: "Process Partial Refund",
+            kind: "process",
+          },
+          {
+            id: "notify-label",
+            label: "Notify Customer: Label Ready",
+            kind: "process",
+          },
+          {
+            id: "refund-method",
+            label: "Refund Method Decision",
+            kind: "decision",
+          },
+          {
+            id: "notify-receipt",
+            label: "Notify Customer: Received",
+            kind: "process",
+          },
+          { id: "restock", label: "Restock Item", kind: "process" },
+          { id: "pickup", label: "Carrier Pickup", kind: "process" },
+          {
+            id: "start",
+            label: "Initiate Return Request",
+            kind: "start",
+          },
+          { id: "end-success", label: "Closure: Success", kind: "end" },
+          {
+            id: "notify-reject",
+            label: "Notify Customer: Rejected",
+            kind: "process",
+          },
+          {
+            id: "eligibility",
+            label: "Eligibility Check",
+            kind: "decision",
+          },
+          { id: "receipt", label: "Warehouse Receipt", kind: "process" },
+          { id: "fraud-check", label: "Fraud Review", kind: "decision" },
+          {
+            id: "label-gen",
+            label: "Generate Return Label",
+            kind: "process",
+          },
+        ],
+        edges: [
+          { id: "e19", source: "notify-final", target: "end-success" },
+          {
+            id: "e15",
+            source: "refund-method",
+            target: "partial-refund",
+            label: "refund",
+          },
+          {
+            id: "e2",
+            source: "eligibility",
+            target: "label-gen",
+            label: "eligible",
+          },
+          {
+            id: "e12",
+            source: "restock-dec",
+            target: "restock",
+            label: "restockable",
+          },
+          { id: "e17", source: "partial-refund", target: "notify-final" },
+          { id: "e6", source: "pickup", target: "receipt" },
+          { id: "e20", source: "notify-reject", target: "end-reject" },
+          { id: "e1", source: "start", target: "eligibility" },
+          {
+            id: "e10",
+            source: "fraud-check",
+            target: "restock-dec",
+            label: "pass",
+          },
+          { id: "e5", source: "notify-label", target: "pickup" },
+          {
+            id: "e13",
+            source: "restock-dec",
+            target: "refund-method",
+            label: "damaged",
+          },
+        ],
+        layout: { direction: "TB", edgeRouting: "orthogonal" },
+      }),
+    );
+
+    expect(validateExcalidrawScene(scene)).toEqual({ ok: true, issues: [] });
+  });
+
   it("routes Agy left-to-right skip edges around intervening row nodes", () => {
     expectFlowchartExportValid({
       id: "enterprise-vendor-onboarding-flow",

packages/diagram/renderer/src/scene.ts

@@ -1469,10 +1469,37 @@ function exteriorLaneRoute(
   const rightLaneX = maxX + HORIZONTAL_GAP / 2 + laneOffset;
   const upperLaneY = minY - VERTICAL_GAP / 2 - laneOffset;
   const lowerLaneY = maxY + VERTICAL_GAP / 2 + laneOffset;
+  const localLaneOffset = ((route.index % 4) * PORT_SPACING) / 2;
+  const localLeftLaneX =
+    Math.min(route.source.x, route.target.x) -
+    HORIZONTAL_GAP / 2 -
+    localLaneOffset;
+  const localRightLaneX =
+    Math.max(
+      route.source.x + route.source.width,
+      route.target.x + route.target.width,
+    ) +
+    HORIZONTAL_GAP / 2 +
+    localLaneOffset;
+  const localUpperLaneY =
+    Math.min(route.source.y, route.target.y) -
+    VERTICAL_GAP / 2 -
+    localLaneOffset;
+  const localLowerLaneY =
+    Math.max(
+      route.source.y + route.source.height,
+      route.target.y + route.target.height,
+    ) +
+    VERTICAL_GAP / 2 +
+    localLaneOffset;
   const preferredX = useLeftLane ? leftLaneX : rightLaneX;
   const alternateX = useLeftLane ? rightLaneX : leftLaneX;
   const preferredY = useUpperLane ? upperLaneY : lowerLaneY;
   const alternateY = useUpperLane ? lowerLaneY : upperLaneY;
+  const preferredLocalX = useLeftLane ? localLeftLaneX : localRightLaneX;
+  const alternateLocalX = useLeftLane ? localRightLaneX : localLeftLaneX;
+  const preferredLocalY = useUpperLane ? localUpperLaneY : localLowerLaneY;
+  const alternateLocalY = useUpperLane ? localLowerLaneY : localUpperLaneY;
   const localStubDistance =
     ROUTE_STUB_LENGTH + (route.index % 4) * PORT_SPACING;
   const stubDistances = [0, localStubDistance];
@@ -1515,6 +1542,18 @@ function exteriorLaneRoute(
     ]);
   };
   const horizontalCandidates = [
+    ...stubDistances.map((stubDistance) =>
+      routeForHorizontalLane(preferredLocalY, stubDistance),
+    ),
+    ...stubDistances.map((stubDistance) =>
+      routeForHorizontalLane(alternateLocalY, stubDistance),
+    ),
+    ...stubDistances.map((stubDistance) =>
+      routeForVerticalLane(preferredLocalX, stubDistance),
+    ),
+    ...stubDistances.map((stubDistance) =>
+      routeForVerticalLane(alternateLocalX, stubDistance),
+    ),
     ...stubDistances.map((stubDistance) =>
       routeForHorizontalLane(preferredY, stubDistance),
     ),
@@ -1529,6 +1568,18 @@ function exteriorLaneRoute(
     ),
   ];
   const verticalCandidates = [
+    ...stubDistances.map((stubDistance) =>
+      routeForVerticalLane(preferredLocalX, stubDistance),
+    ),
+    ...stubDistances.map((stubDistance) =>
+      routeForVerticalLane(alternateLocalX, stubDistance),
+    ),
+    ...stubDistances.map((stubDistance) =>
+      routeForHorizontalLane(preferredLocalY, stubDistance),
+    ),
+    ...stubDistances.map((stubDistance) =>
+      routeForHorizontalLane(alternateLocalY, stubDistance),
+    ),
     ...stubDistances.map((stubDistance) =>
       routeForVerticalLane(preferredX, stubDistance),
     ),

packages/diagram/scenarios/src/index.ts

@@ -1,4 +1,6 @@
 export * from "./lib/evaluate.js";
 export * from "./lib/fixture-client.js";
+export * from "./lib/generation-reliability.js";
+export * from "./lib/generation-registry.js";
 export * from "./lib/prompt.js";
 export * from "./lib/scenarios.js";