feat(execution): classify reference executor errors as permanent or transient

apps/ai-studio/src/data/support-triage-flow.ts

@@ -108,7 +108,8 @@ Use the exact lowercase keyword on the Type line - it drives downstream routing.
                 id: 'branch-general',
                 sourceHandle: 'source:inner:general',
                 label: 'How-to / Other',
-                conditions: [],
+                // A branch with no conditions never matches; an always-true condition is the catch-all.
+                conditions: [{ x: 'always', y: 'always', comparisonOperator: 'isEqual', logicalOperator: 'AND' }],
               },
             ],
           },

apps/execution-worker/README.md

@@ -80,11 +80,28 @@ own: one executor per node type and the database as the store port.
 - **Namespace:** `TEMPORAL_NAMESPACE`, default `default`. Unlike the task queue this is _not_ shared through the plugin: both apps read it through `@workflow-builder/temporal-connection`, but each environment has to set the same value — a mismatch is silent, the worker simply never sees the backend's submissions.
 - **Workflow ID:** `execution-<executionId>` — deterministic, lets the backend cancel by execution ID. Also owned by the package.
 - **Activity timeouts:** DB activities get 30s / 5 retries; node activities (may call LLMs) get 10m / 2 retries. Exported as `DEFAULT_DATABASE_ACTIVITY_PROFILE` and `DEFAULT_NODE_ACTIVITY_PROFILE`.
-- **Retries per failure:** an executor throwing `PermanentNodeExecutionError` stops on its first attempt; `TransientNodeExecutionError` retries within the profile's limit. An unclassified throw keeps today's behavior. Of the reference executors, only the AI Agent's `ai_not_configured` is classified (permanent) so far; the rest are still unclassified.
+- **Retries per failure:** an executor throwing `PermanentNodeExecutionError` stops on its first attempt; `TransientNodeExecutionError` retries within the profile's limit. An unclassified throw keeps the profile's uniform retry. Every failure the reference executors make a judgment on is in the table below.
 - **Sandbox constraint:** `workflows.ts` is bundled into V8 with no Web APIs. It may only re-export from `@workflowbuilder/temporal/workflow`, never from the package root.
 - **Editing the package:** the worker imports its built `dist`, so run `pnpm build:temporal` after changing `packages/temporal/src`.
 - **Deploys that change the emitted event set:** drain in-flight runs first. Replaying an old run's history against a new emit sequence diverges — see [`replay-audit.md`](../../packages/execution-core/replay-audit.md) rule 9.
 
+### Failure classification
+
+Each judgment is made at the throw site that owns the error. The runner and the adapter never infer a class from a status code, so a consumer's own executors are unaffected by this table.
+
+| Failure                                                        | Class     | Code                                        |
+| -------------------------------------------------------------- | --------- | ------------------------------------------- |
+| AI Agent: provider answered 401 or 403                         | permanent | `provider_auth_rejected`                    |
+| AI Agent: provider answered any other 4xx except 408 and 429   | permanent | `provider_rejected_request`                 |
+| AI Agent: provider answered 429                                | transient | `provider_rate_limited`                     |
+| AI Agent: provider answered 5xx                                | transient | `provider_unavailable`                      |
+| AI Agent: provider answered 408, or the connection failed      | transient | `provider_unreachable`                      |
+| AI Agent: `AI_*` variables missing                             | permanent | `ai_not_configured`                         |
+| AI Agent, Decision: template reference malformed or unresolved | permanent | `template_malformed`, `template_unresolved` |
+| Decision: no branch matched                                    | permanent | `no_branch_matched`                         |
+
+The provider's own error is attached as `cause`, and `node_failed` reports the deepest non-empty cause's text, so the provider's message reaches the UI as it did before classification. A refused connection is the exception: the SDK reports it as `Cannot connect to API:` with nothing after the colon, because the reason sits in an `AggregateError` it wraps — one entry per address tried. Only messages survive the activity boundary, so the classifier attaches the first entry (`connect ECONNREFUSED ::1:11434`) as the cause instead of the SDK error. The classifier's own message, which names the HTTP status, is one level up and visible only in Temporal's failure record. 409 is permanent on purpose, unlike the AI SDK's own retry default: no chat provider is known to answer 409 for a condition a retry would clear. Two kinds of SDK error stay unclassified and keep the profile's uniform retry: a response the SDK could not parse (a 2xx with a non-JSON body, typically a proxy answering with HTML) and errors raised without any provider response (a malformed tool call from the model, no output generated), which describe model behaviour a retry can change. Marking a failure transient does not buy extra attempts — the node profile still caps them.
+
 ## Adding a new engine
 
 1. Create `src/engines/<name>/` with:

apps/execution-worker/src/activities/ai-agent.test.ts

@@ -2,10 +2,15 @@ import { APICallError } from 'ai';
 import { MockLanguageModelV3 } from 'ai/test';
 import { describe, expect, it } from 'vitest';
 
-import type { ExecutionContext } from '@workflow-builder/execution-core';
+import {
+  type ExecutionContext,
+  PermanentNodeExecutionError,
+  TransientNodeExecutionError,
+} from '@workflow-builder/execution-core';
 
 import type { AiAgentNode } from '../domain/ai-studio-nodes';
 import { executeAiAgent } from './ai-agent';
+import { apiCallError } from './api-call-error.fixture';
 
 function context(): ExecutionContext {
   return {
@@ -26,6 +31,14 @@ function aiAgentNode(): AiAgentNode {
   };
 }
 
+function failingModel(statusCode: number, message: string): MockLanguageModelV3 {
+  return new MockLanguageModelV3({
+    doGenerate: () => {
+      throw apiCallError(statusCode, message);
+    },
+  });
+}
+
 describe('executeAiAgent', () => {
   it('returns the model text as the node output', async () => {
     const model = new MockLanguageModelV3({
@@ -46,21 +59,40 @@ describe('executeAiAgent', () => {
   });
 
   it('calls the model exactly once on a retryable failure (retries belong to the Temporal activity policy)', async () => {
+    // statusCode 500 makes isRetryable default to true — a failure the SDK itself would
+    // retry, so this assertion fails if client retries ever come back on.
+    const model = failingModel(500, 'Internal Server Error');
+
+    await expect(executeAiAgent(aiAgentNode(), context(), { model })).rejects.toThrow(TransientNodeExecutionError);
+
+    expect(model.doGenerateCalls).toHaveLength(1);
+  });
+
+  it('surfaces a 5xx as a transient failure that keeps the provider error as its cause', async () => {
+    const model = failingModel(503, 'upstream overloaded');
+
+    await expect(executeAiAgent(aiAgentNode(), context(), { model })).rejects.toMatchObject({
+      code: 'provider_unavailable',
+      cause: expect.any(APICallError),
+    });
+  });
+
+  it('surfaces a rejected API key as a permanent failure', async () => {
+    const model = failingModel(401, 'Incorrect API key provided');
+
+    await expect(executeAiAgent(aiAgentNode(), context(), { model })).rejects.toBeInstanceOf(
+      PermanentNodeExecutionError,
+    );
+  });
+
+  it('rethrows an error that is not a provider response unchanged', async () => {
+    const thrown = new Error('mock exploded');
     const model = new MockLanguageModelV3({
       doGenerate: () => {
-        // statusCode 500 makes isRetryable default to true — the error must be one
-        // the SDK would retry, or this test passes even with retries enabled.
-        throw new APICallError({
-          message: 'Internal Server Error',
-          url: 'https://model.invalid/chat/completions',
-          requestBodyValues: {},
-          statusCode: 500,
-        });
+        throw thrown;
       },
     });
 
-    await expect(executeAiAgent(aiAgentNode(), context(), { model })).rejects.toThrow(APICallError);
-
-    expect(model.doGenerateCalls).toHaveLength(1);
+    await expect(executeAiAgent(aiAgentNode(), context(), { model })).rejects.toBe(thrown);
   });
 });

apps/execution-worker/src/activities/ai-agent.ts

@@ -1,9 +1,15 @@
 import { generateText, stepCountIs } from 'ai';
 
-import { type ExecutionContext, type LoggerPort, resolveTemplate } from '@workflow-builder/execution-core';
+import {
+  type ExecutionContext,
+  type LoggerPort,
+  NodeExecutionError,
+  resolveTemplate,
+} from '@workflow-builder/execution-core';
 
 import type { AiAgentNode } from '../domain/ai-studio-nodes';
 import { createWebSearchTool } from '../tools/web-search';
+import { classifyProviderError } from './provider-error';
 
 // Bounds the agentic tool loop so a misbehaving model can't run up cost.
 const MAX_TOOL_STEPS = 4;
@@ -58,14 +64,15 @@ export async function executeAiAgent(node: AiAgentNode, context: ExecutionContex
 
     return { output: { response: result.text } };
   } catch (error) {
-    // Mirror the `node_failed` SSE payload shape so a log line and the event line up by executionId.
+    const failure = classifyProviderError(error);
+    // executionId joins this line to its node_failed event.
     const message = error instanceof Error ? error.message : String(error);
     deps.logger?.error('llm call failed', {
       workflowId: context.workflowId,
       executionId: context.executionId,
       nodeId: node.id,
-      error: { message },
+      error: { message, ...(failure instanceof NodeExecutionError ? { code: failure.code } : {}) },
     });
-    throw error;
+    throw failure;
   }
 }

apps/execution-worker/src/activities/api-call-error.fixture.ts

@@ -0,0 +1,10 @@
+import { APICallError } from 'ai';
+
+export function apiCallError(statusCode?: number, message = 'provider said no'): APICallError {
+  return new APICallError({
+    message,
+    url: 'https://model.invalid/chat/completions',
+    requestBodyValues: {},
+    statusCode,
+  });
+}

apps/execution-worker/src/activities/provider-error.test.ts

@@ -0,0 +1,133 @@
+import { APICallError, InvalidToolInputError, NoOutputGeneratedError, RetryError } from 'ai';
+import { describe, expect, it } from 'vitest';
+
+import { PermanentNodeExecutionError, TransientNodeExecutionError } from '@workflow-builder/execution-core';
+
+import { apiCallError } from './api-call-error.fixture';
+import { classifyProviderError } from './provider-error';
+
+function retryError(errors: Error[]): RetryError {
+  return new RetryError({ message: 'Failed after 3 attempts', reason: 'maxRetriesExceeded', errors });
+}
+
+// Mirrors how the SDK renders a failed fetch: its own text plus the caught error's
+// message — which is where the dangling colon comes from when there is none.
+function unreachable(cause: Error): APICallError {
+  return new APICallError({
+    message: `Cannot connect to API: ${cause.message}`,
+    url: 'http://localhost:11434/v1/chat/completions',
+    requestBodyValues: {},
+    cause,
+  });
+}
+
+describe('classifyProviderError', () => {
+  it.each([
+    [400, PermanentNodeExecutionError, 'provider_rejected_request'],
+    [401, PermanentNodeExecutionError, 'provider_auth_rejected'],
+    [402, PermanentNodeExecutionError, 'provider_rejected_request'],
+    [403, PermanentNodeExecutionError, 'provider_auth_rejected'],
+    [404, PermanentNodeExecutionError, 'provider_rejected_request'],
+    [409, PermanentNodeExecutionError, 'provider_rejected_request'],
+    [413, PermanentNodeExecutionError, 'provider_rejected_request'],
+    [422, PermanentNodeExecutionError, 'provider_rejected_request'],
+    [408, TransientNodeExecutionError, 'provider_unreachable'],
+    [429, TransientNodeExecutionError, 'provider_rate_limited'],
+    [500, TransientNodeExecutionError, 'provider_unavailable'],
+    [502, TransientNodeExecutionError, 'provider_unavailable'],
+    [503, TransientNodeExecutionError, 'provider_unavailable'],
+    [529, TransientNodeExecutionError, 'provider_unavailable'],
+  ])('HTTP %i becomes a %o with code %s', (status, ErrorClass, code) => {
+    const classified = classifyProviderError(apiCallError(status));
+
+    expect(classified).toBeInstanceOf(ErrorClass);
+    expect(classified).toMatchObject({ code, message: expect.stringContaining(`HTTP ${status}`) });
+  });
+
+  it('a provider error without a status code (the connection failed) is transient', () => {
+    const original = apiCallError();
+    const classified = classifyProviderError(original);
+
+    expect(classified).toBeInstanceOf(TransientNodeExecutionError);
+    expect(classified).toMatchObject({ code: 'provider_unreachable', cause: original });
+  });
+
+  it('reports the refused address rather than the SDK text that has none', () => {
+    // A local provider being down: fetch tries both addresses and reports them in an
+    // AggregateError of its own, which has no message.
+    // eslint-disable-next-line unicorn/error-message -- the empty message is the shape under test
+    const refused = new AggregateError([
+      new Error('connect ECONNREFUSED ::1:11434'),
+      new Error('connect ECONNREFUSED 127.0.0.1:11434'),
+    ]);
+
+    const classified = classifyProviderError(unreachable(refused));
+
+    // The cause is what node_failed shows: the runner reports the deepest non-empty
+    // message, and "Cannot connect to API: " would otherwise be the last one standing.
+    expect(classified).toMatchObject({ code: 'provider_unreachable', cause: refused.errors[0] });
+  });
+
+  it('keeps the provider error when the connection failure named a reason itself', () => {
+    const named = unreachable(new AggregateError([new Error('read ECONNRESET')], 'socket hang up'));
+
+    expect(classifyProviderError(named)).toMatchObject({ cause: named });
+  });
+
+  it('keeps the provider error when no entry named a reason either', () => {
+    // eslint-disable-next-line unicorn/error-message -- the empty messages are the shape under test
+    const blank = unreachable(new AggregateError([new Error()]));
+
+    expect(classifyProviderError(blank)).toMatchObject({ cause: blank });
+  });
+
+  it('classifies the provider error inside a RetryError, so SDK retries do not hide the status', () => {
+    const original = apiCallError(401);
+
+    expect(classifyProviderError(retryError([original]))).toMatchObject({
+      code: 'provider_auth_rejected',
+      cause: original,
+    });
+  });
+
+  it("keeps the provider's own error as the cause, so node_failed still shows the provider's text", () => {
+    const original = apiCallError(401, 'Incorrect API key provided');
+
+    expect(classifyProviderError(original)).toMatchObject({ cause: original });
+  });
+
+  it('passes a 2xx the SDK could not parse through unclassified', () => {
+    // The SDK reports a proxy answering with HTML as an APICallError carrying the 200.
+    const unparsable = apiCallError(200, 'Failed to process successful response');
+
+    expect(classifyProviderError(unparsable)).toBe(unparsable);
+  });
+
+  // Raised without any provider response, so they describe model behaviour a retry can
+  // change. Built from the real SDK classes the README names, not from look-alikes.
+  it.each([
+    ['produced no output', new NoOutputGeneratedError({ message: 'No output generated.' })],
+    [
+      'called a tool with malformed input',
+      new InvalidToolInputError({
+        toolName: 'webSearch',
+        toolInput: '{"query":',
+        cause: new Error('Unexpected end of JSON input'),
+      }),
+    ],
+  ])('passes an SDK error saying the model %s through unclassified', (_label, error) => {
+    expect(classifyProviderError(error)).toBe(error);
+  });
+
+  it.each([
+    ['a plain Error', new Error('boom')],
+    ['a non-error value', 'boom'],
+    ['an object that merely looks like an API error', { statusCode: 500, message: 'not from the SDK' }],
+    ['a status below the 4xx floor', apiCallError(0)],
+    ['a redirect the SDK could not follow', apiCallError(302)],
+    ['a RetryError whose last error is not a provider response', retryError([new Error('mock exploded')])],
+    ['a RetryError that captured no error at all', retryError([])],
+  ])('passes %s through unclassified', (_label, error) => {
+    expect(classifyProviderError(error)).toBe(error);
+  });
+});

apps/execution-worker/src/activities/provider-error.ts

@@ -0,0 +1,59 @@
+import { APICallError, RetryError } from 'ai';
+
+import { PermanentNodeExecutionError, TransientNodeExecutionError } from '@workflow-builder/execution-core';
+
+export function classifyProviderError(error: unknown): unknown {
+  // With SDK retries enabled the provider error arrives wrapped in a RetryError;
+  // unwrap it so the status stays readable if maxRetries ever leaves 0.
+  const providerError = RetryError.isInstance(error) ? error.lastError : error;
+  if (!APICallError.isInstance(providerError)) return error;
+
+  const status = providerError.statusCode;
+  const options = { cause: providerError };
+
+  if (status === undefined) {
+    return new TransientNodeExecutionError('provider_unreachable', 'Could not reach the provider', {
+      cause: connectionFailureCause(providerError),
+    });
+  }
+  if (status === 408) {
+    return new TransientNodeExecutionError('provider_unreachable', 'Provider timed out (HTTP 408)', options);
+  }
+  if (status === 429) {
+    return new TransientNodeExecutionError('provider_rate_limited', 'Provider rate limit hit (HTTP 429)', options);
+  }
+  if (status >= 500) {
+    return new TransientNodeExecutionError(
+      'provider_unavailable',
+      `Provider failed to serve the request (HTTP ${status})`,
+      options,
+    );
+  }
+  if (status === 401 || status === 403) {
+    return new PermanentNodeExecutionError(
+      'provider_auth_rejected',
+      `Provider rejected the API key (HTTP ${status})`,
+      options,
+    );
+  }
+  if (status >= 400) {
+    return new PermanentNodeExecutionError(
+      'provider_rejected_request',
+      `Provider rejected the request (HTTP ${status})`,
+      options,
+    );
+  }
+  return error;
+}
+
+// A refused connection reaches the SDK as an AggregateError with no message of its
+// own — one entry per address tried — so APICallError renders it as "Cannot connect
+// to API: ". Only messages cross the activity boundary, so the entry is picked here,
+// at the throw site, while the array still exists.
+function connectionFailureCause(error: APICallError): unknown {
+  const { cause } = error;
+  if (!(cause instanceof AggregateError) || cause.message !== '') return error;
+
+  const entries: unknown[] = cause.errors;
+  return entries.find((entry) => entry instanceof Error && entry.message !== '') ?? error;
+}