Skip to content

fix(v10/core): Stop reporting caller-handled errors from the OpenAI, Anthropic & Google GenAI instrumentations #23638

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nicohrubec merged 1 commit into from
Aug 26, 2026
Merged

fix(v10/core): Stop reporting caller-handled errors from the OpenAI, Anthropic & Google GenAI instrumentations #23638

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-response-error.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import Anthropic from '@anthropic-ai/sdk';
import * as Sentry from '@sentry/node';
import express from 'express';

function startMockAnthropicServer() {
const app = express();
app.use(express.json());

// Anthropic can hand back an error-shaped body on an otherwise successful (HTTP 200) response.
// The SDK resolves it as data, so the caller never sees a thrown error.
// @see https://docs.anthropic.com/en/api/errors#error-shapes
app.post('/anthropic/v1/messages', (_req, res) => {
res
.status(200)
.set('x-request-id', 'mock-response-error')
.json({ type: 'error', error: { type: 'overloaded_error', message: 'Overloaded' } });
});

return new Promise(resolve => {
const server = app.listen(0, () => resolve(server));
});
}

async function run() {
const server = await startMockAnthropicServer();

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const client = new Anthropic({
apiKey: 'mock-api-key',
baseURL: `http://localhost:${server.address().port}/anthropic`,
});

// Resolves with the error-shaped body; no try/catch because nothing is thrown.
await client.messages.create({
model: 'claude-3-haiku-20240307',
messages: [{ role: 'user', content: 'What is the capital of France?' }],
max_tokens: 100,
});
});

await Sentry.flush(2000);
server.close();
}

run();
66 changes: 28 additions & 38 deletions dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,25 +40,13 @@ describe('Anthropic integration', () => {
transaction: 'main',
};

const EXPECTED_MODEL_ERROR = {
exception: {
values: [
{
type: 'Error',
value: '404 Model not found',
},
],
},
};

const EXPECTED_STREAM_EVENT_HANDLER_MESSAGE = {
message: 'stream event from user-added event listener captured',
};

createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => {
test('preserves .withResponse() and .asResponse() for non-streaming and streaming', async () => {
await createRunner()
.ignore('event')
.expect({
transaction: {
transaction: 'main',
Expand Down Expand Up @@ -94,15 +82,7 @@ describe('Anthropic integration', () => {

createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => {
test('creates anthropic related spans with genAI recording disabled', async () => {
const runner = createRunner();

// The orchestrion path only marks the errored span; unlike the OTel path it does not
// capture the handled `error-model` rejection as an event.
if (!isOrchestrionEnabled()) {
runner.expect({ event: EXPECTED_MODEL_ERROR });
}

await runner
await createRunner()
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE })
.expect({
span: container => {
Expand Down Expand Up @@ -149,15 +129,7 @@ describe('Anthropic integration', () => {

createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('creates anthropic related spans with genAI recording enabled', async () => {
const runner = createRunner();

// The orchestrion path only marks the errored span; unlike the OTel path it does not
// capture the handled `error-model` rejection as an event.
if (!isOrchestrionEnabled()) {
runner.expect({ event: EXPECTED_MODEL_ERROR });
}

await runner
await createRunner()
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE })
.expect({
span: container => {
Expand Down Expand Up @@ -237,7 +209,6 @@ describe('Anthropic integration', () => {
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-options.mjs', (createRunner, test) => {
test('creates anthropic related spans with custom options', async () => {
await createRunner()
.expect({ event: EXPECTED_MODEL_ERROR })
.expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS })
.expect({
span: container => {
Expand Down Expand Up @@ -302,7 +273,6 @@ describe('Anthropic integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument.mjs', (createRunner, test) => {
test('streams produce spans with token usage and metadata (PII false)', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: EXPECTED_STREAM_SPANS_PII_FALSE })
.expect({
span: container => {
Expand Down Expand Up @@ -364,7 +334,6 @@ describe('Anthropic integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('streams record response text when PII true', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: EXPECTED_STREAM_SPANS_PII_TRUE })
.expect({
span: container => {
Expand Down Expand Up @@ -415,7 +384,6 @@ describe('Anthropic integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-stream-nested-create.mjs', 'instrument.mjs', (createRunner, test) => {
test('traces a create() invoked from a stream event handler (dedup does not over-suppress)', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -446,7 +414,6 @@ describe('Anthropic integration', () => {
const EXPECTED_TOOL_CALLS_JSON =
'[{"type":"tool_use","id":"tool_weather_1","name":"weather","input":{"city":"Paris"}}]';
await createRunner()
.ignore('event')
.expect({
transaction: {},
})
Expand Down Expand Up @@ -476,7 +443,6 @@ describe('Anthropic integration', () => {
const EXPECTED_TOOL_CALLS_JSON =
'[{"type":"tool_use","id":"tool_weather_2","name":"weather","input":{"city":"Paris"}}]';
await createRunner()
.ignore('event')
.expect({
transaction: {},
})
Expand Down Expand Up @@ -517,6 +483,9 @@ describe('Anthropic integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-stream-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('handles streaming errors correctly', async () => {
await createRunner()
// Stream errors surface via the MessageStream `error` event; attaching that listener stops it
// being raised as an unhandled rejection, so the instrumentation captures it. This test only
// asserts the spans.
.ignore('event')
.expect({ transaction: EXPECTED_STREAM_ERROR_SPANS })
.expect({
Expand Down Expand Up @@ -574,7 +543,6 @@ describe('Anthropic integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('handles tool errors and model retrieval errors correctly', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: EXPECTED_ERROR_SPANS })
.expect({
span: container => {
Expand Down Expand Up @@ -717,7 +685,6 @@ describe('Anthropic integration', () => {
test('extracts system instructions from messages', async () => {
const expectedInstructions = JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]);
await createRunner()
.ignore('event')
.expect({
transaction: {
transaction: 'main',
Expand Down Expand Up @@ -833,4 +800,27 @@ describe('Anthropic integration', () => {
});
},
);
createEsmAndCjsTests(__dirname, 'scenario-response-error.mjs', 'instrument.mjs', (createRunner, test) => {
test('captures error-shaped responses returned as data', async () => {
await createRunner()
// The API returns the error as data on a 200 response, never as a thrown error to the caller,
// so the instrumentation intentionally captures it as an event.
.unordered()
.expect({
event: {
exception: {
values: [
{
value: 'Overloaded',
mechanism: { type: 'auto.ai.anthropic.anthropic_error', handled: false },
},
],
},
},
})
.expect({ transaction: { transaction: 'main' } })
.start()
.completed();
});
});
});
39 changes: 30 additions & 9 deletions dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ describe('Google GenAI integration', () => {
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => {
test('creates google genai related spans with genAI recording disabled', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -88,7 +87,6 @@ describe('Google GenAI integration', () => {
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('creates google genai related spans with genAI recording enabled', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -139,7 +137,6 @@ describe('Google GenAI integration', () => {
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-options.mjs', (createRunner, test) => {
test('creates google genai related spans with custom options', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -176,7 +173,6 @@ describe('Google GenAI integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-options.mjs', (createRunner, test) => {
test('creates google genai related spans with tool calls', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -237,7 +233,21 @@ describe('Google GenAI integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-streaming.mjs', 'instrument.mjs', (createRunner, test) => {
test('creates google genai streaming spans with genAI recording disabled', async () => {
await createRunner()
.ignore('event')
// The provider surfaces blocked content within the stream and never returns it to the caller as
// a thrown error, so the instrumentation intentionally captures it as an event.
.unordered()
.expect({
event: {
exception: {
values: [
{
value: 'Content blocked: The prompt was blocked due to safety concerns',
mechanism: { type: 'auto.ai.google_genai', handled: false },
},
],
},
},
})
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -295,7 +305,21 @@ describe('Google GenAI integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-streaming.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('creates google genai streaming spans with genAI recording enabled', async () => {
await createRunner()
.ignore('event')
// The provider surfaces blocked content within the stream and never returns it to the caller as
// a thrown error, so the instrumentation intentionally captures it as an event.
.unordered()
.expect({
event: {
exception: {
values: [
{
value: 'Content blocked: The prompt was blocked due to safety concerns',
mechanism: { type: 'auto.ai.google_genai', handled: false },
},
],
},
},
})
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -400,7 +424,6 @@ describe('Google GenAI integration', () => {
(createRunner, test) => {
test('extracts system instructions from messages', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand All @@ -424,7 +447,6 @@ describe('Google GenAI integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => {
test('creates google genai embeddings spans with genAI recording disabled', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -463,7 +485,6 @@ describe('Google GenAI integration', () => {
createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('creates google genai embeddings spans with genAI recording enabled', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down
2 changes: 0 additions & 2 deletions dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,6 @@ describe('OpenAI Tool Calls integration', () => {
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => {
test('creates openai tool calls related spans with genAI recording disabled', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down Expand Up @@ -327,7 +326,6 @@ describe('OpenAI Tool Calls integration', () => {
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('creates openai tool calls related spans with genAI recording enabled', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
Expand Down
Loading
Loading