Skip to content

feat(server-utils)!: Clean up gen ai ops #23076

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msonnb merged 1 commit into from
Aug 6, 2026
Merged

feat(server-utils)!: Clean up gen ai ops #23076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 36 additions & 49 deletions dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,59 +1,41 @@
import type { TransactionEvent } from '@sentry/core';
import type { SerializedStreamedSpanContainer } from '@sentry/core';
import { afterAll, describe, expect } from 'vitest';
import { isOrchestrionEnabled } from '../../../utils';
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner';

// The suite runs twice on CI: once with the OTel `Aws` integration (default) and once with the
// orchestrion diagnostics-channel integration auto-injected (`INJECT_ORCHESTRION`). Both emit the
// same gen_ai spans; only the origin differs.
const ORIGIN = isOrchestrionEnabled() ? 'auto.aws.aws_sdk' : 'auto.otel.aws';

const MODEL_ID = 'anthropic.claude-3-5-sonnet-20240620-v1:0';

function assertBedrockSpans(transaction: TransactionEvent): void {
const spans = transaction.spans ?? [];

expect(transaction.transaction).toBe('Test Transaction');

function assertBedrockSpans(container: SerializedStreamedSpanContainer): void {
// Converse (non-streaming)
expect(spans, 'expected a Bedrock Converse span').toContainEqual(
expect.objectContaining({
description: `chat ${MODEL_ID}`,
origin: ORIGIN,
status: 'ok',
data: expect.objectContaining({
'sentry.origin': ORIGIN,
'gen_ai.provider.name': 'aws.bedrock',
'gen_ai.operation.name': 'chat',
'gen_ai.request.model': MODEL_ID,
'gen_ai.request.max_tokens': 100,
'gen_ai.request.temperature': 0.5,
'gen_ai.request.top_p': 0.9,
'gen_ai.usage.input_tokens': 12,
'gen_ai.usage.output_tokens': 8,
'gen_ai.response.finish_reasons': ['end_turn'],
}),
}),
);
const converseSpan = container.items.find(span => span.name === `chat ${MODEL_ID}`);
expect(converseSpan).toBeDefined();
expect(converseSpan!.status).toBe('ok');
expect(converseSpan!.attributes['sentry.origin'].value).toBe('auto.aws.aws_sdk');
expect(converseSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat');
expect(converseSpan!.attributes['gen_ai.provider.name'].value).toBe('aws.bedrock');
expect(converseSpan!.attributes['gen_ai.operation.name'].value).toBe('chat');
expect(converseSpan!.attributes['gen_ai.request.model'].value).toBe(MODEL_ID);
expect(converseSpan!.attributes['gen_ai.request.max_tokens'].value).toBe(100);
expect(converseSpan!.attributes['gen_ai.request.temperature'].value).toBe(0.5);
expect(converseSpan!.attributes['gen_ai.request.top_p'].value).toBe(0.9);
expect(converseSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(12);
expect(converseSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(8);
expect(converseSpan!.attributes['gen_ai.response.finish_reasons'].value).toEqual(['end_turn']);

// InvokeModel (non-streaming, anthropic.claude request/response body)
expect(spans, 'expected a Bedrock InvokeModel span').toContainEqual(
expect.objectContaining({
origin: ORIGIN,
status: 'ok',
data: expect.objectContaining({
'sentry.origin': ORIGIN,
'gen_ai.provider.name': 'aws.bedrock',
'gen_ai.request.model': MODEL_ID,
'gen_ai.request.max_tokens': 100,
'gen_ai.request.temperature': 0.5,
'gen_ai.request.top_p': 0.9,
'gen_ai.usage.input_tokens': 15,
'gen_ai.usage.output_tokens': 9,
'gen_ai.response.finish_reasons': ['end_turn'],
}),
}),
);
const invokeModelSpan = container.items.find(span => span.name === `generate_content ${MODEL_ID}`);
expect(invokeModelSpan).toBeDefined();
expect(invokeModelSpan!.status).toBe('ok');
expect(invokeModelSpan!.attributes['sentry.origin'].value).toBe('auto.aws.aws_sdk');
expect(invokeModelSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content');
expect(invokeModelSpan!.attributes['gen_ai.provider.name'].value).toBe('aws.bedrock');
expect(invokeModelSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content');
expect(invokeModelSpan!.attributes['gen_ai.request.model'].value).toBe(MODEL_ID);
expect(invokeModelSpan!.attributes['gen_ai.request.max_tokens'].value).toBe(100);
expect(invokeModelSpan!.attributes['gen_ai.request.temperature'].value).toBe(0.5);
expect(invokeModelSpan!.attributes['gen_ai.request.top_p'].value).toBe(0.9);
expect(invokeModelSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15);
expect(invokeModelSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(9);
expect(invokeModelSpan!.attributes['gen_ai.response.finish_reasons'].value).toEqual(['end_turn']);
}

describe('awsIntegration - Bedrock', () => {
Expand All @@ -67,7 +49,12 @@ describe('awsIntegration - Bedrock', () => {
'instrument.mjs',
(createTestRunner, test) => {
test('auto-instruments Bedrock Converse and InvokeModel', { timeout: 90_000 }, async () => {
await createTestRunner().ignore('event').expect({ transaction: assertBedrockSpans }).start().completed();
await createTestRunner()
.ignore('event')
.expect({ transaction: transaction => expect(transaction.transaction).toBe('Test Transaction') })
.expect({ span: assertBedrockSpans })
.start()
.completed();
});
},
{ additionalDependencies: { '@aws-sdk/client-bedrock-runtime': '^3.1046.0' } },
Expand Down
88 changes: 39 additions & 49 deletions dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { afterAll, describe, expect } from 'vitest';
import type { TransactionEvent } from '@sentry/core';
import {
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
Expand Down Expand Up @@ -27,16 +28,14 @@ describe('Anthropic integration', () => {
cleanupChildProcesses();
});

const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = {
transaction: 'main',
};

const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = {
transaction: 'main',
};

const EXPECTED_TRANSACTION_WITH_OPTIONS = {
transaction: 'main',
// `models.retrieve` reports a generic `function` op (model retrieval is not an inference call), so it is
// delivered as a plain transaction span rather than extracted into the gen_ai span container.
const expectModelsSpanOnTransaction = (event: TransactionEvent): void => {
expect(event.transaction).toBe('main');
const modelsSpan = (event.spans ?? []).find(span => span.description === 'models claude-3-haiku-20240307');
expect(modelsSpan).toBeDefined();
expect(modelsSpan!.op).toBe('function');
expect(modelsSpan!.status).toBe('ok');
};

const EXPECTED_MODEL_ERROR = {
Expand Down Expand Up @@ -102,10 +101,10 @@ describe('Anthropic integration', () => {
}

await runner
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE })
.expect({ transaction: expectModelsSpanOnTransaction })
.expect({
span: container => {
expect(container.items).toHaveLength(5);
expect(container.items).toHaveLength(4);
const completionSpan = container.items.find(
span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123',
);
Expand All @@ -126,11 +125,6 @@ describe('Anthropic integration', () => {
expect(tokenCountingSpan).toBeDefined();
expect(tokenCountingSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat');

const modelsSpan = container.items.find(span => span.name === 'models claude-3-haiku-20240307');
expect(modelsSpan).toBeDefined();
expect(modelsSpan!.status).toBe('ok');
expect(modelsSpan!.attributes['sentry.op'].value).toBe('gen_ai.models');

const streamingSpan = container.items.find(
span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123',
);
Expand All @@ -157,10 +151,10 @@ describe('Anthropic integration', () => {
}

await runner
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE })
.expect({ transaction: expectModelsSpanOnTransaction })
.expect({
span: container => {
expect(container.items).toHaveLength(5);
expect(container.items).toHaveLength(4);
const completionSpan = container.items.find(
span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123',
);
Expand Down Expand Up @@ -200,11 +194,6 @@ describe('Anthropic integration', () => {
expect(tokenCountingSpan!.status).toBe('ok');
expect(tokenCountingSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat');

const modelsSpan = container.items.find(span => span.name === 'models claude-3-haiku-20240307');
expect(modelsSpan).toBeDefined();
expect(modelsSpan!.status).toBe('ok');
expect(modelsSpan!.attributes['sentry.op'].value).toBe('gen_ai.models');

// TODO: messages.stream() should produce its own distinct gen_ai span, but it
// currently does not (pre-existing bug). Once fixed, add an additional indexed span assertion.
const streamingSpan = container.items.find(
Expand Down Expand Up @@ -240,10 +229,23 @@ describe('Anthropic integration', () => {
}

await runner
.expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS })
.expect({
transaction: event => {
expect(event.transaction).toBe('main');
const modelsSpan = (event.spans ?? []).find(span => span.description === 'models claude-3-haiku-20240307');
expect(modelsSpan).toBeDefined();
expect(modelsSpan!.op).toBe('function');
expect(modelsSpan!.status).toBe('ok');
expect(modelsSpan!.data[GEN_AI_OPERATION_NAME]).toBe('models');
expect(modelsSpan!.data[GEN_AI_PROVIDER_NAME]).toBe('anthropic');
expect(modelsSpan!.data[GEN_AI_REQUEST_MODEL]).toBe('claude-3-haiku-20240307');
expect(modelsSpan!.data[GEN_AI_RESPONSE_ID]).toBe('claude-3-haiku-20240307');
expect(modelsSpan!.data[GEN_AI_RESPONSE_MODEL]).toBe('claude-3-haiku-20240307');
},
})
.expect({
span: container => {
expect(container.items).toHaveLength(5);
expect(container.items).toHaveLength(4);
const completionSpan = container.items.find(
span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123',
);
Expand All @@ -268,16 +270,6 @@ describe('Anthropic integration', () => {
expect(tokenCountingSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat');
expect(tokenCountingSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat');

const modelsSpan = container.items.find(span => span.name === 'models claude-3-haiku-20240307');
expect(modelsSpan).toBeDefined();
expect(modelsSpan!.status).toBe('ok');
expect(modelsSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('models');
expect(modelsSpan!.attributes['sentry.op'].value).toBe('gen_ai.models');
expect(modelsSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic');
expect(modelsSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307');
expect(modelsSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('claude-3-haiku-20240307');
expect(modelsSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307');

const streamingSpan = container.items.find(
span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123',
);
Expand Down Expand Up @@ -556,31 +548,29 @@ describe('Anthropic integration', () => {
});
});

// Additional error scenarios - Tool errors and model retrieval errors
const EXPECTED_ERROR_SPANS = {
transaction: 'main',
};

createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('handles tool errors and model retrieval errors correctly', async () => {
await createRunner()
.ignore('event')
.expect({ transaction: EXPECTED_ERROR_SPANS })
.expect({
transaction: event => {
expect(event.transaction).toBe('main');
const modelErrorSpan = (event.spans ?? []).find(span => span.description === 'models nonexistent-model');
expect(modelErrorSpan).toBeDefined();
expect(modelErrorSpan!.op).toBe('function');
expect(modelErrorSpan!.status).toBe('internal_error');
expect(modelErrorSpan!.data[GEN_AI_REQUEST_MODEL]).toBe('nonexistent-model');
},
})
.expect({
span: container => {
expect(container.items).toHaveLength(3);
expect(container.items).toHaveLength(2);
const invalidFormatSpan = container.items.find(span => span.name === 'chat invalid-format');
expect(invalidFormatSpan).toBeDefined();
expect(invalidFormatSpan!.status).toBe('error');
expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format');
expect(invalidFormatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat');

const modelErrorSpan = container.items.find(span => span.name === 'models nonexistent-model');
expect(modelErrorSpan).toBeDefined();
expect(modelErrorSpan!.status).toBe('error');
expect(modelErrorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('nonexistent-model');
expect(modelErrorSpan!.attributes['sentry.op'].value).toBe('gen_ai.models');

const toolSuccessSpan = container.items.find(span => span.name === 'chat claude-3-haiku-20240307');
expect(toolSuccessSpan).toBeDefined();
expect(toolSuccessSpan!.status).toBe('ok');
Expand Down
21 changes: 12 additions & 9 deletions packages/server-utils/src/ai/anthropic-ai/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,12 @@ import {
} from '@sentry/conventions/attributes';
import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes';
import type { InstrumentedMethodEntry } from '../core/utils';
import { resolveAIRecordingOptions, setTokenUsageAttributes, wrapPromiseWithMethods } from '../core/utils';
import {
getGenAiSpanOp,
resolveAIRecordingOptions,
setTokenUsageAttributes,
wrapPromiseWithMethods,
} from '../core/utils';
import { ANTHROPIC_METHOD_REGISTRY } from './constants';
import { instrumentAsyncIterableStream, instrumentMessageStream } from './streaming';
import type { AnthropicAiOptions, AnthropicAiResponse, AnthropicAiStreamingEvent, ContentBlock } from './types';
Expand Down Expand Up @@ -67,13 +72,11 @@ export function extractRequestAttributes(
if ('top_k' in params) attributes[GEN_AI_REQUEST_TOP_K] = params.top_k;
if ('frequency_penalty' in params) attributes[GEN_AI_REQUEST_FREQUENCY_PENALTY] = params.frequency_penalty;
if ('max_tokens' in params) attributes[GEN_AI_REQUEST_MAX_TOKENS] = params.max_tokens;
} else if (methodPath === 'models.retrieve' || methodPath === 'models.get') {
// `models.retrieve(model-id)` / `models.get(model-id)` pass the model id as a positional arg
attributes[GEN_AI_REQUEST_MODEL] = args[0];
} else {
if (methodPath === 'models.retrieve' || methodPath === 'models.get') {
// models.retrieve(model-id) and models.get(model-id)
attributes[GEN_AI_REQUEST_MODEL] = args[0];
} else {
attributes[GEN_AI_REQUEST_MODEL] = 'unknown';
}
attributes[GEN_AI_REQUEST_MODEL] = 'unknown';
}

return attributes;
Expand Down Expand Up @@ -204,7 +207,7 @@ function handleStreamingRequest<T extends unknown[], R>(
const model = requestAttributes[GEN_AI_REQUEST_MODEL] ?? 'unknown';
const spanConfig = {
name: `${operationName} ${model}`,
op: `gen_ai.${operationName}`,
op: getGenAiSpanOp(operationName),
attributes: requestAttributes as Record<string, SpanAttributeValue>,
};

Expand Down Expand Up @@ -310,7 +313,7 @@ function instrumentMethod<T extends unknown[], R>(
const instrumentedPromise = startSpan(
{
name: `${operationName} ${model}`,
op: `gen_ai.${operationName}`,
op: getGenAiSpanOp(operationName),
attributes: requestAttributes as Record<string, SpanAttributeValue>,
},
span => {
Expand Down
15 changes: 15 additions & 0 deletions packages/server-utils/src/ai/core/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
GEN_AI_USAGE_OUTPUT_TOKENS,
GEN_AI_USAGE_TOTAL_TOKENS,
} from '@sentry/conventions/attributes';
import { GENERAL_FUNCTION_SPAN_OP } from '@sentry/conventions/op';

export interface AIRecordingOptions {
recordInputs?: boolean;
Expand All @@ -41,6 +42,20 @@ export interface InstrumentedMethodEntry {
*/
export type InstrumentedMethodRegistry = Record<string, InstrumentedMethodEntry>;

// Operation names that are not inference calls: `models` retrieves model metadata and `unknown` is
// the fallback for methods with no registered operation. Neither should surface as a `gen_ai.*` op
// (an unknown string must not masquerade as a convention), so they map to the generic `function` op.
// The operation name itself is preserved on `gen_ai.operation.name`.
const NON_INFERENCE_OPERATIONS = new Set(['models', 'unknown']);

/**
* Derive the span op from a gen_ai operation name. Inference operations become `gen_ai.<operation>`;
* non-inference operations (`models`, `unknown`) become the generic `function` op.
*/
export function getGenAiSpanOp(operationName: string): string {
return NON_INFERENCE_OPERATIONS.has(operationName) ? GENERAL_FUNCTION_SPAN_OP : `gen_ai.${operationName}`;
}

@cursor cursor Bot Aug 6, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing migration notes for span ops

Medium Severity

This ! breaking change renames many user-visible span ops (handler → request_handler.* / handler.nestjs, function → framework-specific ops, gen_ai.models / ai.run → function, Bedrock rpc → gen_ai.*, Hono internal http.server → hono.request, Hapi plugin handler → plugin.hapi) but does not update MIGRATION.md. That violates the Breaking Changes review rule: similar op renames are already documented there for dashboards, alerts, and ignoreSpans filters.

Additional Locations (2)
Fix in Cursor Fix in Web

Triggered by project rule: PR Review Guidelines for Cursor Bot

Reviewed by Cursor Bugbot for commit 531ee7b. Configure here.

@msonnb msonnb Aug 6, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

soon curslinger, soon


/**
* Resolves AI recording options by falling back to the client's `dataCollection.genAI` settings.
* Precedence: explicit option > dataCollection.genAI > true (genAI data collected by default)
Expand Down
6 changes: 3 additions & 3 deletions packages/server-utils/src/ai/google-genai/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import {
GEN_AI_USAGE_TOTAL_TOKENS,
} from '@sentry/conventions/attributes';
import type { InstrumentedMethodEntry } from '../core/utils';
import { buildMethodPath, extractSystemInstructions, resolveAIRecordingOptions } from '../core/utils';
import { buildMethodPath, extractSystemInstructions, getGenAiSpanOp, resolveAIRecordingOptions } from '../core/utils';
import { GOOGLE_GENAI_METHOD_REGISTRY, GOOGLE_GENAI_SYSTEM_NAME } from './constants';
import { instrumentStream } from './streaming';
import type { Candidate, ContentPart, GoogleGenAIOptions, GoogleGenAIResponse } from './types';
Expand Down Expand Up @@ -277,7 +277,7 @@ function instrumentMethod<T extends unknown[], R>(
return startSpanManual(
{
name: `${operationName} ${model}`,
op: `gen_ai.${operationName}`,
op: getGenAiSpanOp(operationName),
attributes: requestAttributes,
},
async (span: Span) => {
Expand Down Expand Up @@ -306,7 +306,7 @@ function instrumentMethod<T extends unknown[], R>(
return startSpan(
{
name: `${operationName} ${model}`,
op: `gen_ai.${operationName}`,
op: getGenAiSpanOp(operationName),
attributes: requestAttributes,
},
(span: Span) => {
Expand Down
3 changes: 2 additions & 1 deletion packages/server-utils/src/ai/openai/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import type { InstrumentedMethodEntry } from '../core/utils';
import {
buildMethodPath,
extractSystemInstructions,
getGenAiSpanOp,
resolveAIRecordingOptions,
wrapPromiseWithMethods,
} from '../core/utils';
Expand Down Expand Up @@ -146,7 +147,7 @@ function instrumentMethod<T extends unknown[], R>(

const spanConfig = {
name: `${operationName} ${model}`,
op: `gen_ai.${operationName}`,
op: getGenAiSpanOp(operationName),
attributes: requestAttributes as Record<string, SpanAttributeValue>,
};

Expand Down
Loading
Loading