diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs new file mode 100644 index 000000000000..5e0b6fb5592f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, + transport: loggingTransport, + traceLifecycle: 'stream', +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs new file mode 100644 index 000000000000..2097d76a4eff --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + traceLifecycle: 'static', + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, + transport: loggingTransport, + beforeSendTransaction: event => { + if (event.transaction.includes('/openai/')) { + return null; + } + return event; + }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-chat.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-chat.mjs new file mode 100644 index 000000000000..6031b6861f5b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-chat.mjs @@ -0,0 +1,278 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + // Chat completions endpoint + app.post('/openai/chat/completions', (req, res) => { + const { model, stream } = req.body; + + // Handle error model + if (model === 'error-model') { + res.status(500).set('x-request-id', 'mock-request-error').end('Internal server error'); + return; + } + + if (stream) { + // Streaming response + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + + const chunks = [ + { + id: 'chatcmpl-stream-123', + object: 'chat.completion.chunk', + created: 1677652300, + model: model, + choices: [{ delta: { role: 'assistant', content: '' }, index: 0 }], + }, + { + id: 'chatcmpl-stream-123', + object: 'chat.completion.chunk', + created: 1677652300, + model: model, + choices: [{ delta: { content: 'Hello from OpenAI streaming!' }, index: 0 }], + }, + { + id: 'chatcmpl-stream-123', + object: 'chat.completion.chunk', + created: 1677652300, + model: model, + choices: [{ delta: {}, index: 0, finish_reason: 'stop' }], + usage: { + prompt_tokens: 12, + completion_tokens: 18, + total_tokens: 30, + }, + }, + ]; + + chunks.forEach((chunk, index) => { + setTimeout(() => { + res.write(`data: ${JSON.stringify(chunk)}\n\n`); + if (index === chunks.length - 1) { + res.write('data: [DONE]\n\n'); + res.end(); + } + }, index * 10); + }); + } else { + // Non-streaming response + res.send({ + id: 'chatcmpl-mock123', + object: 'chat.completion', + created: 1677652288, + model: model, + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'Hello from OpenAI mock!', + }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 15, + total_tokens: 25, + }, + }); + } + }); + + // Responses API endpoint + app.post('/openai/responses', (req, res) => { + const { model, stream } = req.body; + + // Handle error model + if (model === 'error-model') { + res.status(500).set('x-request-id', 'mock-request-error').end('Internal server error'); + return; + } + + if (stream) { + // Streaming response - using event-based format with 'response' field + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + + const events = [ + { + type: 'response.created', + response: { + id: 'resp_stream_456', + object: 'response', + created_at: 1677652310, + model: model, + status: 'in_progress', + }, + }, + { + type: 'response.output_text.delta', + delta: 'Streaming response to: Test streaming responses API', + response: { + id: 'resp_stream_456', + model: model, + created_at: 1677652310, + }, + }, + { + type: 'response.completed', + response: { + id: 'resp_stream_456', + object: 'response', + created_at: 1677652310, + model: model, + status: 'completed', + output_text: 'Test streaming responses API', + usage: { + input_tokens: 6, + output_tokens: 10, + total_tokens: 16, + }, + }, + }, + ]; + + events.forEach((event, index) => { + setTimeout(() => { + res.write(`data: ${JSON.stringify(event)}\n\n`); + if (index === events.length - 1) { + res.write('data: [DONE]\n\n'); + res.end(); + } + }, index * 10); + }); + } else { + // Non-streaming response + res.send({ + id: 'resp_mock456', + object: 'response', + created_at: 1677652290, + model: model, + output: [ + { + type: 'message', + id: 'msg_mock_output_1', + status: 'completed', + role: 'assistant', + content: [ + { + type: 'output_text', + text: `Response to: ${req.body.input}`, + annotations: [], + }, + ], + }, + ], + output_text: `Response to: ${req.body.input}`, + status: 'completed', + usage: { + input_tokens: 5, + output_tokens: 8, + total_tokens: 13, + }, + }); + } + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new OpenAI({ + baseURL: `http://localhost:${server.address().port}/openai`, + apiKey: 'mock-api-key', + }); + + // First test: basic chat completion + await client.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'What is the capital of France?' }, + ], + temperature: 0.7, + max_tokens: 100, + }); + + // Second test: responses API + await client.responses.create({ + model: 'gpt-3.5-turbo', + input: 'Translate this to French: Hello', + instructions: 'You are a translator', + }); + + // Third test: error handling in chat completions + try { + await client.chat.completions.create({ + model: 'error-model', + messages: [{ role: 'user', content: 'This will fail' }], + }); + } catch { + // Error is expected and handled + } + + // Fourth test: chat completions streaming + const stream1 = await client.chat.completions.create({ + model: 'gpt-4', + messages: [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'Tell me about streaming' }, + ], + stream: true, + temperature: 0.8, + }); + + // Consume the stream to trigger span instrumentation + for await (const chunk of stream1) { + // Stream chunks are processed automatically by instrumentation + void chunk; // Prevent unused variable warning + } + + // Fifth test: responses API streaming + const stream2 = await client.responses.create({ + model: 'gpt-4', + input: 'Test streaming responses API', + instructions: 'You are a streaming assistant', + stream: true, + }); + + for await (const chunk of stream2) { + void chunk; + } + + // Sixth test: error handling in streaming context + try { + const errorStream = await client.chat.completions.create({ + model: 'error-model', + messages: [{ role: 'user', content: 'This will fail' }], + stream: true, + }); + + // Try to consume the stream (this should not execute) + for await (const chunk of errorStream) { + void chunk; + } + } catch { + // Error is expected and handled + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-embeddings.mjs new file mode 100644 index 000000000000..42c6a94c5199 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-embeddings.mjs @@ -0,0 +1,81 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + // Embeddings endpoint + app.post('/openai/embeddings', (req, res) => { + const { model } = req.body; + + // Handle error model + if (model === 'error-model') { + res.status(404).set('x-request-id', 'mock-request-123').end('Model not found'); + return; + } + + // Return embeddings response + res.send({ + object: 'list', + data: [ + { + object: 'embedding', + embedding: [0.1, 0.2, 0.3], + index: 0, + }, + ], + model: model, + usage: { + prompt_tokens: 10, + total_tokens: 10, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new OpenAI({ + baseURL: `http://localhost:${server.address().port}/openai`, + apiKey: 'mock-api-key', + }); + + // First test: embeddings API + await client.embeddings.create({ + input: 'Embedding test!', + model: 'text-embedding-3-small', + dimensions: 1536, + encoding_format: 'float', + }); + + // Second test: embeddings API error model + try { + await client.embeddings.create({ + input: 'Error embedding test!', + model: 'error-model', + }); + } catch { + // Error is expected and handled + } + + // Third test: embeddings API with multiple inputs + await client.embeddings.create({ + input: ['First input text', 'Second input text', 'Third input text'], + model: 'text-embedding-3-small', + }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts new file mode 100644 index 000000000000..6d7eb552c219 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -0,0 +1,208 @@ +import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; +import { afterAll, expect } from 'vitest'; +import { + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; +import { conditionalTest } from '../../../../utils/index'; + +// openai 7 requires Node.js 22 — its only breaking change over v6 — so this suite is skipped on the +// Node 20 CI leg rather than pinning the whole matrix to the newer runtime. +conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + // The per-attribute extraction is version-independent and already covered by the v4/v5 suite. + // What a new major puts at risk is whether the transformer still matches the resource files at + // all, so these assert that each instrumented `create` produces a span with the right shape. + createEsmAndCjsTests( + __dirname, + 'scenario-chat.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('instruments chat completions, the responses API and streaming on openai v7', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(6); + + const chatCompletionSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan).toBeDefined(); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 25 }); + + // The responses API is a separate instrumented resource file from chat completions. + const responsesSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456', + ); + expect(responsesSpan).toBeDefined(); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + + // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it + // is the part most likely to break if the `Stream` shape changes across a major. + const streamingSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingSpan).toBeDefined(); + expect(streamingSpan!.name).toBe('chat gpt-4'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 12 }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 18 }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); + + const errorSpan = container.items.find(span => span.name === 'chat error-model' && span.status !== 'ok'); + expect(errorSpan).toBeDefined(); + expect(errorSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + openai: '7.5.0', + }, + }, + ); + + // Embeddings publish to a different channel than chat, and match a resource file at the package + // root rather than under a nested directory, so they need their own coverage. + createEsmAndCjsTests( + __dirname, + 'scenario-embeddings.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('instruments the embeddings API on openai v7', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + const embeddingSpans = container.items.filter( + span => span.attributes[GEN_AI_OPERATION_NAME]?.value === 'embeddings', + ); + expect(embeddingSpans).toHaveLength(3); + + const singleEmbeddingSpan = embeddingSpans.find( + span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', + ); + expect(singleEmbeddingSpan).toBeDefined(); + expect(singleEmbeddingSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + + const errorEmbeddingSpan = embeddingSpans.find(span => span.name === 'embeddings error-model'); + expect(errorEmbeddingSpan).toBeDefined(); + expect(errorEmbeddingSpan!.status).not.toBe('ok'); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + openai: '7.5.0', + }, + }, + ); + // Span streaming is the default trace lifecycle, so cover it too. The span buffer flushes on a 5s + // timer per trace, which splits this scenario's spans across envelopes under CI load, and the + // runner asserts against one envelope at a time — so this only asserts on the first call's span, + // which is always in the first flush. The exhaustive assertions above stay on the static lifecycle, + // where every span arrives in a single envelope. + createEsmAndCjsTests( + __dirname, + 'scenario-chat.mjs', + 'instrument-span-streaming.mjs', + (createRunner, test) => { + test('instruments chat completions on openai v7 with span streaming enabled', async () => { + await createRunner() + .ignore('event') + .expect({ + span: container => { + const chatCompletionSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan).toBeDefined(); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + openai: '7.5.0', + }, + }, + ); +}); diff --git a/packages/server-utils/src/orchestrion/config/openai.ts b/packages/server-utils/src/orchestrion/config/openai.ts index 8227611b3615..c43313cbb5af 100644 --- a/packages/server-utils/src/orchestrion/config/openai.ts +++ b/packages/server-utils/src/orchestrion/config/openai.ts @@ -8,25 +8,25 @@ export const openaiConfig = [ // `filePath` exactly, hence one entry per built file (`.js` for `require`, `.mjs` for `import`). ...['resources/chat/completions/completions.js', 'resources/chat/completions/completions.mjs'].map(filePath => ({ channelName: 'chat', - module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Completions', methodName: 'create', kind: 'Auto' as const }, })), // OpenAI responses API — same `create(body, options)` shape as chat completions. ...['resources/responses/responses.js', 'resources/responses/responses.mjs'].map(filePath => ({ channelName: 'chat', - module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Responses', methodName: 'create', kind: 'Auto' as const }, })), // OpenAI embeddings API — same `create(body, options)` shape as chat completions. ...['resources/embeddings.js', 'resources/embeddings.mjs'].map(filePath => ({ channelName: 'embeddings', - module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Embeddings', methodName: 'create', kind: 'Auto' as const }, })), // OpenAI conversations API — same `create(body, options)` shape as chat completions. ...['resources/conversations/conversations.js', 'resources/conversations/conversations.mjs'].map(filePath => ({ channelName: 'chat', - module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Conversations', methodName: 'create', kind: 'Auto' as const }, })), ] satisfies InstrumentationConfig[];