From c45a02e31f08b104f4375bd73ee26772baaeece2 Mon Sep 17 00:00:00 2001 From: Charly Gomez Date: Mon, 24 Aug 2026 13:06:24 +0200 Subject: [PATCH 1/4] fix(server-utils): Support openai v7 in auto-instrumentation The orchestrion config capped `openai` at `<7`, so the code transformer never matched v7 modules and auto-instrumentation silently produced no spans on the current major. Bump the cap to `<8`. v7's only breaking change was requiring Node.js 22, so every instrumented match point is unchanged. Add a unit test covering 4.x-7.x against the real transformer, and an `openai/v7` integration suite pinning 7.5.0, skipped below Node 22. Fixes #23511 Co-Authored-By: Claude Opus 5 (1M context) --- .../suites/tracing/openai/v7/instrument.mjs | 17 ++ .../tracing/openai/v7/scenario-chat.mjs | 278 ++++++++++++++++++ .../tracing/openai/v7/scenario-embeddings.mjs | 81 +++++ .../suites/tracing/openai/v7/test.ts | 166 +++++++++++ .../src/orchestrion/config/openai.ts | 8 +- .../orchestrion/openai-version-range.test.ts | 81 +++++ 6 files changed, 627 insertions(+), 4 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-chat.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-embeddings.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts create mode 100644 packages/server-utils/test/orchestrion/openai-version-range.test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs new file mode 100644 index 000000000000..2097d76a4eff --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + traceLifecycle: 'static', + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, + transport: loggingTransport, + beforeSendTransaction: event => { + if (event.transaction.includes('/openai/')) { + return null; + } + return event; + }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-chat.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-chat.mjs new file mode 100644 index 000000000000..6031b6861f5b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-chat.mjs @@ -0,0 +1,278 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + // Chat completions endpoint + app.post('/openai/chat/completions', (req, res) => { + const { model, stream } = req.body; + + // Handle error model + if (model === 'error-model') { + res.status(500).set('x-request-id', 'mock-request-error').end('Internal server error'); + return; + } + + if (stream) { + // Streaming response + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + + const chunks = [ + { + id: 'chatcmpl-stream-123', + object: 'chat.completion.chunk', + created: 1677652300, + model: model, + choices: [{ delta: { role: 'assistant', content: '' }, index: 0 }], + }, + { + id: 'chatcmpl-stream-123', + object: 'chat.completion.chunk', + created: 1677652300, + model: model, + choices: [{ delta: { content: 'Hello from OpenAI streaming!' }, index: 0 }], + }, + { + id: 'chatcmpl-stream-123', + object: 'chat.completion.chunk', + created: 1677652300, + model: model, + choices: [{ delta: {}, index: 0, finish_reason: 'stop' }], + usage: { + prompt_tokens: 12, + completion_tokens: 18, + total_tokens: 30, + }, + }, + ]; + + chunks.forEach((chunk, index) => { + setTimeout(() => { + res.write(`data: ${JSON.stringify(chunk)}\n\n`); + if (index === chunks.length - 1) { + res.write('data: [DONE]\n\n'); + res.end(); + } + }, index * 10); + }); + } else { + // Non-streaming response + res.send({ + id: 'chatcmpl-mock123', + object: 'chat.completion', + created: 1677652288, + model: model, + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'Hello from OpenAI mock!', + }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 15, + total_tokens: 25, + }, + }); + } + }); + + // Responses API endpoint + app.post('/openai/responses', (req, res) => { + const { model, stream } = req.body; + + // Handle error model + if (model === 'error-model') { + res.status(500).set('x-request-id', 'mock-request-error').end('Internal server error'); + return; + } + + if (stream) { + // Streaming response - using event-based format with 'response' field + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + + const events = [ + { + type: 'response.created', + response: { + id: 'resp_stream_456', + object: 'response', + created_at: 1677652310, + model: model, + status: 'in_progress', + }, + }, + { + type: 'response.output_text.delta', + delta: 'Streaming response to: Test streaming responses API', + response: { + id: 'resp_stream_456', + model: model, + created_at: 1677652310, + }, + }, + { + type: 'response.completed', + response: { + id: 'resp_stream_456', + object: 'response', + created_at: 1677652310, + model: model, + status: 'completed', + output_text: 'Test streaming responses API', + usage: { + input_tokens: 6, + output_tokens: 10, + total_tokens: 16, + }, + }, + }, + ]; + + events.forEach((event, index) => { + setTimeout(() => { + res.write(`data: ${JSON.stringify(event)}\n\n`); + if (index === events.length - 1) { + res.write('data: [DONE]\n\n'); + res.end(); + } + }, index * 10); + }); + } else { + // Non-streaming response + res.send({ + id: 'resp_mock456', + object: 'response', + created_at: 1677652290, + model: model, + output: [ + { + type: 'message', + id: 'msg_mock_output_1', + status: 'completed', + role: 'assistant', + content: [ + { + type: 'output_text', + text: `Response to: ${req.body.input}`, + annotations: [], + }, + ], + }, + ], + output_text: `Response to: ${req.body.input}`, + status: 'completed', + usage: { + input_tokens: 5, + output_tokens: 8, + total_tokens: 13, + }, + }); + } + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new OpenAI({ + baseURL: `http://localhost:${server.address().port}/openai`, + apiKey: 'mock-api-key', + }); + + // First test: basic chat completion + await client.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'What is the capital of France?' }, + ], + temperature: 0.7, + max_tokens: 100, + }); + + // Second test: responses API + await client.responses.create({ + model: 'gpt-3.5-turbo', + input: 'Translate this to French: Hello', + instructions: 'You are a translator', + }); + + // Third test: error handling in chat completions + try { + await client.chat.completions.create({ + model: 'error-model', + messages: [{ role: 'user', content: 'This will fail' }], + }); + } catch { + // Error is expected and handled + } + + // Fourth test: chat completions streaming + const stream1 = await client.chat.completions.create({ + model: 'gpt-4', + messages: [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'Tell me about streaming' }, + ], + stream: true, + temperature: 0.8, + }); + + // Consume the stream to trigger span instrumentation + for await (const chunk of stream1) { + // Stream chunks are processed automatically by instrumentation + void chunk; // Prevent unused variable warning + } + + // Fifth test: responses API streaming + const stream2 = await client.responses.create({ + model: 'gpt-4', + input: 'Test streaming responses API', + instructions: 'You are a streaming assistant', + stream: true, + }); + + for await (const chunk of stream2) { + void chunk; + } + + // Sixth test: error handling in streaming context + try { + const errorStream = await client.chat.completions.create({ + model: 'error-model', + messages: [{ role: 'user', content: 'This will fail' }], + stream: true, + }); + + // Try to consume the stream (this should not execute) + for await (const chunk of errorStream) { + void chunk; + } + } catch { + // Error is expected and handled + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-embeddings.mjs new file mode 100644 index 000000000000..42c6a94c5199 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-embeddings.mjs @@ -0,0 +1,81 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + // Embeddings endpoint + app.post('/openai/embeddings', (req, res) => { + const { model } = req.body; + + // Handle error model + if (model === 'error-model') { + res.status(404).set('x-request-id', 'mock-request-123').end('Model not found'); + return; + } + + // Return embeddings response + res.send({ + object: 'list', + data: [ + { + object: 'embedding', + embedding: [0.1, 0.2, 0.3], + index: 0, + }, + ], + model: model, + usage: { + prompt_tokens: 10, + total_tokens: 10, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new OpenAI({ + baseURL: `http://localhost:${server.address().port}/openai`, + apiKey: 'mock-api-key', + }); + + // First test: embeddings API + await client.embeddings.create({ + input: 'Embedding test!', + model: 'text-embedding-3-small', + dimensions: 1536, + encoding_format: 'float', + }); + + // Second test: embeddings API error model + try { + await client.embeddings.create({ + input: 'Error embedding test!', + model: 'error-model', + }); + } catch { + // Error is expected and handled + } + + // Third test: embeddings API with multiple inputs + await client.embeddings.create({ + input: ['First input text', 'Second input text', 'Third input text'], + model: 'text-embedding-3-small', + }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts new file mode 100644 index 000000000000..a4d753800997 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -0,0 +1,166 @@ +import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; +import { afterAll, expect } from 'vitest'; +import { + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; +import { conditionalTest } from '../../../../utils/index'; + +// openai 7 requires Node.js 22 — its only breaking change over v6 — so this suite is skipped on the +// Node 20 CI leg rather than pinning the whole matrix to the newer runtime. +conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + // The per-attribute extraction is version-independent and already covered by the v4/v5 suite. + // What a new major puts at risk is whether the transformer still matches the resource files at + // all, so these assert that each instrumented `create` produces a span with the right shape. + createEsmAndCjsTests( + __dirname, + 'scenario-chat.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('instruments chat completions, the responses API and streaming on openai v7', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(6); + + const chatCompletionSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan).toBeDefined(); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 25 }); + + // The responses API is a separate instrumented resource file from chat completions. + const responsesSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456', + ); + expect(responsesSpan).toBeDefined(); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + + // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it + // is the part most likely to break if the `Stream` shape changes across a major. + const streamingSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingSpan).toBeDefined(); + expect(streamingSpan!.name).toBe('chat gpt-4'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 12 }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 18 }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); + + const errorSpan = container.items.find(span => span.name === 'chat error-model' && span.status !== 'ok'); + expect(errorSpan).toBeDefined(); + expect(errorSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + openai: '7.5.0', + }, + }, + ); + + // Embeddings publish to a different channel than chat, and match a resource file at the package + // root rather than under a nested directory, so they need their own coverage. + createEsmAndCjsTests( + __dirname, + 'scenario-embeddings.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('instruments the embeddings API on openai v7', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + const embeddingSpans = container.items.filter( + span => span.attributes[GEN_AI_OPERATION_NAME]?.value === 'embeddings', + ); + expect(embeddingSpans).toHaveLength(3); + + const singleEmbeddingSpan = embeddingSpans.find( + span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', + ); + expect(singleEmbeddingSpan).toBeDefined(); + expect(singleEmbeddingSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + + const errorEmbeddingSpan = embeddingSpans.find(span => span.name === 'embeddings error-model'); + expect(errorEmbeddingSpan).toBeDefined(); + expect(errorEmbeddingSpan!.status).not.toBe('ok'); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + openai: '7.5.0', + }, + }, + ); +}); diff --git a/packages/server-utils/src/orchestrion/config/openai.ts b/packages/server-utils/src/orchestrion/config/openai.ts index 1d087688c952..d2a74307249a 100644 --- a/packages/server-utils/src/orchestrion/config/openai.ts +++ b/packages/server-utils/src/orchestrion/config/openai.ts @@ -7,25 +7,25 @@ export const openaiConfig = [ // `filePath` exactly, hence one entry per built file (`.js` for `require`, `.mjs` for `import`). ...['resources/chat/completions/completions.js', 'resources/chat/completions/completions.mjs'].map(filePath => ({ channelName: 'chat', - module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Completions', methodName: 'create', kind: 'Auto' as const }, })), // OpenAI responses API — same `create(body, options)` shape as chat completions. ...['resources/responses/responses.js', 'resources/responses/responses.mjs'].map(filePath => ({ channelName: 'chat', - module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Responses', methodName: 'create', kind: 'Auto' as const }, })), // OpenAI embeddings API — same `create(body, options)` shape as chat completions. ...['resources/embeddings.js', 'resources/embeddings.mjs'].map(filePath => ({ channelName: 'embeddings', - module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Embeddings', methodName: 'create', kind: 'Auto' as const }, })), // OpenAI conversations API — same `create(body, options)` shape as chat completions. ...['resources/conversations/conversations.js', 'resources/conversations/conversations.mjs'].map(filePath => ({ channelName: 'chat', - module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Conversations', methodName: 'create', kind: 'Auto' as const }, })), ] satisfies InstrumentationConfig[]; diff --git a/packages/server-utils/test/orchestrion/openai-version-range.test.ts b/packages/server-utils/test/orchestrion/openai-version-range.test.ts new file mode 100644 index 000000000000..16d357f6336a --- /dev/null +++ b/packages/server-utils/test/orchestrion/openai-version-range.test.ts @@ -0,0 +1,81 @@ +import { createCodeTransformer } from '@apm-js-collab/code-transformer-bundler-plugins/core'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; +import { orchestrionTransformOptions } from '../../src/orchestrion/bundler/options'; + +// The transformer resolves the instrumented package's version from its on-disk `package.json`, +// so each version under test needs its own package root. +const roots: string[] = []; + +// Mirrors the shape of openai's generated resource files: a `class X extends APIResource` whose +// `create(body, options)` returns the client's thenable `APIPromise`. This shape is byte-identical +// across openai 4-7 — v7's only breaking change was requiring Node.js 22 — so a single fixture +// legitimately stands in for every major in the declared range. +function resourceSource(className: string): string { + return `'use strict';\nclass ${className} extends APIResource {\n create(body, options) {\n return this._client.post('/x', { body, ...options });\n }\n}\nexports.${className} = ${className};\n`; +} + +const MATCH_POINTS = [ + { + filePath: 'resources/chat/completions/completions.js', + className: 'Completions', + channel: 'orchestrion:openai:chat', + }, + { filePath: 'resources/responses/responses.js', className: 'Responses', channel: 'orchestrion:openai:chat' }, + { filePath: 'resources/embeddings.js', className: 'Embeddings', channel: 'orchestrion:openai:embeddings' }, + { + filePath: 'resources/conversations/conversations.js', + className: 'Conversations', + channel: 'orchestrion:openai:chat', + }, +] as const; + +function makeOpenAiPackage(version: string): string { + const root = mkdtempSync(join(tmpdir(), 'orch-openai-')); + roots.push(root); + const dir = join(root, 'node_modules', 'openai'); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'package.json'), JSON.stringify({ name: 'openai', version, type: 'commonjs' })); + for (const { filePath, className } of MATCH_POINTS) { + const file = join(dir, filePath); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, resourceSource(className)); + } + return dir; +} + +function transformMatchPoints(version: string): (string | null)[] { + const dir = makeOpenAiPackage(version); + const transformer = createCodeTransformer(orchestrionTransformOptions({})); + return MATCH_POINTS.map(({ filePath, className }) => { + const result = transformer.transform(resourceSource(className), join(dir, filePath)); + return result?.code ?? null; + }); +} + +describe('orchestrion config — openai declared version range', () => { + afterAll(() => { + for (const root of roots) { + rmSync(root, { recursive: true, force: true }); + } + }); + + // Guards against the declared range falling behind upstream: openai 7 shipped while the range + // still said `<7`, which silently dropped instrumentation for everyone on the current major. + it.each(['4.0.0', '5.18.1', '6.49.0', '7.0.0', '7.5.0'])('instruments every match point on openai %s', version => { + const codes = transformMatchPoints(version); + + codes.forEach((code, i) => { + expect(code, `${MATCH_POINTS[i]!.filePath} was not transformed`).not.toBeNull(); + expect(code).toContain(MATCH_POINTS[i]!.channel); + }); + }); + + // The upper bound is deliberate, not incidental: a new major must be checked against the real + // package before it is declared, so it has to stay excluded until someone does that. + it.each(['3.9.0', '8.0.0'])('leaves openai %s untouched, outside the declared range', version => { + expect(transformMatchPoints(version)).toEqual([null, null, null, null]); + }); +}); From 23b371cac6a5c7426cdca8e8fcbdbbf25d6270b5 Mon Sep 17 00:00:00 2001 From: Charly Gomez Date: Mon, 24 Aug 2026 14:25:09 +0200 Subject: [PATCH 2/4] test(server-utils): Use span streaming in openai v7 suite The v7 suite inherited `traceLifecycle: 'static'` from the v6 suite, which carries it from #22589. Span streaming is the default lifecycle, so exercise that instead. Under streaming there are no transaction envelopes and the mock server's own spans stream in alongside the gen_ai ones, so the transaction expectation and the fixed item count are dropped in favour of selecting the spans under test. Co-Authored-By: Claude Opus 5 (1M context) --- .../suites/tracing/openai/v7/instrument.mjs | 8 +------- .../suites/tracing/openai/v7/test.ts | 9 +++------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs index 2097d76a4eff..5e0b6fb5592f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs @@ -2,16 +2,10 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, + traceLifecycle: 'stream', }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index a4d753800997..620ab8b62989 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -33,11 +33,8 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { test('instruments chat completions, the responses API and streaming on openai v7', async () => { await createRunner() .ignore('event') - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(6); - const chatCompletionSpan = container.items.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); @@ -81,8 +78,9 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { value: 'gpt-3.5-turbo', }); - // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it - // is the part most likely to break if the `Stream` shape changes across a major. + // Response streaming (`stream: true`) ends its span from the patched async iterator + // rather than `beforeSpanEnd`, so it is the part most likely to break if openai's + // `Stream` shape changes across a major. const streamingSpan = container.items.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); @@ -123,7 +121,6 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { test('instruments the embeddings API on openai v7', async () => { await createRunner() .ignore('event') - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { const embeddingSpans = container.items.filter( From 75f626549603dbb4e09e3548a3c66b5db52cbdcd Mon Sep 17 00:00:00 2001 From: Charly Gomez Date: Mon, 24 Aug 2026 14:52:03 +0200 Subject: [PATCH 3/4] test(server-utils): Fix flaky span assertions in openai v7 suite The previous commit moved the whole suite to `traceLifecycle: 'stream'`, which made the chat test fail on the Node 26 CI leg. The span buffer flushes on a 5s timer per trace and the runner asserts against one envelope at a time, so once the scenario ran past that window its later spans landed in a second envelope the assertion never saw. Reproduced on Node 22 by delaying the scenario past 5s. Put the exhaustive assertions back on the static lifecycle, where every span arrives in one envelope, and cover streaming with a separate instrument file that asserts only on the first call's span. This mirrors how the v4/v5 suite splits `instrument.mjs` and `instrument-span-streaming.mjs`. Co-Authored-By: Claude Opus 5 (1M context) --- .../openai/v7/instrument-span-streaming.mjs | 11 ++++ .../suites/tracing/openai/v7/instrument.mjs | 8 ++- .../suites/tracing/openai/v7/test.ts | 51 +++++++++++++++++-- 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs new file mode 100644 index 000000000000..5e0b6fb5592f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, + transport: loggingTransport, + traceLifecycle: 'stream', +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs index 5e0b6fb5592f..2097d76a4eff 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs @@ -2,10 +2,16 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ + traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - traceLifecycle: 'stream', + beforeSendTransaction: event => { + if (event.transaction.includes('/openai/')) { + return null; + } + return event; + }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 620ab8b62989..6d7eb552c219 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -33,8 +33,11 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { test('instruments chat completions, the responses API and streaming on openai v7', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { + expect(container.items).toHaveLength(6); + const chatCompletionSpan = container.items.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', ); @@ -78,9 +81,8 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { value: 'gpt-3.5-turbo', }); - // Response streaming (`stream: true`) ends its span from the patched async iterator - // rather than `beforeSpanEnd`, so it is the part most likely to break if openai's - // `Stream` shape changes across a major. + // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it + // is the part most likely to break if the `Stream` shape changes across a major. const streamingSpan = container.items.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', ); @@ -121,6 +123,7 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { test('instruments the embeddings API on openai v7', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { const embeddingSpans = container.items.filter( @@ -160,4 +163,46 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { }, }, ); + // Span streaming is the default trace lifecycle, so cover it too. The span buffer flushes on a 5s + // timer per trace, which splits this scenario's spans across envelopes under CI load, and the + // runner asserts against one envelope at a time — so this only asserts on the first call's span, + // which is always in the first flush. The exhaustive assertions above stay on the static lifecycle, + // where every span arrives in a single envelope. + createEsmAndCjsTests( + __dirname, + 'scenario-chat.mjs', + 'instrument-span-streaming.mjs', + (createRunner, test) => { + test('instruments chat completions on openai v7 with span streaming enabled', async () => { + await createRunner() + .ignore('event') + .expect({ + span: container => { + const chatCompletionSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan).toBeDefined(); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + openai: '7.5.0', + }, + }, + ); }); From 63f33239d04fc644d841c832a922dcc3a287144f Mon Sep 17 00:00:00 2001 From: Charly Gomez Date: Mon, 24 Aug 2026 17:24:03 +0200 Subject: [PATCH 4/4] test(server-utils): Drop openai version-range unit test The declared range will be covered by canary tests against the latest published version instead, which exercises the real package rather than a synthetic one. Refs #23515 Co-Authored-By: Claude Opus 5 (1M context) --- .../orchestrion/openai-version-range.test.ts | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 packages/server-utils/test/orchestrion/openai-version-range.test.ts diff --git a/packages/server-utils/test/orchestrion/openai-version-range.test.ts b/packages/server-utils/test/orchestrion/openai-version-range.test.ts deleted file mode 100644 index 16d357f6336a..000000000000 --- a/packages/server-utils/test/orchestrion/openai-version-range.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { createCodeTransformer } from '@apm-js-collab/code-transformer-bundler-plugins/core'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { afterAll, describe, expect, it } from 'vitest'; -import { orchestrionTransformOptions } from '../../src/orchestrion/bundler/options'; - -// The transformer resolves the instrumented package's version from its on-disk `package.json`, -// so each version under test needs its own package root. -const roots: string[] = []; - -// Mirrors the shape of openai's generated resource files: a `class X extends APIResource` whose -// `create(body, options)` returns the client's thenable `APIPromise`. This shape is byte-identical -// across openai 4-7 — v7's only breaking change was requiring Node.js 22 — so a single fixture -// legitimately stands in for every major in the declared range. -function resourceSource(className: string): string { - return `'use strict';\nclass ${className} extends APIResource {\n create(body, options) {\n return this._client.post('/x', { body, ...options });\n }\n}\nexports.${className} = ${className};\n`; -} - -const MATCH_POINTS = [ - { - filePath: 'resources/chat/completions/completions.js', - className: 'Completions', - channel: 'orchestrion:openai:chat', - }, - { filePath: 'resources/responses/responses.js', className: 'Responses', channel: 'orchestrion:openai:chat' }, - { filePath: 'resources/embeddings.js', className: 'Embeddings', channel: 'orchestrion:openai:embeddings' }, - { - filePath: 'resources/conversations/conversations.js', - className: 'Conversations', - channel: 'orchestrion:openai:chat', - }, -] as const; - -function makeOpenAiPackage(version: string): string { - const root = mkdtempSync(join(tmpdir(), 'orch-openai-')); - roots.push(root); - const dir = join(root, 'node_modules', 'openai'); - mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, 'package.json'), JSON.stringify({ name: 'openai', version, type: 'commonjs' })); - for (const { filePath, className } of MATCH_POINTS) { - const file = join(dir, filePath); - mkdirSync(dirname(file), { recursive: true }); - writeFileSync(file, resourceSource(className)); - } - return dir; -} - -function transformMatchPoints(version: string): (string | null)[] { - const dir = makeOpenAiPackage(version); - const transformer = createCodeTransformer(orchestrionTransformOptions({})); - return MATCH_POINTS.map(({ filePath, className }) => { - const result = transformer.transform(resourceSource(className), join(dir, filePath)); - return result?.code ?? null; - }); -} - -describe('orchestrion config — openai declared version range', () => { - afterAll(() => { - for (const root of roots) { - rmSync(root, { recursive: true, force: true }); - } - }); - - // Guards against the declared range falling behind upstream: openai 7 shipped while the range - // still said `<7`, which silently dropped instrumentation for everyone on the current major. - it.each(['4.0.0', '5.18.1', '6.49.0', '7.0.0', '7.5.0'])('instruments every match point on openai %s', version => { - const codes = transformMatchPoints(version); - - codes.forEach((code, i) => { - expect(code, `${MATCH_POINTS[i]!.filePath} was not transformed`).not.toBeNull(); - expect(code).toContain(MATCH_POINTS[i]!.channel); - }); - }); - - // The upper bound is deliberate, not incidental: a new major must be checked against the real - // package before it is declared, so it has to stay excluded until someone does that. - it.each(['3.9.0', '8.0.0'])('leaves openai %s untouched, outside the declared range', version => { - expect(transformMatchPoints(version)).toEqual([null, null, null, null]); - }); -});