From cab92a62559b19d700cca271ca71aaaacc7c08ef Mon Sep 17 00:00:00 2001 From: Charly Gomez Date: Thu, 6 Aug 2026 13:41:11 +0200 Subject: [PATCH 1/2] fix(node): Support @google/genai v2 in auto-instrumentation The orchestrion config capped `@google/genai` at `<2`, so the code transformer never matched v2 modules and auto-instrumentation silently did nothing (manual instrumentation was unaffected); this bumps the cap to `<3`. Adds a unit test against the real transformer and a `google-genai-v2` integration suite pinning `@google/genai@^2`. Fixes #23066 Co-Authored-By: Claude Opus 4.8 --- .../tracing/google-genai-v2/instrument.mjs | 18 +++ .../google-genai-v2/scenario-embeddings.mjs | 77 +++++++++++ .../tracing/google-genai-v2/scenario.mjs | 113 +++++++++++++++ .../suites/tracing/google-genai-v2/test.ts | 130 ++++++++++++++++++ .../src/orchestrion/config/google-genai.ts | 9 +- .../test/orchestrion/google-genai.test.ts | 75 ++++++++++ 6 files changed, 418 insertions(+), 4 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai-v2/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai-v2/scenario-embeddings.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai-v2/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts create mode 100644 packages/server-utils/test/orchestrion/google-genai.test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/instrument.mjs new file mode 100644 index 000000000000..c83310d623f6 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/instrument.mjs @@ -0,0 +1,18 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + traceLifecycle: 'static', + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, + transport: loggingTransport, + beforeSendTransaction: event => { + // Filter out mock express server transactions + if (event.transaction.includes('/v1beta')) { + return null; + } + return event; + }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/scenario-embeddings.mjs new file mode 100644 index 000000000000..166e741cf199 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/scenario-embeddings.mjs @@ -0,0 +1,77 @@ +import { GoogleGenAI } from '@google/genai'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockGoogleGenAIServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1beta/models/:model\\:batchEmbedContents', (req, res) => { + const model = req.params.model; + + if (model === 'error-model') { + res.status(404).set('x-request-id', 'mock-request-123').end('Model not found'); + return; + } + + res.send({ + embeddings: [ + { + values: [0.1, 0.2, 0.3, 0.4, 0.5], + }, + ], + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockGoogleGenAIServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new GoogleGenAI({ + apiKey: 'mock-api-key', + httpOptions: { baseUrl: `http://localhost:${server.address().port}` }, + }); + + // Test 1: Basic embedContent with string contents + await client.models.embedContent({ + model: 'text-embedding-004', + contents: 'What is the capital of France?', + }); + + // Test 2: Error handling + try { + await client.models.embedContent({ + model: 'error-model', + contents: 'This will fail', + }); + } catch { + // Expected error + } + + // Test 3: embedContent with array contents + await client.models.embedContent({ + model: 'text-embedding-004', + contents: [ + { + role: 'user', + parts: [{ text: 'First input text' }], + }, + { + role: 'user', + parts: [{ text: 'Second input text' }], + }, + ], + }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/scenario.mjs new file mode 100644 index 000000000000..2d7a09e6f638 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/scenario.mjs @@ -0,0 +1,113 @@ +import { GoogleGenAI } from '@google/genai'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockGoogleGenAIServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1beta/models/:model\\:generateContent', (req, res) => { + const model = req.params.model; + + if (model === 'error-model') { + res.status(404).set('x-request-id', 'mock-request-123').end('Model not found'); + return; + } + + res.send({ + candidates: [ + { + content: { + parts: [ + { + text: 'Mock response from Google GenAI!', + }, + ], + role: 'model', + }, + finishReason: 'stop', + index: 0, + }, + ], + usageMetadata: { + promptTokenCount: 8, + candidatesTokenCount: 12, + totalTokenCount: 20, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockGoogleGenAIServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new GoogleGenAI({ + apiKey: 'mock-api-key', + httpOptions: { baseUrl: `http://localhost:${server.address().port}` }, + }); + + // Test 1: chats.create and sendMessage flow + // This should generate two spans: one for chats.create and one for sendMessage + const chat = client.chats.create({ + model: 'gemini-1.5-pro', + config: { + temperature: 0.8, + topP: 0.9, + maxOutputTokens: 150, + systemInstruction: 'You are a friendly robot who likes to be funny.', + }, + history: [ + { + role: 'user', + parts: [{ text: 'Hello, how are you?' }], + }, + ], + }); + + await chat.sendMessage({ + message: 'Tell me a joke', + }); + + // Test 2: models.generateContent + await client.models.generateContent({ + model: 'gemini-1.5-flash', + config: { + temperature: 0.7, + topP: 0.9, + maxOutputTokens: 100, + }, + contents: [ + { + role: 'user', + parts: [{ text: 'What is the capital of France?' }], + }, + ], + }); + + // Test 3: Error handling + try { + await client.models.generateContent({ + model: 'error-model', + contents: [ + { + role: 'user', + parts: [{ text: 'This will fail' }], + }, + ], + }); + } catch { + // Expected error + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts new file mode 100644 index 000000000000..97f65eeb0680 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts @@ -0,0 +1,130 @@ +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MAX_TOKENS, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_REQUEST_TOP_P, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; + +const EXPECTED_ORIGIN = 'auto.ai.google_genai'; + +// `@google/genai` v2 restructured the `Models` class so `embedContent` is a constructor-assigned arrow +// property rather than a class method (v1 shape). The orchestrion config caps at `<3`, so the code +// transformer only injects the diagnostics channels for v2 when the range includes it — this suite pins +// `^2` and re-runs the core auto-instrumentation assertions to guard that path. The v1 suite lives in +// `../google-genai`; the scenario files are byte-identical because v2 kept the public API surface. +describe('Google GenAI integration (v2)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('auto-instruments chat and generateContent on @google/genai v2', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + expect(container.items.map(span => span.name).sort()).toEqual([ + 'chat gemini-1.5-pro', + 'generate_content error-model', + 'generate_content gemini-1.5-flash', + ]); + + const chatSpan = container.items.find(span => span.name === 'chat gemini-1.5-pro'); + expect(chatSpan!.status).toBe('ok'); + expect(chatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(chatSpan!.attributes['sentry.origin'].value).toBe(EXPECTED_ORIGIN); + expect(chatSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(chatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-pro'); + expect(chatSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(chatSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(chatSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + + const generateContentSpan = container.items.find( + span => span.name === 'generate_content gemini-1.5-flash', + ); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(EXPECTED_ORIGIN); + expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-flash'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + + const errorSpan = container.items.find(span => span.name === 'generate_content error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); + }, + }) + .start() + .completed(); + }); + }, + { additionalDependencies: { '@google/genai': '^2' } }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-embeddings.mjs', + 'instrument.mjs', + (createRunner, test) => { + // `embedContent` is the member that changed shape in v2; asserting its span proves the + // `className`/`methodName` selector still matches the constructor-assigned arrow. + test('auto-instruments embedContent on @google/genai v2', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + expect(container.items.map(span => span.name).sort()).toEqual([ + 'embeddings error-model', + 'embeddings text-embedding-004', + 'embeddings text-embedding-004', + ]); + + const successfulSpans = container.items.filter( + span => span.name === 'embeddings text-embedding-004' && span.status === 'ok', + ); + expect(successfulSpans).toHaveLength(2); + for (const span of successfulSpans) { + expect(span.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); + expect(span.attributes['sentry.origin'].value).toBe(EXPECTED_ORIGIN); + expect(span.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('text-embedding-004'); + } + + const errorSpan = container.items.find(span => span.name === 'embeddings error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); + }, + }) + .start() + .completed(); + }); + }, + { additionalDependencies: { '@google/genai': '^2' } }, + ); +}); diff --git a/packages/server-utils/src/orchestrion/config/google-genai.ts b/packages/server-utils/src/orchestrion/config/google-genai.ts index 4be83b0d7e50..ece4f2bd799e 100644 --- a/packages/server-utils/src/orchestrion/config/google-genai.ts +++ b/packages/server-utils/src/orchestrion/config/google-genai.ts @@ -13,14 +13,15 @@ export const googleGenAiConfig = [ ...NODE_DIST_FILES.flatMap(filePath => (['generateContent', 'generateContentStream'] as const).map(expressionName => ({ channelName: 'generate-content', - module: { name: '@google/genai', versionRange: '>=0.10.0 <2', filePath }, + module: { name: '@google/genai', versionRange: '>=0.10.0 <3', filePath }, functionQuery: { expressionName, kind: 'Auto' as const }, })), ), - // `embedContent` and the `Chat` methods are real class methods. + // `embedContent` is a real class method in v1 but a constructor-assigned arrow in v2; the + // `className`/`methodName` selector matches both shapes. The `Chat` methods stay real class methods. ...NODE_DIST_FILES.map(filePath => ({ channelName: 'embed-content', - module: { name: '@google/genai', versionRange: '>=0.10.0 <2', filePath }, + module: { name: '@google/genai', versionRange: '>=0.10.0 <3', filePath }, functionQuery: { className: 'Models', methodName: 'embedContent', kind: 'Auto' as const }, })), // `sendMessage`/`sendMessageStream` internally delegate to `Models.generateContent(Stream)`; the @@ -28,7 +29,7 @@ export const googleGenAiConfig = [ ...NODE_DIST_FILES.flatMap(filePath => (['sendMessage', 'sendMessageStream'] as const).map(methodName => ({ channelName: 'chat', - module: { name: '@google/genai', versionRange: '>=0.10.0 <2', filePath }, + module: { name: '@google/genai', versionRange: '>=0.10.0 <3', filePath }, functionQuery: { className: 'Chat', methodName, kind: 'Auto' as const }, })), ), diff --git a/packages/server-utils/test/orchestrion/google-genai.test.ts b/packages/server-utils/test/orchestrion/google-genai.test.ts new file mode 100644 index 000000000000..a35e08202b38 --- /dev/null +++ b/packages/server-utils/test/orchestrion/google-genai.test.ts @@ -0,0 +1,75 @@ +import { create } from '@apm-js-collab/code-transformer'; +import { describe, expect, it } from 'vitest'; +import { googleGenAiConfig } from '../../src/orchestrion/config/google-genai'; + +// The file every `@google/genai` `node` export condition resolves to for the ESM build; it's one of +// the paths listed in `googleGenAiConfig`, so the real transformer will pick up its configs for it. +const FILE_PATH = 'dist/node/index.mjs'; + +// Minimal fixtures reproducing the exact `Models`/`Chat` member shapes the SDK ships. `embedContent` +// is the one that shifted between majors: a real class method in v1, a constructor-assigned arrow in +// v2. `generateContent(Stream)` are arrows and `sendMessage(Stream)` are class methods in both. +const V1_SOURCE = ` +class Models { + constructor() { + this.generateContent = async (params) => params; + this.generateContentStream = async (params) => params; + } + async embedContent(params) { return params; } +} +class Chat { + async sendMessage(params) { return params; } + async sendMessageStream(params) { return params; } +} +export { Models, Chat }; +`; + +const V2_SOURCE = ` +class Models { + constructor() { + this.generateContent = async (params) => params; + this.generateContentStream = async (params) => params; + this.embedContent = async (params) => params; + } +} +class Chat { + async sendMessage(params) { return params; } + async sendMessageStream(params) { return params; } +} +export { Models, Chat }; +`; + +function injectedChannels(version: string, source: string): string[] { + const matcher = create(googleGenAiConfig); + const transformer = matcher.getTransformer('@google/genai', version, FILE_PATH); + if (!transformer) { + return []; + } + const { code } = transformer.transform(source, 'esm'); + return [...new Set(code.match(/orchestrion:@google\/genai:[a-z-]+/g) ?? [])].sort(); +} + +describe('googleGenAiConfig', () => { + const ALL_CHANNELS = [ + 'orchestrion:@google/genai:chat', + 'orchestrion:@google/genai:embed-content', + 'orchestrion:@google/genai:generate-content', + ]; + + it('instruments every channel in the v1 source shape', () => { + expect(injectedChannels('1.20.0', V1_SOURCE)).toEqual(ALL_CHANNELS); + }); + + it.each(['2.0.0', '2.16.0'])( + 'instruments every channel in the v2 source shape (v%s), including the arrow-property embedContent', + version => { + expect(injectedChannels(version, V2_SOURCE)).toEqual(ALL_CHANNELS); + }, + ); + + it('does not match a version outside the supported range', () => { + const matcher = create(googleGenAiConfig); + expect(matcher.getTransformer('@google/genai', '0.9.0', FILE_PATH)).toBeUndefined(); + expect(matcher.getTransformer('@google/genai', '3.0.0', FILE_PATH)).toBeUndefined(); + }); +}); From e07c422dd675895d2fd68d3dc76651c6f5450ff5 Mon Sep 17 00:00:00 2001 From: Charly Gomez Date: Thu, 6 Aug 2026 14:41:58 +0200 Subject: [PATCH 2/2] address review: drop redundant unit test and confusing comment sentence The transformer-level unit test covered orchestrion internals already exercised by the per-major integration suites. Also removes a trailing comment sentence about `Chat` that belonged to the block below it. Co-Authored-By: Claude Opus 4.8 --- .../src/orchestrion/config/google-genai.ts | 2 +- .../test/orchestrion/google-genai.test.ts | 75 ------------------- 2 files changed, 1 insertion(+), 76 deletions(-) delete mode 100644 packages/server-utils/test/orchestrion/google-genai.test.ts diff --git a/packages/server-utils/src/orchestrion/config/google-genai.ts b/packages/server-utils/src/orchestrion/config/google-genai.ts index ece4f2bd799e..30b3f9c54c25 100644 --- a/packages/server-utils/src/orchestrion/config/google-genai.ts +++ b/packages/server-utils/src/orchestrion/config/google-genai.ts @@ -18,7 +18,7 @@ export const googleGenAiConfig = [ })), ), // `embedContent` is a real class method in v1 but a constructor-assigned arrow in v2; the - // `className`/`methodName` selector matches both shapes. The `Chat` methods stay real class methods. + // `className`/`methodName` selector matches both shapes. ...NODE_DIST_FILES.map(filePath => ({ channelName: 'embed-content', module: { name: '@google/genai', versionRange: '>=0.10.0 <3', filePath }, diff --git a/packages/server-utils/test/orchestrion/google-genai.test.ts b/packages/server-utils/test/orchestrion/google-genai.test.ts deleted file mode 100644 index a35e08202b38..000000000000 --- a/packages/server-utils/test/orchestrion/google-genai.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { create } from '@apm-js-collab/code-transformer'; -import { describe, expect, it } from 'vitest'; -import { googleGenAiConfig } from '../../src/orchestrion/config/google-genai'; - -// The file every `@google/genai` `node` export condition resolves to for the ESM build; it's one of -// the paths listed in `googleGenAiConfig`, so the real transformer will pick up its configs for it. -const FILE_PATH = 'dist/node/index.mjs'; - -// Minimal fixtures reproducing the exact `Models`/`Chat` member shapes the SDK ships. `embedContent` -// is the one that shifted between majors: a real class method in v1, a constructor-assigned arrow in -// v2. `generateContent(Stream)` are arrows and `sendMessage(Stream)` are class methods in both. -const V1_SOURCE = ` -class Models { - constructor() { - this.generateContent = async (params) => params; - this.generateContentStream = async (params) => params; - } - async embedContent(params) { return params; } -} -class Chat { - async sendMessage(params) { return params; } - async sendMessageStream(params) { return params; } -} -export { Models, Chat }; -`; - -const V2_SOURCE = ` -class Models { - constructor() { - this.generateContent = async (params) => params; - this.generateContentStream = async (params) => params; - this.embedContent = async (params) => params; - } -} -class Chat { - async sendMessage(params) { return params; } - async sendMessageStream(params) { return params; } -} -export { Models, Chat }; -`; - -function injectedChannels(version: string, source: string): string[] { - const matcher = create(googleGenAiConfig); - const transformer = matcher.getTransformer('@google/genai', version, FILE_PATH); - if (!transformer) { - return []; - } - const { code } = transformer.transform(source, 'esm'); - return [...new Set(code.match(/orchestrion:@google\/genai:[a-z-]+/g) ?? [])].sort(); -} - -describe('googleGenAiConfig', () => { - const ALL_CHANNELS = [ - 'orchestrion:@google/genai:chat', - 'orchestrion:@google/genai:embed-content', - 'orchestrion:@google/genai:generate-content', - ]; - - it('instruments every channel in the v1 source shape', () => { - expect(injectedChannels('1.20.0', V1_SOURCE)).toEqual(ALL_CHANNELS); - }); - - it.each(['2.0.0', '2.16.0'])( - 'instruments every channel in the v2 source shape (v%s), including the arrow-property embedContent', - version => { - expect(injectedChannels(version, V2_SOURCE)).toEqual(ALL_CHANNELS); - }, - ); - - it('does not match a version outside the supported range', () => { - const matcher = create(googleGenAiConfig); - expect(matcher.getTransformer('@google/genai', '0.9.0', FILE_PATH)).toBeUndefined(); - expect(matcher.getTransformer('@google/genai', '3.0.0', FILE_PATH)).toBeUndefined(); - }); -});