From 8908faafa5c336d657a7856a147e61558c110cee Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Fri, 9 Oct 2026 10:26:29 -0700 Subject: [PATCH 1/2] refactor(gax): organize observability utilities into util, contextUtil, and errorUtil --- core/packages/gax/src/createApiCall.ts | 10 +- core/packages/gax/src/fallback.ts | 2 +- core/packages/gax/src/gax.ts | 5 +- core/packages/gax/src/grpc.ts | 2 +- core/packages/gax/src/index.ts | 13 +- .../gax/src/observability/TracerHelper.ts | 875 +--------- .../{metadataResolver.ts => contextUtil.ts} | 137 +- .../gax/src/observability/errorUtil.ts | 827 ++++++++++ core/packages/gax/src/observability/util.ts | 59 + core/packages/gax/src/util.ts | 132 -- core/packages/gax/test/unit/apiCallable.ts | 404 +---- .../{metadataResolver.ts => contextUtil.ts} | 66 +- core/packages/gax/test/unit/errorUtil.ts | 859 ++++++++++ core/packages/gax/test/unit/otelHarness.ts | 13 +- core/packages/gax/test/unit/tracerHelper.ts | 1467 +---------------- core/packages/gax/test/unit/util.ts | 60 +- 16 files changed, 2120 insertions(+), 2811 deletions(-) rename core/packages/gax/src/observability/{metadataResolver.ts => contextUtil.ts} (67%) create mode 100644 core/packages/gax/src/observability/errorUtil.ts create mode 100644 core/packages/gax/src/observability/util.ts rename core/packages/gax/test/unit/{metadataResolver.ts => contextUtil.ts} (88%) create mode 100644 core/packages/gax/test/unit/errorUtil.ts diff --git a/core/packages/gax/src/createApiCall.ts b/core/packages/gax/src/createApiCall.ts index 08b704b52ca0..40b271a441c4 100644 --- a/core/packages/gax/src/createApiCall.ts +++ b/core/packages/gax/src/createApiCall.ts @@ -39,13 +39,15 @@ import {warn} from './warnings'; import { traceCall, traceAttempt, + ResendRecorder, +} from './observability/TracerHelper'; +import {checkTelemetryEnabled} from './observability/util'; +import { StaticTraceContext, DynamicTraceContext, AttemptTraceContext, - ResendRecorder, -} from './observability/TracerHelper'; -import {resolveStaticTraceContext} from './observability/metadataResolver'; -import {checkTelemetryEnabled} from './util'; + resolveStaticTraceContext, +} from './observability/contextUtil'; /** * Converts an rpc call into an API call governed by the settings. diff --git a/core/packages/gax/src/fallback.ts b/core/packages/gax/src/fallback.ts index 42028ade871a..1796c5b9ca53 100644 --- a/core/packages/gax/src/fallback.ts +++ b/core/packages/gax/src/fallback.ts @@ -32,7 +32,7 @@ import {isNodeJS} from './featureDetection'; import {generateServiceStub} from './fallbackServiceStub'; import {StreamType} from './streamingCalls/streaming'; import {toLowerCamelCase} from './util'; -import {StaticTraceContext} from './observability/TracerHelper'; +import {StaticTraceContext} from './observability/contextUtil'; import {google} from '../protos/http'; import * as IamProtos from '../protos/iam_service'; import * as LocationProtos from '../protos/locations'; diff --git a/core/packages/gax/src/gax.ts b/core/packages/gax/src/gax.ts index 829b2c6bbceb..710e04a976e0 100644 --- a/core/packages/gax/src/gax.ts +++ b/core/packages/gax/src/gax.ts @@ -23,8 +23,9 @@ import type {ResumableUploadContext} from './resumableUpload'; import {warn} from './warnings'; import {GoogleError} from './googleError'; import {BundleOptions} from './bundlingCalls/bundleExecutor'; -import {toLowerCamelCase, checkTelemetryEnabled} from './util'; -import {StaticTraceContext} from './observability/TracerHelper'; +import {toLowerCamelCase} from './util'; +import {checkTelemetryEnabled} from './observability/util'; +import {StaticTraceContext} from './observability/contextUtil'; import {Status} from './status'; import {RequestType} from './apitypes'; diff --git a/core/packages/gax/src/grpc.ts b/core/packages/gax/src/grpc.ts index 2c5e84fdd308..dbed0cfd7154 100644 --- a/core/packages/gax/src/grpc.ts +++ b/core/packages/gax/src/grpc.ts @@ -27,7 +27,7 @@ import * as protobuf from 'protobufjs'; import objectHash from 'object-hash'; import * as gax from './gax'; -import {StaticTraceContext} from './observability/TracerHelper'; +import {StaticTraceContext} from './observability/contextUtil'; import {ClientOptions} from '@grpc/grpc-js/build/src/client'; const googleProtoFilesDir = path.join(__dirname, '..', '..', 'build', 'protos'); diff --git a/core/packages/gax/src/index.ts b/core/packages/gax/src/index.ts index 7718fa4a59a1..89d95d5cab8a 100644 --- a/core/packages/gax/src/index.ts +++ b/core/packages/gax/src/index.ts @@ -126,18 +126,15 @@ export { PaginationResponse, } from './clientInterface'; -export { - makeUUID, - decodeAnyProtosInArray, - decodeProtobufAny, - checkTelemetryEnabled, -} from './util'; +export {makeUUID, decodeAnyProtosInArray, decodeProtobufAny} from './util'; -export {StaticTraceContext, traceCall} from './observability/TracerHelper'; +export {checkTelemetryEnabled} from './observability/util'; export { + StaticTraceContext, resolveStaticTraceContext, clearMetadataCache, -} from './observability/metadataResolver'; +} from './observability/contextUtil'; +export {traceCall} from './observability/TracerHelper'; export {ServiceError, ChannelCredentials} from '@grpc/grpc-js'; export {warn} from './warnings'; diff --git a/core/packages/gax/src/observability/TracerHelper.ts b/core/packages/gax/src/observability/TracerHelper.ts index 96fcd5fc7134..3fa2f4827d55 100644 --- a/core/packages/gax/src/observability/TracerHelper.ts +++ b/core/packages/gax/src/observability/TracerHelper.ts @@ -27,122 +27,21 @@ import { Tracer, } from '@opentelemetry/api'; import {APICallback, GaxCallResult} from '../apitypes'; -import {GoogleError} from '../googleError'; import {Status} from '../status'; import { - connectionCodes, - decodeCodes, - DEPTH_TO_CHECK, - genericClasses, - preConnectionCodes, - redirectCodes, - requestBodyCodes, - requestCodes, -} from '../util'; - -/** - * Static metadata about the Google Cloud client library used to populate - * telemetry span attributes (`gcp.client.*`, `server.*`, `url.domain`). - */ -export interface StaticTraceContext { - /** - * The target GCP service endpoint or domain (e.g. 'storage.googleapis.com'). - * Populates `gcp.client.service`. - */ - gcpClientService?: string; - /** - * The version of the client library (e.g. '1.2.3'). - * Populates `gcp.client.version`. - */ - gcpVersion?: string; - /** - * The GitHub repository name hosting the client library (e.g. 'googleapis/google-cloud-node'). - * Populates `gcp.client.repo`. - */ - gcpRepo?: string; - /** - * The NPM package name of the client library (e.g. '@google-cloud/storage'). - * Populates `gcp.client.artifact`. - */ - gcpArtifact?: string; - /** - * Server domain name or IP address for the RPC call. - * Populates `server.address`. - */ - serverAddress?: string; - /** - * Server port number for the RPC call. - * Populates `server.port`. - */ - serverPort?: number; - /** - * Target service domain (e.g. 'cloudkms.googleapis.com'). - * Populates `url.domain`. - */ - urlDomain?: string; -} - -/** - * Dynamic metadata specific to the individual RPC invocation used to populate - * telemetry span names and attributes. - */ -export interface DynamicTraceContext { - /** - * The name of the client class making the call (e.g. 'StorageClient'). - */ - clientName: string; - /** - * The name of the API method or RPC being invoked (e.g. 'GetObject'). - */ - methodName: string; - /** - * The transport protocol used for the RPC ('grpc' or 'http'). - * Populates `rpc.system.name`. - */ - rpcType: 'grpc' | 'http'; - /** - * Server domain name or IP address for the RPC call. - * Populates `server.address`. - */ - serverAddress?: string; - /** - * Server port number for the RPC call. - * Populates `server.port`. - */ - serverPort?: number; - /** - * Target service domain (e.g. 'cloudkms.googleapis.com'). - * Populates `url.domain`. - */ - urlDomain?: string; -} - -/** - * Dynamic metadata specific to an individual RPC transport attempt (low level network span). - */ -export interface AttemptTraceContext extends DynamicTraceContext { - /** - * The fully-qualified protobuf service name (e.g. 'google.cloud.kms.v1.KeyManagementService'). - * Combined with `methodName` to populate `rpc.method`. - */ - apiName?: string; - /** - * The ordinal resend count for this attempt (0 for the initial attempt, 1 for the first retry, etc.). - * Populates `gcp.grpc.resend_count` (gRPC) or `http.request.resend_count` (HTTP) when > 0; - * omitted from span attributes when 0 or undefined. - */ - resendCount?: number; - /** - * The HTTP request method for REST fallback attempts (e.g. 'GET', 'POST', 'PUT', 'PATCH', 'DELETE'). - * Populates `http.request.method`. - */ - httpMethod?: string; - /** - * The URL path template for REST fallback attempts (e.g. '/v1/{name}:access'). - * Populates `url.template`. - */ - urlTemplate?: string; -} + AttemptTraceContext, + buildInitialAttributes, + DynamicTraceContext, + StaticTraceContext, +} from './contextUtil'; +import { + isPreConnectionFailure, + recordExceptionEvent, + resolveErrorMessage, + resolveErrorType, + resolveHttpStatusCode, + resolveRpcStatusName, +} from './errorUtil'; const CLIENT_REQUEST_SPAN_KEY = createContextKey( 'google-gax-client-request-span', @@ -249,727 +148,6 @@ export function getGaxTracer(): Tracer { return trace.getTracer('google-gax'); } -/** - * Decodes binary gRPC status details (`grpc-status-details-bin`) onto the error - * object if present and not yet parsed. - * - * @param {unknown} e - The error to inspect and populate with parsed gRPC status details. - */ -function ensureGrpcStatusDetailsParsed(e: unknown): void { - if (!e || typeof e !== 'object') { - return; - } - const errWithMeta = e as GoogleError & {statusDetails?: unknown}; - if ( - !errWithMeta.reason && - !errWithMeta.statusDetails && - errWithMeta.metadata && - typeof errWithMeta.metadata.get === 'function' && - (errWithMeta.metadata.get('grpc-status-details-bin') as unknown[])?.length > - 0 - ) { - try { - GoogleError.parseGRPCStatusDetails(errWithMeta); - } catch { - // Ignore decoding errors. - } - } -} - -/** - * Resolves google.rpc.ErrorInfo reason if present on the error or its cause chain. - * Corresponds to Tier 1 in the error.type hierarchy. - * - * @param {unknown} e - The error to inspect. - * @returns {string | undefined} The ErrorInfo reason string if found, or undefined. - */ -export function resolveErrorInfoReason(e: unknown): string | undefined { - ensureGrpcStatusDetailsParsed(e); - - let current: unknown = e; - const seen = new Set(); - for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { - if (!current || typeof current !== 'object') { - break; - } - // Guard against circular cause references. - if (seen.has(current)) { - break; - } - seen.add(current); - const err = current as { - reason?: unknown; - statusDetails?: unknown; - errorInfo?: unknown; - cause?: unknown; - }; - - // Check direct reason property on error. - if (typeof err.reason === 'string' && err.reason.length > 0) { - return err.reason; - } - - // Check errorInfo object on error. - if (err.errorInfo && typeof err.errorInfo === 'object') { - const infoReason = (err.errorInfo as {reason?: unknown}).reason; - if (typeof infoReason === 'string' && infoReason.length > 0) { - return infoReason; - } - } - - // Inspect statusDetails array for reason or errorInfo. - if (Array.isArray(err.statusDetails)) { - for (const detail of err.statusDetails) { - if (detail && typeof detail === 'object') { - if ( - 'reason' in detail && - typeof (detail as {reason?: unknown}).reason === 'string' && - (detail as {reason: string}).reason.length > 0 - ) { - return (detail as {reason: string}).reason; - } - if ( - 'errorInfo' in detail && - detail.errorInfo && - typeof (detail.errorInfo as {reason?: unknown}).reason === 'string' - ) { - return (detail.errorInfo as {reason: string}).reason; - } - } - } - } - - // Traverse error cause chain. - current = err.cause; - } - - return undefined; -} - -/** - * Resolves a server error code received from the backend service: - * - For HTTP: The HTTP status code string (e.g. '400', '403', '503'). - * - For gRPC: The canonical gRPC status code name in uppercase (e.g. 'PERMISSION_DENIED', 'UNAVAILABLE'). - * Corresponds to Tier 2 in the error.type hierarchy. - * - * @param {unknown} e - The error to inspect. - * @param {'grpc' | 'http'} rpcType - The RPC transport protocol. - * @returns {string | undefined} The server error code string if the error came from the server, or undefined. - */ -export function resolveServerErrorCode( - e: unknown, - rpcType: 'grpc' | 'http', -): string | undefined { - if (rpcType === 'http') { - const httpStatus = resolveHttpStatusCode(e); - return httpStatus !== undefined ? httpStatus.toString() : undefined; - } - if (rpcType === 'grpc') { - if ( - isPreConnectionFailure(e) || - resolveClientNetworkOrOperationalError(e) !== undefined - ) { - return undefined; - } - // Exclude client-side aborts across the cause chain (timeouts are already - // matched by resolveClientNetworkOrOperationalError above). - let current: unknown = e; - const seen = new Set(); - for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { - if (!current || typeof current !== 'object' || seen.has(current)) { - break; - } - seen.add(current); - const err = current as {name?: unknown; cause?: unknown}; - if (err.name === 'AbortError') { - return undefined; - } - current = err.cause; - } - return resolveRpcStatusName(e); - } - return undefined; -} - -/** - * Resolves client-side network and operational errors to standard CLIENT_* identifiers. - * Corresponds to Tier 3 in the error.type hierarchy. - * - * @param {unknown} e - The error to inspect. - * @returns {string | undefined} The standardized CLIENT_* error type if matched, or undefined. - */ -export function resolveClientNetworkOrOperationalError( - e: unknown, -): string | undefined { - let current: unknown = e; - const seen = new Set(); - for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { - if (!current || typeof current !== 'object') { - break; - } - if (seen.has(current)) { - break; - } - seen.add(current); - const err = current as { - name?: unknown; - code?: unknown; - message?: unknown; - cause?: unknown; - constructor?: {name?: string}; - }; - - const name = typeof err.name === 'string' ? err.name : undefined; - const constructorName = err.constructor?.name; - const code = typeof err.code === 'string' ? err.code : undefined; - const message = typeof err.message === 'string' ? err.message : ''; - - // 1. CLIENT_TIMEOUT - if ( - name === 'TimeoutError' || - code === 'ETIMEDOUT' || - code === 'ESOCKETTIMEDOUT' || - /timeout.*exceeded|deadline.*exceeded|total timeout/i.test(message) - ) { - return 'CLIENT_TIMEOUT'; - } - - // 2. CLIENT_CONNECTION_ERROR - if ( - (code && connectionCodes.includes(code)) || - (code && code.startsWith('ERR_SSL')) || - name === 'TLSError' - ) { - return 'CLIENT_CONNECTION_ERROR'; - } - - // 3. CLIENT_REQUEST_ERROR - if ( - (code && requestCodes.includes(code)) || - name === 'URIError' || - constructorName === 'URIError' - ) { - return 'CLIENT_REQUEST_ERROR'; - } - - // 4. CLIENT_REQUEST_BODY_ERROR - if (code && requestBodyCodes.includes(code)) { - return 'CLIENT_REQUEST_BODY_ERROR'; - } - - // 5. CLIENT_RESPONSE_DECODE_ERROR - if ( - (code && decodeCodes.includes(code)) || - name === 'DecodeError' || - constructorName === 'DecodeError' || - name === 'SyntaxError' || - constructorName === 'SyntaxError' - ) { - return 'CLIENT_RESPONSE_DECODE_ERROR'; - } - - // 6. CLIENT_REDIRECT_ERROR - if (code && redirectCodes.includes(code)) { - return 'CLIENT_REDIRECT_ERROR'; - } - - // 7. CLIENT_AUTHENTICATION_ERROR - if ( - name === 'GoogleAuthError' || - constructorName === 'GoogleAuthError' || - code === 'ERR_NO_CREDENTIALS' || - code === 'MISSING_CREDENTIALS' - ) { - return 'CLIENT_AUTHENTICATION_ERROR'; - } - - current = err.cause; - } - - return undefined; -} - -/** - * Resolves a language-specific error type name (e.g. AbortError, TypeError, RangeError, CustomRpcError). - * Generic wrapper types (Error, GoogleError, Object, DOMException) are excluded and unwrap e.cause. - * Corresponds to Tier 4 in the error.type hierarchy. - * - * @param {unknown} e - The error to inspect. - * @returns {string | undefined} The language-specific error class or name if found, or undefined. - */ -export function resolveLanguageSpecificErrorType( - e: unknown, -): string | undefined { - let current: unknown = e; - const seen = new Set(); - for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { - if (!current || typeof current !== 'object') { - break; - } - // Guard against circular cause references. - if (seen.has(current)) { - break; - } - seen.add(current); - const err = current as { - name?: unknown; - constructor?: {name?: string}; - cause?: unknown; - }; - - // Prioritize AbortError regardless of class hierarchy. - if (err.name === 'AbortError') { - return 'AbortError'; - } - - // Prefer specific constructor class name over generic base wrappers. - const className = err.constructor?.name; - if (className && !genericClasses.includes(className)) { - return className; - } - - // Fall back to non-generic error name if available. - if ( - typeof err.name === 'string' && - err.name.length > 0 && - !genericClasses.includes(err.name) - ) { - return err.name; - } - - // Unwrap cause chain when encountering generic wrapper classes. - current = err.cause; - } - - return undefined; -} - -/** - * Resolves the OpenTelemetry `error.type` attribute according to the 5-tier hierarchy: - * 1. google.rpc.ErrorInfo.reason - * 2. Specific Server Error Code (HTTP status code or gRPC status name) - * 3. Client-Side Network/Operational Errors (CLIENT_* standardized strings) - * 4. Language-specific error type (e.g. AbortError, RangeError, TypeError, CustomRpcError) - * 5. Internal Fallback ("INTERNAL") - * - * @param {unknown} e - The error to classify. - * @param {'grpc' | 'http'} rpcType - The RPC transport protocol. - * @returns {string} The resolved `error.type` value. - */ -export function resolveErrorType(e: unknown, rpcType: 'grpc' | 'http'): string { - // Tier 1: google.rpc.ErrorInfo.reason - const errorInfoReason = resolveErrorInfoReason(e); - if (errorInfoReason) { - return errorInfoReason; - } - - // Tier 2: Specific Server Error Code - const serverErrorCode = resolveServerErrorCode(e, rpcType); - if (serverErrorCode) { - return serverErrorCode; - } - - // Tier 3: Client-Side Network/Operational Errors - const clientError = resolveClientNetworkOrOperationalError(e); - if (clientError) { - return clientError; - } - - // Tier 4: Language-specific error type - const languageError = resolveLanguageSpecificErrorType(e); - if (languageError) { - return languageError; - } - - // Tier 5: Internal Fallback - return 'INTERNAL'; -} - -/** - * Resolves the exception type name for a failed call. Prefers specific - * exception names (e.g. `AbortError`, `TimeoutError`, `TypeError`) and - * constructor names over generic `Error`, falling back to `e.name` when - * the constructor is generic or `DOMException`. - * - * @param {Error} e - The Error instance. - * @returns {string} The resolved `exception.type` string. - */ -function resolveExceptionType(e: Error): string { - if (e.name && (e.name === 'AbortError' || e.name === 'TimeoutError')) { - return e.name; - } - const className = e.constructor?.name; - if ( - className && - className !== 'Error' && - className !== 'Object' && - className !== 'DOMException' - ) { - return className; - } - return e.name || 'Error'; -} - -/** - * Resolves a Node system error code (e.g. `ECONNREFUSED`), checking the error - * itself and any underlying cause attached by fallback wrapping. - * - * @param {unknown} e - The error to inspect. - * @returns {string | undefined} The system error code string if present, or undefined. - */ -function resolveSystemErrorCode(e: unknown): string | undefined { - let current: unknown = e; - const seen = new Set(); - for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { - if (!current || typeof current !== 'object') { - break; - } - if (seen.has(current)) { - break; - } - seen.add(current); - const code = (current as {code?: unknown}).code; - if (typeof code === 'string' && code.length > 0) { - return code; - } - current = (current as {cause?: unknown}).cause; - } - return undefined; -} - -/** - * Resolves the canonical gRPC status name for a failed call. Status 0 (OK) - * and codes outside the `Status` enum are treated as absent for failed calls. - * - * @param {unknown} e - The error to inspect. - * @returns {string | undefined} The canonical gRPC status name if present, or undefined. - */ -function resolveRpcStatusName(e: unknown): string | undefined { - let current: unknown = e; - const seen = new Set(); - for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { - if (!current || typeof current !== 'object') { - break; - } - if (seen.has(current)) { - break; - } - seen.add(current); - const code = (current as {code?: unknown}).code; - if ( - typeof code === 'number' && - code !== Status.OK && - Status[code] !== undefined - ) { - return Status[code]; - } - current = (current as {cause?: unknown}).cause; - } - return undefined; -} - -/** - * Reads the HTTP response status recorded on a fallback error. - * - * @param {unknown} e - The error to inspect. - * @returns {number | undefined} The numeric HTTP status code if present, or undefined. - */ -function resolveHttpStatusCode(e: unknown): number | undefined { - let current: unknown = e; - const seen = new Set(); - for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { - if (!current || typeof current !== 'object') { - break; - } - if (seen.has(current)) { - break; - } - seen.add(current); - const code = (current as {httpStatusCode?: unknown}).httpStatusCode; - if (typeof code === 'number') { - return code; - } - current = (current as {cause?: unknown}).cause; - } - return undefined; -} - -/** - * Determines if a failure occurred on the client side before DNS resolution - * or connection establishment. - * - * @param {unknown} e - The error to inspect. - * @returns {boolean} True if the error occurred before connection establishment. - */ -export function isPreConnectionFailure(e: unknown): boolean { - let current: unknown = e; - const seen = new Set(); - - for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { - // Server status code indicates a response was received. - if ( - resolveHttpStatusCode(current) !== undefined || - resolveRpcStatusName(current) !== undefined - ) { - return false; - } - - // Non-Error throws or objects without stack are client failures. - if ( - !current || - !( - current instanceof Error || - (typeof current === 'object' && 'stack' in current) - ) - ) { - return true; - } - - // Guard against circular cause references. - if (seen.has(current)) { - return false; - } - seen.add(current); - - // Client-side validation errors happen before connection. - const err = current as {name?: unknown; cause?: unknown}; - if ( - current instanceof TypeError || - current instanceof RangeError || - current instanceof URIError || - err.name === 'TypeError' || - err.name === 'RangeError' || - err.name === 'URIError' - ) { - return true; - } - - // Check for network error codes occurring before connection establishment. - const systemCode = resolveSystemErrorCode(current); - if (systemCode && preConnectionCodes.includes(systemCode)) { - return true; - } - - // Unwrap GoogleError wrappers to inspect underlying cause. - if ( - (current instanceof GoogleError || - (current as {constructor?: {name?: string}}).constructor?.name === - 'GoogleError') && - err.cause - ) { - current = err.cause; - } else { - return false; - } - } - - return false; -} - -/** - * Resolves a human-readable description for non-Error throws, extracting - * `message` if present or falling back to `String(e)`. - * - * @param {unknown} e - The non-Error value thrown or reported. - * @returns {string} The resolved error message string. - */ -function resolveErrorMessage(e: unknown): string { - const message = (e as {message?: unknown} | null)?.message; - return typeof message === 'string' ? message : String(e); -} - -/** - * Determines whether a failure is a server-side error (i.e. a server response arrived), - * excluding pre-connection failures, client network/operational errors, and aborts/timeouts. - * - * @param {unknown} e - The error to inspect. - * @param {'grpc' | 'http'} rpcType - The RPC transport protocol. - * @returns {boolean} True if a server error code was resolved for the transport. - */ -export function isServerSideError( - e: unknown, - rpcType: 'grpc' | 'http', -): boolean { - return resolveServerErrorCode(e, rpcType) !== undefined; -} - -/** - * Checks whether a value is a Node.js Buffer instance. - * - * @param {unknown} val - The value to check. - * @returns {boolean} True if `val` is a Buffer. - */ -function isBuffer(val: unknown): val is Buffer { - return typeof Buffer !== 'undefined' && Buffer.isBuffer(val); -} - -/** - * Safely converts a value to a JSON string without throwing exceptions on - * circular references, BigInt values, or non-serializable properties. - * - * @param {unknown} value - The value to serialize. - * @returns {string | undefined} The serialized JSON string, or undefined if `value` is undefined or unserializable. - */ -export function safeJsonStringify(value: unknown): string | undefined { - if (value === undefined) { - return undefined; - } - try { - const ancestors: unknown[] = []; - return JSON.stringify( - value, - function (this: unknown, _key: string, val: unknown) { - if (typeof val === 'bigint') { - return val.toString(); - } - if (typeof val !== 'object' || val === null) { - return val; - } - // Track object ancestry to replace circular references. - if (ancestors.includes(this)) { - while ( - ancestors.length > 0 && - ancestors[ancestors.length - 1] !== this - ) { - ancestors.pop(); - } - } - if (ancestors.includes(val)) { - return '[Circular]'; - } - ancestors.push(val); - return val; - }, - ); - } catch { - try { - return String(value); - } catch { - return undefined; - } - } -} - -/** - * Extracts and formats server-side error details (message, status details, and metadata) - * attached by GFE/backend on server-side errors. Note that only `message` is used on - * low level network `exception` span events (`exception.stacktrace` is not recorded on spans). - * - * @param {Error} e - The server Error instance. - * @returns {{message: string; stacktrace?: string}} The resolved server message and formatted details string. - */ -export function resolveServerExceptionDetails(e: Error): { - message: string; - stacktrace?: string; -} { - ensureGrpcStatusDetailsParsed(e); - - const errObj = e as { - details?: unknown; - statusDetails?: unknown; - metadata?: unknown; - cause?: unknown; - }; - - const causeObj = - errObj.cause && typeof errObj.cause === 'object' - ? (errObj.cause as { - details?: unknown; - statusDetails?: unknown; - metadata?: unknown; - }) - : undefined; - - // Prefer server error details over generic error message. - const serverDetails = errObj.details ?? causeObj?.details; - const message = - typeof serverDetails === 'string' && serverDetails.length > 0 - ? serverDetails - : e.message; - - // Serialize status details if present. - const rawStatusDetails = errObj.statusDetails ?? causeObj?.statusDetails; - let statusDetailsStr: string | undefined; - if (rawStatusDetails !== undefined && rawStatusDetails !== null) { - statusDetailsStr = - typeof rawStatusDetails === 'string' - ? rawStatusDetails - : safeJsonStringify(rawStatusDetails); - } - - // Serialize backend metadata, encoding Buffer values as base64. - const rawMetadata = errObj.metadata ?? causeObj?.metadata; - let metadataStr: string | undefined; - if (rawMetadata && typeof rawMetadata === 'object') { - try { - let map: Record; - if (typeof (rawMetadata as {getMap?: unknown}).getMap === 'function') { - map = (rawMetadata as {getMap: () => Record}).getMap(); - } else { - map = rawMetadata as Record; - } - const cleanMap: Record = {}; - for (const [key, val] of Object.entries(map)) { - if (isBuffer(val)) { - cleanMap[key] = val.toString('base64'); - } else if (Array.isArray(val)) { - cleanMap[key] = val.map(item => - isBuffer(item) ? item.toString('base64') : item, - ); - } else { - cleanMap[key] = val; - } - } - metadataStr = safeJsonStringify(cleanMap); - } catch { - metadataStr = safeJsonStringify(rawMetadata); - } - } - - // Combine status details and metadata into a formatted stacktrace string. - const stacktraceParts: string[] = []; - if (statusDetailsStr) { - stacktraceParts.push(`status_details: ${statusDetailsStr}`); - } - if (metadataStr) { - stacktraceParts.push(`metadata: ${metadataStr}`); - } - const stacktrace = - stacktraceParts.length > 0 ? stacktraceParts.join('\n') : undefined; - - return { - message, - stacktrace, - }; -} - -/** - * Records an `exception` span event with `exception.type` (and `exception.message` - * when `includeMessage` is true, such as on low level network attempt spans). - * - * @param {Span} span - The span on which to record the exception event. - * @param {Error} e - The error instance. - * @param {'grpc' | 'http'} rpcType - The RPC transport protocol. - * @param {boolean} [includeMessage=false] - Whether to include `exception.message` on the event - * (true for low level network attempt spans, false for client request spans). - */ -function recordExceptionEvent( - span: Span, - e: Error, - rpcType: 'grpc' | 'http', - includeMessage = false, -): void { - const attributes: Attributes = { - 'exception.type': resolveExceptionType(e), - }; - if (includeMessage) { - attributes['exception.message'] = isServerSideError(e, rpcType) - ? resolveServerExceptionDetails(e).message - : e.message; - } - span.addEvent('exception', attributes); -} - /** * Checks if a value behaves like a Promise or Thenable. * @@ -1163,33 +341,6 @@ function resolveEndpointAndDomain( return {rawAddress, rawPort, urlDomain}; } -/** - * Builds the common initial client, transport, and domain attributes shared by - * client request spans and low level network attempt spans. - * - * @param {DynamicTraceContext} dynamicArgs - Dynamic trace context for the call or attempt. - * @param {StaticTraceContext} staticArgs - Static trace context for the client library. - * @param {string} [urlDomain] - Resolved `url.domain` attribute value, if available. - * @returns {Attributes} The initial span attributes map. - */ -function buildInitialAttributes( - dynamicArgs: DynamicTraceContext, - staticArgs: StaticTraceContext, - urlDomain?: string, -): Attributes { - const attributes: Attributes = { - 'gcp.client.service': staticArgs.gcpClientService, - 'gcp.client.version': staticArgs.gcpVersion, - 'gcp.client.repo': staticArgs.gcpRepo, - 'gcp.client.artifact': staticArgs.gcpArtifact, - 'rpc.system.name': dynamicArgs.rpcType, - }; - if (urlDomain !== undefined) { - attributes['url.domain'] = urlDomain; - } - return attributes; -} - /** * Returns the transport-specific resend count attribute key. * diff --git a/core/packages/gax/src/observability/metadataResolver.ts b/core/packages/gax/src/observability/contextUtil.ts similarity index 67% rename from core/packages/gax/src/observability/metadataResolver.ts rename to core/packages/gax/src/observability/contextUtil.ts index bb9f082a25fa..5fe8d6a83f27 100644 --- a/core/packages/gax/src/observability/metadataResolver.ts +++ b/core/packages/gax/src/observability/contextUtil.ts @@ -14,12 +14,115 @@ * limitations under the License. */ -import {StaticTraceContext} from './TracerHelper'; +import {Attributes} from '@opentelemetry/api'; import {CallSettings} from '../gax'; import {ignoredClientHeaderTokens} from '../util'; - export const DEFAULT_GCP_REPO = 'googleapis/google-cloud-node'; +/** + * Static metadata about the Google Cloud client library used to populate + * telemetry span attributes (`gcp.client.*`, `server.*`, `url.domain`). + */ +export interface StaticTraceContext { + /** + * The target GCP service endpoint or domain (e.g. 'storage.googleapis.com'). + * Populates `gcp.client.service`. + */ + gcpClientService?: string; + /** + * The version of the client library (e.g. '1.2.3'). + * Populates `gcp.client.version`. + */ + gcpVersion?: string; + /** + * The GitHub repository name hosting the client library (e.g. 'googleapis/google-cloud-node'). + * Populates `gcp.client.repo`. + */ + gcpRepo?: string; + /** + * The NPM package name of the client library (e.g. '@google-cloud/storage'). + * Populates `gcp.client.artifact`. + */ + gcpArtifact?: string; + /** + * Server domain name or IP address for the RPC call. + * Populates `server.address`. + */ + serverAddress?: string; + /** + * Server port number for the RPC call. + * Populates `server.port`. + */ + serverPort?: number; + /** + * Target service domain (e.g. 'cloudkms.googleapis.com'). + * Populates `url.domain`. + */ + urlDomain?: string; +} + +/** + * Dynamic metadata specific to the individual RPC invocation used to populate + * telemetry span names and attributes. + */ +export interface DynamicTraceContext { + /** + * The name of the client class making the call (e.g. 'StorageClient'). + */ + clientName: string; + /** + * The name of the API method or RPC being invoked (e.g. 'GetObject'). + */ + methodName: string; + /** + * The transport protocol used for the RPC ('grpc' or 'http'). + * Populates `rpc.system.name`. + */ + rpcType: 'grpc' | 'http'; + /** + * Server domain name or IP address for the RPC call. + * Populates `server.address`. + */ + serverAddress?: string; + /** + * Server port number for the RPC call. + * Populates `server.port`. + */ + serverPort?: number; + /** + * Target service domain (e.g. 'cloudkms.googleapis.com'). + * Populates `url.domain`. + */ + urlDomain?: string; +} + +/** + * Dynamic metadata specific to an individual RPC transport attempt (low level network span). + */ +export interface AttemptTraceContext extends DynamicTraceContext { + /** + * The fully-qualified protobuf service name (e.g. 'google.cloud.kms.v1.KeyManagementService'). + * Combined with `methodName` to populate `rpc.method`. + */ + apiName?: string; + /** + * The ordinal resend count for this attempt (0 for the initial attempt, 1 for the first retry, etc.). + * Populates `gcp.grpc.resend_count` (gRPC) or `http.request.resend_count` (HTTP) when > 0; + * omitted from span attributes when 0 or undefined. + */ + resendCount?: number; + /** + * The HTTP request method for REST fallback attempts (e.g. 'GET', 'POST', 'PUT', 'PATCH', 'DELETE'). + * Populates `http.request.method`. + */ + httpMethod?: string; + /** + * The URL path template for REST fallback attempts (e.g. '/v1/{name}:access'). + * Populates `url.template`. + */ + urlTemplate?: string; +} + /** * Derives the GCP client service name from an NPM package name. */ @@ -236,7 +339,8 @@ export function resolveStaticTraceContext( ): StaticTraceContext { const envMeta = extractFromEnvironment(); const explicit = settings?.otherArgs?.internalTelemetryInfo as - StaticTraceContext | undefined; + | StaticTraceContext + | undefined; let dynamic: StaticTraceContext; if (settings?.apiName) { @@ -276,3 +380,30 @@ export function resolveStaticTraceContext( return context; } + +/** + * Builds the common initial client, transport, and domain attributes shared by + * client request spans and low level network attempt spans. + * + * @param {DynamicTraceContext} dynamicArgs - Dynamic trace context for the call or attempt. + * @param {StaticTraceContext} staticArgs - Static trace context for the client library. + * @param {string} [urlDomain] - Resolved `url.domain` attribute value, if available. + * @returns {Attributes} The initial span attributes map. + */ +export function buildInitialAttributes( + dynamicArgs: DynamicTraceContext, + staticArgs: StaticTraceContext, + urlDomain?: string, +): Attributes { + const attributes: Attributes = { + 'gcp.client.service': staticArgs.gcpClientService, + 'gcp.client.version': staticArgs.gcpVersion, + 'gcp.client.repo': staticArgs.gcpRepo, + 'gcp.client.artifact': staticArgs.gcpArtifact, + 'rpc.system.name': dynamicArgs.rpcType, + }; + if (urlDomain !== undefined) { + attributes['url.domain'] = urlDomain; + } + return attributes; +} diff --git a/core/packages/gax/src/observability/errorUtil.ts b/core/packages/gax/src/observability/errorUtil.ts new file mode 100644 index 000000000000..13046f1ebf6b --- /dev/null +++ b/core/packages/gax/src/observability/errorUtil.ts @@ -0,0 +1,827 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Attributes, Span} from '@opentelemetry/api'; +import {GoogleError} from '../googleError'; +import {Status} from '../status'; +/** + * System and network error codes mapped to CLIENT_CONNECTION_ERROR. + */ +export const connectionCodes = [ + 'ENOTFOUND', + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'ENETDOWN', + 'EPIPE', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_SOCKET', + 'CERT_HAS_EXPIRED', + 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + 'DEPTH_ZERO_SELF_SIGNED_CERT', + 'SELF_SIGNED_CERT_IN_CHAIN', + 'ERR_TLS_CERT_ALTNAME_INVALID', + 'UNABLE_TO_GET_ISSUER_CERT', +]; + +/** + * System and client request error codes mapped to CLIENT_REQUEST_ERROR. + */ +export const requestCodes = [ + 'ERR_INVALID_ARG_TYPE', + 'ERR_INVALID_URL', + 'ERR_HTTP_INVALID_HEADER_VALUE', + 'ERR_INVALID_HTTP_TOKEN', + 'ERR_INVALID_PROTOCOL', + 'ERR_INVALID_ARG_VALUE', +]; + +/** + * Stream and body error codes mapped to CLIENT_REQUEST_BODY_ERROR. + */ +export const requestBodyCodes = [ + 'ERR_STREAM_WRITE_AFTER_END', + 'ERR_STREAM_DESTROYED', + 'ERR_STREAM_ALREADY_FINISHED', + 'ERR_STREAM_CANNOT_PIPE', + 'ERR_STREAM_NULL_VALUES', + 'ERR_STREAM_PREMATURE_CLOSE', +]; + +/** + * Buffer error codes mapped to CLIENT_RESPONSE_DECODE_ERROR. + */ +export const decodeCodes = ['ERR_BUFFER_OUT_OF_BOUNDS']; + +/** + * Redirect error codes mapped to CLIENT_REDIRECT_ERROR. + */ +export const redirectCodes = [ + 'ERR_TOO_MANY_REDIRECTS', + 'ERR_FR_TOO_MANY_REDIRECTS', +]; + +/** + * Generic class / constructor names that should be unwrapped to find more specific error causes. + */ +export const genericClasses = [ + 'Error', + 'GoogleError', + 'Object', + 'DOMException', +]; + +/** + * Error codes indicating failures prior to network connection establishment. + */ +export const preConnectionCodes = [ + 'ENOTFOUND', + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'ERR_INVALID_ARG_TYPE', + 'ERR_INVALID_URL', +]; + +/** + * Maximum traversal depth for unwrapping nested error causes. + */ +export const DEPTH_TO_CHECK = 10; + +/** + * Decodes binary gRPC status details (`grpc-status-details-bin`) onto the error + * object if present and not yet parsed. + * + * @param {unknown} e - The error to inspect and populate with parsed gRPC status details. + */ +export function ensureGrpcStatusDetailsParsed(e: unknown): void { + if (!e || typeof e !== 'object') { + return; + } + const errWithMeta = e as GoogleError & {statusDetails?: unknown}; + if ( + !errWithMeta.reason && + !errWithMeta.statusDetails && + errWithMeta.metadata && + typeof errWithMeta.metadata.get === 'function' && + (errWithMeta.metadata.get('grpc-status-details-bin') as unknown[])?.length > + 0 + ) { + try { + GoogleError.parseGRPCStatusDetails(errWithMeta); + } catch { + // Ignore decoding errors. + } + } +} + +/** + * Resolves google.rpc.ErrorInfo reason if present on the error or its cause chain. + * Corresponds to Tier 1 in the error.type hierarchy. + * + * @param {unknown} e - The error to inspect. + * @returns {string | undefined} The ErrorInfo reason string if found, or undefined. + */ +export function resolveErrorInfoReason(e: unknown): string | undefined { + ensureGrpcStatusDetailsParsed(e); + + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object') { + break; + } + // Guard against circular cause references. + if (seen.has(current)) { + break; + } + seen.add(current); + const err = current as { + reason?: unknown; + statusDetails?: unknown; + errorInfo?: unknown; + cause?: unknown; + }; + + // Check direct reason property on error. + if (typeof err.reason === 'string' && err.reason.length > 0) { + return err.reason; + } + + // Check errorInfo object on error. + if (err.errorInfo && typeof err.errorInfo === 'object') { + const infoReason = (err.errorInfo as {reason?: unknown}).reason; + if (typeof infoReason === 'string' && infoReason.length > 0) { + return infoReason; + } + } + + // Inspect statusDetails array for reason or errorInfo. + if (Array.isArray(err.statusDetails)) { + for (const detail of err.statusDetails) { + if (detail && typeof detail === 'object') { + if ( + 'reason' in detail && + typeof (detail as {reason?: unknown}).reason === 'string' && + (detail as {reason: string}).reason.length > 0 + ) { + return (detail as {reason: string}).reason; + } + if ( + 'errorInfo' in detail && + detail.errorInfo && + typeof (detail.errorInfo as {reason?: unknown}).reason === 'string' + ) { + return (detail.errorInfo as {reason: string}).reason; + } + } + } + } + + // Traverse error cause chain. + current = err.cause; + } + + return undefined; +} + +/** + * Resolves a server error code received from the backend service: + * - For HTTP: The HTTP status code string (e.g. '400', '403', '503'). + * - For gRPC: The canonical gRPC status code name in uppercase (e.g. 'PERMISSION_DENIED', 'UNAVAILABLE'). + * Corresponds to Tier 2 in the error.type hierarchy. + * + * @param {unknown} e - The error to inspect. + * @param {'grpc' | 'http'} rpcType - The RPC transport protocol. + * @returns {string | undefined} The server error code string if the error came from the server, or undefined. + */ +export function resolveServerErrorCode( + e: unknown, + rpcType: 'grpc' | 'http', +): string | undefined { + if (rpcType === 'http') { + const httpStatus = resolveHttpStatusCode(e); + return httpStatus !== undefined ? httpStatus.toString() : undefined; + } + if (rpcType === 'grpc') { + if ( + isPreConnectionFailure(e) || + resolveClientNetworkOrOperationalError(e) !== undefined + ) { + return undefined; + } + // Exclude client-side aborts across the cause chain (timeouts are already + // matched by resolveClientNetworkOrOperationalError above). + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object' || seen.has(current)) { + break; + } + seen.add(current); + const err = current as {name?: unknown; cause?: unknown}; + if (err.name === 'AbortError') { + return undefined; + } + current = err.cause; + } + return resolveRpcStatusName(e); + } + return undefined; +} + +/** + * Resolves client-side network and operational errors to standard CLIENT_* identifiers. + * Corresponds to Tier 3 in the error.type hierarchy. + * + * @param {unknown} e - The error to inspect. + * @returns {string | undefined} The standardized CLIENT_* error type if matched, or undefined. + */ +export function resolveClientNetworkOrOperationalError( + e: unknown, +): string | undefined { + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object') { + break; + } + if (seen.has(current)) { + break; + } + seen.add(current); + const err = current as { + name?: unknown; + code?: unknown; + message?: unknown; + cause?: unknown; + constructor?: {name?: string}; + }; + + const name = typeof err.name === 'string' ? err.name : undefined; + const constructorName = err.constructor?.name; + const code = typeof err.code === 'string' ? err.code : undefined; + const message = typeof err.message === 'string' ? err.message : ''; + + // 1. CLIENT_TIMEOUT + if ( + name === 'TimeoutError' || + code === 'ETIMEDOUT' || + code === 'ESOCKETTIMEDOUT' || + /timeout.*exceeded|deadline.*exceeded|total timeout/i.test(message) + ) { + return 'CLIENT_TIMEOUT'; + } + + // 2. CLIENT_CONNECTION_ERROR + if ( + (code && connectionCodes.includes(code)) || + (code && code.startsWith('ERR_SSL')) || + name === 'TLSError' + ) { + return 'CLIENT_CONNECTION_ERROR'; + } + + // 3. CLIENT_REQUEST_ERROR + if ( + (code && requestCodes.includes(code)) || + name === 'URIError' || + constructorName === 'URIError' + ) { + return 'CLIENT_REQUEST_ERROR'; + } + + // 4. CLIENT_REQUEST_BODY_ERROR + if (code && requestBodyCodes.includes(code)) { + return 'CLIENT_REQUEST_BODY_ERROR'; + } + + // 5. CLIENT_RESPONSE_DECODE_ERROR + if ( + (code && decodeCodes.includes(code)) || + name === 'DecodeError' || + constructorName === 'DecodeError' || + name === 'SyntaxError' || + constructorName === 'SyntaxError' + ) { + return 'CLIENT_RESPONSE_DECODE_ERROR'; + } + + // 6. CLIENT_REDIRECT_ERROR + if (code && redirectCodes.includes(code)) { + return 'CLIENT_REDIRECT_ERROR'; + } + + // 7. CLIENT_AUTHENTICATION_ERROR + if ( + name === 'GoogleAuthError' || + constructorName === 'GoogleAuthError' || + code === 'ERR_NO_CREDENTIALS' || + code === 'MISSING_CREDENTIALS' + ) { + return 'CLIENT_AUTHENTICATION_ERROR'; + } + + current = err.cause; + } + + return undefined; +} + +/** + * Resolves a language-specific error type name (e.g. AbortError, TypeError, RangeError, CustomRpcError). + * Generic wrapper types (Error, GoogleError, Object, DOMException) are excluded and unwrap e.cause. + * Corresponds to Tier 4 in the error.type hierarchy. + * + * @param {unknown} e - The error to inspect. + * @returns {string | undefined} The language-specific error class or name if found, or undefined. + */ +export function resolveLanguageSpecificErrorType( + e: unknown, +): string | undefined { + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object') { + break; + } + // Guard against circular cause references. + if (seen.has(current)) { + break; + } + seen.add(current); + const err = current as { + name?: unknown; + constructor?: {name?: string}; + cause?: unknown; + }; + + // Prioritize AbortError regardless of class hierarchy. + if (err.name === 'AbortError') { + return 'AbortError'; + } + + // Prefer specific constructor class name over generic base wrappers. + const className = err.constructor?.name; + if (className && !genericClasses.includes(className)) { + return className; + } + + // Fall back to non-generic error name if available. + if ( + typeof err.name === 'string' && + err.name.length > 0 && + !genericClasses.includes(err.name) + ) { + return err.name; + } + + // Unwrap cause chain when encountering generic wrapper classes. + current = err.cause; + } + + return undefined; +} + +/** + * Resolves the OpenTelemetry `error.type` attribute according to the 5-tier hierarchy: + * 1. google.rpc.ErrorInfo.reason + * 2. Specific Server Error Code (HTTP status code or gRPC status name) + * 3. Client-Side Network/Operational Errors (CLIENT_* standardized strings) + * 4. Language-specific error type (e.g. AbortError, RangeError, TypeError, CustomRpcError) + * 5. Internal Fallback ("INTERNAL") + * + * @param {unknown} e - The error to classify. + * @param {'grpc' | 'http'} rpcType - The RPC transport protocol. + * @returns {string} The resolved `error.type` value. + */ +export function resolveErrorType(e: unknown, rpcType: 'grpc' | 'http'): string { + // Tier 1: google.rpc.ErrorInfo.reason + const errorInfoReason = resolveErrorInfoReason(e); + if (errorInfoReason) { + return errorInfoReason; + } + + // Tier 2: Specific Server Error Code + const serverErrorCode = resolveServerErrorCode(e, rpcType); + if (serverErrorCode) { + return serverErrorCode; + } + + // Tier 3: Client-Side Network/Operational Errors + const clientError = resolveClientNetworkOrOperationalError(e); + if (clientError) { + return clientError; + } + + // Tier 4: Language-specific error type + const languageError = resolveLanguageSpecificErrorType(e); + if (languageError) { + return languageError; + } + + // Tier 5: Internal Fallback + return 'INTERNAL'; +} + +/** + * Resolves the exception type name for a failed call. Prefers specific + * exception names (e.g. `AbortError`, `TimeoutError`, `TypeError`) and + * constructor names over generic `Error`, falling back to `e.name` when + * the constructor is generic or `DOMException`. + * + * @param {Error} e - The Error instance. + * @returns {string} The resolved `exception.type` string. + */ +export function resolveExceptionType(e: Error): string { + if (e.name && (e.name === 'AbortError' || e.name === 'TimeoutError')) { + return e.name; + } + const className = e.constructor?.name; + if ( + className && + className !== 'Error' && + className !== 'Object' && + className !== 'DOMException' + ) { + return className; + } + return e.name || 'Error'; +} + +/** + * Resolves a Node system error code (e.g. `ECONNREFUSED`), checking the error + * itself and any underlying cause attached by fallback wrapping. + * + * @param {unknown} e - The error to inspect. + * @returns {string | undefined} The system error code string if present, or undefined. + */ +export function resolveSystemErrorCode(e: unknown): string | undefined { + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object') { + break; + } + if (seen.has(current)) { + break; + } + seen.add(current); + const code = (current as {code?: unknown}).code; + if (typeof code === 'string' && code.length > 0) { + return code; + } + current = (current as {cause?: unknown}).cause; + } + return undefined; +} + +/** + * Resolves the canonical gRPC status name for a failed call. Status 0 (OK) + * and codes outside the `Status` enum are treated as absent for failed calls. + * + * @param {unknown} e - The error to inspect. + * @returns {string | undefined} The canonical gRPC status name if present, or undefined. + */ +export function resolveRpcStatusName(e: unknown): string | undefined { + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object') { + break; + } + if (seen.has(current)) { + break; + } + seen.add(current); + const code = (current as {code?: unknown}).code; + if ( + typeof code === 'number' && + code !== Status.OK && + Status[code] !== undefined + ) { + return Status[code]; + } + current = (current as {cause?: unknown}).cause; + } + return undefined; +} + +/** + * Reads the HTTP response status recorded on a fallback error. + * + * @param {unknown} e - The error to inspect. + * @returns {number | undefined} The numeric HTTP status code if present, or undefined. + */ +export function resolveHttpStatusCode(e: unknown): number | undefined { + let current: unknown = e; + const seen = new Set(); + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + if (!current || typeof current !== 'object') { + break; + } + if (seen.has(current)) { + break; + } + seen.add(current); + const code = (current as {httpStatusCode?: unknown}).httpStatusCode; + if (typeof code === 'number') { + return code; + } + current = (current as {cause?: unknown}).cause; + } + return undefined; +} + +/** + * Determines if a failure occurred on the client side before DNS resolution + * or connection establishment. + * + * @param {unknown} e - The error to inspect. + * @returns {boolean} True if the error occurred before connection establishment. + */ +export function isPreConnectionFailure(e: unknown): boolean { + let current: unknown = e; + const seen = new Set(); + + for (let depth = 0; depth < DEPTH_TO_CHECK; depth++) { + // Server status code indicates a response was received. + if ( + resolveHttpStatusCode(current) !== undefined || + resolveRpcStatusName(current) !== undefined + ) { + return false; + } + + // Non-Error throws or objects without stack are client failures. + if ( + !current || + !( + current instanceof Error || + (typeof current === 'object' && 'stack' in current) + ) + ) { + return true; + } + + // Guard against circular cause references. + if (seen.has(current)) { + return false; + } + seen.add(current); + + // Client-side validation errors happen before connection. + const err = current as {name?: unknown; cause?: unknown}; + if ( + current instanceof TypeError || + current instanceof RangeError || + current instanceof URIError || + err.name === 'TypeError' || + err.name === 'RangeError' || + err.name === 'URIError' + ) { + return true; + } + + // Check for network error codes occurring before connection establishment. + const systemCode = resolveSystemErrorCode(current); + if (systemCode && preConnectionCodes.includes(systemCode)) { + return true; + } + + // Unwrap GoogleError wrappers to inspect underlying cause. + if ( + (current instanceof GoogleError || + (current as {constructor?: {name?: string}}).constructor?.name === + 'GoogleError') && + err.cause + ) { + current = err.cause; + } else { + return false; + } + } + + return false; +} + +/** + * Resolves a human-readable description for non-Error throws, extracting + * `message` if present or falling back to `String(e)`. + * + * @param {unknown} e - The non-Error value thrown or reported. + * @returns {string} The resolved error message string. + */ +export function resolveErrorMessage(e: unknown): string { + const message = (e as {message?: unknown} | null)?.message; + return typeof message === 'string' ? message : String(e); +} + +/** + * Determines whether a failure is a server-side error (i.e. a server response arrived), + * excluding pre-connection failures, client network/operational errors, and aborts/timeouts. + * + * @param {unknown} e - The error to inspect. + * @param {'grpc' | 'http'} rpcType - The RPC transport protocol. + * @returns {boolean} True if a server error code was resolved for the transport. + */ +export function isServerSideError( + e: unknown, + rpcType: 'grpc' | 'http', +): boolean { + return resolveServerErrorCode(e, rpcType) !== undefined; +} + +/** + * Checks whether a value is a Node.js Buffer instance. + * + * @param {unknown} val - The value to check. + * @returns {boolean} True if `val` is a Buffer. + */ +function isBuffer(val: unknown): val is Buffer { + return typeof Buffer !== 'undefined' && Buffer.isBuffer(val); +} + +/** + * Safely converts a value to a JSON string without throwing exceptions on + * circular references, BigInt values, or non-serializable properties. + * + * @param {unknown} value - The value to serialize. + * @returns {string | undefined} The serialized JSON string, or undefined if `value` is undefined or unserializable. + */ +export function safeJsonStringify(value: unknown): string | undefined { + if (value === undefined) { + return undefined; + } + try { + const ancestors: unknown[] = []; + return JSON.stringify( + value, + function (this: unknown, _key: string, val: unknown) { + if (typeof val === 'bigint') { + return val.toString(); + } + if (typeof val !== 'object' || val === null) { + return val; + } + // Track object ancestry to replace circular references. + if (ancestors.includes(this)) { + while ( + ancestors.length > 0 && + ancestors[ancestors.length - 1] !== this + ) { + ancestors.pop(); + } + } + if (ancestors.includes(val)) { + return '[Circular]'; + } + ancestors.push(val); + return val; + }, + ); + } catch { + try { + return String(value); + } catch { + return undefined; + } + } +} + +/** + * Extracts and formats server-side error details (message, status details, and metadata) + * attached by GFE/backend on server-side errors. Note that only `message` is used on + * low level network `exception` span events (`exception.stacktrace` is not recorded on spans). + * + * @param {Error} e - The server Error instance. + * @returns {{message: string; stacktrace?: string}} The resolved server message and formatted details string. + */ +export function resolveServerExceptionDetails(e: Error): { + message: string; + stacktrace?: string; +} { + ensureGrpcStatusDetailsParsed(e); + + const errObj = e as { + details?: unknown; + statusDetails?: unknown; + metadata?: unknown; + cause?: unknown; + }; + + const causeObj = + errObj.cause && typeof errObj.cause === 'object' + ? (errObj.cause as { + details?: unknown; + statusDetails?: unknown; + metadata?: unknown; + }) + : undefined; + + // Prefer server error details over generic error message. + const serverDetails = errObj.details ?? causeObj?.details; + const message = + typeof serverDetails === 'string' && serverDetails.length > 0 + ? serverDetails + : e.message; + + // Serialize status details if present. + const rawStatusDetails = errObj.statusDetails ?? causeObj?.statusDetails; + let statusDetailsStr: string | undefined; + if (rawStatusDetails !== undefined && rawStatusDetails !== null) { + statusDetailsStr = + typeof rawStatusDetails === 'string' + ? rawStatusDetails + : safeJsonStringify(rawStatusDetails); + } + + // Serialize backend metadata, encoding Buffer values as base64. + const rawMetadata = errObj.metadata ?? causeObj?.metadata; + let metadataStr: string | undefined; + if (rawMetadata && typeof rawMetadata === 'object') { + try { + let map: Record; + if (typeof (rawMetadata as {getMap?: unknown}).getMap === 'function') { + map = (rawMetadata as {getMap: () => Record}).getMap(); + } else { + map = rawMetadata as Record; + } + const cleanMap: Record = {}; + for (const [key, val] of Object.entries(map)) { + if (isBuffer(val)) { + cleanMap[key] = val.toString('base64'); + } else if (Array.isArray(val)) { + cleanMap[key] = val.map(item => + isBuffer(item) ? item.toString('base64') : item, + ); + } else { + cleanMap[key] = val; + } + } + metadataStr = safeJsonStringify(cleanMap); + } catch { + metadataStr = safeJsonStringify(rawMetadata); + } + } + + // Combine status details and metadata into a formatted stacktrace string. + const stacktraceParts: string[] = []; + if (statusDetailsStr) { + stacktraceParts.push(`status_details: ${statusDetailsStr}`); + } + if (metadataStr) { + stacktraceParts.push(`metadata: ${metadataStr}`); + } + const stacktrace = + stacktraceParts.length > 0 ? stacktraceParts.join('\n') : undefined; + + return { + message, + stacktrace, + }; +} + +/** + * Records an `exception` span event with `exception.type` (and `exception.message` + * when `includeMessage` is true, such as on low level network attempt spans). + * + * @param {Span} span - The span on which to record the exception event. + * @param {Error} e - The error instance. + * @param {'grpc' | 'http'} rpcType - The RPC transport protocol. + * @param {boolean} [includeMessage=false] - Whether to include `exception.message` on the event + * (true for low level network attempt spans, false for client request spans). + */ +export function recordExceptionEvent( + span: Span, + e: Error, + rpcType: 'grpc' | 'http', + includeMessage = false, +): void { + const attributes: Attributes = { + 'exception.type': resolveExceptionType(e), + }; + if (includeMessage) { + attributes['exception.message'] = isServerSideError(e, rpcType) + ? resolveServerExceptionDetails(e).message + : e.message; + } + span.addEvent('exception', attributes); +} diff --git a/core/packages/gax/src/observability/util.ts b/core/packages/gax/src/observability/util.ts new file mode 100644 index 000000000000..dbe43d63e624 --- /dev/null +++ b/core/packages/gax/src/observability/util.ts @@ -0,0 +1,59 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {CallSettings} from '../gax'; + +/** + * Checks if telemetry tracing is enabled. + * + * If `GOOGLE_SDK_NODE_ENABLE_TRACING` is explicitly set, then the client option + * doesn't matter (only the environment variable determines whether tracing is enabled). + * The client option is only checked if the environment variable isn't set. + * + * Tracing can be enabled purely through environment variables without + * requiring client libraries to pass generator parameters, as metadata + * is resolved dynamically at runtime. + * + * If the environment variable is not set, tracing requires the client option + * and the client must have supplied `internalTelemetryInfo` (the extra protoc param). + * + * @param settings + * @returns true if telemetry tracing is enabled, false otherwise + */ +export function checkTelemetryEnabled(settings?: CallSettings): boolean { + // `process` is undeclared in browsers and some edge runtimes, where reading + // it would throw a ReferenceError rather than yield undefined, so it is + // reached through a `typeof` guard and stands in as an empty environment. + const env: Record = + typeof process === 'object' && typeof process.env === 'object' + ? process.env + : {}; + const envOptIn = env.GOOGLE_SDK_NODE_ENABLE_TRACING?.trim(); + + if (envOptIn !== undefined && envOptIn !== '') { + const lower = envOptIn.toLowerCase(); + return lower === 'true' || lower === '1'; + } + + const clientOptIn = Boolean( + settings?.enableTelemetryTracing || + settings?.otherArgs?.enableTelemetryTracing, + ); + + return ( + clientOptIn && settings?.otherArgs?.internalTelemetryInfo !== undefined + ); +} diff --git a/core/packages/gax/src/util.ts b/core/packages/gax/src/util.ts index e44fb7c20645..3609b6257500 100644 --- a/core/packages/gax/src/util.ts +++ b/core/packages/gax/src/util.ts @@ -14,56 +14,12 @@ * limitations under the License. */ -import {CallSettings} from './gax'; - const PROTO_TYPE_PREFIX = 'type.googleapis.com/'; const NUM_OF_PARTS_IN_PROTO_TYPE_NAME = 2; const randomUUID = () => globalThis.crypto?.randomUUID() || require('crypto').randomUUID(); -/** - * Checks if telemetry tracing is enabled. - * - * If `GOOGLE_SDK_NODE_ENABLE_TRACING` is explicitly set, then the client option - * doesn't matter (only the environment variable determines whether tracing is enabled). - * The client option is only checked if the environment variable isn't set. - * - * Tracing can be enabled purely through environment variables without - * requiring client libraries to pass generator parameters, as metadata - * is resolved dynamically at runtime. - * - * If the environment variable is not set, tracing requires the client option - * and the client must have supplied `internalTelemetryInfo` (the extra protoc param). - * - * @param settings - * @returns true if telemetry tracing is enabled, false otherwise - */ -export function checkTelemetryEnabled(settings?: CallSettings): boolean { - // `process` is undeclared in browsers and some edge runtimes, where reading - // it would throw a ReferenceError rather than yield undefined, so it is - // reached through a `typeof` guard and stands in as an empty environment. - const env: Record = - typeof process === 'object' && typeof process.env === 'object' - ? process.env - : {}; - const envOptIn = env.GOOGLE_SDK_NODE_ENABLE_TRACING?.trim(); - - if (envOptIn !== undefined && envOptIn !== '') { - const lower = envOptIn.toLowerCase(); - return lower === 'true' || lower === '1'; - } - - const clientOptIn = Boolean( - settings?.enableTelemetryTracing || - settings?.otherArgs?.enableTelemetryTracing, - ); - - return ( - clientOptIn && settings?.otherArgs?.internalTelemetryInfo !== undefined - ); -} - function words(str: string, normalize = false) { if (normalize) { // strings like somethingABCSomething are special case for protobuf.js, @@ -206,89 +162,6 @@ export const decodeAnyProtosInArray = ( return protoListDecoded; }; -/** - * System and network error codes mapped to CLIENT_CONNECTION_ERROR. - */ -export const connectionCodes = [ - 'ENOTFOUND', - 'EAI_AGAIN', - 'ECONNREFUSED', - 'ECONNRESET', - 'EHOSTUNREACH', - 'ENETUNREACH', - 'ENETDOWN', - 'EPIPE', - 'UND_ERR_CONNECT_TIMEOUT', - 'UND_ERR_SOCKET', - 'CERT_HAS_EXPIRED', - 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', - 'DEPTH_ZERO_SELF_SIGNED_CERT', - 'SELF_SIGNED_CERT_IN_CHAIN', - 'ERR_TLS_CERT_ALTNAME_INVALID', - 'UNABLE_TO_GET_ISSUER_CERT', -]; - -/** - * System and client request error codes mapped to CLIENT_REQUEST_ERROR. - */ -export const requestCodes = [ - 'ERR_INVALID_ARG_TYPE', - 'ERR_INVALID_URL', - 'ERR_HTTP_INVALID_HEADER_VALUE', - 'ERR_INVALID_HTTP_TOKEN', - 'ERR_INVALID_PROTOCOL', - 'ERR_INVALID_ARG_VALUE', -]; - -/** - * Stream and body error codes mapped to CLIENT_REQUEST_BODY_ERROR. - */ -export const requestBodyCodes = [ - 'ERR_STREAM_WRITE_AFTER_END', - 'ERR_STREAM_DESTROYED', - 'ERR_STREAM_ALREADY_FINISHED', - 'ERR_STREAM_CANNOT_PIPE', - 'ERR_STREAM_NULL_VALUES', - 'ERR_STREAM_PREMATURE_CLOSE', -]; - -/** - * Buffer error codes mapped to CLIENT_RESPONSE_DECODE_ERROR. - */ -export const decodeCodes = ['ERR_BUFFER_OUT_OF_BOUNDS']; - -/** - * Redirect error codes mapped to CLIENT_REDIRECT_ERROR. - */ -export const redirectCodes = [ - 'ERR_TOO_MANY_REDIRECTS', - 'ERR_FR_TOO_MANY_REDIRECTS', -]; - -/** - * Generic class / constructor names that should be unwrapped to find more specific error causes. - */ -export const genericClasses = [ - 'Error', - 'GoogleError', - 'Object', - 'DOMException', -]; - -/** - * Error codes indicating failures prior to network connection establishment. - */ -export const preConnectionCodes = [ - 'ENOTFOUND', - 'EAI_AGAIN', - 'ECONNREFUSED', - 'ECONNRESET', - 'EHOSTUNREACH', - 'ENETUNREACH', - 'ERR_INVALID_ARG_TYPE', - 'ERR_INVALID_URL', -]; - /** * Standard or infrastructure tokens in the x-goog-api-client header that should * be ignored when resolving the client artifact/package name. @@ -303,8 +176,3 @@ export const ignoredClientHeaderTokens = [ 'gapic', 'gccl', ]; - -/** - * Maximum traversal depth for unwrapping nested error causes. - */ -export const DEPTH_TO_CHECK = 10; diff --git a/core/packages/gax/test/unit/apiCallable.ts b/core/packages/gax/test/unit/apiCallable.ts index d99265238a92..82ed8a26b004 100644 --- a/core/packages/gax/test/unit/apiCallable.ts +++ b/core/packages/gax/test/unit/apiCallable.ts @@ -35,15 +35,16 @@ import { createApiCall as fallbackCreateApiCall, GrpcClient as FallbackGrpcClient, } from '../../src/fallback'; -import {GrpcClient} from '../../src/grpc'; import {PageDescriptor, StreamDescriptor} from '../../src/descriptor'; import {StreamType} from '../../src/streamingCalls/streaming'; import * as gax from '../../src/gax'; import {GoogleError} from '../../src/googleError'; import {OtelHarness} from './otelHarness'; import * as tracerHelper from '../../src/observability/TracerHelper'; -import {StaticTraceContext} from '../../src/observability/TracerHelper'; -import {clearMetadataCache} from '../../src/observability/metadataResolver'; +import { + StaticTraceContext, + clearMetadataCache, +} from '../../src/observability/contextUtil'; import * as utils from './utils'; import * as retries from '../../src/normalCalls/retries'; @@ -571,10 +572,31 @@ describe('createApiCall', () => { const span = spans[1]; assert.strictEqual(attemptSpan.name, 'google.example.v1.Echo/Echo'); assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); + assert.strictEqual( + attemptSpan.spanContext().traceId, + span.spanContext().traceId, + ); assert.strictEqual( attemptSpan.parentSpanContext?.spanId, span.spanContext().spanId, ); + assert.strictEqual( + attemptSpan.attributes['url.domain'], + 'echo.googleapis.com', + ); + harness.assertServerAddressAndPort( + {address: 'echo.googleapis.com', port: 443}, + {span: attemptSpan}, + ); + assert.strictEqual( + attemptSpan.attributes['rpc.method'], + 'google.example.v1.Echo/Echo', + ); + assert.strictEqual( + attemptSpan.attributes['http.request.method'], + undefined, + ); + harness.assertResponseStatus({rpcStatus: 'OK'}, {span: attemptSpan}); assert.strictEqual(span.name, 'EchoClient.Echo'); assert.strictEqual(span.kind, SpanKind.INTERNAL); assert.strictEqual(span.ended, true); @@ -592,7 +614,12 @@ describe('createApiCall', () => { '@google-cloud/echo', ); assert.strictEqual(span.attributes['rpc.system.name'], 'grpc'); + assert.strictEqual( + span.attributes['rpc.method'], + 'google.example.v1.Echo/Echo', + ); assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); + harness.assertResponseStatus({rpcStatus: 'OK'}, {span}); }); it('enables tracing purely through GOOGLE_SDK_NODE_ENABLE_TRACING and resolves static metadata dynamically at runtime', async () => { @@ -721,10 +748,35 @@ describe('createApiCall', () => { const attemptSpan = spans[0]; const span = spans[1]; assert.strictEqual(attemptSpan.name, 'POST'); + assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); + assert.strictEqual( + attemptSpan.spanContext().traceId, + span.spanContext().traceId, + ); + assert.strictEqual( + attemptSpan.parentSpanContext?.spanId, + span.spanContext().spanId, + ); assert.strictEqual(attemptSpan.attributes['rpc.system.name'], 'http'); + assert.strictEqual( + attemptSpan.attributes['http.request.method'], + 'POST', + ); + assert.strictEqual(attemptSpan.attributes['rpc.method'], undefined); + harness.assertServerAddressAndPort( + {address: 'echo.googleapis.com', port: 443}, + {span: attemptSpan}, + ); + harness.assertResponseStatus({httpStatus: 200}, {span: attemptSpan}); assert.strictEqual(span.name, 'EchoClient.Echo'); + assert.strictEqual(span.kind, SpanKind.INTERNAL); assert.strictEqual(span.ended, true); assert.strictEqual(span.attributes['rpc.system.name'], 'http'); + assert.strictEqual( + span.attributes['rpc.method'], + 'google.example.v1.Echo/Echo', + ); + harness.assertResponseStatus({httpStatus: 200}, {span}); }); it('overrides an explicit _fallback argument, since the call is a fallback call by definition', async () => { @@ -1274,7 +1326,7 @@ describe('createApiCall', () => { ) { attempts++; if (attempts < 3) { - const error = new GoogleError(); + const error = new GoogleError('transient failure'); error.code = FAKE_STATUS_CODE_1; callback(error); } else { @@ -1302,8 +1354,37 @@ describe('createApiCall', () => { ); assert.strictEqual(attemptSpans.length, attempts); attemptSpans.forEach((attemptSpan, idx) => { + assert.strictEqual(attemptSpan.kind, SpanKind.CLIENT); + assert.strictEqual( + attemptSpan.parentSpanContext?.spanId, + span.spanContext().spanId, + ); harness.assertResendCount(idx, {span: attemptSpan}); + if (idx < attempts - 1) { + assert.strictEqual( + attemptSpan.status.code, + SpanStatusCode.ERROR, + ); + assert.strictEqual( + attemptSpan.status.message, + 'transient failure', + ); + assert.strictEqual(attemptSpan.events.length, 1); + assert.strictEqual(attemptSpan.events[0].name, 'exception'); + } else { + assert.strictEqual( + attemptSpan.status.code, + SpanStatusCode.UNSET, + ); + harness.assertResponseStatus( + transport.rpcType === 'http' + ? {httpStatus: 200} + : {rpcStatus: 'OK'}, + {span: attemptSpan}, + ); + } }); + assert.strictEqual(span.status.code, SpanStatusCode.UNSET); harness.assertResponseStatus( transport.rpcType === 'http' ? {httpStatus: 200} @@ -1687,197 +1768,6 @@ describe('createApiCall', () => { }); describe('client request span to low level network per-attempt span correlation', () => { - it('emits a low level network gRPC attempt span parented to its client request span', async () => { - const grpcClient = new GrpcClient({ - servicePath: 'echo.googleapis.com', - port: 443, - }); - const defaults = grpcClient.constructSettings( - 'google.example.v1.Echo', - { - interfaces: { - 'google.example.v1.Echo': { - methods: { - Echo: {timeout_millis: 5000}, - }, - }, - }, - }, - {}, - {'x-goog-api-client': 'test'}, - true, - telemetryInfo, - ); - - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - callback(null, {echo: 'ok'}); - return {cancel: () => {}}; - }; - - const apiCall = gaxCreateApiCall( - stubFunc as unknown as GRPCCall, - defaults.echo, - ); - await apiCall({message: 'hello'}, undefined); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 2); - - const networkSpan = spans.find( - s => s.name === 'google.example.v1.Echo/Echo', - )!; - const clientRequestSpan = spans.find( - s => s.name === 'EchoClient.Echo', - )!; - assert.ok(networkSpan); - assert.ok(clientRequestSpan); - - assert.strictEqual(clientRequestSpan.kind, SpanKind.INTERNAL); - assert.strictEqual(networkSpan.kind, SpanKind.CLIENT); - assert.strictEqual( - networkSpan.spanContext().traceId, - clientRequestSpan.spanContext().traceId, - ); - assert.strictEqual( - networkSpan.parentSpanContext?.spanId, - clientRequestSpan.spanContext().spanId, - ); - assert.strictEqual( - networkSpan.attributes['url.domain'], - 'echo.googleapis.com', - ); - assert.strictEqual( - networkSpan.attributes['server.address'], - 'echo.googleapis.com', - ); - assert.strictEqual(networkSpan.attributes['server.port'], 443); - assert.strictEqual( - networkSpan.attributes['rpc.method'], - 'google.example.v1.Echo/Echo', - ); - assert.strictEqual( - clientRequestSpan.attributes['rpc.method'], - 'google.example.v1.Echo/Echo', - ); - assert.strictEqual( - networkSpan.attributes['http.request.method'], - undefined, - ); - assert.strictEqual( - networkSpan.attributes['rpc.response.status_code'], - 'OK', - ); - assert.strictEqual( - networkSpan.attributes['grpc.response.status_code'], - undefined, - ); - assert.strictEqual( - networkSpan.attributes['http.response.status_code'], - undefined, - ); - }); - - it('emits a low level network HTTP/REST attempt span parented to its client request span', async () => { - const fallbackClient = new FallbackGrpcClient({ - servicePath: 'echo.googleapis.com', - port: 443, - }); - const defaults = fallbackClient.constructSettings( - 'google.example.v1.Echo', - { - interfaces: { - 'google.example.v1.Echo': { - methods: { - Echo: {timeout_millis: 5000}, - }, - }, - }, - }, - {}, - {'x-goog-api-client': 'test'}, - true, - telemetryInfo, - ); - - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - callback(null, {echo: 'ok'}); - return {cancel: () => {}}; - }; - - const apiCall = gaxCreateApiCall( - stubFunc as unknown as GRPCCall, - defaults.echo, - undefined, - 'rest', - ); - await apiCall({message: 'hello'}, undefined); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 2); - - const networkSpan = spans.find(s => s.name === 'POST')!; - const clientRequestSpan = spans.find( - s => s.name === 'EchoClient.Echo', - )!; - assert.ok(networkSpan); - assert.ok(clientRequestSpan); - - assert.strictEqual(clientRequestSpan.kind, SpanKind.INTERNAL); - assert.strictEqual(networkSpan.kind, SpanKind.CLIENT); - assert.strictEqual( - networkSpan.spanContext().traceId, - clientRequestSpan.spanContext().traceId, - ); - assert.strictEqual( - networkSpan.parentSpanContext?.spanId, - clientRequestSpan.spanContext().spanId, - ); - assert.strictEqual( - networkSpan.attributes['url.domain'], - 'echo.googleapis.com', - ); - assert.strictEqual( - networkSpan.attributes['server.address'], - 'echo.googleapis.com', - ); - assert.strictEqual(networkSpan.attributes['server.port'], 443); - assert.strictEqual( - networkSpan.attributes['http.request.method'], - 'POST', - ); - assert.strictEqual(networkSpan.attributes['rpc.method'], undefined); - assert.strictEqual( - clientRequestSpan.attributes['rpc.method'], - 'google.example.v1.Echo/Echo', - ); - assert.strictEqual( - networkSpan.attributes['rpc.response.status_code'], - undefined, - ); - assert.strictEqual( - networkSpan.attributes['http.response.status_code'], - 200, - ); - assert.strictEqual( - clientRequestSpan.attributes['rpc.response.status_code'], - undefined, - ); - assert.strictEqual( - clientRequestSpan.attributes['http.response.status_code'], - 200, - ); - }); - it('populates url.template and updates low level network span name when using a real REST fallback stub', async () => { const authClient = new PassThroughClient(); const fallbackClient = new FallbackGrpcClient({ @@ -2030,130 +1920,6 @@ describe('createApiCall', () => { clientRequestExpand.spanContext().spanId, ); }); - - it('emits one low level network attempt span per retry attempt, all parented to the single client request span', async () => { - const retryOptions = gax.createRetryOptions( - [status.UNAVAILABLE], - gax.createBackoffSettings(1, 1.1, 5, 100, 1.0, 100, 1000), - ); - - const settings = new gax.CallSettings({ - apiName: 'google.example.v1.Echo', - retry: retryOptions, - enableTelemetryTracing: true, - otherArgs: { - internalTelemetryInfo: telemetryInfo, - internalMethodName: 'Echo', - }, - }); - - let attempt = 0; - const stubFunc = ( - argument: {}, - metadata: {}, - options: {}, - callback: Function, - ): GRPCCallResult => { - attempt++; - if (attempt === 1) { - const err = new GoogleError('transient failure'); - err.code = status.UNAVAILABLE; - callback(err); - } else { - callback(null, {echo: 'recovered'}); - } - return {cancel: () => {}}; - }; - - const apiCall = gaxCreateApiCall( - stubFunc as unknown as GRPCCall, - settings, - ); - await apiCall({message: 'retry-me'}, undefined); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 3); - - const clientRequestSpan = spans.find( - s => s.name === 'EchoClient.Echo', - )!; - const networkSpans = spans.filter( - s => s.name === 'google.example.v1.Echo/Echo', - ); - assert.ok(clientRequestSpan); - assert.strictEqual(networkSpans.length, 2); - - // First attempt failed with UNAVAILABLE (resend_count omitted on initial attempt) - assert.strictEqual(networkSpans[0].kind, SpanKind.CLIENT); - assert.strictEqual(networkSpans[0].status.code, SpanStatusCode.ERROR); - assert.strictEqual(networkSpans[0].status.message, 'transient failure'); - assert.strictEqual( - networkSpans[0].attributes['status.message'], - 'transient failure', - ); - assert.strictEqual( - networkSpans[0].attributes['gcp.grpc.resend_count'], - undefined, - ); - assert.strictEqual( - networkSpans[0].attributes['error.type'], - 'UNAVAILABLE', - ); - assert.strictEqual( - networkSpans[0].attributes['rpc.response.status_code'], - 'UNAVAILABLE', - ); - assert.strictEqual( - networkSpans[0].attributes['grpc.response.status_code'], - undefined, - ); - assert.strictEqual(networkSpans[0].events.length, 1); - assert.strictEqual(networkSpans[0].events[0].name, 'exception'); - assert.strictEqual( - networkSpans[0].events[0].attributes?.['exception.type'], - 'GoogleError', - ); - assert.strictEqual( - networkSpans[0].parentSpanContext?.spanId, - clientRequestSpan.spanContext().spanId, - ); - - // Second attempt succeeded with OK (resend_count = 1) - assert.strictEqual(networkSpans[1].kind, SpanKind.CLIENT); - assert.strictEqual(networkSpans[1].status.code, SpanStatusCode.UNSET); - assert.strictEqual( - networkSpans[1].attributes['gcp.grpc.resend_count'], - 1, - ); - assert.strictEqual( - networkSpans[1].attributes['rpc.response.status_code'], - 'OK', - ); - assert.strictEqual( - networkSpans[1].attributes['grpc.response.status_code'], - undefined, - ); - assert.strictEqual( - networkSpans[1].parentSpanContext?.spanId, - clientRequestSpan.spanContext().spanId, - ); - - // Overall client request span succeeded (resend_count omitted on client request span) - assert.strictEqual(clientRequestSpan.kind, SpanKind.INTERNAL); - assert.strictEqual(clientRequestSpan.status.code, SpanStatusCode.UNSET); - assert.strictEqual( - clientRequestSpan.attributes['gcp.grpc.resend_count'], - undefined, - ); - assert.strictEqual( - clientRequestSpan.attributes['rpc.response.status_code'], - 'OK', - ); - assert.strictEqual( - clientRequestSpan.attributes['grpc.response.status_code'], - undefined, - ); - }); }); }); }); diff --git a/core/packages/gax/test/unit/metadataResolver.ts b/core/packages/gax/test/unit/contextUtil.ts similarity index 88% rename from core/packages/gax/test/unit/metadataResolver.ts rename to core/packages/gax/test/unit/contextUtil.ts index 1493a23f7942..72897311a2df 100644 --- a/core/packages/gax/test/unit/metadataResolver.ts +++ b/core/packages/gax/test/unit/contextUtil.ts @@ -17,18 +17,20 @@ import * as assert from 'assert'; import {describe, it, beforeEach, afterEach} from 'mocha'; import { + buildInitialAttributes, clearMetadataCache, + DynamicTraceContext, extractClientServiceFromPackageName, extractFromEnvironment, extractFromSettings, extractServiceFromApiName, resolveStaticTraceContext, + StaticTraceContext, DEFAULT_GCP_REPO, -} from '../../src/observability/metadataResolver'; +} from '../../src/observability/contextUtil'; import {CallSettings} from '../../src/gax'; -import {StaticTraceContext} from '../../src/observability/TracerHelper'; -describe('metadataResolver', () => { +describe('contextUtil', () => { beforeEach(() => { clearMetadataCache(); delete process.env.GOOGLE_SDK_NODE_CLIENT_SERVICE; @@ -381,4 +383,62 @@ describe('metadataResolver', () => { assert.strictEqual(metadata.serverPort, 8443); }); }); + + describe('buildInitialAttributes', () => { + it('builds initial client, transport, and url.domain attributes', () => { + const dynamicArgs: DynamicTraceContext = { + clientName: 'StorageClient', + methodName: 'GetObject', + rpcType: 'grpc', + }; + const staticArgs: StaticTraceContext = { + gcpClientService: 'storage', + gcpVersion: '1.2.3', + gcpRepo: DEFAULT_GCP_REPO, + gcpArtifact: '@google-cloud/storage', + }; + + assert.deepStrictEqual( + buildInitialAttributes( + dynamicArgs, + staticArgs, + 'storage.googleapis.com', + ), + { + 'gcp.client.service': 'storage', + 'gcp.client.version': '1.2.3', + 'gcp.client.repo': DEFAULT_GCP_REPO, + 'gcp.client.artifact': '@google-cloud/storage', + 'rpc.system.name': 'grpc', + 'url.domain': 'storage.googleapis.com', + }, + ); + + assert.deepStrictEqual(buildInitialAttributes(dynamicArgs, staticArgs), { + 'gcp.client.service': 'storage', + 'gcp.client.version': '1.2.3', + 'gcp.client.repo': DEFAULT_GCP_REPO, + 'gcp.client.artifact': '@google-cloud/storage', + 'rpc.system.name': 'grpc', + }); + + assert.deepStrictEqual( + buildInitialAttributes( + { + clientName: 'ComputeClient', + methodName: 'InsertInstance', + rpcType: 'http', + }, + {}, + ), + { + 'gcp.client.service': undefined, + 'gcp.client.version': undefined, + 'gcp.client.repo': undefined, + 'gcp.client.artifact': undefined, + 'rpc.system.name': 'http', + }, + ); + }); + }); }); diff --git a/core/packages/gax/test/unit/errorUtil.ts b/core/packages/gax/test/unit/errorUtil.ts new file mode 100644 index 000000000000..2ed429e50f36 --- /dev/null +++ b/core/packages/gax/test/unit/errorUtil.ts @@ -0,0 +1,859 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'assert'; +import * as vm from 'vm'; +import {describe, it} from 'mocha'; +import * as grpc from '@grpc/grpc-js'; +import {Span} from '@opentelemetry/api'; +import {GoogleError} from '../../src/googleError'; +import {Status} from '../../src/status'; +import { + connectionCodes, + requestCodes, + requestBodyCodes, + decodeCodes, + redirectCodes, + genericClasses, + preConnectionCodes, + DEPTH_TO_CHECK, + resolveErrorInfoReason, + resolveServerErrorCode, + isServerSideError, + resolveClientNetworkOrOperationalError, + resolveLanguageSpecificErrorType, + resolveErrorType, + resolveExceptionType, + resolveSystemErrorCode, + resolveRpcStatusName, + resolveHttpStatusCode, + resolveErrorMessage, + resolveServerExceptionDetails, + recordExceptionEvent, + isPreConnectionFailure, + safeJsonStringify, +} from '../../src/observability/errorUtil'; + +describe('errorUtil', () => { + describe('error code and class constants', () => { + it('defines connectionCodes containing common network errors', () => { + assert(Array.isArray(connectionCodes)); + assert(connectionCodes.includes('ECONNREFUSED')); + assert(connectionCodes.includes('ENOTFOUND')); + }); + + it('defines requestCodes containing request error codes', () => { + assert(Array.isArray(requestCodes)); + assert(requestCodes.includes('ERR_INVALID_ARG_TYPE')); + assert(requestCodes.includes('ERR_INVALID_URL')); + }); + + it('defines requestBodyCodes containing stream error codes', () => { + assert(Array.isArray(requestBodyCodes)); + assert(requestBodyCodes.includes('ERR_STREAM_WRITE_AFTER_END')); + }); + + it('defines decodeCodes containing buffer error codes', () => { + assert(Array.isArray(decodeCodes)); + assert(decodeCodes.includes('ERR_BUFFER_OUT_OF_BOUNDS')); + }); + + it('defines redirectCodes containing redirect error codes', () => { + assert(Array.isArray(redirectCodes)); + assert(redirectCodes.includes('ERR_TOO_MANY_REDIRECTS')); + }); + + it('defines genericClasses containing standard base error types', () => { + assert(Array.isArray(genericClasses)); + assert(genericClasses.includes('Error')); + assert(genericClasses.includes('GoogleError')); + assert(genericClasses.includes('Object')); + assert(genericClasses.includes('DOMException')); + }); + + it('defines preConnectionCodes containing pre-connection error codes', () => { + assert(Array.isArray(preConnectionCodes)); + assert(preConnectionCodes.includes('ECONNREFUSED')); + assert(preConnectionCodes.includes('ENOTFOUND')); + assert(preConnectionCodes.includes('ERR_INVALID_URL')); + }); + + it('defines DEPTH_TO_CHECK constant as 10', () => { + assert.strictEqual(DEPTH_TO_CHECK, 10); + }); + }); + + describe('error classification and resolution helpers', () => { + it('resolveErrorInfoReason handles direct reason, statusDetails, and cause', () => { + assert.strictEqual( + resolveErrorInfoReason({reason: 'RATE_LIMIT_EXCEEDED'}), + 'RATE_LIMIT_EXCEEDED', + ); + assert.strictEqual( + resolveErrorInfoReason({ + statusDetails: [{reason: 'SERVICE_DISABLED'}], + }), + 'SERVICE_DISABLED', + ); + assert.strictEqual( + resolveErrorInfoReason({ + cause: {reason: 'ACCESS_TOKEN_EXPIRED'}, + }), + 'ACCESS_TOKEN_EXPIRED', + ); + assert.strictEqual(resolveErrorInfoReason(new Error('none')), undefined); + }); + + it('resolveServerErrorCode handles http and grpc', () => { + const httpErr = Object.assign(new Error('Not Found'), { + httpStatusCode: 404, + }); + const grpcErr = Object.assign(new Error('Not Found'), { + code: Status.NOT_FOUND, + }); + assert.strictEqual(resolveServerErrorCode(httpErr, 'http'), '404'); + assert.strictEqual(resolveServerErrorCode(grpcErr, 'grpc'), 'NOT_FOUND'); + assert.strictEqual(resolveServerErrorCode(grpcErr, 'http'), undefined); + assert.strictEqual(resolveServerErrorCode(httpErr, 'grpc'), undefined); + }); + + it('resolveServerErrorCode handles plain error objects without stack property', () => { + assert.strictEqual( + resolveServerErrorCode({httpStatusCode: 500}, 'http'), + '500', + ); + assert.strictEqual( + resolveServerErrorCode({code: Status.NOT_FOUND}, 'grpc'), + 'NOT_FOUND', + ); + assert.strictEqual( + resolveServerErrorCode({code: 5}, 'grpc'), + 'NOT_FOUND', + ); + assert.strictEqual( + resolveServerErrorCode({httpStatusCode: 500}, 'grpc'), + undefined, + ); + assert.strictEqual(resolveServerErrorCode({code: 5}, 'http'), undefined); + }); + + it('isServerSideError recognizes plain error objects with valid status codes', () => { + assert.strictEqual(isServerSideError({httpStatusCode: 500}, 'http'), true); + assert.strictEqual( + isServerSideError({code: Status.NOT_FOUND}, 'grpc'), + true, + ); + assert.strictEqual(isServerSideError({code: 5}, 'grpc'), true); + assert.strictEqual( + isServerSideError({httpStatusCode: 500}, 'grpc'), + false, + ); + assert.strictEqual(isServerSideError({code: 5}, 'http'), false); + assert.strictEqual(isServerSideError({message: 'no code'}, 'grpc'), false); + assert.strictEqual(isServerSideError({message: 'no code'}, 'http'), false); + }); + + it('resolveClientNetworkOrOperationalError maps timeout, connection, request, body, decode, redirect, and auth errors', () => { + // CLIENT_TIMEOUT + assert.strictEqual( + resolveClientNetworkOrOperationalError( + new DOMException('timed out', 'TimeoutError'), + ), + 'CLIENT_TIMEOUT', + ); + assert.strictEqual( + resolveClientNetworkOrOperationalError({code: 'ETIMEDOUT'}), + 'CLIENT_TIMEOUT', + ); + assert.strictEqual( + resolveClientNetworkOrOperationalError({code: 'ESOCKETTIMEDOUT'}), + 'CLIENT_TIMEOUT', + ); + assert.strictEqual( + resolveClientNetworkOrOperationalError( + new GoogleError('Total timeout of API exceeded before completion'), + ), + 'CLIENT_TIMEOUT', + ); + + // CLIENT_CONNECTION_ERROR + for (const code of [ + 'ENOTFOUND', + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'CERT_HAS_EXPIRED', + 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + 'ERR_TLS_CERT_ALTNAME_INVALID', + 'ERR_SSL_PROTOCOL_ERROR', + ]) { + assert.strictEqual( + resolveClientNetworkOrOperationalError({code}), + 'CLIENT_CONNECTION_ERROR', + `Expected ${code} to resolve to CLIENT_CONNECTION_ERROR`, + ); + } + + // CLIENT_REQUEST_ERROR + assert.strictEqual( + resolveClientNetworkOrOperationalError({ + code: 'ERR_INVALID_ARG_TYPE', + }), + 'CLIENT_REQUEST_ERROR', + ); + assert.strictEqual( + resolveClientNetworkOrOperationalError({code: 'ERR_INVALID_URL'}), + 'CLIENT_REQUEST_ERROR', + ); + assert.strictEqual( + resolveClientNetworkOrOperationalError( + new URIError('malformed URI sequence'), + ), + 'CLIENT_REQUEST_ERROR', + ); + + // CLIENT_REQUEST_BODY_ERROR + assert.strictEqual( + resolveClientNetworkOrOperationalError({ + code: 'ERR_STREAM_WRITE_AFTER_END', + }), + 'CLIENT_REQUEST_BODY_ERROR', + ); + + // CLIENT_RESPONSE_DECODE_ERROR + class DecodeError extends Error {} + assert.strictEqual( + resolveClientNetworkOrOperationalError( + new DecodeError('failed to decode response'), + ), + 'CLIENT_RESPONSE_DECODE_ERROR', + ); + assert.strictEqual( + resolveClientNetworkOrOperationalError( + new SyntaxError('Unexpected token < in JSON'), + ), + 'CLIENT_RESPONSE_DECODE_ERROR', + ); + + // CLIENT_REDIRECT_ERROR + assert.strictEqual( + resolveClientNetworkOrOperationalError({ + code: 'ERR_TOO_MANY_REDIRECTS', + }), + 'CLIENT_REDIRECT_ERROR', + ); + + // CLIENT_AUTHENTICATION_ERROR + class GoogleAuthError extends Error {} + assert.strictEqual( + resolveClientNetworkOrOperationalError( + new GoogleAuthError('Missing credentials'), + ), + 'CLIENT_AUTHENTICATION_ERROR', + ); + assert.strictEqual( + resolveClientNetworkOrOperationalError({code: 'ERR_NO_CREDENTIALS'}), + 'CLIENT_AUTHENTICATION_ERROR', + ); + + assert.strictEqual( + resolveClientNetworkOrOperationalError(new Error('normal error')), + undefined, + ); + }); + + it('resolveLanguageSpecificErrorType excludes generic classes and unwraps cause', () => { + class CustomRpcError extends Error {} + assert.strictEqual( + resolveLanguageSpecificErrorType( + new DOMException('Operation aborted', 'AbortError'), + ), + 'AbortError', + ); + assert.strictEqual( + resolveLanguageSpecificErrorType(new RangeError('out of bounds')), + 'RangeError', + ); + assert.strictEqual( + resolveLanguageSpecificErrorType(new CustomRpcError('custom')), + 'CustomRpcError', + ); + assert.strictEqual( + resolveLanguageSpecificErrorType(new Error('generic')), + undefined, + ); + const intermediate = new GoogleError('intermediate'); + intermediate.cause = new TypeError('inner'); + const outer = new GoogleError('outer'); + outer.cause = intermediate; + assert.strictEqual(resolveLanguageSpecificErrorType(outer), 'TypeError'); + }); + + describe('resolveErrorType', () => { + it('implements the 5-tier precedence hierarchy', () => { + // Tier 1 beats Tier 2 (gRPC and HTTP) + const t1Grpc = Object.assign(new GoogleError('foo'), { + reason: 'MY_REASON', + code: Status.INVALID_ARGUMENT, + }); + assert.strictEqual(resolveErrorType(t1Grpc, 'grpc'), 'MY_REASON'); + + const t1Http = Object.assign(new GoogleError('service disabled'), { + reason: 'SERVICE_DISABLED', + httpStatusCode: 403, + }); + assert.strictEqual( + resolveErrorType(t1Http, 'http'), + 'SERVICE_DISABLED', + ); + + // Tier 2 beats Tier 3 + const t2 = Object.assign(new GoogleError('foo'), { + code: Status.PERMISSION_DENIED, + }); + assert.strictEqual(resolveErrorType(t2, 'grpc'), 'PERMISSION_DENIED'); + + // Tier 3 beats Tier 4 + const t3 = Object.assign(new TypeError('invalid type'), { + code: 'ERR_INVALID_ARG_TYPE', + }); + assert.strictEqual(resolveErrorType(t3, 'grpc'), 'CLIENT_REQUEST_ERROR'); + + // Tier 4 beats Tier 5 + const t4 = new TypeError('normal type error'); + assert.strictEqual(resolveErrorType(t4, 'grpc'), 'TypeError'); + + // Tier 5 fallback + assert.strictEqual( + resolveErrorType(new Error('generic'), 'grpc'), + 'INTERNAL', + ); + assert.strictEqual( + resolveErrorType(new GoogleError('unexpected'), 'grpc'), + 'INTERNAL', + ); + assert.strictEqual(resolveErrorType('string', 'grpc'), 'INTERNAL'); + }); + + it('resolves error.type by transport and unwraps client-side causes appropriately', () => { + // Unmapped HTTP status is preserved rather than collapsed to mapped gRPC status + const teapot = Object.assign(new GoogleError('teapot'), { + code: Status.FAILED_PRECONDITION, + httpStatusCode: 418, + }); + assert.strictEqual(resolveErrorType(teapot, 'http'), '418'); + assert.strictEqual( + resolveErrorType(teapot, 'grpc'), + 'FAILED_PRECONDITION', + ); + + // HTTP call without HTTP response falls through to Tier 3 client error + const deadlineErr = Object.assign(new GoogleError('Deadline exceeded'), { + code: Status.DEADLINE_EXCEEDED, + }); + assert.strictEqual( + resolveErrorType(deadlineErr, 'http'), + 'CLIENT_TIMEOUT', + ); + + // Wrapped cause takes precedence when outer GoogleError only has mapped gRPC code + const connCause = Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }); + const wrappedConn = Object.assign(new GoogleError(connCause.message), { + code: Status.UNAVAILABLE, + cause: connCause, + }); + assert.strictEqual( + resolveErrorType(wrappedConn, 'http'), + 'CLIENT_CONNECTION_ERROR', + ); + assert.strictEqual( + resolveErrorType(wrappedConn, 'grpc'), + 'CLIENT_CONNECTION_ERROR', + ); + + const abortCause = new DOMException('Operation aborted', 'AbortError'); + const wrappedAbort = Object.assign(new GoogleError('aborted'), { + code: Status.CANCELLED, + cause: abortCause, + }); + assert.strictEqual(resolveErrorType(wrappedAbort, 'grpc'), 'AbortError'); + + // Outer HTTP response status takes precedence over inner cause exception type + const httpWithCause = Object.assign( + new GoogleError('service unavailable'), + { + code: Status.UNAVAILABLE, + httpStatusCode: 503, + cause: new TypeError('inner'), + }, + ); + assert.strictEqual(resolveErrorType(httpWithCause, 'http'), '503'); + }); + + it('falls back to INTERNAL for zero, out-of-range, or empty string status codes', () => { + assert.strictEqual( + resolveErrorType(Object.assign(new Error('unset'), {code: 0}), 'grpc'), + 'INTERNAL', + ); + assert.strictEqual( + resolveErrorType( + Object.assign(new Error('bogus'), {code: 4242}), + 'grpc', + ), + 'INTERNAL', + ); + assert.strictEqual( + resolveErrorType( + Object.assign(new Error('empty'), {code: ''}), + 'grpc', + ), + 'INTERNAL', + ); + }); + }); + + describe('resolveExceptionType', () => { + it('names the class rather than the status code or mutated name', () => { + const errorWithCode = Object.assign(new GoogleError('not found'), { + code: Status.NOT_FOUND, + name: 'CustomRpcError', + }); + assert.strictEqual(resolveExceptionType(errorWithCode), 'GoogleError'); + + class RetryError extends GoogleError {} + const subclassErr = new RetryError('retries exhausted'); + assert.strictEqual(subclassErr.name, 'Error'); + assert.strictEqual(resolveExceptionType(subclassErr), 'RetryError'); + }); + + it('resolves built-in classes, falls back to name on bare Error, and reports Error when unset', () => { + assert.strictEqual( + resolveExceptionType(new TypeError('bad type')), + 'TypeError', + ); + const namedErr = new Error('aborted'); + namedErr.name = 'AbortError'; + assert.strictEqual(resolveExceptionType(namedErr), 'AbortError'); + assert.strictEqual(resolveExceptionType(new Error('generic')), 'Error'); + }); + }); + + it('resolveSystemErrorCode, resolveRpcStatusName, resolveHttpStatusCode, and resolveErrorMessage inspect errors and causes', () => { + const wrappedSys = new GoogleError('wrapped'); + wrappedSys.cause = {code: 'ECONNREFUSED'}; + assert.strictEqual(resolveSystemErrorCode(wrappedSys), 'ECONNREFUSED'); + + assert.strictEqual( + resolveRpcStatusName({code: Status.UNAVAILABLE}), + 'UNAVAILABLE', + ); + assert.strictEqual(resolveRpcStatusName({code: Status.OK}), undefined); + + assert.strictEqual(resolveHttpStatusCode({httpStatusCode: 503}), 503); + + // resolveErrorMessage cases: Error, error-shaped object, cross-realm Error, non-string message, empty message, primitive + assert.strictEqual(resolveErrorMessage({message: 'msg'}), 'msg'); + const crossRealm = vm.runInNewContext( + "new Error('cross realm boom')", + ) as Error; + assert.strictEqual(crossRealm instanceof Error, false); + assert.strictEqual(resolveErrorMessage(crossRealm), 'cross realm boom'); + assert.strictEqual( + resolveErrorMessage({message: {nested: 'object'}}), + '[object Object]', + ); + assert.strictEqual(resolveErrorMessage(new Error()), ''); + assert.strictEqual(resolveErrorMessage('raw'), 'raw'); + }); + + it('recordExceptionEvent records exception.type and optionally exception.message', () => { + const events: Array<{name: string; attributes?: Record}> = + []; + const fakeSpan = { + addEvent(name: string, attributes?: Record) { + events.push({name, attributes}); + }, + } as unknown as Span; + + const serverErr = Object.assign(new GoogleError('Wrapper message'), { + code: Status.INVALID_ARGUMENT, + details: 'Detailed backend message', + }); + + recordExceptionEvent(fakeSpan, serverErr, 'grpc', false); + assert.deepStrictEqual(events[0], { + name: 'exception', + attributes: { + 'exception.type': 'GoogleError', + }, + }); + + recordExceptionEvent(fakeSpan, serverErr, 'grpc', true); + assert.deepStrictEqual(events[1], { + name: 'exception', + attributes: { + 'exception.type': 'GoogleError', + 'exception.message': 'Detailed backend message', + }, + }); + }); + }); + + describe('safeJsonStringify', () => { + it('returns undefined for undefined and serializes primitives', () => { + assert.strictEqual(safeJsonStringify(undefined), undefined); + assert.strictEqual(safeJsonStringify(null), 'null'); + assert.strictEqual(safeJsonStringify(123), '123'); + assert.strictEqual(safeJsonStringify('abc'), '"abc"'); + assert.strictEqual(safeJsonStringify(true), 'true'); + }); + + it('serializes BigInt values as strings', () => { + const obj = {big: 9007199254740993n, list: [1n, 2n]}; + assert.strictEqual( + safeJsonStringify(obj), + '{"big":"9007199254740993","list":["1","2"]}', + ); + }); + + it('handles circular references in objects', () => { + const circular: Record = {name: 'test'}; + circular.self = circular; + assert.strictEqual( + safeJsonStringify(circular), + '{"name":"test","self":"[Circular]"}', + ); + }); + + it('handles circular references in arrays', () => { + const arr: unknown[] = [1, 2]; + arr.push(arr); + assert.strictEqual(safeJsonStringify(arr), '[1,2,"[Circular]"]'); + }); + + it('handles deep circular references', () => { + const root: Record = { + a: { + b: { + c: {}, + }, + }, + }; + (root.a as Record).b = root; + assert.strictEqual(safeJsonStringify(root), '{"a":{"b":"[Circular]"}}'); + }); + + it('does not falsely mark shared references in DAGs as circular', () => { + const shared = {sharedKey: 'sharedValue'}; + const dag = {first: shared, second: shared}; + assert.strictEqual( + safeJsonStringify(dag), + '{"first":{"sharedKey":"sharedValue"},"second":{"sharedKey":"sharedValue"}}', + ); + }); + + it('safely handles non-serializable objects and throwing getters without throwing', () => { + const badObj = { + get throwingProp() { + throw new Error('getter threw'); + }, + }; + const result = safeJsonStringify(badObj); + assert.strictEqual(result, '[object Object]'); + }); + + it('safely handles objects with throwing toString without throwing', () => { + const badObj = { + get throwingProp() { + throw new Error('getter threw'); + }, + toString() { + throw new Error('toString threw'); + }, + }; + assert.strictEqual(safeJsonStringify(badObj), undefined); + }); + }); + + describe('isPreConnectionFailure', () => { + it('handles circular references in cause without infinite recursion', () => { + const circularError = new GoogleError('circular error'); + circularError.cause = circularError; + assert.strictEqual(isPreConnectionFailure(circularError), false); + + const errA = new GoogleError('errA'); + const errB = new GoogleError('errB'); + errA.cause = errB; + errB.cause = errA; + assert.strictEqual(isPreConnectionFailure(errA), false); + }); + + it('handles deep cause chains (> DEPTH_TO_CHECK depth) without stack overflow', () => { + const root = new GoogleError('root'); + let current = root; + for (let i = 0; i < 20; i++) { + const next = new GoogleError(`level-${i}`); + current.cause = next; + current = next; + } + assert.strictEqual(isPreConnectionFailure(root), false); + }); + + it('respects DEPTH_TO_CHECK limit when unwrapping cause chain', () => { + // Error within DEPTH_TO_CHECK levels is detected + const rootWithin = new GoogleError('root'); + let currWithin = rootWithin; + for (let i = 0; i < DEPTH_TO_CHECK - 2; i++) { + const next = new GoogleError(`level-${i}`); + currWithin.cause = next; + currWithin = next; + } + currWithin.cause = new TypeError('client validation failure'); + assert.strictEqual(isPreConnectionFailure(rootWithin), true); + + // Error deeper than DEPTH_TO_CHECK levels is not reached + const rootDeeper = new GoogleError('root'); + let currDeeper = rootDeeper; + for (let i = 0; i < DEPTH_TO_CHECK + 5; i++) { + const next = new GoogleError(`level-${i}`); + currDeeper.cause = next; + currDeeper = next; + } + currDeeper.cause = new TypeError('too deep to find'); + assert.strictEqual(isPreConnectionFailure(rootDeeper), false); + }); + + it('identifies pre-connection errors directly and wrapped in GoogleError', () => { + assert.strictEqual( + isPreConnectionFailure(new TypeError('type err')), + true, + ); + assert.strictEqual( + isPreConnectionFailure(new RangeError('range err')), + true, + ); + assert.strictEqual(isPreConnectionFailure(new URIError('uri err')), true); + + const wrappedType = new GoogleError('wrapped'); + wrappedType.cause = new TypeError('inner'); + assert.strictEqual(isPreConnectionFailure(wrappedType), true); + + const netErr = Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }); + const wrappedNet = new GoogleError('wrapped connection'); + wrappedNet.cause = netErr; + assert.strictEqual(isPreConnectionFailure(wrappedNet), true); + }); + + it('returns false for server-side errors', () => { + const grpcErr = Object.assign(new GoogleError('grpc err'), { + code: Status.UNAVAILABLE, + }); + assert.strictEqual(isPreConnectionFailure(grpcErr), false); + + const httpErr = Object.assign(new GoogleError('http err'), { + httpStatusCode: 503, + }); + assert.strictEqual(isPreConnectionFailure(httpErr), false); + }); + + it('returns true for non-Errors and primitives', () => { + assert.strictEqual(isPreConnectionFailure(null), true); + assert.strictEqual(isPreConnectionFailure('string throw'), true); + assert.strictEqual(isPreConnectionFailure(123), true); + }); + }); + + describe('resolveServerExceptionDetails', () => { + it('formats status details and metadata in resolveServerExceptionDetails for gRPC server errors', () => { + const metadata = new grpc.Metadata(); + metadata.set('x-goog-request-id', 'req-123'); + const error = Object.assign(new GoogleError('Detailed server failure'), { + code: Status.INVALID_ARGUMENT, + details: 'Field name is invalid', + statusDetails: [{field: 'name', description: 'must be non-empty'}], + metadata, + reason: 'INVALID_FIELD', + domain: 'googleapis.com', + errorInfoMetadata: {consumer: 'projects/123'}, + }); + + const details = resolveServerExceptionDetails(error); + assert.strictEqual(details.message, 'Field name is invalid'); + assert.strictEqual( + details.stacktrace, + 'status_details: [{"field":"name","description":"must be non-empty"}]\n' + + 'metadata: {"x-goog-request-id":"req-123"}', + ); + }); + + it('formats status details and metadata in resolveServerExceptionDetails for HTTP fallback server errors', () => { + const error = Object.assign(new GoogleError('HTTP server error'), { + httpStatusCode: 400, + details: 'Bad HTTP Request', + statusDetails: 'Quota exceeded', + metadata: {'content-type': 'application/json'}, + }); + + const details = resolveServerExceptionDetails(error); + assert.strictEqual(details.message, 'Bad HTTP Request'); + assert.strictEqual( + details.stacktrace, + 'status_details: Quota exceeded\n' + + 'metadata: {"content-type":"application/json"}', + ); + }); + + it('encodes Buffer and Array of Buffers in metadata as base64', () => { + const error = Object.assign(new GoogleError('Buffer error'), { + metadata: { + 'bin-key': Buffer.from('hello'), + 'bin-array': [Buffer.from('foo'), 'bar', Buffer.from('baz')], + 'regular-key': 'value', + }, + }); + const {stacktrace} = resolveServerExceptionDetails(error); + assert.strictEqual( + stacktrace, + 'metadata: {"bin-key":"aGVsbG8=","bin-array":["Zm9v","bar","YmF6"],"regular-key":"value"}', + ); + }); + + it('formats metadata safely without ReferenceError when Buffer is not defined', () => { + const originalBuffer = (globalThis as Record).Buffer; + try { + delete (globalThis as Record).Buffer; + const error = Object.assign(new GoogleError('Browser error'), { + metadata: { + 'content-type': 'application/json', + headers: ['x-goog-request-id', 'req-456'], + }, + }); + const {stacktrace} = resolveServerExceptionDetails(error); + assert.strictEqual( + stacktrace, + 'metadata: {"content-type":"application/json","headers":["x-goog-request-id","req-456"]}', + ); + } finally { + (globalThis as Record).Buffer = originalBuffer; + } + }); + + it('safely formats statusDetails containing circular references', () => { + const detail: Record = { + reason: 'RATE_LIMIT_EXCEEDED', + }; + detail.self = detail; + const error = Object.assign( + new GoogleError('Circular statusDetails error'), + { + statusDetails: [detail], + }, + ); + const {stacktrace} = resolveServerExceptionDetails(error); + assert.strictEqual( + stacktrace, + 'status_details: [{"reason":"RATE_LIMIT_EXCEEDED","self":"[Circular]"}]', + ); + }); + + it('safely formats statusDetails containing BigInt values', () => { + const error = Object.assign( + new GoogleError('BigInt statusDetails error'), + { + statusDetails: [{retryDelayNanos: 5000000000n}], + }, + ); + const {stacktrace} = resolveServerExceptionDetails(error); + assert.strictEqual( + stacktrace, + 'status_details: [{"retryDelayNanos":"5000000000"}]', + ); + }); + + it('safely formats metadata containing circular references', () => { + const meta: Record = {'x-request-id': 'req-123'}; + meta.self = meta; + const error = Object.assign(new GoogleError('Circular metadata error'), { + metadata: meta, + }); + const {stacktrace} = resolveServerExceptionDetails(error); + assert.strictEqual( + stacktrace, + 'metadata: {"x-request-id":"req-123","self":{"x-request-id":"req-123","self":"[Circular]"}}', + ); + }); + + it('safely formats metadata containing BigInt values', () => { + const error = Object.assign(new GoogleError('BigInt metadata error'), { + metadata: {'quota-consumed': 1000000000000n}, + }); + const {stacktrace} = resolveServerExceptionDetails(error); + assert.strictEqual( + stacktrace, + 'metadata: {"quota-consumed":"1000000000000"}', + ); + }); + + it('safely handles non-serializable throwing properties in metadata without throwing', () => { + const badMeta = { + get bad() { + throw new Error('boom'); + }, + }; + const error = Object.assign(new GoogleError('Throwing metadata error'), { + metadata: badMeta, + }); + const {stacktrace} = resolveServerExceptionDetails(error); + assert.strictEqual(stacktrace, 'metadata: [object Object]'); + }); + }); + + describe('cause traversal circular reference protection in helpers', () => { + it('resolveErrorInfoReason terminates cleanly with circular cause', () => { + const err: Record = {message: 'test'}; + err.cause = err; + assert.strictEqual(resolveErrorInfoReason(err), undefined); + }); + + it('resolveClientNetworkOrOperationalError terminates cleanly with circular cause', () => { + const err: Record = {message: 'test'}; + err.cause = err; + assert.strictEqual( + resolveClientNetworkOrOperationalError(err), + undefined, + ); + }); + + it('resolveLanguageSpecificErrorType terminates cleanly with circular cause', () => { + const err = new GoogleError('test'); + err.cause = err; + assert.strictEqual(resolveLanguageSpecificErrorType(err), undefined); + }); + + it('isServerSideError terminates cleanly with circular cause', () => { + const err = new GoogleError('test'); + err.cause = err; + assert.strictEqual(isServerSideError(err, 'grpc'), false); + assert.strictEqual(isServerSideError(err, 'http'), false); + }); + }); +}); diff --git a/core/packages/gax/test/unit/otelHarness.ts b/core/packages/gax/test/unit/otelHarness.ts index 3eafbf358fd7..b78e2ed9c6cf 100644 --- a/core/packages/gax/test/unit/otelHarness.ts +++ b/core/packages/gax/test/unit/otelHarness.ts @@ -206,11 +206,9 @@ export class OtelHarness { * asserts that no HTTP status was reported, which is the expected result for * a failure that never received a response, such as an expired deadline. * - * @param {object} expected - Expected status and optional server endpoint values. + * @param {object} expected - Expected status values. * @param {string} [expected.rpcStatus] - gRPC status name, e.g. 'OK' or 'NOT_FOUND'. Undefined if no response arrived. * @param {number} [expected.httpStatus] - HTTP status expected on a fallback span. - * @param {string} [expected.serverAddress] - Optional expected server.address (delegates to assertServerAddressAndPort). - * @param {number} [expected.serverPort] - Optional expected server.port (delegates to assertServerAddressAndPort). * @param {object} [options] - Span selection. * @param {string} [options.tracerName] - Restrict the lookup to one instrumentation scope. * @param {ReadableSpan} [options.span] - Span to check; defaults to the only exported span. @@ -219,8 +217,6 @@ export class OtelHarness { expected: { rpcStatus?: string; httpStatus?: number; - serverAddress?: string; - serverPort?: number; }, options: {tracerName?: string; span?: ReadableSpan} = {}, ): void { @@ -229,13 +225,6 @@ export class OtelHarness { const transport = target.attributes['rpc.system.name']; const where = `span '${target.name}'`; - if ('serverAddress' in expected || 'serverPort' in expected) { - this.assertServerAddressAndPort( - {address: expected.serverAddress, port: expected.serverPort}, - options, - ); - } - assert.ok( transport === 'grpc' || transport === 'http', `${where} has rpc.system.name ${JSON.stringify(transport)}; the ` + diff --git a/core/packages/gax/test/unit/tracerHelper.ts b/core/packages/gax/test/unit/tracerHelper.ts index ae1734c5ea1f..7caaf848bb7f 100644 --- a/core/packages/gax/test/unit/tracerHelper.ts +++ b/core/packages/gax/test/unit/tracerHelper.ts @@ -15,12 +15,10 @@ */ import * as assert from 'assert'; -import * as vm from 'vm'; import {EventEmitter} from 'events'; import {Duplex, Writable} from 'stream'; import {SpanKind, SpanStatusCode, trace} from '@opentelemetry/api'; import {describe, it, beforeEach, afterEach} from 'mocha'; -import * as grpc from '@grpc/grpc-js'; import { getGaxTracer, traceCall, @@ -28,19 +26,12 @@ import { setAttemptHttpMethod, handlePromise, handleStream, +} from '../../src/observability/TracerHelper'; +import { AttemptTraceContext, DynamicTraceContext, StaticTraceContext, - resolveErrorInfoReason, - resolveServerErrorCode, - isServerSideError, - resolveClientNetworkOrOperationalError, - resolveLanguageSpecificErrorType, - resolveErrorType, - resolveServerExceptionDetails, - isPreConnectionFailure, - safeJsonStringify, -} from '../../src/observability/TracerHelper'; +} from '../../src/observability/contextUtil'; import { ResultTuple, APICallback, @@ -50,7 +41,6 @@ import { } from '../../src/apitypes'; import {GoogleError} from '../../src/googleError'; import {Status} from '../../src/status'; -import {DEPTH_TO_CHECK} from '../../src/util'; import {OngoingCallPromise} from '../../src/call'; import {OtelHarness} from './otelHarness'; @@ -298,922 +288,13 @@ describe('TracerHelper', () => { 'CustomRpcError', ); assert.strictEqual( - span.events[0].attributes?.['exception.message'], - undefined, - ); - assert.strictEqual( - span.events[0].attributes?.['exception.stacktrace'], - undefined, - ); - }); - - // In production, grpc-js builds failures as a plain Error, so the class name - // carries no information; the status code is the only stable identifier. - it('derives error.type from the gRPC status code on a grpc-js error', async () => { - // Shape produced by grpc-js callErrorFromStatus: - // Object.assign(new Error(message), status). - const error = Object.assign( - new Error('5 NOT_FOUND: object does not exist'), - {code: 5}, - ); - - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticArgs, async () => { - throw error; - }); - }); - - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual(error.constructor.name, 'Error'); - assert.strictEqual(span.attributes['error.type'], 'NOT_FOUND'); - assert.strictEqual(span.attributes['exception.type'], undefined); - }); - - // error.type reports the identifier belonging to the transport that - // failed: the gRPC status name on a gRPC call, the received HTTP status on - // a fallback one. The same logical failure therefore reads differently on - // the two, which is deliberate — each names the status its own protocol - // actually returned. - describe('error.type by transport', () => { - const httpDynamicArgs: DynamicTraceContext = { - clientName: 'ComputeClient', - methodName: 'InsertInstance', - rpcType: 'http', - }; - - const errorTypeOf = async ( - thrown: unknown, - args: DynamicTraceContext = dynamicArgs, - ) => { - await assert.rejects(async () => { - await traceCall(args, staticArgs, async () => { - throw thrown; - }); - }); - return harness.requireSingleSpan('google-gax').attributes['error.type']; - }; - - it('reports the gRPC status name on a grpc call', async () => { - const error = new GoogleError('object does not exist'); - error.code = Status.NOT_FOUND; - - assert.strictEqual(await errorTypeOf(error), 'NOT_FOUND'); - }); - - it('reports the received status as a string on a fallback call', async () => { - const error = new GoogleError('service unavailable'); - error.code = Status.UNAVAILABLE; - error.httpStatusCode = 503; - - assert.strictEqual(await errorTypeOf(error, httpDynamicArgs), '503'); - }); - - it('reports the received status, not the status it was mapped to', async () => { - // 418 is unmapped, so rpcCodeFromHttpStatusCode collapses it onto - // FAILED_PRECONDITION. Reporting the mapped code would lose the only - // record of what the server actually answered. - const error = new GoogleError('teapot'); - error.code = Status.FAILED_PRECONDITION; - error.httpStatusCode = 418; - - assert.strictEqual(await errorTypeOf(error, httpDynamicArgs), '418'); - }); - - it('never reports an HTTP status on a grpc call', async () => { - // A gRPC call has no HTTP status. Even if one is somehow attached, the - // gRPC status is the identifier that belongs to this transport. - const error = new GoogleError('object does not exist'); - error.code = Status.NOT_FOUND; - error.httpStatusCode = 404; - - assert.strictEqual(await errorTypeOf(error), 'NOT_FOUND'); - }); - - it('falls through to Tier 3 client error when no HTTP response arrived', async () => { - // An expired deadline on the fallback transport: a real gRPC status, - // but no response and so no HTTP status. The gRPC code is not this - // transport's identifier, so CLIENT_TIMEOUT is reported per Tier 3. - const error = new GoogleError('Deadline exceeded'); - error.code = Status.DEADLINE_EXCEEDED; - - assert.strictEqual( - await errorTypeOf(error, httpDynamicArgs), - 'CLIENT_TIMEOUT', - ); - harness.assertResponseStatus({}); - }); - - it('prefers a system error code to the class on either transport', async () => { - // A refused connection never reaches a server, so neither transport - // has a status to report. 'CLIENT_CONNECTION_ERROR' is reported per Tier 3. - const error = Object.assign(new Error('connect ECONNREFUSED'), { - code: 'ECONNREFUSED', - }); - - assert.strictEqual( - await errorTypeOf(error, httpDynamicArgs), - 'CLIENT_CONNECTION_ERROR', - ); - harness.reset(); - assert.strictEqual(await errorTypeOf(error), 'CLIENT_CONNECTION_ERROR'); - }); - - it('preserves a system error code wrapped on cause by fallback _toGoogleError', async () => { - // On the REST fallback path, _toGoogleError wraps the fetch error in - // a GoogleError with a numeric e.code and puts the raw error on - // e.cause. - const fetchError = Object.assign(new Error('connect ECONNREFUSED'), { - code: 'ECONNREFUSED', - }); - const error = new GoogleError(fetchError.message); - error.code = Status.UNAVAILABLE; - error.cause = fetchError; - - assert.strictEqual( - await errorTypeOf(error, httpDynamicArgs), - 'CLIENT_CONNECTION_ERROR', - ); - harness.assertResponseStatus({}); - }); - - it('checks e.cause when the outer error is a GoogleError', async () => { - const fetchError = new TypeError('Failed to fetch'); - const error = new GoogleError(fetchError.message); - error.cause = fetchError; - - assert.strictEqual( - await errorTypeOf(error, httpDynamicArgs), - 'TypeError', - ); - }); - - it('resolves client-side error.type from cause when outer GoogleError has a gRPC status code', async () => { - const abortError = new DOMException('Operation aborted', 'AbortError'); - const timeoutError = new DOMException('Timed out', 'TimeoutError'); - const nodeCodeError = Object.assign( - new TypeError('Invalid argument type'), - {code: 'ERR_INVALID_ARG_TYPE'}, - ); - - const cancelledError = new GoogleError(abortError.message); - cancelledError.code = Status.CANCELLED; - cancelledError.cause = abortError; - - const deadlineError = new GoogleError(timeoutError.message); - deadlineError.code = Status.DEADLINE_EXCEEDED; - deadlineError.cause = timeoutError; - - const invalidArgError = new GoogleError(nodeCodeError.message); - invalidArgError.code = Status.INVALID_ARGUMENT; - invalidArgError.cause = nodeCodeError; - - assert.strictEqual(await errorTypeOf(cancelledError), 'AbortError'); - harness.reset(); - assert.strictEqual(await errorTypeOf(deadlineError), 'CLIENT_TIMEOUT'); - harness.reset(); - assert.strictEqual( - await errorTypeOf(invalidArgError), - 'CLIENT_REQUEST_ERROR', - ); - }); - - it('unwraps nested causes when intermediate errors are GoogleErrors', async () => { - const rootError = new TypeError('root failure'); - const intermediateError = new GoogleError('intermediate'); - intermediateError.cause = rootError; - const outerError = new GoogleError('outer'); - outerError.cause = intermediateError; - - assert.strictEqual( - await errorTypeOf(outerError, httpDynamicArgs), - 'TypeError', - ); - }); - - it('prefers protocol status on the outer GoogleError over e.cause exception type', async () => { - const innerError = new TypeError('inner'); - const error = new GoogleError('service unavailable'); - error.code = Status.UNAVAILABLE; - error.httpStatusCode = 503; - error.cause = innerError; - - assert.strictEqual(await errorTypeOf(error, httpDynamicArgs), '503'); - }); - }); - - describe('error.type 5-tier resolution hierarchy', () => { - const httpDynamicArgs: DynamicTraceContext = { - clientName: 'ComputeClient', - methodName: 'InsertInstance', - rpcType: 'http', - }; - - const errorTypeOf = async ( - thrown: unknown, - args: DynamicTraceContext = dynamicArgs, - ) => { - await assert.rejects(async () => { - await traceCall(args, staticArgs, async () => { - throw thrown; - }); - }); - return harness.requireSingleSpan('google-gax').attributes['error.type']; - }; - - describe('Tier 1: google.rpc.ErrorInfo.reason', () => { - it('uses ErrorInfo.reason when available on gRPC, taking precedence over status code', async () => { - const error = Object.assign(new GoogleError('rate limit exceeded'), { - code: Status.RESOURCE_EXHAUSTED, - reason: 'RATE_LIMIT_EXCEEDED', - }); - assert.strictEqual( - await errorTypeOf(error, dynamicArgs), - 'RATE_LIMIT_EXCEEDED', - ); - }); - - it('uses ErrorInfo.reason when available on HTTP, taking precedence over status code', async () => { - const error = Object.assign(new GoogleError('service disabled'), { - httpStatusCode: 403, - reason: 'SERVICE_DISABLED', - }); - assert.strictEqual( - await errorTypeOf(error, httpDynamicArgs), - 'SERVICE_DISABLED', - ); - }); - - it('extracts reason from statusDetails array', async () => { - const error = Object.assign(new GoogleError('detailed failure'), { - code: Status.FAILED_PRECONDITION, - statusDetails: [{reason: 'RESOURCE_PROJECT_INVALID'}], - }); - assert.strictEqual( - await errorTypeOf(error, dynamicArgs), - 'RESOURCE_PROJECT_INVALID', - ); - }); - - it('extracts reason from cause chain', async () => { - const innerError = Object.assign(new Error('inner'), { - reason: 'ACCESS_TOKEN_EXPIRED', - }); - const error = new GoogleError('outer'); - error.cause = innerError; - assert.strictEqual( - await errorTypeOf(error, dynamicArgs), - 'ACCESS_TOKEN_EXPIRED', - ); - }); - }); - - describe('Tier 2: Specific Server Error Code', () => { - it('uses HTTP status code string for HTTP calls', async () => { - const error400 = Object.assign(new GoogleError('Bad Request'), { - httpStatusCode: 400, - }); - const error503 = Object.assign( - new GoogleError('Service Unavailable'), - { - httpStatusCode: 503, - }, - ); - assert.strictEqual( - await errorTypeOf(error400, httpDynamicArgs), - '400', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(error503, httpDynamicArgs), - '503', - ); - }); - - it('uses uppercase gRPC canonical status name for gRPC calls', async () => { - const permDenied = Object.assign(new Error('Permission Denied'), { - code: Status.PERMISSION_DENIED, - }); - const unavailable = Object.assign(new Error('Unavailable'), { - code: Status.UNAVAILABLE, - }); - assert.strictEqual( - await errorTypeOf(permDenied, dynamicArgs), - 'PERMISSION_DENIED', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(unavailable, dynamicArgs), - 'UNAVAILABLE', - ); - }); - }); - - describe('Tier 3: Client-Side Network/Operational Errors', () => { - it('resolves CLIENT_TIMEOUT for timeout conditions', async () => { - const domTimeout = new DOMException('timed out', 'TimeoutError'); - const etimedout = Object.assign(new Error('connection timed out'), { - code: 'ETIMEDOUT', - }); - const esocket = Object.assign(new Error('socket timed out'), { - code: 'ESOCKETTIMEDOUT', - }); - const gaxTimeout = new GoogleError( - 'Total timeout of API exceeded before completion', - ); - - assert.strictEqual( - await errorTypeOf(domTimeout, dynamicArgs), - 'CLIENT_TIMEOUT', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(etimedout, dynamicArgs), - 'CLIENT_TIMEOUT', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(esocket, dynamicArgs), - 'CLIENT_TIMEOUT', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(gaxTimeout, dynamicArgs), - 'CLIENT_TIMEOUT', - ); - }); - - it('resolves CLIENT_CONNECTION_ERROR for DNS, socket, connection, and TLS errors', async () => { - const codes = [ - 'ENOTFOUND', - 'EAI_AGAIN', - 'ECONNREFUSED', - 'ECONNRESET', - 'EHOSTUNREACH', - 'ENETUNREACH', - 'CERT_HAS_EXPIRED', - 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', - 'ERR_TLS_CERT_ALTNAME_INVALID', - 'ERR_SSL_PROTOCOL_ERROR', - ]; - - for (const code of codes) { - const err = Object.assign(new Error(`error with ${code}`), {code}); - assert.strictEqual( - await errorTypeOf(err, dynamicArgs), - 'CLIENT_CONNECTION_ERROR', - `Expected ${code} to resolve to CLIENT_CONNECTION_ERROR`, - ); - harness.reset(); - } - }); - - it('resolves CLIENT_REQUEST_ERROR for client parameter and request formatting errors', async () => { - const argTypeErr = Object.assign(new TypeError('invalid type'), { - code: 'ERR_INVALID_ARG_TYPE', - }); - const invalidUrlErr = Object.assign(new Error('invalid url'), { - code: 'ERR_INVALID_URL', - }); - const uriErr = new URIError('malformed URI sequence'); - - assert.strictEqual( - await errorTypeOf(argTypeErr, dynamicArgs), - 'CLIENT_REQUEST_ERROR', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(invalidUrlErr, dynamicArgs), - 'CLIENT_REQUEST_ERROR', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(uriErr, dynamicArgs), - 'CLIENT_REQUEST_ERROR', - ); - }); - - it('resolves CLIENT_REQUEST_BODY_ERROR for stream body errors', async () => { - const streamErr = Object.assign(new Error('write after end'), { - code: 'ERR_STREAM_WRITE_AFTER_END', - }); - assert.strictEqual( - await errorTypeOf(streamErr, dynamicArgs), - 'CLIENT_REQUEST_BODY_ERROR', - ); - }); - - it('resolves CLIENT_RESPONSE_DECODE_ERROR for response decoding errors', async () => { - class DecodeError extends Error {} - const decodeErr = new DecodeError('failed to decode response'); - const syntaxErr = new SyntaxError('Unexpected token < in JSON'); - - assert.strictEqual( - await errorTypeOf(decodeErr, dynamicArgs), - 'CLIENT_RESPONSE_DECODE_ERROR', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(syntaxErr, dynamicArgs), - 'CLIENT_RESPONSE_DECODE_ERROR', - ); - }); - - it('resolves CLIENT_REDIRECT_ERROR for redirect errors', async () => { - const redirectErr = Object.assign(new Error('too many redirects'), { - code: 'ERR_TOO_MANY_REDIRECTS', - }); - assert.strictEqual( - await errorTypeOf(redirectErr, dynamicArgs), - 'CLIENT_REDIRECT_ERROR', - ); - }); - - it('resolves CLIENT_AUTHENTICATION_ERROR for authentication errors', async () => { - class GoogleAuthError extends Error {} - const authErr = new GoogleAuthError('Missing credentials'); - const codeAuthErr = Object.assign(new Error('No credentials'), { - code: 'ERR_NO_CREDENTIALS', - }); - - assert.strictEqual( - await errorTypeOf(authErr, dynamicArgs), - 'CLIENT_AUTHENTICATION_ERROR', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(codeAuthErr, dynamicArgs), - 'CLIENT_AUTHENTICATION_ERROR', - ); - }); - }); - - describe('Tier 4: Language-specific error type', () => { - it('resolves AbortError, TypeError, RangeError, and custom error classes', async () => { - class CustomRpcError extends Error {} - const abortErr = new DOMException('Operation aborted', 'AbortError'); - const typeErr = new TypeError('parameter error'); - const rangeErr = new RangeError('value out of range'); - const customErr = new CustomRpcError('custom error'); - - assert.strictEqual( - await errorTypeOf(abortErr, dynamicArgs), - 'AbortError', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(typeErr, dynamicArgs), - 'TypeError', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(rangeErr, dynamicArgs), - 'RangeError', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(customErr, dynamicArgs), - 'CustomRpcError', - ); - }); - }); - - describe('Tier 5: Internal Fallback', () => { - it('falls back to INTERNAL for bare Error, bare GoogleError, and non-Errors', async () => { - const bareError = new Error('internal logic error'); - const bareGoogleError = new GoogleError('unexpected error'); - - assert.strictEqual( - await errorTypeOf(bareError, dynamicArgs), - 'INTERNAL', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf(bareGoogleError, dynamicArgs), - 'INTERNAL', - ); - harness.reset(); - assert.strictEqual( - await errorTypeOf('unexpected string', dynamicArgs), - 'INTERNAL', - ); - }); - }); - - describe('helper functions direct unit tests', () => { - it('resolveErrorInfoReason handles direct reason, statusDetails, and cause', () => { - assert.strictEqual( - resolveErrorInfoReason({reason: 'RATE_LIMIT_EXCEEDED'}), - 'RATE_LIMIT_EXCEEDED', - ); - assert.strictEqual( - resolveErrorInfoReason({ - statusDetails: [{reason: 'SERVICE_DISABLED'}], - }), - 'SERVICE_DISABLED', - ); - assert.strictEqual( - resolveErrorInfoReason({ - cause: {reason: 'ACCESS_TOKEN_EXPIRED'}, - }), - 'ACCESS_TOKEN_EXPIRED', - ); - assert.strictEqual( - resolveErrorInfoReason(new Error('none')), - undefined, - ); - }); - - it('resolveServerErrorCode handles http and grpc', () => { - const httpErr = Object.assign(new Error('Not Found'), { - httpStatusCode: 404, - }); - const grpcErr = Object.assign(new Error('Not Found'), { - code: Status.NOT_FOUND, - }); - assert.strictEqual(resolveServerErrorCode(httpErr, 'http'), '404'); - assert.strictEqual( - resolveServerErrorCode(grpcErr, 'grpc'), - 'NOT_FOUND', - ); - assert.strictEqual( - resolveServerErrorCode(grpcErr, 'http'), - undefined, - ); - assert.strictEqual( - resolveServerErrorCode(httpErr, 'grpc'), - undefined, - ); - }); - - it('resolveServerErrorCode handles plain error objects without stack property', () => { - assert.strictEqual( - resolveServerErrorCode({httpStatusCode: 500}, 'http'), - '500', - ); - assert.strictEqual( - resolveServerErrorCode({code: Status.NOT_FOUND}, 'grpc'), - 'NOT_FOUND', - ); - assert.strictEqual( - resolveServerErrorCode({code: 5}, 'grpc'), - 'NOT_FOUND', - ); - assert.strictEqual( - resolveServerErrorCode({httpStatusCode: 500}, 'grpc'), - undefined, - ); - assert.strictEqual( - resolveServerErrorCode({code: 5}, 'http'), - undefined, - ); - }); - - it('isServerSideError recognizes plain error objects with valid status codes', () => { - assert.strictEqual( - isServerSideError({httpStatusCode: 500}, 'http'), - true, - ); - assert.strictEqual( - isServerSideError({code: Status.NOT_FOUND}, 'grpc'), - true, - ); - assert.strictEqual(isServerSideError({code: 5}, 'grpc'), true); - assert.strictEqual( - isServerSideError({httpStatusCode: 500}, 'grpc'), - false, - ); - assert.strictEqual(isServerSideError({code: 5}, 'http'), false); - assert.strictEqual( - isServerSideError({message: 'no code'}, 'grpc'), - false, - ); - assert.strictEqual( - isServerSideError({message: 'no code'}, 'http'), - false, - ); - }); - - it('resolveClientNetworkOrOperationalError maps codes and classes', () => { - assert.strictEqual( - resolveClientNetworkOrOperationalError({code: 'ECONNREFUSED'}), - 'CLIENT_CONNECTION_ERROR', - ); - assert.strictEqual( - resolveClientNetworkOrOperationalError({name: 'TimeoutError'}), - 'CLIENT_TIMEOUT', - ); - assert.strictEqual( - resolveClientNetworkOrOperationalError({ - code: 'ERR_INVALID_ARG_TYPE', - }), - 'CLIENT_REQUEST_ERROR', - ); - assert.strictEqual( - resolveClientNetworkOrOperationalError(new Error('normal error')), - undefined, - ); - }); - - it('resolveLanguageSpecificErrorType excludes generic classes and unwraps cause', () => { - assert.strictEqual( - resolveLanguageSpecificErrorType(new RangeError('out of bounds')), - 'RangeError', - ); - assert.strictEqual( - resolveLanguageSpecificErrorType(new Error('generic')), - undefined, - ); - const outer = new GoogleError('outer'); - outer.cause = new TypeError('inner'); - assert.strictEqual( - resolveLanguageSpecificErrorType(outer), - 'TypeError', - ); - }); - - it('resolveErrorType implements the 5 tiers', () => { - // Tier 1 beats Tier 2 - const t1 = Object.assign(new GoogleError('foo'), { - reason: 'MY_REASON', - code: Status.INVALID_ARGUMENT, - }); - assert.strictEqual(resolveErrorType(t1, 'grpc'), 'MY_REASON'); - - // Tier 2 beats Tier 3 - const t2 = Object.assign(new GoogleError('foo'), { - code: Status.PERMISSION_DENIED, - }); - assert.strictEqual(resolveErrorType(t2, 'grpc'), 'PERMISSION_DENIED'); - - // Tier 3 beats Tier 4 - const t3 = Object.assign(new TypeError('invalid type'), { - code: 'ERR_INVALID_ARG_TYPE', - }); - assert.strictEqual( - resolveErrorType(t3, 'grpc'), - 'CLIENT_REQUEST_ERROR', - ); - - // Tier 4 beats Tier 5 - const t4 = new TypeError('normal type error'); - assert.strictEqual(resolveErrorType(t4, 'grpc'), 'TypeError'); - - // Tier 5 fallback - assert.strictEqual( - resolveErrorType(new Error('generic'), 'grpc'), - 'INTERNAL', - ); - assert.strictEqual(resolveErrorType('string', 'grpc'), 'INTERNAL'); - }); - - describe('safeJsonStringify', () => { - it('returns undefined for undefined and serializes primitives', () => { - assert.strictEqual(safeJsonStringify(undefined), undefined); - assert.strictEqual(safeJsonStringify(null), 'null'); - assert.strictEqual(safeJsonStringify(123), '123'); - assert.strictEqual(safeJsonStringify('abc'), '"abc"'); - assert.strictEqual(safeJsonStringify(true), 'true'); - }); - - it('serializes BigInt values as strings', () => { - const obj = {big: 9007199254740993n, list: [1n, 2n]}; - assert.strictEqual( - safeJsonStringify(obj), - '{"big":"9007199254740993","list":["1","2"]}', - ); - }); - - it('handles circular references in objects', () => { - const circular: Record = {name: 'test'}; - circular.self = circular; - assert.strictEqual( - safeJsonStringify(circular), - '{"name":"test","self":"[Circular]"}', - ); - }); - - it('handles circular references in arrays', () => { - const arr: unknown[] = [1, 2]; - arr.push(arr); - assert.strictEqual(safeJsonStringify(arr), '[1,2,"[Circular]"]'); - }); - - it('handles deep circular references', () => { - const root: Record = { - a: { - b: { - c: {}, - }, - }, - }; - (root.a as Record).b = root; - assert.strictEqual( - safeJsonStringify(root), - '{"a":{"b":"[Circular]"}}', - ); - }); - - it('does not falsely mark shared references in DAGs as circular', () => { - const shared = {sharedKey: 'sharedValue'}; - const dag = {first: shared, second: shared}; - assert.strictEqual( - safeJsonStringify(dag), - '{"first":{"sharedKey":"sharedValue"},"second":{"sharedKey":"sharedValue"}}', - ); - }); - - it('safely handles non-serializable objects and throwing getters without throwing', () => { - const badObj = { - get throwingProp() { - throw new Error('getter threw'); - }, - }; - const result = safeJsonStringify(badObj); - assert.strictEqual(result, '[object Object]'); - }); - - it('safely handles objects with throwing toString without throwing', () => { - const badObj = { - get throwingProp() { - throw new Error('getter threw'); - }, - toString() { - throw new Error('toString threw'); - }, - }; - assert.strictEqual(safeJsonStringify(badObj), undefined); - }); - }); - - describe('isPreConnectionFailure', () => { - it('handles circular references in cause without infinite recursion', () => { - const circularError = new GoogleError('circular error'); - circularError.cause = circularError; - assert.strictEqual(isPreConnectionFailure(circularError), false); - - const errA = new GoogleError('errA'); - const errB = new GoogleError('errB'); - errA.cause = errB; - errB.cause = errA; - assert.strictEqual(isPreConnectionFailure(errA), false); - }); - - it('handles deep cause chains (> DEPTH_TO_CHECK depth) without stack overflow', () => { - const root = new GoogleError('root'); - let current = root; - for (let i = 0; i < 20; i++) { - const next = new GoogleError(`level-${i}`); - current.cause = next; - current = next; - } - assert.strictEqual(isPreConnectionFailure(root), false); - }); - - it('respects DEPTH_TO_CHECK limit when unwrapping cause chain', () => { - // Error within DEPTH_TO_CHECK levels is detected - const rootWithin = new GoogleError('root'); - let currWithin = rootWithin; - for (let i = 0; i < DEPTH_TO_CHECK - 2; i++) { - const next = new GoogleError(`level-${i}`); - currWithin.cause = next; - currWithin = next; - } - currWithin.cause = new TypeError('client validation failure'); - assert.strictEqual(isPreConnectionFailure(rootWithin), true); - - // Error deeper than DEPTH_TO_CHECK levels is not reached - const rootDeeper = new GoogleError('root'); - let currDeeper = rootDeeper; - for (let i = 0; i < DEPTH_TO_CHECK + 5; i++) { - const next = new GoogleError(`level-${i}`); - currDeeper.cause = next; - currDeeper = next; - } - currDeeper.cause = new TypeError('too deep to find'); - assert.strictEqual(isPreConnectionFailure(rootDeeper), false); - }); - - it('identifies pre-connection errors directly and wrapped in GoogleError', () => { - assert.strictEqual( - isPreConnectionFailure(new TypeError('type err')), - true, - ); - assert.strictEqual( - isPreConnectionFailure(new RangeError('range err')), - true, - ); - assert.strictEqual( - isPreConnectionFailure(new URIError('uri err')), - true, - ); - - const wrappedType = new GoogleError('wrapped'); - wrappedType.cause = new TypeError('inner'); - assert.strictEqual(isPreConnectionFailure(wrappedType), true); - - const netErr = Object.assign(new Error('connect ECONNREFUSED'), { - code: 'ECONNREFUSED', - }); - const wrappedNet = new GoogleError('wrapped connection'); - wrappedNet.cause = netErr; - assert.strictEqual(isPreConnectionFailure(wrappedNet), true); - }); - - it('returns false for server-side errors', () => { - const grpcErr = Object.assign(new GoogleError('grpc err'), { - code: Status.UNAVAILABLE, - }); - assert.strictEqual(isPreConnectionFailure(grpcErr), false); - - const httpErr = Object.assign(new GoogleError('http err'), { - httpStatusCode: 503, - }); - assert.strictEqual(isPreConnectionFailure(httpErr), false); - }); - - it('returns true for non-Errors and primitives', () => { - assert.strictEqual(isPreConnectionFailure(null), true); - assert.strictEqual(isPreConnectionFailure('string throw'), true); - assert.strictEqual(isPreConnectionFailure(123), true); - }); - }); - - describe('cause traversal circular reference protection in helpers', () => { - it('resolveErrorInfoReason terminates cleanly with circular cause', () => { - const err: Record = {message: 'test'}; - err.cause = err; - assert.strictEqual(resolveErrorInfoReason(err), undefined); - }); - - it('resolveClientNetworkOrOperationalError terminates cleanly with circular cause', () => { - const err: Record = {message: 'test'}; - err.cause = err; - assert.strictEqual( - resolveClientNetworkOrOperationalError(err), - undefined, - ); - }); - - it('resolveLanguageSpecificErrorType terminates cleanly with circular cause', () => { - const err = new GoogleError('test'); - err.cause = err; - assert.strictEqual( - resolveLanguageSpecificErrorType(err), - undefined, - ); - }); - - it('isServerSideError terminates cleanly with circular cause', () => { - const err = new GoogleError('test'); - err.cause = err; - assert.strictEqual(isServerSideError(err, 'grpc'), false); - assert.strictEqual(isServerSideError(err, 'http'), false); - }); - }); - }); - }); - - // Fallback branches. Each of these must not produce a bogus error.type. - it('treats a zero status code as absent rather than as OK and falls back to INTERNAL', async () => { - // Zero is the proto3 default for an unset code, which is why - // GoogleError.parseHttpError deletes the field. Reporting 'OK' as the - // type of a failed call would be actively wrong. - const error = Object.assign(new Error('unset code'), {code: 0}); - - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticArgs, async () => { - throw error; - }); - }); - - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual(span.attributes['error.type'], 'INTERNAL'); - }); - - it('falls back to INTERNAL for a code outside the Status range', async () => { - const error = Object.assign(new Error('bogus code'), {code: 4242}); - - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticArgs, async () => { - throw error; - }); - }); - - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual(span.attributes['error.type'], 'INTERNAL'); - }); - - it('falls back to INTERNAL for an empty string code', async () => { - const error = Object.assign(new Error('empty code'), {code: ''}); - - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticArgs, async () => { - throw error; - }); - }); - - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual(span.attributes['error.type'], 'INTERNAL'); + span.events[0].attributes?.['exception.message'], + undefined, + ); + assert.strictEqual( + span.events[0].attributes?.['exception.stacktrace'], + undefined, + ); }); it('reports the INTERNAL error.type when a non-Error is thrown', async () => { @@ -1311,238 +392,18 @@ describe('TracerHelper', () => { ); }); - // exception.type names the class that was raised. semconv asks for the - // fully-qualified class name "if applicable", and it is not applicable - // in JavaScript: there is no namespace to qualify with, so the - // constructor name is the whole identity. It also asks for the dynamic - // type over the static one, which is why the constructor is preferred - // over `name` — a mutable property, not a type. - describe('exception.type', () => { - const exceptionTypeOf = async (thrown: unknown) => { - const span = await failWith(thrown); - return span.events[0]?.attributes?.['exception.type']; - }; - - it('names the class rather than the status code it carries', async () => { - // The regression this guards: the SDK reads `code` before `name`, so - // handing it a status-bearing error reports the stringified number. - const error = new GoogleError('object does not exist'); - error.code = Status.NOT_FOUND; - - assert.strictEqual(await exceptionTypeOf(error), 'GoogleError'); - }); - - it('names the class of a subclass that never assigns name', async () => { - // The shape every gax error has: `GoogleError` and its subclasses - // inherit the literal 'Error' from Error.prototype, so `name` would - // report 'Error' for all of them and collapse the dimension. - class RetryError extends GoogleError {} - const error = new RetryError('retries exhausted'); - - assert.strictEqual(error.name, 'Error'); - assert.strictEqual(await exceptionTypeOf(error), 'RetryError'); - }); - - it('falls back to name when the class is the bare Error', async () => { - // grpc-js builds failures as Object.assign(new Error(msg), status), - // so the class is 'Error' and carries nothing. A name the caller - // assigned is the only description of the failure left. - const error = new Error('RPC Failed'); - error.name = 'CustomRpcError'; - - assert.strictEqual(await exceptionTypeOf(error), 'CustomRpcError'); - }); - - it('prefers the class over a name assigned on top of it', async () => { - // `name` is a mutable data property and may say anything; the class - // is the dynamic type semconv asks to be preferred. - const error = new GoogleError('object does not exist'); - error.name = 'CustomRpcError'; - - assert.strictEqual(await exceptionTypeOf(error), 'GoogleError'); - }); - - it('reports a built-in error class as itself', async () => { - assert.strictEqual( - await exceptionTypeOf(new TypeError('not a function')), - 'TypeError', - ); - }); - - it('reports Error for a bare Error with nothing else to go on', async () => { - assert.strictEqual(await exceptionTypeOf(new Error('boom')), 'Error'); - }); - - it('formats status details and metadata in resolveServerExceptionDetails for gRPC server errors', () => { - const metadata = new grpc.Metadata(); - metadata.set('x-goog-request-id', 'req-123'); - const error = Object.assign( - new GoogleError('Detailed server failure'), - { - code: Status.INVALID_ARGUMENT, - details: 'Field name is invalid', - statusDetails: [ - {field: 'name', description: 'must be non-empty'}, - ], - metadata, - reason: 'INVALID_FIELD', - domain: 'googleapis.com', - errorInfoMetadata: {consumer: 'projects/123'}, - }, - ); - - const details = resolveServerExceptionDetails(error); - assert.strictEqual(details.message, 'Field name is invalid'); - assert.strictEqual( - details.stacktrace, - 'status_details: [{"field":"name","description":"must be non-empty"}]\n' + - 'metadata: {"x-goog-request-id":"req-123"}', - ); - }); - - it('formats status details and metadata in resolveServerExceptionDetails for HTTP fallback server errors', () => { - const error = Object.assign(new GoogleError('HTTP server error'), { - httpStatusCode: 400, - details: 'Bad HTTP Request', - statusDetails: 'Quota exceeded', - metadata: {'content-type': 'application/json'}, - }); - - const details = resolveServerExceptionDetails(error); - assert.strictEqual(details.message, 'Bad HTTP Request'); - assert.strictEqual( - details.stacktrace, - 'status_details: Quota exceeded\n' + - 'metadata: {"content-type":"application/json"}', - ); - }); + it('survives traceCall when thrown error has circular cause', async () => { + const circularError = new GoogleError('wrapper error'); + circularError.cause = circularError; - it('encodes Buffer and Array of Buffers in metadata as base64', () => { - const error = Object.assign(new GoogleError('Buffer error'), { - metadata: { - 'bin-key': Buffer.from('hello'), - 'bin-array': [Buffer.from('foo'), 'bar', Buffer.from('baz')], - 'regular-key': 'value', - }, + await assert.rejects(async () => { + await traceCall(dynamicArgs, staticArgs, async () => { + throw circularError; }); - const {stacktrace} = resolveServerExceptionDetails(error); - assert.strictEqual( - stacktrace, - 'metadata: {"bin-key":"aGVsbG8=","bin-array":["Zm9v","bar","YmF6"],"regular-key":"value"}', - ); - }); - - it('formats metadata safely without ReferenceError when Buffer is not defined', () => { - const originalBuffer = (globalThis as Record).Buffer; - try { - delete (globalThis as Record).Buffer; - const error = Object.assign(new GoogleError('Browser error'), { - metadata: { - 'content-type': 'application/json', - headers: ['x-goog-request-id', 'req-456'], - }, - }); - const {stacktrace} = resolveServerExceptionDetails(error); - assert.strictEqual( - stacktrace, - 'metadata: {"content-type":"application/json","headers":["x-goog-request-id","req-456"]}', - ); - } finally { - (globalThis as Record).Buffer = originalBuffer; - } - }); - - it('safely formats statusDetails containing circular references', () => { - const detail: Record = { - reason: 'RATE_LIMIT_EXCEEDED', - }; - detail.self = detail; - const error = Object.assign( - new GoogleError('Circular statusDetails error'), - { - statusDetails: [detail], - }, - ); - const {stacktrace} = resolveServerExceptionDetails(error); - assert.strictEqual( - stacktrace, - 'status_details: [{"reason":"RATE_LIMIT_EXCEEDED","self":"[Circular]"}]', - ); - }); - - it('safely formats statusDetails containing BigInt values', () => { - const error = Object.assign( - new GoogleError('BigInt statusDetails error'), - { - statusDetails: [{retryDelayNanos: 5000000000n}], - }, - ); - const {stacktrace} = resolveServerExceptionDetails(error); - assert.strictEqual( - stacktrace, - 'status_details: [{"retryDelayNanos":"5000000000"}]', - ); - }); - - it('safely formats metadata containing circular references', () => { - const meta: Record = {'x-request-id': 'req-123'}; - meta.self = meta; - const error = Object.assign( - new GoogleError('Circular metadata error'), - { - metadata: meta, - }, - ); - const {stacktrace} = resolveServerExceptionDetails(error); - assert.strictEqual( - stacktrace, - 'metadata: {"x-request-id":"req-123","self":{"x-request-id":"req-123","self":"[Circular]"}}', - ); - }); - - it('safely formats metadata containing BigInt values', () => { - const error = Object.assign( - new GoogleError('BigInt metadata error'), - { - metadata: {'quota-consumed': 1000000000000n}, - }, - ); - const {stacktrace} = resolveServerExceptionDetails(error); - assert.strictEqual( - stacktrace, - 'metadata: {"quota-consumed":"1000000000000"}', - ); - }); - - it('safely handles non-serializable throwing properties in metadata without throwing', () => { - const badMeta = { - get bad() { - throw new Error('boom'); - }, - }; - const error = Object.assign( - new GoogleError('Throwing metadata error'), - { - metadata: badMeta, - }, - ); - const {stacktrace} = resolveServerExceptionDetails(error); - assert.strictEqual(stacktrace, 'metadata: [object Object]'); }); - it('survives traceCall when thrown error has circular cause', async () => { - const circularError = new GoogleError('wrapper error'); - circularError.cause = circularError; - - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticArgs, async () => { - throw circularError; - }); - }); - - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual(span.status.code, SpanStatusCode.ERROR); - }); + const span = harness.requireSingleSpan('google-gax'); + assert.strictEqual(span.status.code, SpanStatusCode.ERROR); }); it('reports error.type consistently with the RPC status', async () => { @@ -1589,56 +450,6 @@ describe('TracerHelper', () => { assert.strictEqual(span.status.message, '[object Object]'); }); - it('describes an error-shaped object from its message property', async () => { - // `code` is already read off whatever was thrown rather than off an - // Error; the description is resolved the same way, so an object that - // carries a perfectly good message is not reduced to '[object Object]'. - const span = await failWith({ - code: Status.NOT_FOUND, - message: 'object does not exist', - }); - - assert.strictEqual(span.status.message, 'object does not exist'); - assert.strictEqual(span.attributes['error.type'], 'NOT_FOUND'); - harness.assertResponseStatus({rpcStatus: 'NOT_FOUND'}, {span}); - }); - - it('describes an Error that crossed a realm boundary', async () => { - // A value from another realm fails `instanceof Error`, so it takes the - // non-Error branch despite being a real Error. Reading `message` - // directly keeps the description intact; String() would have prefixed - // it with the class name. - const crossRealm = vm.runInNewContext( - "new Error('cross realm boom')", - ) as Error; - assert.strictEqual(crossRealm instanceof Error, false); - - const span = await failWith(crossRealm); - - assert.strictEqual(span.status.message, 'cross realm boom'); - }); - - it('falls back to String() for a message that is not a string', async () => { - // A non-string `message` is not a description, and passing it through - // would hand OTel a value its status API does not accept. - const span = await failWith({message: {nested: 'object'}}); - - assert.strictEqual(span.status.message, '[object Object]'); - assert.strictEqual(span.status.code, SpanStatusCode.ERROR); - }); - - it('keeps the description empty when the error has no message', async () => { - // `new Error()` has an empty message, and it must be reported as such. - // Substituting a placeholder would invent a description that no error - // actually carried; the failure is already conveyed by the status code - // and error.type. - const span = await failWith(new Error()); - - assert.strictEqual(span.status.code, SpanStatusCode.ERROR); - assert.strictEqual(span.status.message, ''); - assert.strictEqual(span.attributes['error.type'], 'INTERNAL'); - }); - it('keeps the first description when a second, different failure arrives', async () => { // The first failure is the one that ended the call; a later error is // fallout from the teardown. Overwriting the description would replace @@ -1671,43 +482,6 @@ describe('TracerHelper', () => { }); }); - it('handles missing optional static arguments gracefully', async () => { - const emptyStaticArgs: StaticTraceContext = {}; - const result = await traceCall(dynamicArgs, emptyStaticArgs, async () => { - return 42; - }); - - assert.strictEqual(result, 42); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - - const span = spans[0]; - assert.strictEqual(span.name, 'StorageClient.GetObject'); - assert.strictEqual(span.ended, true); - assert.strictEqual(span.attributes['gcp.client.service'], undefined); - assert.strictEqual(span.attributes['gcp.client.version'], undefined); - assert.strictEqual(span.attributes['gcp.client.repo'], undefined); - assert.strictEqual(span.attributes['gcp.client.artifact'], undefined); - assert.strictEqual(span.attributes['rpc.system.name'], 'grpc'); - }); - - it('supports http rpcType', async () => { - const httpDynamicArgs: DynamicTraceContext = { - clientName: 'ComputeClient', - methodName: 'InsertInstance', - rpcType: 'http', - }; - - await traceCall(httpDynamicArgs, staticArgs, async () => { - return 'ok'; - }); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - assert.strictEqual(spans[0].attributes['rpc.system.name'], 'http'); - }); - describe('response status attributes', () => { const httpDynamicArgs: DynamicTraceContext = { clientName: 'ComputeClient', @@ -1781,34 +555,6 @@ describe('TracerHelper', () => { assert.strictEqual(spans[0].status.message, 'Deadline exceeded'); }); - it('does not set response status codes for a failure carrying no gRPC status', async () => { - const error = Object.assign(new Error('connect ECONNREFUSED'), { - code: 'ECONNREFUSED', - }); - - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticArgs, async () => { - throw error; - }); - }); - - harness.assertResponseStatus({rpcStatus: undefined}); - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans[0].status.message, 'connect ECONNREFUSED'); - assert.strictEqual( - spans[0].attributes['grpc.response.status_code'], - undefined, - ); - assert.strictEqual( - spans[0].attributes['rpc.response.status_code'], - undefined, - ); - assert.strictEqual( - spans[0].attributes['http.response.status_code'], - undefined, - ); - }); - it('does not set response status codes for a zero status code', async () => { // Zero is the proto3 default for an unset code, so a failed call must // not be labelled OK or report a response status. @@ -1823,59 +569,25 @@ describe('TracerHelper', () => { harness.assertResponseStatus({rpcStatus: undefined}); }); - it('does not set response status codes for client-side errors on gRPC', async () => { - const error = new TypeError('client parameter validation failed'); - + it('does not set response status codes for client-side errors on gRPC or HTTP', async () => { + const connErr = Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }); await assert.rejects(async () => { await traceCall(dynamicArgs, staticArgs, async () => { - throw error; + throw connErr; }); }); - harness.assertResponseStatus({rpcStatus: undefined}); - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual( - span.status.message, - 'client parameter validation failed', - ); - assert.strictEqual( - span.attributes['grpc.response.status_code'], - undefined, - ); - assert.strictEqual( - span.attributes['rpc.response.status_code'], - undefined, - ); - assert.strictEqual( - span.attributes['http.response.status_code'], - undefined, - ); - }); - - it('does not set response status codes for client-side errors on HTTP', async () => { - const error = new TypeError('client-side http error'); + harness.reset(); + const typeErr = new TypeError('client-side http error'); await assert.rejects(async () => { await traceCall(httpDynamicArgs, staticArgs, async () => { - throw error; + throw typeErr; }); }); - harness.assertResponseStatus({}); - const span = harness.requireSingleSpan('google-gax'); - assert.strictEqual(span.status.message, 'client-side http error'); - assert.strictEqual( - span.attributes['grpc.response.status_code'], - undefined, - ); - assert.strictEqual( - span.attributes['rpc.response.status_code'], - undefined, - ); - assert.strictEqual( - span.attributes['http.response.status_code'], - undefined, - ); }); it('reports server.address and server.port when call succeeds', async () => { @@ -1931,64 +643,36 @@ describe('TracerHelper', () => { }); }); - it('omits server.address and server.port for a DNS resolution failure (ENOTFOUND)', async () => { + it('omits server.address and server.port for pre-connection and client validation failures', async () => { const staticWithServer: StaticTraceContext = { ...staticArgs, serverAddress: 'storage.googleapis.com', serverPort: 443, }; - const error = Object.assign(new Error('getaddrinfo ENOTFOUND'), { - code: 'ENOTFOUND', - }); - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticWithServer, async () => { - throw error; - }); - }); - harness.assertServerAddressAndPort({ - address: undefined, - port: undefined, - }); - }); + const errors = [ + Object.assign(new Error('getaddrinfo ENOTFOUND'), { + code: 'ENOTFOUND', + }), + Object.assign(new Error('connect ECONNREFUSED'), { + code: 'ECONNREFUSED', + }), + Object.assign(new TypeError('Invalid argument'), { + code: 'ERR_INVALID_ARG_TYPE', + }), + ]; - it('omits server.address and server.port for a connection failure (ECONNREFUSED)', async () => { - const staticWithServer: StaticTraceContext = { - ...staticArgs, - serverAddress: 'storage.googleapis.com', - serverPort: 443, - }; - const error = Object.assign(new Error('connect ECONNREFUSED'), { - code: 'ECONNREFUSED', - }); - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticWithServer, async () => { - throw error; + for (const error of errors) { + await assert.rejects(async () => { + await traceCall(dynamicArgs, staticWithServer, async () => { + throw error; + }); }); - }); - harness.assertServerAddressAndPort({ - address: undefined, - port: undefined, - }); - }); - - it('omits server.address and server.port for client validation errors', async () => { - const staticWithServer: StaticTraceContext = { - ...staticArgs, - serverAddress: 'storage.googleapis.com', - serverPort: 443, - }; - const error = Object.assign(new TypeError('Invalid argument'), { - code: 'ERR_INVALID_ARG_TYPE', - }); - await assert.rejects(async () => { - await traceCall(dynamicArgs, staticWithServer, async () => { - throw error; + harness.assertServerAddressAndPort({ + address: undefined, + port: undefined, }); - }); - harness.assertServerAddressAndPort({ - address: undefined, - port: undefined, - }); + harness.reset(); + } }); it('reports the status of a stream failure', () => { @@ -2086,18 +770,6 @@ describe('TracerHelper', () => { }); }); - it('manages span lifetime for resolved promises', async () => { - const result = await traceCall(dynamicArgs, staticArgs, () => - Promise.resolve('async-result'), - ); - assert.strictEqual(result, 'async-result'); - - const spans = harness.getSpans('google-gax'); - assert.strictEqual(spans.length, 1); - assert.strictEqual(spans[0].ended, true); - assert.strictEqual(spans[0].events.length, 0); - }); - it('does not end span prematurely for pending asynchronous promises', async () => { let resolvePromise: (val: string) => void; const asyncPromise = new Promise(resolve => { @@ -3263,22 +1935,17 @@ describe('TracerHelper', () => { assert.strictEqual(span.name, 'google.example.v1.Echo/Echo'); assert.strictEqual(span.kind, SpanKind.CLIENT); assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); - assert.strictEqual( - span.attributes['server.address'], - 'echo.googleapis.com', + harness.assertServerAddressAndPort( + {address: 'echo.googleapis.com', port: 443}, + {span}, ); - assert.strictEqual(span.attributes['server.port'], 443); assert.strictEqual(span.attributes['rpc.system.name'], 'grpc'); assert.strictEqual( span.attributes['rpc.method'], 'google.example.v1.Echo/Echo', ); assert.strictEqual(span.attributes['http.request.method'], undefined); - assert.strictEqual(span.attributes['rpc.response.status_code'], 'OK'); - assert.strictEqual( - span.attributes['grpc.response.status_code'], - undefined, - ); + harness.assertResponseStatus({rpcStatus: 'OK'}, {span}); assert.strictEqual( span.attributes['gcp.client.repo'], 'googleapis/google-cloud-node', @@ -3326,20 +1993,12 @@ describe('TracerHelper', () => { assert.strictEqual(spans[0].name, 'POST'); assert.strictEqual(spans[0].attributes['http.request.method'], 'POST'); assert.strictEqual(spans[0].attributes['rpc.method'], undefined); - assert.strictEqual(spans[0].attributes['http.response.status_code'], 200); - assert.strictEqual( - spans[0].attributes['rpc.response.status_code'], - undefined, - ); + harness.assertResponseStatus({httpStatus: 200}, {span: spans[0]}); assert.strictEqual(spans[0].attributes['status.message'], undefined); assert.strictEqual(spans[1].name, 'GET'); assert.strictEqual(spans[1].attributes['http.request.method'], 'GET'); assert.strictEqual(spans[1].attributes['rpc.method'], undefined); - assert.strictEqual(spans[1].attributes['http.response.status_code'], 200); - assert.strictEqual( - spans[1].attributes['rpc.response.status_code'], - undefined, - ); + harness.assertResponseStatus({httpStatus: 200}, {span: spans[1]}); assert.strictEqual(spans[1].attributes['status.message'], undefined); }); @@ -3444,8 +2103,10 @@ describe('TracerHelper', () => { assert.strictEqual(span.name, 'google.example.v1.Echo/Echo'); assert.strictEqual(span.kind, SpanKind.CLIENT); assert.strictEqual(span.attributes['url.domain'], 'echo.googleapis.com'); - assert.strictEqual(span.attributes['server.address'], undefined); - assert.strictEqual(span.attributes['server.port'], undefined); + harness.assertServerAddressAndPort( + {address: undefined, port: undefined}, + {span}, + ); assert.strictEqual( span.attributes['error.type'], 'CLIENT_CONNECTION_ERROR', @@ -3557,16 +2218,8 @@ describe('TracerHelper', () => { const spans = harness.getSpans('google-gax'); assert.strictEqual(spans.length, 2); - assert.strictEqual(spans[0].attributes['gcp.grpc.resend_count'], 2); - assert.strictEqual( - spans[0].attributes['http.request.resend_count'], - undefined, - ); - assert.strictEqual(spans[1].attributes['http.request.resend_count'], 3); - assert.strictEqual( - spans[1].attributes['gcp.grpc.resend_count'], - undefined, - ); + harness.assertResendCount(2, {span: spans[0]}); + harness.assertResendCount(3, {span: spans[1]}); }); it('sets rpc.method on both gRPC and HTTP client request spans based on the last low level network child span', async () => { diff --git a/core/packages/gax/test/unit/util.ts b/core/packages/gax/test/unit/util.ts index 48be4e9f6a54..989acee0a9bd 100644 --- a/core/packages/gax/test/unit/util.ts +++ b/core/packages/gax/test/unit/util.ts @@ -24,18 +24,10 @@ import { getProtoNameFromFullName, decodeProtobufAny, decodeAnyProtosInArray, - checkTelemetryEnabled, - connectionCodes, - requestCodes, - requestBodyCodes, - decodeCodes, - redirectCodes, - genericClasses, - preConnectionCodes, ignoredClientHeaderTokens, - DEPTH_TO_CHECK, } from '../../src/util'; -import {StaticTraceContext} from '../../src/observability/TracerHelper'; +import {checkTelemetryEnabled} from '../../src/observability/util'; +import {StaticTraceContext} from '../../src/observability/contextUtil'; import {CallSettings} from '../../src/gax'; import * as protobuf from 'protobufjs'; import protosJson from '../../protos/status.json'; @@ -367,58 +359,12 @@ describe('util.ts', () => { }); }); - describe('error code and class constants', () => { - it('defines connectionCodes containing common network errors', () => { - assert(Array.isArray(connectionCodes)); - assert(connectionCodes.includes('ECONNREFUSED')); - assert(connectionCodes.includes('ENOTFOUND')); - }); - - it('defines requestCodes containing request error codes', () => { - assert(Array.isArray(requestCodes)); - assert(requestCodes.includes('ERR_INVALID_ARG_TYPE')); - assert(requestCodes.includes('ERR_INVALID_URL')); - }); - - it('defines requestBodyCodes containing stream error codes', () => { - assert(Array.isArray(requestBodyCodes)); - assert(requestBodyCodes.includes('ERR_STREAM_WRITE_AFTER_END')); - }); - - it('defines decodeCodes containing buffer error codes', () => { - assert(Array.isArray(decodeCodes)); - assert(decodeCodes.includes('ERR_BUFFER_OUT_OF_BOUNDS')); - }); - - it('defines redirectCodes containing redirect error codes', () => { - assert(Array.isArray(redirectCodes)); - assert(redirectCodes.includes('ERR_TOO_MANY_REDIRECTS')); - }); - - it('defines genericClasses containing standard base error types', () => { - assert(Array.isArray(genericClasses)); - assert(genericClasses.includes('Error')); - assert(genericClasses.includes('GoogleError')); - assert(genericClasses.includes('Object')); - assert(genericClasses.includes('DOMException')); - }); - - it('defines preConnectionCodes containing pre-connection error codes', () => { - assert(Array.isArray(preConnectionCodes)); - assert(preConnectionCodes.includes('ECONNREFUSED')); - assert(preConnectionCodes.includes('ENOTFOUND')); - assert(preConnectionCodes.includes('ERR_INVALID_URL')); - }); - + describe('ignoredClientHeaderTokens', () => { it('defines ignoredClientHeaderTokens containing ignored client header tokens', () => { assert(Array.isArray(ignoredClientHeaderTokens)); assert(ignoredClientHeaderTokens.includes('gl-node')); assert(ignoredClientHeaderTokens.includes('gax')); assert(ignoredClientHeaderTokens.includes('gapic')); }); - - it('defines DEPTH_TO_CHECK constant as 10', () => { - assert.strictEqual(DEPTH_TO_CHECK, 10); - }); }); }); From dda98d30a8259162576ee85f968cdc39fdf17807 Mon Sep 17 00:00:00 2001 From: Shivanee Persaud Date: Fri, 9 Oct 2026 10:42:00 -0700 Subject: [PATCH 2/2] fix(gax): ensure detail.errorInfo.reason is non-empty and fix lint formatting --- .../gax/src/observability/contextUtil.ts | 3 +- .../gax/src/observability/errorUtil.ts | 5 +- core/packages/gax/test/unit/apiCallable.ts | 5 +- core/packages/gax/test/unit/errorUtil.ts | 66 +++++++++++++++---- 4 files changed, 61 insertions(+), 18 deletions(-) diff --git a/core/packages/gax/src/observability/contextUtil.ts b/core/packages/gax/src/observability/contextUtil.ts index 5fe8d6a83f27..4f37b74b744a 100644 --- a/core/packages/gax/src/observability/contextUtil.ts +++ b/core/packages/gax/src/observability/contextUtil.ts @@ -339,8 +339,7 @@ export function resolveStaticTraceContext( ): StaticTraceContext { const envMeta = extractFromEnvironment(); const explicit = settings?.otherArgs?.internalTelemetryInfo as - | StaticTraceContext - | undefined; + StaticTraceContext | undefined; let dynamic: StaticTraceContext; if (settings?.apiName) { diff --git a/core/packages/gax/src/observability/errorUtil.ts b/core/packages/gax/src/observability/errorUtil.ts index 13046f1ebf6b..92160535e165 100644 --- a/core/packages/gax/src/observability/errorUtil.ts +++ b/core/packages/gax/src/observability/errorUtil.ts @@ -187,7 +187,10 @@ export function resolveErrorInfoReason(e: unknown): string | undefined { if ( 'errorInfo' in detail && detail.errorInfo && - typeof (detail.errorInfo as {reason?: unknown}).reason === 'string' + typeof detail.errorInfo === 'object' && + typeof (detail.errorInfo as {reason?: unknown}).reason === + 'string' && + (detail.errorInfo as {reason: string}).reason.length > 0 ) { return (detail.errorInfo as {reason: string}).reason; } diff --git a/core/packages/gax/test/unit/apiCallable.ts b/core/packages/gax/test/unit/apiCallable.ts index 82ed8a26b004..863c00d38384 100644 --- a/core/packages/gax/test/unit/apiCallable.ts +++ b/core/packages/gax/test/unit/apiCallable.ts @@ -758,10 +758,7 @@ describe('createApiCall', () => { span.spanContext().spanId, ); assert.strictEqual(attemptSpan.attributes['rpc.system.name'], 'http'); - assert.strictEqual( - attemptSpan.attributes['http.request.method'], - 'POST', - ); + assert.strictEqual(attemptSpan.attributes['http.request.method'], 'POST'); assert.strictEqual(attemptSpan.attributes['rpc.method'], undefined); harness.assertServerAddressAndPort( {address: 'echo.googleapis.com', port: 443}, diff --git a/core/packages/gax/test/unit/errorUtil.ts b/core/packages/gax/test/unit/errorUtil.ts index 2ed429e50f36..f59166e89636 100644 --- a/core/packages/gax/test/unit/errorUtil.ts +++ b/core/packages/gax/test/unit/errorUtil.ts @@ -108,6 +108,27 @@ describe('errorUtil', () => { }), 'SERVICE_DISABLED', ); + assert.strictEqual( + resolveErrorInfoReason({ + statusDetails: [{errorInfo: {reason: 'QUOTA_EXCEEDED'}}], + }), + 'QUOTA_EXCEEDED', + ); + assert.strictEqual( + resolveErrorInfoReason({ + statusDetails: [ + {errorInfo: {reason: ''}}, + {errorInfo: {reason: 'VALID_REASON'}}, + ], + }), + 'VALID_REASON', + ); + assert.strictEqual( + resolveErrorInfoReason({ + statusDetails: [{errorInfo: {reason: ''}}], + }), + undefined, + ); assert.strictEqual( resolveErrorInfoReason({ cause: {reason: 'ACCESS_TOKEN_EXPIRED'}, @@ -151,7 +172,10 @@ describe('errorUtil', () => { }); it('isServerSideError recognizes plain error objects with valid status codes', () => { - assert.strictEqual(isServerSideError({httpStatusCode: 500}, 'http'), true); + assert.strictEqual( + isServerSideError({httpStatusCode: 500}, 'http'), + true, + ); assert.strictEqual( isServerSideError({code: Status.NOT_FOUND}, 'grpc'), true, @@ -162,8 +186,14 @@ describe('errorUtil', () => { false, ); assert.strictEqual(isServerSideError({code: 5}, 'http'), false); - assert.strictEqual(isServerSideError({message: 'no code'}, 'grpc'), false); - assert.strictEqual(isServerSideError({message: 'no code'}, 'http'), false); + assert.strictEqual( + isServerSideError({message: 'no code'}, 'grpc'), + false, + ); + assert.strictEqual( + isServerSideError({message: 'no code'}, 'http'), + false, + ); }); it('resolveClientNetworkOrOperationalError maps timeout, connection, request, body, decode, redirect, and auth errors', () => { @@ -332,7 +362,10 @@ describe('errorUtil', () => { const t3 = Object.assign(new TypeError('invalid type'), { code: 'ERR_INVALID_ARG_TYPE', }); - assert.strictEqual(resolveErrorType(t3, 'grpc'), 'CLIENT_REQUEST_ERROR'); + assert.strictEqual( + resolveErrorType(t3, 'grpc'), + 'CLIENT_REQUEST_ERROR', + ); // Tier 4 beats Tier 5 const t4 = new TypeError('normal type error'); @@ -363,9 +396,12 @@ describe('errorUtil', () => { ); // HTTP call without HTTP response falls through to Tier 3 client error - const deadlineErr = Object.assign(new GoogleError('Deadline exceeded'), { - code: Status.DEADLINE_EXCEEDED, - }); + const deadlineErr = Object.assign( + new GoogleError('Deadline exceeded'), + { + code: Status.DEADLINE_EXCEEDED, + }, + ); assert.strictEqual( resolveErrorType(deadlineErr, 'http'), 'CLIENT_TIMEOUT', @@ -393,7 +429,10 @@ describe('errorUtil', () => { code: Status.CANCELLED, cause: abortCause, }); - assert.strictEqual(resolveErrorType(wrappedAbort, 'grpc'), 'AbortError'); + assert.strictEqual( + resolveErrorType(wrappedAbort, 'grpc'), + 'AbortError', + ); // Outer HTTP response status takes precedence over inner cause exception type const httpWithCause = Object.assign( @@ -409,7 +448,10 @@ describe('errorUtil', () => { it('falls back to INTERNAL for zero, out-of-range, or empty string status codes', () => { assert.strictEqual( - resolveErrorType(Object.assign(new Error('unset'), {code: 0}), 'grpc'), + resolveErrorType( + Object.assign(new Error('unset'), {code: 0}), + 'grpc', + ), 'INTERNAL', ); assert.strictEqual( @@ -484,8 +526,10 @@ describe('errorUtil', () => { }); it('recordExceptionEvent records exception.type and optionally exception.message', () => { - const events: Array<{name: string; attributes?: Record}> = - []; + const events: Array<{ + name: string; + attributes?: Record; + }> = []; const fakeSpan = { addEvent(name: string, attributes?: Record) { events.push({name, attributes});