Skip to content

Commit 402f730

Browse files
committed
feat(api_core): centralize telemetry error attribute and trace context builders in _observability
- Move error attribute extraction from gapic_v1.method into _observability.build_telemetry_error_attributes with single-pass exception unwrapping, rpc.response.status_code resolution, native GoogleAPICallError property access, raw gRPC RpcError trailing_metadata parsing, and optional exception.* fields. - Update gapic_v1.method and tests to call _observability.build_telemetry_error_attributes directly without separate _extract_status_code calls. - Add has_opentelemetry() (with module-level caching) and extract_trace_context() in _observability. - Add unit tests achieving 100% statement and branch coverage.
1 parent 0c3fa2b commit 402f730

5 files changed

Lines changed: 767 additions & 302 deletions

File tree

‎packages/google-api-core/google/api_core/_observability.py‎

Lines changed: 230 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@
1919
from __future__ import annotations
2020

2121
import contextlib
22+
import traceback
2223
import urllib.parse
2324
from typing import TYPE_CHECKING, Any, Callable, Sequence
2425

25-
from google.api_core import _feature_gating_helpers
26+
from google.api_core import _feature_gating_helpers, exceptions
2627
from google.api_core.client_options import ClientOptions
2728

2829
try:
@@ -41,6 +42,22 @@
4142
from google.api_core.grpc_helpers import ClientInterceptor # noqa: F401
4243

4344
_TRACER_PROVIDER = "tracer_provider"
45+
_HAS_OPENTELEMETRY: bool | None = None
46+
47+
48+
def has_opentelemetry() -> bool:
49+
"""Checks once per process if opentelemetry-api is installed."""
50+
global _HAS_OPENTELEMETRY
51+
if _HAS_OPENTELEMETRY is None:
52+
try:
53+
# Ignored by mypy: Optional dependency only checked at runtime
54+
# flake8: Imported only to verify package installation
55+
import opentelemetry.trace # type: ignore[import-not-found] # noqa: F401
56+
57+
_HAS_OPENTELEMETRY = True
58+
except ImportError:
59+
_HAS_OPENTELEMETRY = False
60+
return _HAS_OPENTELEMETRY
4461

4562

4663
def is_otel_capabilities_enabled(
@@ -74,6 +91,218 @@ def is_otel_capabilities_enabled(
7491
return False
7592

7693

94+
def extract_trace_context(span: Any = None) -> dict[str, Any]:
95+
"""Extracts trace_id, span_id, and Cloud Logging trace correlation fields.
96+
97+
If `span` is None, falls back to `opentelemetry.trace.get_current_span()`
98+
when OpenTelemetry is installed. Returns an empty dict if no valid recording
99+
span context exists or if OpenTelemetry is not installed.
100+
101+
Args:
102+
span (Optional[Any]): The active OpenTelemetry span, if any.
103+
104+
Returns:
105+
dict[str, Any]: Trace and span correlation attributes.
106+
"""
107+
if span is None and has_opentelemetry():
108+
try:
109+
# Ignored by mypy: Optional dependency only loaded after
110+
# has_opentelemetry() check
111+
from opentelemetry import trace # type: ignore[import-not-found]
112+
113+
span = trace.get_current_span()
114+
# Defensive fail-open: telemetry correlation must never crash an RPC call
115+
# if a custom or broken OpenTelemetry TracerProvider raises at runtime.
116+
except Exception:
117+
span = None
118+
119+
if span is None or not hasattr(span, "get_span_context"):
120+
return {}
121+
122+
try:
123+
ctx = span.get_span_context()
124+
# Defensive fail-open: guard against broken third-party or mock span objects.
125+
except Exception:
126+
return {}
127+
128+
if ctx is None or not getattr(ctx, "is_valid", False):
129+
return {}
130+
131+
try:
132+
trace_id_hex = f"{ctx.trace_id:032x}"
133+
span_id_hex = f"{ctx.span_id:016x}"
134+
trace_flags = getattr(ctx, "trace_flags", None)
135+
if trace_flags is not None and hasattr(trace_flags, "sampled"):
136+
sampled = bool(trace_flags.sampled)
137+
elif isinstance(trace_flags, int):
138+
sampled = bool(trace_flags & 0x01)
139+
else:
140+
sampled = False
141+
except (TypeError, ValueError, AttributeError):
142+
return {}
143+
144+
return {
145+
"trace_id": trace_id_hex,
146+
"span_id": span_id_hex,
147+
"logging.googleapis.com/trace": trace_id_hex,
148+
"logging.googleapis.com/spanId": span_id_hex,
149+
"logging.googleapis.com/trace_sampled": sampled,
150+
}
151+
152+
153+
def _extract_error_info_fields(target_exc: Any) -> tuple[Any, Any, Any]:
154+
"""Resolves (domain, reason, metadata) from an exception, ErrorInfo, or dict."""
155+
# Prefer native GoogleAPICallError properties (.domain, .reason, .metadata)
156+
# when present; fall back to .error_info or raw gRPC trailing_metadata.
157+
error_info = getattr(target_exc, "error_info", None)
158+
if error_info is None and not (
159+
getattr(target_exc, "reason", None)
160+
or getattr(target_exc, "domain", None)
161+
or getattr(target_exc, "metadata", None)
162+
):
163+
rpc_call = (
164+
target_exc
165+
if hasattr(target_exc, "trailing_metadata")
166+
else getattr(target_exc, "response", None)
167+
)
168+
if rpc_call is not None and hasattr(rpc_call, "trailing_metadata"):
169+
try:
170+
_, error_info = exceptions._parse_grpc_error_details(rpc_call)
171+
# Defensive fail-open: malformed trailing metadata must not abort
172+
# error attribute extraction.
173+
except Exception:
174+
pass
175+
176+
source = error_info if error_info is not None else target_exc
177+
if isinstance(source, dict):
178+
return (
179+
source.get("domain"),
180+
source.get("reason"),
181+
source.get("metadata"),
182+
)
183+
return (
184+
getattr(source, "domain", None),
185+
getattr(source, "reason", None),
186+
getattr(source, "metadata", None),
187+
)
188+
189+
190+
def _extract_error_message(
191+
target_exc: Any, http_response: Any = None, fallback: str | None = None
192+
) -> str | None:
193+
"""Resolves the human-readable error message across HTTP, gRPC, and Python errors."""
194+
http_msg = (
195+
_TraceContext._extract_http_error_message(http_response)
196+
if http_response is not None
197+
else None
198+
)
199+
details = getattr(target_exc, "details", None)
200+
details_msg = details() if callable(details) else details
201+
exc_msg = str(target_exc) if isinstance(target_exc, BaseException) else None
202+
raw_message = (
203+
http_msg
204+
or getattr(target_exc, "message", None)
205+
or details_msg
206+
or fallback
207+
or exc_msg
208+
)
209+
return str(raw_message) if raw_message else None
210+
211+
212+
def build_telemetry_error_attributes(
213+
exc: BaseException | None,
214+
status_code: str | None = None,
215+
status_message: str | None = None,
216+
include_exception: bool = False,
217+
) -> dict[str, Any]:
218+
"""Extracts rpc.response.status_code, gcp.errors.*, error.type, status.message, and exception.* fields.
219+
220+
Error details and ErrorInfo structures are resolved by inspecting:
221+
* Chained exceptions: Unwraps RetryError or __cause__ to the root error.
222+
* Canonical status code: Resolves rpc.response.status_code via
223+
exceptions._extract_status_code.
224+
* GoogleAPICallError & ErrorInfo attributes: Reads .domain, .reason, and
225+
.metadata directly (or from .error_info / gRPC trailing_metadata).
226+
* Unified attribute extraction: Populates gcp.errors.domain, error.type,
227+
gcp.errors.metadata.*, status.message, and optional exception.* fields.
228+
229+
Args:
230+
exc (Optional[BaseException]): An exception (e.g. GoogleAPICallError
231+
or grpc.RpcError) or ErrorInfo object.
232+
status_code (Optional[str]): Optional pre-extracted or HTTP status code
233+
fallback when exc is None or lacks a status code.
234+
status_message (Optional[str]): Optional status message fallback when
235+
exc is None or lacks a message.
236+
include_exception (bool): When True, also populates exception.type,
237+
exception.message, and exception.stacktrace.
238+
239+
Returns:
240+
dict[str, Any]: Extracted error and exception attributes.
241+
"""
242+
if exc is None:
243+
if status_code is None:
244+
return {}
245+
code_str = str(status_code)
246+
return {
247+
"rpc.response.status_code": code_str,
248+
"error.type": code_str,
249+
"status.message": status_message or f"HTTP {code_str}",
250+
}
251+
252+
# Unwrap chained exceptions once and resolve the canonical status code
253+
target_exc = getattr(exc, "cause", None) or getattr(exc, "__cause__", None) or exc
254+
resolved_code = exceptions._extract_status_code(target_exc, default=status_code)
255+
domain, reason, metadata = _extract_error_info_fields(target_exc)
256+
257+
# Identify HTTP transport responses (excluding gRPC Call objects)
258+
http_response = getattr(target_exc, "response", None)
259+
http_status = (
260+
getattr(http_response, "status_code", getattr(http_response, "status", None))
261+
if http_response is not None and not hasattr(http_response, "trailing_metadata")
262+
else None
263+
)
264+
265+
# Populate OpenTelemetry semantic error attributes and ErrorInfo metadata
266+
attrs: dict[str, Any] = {"rpc.response.status_code": resolved_code}
267+
if domain:
268+
attrs["gcp.errors.domain"] = domain
269+
if reason:
270+
attrs["error.type"] = reason
271+
elif (
272+
http_status is not None
273+
and not isinstance(http_status, bool)
274+
and str(http_status).isdigit()
275+
):
276+
attrs["error.type"] = str(http_status)
277+
else:
278+
# Every failed span or log records a low-cardinality error.type; reuse
279+
# the resolved canonical status code name or exception class name.
280+
attrs["error.type"] = resolved_code
281+
282+
if metadata and hasattr(metadata, "items"):
283+
for k, v in metadata.items():
284+
attrs[f"gcp.errors.metadata.{k}"] = str(v)
285+
286+
# Extract human-readable error description for cross-language parity
287+
message = _extract_error_message(
288+
target_exc,
289+
http_response if http_status is not None else None,
290+
status_message,
291+
)
292+
if message:
293+
attrs["status.message"] = message
294+
295+
# Populate exception.* diagnostic fields for structured logs when requested
296+
if include_exception and isinstance(exc, BaseException):
297+
attrs["exception.type"] = exc.__class__.__name__
298+
attrs["exception.message"] = str(exc)
299+
attrs["exception.stacktrace"] = "".join(
300+
traceback.format_exception(type(exc), exc, exc.__traceback__)
301+
)
302+
303+
return attrs
304+
305+
77306
def _extract_endpoint_attributes(
78307
client_options: ClientOptions | dict[str, Any] | None = None,
79308
default_host: str | None = None,

‎packages/google-api-core/google/api_core/gapic_v1/method.py‎

Lines changed: 5 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@
2121
import contextlib
2222
import enum
2323
import functools
24-
from typing import Any, List, Optional, Tuple
24+
from typing import List, Tuple
2525

26-
from google.api_core import _observability, exceptions, grpc_helpers
26+
from google.api_core import _observability, grpc_helpers
2727
from google.api_core.gapic_v1 import client_info
2828
from google.api_core.gapic_v1.client_info import METRICS_METADATA_KEY
2929
from google.api_core.timeout import TimeToDeadlineTimeout
@@ -126,98 +126,6 @@ def _extract_rpc_identity(
126126
return method_str, service, method
127127

128128

129-
def _extract_error_attributes(exc: Optional[BaseException]) -> dict[str, Any]:
130-
"""Extract gcp.errors.* and error.type attributes from an exception.
131-
132-
Error details and ErrorInfo structures are resolved by inspecting the following locations:
133-
* Chained exceptions: Unwraps RetryError or __cause__ to the root exception.
134-
* GoogleAPICallError attributes: Reads ErrorInfo from ._error_info or .error_info.
135-
* Native gRPC trailing metadata: Parses google.rpc.Status binary details from trailing_metadata.
136-
* Unified attribute extraction: Extracts domain, reason, and metadata from ErrorInfo or exception attributes.
137-
138-
Args:
139-
exc (Optional[BaseException]): An exception (such as GoogleAPICallError or grpc.RpcError) or ErrorInfo object.
140-
141-
Returns:
142-
dict[str, Any]: Extracted error attributes (e.g. gcp.errors.domain, error.type, gcp.errors.metadata.*).
143-
"""
144-
attrs: dict[str, Any] = {}
145-
if exc is None:
146-
return attrs
147-
148-
# 1. Unwrap chained exceptions: unwrap RetryError or __cause__ to the root failure
149-
target_exc = getattr(exc, "cause", None) or getattr(exc, "__cause__", None) or exc
150-
151-
# 2. Check GoogleAPICallError ErrorInfo attributes
152-
error_info = getattr(target_exc, "_error_info", None) or getattr(
153-
target_exc, "error_info", None
154-
)
155-
156-
# 3. Check native gRPC trailing metadata for binary google.rpc.Status details
157-
if error_info is None:
158-
rpc_call = (
159-
target_exc
160-
if hasattr(target_exc, "trailing_metadata")
161-
else getattr(target_exc, "response", None)
162-
)
163-
if rpc_call is not None and hasattr(rpc_call, "trailing_metadata"):
164-
try:
165-
_, error_info = exceptions._parse_grpc_error_details(rpc_call)
166-
except Exception:
167-
pass
168-
169-
# 4. Unified attribute extraction: extract domain, reason, and metadata from ErrorInfo or exception attributes
170-
http_response = getattr(target_exc, "response", None)
171-
http_status = (
172-
getattr(http_response, "status_code", getattr(http_response, "status", None))
173-
if http_response is not None and not hasattr(http_response, "trailing_metadata")
174-
else None
175-
)
176-
177-
source = error_info or target_exc
178-
if isinstance(source, dict):
179-
domain = source.get("domain")
180-
reason = source.get("reason")
181-
metadata = source.get("metadata")
182-
else:
183-
domain = getattr(source, "domain", None)
184-
reason = getattr(source, "reason", None)
185-
metadata = getattr(source, "metadata", None)
186-
187-
if domain:
188-
attrs["gcp.errors.domain"] = domain
189-
if reason:
190-
attrs["error.type"] = reason
191-
elif isinstance(http_status, int):
192-
attrs["error.type"] = str(http_status)
193-
else:
194-
# Fallback per OpenTelemetry Semantic Conventions: every failed span should record
195-
# a low-cardinality error.type. Use canonical status code name or exception class name.
196-
status_code = exceptions._extract_status_code(target_exc)
197-
attrs["error.type"] = status_code or target_exc.__class__.__name__
198-
if metadata:
199-
for k, v in metadata.items():
200-
attrs[f"gcp.errors.metadata.{k}"] = str(v)
201-
202-
# 5. Extract human-readable error description for cross-language PRD parity
203-
message = (
204-
_observability._TraceContext._extract_http_error_message(http_response)
205-
if http_status is not None
206-
else None
207-
)
208-
if not message:
209-
message = getattr(target_exc, "message", None)
210-
if not message and hasattr(target_exc, "details"):
211-
details = target_exc.details
212-
message = details() if callable(details) else details
213-
if not message and isinstance(target_exc, BaseException):
214-
message = str(target_exc)
215-
if message:
216-
attrs["status.message"] = str(message)
217-
218-
return attrs
219-
220-
221129
class _GapicCallable(object):
222130
"""Callable that applies retry, timeout, and metadata logic.
223131
@@ -381,10 +289,9 @@ def _trace_span(self):
381289
raise
382290
except BaseException as exc:
383291
if span is not None and hasattr(span, "set_attribute"):
384-
span.set_attribute(
385-
"rpc.response.status_code", exceptions._extract_status_code(exc)
386-
)
387-
for k, v in _extract_error_attributes(exc).items():
292+
for k, v in _observability.build_telemetry_error_attributes(
293+
exc
294+
).items():
388295
span.set_attribute(k, v)
389296
raise
390297

‎packages/google-api-core/tests/conftest.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ def mock_otel(monkeypatch):
4343

4444
mock_trace = mock.Mock()
4545
mock_trace.get_tracer.return_value = mock_tracer
46+
mock_trace.get_current_span.return_value = mock_span
4647
mock_trace.SpanKind.CLIENT = "CLIENT"
4748
mock_trace.SpanKind.INTERNAL = "INTERNAL"
4849
mock_trace.StatusCode.ERROR = "ERROR"

0 commit comments

Comments
 (0)