feat(observability): [WIP] implement universal 4-path OpenTelemetry tracing

.github/workflows/gapic-generator-tests.yml

@@ -71,7 +71,7 @@ jobs:
         with:
           python-version: "${{ matrix.python }}"
           allow-prereleases: true
-          # Caches compiled wheels locally to prevent building heavy libraries 
+          # Caches compiled wheels locally to prevent building heavy libraries
           # such as grpcio, which we build from scratch on every run for Python 3.15+.
           # Follow https://github.com/grpc/grpc/issues/41010 for updates.
           cache: 'pip'
@@ -142,6 +142,68 @@ jobs:
           run: |
             cd packages/gapic-generator
             nox -s ${{ matrix.target }}-${MATRIX_PYTHON}
+        - name: Upload Raw Spans Artifact
+          if: always()
+          uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+          with:
+            name: raw-spans-${{ matrix.target }}-${{ matrix.python }}${{ matrix.logging_scope && format('-{0}', matrix.logging_scope) || '' }}
+            path: packages/gapic-generator/tests/system/raw_spans_output.json
+            if-no-files-found: ignore
+
+  showcase-canary:
+      needs: python_config
+      strategy:
+        # Run canary tests with local monorepo google-api-core on bookend runtimes
+        matrix:
+          python: ["${{ needs.python_config.outputs.oldest_python }}", "${{ needs.python_config.outputs.latest_stable_python }}"]
+          target: [showcase, showcase_w_rest_async]
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+          with:
+            persist-credentials: false
+        - name: Set up Python "${{ matrix.python }}"
+          uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+          with:
+            python-version: "${{ matrix.python }}"
+            cache: 'pip'
+        - name: Install system dependencies.
+          run: |
+            sudo apt-get update
+            sudo apt-get install -y curl pandoc unzip
+        - name: Install showcase
+          run: |
+            sudo mkdir -p /usr/src/showcase
+            sudo chown -R ${USER} /usr/src/
+            curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_VERSION}/gapic-showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz
+            cd /usr/src/showcase/
+            tar -xf showcase-*
+            ./gapic-showcase run &
+            cd -
+        - name: Install nox.
+          run: python -m pip install nox
+        - name: Install protoc
+          run: |
+              sudo mkdir -p /usr/src/protoc/
+              sudo chown -R ${USER} /usr/src/
+              curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc-${PROTOC_VERSION}.zip
+              cd /usr/src/protoc/
+              unzip protoc-${PROTOC_VERSION}.zip
+              sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
+        - name: Run showcase canary with local google-api-core.
+          env:
+            INSTALL_LOCAL_CORE: "true"
+            MATRIX_PYTHON: ${{ matrix.python }}
+          run: |
+            cd packages/gapic-generator
+            nox -s ${{ matrix.target }}-${MATRIX_PYTHON}
+        - name: Upload Raw Spans Artifact
+          if: always()
+          uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+          with:
+            name: raw-spans-canary-${{ matrix.target }}-${{ matrix.python }}
+            path: packages/gapic-generator/tests/system/raw_spans_output.json
+            if-no-files-found: ignore
 
   showcase-mypy:
     needs: python_config
@@ -210,7 +272,7 @@ jobs:
         with:
           python-version: ${{ needs.python_config.outputs.prerelease_python }}
           allow-prereleases: true
-          # Caches compiled wheels locally to prevent building heavy libraries 
+          # Caches compiled wheels locally to prevent building heavy libraries
           # such as grpcio, which we build from scratch on every run for Python 3.15+.
           # Follow https://github.com/grpc/grpc/issues/41010 for updates.
           cache: 'pip'
@@ -242,7 +304,7 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           allow-prereleases: true
-          # Caches compiled wheels locally to prevent building heavy libraries 
+          # Caches compiled wheels locally to prevent building heavy libraries
           # such as grpcio, which we build from scratch on every run for Python 3.15+.
           # Follow https://github.com/grpc/grpc/issues/41010 for updates.
           cache: 'pip'
@@ -287,13 +349,13 @@ jobs:
 
       - name: Run Bazel Integration Tests
         run: |
-          # We need to move into the package directory if the 
+          # We need to move into the package directory if the
           # WORKSPACE file is located there.
           cd packages/gapic-generator
           bazelisk test //tests/integration/... --test_output=errors
 
   # Final job to simplify branch protection rules.
-  # This job will be reported as 'Success' if all its dependencies either 
+  # This job will be reported as 'Success' if all its dependencies either
   # pass or are skipped due to path filtering.
   status:
     needs: [check_changes, python_config, showcase-unit, showcase, showcase-mypy, goldens, goldens-prerelease, fragment-snippet, integration]
@@ -314,4 +376,3 @@ jobs:
             exit 1
           fi
           echo "All checks passed or were successfully skipped."
-

.pre-commit-config.yaml

@@ -14,6 +14,7 @@
 #
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
+exclude: '^packages/gapic-generator/tests/integration/goldens/'
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0

ci/get_package_shards.py

@@ -54,8 +54,11 @@
 # extra name 'mssql-pymssql' under strict uv PEP 621 parsing in sqlalchemy==2.1.0rc2,
 # pulled via global UV_PRERELEASE=allow). Awaiting team feedback on a long-term
 # solution (e.g. package migration out of the monorepo or adjusting workflow settings).
+# NOTE: 'gcp-sphinx-docfx-yaml' is temporarily excluded due to legacy build dependencies
+# (unify/untokenize) failing to build under modern setuptools/uv.
 EXCLUDED_PACKAGES = {
     "sqlalchemy-bigquery",
+    "gcp-sphinx-docfx-yaml",
 }
 
 

packages/gapic-generator/gapic/schema/mixins.py

@@ -19,50 +19,60 @@
         "DeleteOperation",
         request_type="operations_pb2.DeleteOperationRequest",
         response_type="None",
+        rpc_name="google.longrunning.Operations/DeleteOperation",
     ),
     "WaitOperation": wrappers.MixinMethod(
         "WaitOperation",
         request_type="operations_pb2.WaitOperationRequest",
         response_type="operations_pb2.Operation",
+        rpc_name="google.longrunning.Operations/WaitOperation",
     ),
     "ListOperations": wrappers.MixinMethod(
         "ListOperations",
         request_type="operations_pb2.ListOperationsRequest",
         response_type="operations_pb2.ListOperationsResponse",
+        rpc_name="google.longrunning.Operations/ListOperations",
     ),
     "CancelOperation": wrappers.MixinMethod(
         "CancelOperation",
         request_type="operations_pb2.CancelOperationRequest",
         response_type="None",
+        rpc_name="google.longrunning.Operations/CancelOperation",
     ),
     "GetOperation": wrappers.MixinMethod(
         "GetOperation",
         request_type="operations_pb2.GetOperationRequest",
         response_type="operations_pb2.Operation",
+        rpc_name="google.longrunning.Operations/GetOperation",
     ),
     "TestIamPermissions": wrappers.MixinMethod(
         "TestIamPermissions",
         request_type="iam_policy_pb2.TestIamPermissionsRequest",
         response_type="iam_policy_pb2.TestIamPermissionsResponse",
+        rpc_name="google.iam.v1.IAMPolicy/TestIamPermissions",
     ),
     "GetIamPolicy": wrappers.MixinMethod(
         "GetIamPolicy",
         request_type="iam_policy_pb2.GetIamPolicyRequest",
         response_type="policy_pb2.Policy",
+        rpc_name="google.iam.v1.IAMPolicy/GetIamPolicy",
     ),
     "SetIamPolicy": wrappers.MixinMethod(
         "SetIamPolicy",
         request_type="iam_policy_pb2.SetIamPolicyRequest",
         response_type="policy_pb2.Policy",
+        rpc_name="google.iam.v1.IAMPolicy/SetIamPolicy",
     ),
     "ListLocations": wrappers.MixinMethod(
         "ListLocations",
         request_type="locations_pb2.ListLocationsRequest",
         response_type="locations_pb2.ListLocationsResponse",
+        rpc_name="google.cloud.location.Locations/ListLocations",
     ),
     "GetLocation": wrappers.MixinMethod(
         "GetLocation",
         request_type="locations_pb2.GetLocationRequest",
         response_type="locations_pb2.Location",
+        rpc_name="google.cloud.location.Locations/GetLocation",
     ),
 }

packages/gapic-generator/gapic/schema/wrappers.py

@@ -1463,6 +1463,8 @@ class MixinMethod:
     name: str
     request_type: str
     response_type: str
+    rpc_name: str = ""
+
 
 
 @dataclasses.dataclass(frozen=True)

packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/_compat.py.j2

@@ -5,12 +5,12 @@
 """A compatibility module for older versions of google-api-core."""
 {% set has_auto_populated_fields = api.all_method_settings.values()|map(attribute="auto_populated_fields", default=[])|select|list %}
 {# TODO(https://github.com/googleapis/google-cloud-python/issues/17813):
-Clean up this file/functions when the minimum supported version of 
+Clean up this file/functions when the minimum supported version of
 google-api-core has the functions in `_compat.py.j2`. #}
-{# TODO(https://github.com/googleapis/google-cloud-python/issues/17884): 
-Add conditional logic to check if static code exists in google-api-core and use it from there, 
+{# TODO(https://github.com/googleapis/google-cloud-python/issues/17884):
+Add conditional logic to check if static code exists in google-api-core and use it from there,
 falling back to the local implementation if not present. #}
-{# TODO(https://github.com/googleapis/google-cloud-python/issues/17883): 
+{# TODO(https://github.com/googleapis/google-cloud-python/issues/17883):
 Backfill compatibility functions being removed from the client layer. #}
 
 import os
@@ -20,12 +20,9 @@ import uuid
 import google.protobuf.message
 {% endif %}
 
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Sequence, Tuple
 {% if has_auto_populated_fields %}
 from typing import TYPE_CHECKING, Union
-
-if TYPE_CHECKING:  # pragma: NO COVER
-    import proto  # type: ignore[import-untyped]
 {% endif %}
 
 from google.api_core import path_template
@@ -34,9 +31,117 @@ from google.auth.exceptions import MutualTLSChannelError
 from google.protobuf import json_format
 from urllib.parse import urlparse, urlunparse
 
+{% if has_auto_populated_fields %}
+if TYPE_CHECKING:  # pragma: NO COVER
+    import proto  # type: ignore[import-untyped]
+{% endif %}
+
+# The _observability module was introduced in google-api-core 2.36.0+.
+# On older versions of google-api-core or when type-checking against them,
+# mypy may flag attr-defined or assignment errors when fallback to None occurs.
+try:
+    from google.api_core import _observability  # type: ignore[attr-defined]
+except ImportError:  # pragma: NO COVER
+    _observability = None  # type: ignore[assignment]
+
+if _observability is not None and hasattr(_observability, "trace_http_request"):
+    trace_http_request = _observability.trace_http_request
+else:  # pragma: NO COVER
+    # Fallback for older versions of google-api-core without HTTP tracing.
+    class _FallbackTraceContext:
+        def __enter__(self) -> "_FallbackTraceContext":
+            return self
+
+        def __exit__(self, *args: Any) -> None:
+            pass
+
+        def record_response(self, response: Any) -> None:
+            pass
+
+        record_http_response = record_response
+
+        def record_error(self, exc: BaseException | None) -> None:
+            pass
+
+        record_http_error = record_error
+
+    def trace_http_request(*args: Any, **kwargs: Any) -> _FallbackTraceContext:
+        return _FallbackTraceContext()
+
+try:
+    from google.api_core import grpc_helpers_async
+except ImportError:  # pragma: NO COVER
+    grpc_helpers_async = None  # type: ignore[assignment]
+
+
+def _fallback_apply_interceptors(
+    channel: Any, interceptors: Optional[Sequence[Any]] = None
+) -> Any:  # pragma: NO COVER
+    """Fallback helper to attach interceptors to an existing async gRPC channel.
+
+    Synchronous gRPC channels allow post-creation wrapping via
+    `grpc.intercept_channel(channel, *interceptors)`. However, asynchronous
+    channels (`grpc.aio.Channel`) in grpcio are immutable post-creation and
+    normally require interceptors at initialization.
+
+    When generated client libraries run against older versions of `google-api-core`
+    (< 2.36.0) where `grpc_helpers_async.apply_channel_interceptors` is not yet
+    available, this fallback mutates the channel's internal interceptor lists
+    directly so telemetry and auth interceptors remain operational.
+
+    Args:
+        channel (Any): The gRPC channel instance (typically a `grpc.aio.Channel`
+            or mock).
+        interceptors (Optional[Sequence[Any]]): A sequence of gRPC interceptor
+            instances or callables to register on the channel.
+
+    Returns:
+        Any: The channel instance with interceptors registered.
+    """
+    mapping = (
+        ("intercept_unary_unary", "_unary_unary_interceptors"),
+        ("intercept_unary_stream", "_unary_stream_interceptors"),
+        ("intercept_stream_unary", "_stream_unary_interceptors"),
+        ("intercept_stream_stream", "_stream_stream_interceptors"),
+    )
+    if interceptors:
+        for interceptor in interceptors:
+            matched = False
+
+            # Path 1: Interface-matched routing.
+            # Inspect whether the interceptor implements specific gRPC method hooks
+            # (unary-unary, unary-stream, stream-unary, stream-stream). If the channel
+            # maintains a corresponding internal list, append the interceptor (preventing duplicates).
+            # A single interceptor can handle multiple call patterns, so check all mappings.
+            for method_name, attr_name in mapping:
+                if hasattr(interceptor, method_name) and hasattr(channel, attr_name):
+                    target_list = getattr(channel, attr_name)
+                    if isinstance(target_list, list) and interceptor not in target_list:
+                        target_list.append(interceptor)
+                    matched = True
+
+            # Path 2: Duck-typed / callable fallback.
+            # If the interceptor did not implement any of the specific method signatures
+            # above (e.g., generic callables, untyped wrappers, or test doubles), fall back
+            # to registering it in the primary unary-unary interceptor list if present.
+            if not matched and hasattr(channel, "_unary_unary_interceptors"):
+                unary_interceptors = channel._unary_unary_interceptors
+                if isinstance(unary_interceptors, list) and interceptor not in unary_interceptors:
+                    unary_interceptors.append(interceptor)
+    return channel
+
+
+if grpc_helpers_async is not None and hasattr(
+    grpc_helpers_async, "apply_channel_interceptors"
+):
+    apply_channel_interceptors = grpc_helpers_async.apply_channel_interceptors  # pragma: NO COVER
+else:  # pragma: NO COVER
+    apply_channel_interceptors = _fallback_apply_interceptors
+
+
 try:
     # note: `#type: ignore` is added because the return type for `should_use_client_cert`
-    # is different than that of the fallback implementation below. This will be removed once 
+    # is different than that of the fallback implementation below. This will be removed once
     # we bump the minimum supported version of google-auth.
     from google.auth.transport.mtls import should_use_client_cert  # type: ignore
 except ImportError:  # pragma: NO COVER